test_process.py 86 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017
  1. import os
  2. import signal
  3. import time
  4. import unittest
  5. import sys
  6. import errno
  7. from mock import Mock, patch, sentinel
  8. from supervisor.tests.base import DummyOptions
  9. from supervisor.tests.base import DummyPConfig
  10. from supervisor.tests.base import DummyProcess
  11. from supervisor.tests.base import DummyPGroupConfig
  12. from supervisor.tests.base import DummyDispatcher
  13. from supervisor.tests.base import DummyEvent
  14. from supervisor.tests.base import DummyFCGIGroupConfig
  15. from supervisor.tests.base import DummySocketConfig
  16. from supervisor.tests.base import DummyProcessGroup
  17. from supervisor.tests.base import DummyFCGIProcessGroup
  18. from supervisor.process import Subprocess
  19. from supervisor.options import BadCommand
  20. class SubprocessTests(unittest.TestCase):
  21. def _getTargetClass(self):
  22. from supervisor.process import Subprocess
  23. return Subprocess
  24. def _makeOne(self, *arg, **kw):
  25. return self._getTargetClass()(*arg, **kw)
  26. def tearDown(self):
  27. from supervisor.events import clear
  28. clear()
  29. def test_getProcessStateDescription(self):
  30. from supervisor.states import ProcessStates
  31. from supervisor.process import getProcessStateDescription
  32. for statename, code in ProcessStates.__dict__.items():
  33. if isinstance(code, int):
  34. self.assertEqual(getProcessStateDescription(code), statename)
  35. def test_ctor(self):
  36. options = DummyOptions()
  37. config = DummyPConfig(options, 'cat', 'bin/cat',
  38. stdout_logfile='/tmp/temp123.log',
  39. stderr_logfile='/tmp/temp456.log')
  40. instance = self._makeOne(config)
  41. self.assertEqual(instance.config, config)
  42. self.assertEqual(instance.config.options, options)
  43. self.assertEqual(instance.laststart, 0)
  44. self.assertEqual(instance.pid, 0)
  45. self.assertEqual(instance.laststart, 0)
  46. self.assertEqual(instance.laststop, 0)
  47. self.assertEqual(instance.delay, 0)
  48. self.assertEqual(instance.administrative_stop, 0)
  49. self.assertEqual(instance.killing, 0)
  50. self.assertEqual(instance.backoff, 0)
  51. self.assertEqual(instance.pipes, {})
  52. self.assertEqual(instance.dispatchers, {})
  53. self.assertEqual(instance.spawnerr, None)
  54. def test_repr(self):
  55. options = DummyOptions()
  56. config = DummyPConfig(options, 'cat', 'bin/cat')
  57. instance = self._makeOne(config)
  58. s = repr(instance)
  59. self.assertTrue(s.startswith('<Subprocess at'))
  60. self.assertTrue(s.endswith('with name cat in state STOPPED>'))
  61. def test_reopenlogs(self):
  62. options = DummyOptions()
  63. config = DummyPConfig(options, 'test', '/test')
  64. instance = self._makeOne(config)
  65. instance.dispatchers = {0:DummyDispatcher(readable=True),
  66. 1:DummyDispatcher(writable=True)}
  67. instance.reopenlogs()
  68. self.assertEqual(instance.dispatchers[0].logs_reopened, True)
  69. self.assertEqual(instance.dispatchers[1].logs_reopened, False)
  70. def test_removelogs(self):
  71. options = DummyOptions()
  72. config = DummyPConfig(options, 'test', '/test')
  73. instance = self._makeOne(config)
  74. instance.dispatchers = {0:DummyDispatcher(readable=True),
  75. 1:DummyDispatcher(writable=True)}
  76. instance.removelogs()
  77. self.assertEqual(instance.dispatchers[0].logs_removed, True)
  78. self.assertEqual(instance.dispatchers[1].logs_removed, False)
  79. def test_drain(self):
  80. options = DummyOptions()
  81. config = DummyPConfig(options, 'test', '/test',
  82. stdout_logfile='/tmp/foo',
  83. stderr_logfile='/tmp/bar')
  84. instance = self._makeOne(config)
  85. instance.dispatchers = {0:DummyDispatcher(readable=True),
  86. 1:DummyDispatcher(writable=True)}
  87. instance.drain()
  88. self.assertTrue(instance.dispatchers[0].read_event_handled)
  89. self.assertTrue(instance.dispatchers[1].write_event_handled)
  90. def test_get_execv_args_bad_command_extraquote(self):
  91. options = DummyOptions()
  92. config = DummyPConfig(options, 'extraquote', 'extraquote"')
  93. instance = self._makeOne(config)
  94. self.assertRaises(BadCommand, instance.get_execv_args)
  95. def test_get_execv_args_bad_command_empty(self):
  96. options = DummyOptions()
  97. config = DummyPConfig(options, 'empty', '')
  98. instance = self._makeOne(config)
  99. self.assertRaises(BadCommand, instance.get_execv_args)
  100. def test_get_execv_args_bad_command_whitespaceonly(self):
  101. options = DummyOptions()
  102. config = DummyPConfig(options, 'whitespaceonly', ' \t ')
  103. instance = self._makeOne(config)
  104. self.assertRaises(BadCommand, instance.get_execv_args)
  105. def test_get_execv_args_abs_missing(self):
  106. options = DummyOptions()
  107. config = DummyPConfig(options, 'notthere', '/notthere')
  108. instance = self._makeOne(config)
  109. args = instance.get_execv_args()
  110. self.assertEqual(args, ('/notthere', ['/notthere']))
  111. def test_get_execv_args_abs_withquotes_missing(self):
  112. options = DummyOptions()
  113. config = DummyPConfig(options, 'notthere', '/notthere "an argument"')
  114. instance = self._makeOne(config)
  115. args = instance.get_execv_args()
  116. self.assertEqual(args, ('/notthere', ['/notthere', 'an argument']))
  117. def test_get_execv_args_rel_missing(self):
  118. options = DummyOptions()
  119. config = DummyPConfig(options, 'notthere', 'notthere')
  120. instance = self._makeOne(config)
  121. args = instance.get_execv_args()
  122. self.assertEqual(args, ('notthere', ['notthere']))
  123. def test_get_execv_args_rel_withquotes_missing(self):
  124. options = DummyOptions()
  125. config = DummyPConfig(options, 'notthere', 'notthere "an argument"')
  126. instance = self._makeOne(config)
  127. args = instance.get_execv_args()
  128. self.assertEqual(args, ('notthere', ['notthere', 'an argument']))
  129. def test_get_execv_args_abs(self):
  130. executable = '/bin/sh foo'
  131. options = DummyOptions()
  132. config = DummyPConfig(options, 'sh', executable)
  133. instance = self._makeOne(config)
  134. args = instance.get_execv_args()
  135. self.assertEqual(len(args), 2)
  136. self.assertEqual(args[0], '/bin/sh')
  137. self.assertEqual(args[1], ['/bin/sh', 'foo'])
  138. def test_get_execv_args_rel(self):
  139. executable = 'sh foo'
  140. options = DummyOptions()
  141. config = DummyPConfig(options, 'sh', executable)
  142. instance = self._makeOne(config)
  143. args = instance.get_execv_args()
  144. self.assertEqual(len(args), 2)
  145. self.assertEqual(args[0], '/bin/sh')
  146. self.assertEqual(args[1], ['sh', 'foo'])
  147. def test_record_spawnerr(self):
  148. options = DummyOptions()
  149. config = DummyPConfig(options, 'test', '/test')
  150. instance = self._makeOne(config)
  151. instance.record_spawnerr('foo')
  152. self.assertEqual(instance.spawnerr, 'foo')
  153. self.assertEqual(options.logger.data[0], 'spawnerr: foo')
  154. def test_spawn_already_running(self):
  155. options = DummyOptions()
  156. config = DummyPConfig(options, 'sh', '/bin/sh')
  157. instance = self._makeOne(config)
  158. instance.pid = True
  159. from supervisor.states import ProcessStates
  160. instance.state = ProcessStates.RUNNING
  161. result = instance.spawn()
  162. self.assertEqual(result, None)
  163. self.assertEqual(options.logger.data[0], "process 'sh' already running")
  164. self.assertEqual(instance.state, ProcessStates.RUNNING)
  165. def test_spawn_fail_check_execv_args(self):
  166. options = DummyOptions()
  167. config = DummyPConfig(options, 'bad', '/bad/filename')
  168. instance = self._makeOne(config)
  169. from supervisor.states import ProcessStates
  170. instance.state = ProcessStates.BACKOFF
  171. from supervisor import events
  172. L = []
  173. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  174. result = instance.spawn()
  175. self.assertEqual(result, None)
  176. self.assertEqual(instance.spawnerr, 'bad filename')
  177. self.assertEqual(options.logger.data[0], "spawnerr: bad filename")
  178. self.assertTrue(instance.delay)
  179. self.assertTrue(instance.backoff)
  180. from supervisor.states import ProcessStates
  181. self.assertEqual(instance.state, ProcessStates.BACKOFF)
  182. self.assertEqual(len(L), 2)
  183. event1 = L[0]
  184. event2 = L[1]
  185. self.assertEqual(event1.__class__, events.ProcessStateStartingEvent)
  186. self.assertEqual(event2.__class__, events.ProcessStateBackoffEvent)
  187. def test_spawn_fail_make_pipes_emfile(self):
  188. options = DummyOptions()
  189. options.make_pipes_error = errno.EMFILE
  190. config = DummyPConfig(options, 'good', '/good/filename')
  191. instance = self._makeOne(config)
  192. from supervisor.states import ProcessStates
  193. instance.state = ProcessStates.BACKOFF
  194. from supervisor import events
  195. L = []
  196. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  197. result = instance.spawn()
  198. self.assertEqual(result, None)
  199. self.assertEqual(instance.spawnerr,
  200. "too many open files to spawn 'good'")
  201. self.assertEqual(options.logger.data[0],
  202. "spawnerr: too many open files to spawn 'good'")
  203. self.assertTrue(instance.delay)
  204. self.assertTrue(instance.backoff)
  205. from supervisor.states import ProcessStates
  206. self.assertEqual(instance.state, ProcessStates.BACKOFF)
  207. self.assertEqual(len(L), 2)
  208. event1, event2 = L
  209. self.assertEqual(event1.__class__, events.ProcessStateStartingEvent)
  210. self.assertEqual(event2.__class__, events.ProcessStateBackoffEvent)
  211. def test_spawn_fail_make_pipes_other(self):
  212. options = DummyOptions()
  213. options.make_pipes_error = 1
  214. config = DummyPConfig(options, 'good', '/good/filename')
  215. instance = self._makeOne(config)
  216. from supervisor.states import ProcessStates
  217. instance.state = ProcessStates.BACKOFF
  218. from supervisor import events
  219. L = []
  220. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  221. result = instance.spawn()
  222. self.assertEqual(result, None)
  223. self.assertEqual(instance.spawnerr,
  224. 'unknown error making dispatchers: EPERM')
  225. self.assertEqual(options.logger.data[0],
  226. "spawnerr: unknown error making dispatchers: EPERM")
  227. self.assertTrue(instance.delay)
  228. self.assertTrue(instance.backoff)
  229. from supervisor.states import ProcessStates
  230. self.assertEqual(instance.state, ProcessStates.BACKOFF)
  231. self.assertEqual(len(L), 2)
  232. event1, event2 = L
  233. self.assertEqual(event1.__class__, events.ProcessStateStartingEvent)
  234. self.assertEqual(event2.__class__, events.ProcessStateBackoffEvent)
  235. def test_spawn_fail_make_dispatchers_eisdir(self):
  236. options = DummyOptions()
  237. config = DummyPConfig(options, name='cat', command='/bin/cat',
  238. stdout_logfile='/a/directory') # not a file
  239. def raise_eisdir(envelope):
  240. raise IOError(errno.EISDIR)
  241. config.make_dispatchers = raise_eisdir
  242. instance = self._makeOne(config)
  243. from supervisor.states import ProcessStates
  244. instance.state = ProcessStates.BACKOFF
  245. from supervisor import events
  246. L = []
  247. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  248. result = instance.spawn()
  249. self.assertEqual(result, None)
  250. self.assertEqual(instance.spawnerr,
  251. "unknown error making dispatchers: EISDIR")
  252. self.assertEqual(options.logger.data[0],
  253. "spawnerr: unknown error making dispatchers: EISDIR")
  254. self.assertTrue(instance.delay)
  255. self.assertTrue(instance.backoff)
  256. from supervisor.states import ProcessStates
  257. self.assertEqual(instance.state, ProcessStates.BACKOFF)
  258. self.assertEqual(len(L), 2)
  259. event1, event2 = L
  260. self.assertEqual(event1.__class__, events.ProcessStateStartingEvent)
  261. self.assertEqual(event2.__class__, events.ProcessStateBackoffEvent)
  262. def test_spawn_fork_fail_eagain(self):
  263. options = DummyOptions()
  264. options.fork_error = errno.EAGAIN
  265. config = DummyPConfig(options, 'good', '/good/filename')
  266. instance = self._makeOne(config)
  267. from supervisor.states import ProcessStates
  268. instance.state = ProcessStates.BACKOFF
  269. from supervisor import events
  270. L = []
  271. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  272. result = instance.spawn()
  273. self.assertEqual(result, None)
  274. self.assertEqual(instance.spawnerr,
  275. "Too many processes in process table to spawn 'good'")
  276. self.assertEqual(options.logger.data[0],
  277. "spawnerr: Too many processes in process table to spawn 'good'")
  278. self.assertEqual(len(options.parent_pipes_closed), 6)
  279. self.assertEqual(len(options.child_pipes_closed), 6)
  280. self.assertTrue(instance.delay)
  281. self.assertTrue(instance.backoff)
  282. from supervisor.states import ProcessStates
  283. self.assertEqual(instance.state, ProcessStates.BACKOFF)
  284. self.assertEqual(len(L), 2)
  285. event1, event2 = L
  286. self.assertEqual(event1.__class__, events.ProcessStateStartingEvent)
  287. self.assertEqual(event2.__class__, events.ProcessStateBackoffEvent)
  288. def test_spawn_fork_fail_other(self):
  289. options = DummyOptions()
  290. options.fork_error = 1
  291. config = DummyPConfig(options, 'good', '/good/filename')
  292. instance = self._makeOne(config)
  293. from supervisor.states import ProcessStates
  294. instance.state = ProcessStates.BACKOFF
  295. from supervisor import events
  296. L = []
  297. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  298. result = instance.spawn()
  299. self.assertEqual(result, None)
  300. self.assertEqual(instance.spawnerr,
  301. 'unknown error during fork: EPERM')
  302. self.assertEqual(options.logger.data[0],
  303. "spawnerr: unknown error during fork: EPERM")
  304. self.assertEqual(len(options.parent_pipes_closed), 6)
  305. self.assertEqual(len(options.child_pipes_closed), 6)
  306. self.assertTrue(instance.delay)
  307. self.assertTrue(instance.backoff)
  308. from supervisor.states import ProcessStates
  309. self.assertEqual(instance.state, ProcessStates.BACKOFF)
  310. self.assertEqual(len(L), 2)
  311. event1, event2 = L
  312. self.assertEqual(event1.__class__, events.ProcessStateStartingEvent)
  313. self.assertEqual(event2.__class__, events.ProcessStateBackoffEvent)
  314. def test_spawn_as_child_setuid_ok(self):
  315. options = DummyOptions()
  316. options.forkpid = 0
  317. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  318. instance = self._makeOne(config)
  319. result = instance.spawn()
  320. self.assertEqual(result, None)
  321. self.assertEqual(options.parent_pipes_closed, None)
  322. self.assertEqual(options.child_pipes_closed, None)
  323. self.assertEqual(options.pgrp_set, True)
  324. self.assertEqual(len(options.duped), 3)
  325. self.assertEqual(len(options.fds_closed), options.minfds - 3)
  326. self.assertEqual(options.privsdropped, 1)
  327. self.assertEqual(options.execv_args,
  328. ('/good/filename', ['/good/filename']) )
  329. self.assertEqual(options.execve_called, True)
  330. # if the real execve() succeeds, the code that writes the
  331. # "was not spawned" message won't be reached. this assertion
  332. # is to test that no other errors were written.
  333. self.assertEqual(options.written,
  334. {2: "supervisor: child process was not spawned\n"})
  335. def test_spawn_as_child_setuid_fail(self):
  336. options = DummyOptions()
  337. options.forkpid = 0
  338. options.setuid_msg = 'failure reason'
  339. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  340. instance = self._makeOne(config)
  341. result = instance.spawn()
  342. self.assertEqual(result, None)
  343. self.assertEqual(options.parent_pipes_closed, None)
  344. self.assertEqual(options.child_pipes_closed, None)
  345. self.assertEqual(options.pgrp_set, True)
  346. self.assertEqual(len(options.duped), 3)
  347. self.assertEqual(len(options.fds_closed), options.minfds - 3)
  348. self.assertEqual(options.written,
  349. {2: "supervisor: couldn't setuid to 1: failure reason\n"
  350. "supervisor: child process was not spawned\n"})
  351. self.assertEqual(options.privsdropped, None)
  352. self.assertEqual(options.execve_called, False)
  353. self.assertEqual(options._exitcode, 127)
  354. def test_spawn_as_child_cwd_ok(self):
  355. options = DummyOptions()
  356. options.forkpid = 0
  357. config = DummyPConfig(options, 'good', '/good/filename',
  358. directory='/tmp')
  359. instance = self._makeOne(config)
  360. result = instance.spawn()
  361. self.assertEqual(result, None)
  362. self.assertEqual(options.parent_pipes_closed, None)
  363. self.assertEqual(options.child_pipes_closed, None)
  364. self.assertEqual(options.pgrp_set, True)
  365. self.assertEqual(len(options.duped), 3)
  366. self.assertEqual(len(options.fds_closed), options.minfds - 3)
  367. self.assertEqual(options.execv_args,
  368. ('/good/filename', ['/good/filename']) )
  369. self.assertEqual(options.changed_directory, True)
  370. self.assertEqual(options.execve_called, True)
  371. # if the real execve() succeeds, the code that writes the
  372. # "was not spawned" message won't be reached. this assertion
  373. # is to test that no other errors were written.
  374. self.assertEqual(options.written,
  375. {2: "supervisor: child process was not spawned\n"})
  376. def test_spawn_as_child_sets_umask(self):
  377. options = DummyOptions()
  378. options.forkpid = 0
  379. config = DummyPConfig(options, 'good', '/good/filename', umask=002)
  380. instance = self._makeOne(config)
  381. result = instance.spawn()
  382. self.assertEqual(result, None)
  383. self.assertEqual(options.execv_args,
  384. ('/good/filename', ['/good/filename']) )
  385. self.assertEqual(options.umaskset, 002)
  386. self.assertEqual(options.execve_called, True)
  387. # if the real execve() succeeds, the code that writes the
  388. # "was not spawned" message won't be reached. this assertion
  389. # is to test that no other errors were written.
  390. self.assertEqual(options.written,
  391. {2: "supervisor: child process was not spawned\n"})
  392. def test_spawn_as_child_cwd_fail(self):
  393. options = DummyOptions()
  394. options.forkpid = 0
  395. options.chdir_error = 2
  396. config = DummyPConfig(options, 'good', '/good/filename',
  397. directory='/tmp')
  398. instance = self._makeOne(config)
  399. result = instance.spawn()
  400. self.assertEqual(result, None)
  401. self.assertEqual(options.parent_pipes_closed, None)
  402. self.assertEqual(options.child_pipes_closed, None)
  403. self.assertEqual(options.pgrp_set, True)
  404. self.assertEqual(len(options.duped), 3)
  405. self.assertEqual(len(options.fds_closed), options.minfds - 3)
  406. self.assertEqual(options.execv_args, None)
  407. out = {2: "supervisor: couldn't chdir to /tmp: ENOENT\n"
  408. "supervisor: child process was not spawned\n"}
  409. self.assertEqual(options.written, out)
  410. self.assertEqual(options._exitcode, 127)
  411. self.assertEqual(options.changed_directory, False)
  412. self.assertEqual(options.execve_called, False)
  413. def test_spawn_as_child_execv_fail_oserror(self):
  414. options = DummyOptions()
  415. options.forkpid = 0
  416. options.execv_error = 1
  417. config = DummyPConfig(options, 'good', '/good/filename')
  418. instance = self._makeOne(config)
  419. result = instance.spawn()
  420. self.assertEqual(result, None)
  421. self.assertEqual(options.parent_pipes_closed, None)
  422. self.assertEqual(options.child_pipes_closed, None)
  423. self.assertEqual(options.pgrp_set, True)
  424. self.assertEqual(len(options.duped), 3)
  425. self.assertEqual(len(options.fds_closed), options.minfds - 3)
  426. out = {2: "supervisor: couldn't exec /good/filename: EPERM\n"
  427. "supervisor: child process was not spawned\n"}
  428. self.assertEqual(options.written, out)
  429. self.assertEqual(options.privsdropped, None)
  430. self.assertEqual(options._exitcode, 127)
  431. def test_spawn_as_child_execv_fail_runtime_error(self):
  432. options = DummyOptions()
  433. options.forkpid = 0
  434. options.execv_error = 2
  435. config = DummyPConfig(options, 'good', '/good/filename')
  436. instance = self._makeOne(config)
  437. result = instance.spawn()
  438. self.assertEqual(result, None)
  439. self.assertEqual(options.parent_pipes_closed, None)
  440. self.assertEqual(options.child_pipes_closed, None)
  441. self.assertEqual(options.pgrp_set, True)
  442. self.assertEqual(len(options.duped), 3)
  443. self.assertEqual(len(options.fds_closed), options.minfds - 3)
  444. msg = options.written[2] # dict, 2 is fd #
  445. head = "supervisor: couldn't exec /good/filename:"
  446. self.assertTrue(msg.startswith(head))
  447. self.assertTrue("exceptions.RuntimeError" in msg)
  448. self.assertEqual(options.privsdropped, None)
  449. self.assertEqual(options._exitcode, 127)
  450. def test_spawn_as_child_uses_pconfig_environment(self):
  451. options = DummyOptions()
  452. options.forkpid = 0
  453. config = DummyPConfig(options, 'cat', '/bin/cat',
  454. environment={'_TEST_':'1'})
  455. instance = self._makeOne(config)
  456. result = instance.spawn()
  457. self.assertEqual(result, None)
  458. self.assertEqual(options.execv_args, ('/bin/cat', ['/bin/cat']) )
  459. self.assertEqual(options.execv_environment['_TEST_'], '1')
  460. def test_spawn_as_child_environment_supervisor_envvars(self):
  461. options = DummyOptions()
  462. options.forkpid = 0
  463. config = DummyPConfig(options, 'cat', '/bin/cat')
  464. instance = self._makeOne(config)
  465. class Dummy:
  466. name = 'dummy'
  467. instance.group = Dummy()
  468. instance.group.config = Dummy()
  469. result = instance.spawn()
  470. self.assertEqual(result, None)
  471. self.assertEqual(options.execv_args, ('/bin/cat', ['/bin/cat']) )
  472. self.assertEqual(
  473. options.execv_environment['SUPERVISOR_ENABLED'], '1')
  474. self.assertEqual(
  475. options.execv_environment['SUPERVISOR_PROCESS_NAME'], 'cat')
  476. self.assertEqual(
  477. options.execv_environment['SUPERVISOR_GROUP_NAME'], 'dummy')
  478. self.assertEqual(
  479. options.execv_environment['SUPERVISOR_SERVER_URL'],
  480. 'http://localhost:9001')
  481. def test_spawn_as_child_stderr_redirected(self):
  482. options = DummyOptions()
  483. options.forkpid = 0
  484. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  485. config.redirect_stderr = True
  486. instance = self._makeOne(config)
  487. result = instance.spawn()
  488. self.assertEqual(result, None)
  489. self.assertEqual(options.parent_pipes_closed, None)
  490. self.assertEqual(options.child_pipes_closed, None)
  491. self.assertEqual(options.pgrp_set, True)
  492. self.assertEqual(len(options.duped), 2)
  493. self.assertEqual(len(options.fds_closed), options.minfds - 3)
  494. self.assertEqual(options.privsdropped, 1)
  495. self.assertEqual(options.execv_args,
  496. ('/good/filename', ['/good/filename']) )
  497. self.assertEqual(options.execve_called, True)
  498. # if the real execve() succeeds, the code that writes the
  499. # "was not spawned" message won't be reached. this assertion
  500. # is to test that no other errors were written.
  501. self.assertEqual(options.written,
  502. {2: "supervisor: child process was not spawned\n"})
  503. def test_spawn_as_parent(self):
  504. options = DummyOptions()
  505. options.forkpid = 10
  506. config = DummyPConfig(options, 'good', '/good/filename')
  507. instance = self._makeOne(config)
  508. result = instance.spawn()
  509. self.assertEqual(result, 10)
  510. self.assertEqual(instance.dispatchers[4].__class__, DummyDispatcher)
  511. self.assertEqual(instance.dispatchers[5].__class__, DummyDispatcher)
  512. self.assertEqual(instance.dispatchers[7].__class__, DummyDispatcher)
  513. self.assertEqual(instance.pipes['stdin'], 4)
  514. self.assertEqual(instance.pipes['stdout'], 5)
  515. self.assertEqual(instance.pipes['stderr'], 7)
  516. self.assertEqual(options.parent_pipes_closed, None)
  517. self.assertEqual(len(options.child_pipes_closed), 6)
  518. self.assertEqual(options.logger.data[0], "spawned: 'good' with pid 10")
  519. self.assertEqual(instance.spawnerr, None)
  520. self.assertTrue(instance.delay)
  521. self.assertEqual(instance.config.options.pidhistory[10], instance)
  522. from supervisor.states import ProcessStates
  523. self.assertEqual(instance.state, ProcessStates.STARTING)
  524. def test_spawn_redirect_stderr(self):
  525. options = DummyOptions()
  526. options.forkpid = 10
  527. config = DummyPConfig(options, 'good', '/good/filename',
  528. redirect_stderr=True)
  529. instance = self._makeOne(config)
  530. result = instance.spawn()
  531. self.assertEqual(result, 10)
  532. self.assertEqual(instance.dispatchers[4].__class__, DummyDispatcher)
  533. self.assertEqual(instance.dispatchers[5].__class__, DummyDispatcher)
  534. self.assertEqual(instance.pipes['stdin'], 4)
  535. self.assertEqual(instance.pipes['stdout'], 5)
  536. self.assertEqual(instance.pipes['stderr'], None)
  537. def test_write(self):
  538. executable = '/bin/cat'
  539. options = DummyOptions()
  540. config = DummyPConfig(options, 'output', executable)
  541. instance = self._makeOne(config)
  542. sent = 'a' * (1 << 13)
  543. self.assertRaises(OSError, instance.write, sent)
  544. options.forkpid = 1
  545. instance.spawn()
  546. instance.write(sent)
  547. stdin_fd = instance.pipes['stdin']
  548. self.assertEqual(sent, instance.dispatchers[stdin_fd].input_buffer)
  549. instance.killing = True
  550. self.assertRaises(OSError, instance.write, sent)
  551. def test_write_dispatcher_closed(self):
  552. executable = '/bin/cat'
  553. options = DummyOptions()
  554. config = DummyPConfig(options, 'output', executable)
  555. instance = self._makeOne(config)
  556. sent = 'a' * (1 << 13)
  557. self.assertRaises(OSError, instance.write, sent)
  558. options.forkpid = 1
  559. instance.spawn()
  560. stdin_fd = instance.pipes['stdin']
  561. instance.dispatchers[stdin_fd].close()
  562. self.assertRaises(OSError, instance.write, sent)
  563. def test_write_stdin_fd_none(self):
  564. executable = '/bin/cat'
  565. options = DummyOptions()
  566. config = DummyPConfig(options, 'output', executable)
  567. instance = self._makeOne(config)
  568. options.forkpid = 1
  569. instance.spawn()
  570. stdin_fd = instance.pipes['stdin']
  571. instance.dispatchers[stdin_fd].close()
  572. instance.pipes['stdin'] = None
  573. try:
  574. instance.write('foo')
  575. self.fail('nothing raised')
  576. except OSError, exc:
  577. self.assertEqual(exc.args[0], errno.EPIPE)
  578. self.assertEqual(exc.args[1], 'Process has no stdin channel')
  579. def test_write_dispatcher_flush_raises_epipe(self):
  580. executable = '/bin/cat'
  581. options = DummyOptions()
  582. config = DummyPConfig(options, 'output', executable)
  583. instance = self._makeOne(config)
  584. sent = 'a' * (1 << 13)
  585. self.assertRaises(OSError, instance.write, sent)
  586. options.forkpid = 1
  587. instance.spawn()
  588. stdin_fd = instance.pipes['stdin']
  589. instance.dispatchers[stdin_fd].flush_error = errno.EPIPE
  590. self.assertRaises(OSError, instance.write, sent)
  591. def _dont_test_spawn_and_kill(self):
  592. # this is a functional test
  593. from supervisor.tests.base import makeSpew
  594. try:
  595. called = 0
  596. def foo(*args):
  597. called = 1
  598. signal.signal(signal.SIGCHLD, foo)
  599. executable = makeSpew()
  600. options = DummyOptions()
  601. config = DummyPConfig(options, 'spew', executable)
  602. instance = self._makeOne(config)
  603. result = instance.spawn()
  604. msg = options.logger.data[0]
  605. self.assertTrue(msg.startswith("spawned: 'spew' with pid"))
  606. self.assertEqual(len(instance.pipes), 6)
  607. self.assertTrue(instance.pid)
  608. self.assertEqual(instance.pid, result)
  609. origpid = instance.pid
  610. while 1:
  611. try:
  612. data = os.popen('ps').read()
  613. break
  614. except IOError, why:
  615. if why.args[0] != errno.EINTR:
  616. raise
  617. # try again ;-)
  618. time.sleep(0.1) # arbitrary, race condition possible
  619. self.assertTrue(data.find(repr(origpid)) != -1 )
  620. msg = instance.kill(signal.SIGTERM)
  621. time.sleep(0.1) # arbitrary, race condition possible
  622. self.assertEqual(msg, None)
  623. pid, sts = os.waitpid(-1, os.WNOHANG)
  624. data = os.popen('ps').read()
  625. self.assertEqual(data.find(repr(origpid)), -1) # dubious
  626. finally:
  627. try:
  628. os.remove(executable)
  629. except:
  630. pass
  631. signal.signal(signal.SIGCHLD, signal.SIG_DFL)
  632. def test_stop(self):
  633. options = DummyOptions()
  634. config = DummyPConfig(options, 'test', '/test')
  635. instance = self._makeOne(config)
  636. instance.pid = 11
  637. dispatcher = DummyDispatcher(writable=True)
  638. instance.dispatchers = {'foo':dispatcher}
  639. from supervisor.states import ProcessStates
  640. instance.state = ProcessStates.RUNNING
  641. instance.laststopreport = time.time()
  642. instance.stop()
  643. self.assertEqual(instance.administrative_stop, 1)
  644. self.assertEqual(instance.laststopreport, 0)
  645. self.assertTrue(instance.delay)
  646. self.assertEqual(options.logger.data[0], 'killing test (pid 11) with '
  647. 'signal SIGTERM')
  648. self.assertEqual(instance.killing, 1)
  649. self.assertEqual(options.kills[11], signal.SIGTERM)
  650. def test_stop_not_in_stoppable_state_error(self):
  651. options = DummyOptions()
  652. config = DummyPConfig(options, 'test', '/test')
  653. instance = self._makeOne(config)
  654. instance.pid = 11
  655. dispatcher = DummyDispatcher(writable=True)
  656. instance.dispatchers = {'foo':dispatcher}
  657. from supervisor.states import ProcessStates
  658. instance.state = ProcessStates.STOPPED
  659. try:
  660. instance.stop()
  661. self.fail('nothing raised')
  662. except AssertionError, exc:
  663. self.assertEqual(exc.args[0], 'Assertion failed for test: '
  664. 'STOPPED not in RUNNING STARTING STOPPING')
  665. def test_stop_report_logs_nothing_if_not_stopping_state(self):
  666. options = DummyOptions()
  667. config = DummyPConfig(options, 'test', '/test')
  668. instance = self._makeOne(config)
  669. instance.pid = 11
  670. dispatcher = DummyDispatcher(writable=True)
  671. instance.dispatchers = {'foo':dispatcher}
  672. from supervisor.states import ProcessStates
  673. instance.state = ProcessStates.STOPPED
  674. instance.stop_report()
  675. self.assertEqual(len(options.logger.data), 0)
  676. def test_stop_report_logs_throttled_by_laststopreport(self):
  677. options = DummyOptions()
  678. config = DummyPConfig(options, 'test', '/test')
  679. instance = self._makeOne(config)
  680. instance.pid = 11
  681. dispatcher = DummyDispatcher(writable=True)
  682. instance.dispatchers = {'foo':dispatcher}
  683. from supervisor.states import ProcessStates
  684. instance.state = ProcessStates.STOPPING
  685. self.assertEqual(instance.laststopreport, 0)
  686. instance.stop_report()
  687. self.assertEqual(len(options.logger.data), 1)
  688. self.assertEqual(options.logger.data[0], 'waiting for test to stop')
  689. self.assertNotEqual(instance.laststopreport, 0)
  690. instance.stop_report()
  691. self.assertEqual(len(options.logger.data), 1) # throttled
  692. def test_give_up(self):
  693. options = DummyOptions()
  694. config = DummyPConfig(options, 'test', '/test')
  695. instance = self._makeOne(config)
  696. L = []
  697. from supervisor.states import ProcessStates
  698. from supervisor import events
  699. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  700. instance.state = ProcessStates.BACKOFF
  701. instance.give_up()
  702. self.assertEqual(instance.system_stop, 1)
  703. self.assertFalse(instance.delay)
  704. self.assertFalse(instance.backoff)
  705. self.assertEqual(instance.state, ProcessStates.FATAL)
  706. self.assertEqual(len(L), 1)
  707. event = L[0]
  708. self.assertEqual(event.__class__, events.ProcessStateFatalEvent)
  709. def test_kill_nopid(self):
  710. options = DummyOptions()
  711. config = DummyPConfig(options, 'test', '/test')
  712. instance = self._makeOne(config)
  713. instance.kill(signal.SIGTERM)
  714. self.assertEqual(options.logger.data[0],
  715. 'attempted to kill test with sig SIGTERM but it wasn\'t running')
  716. self.assertEqual(instance.killing, 0)
  717. def test_kill_error(self):
  718. options = DummyOptions()
  719. config = DummyPConfig(options, 'test', '/test')
  720. options.kill_error = 1
  721. instance = self._makeOne(config)
  722. L = []
  723. from supervisor.states import ProcessStates
  724. from supervisor import events
  725. events.subscribe(events.ProcessStateEvent,
  726. lambda x: L.append(x))
  727. instance.pid = 11
  728. instance.state = ProcessStates.RUNNING
  729. instance.kill(signal.SIGTERM)
  730. self.assertEqual(options.logger.data[0], 'killing test (pid 11) with '
  731. 'signal SIGTERM')
  732. self.assertTrue(options.logger.data[1].startswith(
  733. 'unknown problem killing test'))
  734. self.assertEqual(instance.killing, 0)
  735. self.assertEqual(len(L), 2)
  736. event1 = L[0]
  737. event2 = L[1]
  738. self.assertEqual(event1.__class__, events.ProcessStateStoppingEvent)
  739. self.assertEqual(event2.__class__, events.ProcessStateUnknownEvent)
  740. def test_kill_from_starting(self):
  741. options = DummyOptions()
  742. config = DummyPConfig(options, 'test', '/test')
  743. instance = self._makeOne(config)
  744. instance.pid = 11
  745. L = []
  746. from supervisor.states import ProcessStates
  747. from supervisor import events
  748. events.subscribe(events.ProcessStateEvent,lambda x: L.append(x))
  749. instance.state = ProcessStates.STARTING
  750. instance.kill(signal.SIGTERM)
  751. self.assertEqual(options.logger.data[0], 'killing test (pid 11) with '
  752. 'signal SIGTERM')
  753. self.assertEqual(instance.killing, 1)
  754. self.assertEqual(options.kills[11], signal.SIGTERM)
  755. self.assertEqual(len(L), 1)
  756. event = L[0]
  757. self.assertEqual(event.__class__, events.ProcessStateStoppingEvent)
  758. def test_kill_from_running(self):
  759. options = DummyOptions()
  760. config = DummyPConfig(options, 'test', '/test')
  761. instance = self._makeOne(config)
  762. instance.pid = 11
  763. L = []
  764. from supervisor.states import ProcessStates
  765. from supervisor import events
  766. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  767. instance.state = ProcessStates.RUNNING
  768. instance.kill(signal.SIGTERM)
  769. self.assertEqual(options.logger.data[0], 'killing test (pid 11) with '
  770. 'signal SIGTERM')
  771. self.assertEqual(instance.killing, 1)
  772. self.assertEqual(options.kills[11], signal.SIGTERM)
  773. self.assertEqual(len(L), 1)
  774. event = L[0]
  775. self.assertEqual(event.__class__, events.ProcessStateStoppingEvent)
  776. def test_kill_from_stopping(self):
  777. options = DummyOptions()
  778. config = DummyPConfig(options, 'test', '/test')
  779. instance = self._makeOne(config)
  780. instance.pid = 11
  781. L = []
  782. from supervisor.states import ProcessStates
  783. from supervisor import events
  784. events.subscribe(events.Event,lambda x: L.append(x))
  785. instance.state = ProcessStates.STOPPING
  786. instance.kill(signal.SIGKILL)
  787. self.assertEqual(options.logger.data[0], 'killing test (pid 11) with '
  788. 'signal SIGKILL')
  789. self.assertEqual(instance.killing, 1)
  790. self.assertEqual(options.kills[11], signal.SIGKILL)
  791. self.assertEqual(L, []) # no event because we didn't change state
  792. def test_kill_from_backoff(self):
  793. options = DummyOptions()
  794. config = DummyPConfig(options, 'test', '/test')
  795. instance = self._makeOne(config)
  796. L = []
  797. from supervisor.states import ProcessStates
  798. from supervisor import events
  799. events.subscribe(events.Event, L.append)
  800. instance.state = ProcessStates.BACKOFF
  801. instance.kill(signal.SIGKILL)
  802. self.assertEqual(options.logger.data[0],
  803. 'Attempted to kill test, which is in BACKOFF state.')
  804. self.assertEqual(instance.killing, 0)
  805. event = L[0]
  806. self.assertEqual(event.__class__, events.ProcessStateStoppedEvent)
  807. def test_kill_from_stopping_w_killasgroup(self):
  808. options = DummyOptions()
  809. config = DummyPConfig(options, 'test', '/test', killasgroup=True)
  810. instance = self._makeOne(config)
  811. instance.pid = 11
  812. L = []
  813. from supervisor.states import ProcessStates
  814. from supervisor import events
  815. events.subscribe(events.Event,lambda x: L.append(x))
  816. instance.state = ProcessStates.STOPPING
  817. instance.kill(signal.SIGKILL)
  818. self.assertEqual(options.logger.data[0], 'killing test (pid 11) '
  819. 'process group with signal SIGKILL')
  820. self.assertEqual(instance.killing, 1)
  821. self.assertEqual(options.kills[-11], signal.SIGKILL)
  822. self.assertEqual(L, []) # no event because we didn't change state
  823. def test_stopasgroup(self):
  824. options = DummyOptions()
  825. config = DummyPConfig(options, 'test', '/test', stopasgroup=True)
  826. instance = self._makeOne(config)
  827. instance.pid = 11
  828. L = []
  829. from supervisor.states import ProcessStates
  830. from supervisor import events
  831. events.subscribe(events.Event,lambda x: L.append(x))
  832. instance.state = ProcessStates.RUNNING
  833. instance.kill(signal.SIGTERM)
  834. self.assertEqual(options.logger.data[0], 'killing test (pid 11) '
  835. 'process group with signal SIGTERM')
  836. self.assertEqual(instance.killing, 1)
  837. self.assertEqual(options.kills[-11], signal.SIGTERM)
  838. self.assertEqual(len(L), 1)
  839. event = L[0]
  840. self.assertEqual(event.__class__, events.ProcessStateStoppingEvent)
  841. self.assertEqual(event.extra_values, [('pid', 11)])
  842. self.assertEqual(event.from_state, ProcessStates.RUNNING)
  843. def test_finish_stopping_state(self):
  844. options = DummyOptions()
  845. config = DummyPConfig(options, 'notthere', '/notthere',
  846. stdout_logfile='/tmp/foo')
  847. instance = self._makeOne(config)
  848. instance.waitstatus = (123, 1) # pid, waitstatus
  849. instance.config.options.pidhistory[123] = instance
  850. instance.killing = True
  851. pipes = {'stdout':'','stderr':''}
  852. instance.pipes = pipes
  853. from supervisor.states import ProcessStates
  854. from supervisor import events
  855. instance.state = ProcessStates.STOPPING
  856. L = []
  857. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  858. instance.pid = 123
  859. instance.finish(123, 1)
  860. self.assertFalse(instance.killing)
  861. self.assertEqual(instance.pid, 0)
  862. self.assertEqual(options.parent_pipes_closed, pipes)
  863. self.assertEqual(instance.pipes, {})
  864. self.assertEqual(instance.dispatchers, {})
  865. self.assertEqual(options.logger.data[0], 'stopped: notthere '
  866. '(terminated by SIGHUP)')
  867. self.assertEqual(instance.exitstatus, -1)
  868. self.assertEqual(len(L), 1)
  869. event = L[0]
  870. self.assertEqual(event.__class__, events.ProcessStateStoppedEvent)
  871. self.assertEqual(event.extra_values, [('pid', 123)])
  872. self.assertEqual(event.from_state, ProcessStates.STOPPING)
  873. def test_finish_running_state_exit_expected(self):
  874. options = DummyOptions()
  875. config = DummyPConfig(options, 'notthere', '/notthere',
  876. stdout_logfile='/tmp/foo')
  877. instance = self._makeOne(config)
  878. instance.config.options.pidhistory[123] = instance
  879. pipes = {'stdout':'','stderr':''}
  880. instance.pipes = pipes
  881. instance.config.exitcodes =[-1]
  882. from supervisor.states import ProcessStates
  883. from supervisor import events
  884. instance.state = ProcessStates.RUNNING
  885. L = []
  886. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  887. instance.pid = 123
  888. instance.finish(123, 1)
  889. self.assertFalse(instance.killing)
  890. self.assertEqual(instance.pid, 0)
  891. self.assertEqual(options.parent_pipes_closed, pipes)
  892. self.assertEqual(instance.pipes, {})
  893. self.assertEqual(instance.dispatchers, {})
  894. self.assertEqual(options.logger.data[0],
  895. 'exited: notthere (terminated by SIGHUP; expected)')
  896. self.assertEqual(instance.exitstatus, -1)
  897. self.assertEqual(len(L), 1)
  898. event = L[0]
  899. self.assertEqual(event.__class__,
  900. events.ProcessStateExitedEvent)
  901. self.assertEqual(event.expected, True)
  902. self.assertEqual(event.extra_values, [('expected', True), ('pid', 123)])
  903. self.assertEqual(event.from_state, ProcessStates.RUNNING)
  904. def test_finish_starting_state_laststart_in_future(self):
  905. options = DummyOptions()
  906. config = DummyPConfig(options, 'notthere', '/notthere',
  907. stdout_logfile='/tmp/foo')
  908. instance = self._makeOne(config)
  909. instance.config.options.pidhistory[123] = instance
  910. pipes = {'stdout':'','stderr':''}
  911. instance.pipes = pipes
  912. instance.config.exitcodes =[-1]
  913. instance.laststart = time.time() + 3600 # 1 hour into the future
  914. from supervisor.states import ProcessStates
  915. from supervisor import events
  916. instance.state = ProcessStates.STARTING
  917. L = []
  918. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  919. instance.pid = 123
  920. instance.finish(123, 1)
  921. self.assertFalse(instance.killing)
  922. self.assertEqual(instance.pid, 0)
  923. self.assertEqual(options.parent_pipes_closed, pipes)
  924. self.assertEqual(instance.pipes, {})
  925. self.assertEqual(instance.dispatchers, {})
  926. self.assertEqual(options.logger.data[0],
  927. "process 'notthere' (123) laststart time is in the "
  928. "future, don't know how long process was running so "
  929. "assuming it did not exit too quickly")
  930. self.assertEqual(options.logger.data[1],
  931. 'exited: notthere (terminated by SIGHUP; expected)')
  932. self.assertEqual(instance.exitstatus, -1)
  933. self.assertEqual(len(L), 2)
  934. event = L[0]
  935. self.assertEqual(event.__class__, events.ProcessStateRunningEvent)
  936. self.assertEqual(event.expected, True)
  937. self.assertEqual(event.extra_values, [('pid', 123)])
  938. self.assertEqual(event.from_state, ProcessStates.STARTING)
  939. event = L[1]
  940. self.assertEqual(event.__class__, events.ProcessStateExitedEvent)
  941. self.assertEqual(event.expected, True)
  942. self.assertEqual(event.extra_values, [('expected', True), ('pid', 123)])
  943. self.assertEqual(event.from_state, ProcessStates.RUNNING)
  944. def test_finish_starting_state_exited_too_quickly(self):
  945. options = DummyOptions()
  946. config = DummyPConfig(options, 'notthere', '/notthere',
  947. stdout_logfile='/tmp/foo', startsecs=10)
  948. instance = self._makeOne(config)
  949. instance.config.options.pidhistory[123] = instance
  950. pipes = {'stdout':'','stderr':''}
  951. instance.pipes = pipes
  952. instance.config.exitcodes =[-1]
  953. instance.laststart = time.time()
  954. from supervisor.states import ProcessStates
  955. from supervisor import events
  956. instance.state = ProcessStates.STARTING
  957. L = []
  958. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  959. instance.pid = 123
  960. instance.finish(123, 1)
  961. self.assertFalse(instance.killing)
  962. self.assertEqual(instance.pid, 0)
  963. self.assertEqual(options.parent_pipes_closed, pipes)
  964. self.assertEqual(instance.pipes, {})
  965. self.assertEqual(instance.dispatchers, {})
  966. self.assertEqual(options.logger.data[0],
  967. 'exited: notthere (terminated by SIGHUP; not expected)')
  968. self.assertEqual(instance.exitstatus, None)
  969. self.assertEqual(len(L), 1)
  970. event = L[0]
  971. self.assertEqual(event.__class__, events.ProcessStateBackoffEvent)
  972. self.assertEqual(event.from_state, ProcessStates.STARTING)
  973. def test_finish_running_state_laststart_in_future(self):
  974. options = DummyOptions()
  975. config = DummyPConfig(options, 'notthere', '/notthere',
  976. stdout_logfile='/tmp/foo')
  977. instance = self._makeOne(config)
  978. instance.config.options.pidhistory[123] = instance
  979. pipes = {'stdout':'','stderr':''}
  980. instance.pipes = pipes
  981. instance.config.exitcodes =[-1]
  982. instance.laststart = time.time() + 3600 # 1 hour into the future
  983. from supervisor.states import ProcessStates
  984. from supervisor import events
  985. instance.state = ProcessStates.RUNNING
  986. L = []
  987. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  988. instance.pid = 123
  989. instance.finish(123, 1)
  990. self.assertFalse(instance.killing)
  991. self.assertEqual(instance.pid, 0)
  992. self.assertEqual(options.parent_pipes_closed, pipes)
  993. self.assertEqual(instance.pipes, {})
  994. self.assertEqual(instance.dispatchers, {})
  995. self.assertEqual(options.logger.data[0],
  996. "process 'notthere' (123) laststart time is in the "
  997. "future, don't know how long process was running so "
  998. "assuming it did not exit too quickly")
  999. self.assertEqual(options.logger.data[1],
  1000. 'exited: notthere (terminated by SIGHUP; expected)')
  1001. self.assertEqual(instance.exitstatus, -1)
  1002. self.assertEqual(len(L), 1)
  1003. event = L[0]
  1004. self.assertEqual(event.__class__,
  1005. events.ProcessStateExitedEvent)
  1006. self.assertEqual(event.expected, True)
  1007. self.assertEqual(event.extra_values, [('expected', True), ('pid', 123)])
  1008. self.assertEqual(event.from_state, ProcessStates.RUNNING)
  1009. def test_finish_with_current_event_sends_rejected(self):
  1010. from supervisor import events
  1011. L = []
  1012. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1013. events.subscribe(events.EventRejectedEvent, lambda x: L.append(x))
  1014. options = DummyOptions()
  1015. config = DummyPConfig(options, 'notthere', '/notthere',
  1016. stdout_logfile='/tmp/foo', startsecs=10)
  1017. instance = self._makeOne(config)
  1018. from supervisor.states import ProcessStates
  1019. instance.state = ProcessStates.RUNNING
  1020. event = DummyEvent()
  1021. instance.event = event
  1022. instance.finish(123, 1)
  1023. self.assertEqual(len(L), 2)
  1024. event1, event2 = L
  1025. self.assertEqual(event1.__class__,
  1026. events.ProcessStateExitedEvent)
  1027. self.assertEqual(event2.__class__, events.EventRejectedEvent)
  1028. self.assertEqual(event2.process, instance)
  1029. self.assertEqual(event2.event, event)
  1030. self.assertEqual(instance.event, None)
  1031. def test_set_uid_no_uid(self):
  1032. options = DummyOptions()
  1033. config = DummyPConfig(options, 'test', '/test')
  1034. instance = self._makeOne(config)
  1035. instance.set_uid()
  1036. self.assertEqual(options.privsdropped, None)
  1037. def test_set_uid(self):
  1038. options = DummyOptions()
  1039. config = DummyPConfig(options, 'test', '/test', uid=1)
  1040. instance = self._makeOne(config)
  1041. msg = instance.set_uid()
  1042. self.assertEqual(options.privsdropped, 1)
  1043. self.assertEqual(msg, None)
  1044. def test_cmp_bypriority(self):
  1045. options = DummyOptions()
  1046. config = DummyPConfig(options, 'notthere', '/notthere',
  1047. stdout_logfile='/tmp/foo',
  1048. priority=1)
  1049. instance = self._makeOne(config)
  1050. config = DummyPConfig(options, 'notthere1', '/notthere',
  1051. stdout_logfile='/tmp/foo',
  1052. priority=2)
  1053. instance1 = self._makeOne(config)
  1054. config = DummyPConfig(options, 'notthere2', '/notthere',
  1055. stdout_logfile='/tmp/foo',
  1056. priority=3)
  1057. instance2 = self._makeOne(config)
  1058. L = [instance2, instance, instance1]
  1059. L.sort()
  1060. self.assertEqual(L, [instance, instance1, instance2])
  1061. def test_transition_stopped_to_starting_supervisor_stopping(self):
  1062. from supervisor import events
  1063. L = []
  1064. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1065. from supervisor.states import ProcessStates, SupervisorStates
  1066. options = DummyOptions()
  1067. options.mood = SupervisorStates.SHUTDOWN
  1068. # this should not be spawned, as supervisor is shutting down
  1069. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1070. process = self._makeOne(pconfig)
  1071. process.laststart = 0
  1072. process.state = ProcessStates.STOPPED
  1073. process.transition()
  1074. self.assertEqual(process.state, ProcessStates.STOPPED)
  1075. self.assertEqual(L, [])
  1076. def test_transition_stopped_to_starting_supervisor_running(self):
  1077. from supervisor import events
  1078. L = []
  1079. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1080. from supervisor.states import ProcessStates, SupervisorStates
  1081. options = DummyOptions()
  1082. options.mood = SupervisorStates.RUNNING
  1083. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1084. process = self._makeOne(pconfig)
  1085. process.laststart = 0
  1086. process.state = ProcessStates.STOPPED
  1087. process.transition()
  1088. self.assertEqual(process.state, ProcessStates.STARTING)
  1089. self.assertEqual(len(L), 1)
  1090. event = L[0]
  1091. self.assertEqual(event.__class__, events.ProcessStateStartingEvent)
  1092. def test_transition_exited_to_starting_supervisor_stopping(self):
  1093. from supervisor import events
  1094. L = []
  1095. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1096. from supervisor.states import ProcessStates, SupervisorStates
  1097. options = DummyOptions()
  1098. options.mood = SupervisorStates.SHUTDOWN
  1099. # this should not be spawned, as supervisor is shutting down
  1100. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1101. from supervisor.datatypes import RestartUnconditionally
  1102. pconfig.autorestart = RestartUnconditionally
  1103. process = self._makeOne(pconfig)
  1104. process.laststart = 1
  1105. process.system_stop = 1
  1106. process.state = ProcessStates.EXITED
  1107. process.transition()
  1108. self.assertEqual(process.state, ProcessStates.EXITED)
  1109. self.assertEqual(process.system_stop, 1)
  1110. self.assertEqual(L, [])
  1111. def test_transition_exited_to_starting_uncond_supervisor_running(self):
  1112. from supervisor import events
  1113. L = []
  1114. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1115. from supervisor.states import ProcessStates
  1116. options = DummyOptions()
  1117. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1118. from supervisor.datatypes import RestartUnconditionally
  1119. pconfig.autorestart = RestartUnconditionally
  1120. process = self._makeOne(pconfig)
  1121. process.laststart = 1
  1122. process.state = ProcessStates.EXITED
  1123. process.transition()
  1124. self.assertEqual(process.state, ProcessStates.STARTING)
  1125. self.assertEqual(len(L), 1)
  1126. event = L[0]
  1127. self.assertEqual(event.__class__, events.ProcessStateStartingEvent)
  1128. def test_transition_exited_to_starting_condit_supervisor_running(self):
  1129. from supervisor import events
  1130. L = []
  1131. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1132. from supervisor.states import ProcessStates
  1133. options = DummyOptions()
  1134. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1135. from supervisor.datatypes import RestartWhenExitUnexpected
  1136. pconfig.autorestart = RestartWhenExitUnexpected
  1137. process = self._makeOne(pconfig)
  1138. process.laststart = 1
  1139. process.state = ProcessStates.EXITED
  1140. process.exitstatus = 'bogus'
  1141. process.transition()
  1142. self.assertEqual(process.state, ProcessStates.STARTING)
  1143. self.assertEqual(len(L), 1)
  1144. event = L[0]
  1145. self.assertEqual(event.__class__, events.ProcessStateStartingEvent)
  1146. def test_transition_exited_to_starting_condit_fls_supervisor_running(self):
  1147. from supervisor import events
  1148. L = []
  1149. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1150. from supervisor.states import ProcessStates
  1151. options = DummyOptions()
  1152. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1153. from supervisor.datatypes import RestartWhenExitUnexpected
  1154. pconfig.autorestart = RestartWhenExitUnexpected
  1155. process = self._makeOne(pconfig)
  1156. process.laststart = 1
  1157. process.state = ProcessStates.EXITED
  1158. process.exitstatus = 0
  1159. process.transition()
  1160. self.assertEqual(process.state, ProcessStates.EXITED)
  1161. self.assertEqual(L, [])
  1162. def test_transition_backoff_to_starting_supervisor_stopping(self):
  1163. from supervisor import events
  1164. L = []
  1165. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1166. from supervisor.states import ProcessStates, SupervisorStates
  1167. options = DummyOptions()
  1168. options.mood = SupervisorStates.SHUTDOWN
  1169. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1170. process = self._makeOne(pconfig)
  1171. process.laststart = 1
  1172. process.delay = 0
  1173. process.backoff = 0
  1174. process.state = ProcessStates.BACKOFF
  1175. process.transition()
  1176. self.assertEqual(process.state, ProcessStates.BACKOFF)
  1177. self.assertEqual(L, [])
  1178. def test_transition_backoff_to_starting_supervisor_running(self):
  1179. from supervisor import events
  1180. L = []
  1181. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1182. from supervisor.states import ProcessStates, SupervisorStates
  1183. options = DummyOptions()
  1184. options.mood = SupervisorStates.RUNNING
  1185. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1186. process = self._makeOne(pconfig)
  1187. process.laststart = 1
  1188. process.delay = 0
  1189. process.backoff = 0
  1190. process.state = ProcessStates.BACKOFF
  1191. process.transition()
  1192. self.assertEqual(process.state, ProcessStates.STARTING)
  1193. self.assertEqual(len(L), 1)
  1194. self.assertEqual(L[0].__class__, events.ProcessStateStartingEvent)
  1195. def test_transition_backoff_to_starting_supervisor_running_notyet(self):
  1196. from supervisor import events
  1197. L = []
  1198. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1199. from supervisor.states import ProcessStates, SupervisorStates
  1200. options = DummyOptions()
  1201. options.mood = SupervisorStates.RUNNING
  1202. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1203. process = self._makeOne(pconfig)
  1204. process.laststart = 1
  1205. process.delay = sys.maxint
  1206. process.backoff = 0
  1207. process.state = ProcessStates.BACKOFF
  1208. process.transition()
  1209. self.assertEqual(process.state, ProcessStates.BACKOFF)
  1210. self.assertEqual(L, [])
  1211. def test_transition_starting_to_running(self):
  1212. from supervisor import events
  1213. L = []
  1214. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1215. from supervisor.states import ProcessStates
  1216. options = DummyOptions()
  1217. # this should go from STARTING to RUNNING via transition()
  1218. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1219. process = self._makeOne(pconfig)
  1220. process.backoff = 1
  1221. process.delay = 1
  1222. process.system_stop = 0
  1223. process.laststart = 1
  1224. process.pid = 1
  1225. process.stdout_buffer = 'abc'
  1226. process.stderr_buffer = 'def'
  1227. process.state = ProcessStates.STARTING
  1228. process.transition()
  1229. # this implies RUNNING
  1230. self.assertEqual(process.backoff, 0)
  1231. self.assertEqual(process.delay, 0)
  1232. self.assertEqual(process.system_stop, 0)
  1233. self.assertEqual(options.logger.data[0],
  1234. 'success: process entered RUNNING state, process has '
  1235. 'stayed up for > than 10 seconds (startsecs)')
  1236. self.assertEqual(len(L), 1)
  1237. event = L[0]
  1238. self.assertEqual(event.__class__, events.ProcessStateRunningEvent)
  1239. def test_transition_backoff_to_fatal(self):
  1240. from supervisor import events
  1241. L = []
  1242. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1243. from supervisor.states import ProcessStates
  1244. options = DummyOptions()
  1245. # this should go from BACKOFF to FATAL via transition()
  1246. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1247. process = self._makeOne(pconfig)
  1248. process.laststart = 1
  1249. process.backoff = 10000
  1250. process.delay = 1
  1251. process.system_stop = 0
  1252. process.stdout_buffer = 'abc'
  1253. process.stderr_buffer = 'def'
  1254. process.state = ProcessStates.BACKOFF
  1255. process.transition()
  1256. # this implies FATAL
  1257. self.assertEqual(process.backoff, 0)
  1258. self.assertEqual(process.delay, 0)
  1259. self.assertEqual(process.system_stop, 1)
  1260. self.assertEqual(options.logger.data[0],
  1261. 'gave up: process entered FATAL state, too many start'
  1262. ' retries too quickly')
  1263. self.assertEqual(len(L), 1)
  1264. event = L[0]
  1265. self.assertEqual(event.__class__, events.ProcessStateFatalEvent)
  1266. def test_transition_stops_unkillable_notyet(self):
  1267. from supervisor import events
  1268. L = []
  1269. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1270. from supervisor.states import ProcessStates
  1271. options = DummyOptions()
  1272. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1273. process = self._makeOne(pconfig)
  1274. process.delay = sys.maxint
  1275. process.state = ProcessStates.STOPPING
  1276. process.transition()
  1277. self.assertEqual(process.state, ProcessStates.STOPPING)
  1278. self.assertEqual(L, [])
  1279. def test_transition_stops_unkillable(self):
  1280. from supervisor import events
  1281. L = []
  1282. events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
  1283. from supervisor.states import ProcessStates
  1284. options = DummyOptions()
  1285. pconfig = DummyPConfig(options, 'process', 'process','/bin/process')
  1286. process = self._makeOne(pconfig)
  1287. process.delay = 0
  1288. process.pid = 1
  1289. process.killing = 0
  1290. process.state = ProcessStates.STOPPING
  1291. process.transition()
  1292. self.assertEqual(process.killing, 1)
  1293. self.assertNotEqual(process.delay, 0)
  1294. self.assertEqual(process.state, ProcessStates.STOPPING)
  1295. self.assertEqual(options.logger.data[0],
  1296. "killing 'process' (1) with SIGKILL")
  1297. import signal
  1298. self.assertEqual(options.kills[1], signal.SIGKILL)
  1299. self.assertEqual(L, [])
  1300. def test_change_state_doesnt_notify_if_no_state_change(self):
  1301. options = DummyOptions()
  1302. config = DummyPConfig(options, 'test', '/test')
  1303. instance = self._makeOne(config)
  1304. instance.state = 10
  1305. self.assertEqual(instance.change_state(10), False)
  1306. def test_change_state_sets_backoff_and_delay(self):
  1307. from supervisor.states import ProcessStates
  1308. options = DummyOptions()
  1309. config = DummyPConfig(options, 'test', '/test')
  1310. instance = self._makeOne(config)
  1311. instance.state = 10
  1312. instance.change_state(ProcessStates.BACKOFF)
  1313. self.assertEqual(instance.backoff, 1)
  1314. self.assertTrue(instance.delay > 0)
  1315. class FastCGISubprocessTests(unittest.TestCase):
  1316. def _getTargetClass(self):
  1317. from supervisor.process import FastCGISubprocess
  1318. return FastCGISubprocess
  1319. def _makeOne(self, *arg, **kw):
  1320. return self._getTargetClass()(*arg, **kw)
  1321. def tearDown(self):
  1322. from supervisor.events import clear
  1323. clear()
  1324. def test_no_group(self):
  1325. options = DummyOptions()
  1326. options.forkpid = 0
  1327. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  1328. instance = self._makeOne(config)
  1329. self.assertRaises(NotImplementedError, instance.spawn)
  1330. def test_no_socket_manager(self):
  1331. options = DummyOptions()
  1332. options.forkpid = 0
  1333. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  1334. instance = self._makeOne(config)
  1335. instance.group = DummyProcessGroup(DummyPGroupConfig(options))
  1336. self.assertRaises(NotImplementedError, instance.spawn)
  1337. def test_prepare_child_fds(self):
  1338. options = DummyOptions()
  1339. options.forkpid = 0
  1340. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  1341. instance = self._makeOne(config)
  1342. sock_config = DummySocketConfig(7)
  1343. gconfig = DummyFCGIGroupConfig(options, 'whatever', 999, None,
  1344. sock_config)
  1345. instance.group = DummyFCGIProcessGroup(gconfig)
  1346. result = instance.spawn()
  1347. self.assertEqual(result, None)
  1348. self.assertEqual(len(options.duped), 3)
  1349. self.assertEqual(options.duped[7], 0)
  1350. self.assertEqual(options.duped[instance.pipes['child_stdout']], 1)
  1351. self.assertEqual(options.duped[instance.pipes['child_stderr']], 2)
  1352. self.assertEqual(len(options.fds_closed), options.minfds - 3)
  1353. def test_prepare_child_fds_stderr_redirected(self):
  1354. options = DummyOptions()
  1355. options.forkpid = 0
  1356. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  1357. config.redirect_stderr = True
  1358. instance = self._makeOne(config)
  1359. sock_config = DummySocketConfig(13)
  1360. gconfig = DummyFCGIGroupConfig(options, 'whatever', 999, None,
  1361. sock_config)
  1362. instance.group = DummyFCGIProcessGroup(gconfig)
  1363. result = instance.spawn()
  1364. self.assertEqual(result, None)
  1365. self.assertEqual(len(options.duped), 2)
  1366. self.assertEqual(options.duped[13], 0)
  1367. self.assertEqual(len(options.fds_closed), options.minfds - 3)
  1368. def test_before_spawn_gets_socket_ref(self):
  1369. options = DummyOptions()
  1370. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  1371. instance = self._makeOne(config)
  1372. sock_config = DummySocketConfig(7)
  1373. gconfig = DummyFCGIGroupConfig(options, 'whatever', 999, None,
  1374. sock_config)
  1375. instance.group = DummyFCGIProcessGroup(gconfig)
  1376. self.assertTrue(instance.fcgi_sock is None)
  1377. instance.before_spawn()
  1378. self.assertFalse(instance.fcgi_sock is None)
  1379. def test_after_finish_removes_socket_ref(self):
  1380. options = DummyOptions()
  1381. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  1382. instance = self._makeOne(config)
  1383. instance.fcgi_sock = 'hello'
  1384. instance.after_finish()
  1385. self.assertTrue(instance.fcgi_sock is None)
  1386. #Patch Subprocess.finish() method for this test to verify override
  1387. @patch.object(Subprocess, 'finish', Mock(return_value=sentinel.finish_result))
  1388. def test_finish_override(self):
  1389. options = DummyOptions()
  1390. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  1391. instance = self._makeOne(config)
  1392. instance.after_finish = Mock()
  1393. result = instance.finish(sentinel.pid, sentinel.sts)
  1394. self.assertEqual(sentinel.finish_result, result,
  1395. 'FastCGISubprocess.finish() did not pass thru result')
  1396. self.assertEqual(1, instance.after_finish.call_count,
  1397. 'FastCGISubprocess.after_finish() not called once')
  1398. finish_mock = Subprocess.finish
  1399. self.assertEqual(1, finish_mock.call_count,
  1400. 'Subprocess.finish() not called once')
  1401. pid_arg = finish_mock.call_args[0][1]
  1402. sts_arg = finish_mock.call_args[0][2]
  1403. self.assertEqual(sentinel.pid, pid_arg,
  1404. 'Subprocess.finish() pid arg was not passed')
  1405. self.assertEqual(sentinel.sts, sts_arg,
  1406. 'Subprocess.finish() sts arg was not passed')
  1407. #Patch Subprocess.spawn() method for this test to verify override
  1408. @patch.object(Subprocess, 'spawn', Mock(return_value=sentinel.ppid))
  1409. def test_spawn_override_success(self):
  1410. options = DummyOptions()
  1411. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  1412. instance = self._makeOne(config)
  1413. instance.before_spawn = Mock()
  1414. result = instance.spawn()
  1415. self.assertEqual(sentinel.ppid, result,
  1416. 'FastCGISubprocess.spawn() did not pass thru result')
  1417. self.assertEqual(1, instance.before_spawn.call_count,
  1418. 'FastCGISubprocess.before_spawn() not called once')
  1419. spawn_mock = Subprocess.spawn
  1420. self.assertEqual(1, spawn_mock.call_count,
  1421. 'Subprocess.spawn() not called once')
  1422. #Patch Subprocess.spawn() method for this test to verify error handling
  1423. @patch.object(Subprocess, 'spawn', Mock(return_value=None))
  1424. def test_spawn_error(self):
  1425. options = DummyOptions()
  1426. config = DummyPConfig(options, 'good', '/good/filename', uid=1)
  1427. instance = self._makeOne(config)
  1428. instance.before_spawn = Mock()
  1429. instance.fcgi_sock = 'nuke me on error'
  1430. result = instance.spawn()
  1431. self.assertEqual(None, result,
  1432. 'FastCGISubprocess.spawn() did return None on error')
  1433. self.assertEqual(1, instance.before_spawn.call_count,
  1434. 'FastCGISubprocess.before_spawn() not called once')
  1435. self.assertEqual(None, instance.fcgi_sock,
  1436. 'FastCGISubprocess.spawn() did not remove sock ref on error')
  1437. class ProcessGroupBaseTests(unittest.TestCase):
  1438. def _getTargetClass(self):
  1439. from supervisor.process import ProcessGroupBase
  1440. return ProcessGroupBase
  1441. def _makeOne(self, *args, **kw):
  1442. return self._getTargetClass()(*args, **kw)
  1443. def test_get_unstopped_processes(self):
  1444. options = DummyOptions()
  1445. from supervisor.states import ProcessStates
  1446. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1447. process1 = DummyProcess(pconfig1, state=ProcessStates.STOPPING)
  1448. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1449. group = self._makeOne(gconfig)
  1450. group.processes = { 'process1': process1 }
  1451. unstopped = group.get_unstopped_processes()
  1452. self.assertEqual(unstopped, [process1])
  1453. def test_stop_all(self):
  1454. from supervisor.states import ProcessStates
  1455. options = DummyOptions()
  1456. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1457. process1 = DummyProcess(pconfig1, state=ProcessStates.STOPPED)
  1458. pconfig2 = DummyPConfig(options, 'process2', 'process2','/bin/process2')
  1459. process2 = DummyProcess(pconfig2, state=ProcessStates.RUNNING)
  1460. pconfig3 = DummyPConfig(options, 'process3', 'process3','/bin/process3')
  1461. process3 = DummyProcess(pconfig3, state=ProcessStates.STARTING)
  1462. pconfig4 = DummyPConfig(options, 'process4', 'process4','/bin/process4')
  1463. process4 = DummyProcess(pconfig4, state=ProcessStates.BACKOFF)
  1464. process4.delay = 1000
  1465. process4.backoff = 10
  1466. gconfig = DummyPGroupConfig(
  1467. options,
  1468. pconfigs=[pconfig1, pconfig2, pconfig3, pconfig4])
  1469. group = self._makeOne(gconfig)
  1470. group.processes = {'process1': process1, 'process2': process2,
  1471. 'process3':process3, 'process4':process4}
  1472. group.stop_all()
  1473. self.assertEqual(process1.stop_called, False)
  1474. self.assertEqual(process2.stop_called, True)
  1475. self.assertEqual(process3.stop_called, True)
  1476. self.assertEqual(process4.stop_called, False)
  1477. self.assertEqual(process4.state, ProcessStates.FATAL)
  1478. def test_get_dispatchers(self):
  1479. options = DummyOptions()
  1480. from supervisor.states import ProcessStates
  1481. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1482. process1 = DummyProcess(pconfig1, state=ProcessStates.STOPPING)
  1483. process1.dispatchers = {4:None}
  1484. pconfig2 = DummyPConfig(options, 'process2', 'process2','/bin/process2')
  1485. process2 = DummyProcess(pconfig2, state=ProcessStates.STOPPING)
  1486. process2.dispatchers = {5:None}
  1487. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1, pconfig2])
  1488. group = self._makeOne(gconfig)
  1489. group.processes = { 'process1': process1, 'process2': process2 }
  1490. result= group.get_dispatchers()
  1491. self.assertEqual(result, {4:None, 5:None})
  1492. def test_reopenlogs(self):
  1493. options = DummyOptions()
  1494. from supervisor.states import ProcessStates
  1495. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1496. process1 = DummyProcess(pconfig1, state=ProcessStates.STOPPING)
  1497. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1498. group = self._makeOne(gconfig)
  1499. group.processes = {'process1': process1}
  1500. group.reopenlogs()
  1501. self.assertEqual(process1.logs_reopened, True)
  1502. def test_removelogs(self):
  1503. options = DummyOptions()
  1504. from supervisor.states import ProcessStates
  1505. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1506. process1 = DummyProcess(pconfig1, state=ProcessStates.STOPPING)
  1507. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1508. group = self._makeOne(gconfig)
  1509. group.processes = {'process1': process1}
  1510. group.removelogs()
  1511. self.assertEqual(process1.logsremoved, True)
  1512. def test_ordering_and_comparison(self):
  1513. options = DummyOptions()
  1514. gconfig1 = DummyPGroupConfig(options)
  1515. group1 = self._makeOne(gconfig1)
  1516. gconfig2 = DummyPGroupConfig(options)
  1517. group2 = self._makeOne(gconfig2)
  1518. config3 = DummyPGroupConfig(options)
  1519. group3 = self._makeOne(config3)
  1520. group1.config.priority = 5
  1521. group2.config.priority = 1
  1522. group3.config.priority = 5
  1523. L = [group1, group2]
  1524. L.sort()
  1525. self.assertEqual(L, [group2, group1])
  1526. self.assertNotEqual(group1, group2)
  1527. self.assertEqual(group1, group3)
  1528. class ProcessGroupTests(ProcessGroupBaseTests):
  1529. def _getTargetClass(self):
  1530. from supervisor.process import ProcessGroup
  1531. return ProcessGroup
  1532. def test_repr(self):
  1533. options = DummyOptions()
  1534. gconfig = DummyPGroupConfig(options)
  1535. group = self._makeOne(gconfig)
  1536. s = repr(group)
  1537. self.assertTrue(s.startswith(
  1538. '<supervisor.process.ProcessGroup instance at'), s)
  1539. self.assertTrue(s.endswith('named whatever>'), s)
  1540. def test_transition(self):
  1541. options = DummyOptions()
  1542. from supervisor.states import ProcessStates
  1543. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1544. process1 = DummyProcess(pconfig1, state=ProcessStates.STOPPING)
  1545. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1546. group = self._makeOne(gconfig)
  1547. group.processes = {'process1': process1}
  1548. group.transition()
  1549. self.assertEqual(process1.transitioned, True)
  1550. class FastCGIProcessGroupTests(unittest.TestCase):
  1551. def _getTargetClass(self):
  1552. from supervisor.process import FastCGIProcessGroup
  1553. return FastCGIProcessGroup
  1554. def _makeOne(self, config, **kwargs):
  1555. cls = self._getTargetClass()
  1556. return cls(config, **kwargs)
  1557. def test___init__without_socket_error(self):
  1558. options = DummyOptions()
  1559. gconfig = DummyPGroupConfig(options)
  1560. gconfig.socket_config = None
  1561. class DummySocketManager(object):
  1562. def __init__(self, config, logger): pass
  1563. def get_socket(self): pass
  1564. self._makeOne(gconfig, socketManager=DummySocketManager)
  1565. # doesn't fail with exception
  1566. def test___init__with_socket_error(self):
  1567. options = DummyOptions()
  1568. gconfig = DummyPGroupConfig(options)
  1569. gconfig.socket_config = None
  1570. class DummySocketManager(object):
  1571. def __init__(self, config, logger): pass
  1572. def get_socket(self):
  1573. raise KeyError(5)
  1574. def config(self):
  1575. return 'config'
  1576. self.assertRaises(
  1577. ValueError,
  1578. self._makeOne, gconfig, socketManager=DummySocketManager
  1579. )
  1580. class EventListenerPoolTests(ProcessGroupBaseTests):
  1581. def setUp(self):
  1582. from supervisor.events import clear
  1583. clear()
  1584. def tearDown(self):
  1585. from supervisor.events import clear
  1586. clear()
  1587. def _getTargetClass(self):
  1588. from supervisor.process import EventListenerPool
  1589. return EventListenerPool
  1590. def test_ctor(self):
  1591. options = DummyOptions()
  1592. gconfig = DummyPGroupConfig(options)
  1593. class EventType:
  1594. pass
  1595. gconfig.pool_events = (EventType,)
  1596. pool = self._makeOne(gconfig)
  1597. from supervisor import events
  1598. self.assertEqual(len(events.callbacks), 2)
  1599. self.assertEqual(events.callbacks[0],
  1600. (EventType, pool._acceptEvent))
  1601. self.assertEqual(events.callbacks[1],
  1602. (events.EventRejectedEvent, pool.handle_rejected))
  1603. self.assertEqual(pool.serial, -1)
  1604. def test__eventEnvelope(self):
  1605. options = DummyOptions()
  1606. options.identifier = 'thesupervisorname'
  1607. gconfig = DummyPGroupConfig(options)
  1608. gconfig.name = 'thepoolname'
  1609. pool = self._makeOne(gconfig)
  1610. from supervisor import events
  1611. result = pool._eventEnvelope(
  1612. events.EventTypes.PROCESS_COMMUNICATION_STDOUT, 80, 20, 'payload\n')
  1613. header, payload = result.split('\n', 1)
  1614. headers = header.split()
  1615. self.assertEqual(headers[0], 'ver:3.0')
  1616. self.assertEqual(headers[1], 'server:thesupervisorname')
  1617. self.assertEqual(headers[2], 'serial:80')
  1618. self.assertEqual(headers[3], 'pool:thepoolname')
  1619. self.assertEqual(headers[4], 'poolserial:20')
  1620. self.assertEqual(headers[5], 'eventname:PROCESS_COMMUNICATION_STDOUT')
  1621. self.assertEqual(headers[6], 'len:8')
  1622. self.assertEqual(payload, 'payload\n')
  1623. def test_handle_rejected_no_overflow(self):
  1624. options = DummyOptions()
  1625. gconfig = DummyPGroupConfig(options)
  1626. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1627. process1 = DummyProcess(pconfig1)
  1628. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1629. pool = self._makeOne(gconfig)
  1630. pool.processes = {'process1': process1}
  1631. pool.event_buffer = [None, None]
  1632. class DummyEvent1:
  1633. serial = 'abc'
  1634. class DummyEvent2:
  1635. process = process1
  1636. event = DummyEvent1()
  1637. dummyevent = DummyEvent2()
  1638. dummyevent.serial = 1
  1639. pool.handle_rejected(dummyevent)
  1640. self.assertEqual(pool.event_buffer, [dummyevent.event, None, None])
  1641. def test_handle_rejected_event_buffer_overflowed(self):
  1642. options = DummyOptions()
  1643. gconfig = DummyPGroupConfig(options)
  1644. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1645. process1 = DummyProcess(pconfig1)
  1646. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1647. gconfig.buffer_size = 3
  1648. pool = self._makeOne(gconfig)
  1649. pool.processes = {'process1': process1}
  1650. class DummyEvent:
  1651. def __init__(self, serial):
  1652. self.serial = serial
  1653. class DummyRejectedEvent:
  1654. def __init__(self, serial):
  1655. self.process = process1
  1656. self.event = DummyEvent(serial)
  1657. event_a = DummyEvent('a')
  1658. event_b = DummyEvent('b')
  1659. event_c = DummyEvent('c')
  1660. rej_event = DummyRejectedEvent('rejected')
  1661. pool.event_buffer = [event_a, event_b, event_c]
  1662. pool.handle_rejected(rej_event)
  1663. serials = [ x.serial for x in pool.event_buffer ]
  1664. # we popped a, and we inserted the rejected event into the 1st pos
  1665. self.assertEqual(serials, ['rejected', 'b', 'c'])
  1666. self.assertEqual(pool.config.options.logger.data[0],
  1667. 'pool whatever event buffer overflowed, discarding event a')
  1668. def test_dispatch_pipe_error(self):
  1669. options = DummyOptions()
  1670. gconfig = DummyPGroupConfig(options)
  1671. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1672. from supervisor.states import EventListenerStates
  1673. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1674. pool = self._makeOne(gconfig)
  1675. process1 = pool.processes['process1']
  1676. process1.write_error = errno.EPIPE
  1677. process1.listener_state = EventListenerStates.READY
  1678. event = DummyEvent()
  1679. pool._acceptEvent(event)
  1680. pool.dispatch()
  1681. self.assertEqual(process1.listener_state, EventListenerStates.READY)
  1682. self.assertEqual(pool.event_buffer, [event])
  1683. self.assertEqual(options.logger.data[0],
  1684. 'rebuffering event abc for pool whatever (bufsize 0)')
  1685. def test__acceptEvent_attaches_pool_serial_and_serial(self):
  1686. from supervisor.process import GlobalSerial
  1687. options = DummyOptions()
  1688. gconfig = DummyPGroupConfig(options)
  1689. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1690. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1691. pool = self._makeOne(gconfig)
  1692. process1 = pool.processes['process1']
  1693. from supervisor.states import EventListenerStates
  1694. process1.listener_state = EventListenerStates.READY
  1695. event = DummyEvent(None)
  1696. pool._acceptEvent(event)
  1697. self.assertEqual(event.serial, GlobalSerial.serial)
  1698. self.assertEqual(event.pool_serials['whatever'], pool.serial)
  1699. def test_repr(self):
  1700. options = DummyOptions()
  1701. gconfig = DummyPGroupConfig(options)
  1702. pool = self._makeOne(gconfig)
  1703. s = repr(pool)
  1704. self.assertTrue(s.startswith(
  1705. '<supervisor.process.EventListenerPool instance at'))
  1706. self.assertTrue(s.endswith('named whatever>'))
  1707. def test_transition_nobody_ready(self):
  1708. options = DummyOptions()
  1709. from supervisor.states import ProcessStates
  1710. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1711. process1 = DummyProcess(pconfig1, state=ProcessStates.STARTING)
  1712. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1713. pool = self._makeOne(gconfig)
  1714. pool.processes = {'process1': process1}
  1715. event = DummyEvent()
  1716. event.serial = 'a'
  1717. from supervisor.states import EventListenerStates
  1718. process1.listener_state = EventListenerStates.BUSY
  1719. pool._acceptEvent(event)
  1720. pool.transition()
  1721. self.assertEqual(process1.transitioned, True)
  1722. self.assertEqual(pool.event_buffer, [event])
  1723. def test_transition_event_proc_not_running(self):
  1724. options = DummyOptions()
  1725. from supervisor.states import ProcessStates
  1726. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1727. process1 = DummyProcess(pconfig1, state=ProcessStates.STARTING)
  1728. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1729. pool = self._makeOne(gconfig)
  1730. pool.processes = {'process1': process1}
  1731. event = DummyEvent()
  1732. from supervisor.states import EventListenerStates
  1733. event.serial = 1
  1734. process1.listener_state = EventListenerStates.READY
  1735. pool._acceptEvent(event)
  1736. pool.transition()
  1737. self.assertEqual(process1.transitioned, True)
  1738. self.assertEqual(pool.event_buffer, [event])
  1739. self.assertEqual(process1.stdin_buffer, '')
  1740. self.assertEqual(process1.listener_state, EventListenerStates.READY)
  1741. def test_transition_event_proc_running(self):
  1742. options = DummyOptions()
  1743. from supervisor.states import ProcessStates
  1744. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1745. process1 = DummyProcess(pconfig1, state=ProcessStates.RUNNING)
  1746. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1747. pool = self._makeOne(gconfig)
  1748. pool.processes = {'process1': process1}
  1749. event = DummyEvent()
  1750. from supervisor.states import EventListenerStates
  1751. process1.listener_state = EventListenerStates.READY
  1752. class DummyGroup:
  1753. config = gconfig
  1754. process1.group = DummyGroup
  1755. pool._acceptEvent(event)
  1756. pool.transition()
  1757. self.assertEqual(process1.transitioned, True)
  1758. self.assertEqual(pool.event_buffer, [])
  1759. header, payload = process1.stdin_buffer.split('\n', 1)
  1760. self.assertEqual(payload, 'dummy event', payload)
  1761. self.assertEqual(process1.listener_state, EventListenerStates.BUSY)
  1762. self.assertEqual(process1.event, event)
  1763. def test_transition_event_proc_running_with_dispatch_throttle_notyet(self):
  1764. options = DummyOptions()
  1765. from supervisor.states import ProcessStates
  1766. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1767. process1 = DummyProcess(pconfig1, state=ProcessStates.RUNNING)
  1768. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1769. pool = self._makeOne(gconfig)
  1770. pool.dispatch_throttle = 5
  1771. pool.last_dispatch = time.time()
  1772. pool.processes = {'process1': process1}
  1773. event = DummyEvent()
  1774. from supervisor.states import EventListenerStates
  1775. process1.listener_state = EventListenerStates.READY
  1776. class DummyGroup:
  1777. config = gconfig
  1778. process1.group = DummyGroup
  1779. pool._acceptEvent(event)
  1780. pool.transition()
  1781. self.assertEqual(process1.transitioned, True)
  1782. self.assertEqual(pool.event_buffer, [event]) # not popped
  1783. def test_transition_event_proc_running_with_dispatch_throttle_ready(self):
  1784. options = DummyOptions()
  1785. from supervisor.states import ProcessStates
  1786. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1787. process1 = DummyProcess(pconfig1, state=ProcessStates.RUNNING)
  1788. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1789. pool = self._makeOne(gconfig)
  1790. pool.dispatch_throttle = 5
  1791. pool.last_dispatch = time.time() - 1000
  1792. pool.processes = {'process1': process1}
  1793. event = DummyEvent()
  1794. from supervisor.states import EventListenerStates
  1795. process1.listener_state = EventListenerStates.READY
  1796. class DummyGroup:
  1797. config = gconfig
  1798. process1.group = DummyGroup
  1799. pool._acceptEvent(event)
  1800. pool.transition()
  1801. self.assertEqual(process1.transitioned, True)
  1802. self.assertEqual(pool.event_buffer, [])
  1803. header, payload = process1.stdin_buffer.split('\n', 1)
  1804. self.assertEqual(payload, 'dummy event', payload)
  1805. self.assertEqual(process1.listener_state, EventListenerStates.BUSY)
  1806. self.assertEqual(process1.event, event)
  1807. def test__dispatchEvent_notready(self):
  1808. options = DummyOptions()
  1809. from supervisor.states import ProcessStates
  1810. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1811. process1 = DummyProcess(pconfig1, state=ProcessStates.STOPPED)
  1812. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1813. pool = self._makeOne(gconfig)
  1814. pool.processes = {'process1': process1}
  1815. event = DummyEvent()
  1816. pool._acceptEvent(event)
  1817. self.assertEqual(pool._dispatchEvent(event), False)
  1818. def test__dispatchEvent_proc_write_raises_non_EPIPE_OSError(self):
  1819. options = DummyOptions()
  1820. from supervisor.states import ProcessStates
  1821. pconfig1 = DummyPConfig(options, 'process1', 'process1','/bin/process1')
  1822. process1 = DummyProcess(pconfig1, state=ProcessStates.RUNNING)
  1823. def raise_epipe(envelope):
  1824. raise OSError(errno.EAGAIN)
  1825. process1.write = raise_epipe
  1826. gconfig = DummyPGroupConfig(options, pconfigs=[pconfig1])
  1827. pool = self._makeOne(gconfig)
  1828. pool.processes = {'process1': process1}
  1829. event = DummyEvent()
  1830. from supervisor.states import EventListenerStates
  1831. process1.listener_state = EventListenerStates.READY
  1832. class DummyGroup:
  1833. config = gconfig
  1834. process1.group = DummyGroup
  1835. pool._acceptEvent(event)
  1836. self.assertRaises(OSError, pool._dispatchEvent, event)
  1837. class test_new_serial(unittest.TestCase):
  1838. def _callFUT(self, inst):
  1839. from supervisor.process import new_serial
  1840. return new_serial(inst)
  1841. def test_inst_serial_is_maxint(self):
  1842. class Inst(object):
  1843. def __init__(self):
  1844. self.serial = sys.maxint
  1845. inst = Inst()
  1846. result = self._callFUT(inst)
  1847. self.assertEqual(inst.serial, 0)
  1848. self.assertEqual(result, 0)
  1849. def test_inst_serial_is_not_maxint(self):
  1850. class Inst(object):
  1851. def __init__(self):
  1852. self.serial = 1
  1853. inst = Inst()
  1854. result = self._callFUT(inst)
  1855. self.assertEqual(inst.serial, 2)
  1856. self.assertEqual(result, 2)