fix(supervision): reap owned session workers after monitor crash
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
d7397ed6cf
commit
e9b70b00b6
7 files changed
+206
-10
No files matched your search
@@ -3,6 +3,7 @@
|
||||
Only a temporary COPY of the entrypoint is redirected to fake executables;
|
||||
production has no test switches, real network probes, or accelerated timers.
|
||||
"""
|
||||
import ctypes
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
@@ -13,7 +14,7 @@ import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
FAKE = r'''#!/usr/bin/env python3
|
||||
import os, pathlib, signal, sys, time
|
||||
import ctypes, os, pathlib, signal, subprocess, sys, time
|
||||
root = pathlib.Path(os.environ['FIXTURE'])
|
||||
name = pathlib.Path(sys.argv[0]).name
|
||||
scenario = os.environ['SCENARIO']
|
||||
@@ -29,6 +30,25 @@ def linger(code=0):
|
||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(code))
|
||||
while True:
|
||||
time.sleep(.01)
|
||||
def assert_workers_gone():
|
||||
if (root / 'workers').exists():
|
||||
for pid in (root / 'workers').read_text().splitlines():
|
||||
stat = pathlib.Path('/proc/' + pid + '/stat')
|
||||
assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('old worker still live', pid)
|
||||
if (root / 'peer').exists():
|
||||
os.kill(int((root / 'peer').read_text()), 0)
|
||||
if name in ('blockcheck', 'nfq'):
|
||||
# Model Go's Setpgid check child and its NFQ worker, not a daemonized sysv
|
||||
# process. Comm deliberately contains spaces/parentheses to test /proc parsing.
|
||||
ctypes.CDLL(None).prctl(15, b'bc ) (fixture)', 0, 0, 0)
|
||||
assert os.getsid(0) != os.getpgrp(), 'fixture did not split PGID from SID'
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
with (root / 'workers').open('a') as f:
|
||||
f.write(str(os.getpid()) + '\n')
|
||||
if name == 'blockcheck':
|
||||
subprocess.Popen([str(root / 'bin' / 'nfq')])
|
||||
while True:
|
||||
time.sleep(.01)
|
||||
if name == 'awk':
|
||||
sys.exit(0)
|
||||
if name == 'date':
|
||||
@@ -64,6 +84,8 @@ if name in ('ss-server', 'ss-local'):
|
||||
sys.exit(137)
|
||||
linger()
|
||||
if name == 'zapret2':
|
||||
if scenario == 'orphans':
|
||||
assert_workers_gone()
|
||||
assert not (root / 'prepared').exists(), 'sourced prepared config'
|
||||
log(sys.argv[1])
|
||||
if scenario == 'stopfail' and sys.argv[1] == 'stop':
|
||||
@@ -77,6 +99,8 @@ if command == 'init':
|
||||
elif command == 'reconcile':
|
||||
n = increment('reconciles')
|
||||
log('reconcile')
|
||||
if scenario == 'orphans':
|
||||
assert_workers_gone()
|
||||
if scenario == 'prestartfail':
|
||||
sys.exit(78)
|
||||
if scenario in ('unknown', 'unknown75') and n == 2:
|
||||
@@ -87,6 +111,13 @@ elif command == 'monitor':
|
||||
f.write(str(os.getpid()) + '\n')
|
||||
n = increment('monitor_runs')
|
||||
log('monitor ' + str(n))
|
||||
assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session'
|
||||
if n == 1 and scenario == 'orphans':
|
||||
subprocess.Popen([str(root / 'bin' / 'blockcheck')], preexec_fn=os.setpgrp)
|
||||
while not (root / 'workers').exists() or len((root / 'workers').read_text().splitlines()) != 2:
|
||||
time.sleep(.01)
|
||||
(root / 'prepared').touch()
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'):
|
||||
sys.exit(75)
|
||||
if scenario.startswith('fatal'):
|
||||
@@ -117,7 +148,7 @@ def run(scenario, expected):
|
||||
fake = binary / 'fake'
|
||||
fake.write_text(FAKE)
|
||||
fake.chmod(0o755)
|
||||
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2'):
|
||||
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2', 'blockcheck', 'nfq'):
|
||||
(binary / name).symlink_to(fake)
|
||||
configs = root / 'private'
|
||||
configs.mkdir()
|
||||
@@ -135,12 +166,36 @@ def run(scenario, expected):
|
||||
entry.write_text(text)
|
||||
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
|
||||
SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config'))
|
||||
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15)
|
||||
peer = None
|
||||
if scenario == 'orphans':
|
||||
# Local fixture is not PID 1. Adopt orphan workers here so the test
|
||||
# can reap their verified-dead zombies rather than burden host PID 1.
|
||||
assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0
|
||||
peer = subprocess.Popen([shutil.which('sleep'), '300'], start_new_session=True)
|
||||
(root / 'peer').write_text(str(peer.pid))
|
||||
try:
|
||||
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15)
|
||||
if peer:
|
||||
assert peer.poll() is None, 'unowned helper was killed'
|
||||
finally:
|
||||
if peer:
|
||||
peer.terminate()
|
||||
peer.wait(timeout=2)
|
||||
events = (root / 'events').read_text().splitlines()
|
||||
assert result.returncode == expected, (scenario, result.returncode, result.stderr, events)
|
||||
assert b'ab' * 32 not in result.stdout + result.stderr
|
||||
for pid in (root / 'children').read_text().splitlines():
|
||||
assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid)
|
||||
if scenario == 'orphans':
|
||||
for pid in (root / 'workers').read_text().splitlines():
|
||||
stat = Path(f'/proc/{pid}/stat')
|
||||
assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('orphan survived', pid)
|
||||
try:
|
||||
reaped, _ = os.waitpid(int(pid), os.WNOHANG)
|
||||
assert reaped == int(pid), ('worker not reaped', pid)
|
||||
except ChildProcessError:
|
||||
assert not stat.exists(), ('worker was not adopted/reaped', pid)
|
||||
assert ctypes.CDLL(None).prctl(36, 0, 0, 0, 0) == 0
|
||||
# Every activation and stop must immediately follow successful reconcile.
|
||||
for i, event in enumerate(events):
|
||||
if event in ('start', 'stop'):
|
||||
@@ -148,7 +203,7 @@ def run(scenario, expected):
|
||||
return events
|
||||
|
||||
|
||||
for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window'):
|
||||
for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window', 'orphans'):
|
||||
events = run(scenario, 0)
|
||||
assert events.count('start') == 2, (scenario, events)
|
||||
assert events.count('ss-server') == 2 and events.count('ss-local') == 2
|
||||
@@ -220,4 +275,6 @@ if busybox:
|
||||
for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')):
|
||||
result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2)
|
||||
assert result.returncode == 64
|
||||
result = subprocess.run(['/bin/sh', '-c', 'helper=$1; shift; . "$helper"; proc_identity "$$"; own=$proc_sid; if signal_session "$own" KILL || signal_session 1 KILL; then exit 99; fi', 'check', str(ROOT / 'managed-target-supervision.sh')], capture_output=True, timeout=2)
|
||||
assert result.returncode == 0, ('unsafe session accepted', result.returncode)
|
||||
print('entrypoint supervision fixtures passed')
|
||||
Reference in new issue
Block a user