Files
zapret-autotune/deploy/managed-target/check_supervision.py
T
ZeusinaandGPT-6 Bot 643d0c6220 fix: prevent automatic retries of failed startup apply
Latch explicit resource apply once per PID 1 lifetime. Fail closed before launch and permanently pause after authoritative cleanup on failed or incomplete apply; retain reconciliation on successful monitor and service restarts.

Cover deterministic 180-second retry windows, launch/ACK/child/session/service failures, cancellation, successful restarts and fresh-PID1 flags. Document RAM-only reset limits and repair-only startup; no remote health acceptance is claimed.

Co-authored-by: GPT-6 <gpt-6@openai.com>
2026-10-15 08:05:27 +03:00

732 lines
36 KiB
Python

"""Isolated shell fixtures: python3 deploy/managed-target/check_supervision.py.
Only a temporary COPY of the entrypoint is redirected to fake executables;
production has no test switches, real network probes, or accelerated timers.
"""
import ctypes
import atexit
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import time
ROOT = Path(__file__).resolve().parents[2]
# Real readiness CLI checks root ownership. Use an isolated user namespace for
# offline fixtures when invoked as an ordinary user; no host-root requirement.
if os.geteuid() != 0:
os.execvp('unshare', ['unshare', '-Ur', 'python3', str(Path(__file__).resolve())])
CLI_TMP = tempfile.TemporaryDirectory(prefix='dns-readiness-cli-')
atexit.register(CLI_TMP.cleanup)
DNS_CLI = Path(CLI_TMP.name) / 'autotune'
subprocess.run(['go', 'build', '-o', str(DNS_CLI), './cmd/zapret-autotune'],
cwd=ROOT, env=dict(os.environ, GOPROXY='off', GOTOOLCHAIN='local'), check=True)
FAKE = r'''#!/usr/bin/env python3
import ctypes, json, os, pathlib, signal, subprocess, sys, time
root = pathlib.Path(os.environ['FIXTURE'])
name = pathlib.Path(sys.argv[0]).name
scenario = os.environ['SCENARIO']
def log(message):
with (root / 'events').open('a') as f:
f.write(message + '\n')
def increment(key):
path = root / key
n = int(path.read_text()) + 1 if path.exists() else 1
path.write_text(str(n))
return n
def linger(code=0):
def stop(*_):
if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh' and scenario not in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink', 'dns-prestart', 'generic-apply-dns-loss', 'generic-apply-dns-restart'):
os.kill(int((root / 'dns-pid').read_text()), 0)
sys.exit(code)
signal.signal(signal.SIGTERM, stop)
while True:
time.sleep(.01)
def assert_workers_gone():
if (root / 'workers').exists():
for pid in (root / 'workers').read_text().splitlines():
stat = pathlib.Path('/proc/' + pid + '/stat')
assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('old worker still live', pid)
if (root / 'peer').exists():
os.kill(int((root / 'peer').read_text()), 0)
if name in ('blockcheck', 'nfq'):
# Model Go's Setpgid check child and its NFQ worker, not a daemonized sysv
# process. Comm deliberately contains spaces/parentheses to test /proc parsing.
ctypes.CDLL(None).prctl(15, b'bc ) (fixture)', 0, 0, 0)
assert os.getsid(0) != os.getpgrp(), 'fixture did not split PGID from SID'
signal.signal(signal.SIGTERM, signal.SIG_IGN)
with (root / 'workers').open('a') as f:
f.write(str(os.getpid()) + '\n')
if name == 'blockcheck':
subprocess.Popen([str(root / 'bin' / 'nfq')])
while True:
time.sleep(.01)
if name == 'awk':
sys.exit(0)
if name == 'date':
n = increment('clock')
elapsed = int((root / 'elapsed').read_text()) if (root / 'elapsed').exists() else 0
print(100000 + elapsed + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0)))
sys.exit(0)
if name == 'sleep':
seconds = int(sys.argv[1])
if seconds == 60 and scenario.startswith('generic-apply-'):
# Three former 60s retry windows; the failed producer becomes healthy
# after its first call, so any unintended retry would reach native READY.
n = increment('blocked_windows')
(root / 'elapsed').write_text(str(n * 60))
log('blocked-window ' + str(n))
if n == 3:
os.kill(os.getppid(), signal.SIGTERM)
time.sleep(.01)
sys.exit(0)
if scenario.startswith('qualify-') and (root / 'qualified').exists():
log('safe-idle')
os.kill(os.getppid(), signal.SIGTERM)
time.sleep(.02)
sys.exit(0)
if scenario.startswith('generic-budget-') and seconds != 60:
os.execv(os.environ['REAL_SLEEP'], [os.environ['REAL_SLEEP'], *sys.argv[1:]])
if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
log('generic-pause')
(root / 'budget-pause').touch()
if scenario != 'generic-ss':
os.kill(os.getppid(), signal.SIGTERM)
time.sleep(.01)
sys.exit(0)
if seconds in (15, 30):
log('cooldown ' + str(seconds))
if scenario == 'cancel':
os.kill(os.getppid(), signal.SIGTERM)
time.sleep(1)
else:
time.sleep(.01)
else:
time.sleep(.01 if seconds == 1 else 2)
sys.exit(0)
if name == 'timeout':
os.execv(sys.argv[4], sys.argv[4:])
if name in ('ss-server', 'ss-local'):
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
assert 'SS_PASSWORD' not in os.environ
assert sys.argv[1] == '-c'
dns = ('127.0.0.1:5353' if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh'
else os.environ.get('SS_DNS_SERVER', '')) if name == 'ss-server' else ''
assert sys.argv[3:] == (['-d', dns] if dns else [])
if dns == '127.0.0.1:5353':
assert (root / 'private' / 'dns-ready').read_text().strip() == (root / 'dns-pid').read_text()
os.kill(int((root / 'dns-pid').read_text()), 0)
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
log(name)
if name == 'ss-local':
(root / 'ss-pid').write_text(str(os.getpid()))
if scenario.startswith('generic-budget-'):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
time.sleep(.01)
if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail', 'generic-ss') and increment('ss_runs') <= (3 if scenario == 'generic-ss' else 1):
# Let startup/monitor begin before simulating a child crash.
while not (root / 'monitor_runs').exists() or (scenario == 'generic-ss' and int((root / 'monitor_runs').read_text()) < int((root / 'ss_runs').read_text())):
time.sleep(.01)
time.sleep(.03)
sys.exit(137)
linger()
if name == 'zapret2':
if scenario == 'orphans':
assert_workers_gone()
assert not (root / 'prepared').exists(), 'sourced prepared config'
log(sys.argv[1])
if scenario == 'stopfail' and sys.argv[1] == 'stop':
sys.exit(1)
sys.exit(0)
assert name == 'autotune'
if sys.argv[1] == 'qualification-race-barrier':
# Deterministic interleaving: child has exited, shell has not reached wait.
pid = int(sys.argv[2])
deadline = time.monotonic() + 5
while True:
stat = pathlib.Path('/proc/' + str(pid) + '/stat')
if not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z':
break
assert time.monotonic() < deadline, 'qualifier did not exit at barrier'
time.sleep(.001)
log('term-before-parent-reap')
os.kill(os.getppid(), signal.SIGTERM)
sys.exit(0)
if sys.argv[1] == 'qualification-completion-check':
ack = root / 'completion-ack'
if not ack.exists() or ack.is_symlink() or ack.stat().st_mode & 0o777 != 0o600:
sys.exit(78)
payload = json.loads(ack.read_text())
assert payload == {'pid': int(sys.argv[2]), 'code': int(sys.argv[3]), 'cleanup_complete': True}
log('completion-checked')
sys.exit(0)
if sys.argv[1:] == ['target-config-check']:
sys.exit(0)
if sys.argv[1:] == ['apply-launch']:
log('apply-launch')
sys.exit(0)
if sys.argv[1] == 'dns-forwarder':
if sys.argv[2] in ('readiness-check', 'readiness-cleanup'):
# exec preserves PID1's actual inherited anchors; no simulated unlink.
os.execv(os.environ['DNS_CLI'], [os.environ['DNS_CLI'], *sys.argv[1:]])
mode = os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh')
if sys.argv[2] == 'mode':
print(mode)
sys.exit(0)
if sys.argv[2] == 'address':
print('127.0.0.1:5353' if mode == 'doh' else os.environ.get('SS_DNS_SERVER', ''))
sys.exit(0)
assert sys.argv[2:] == ['serve'] and mode == 'doh'
assert 'SS_PASSWORD' not in os.environ
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
(root / 'dns-pid').write_text(str(os.getpid()))
log('dns-start')
if scenario == 'dns-early':
sys.exit(1)
ready = root / 'private' / 'dns-ready'
owned = None
if scenario == 'dns-preack-foreign':
ready.write_text('foreign evidence\n')
sys.exit(1)
if scenario not in ('dns-noack', 'dns-term', 'dns-int'):
with ready.open('x') as f:
os.chmod(ready, 0o600)
f.write(str(os.getpid() + (1 if scenario == 'dns-wrongack' else 0)) + '\n')
owned = ready.stat()
if scenario == 'dns-permissions':
ready.chmod(0o644)
if scenario == 'dns-symlink':
payload = ready.with_name('dns-payload')
ready.rename(payload)
ready.symlink_to(payload)
log('dns-ready')
def stop_dns(*_):
assert not (root / 'prepared').exists(), 'DNS stopped before restoration'
# SS children have been joined, not merely signalled.
for pid in (root / 'children').read_text().splitlines():
if int(pid) != os.getpid():
assert not pathlib.Path('/proc/' + pid).exists(), ('DNS stopped before child join', pid)
log('dns-stop')
if ready.exists() and not ready.is_symlink() and owned and os.path.samestat(owned, ready.stat()):
ready.unlink()
sys.exit(0)
signal.signal(signal.SIGTERM, stop_dns)
if scenario.startswith('generic-budget-'):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
if scenario in ('dns-term', 'dns-int'):
os.kill(os.getppid(), signal.SIGTERM if scenario == 'dns-term' else signal.SIGINT)
if scenario in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink') and increment('dns_runs') == 1:
while not (root / 'monitor_runs').exists():
time.sleep(.01)
if scenario.startswith('dns-replace'):
# Retain old inode so a replacement cannot reuse its number.
ready.rename(ready.with_name('retired-ready'))
payload = ready.with_name('foreign-ready')
payload.write_text('foreign evidence\n')
payload.chmod(0o600)
if scenario == 'dns-replace-symlink':
ready.symlink_to(payload)
else:
payload.rename(ready)
os.kill(os.getpid(), signal.SIGKILL)
while True:
time.sleep(.01)
if sys.argv[1] == 'qualify':
assert sys.argv[2:] == ['--config', '/operator/resources.json', '--strategies', '/operator/strategies.json', '--isolated']
assert os.getsid(0) == os.getpid(), 'qualifier lacks owned session'
assert 'SS_PASSWORD' not in os.environ
events = (root / 'events').read_text().splitlines()
assert 'start' not in events and 'stop' not in events
assert events[-1] == 'reconcile', events
log('qualify')
assert increment('qualification_runs') == 1, 'qualification restarted'
def acknowledge(code):
ack = root / 'completion-ack'
ack.write_text(json.dumps({'pid': os.getpid(), 'code': code, 'cleanup_complete': True}))
ack.chmod(0o600)
if not scenario.endswith('-race'):
(root / 'qualified').touch()
if scenario in ('qualify-unsafe', 'qualify-unsafe-race'):
if not scenario.endswith('-race'):
(root / 'qualified').touch()
sys.exit(78)
if scenario in ('qualify-blocked', 'qualify-blocked-race'):
acknowledge(78)
sys.exit(78)
if scenario == 'qualify-cancel':
def restored(*_):
log('qualification-restored')
acknowledge(0)
sys.exit(0)
signal.signal(signal.SIGTERM, restored)
os.kill(os.getppid(), signal.SIGTERM)
while True:
time.sleep(.01)
log('qualification-restored')
acknowledge(0)
sys.exit(0)
command = sys.argv[2]
if command == 'init':
log('init')
(root / 'config').touch()
elif command == 'reconcile':
n = increment('reconciles')
log('reconcile')
if scenario.startswith('generic-budget-') and n == 2:
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
time.sleep(.01)
if scenario == 'orphans':
assert_workers_gone()
if scenario in ('prestartfail', 'generic-unsafe'):
sys.exit(78)
if scenario == 'dns-prestart':
os.kill(int((root / 'dns-pid').read_text()), signal.SIGKILL)
time.sleep(.03)
if scenario in ('unknown', 'unknown75') and n == 2:
sys.exit(78)
(root / 'prepared').unlink(missing_ok=True)
elif command == 'monitor':
if scenario.startswith('generic-'):
assert os.environ['ZAPRET_AUTOTUNE_RESOURCE_CONFIG'] == '/operator/resources.json'
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
n = increment('monitor_runs')
log('monitor ' + str(n))
if n == 1 and scenario == 'generic-apply-monitor-restart':
sys.exit(75)
if n == 1 and scenario == 'generic-apply-ss-restart':
os.kill(int((root / 'ss-pid').read_text()), signal.SIGKILL)
linger()
if n == 1 and scenario == 'generic-apply-dns-restart':
os.kill(int((root / 'dns-pid').read_text()), signal.SIGKILL)
linger()
assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session'
if scenario.startswith('generic-budget-'):
time.sleep(.03)
os.kill(os.getppid(), signal.SIGTERM)
if scenario == 'generic-budget-monitor':
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
time.sleep(.01)
linger()
if scenario == 'generic-ss' and n <= 3:
linger()
if scenario in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink') and n == 1:
linger()
if scenario == 'dns-outage':
# Upstream SERVFAIL is modelled by a live DNS child, not child failure.
assert (root / 'dns-pid').exists()
if n == 1 and scenario == 'orphans':
subprocess.Popen([str(root / 'bin' / 'blockcheck')], preexec_fn=os.setpgrp)
while not (root / 'workers').exists() or len((root / 'workers').read_text().splitlines()) != 2:
time.sleep(.01)
(root / 'prepared').touch()
os.kill(os.getpid(), signal.SIGKILL)
if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'):
sys.exit(75)
if scenario.startswith('fatal'):
sys.exit(int(scenario[5:]))
if n == 1 and scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'unknown', 'window'):
if scenario.startswith('crash') or scenario == 'unknown':
(root / 'prepared').touch()
code = int(scenario[5:]) if scenario.startswith('crash') and scenario != 'crash' else 2
sys.exit(code if scenario.startswith('crash') or scenario == 'unknown' else 75)
if scenario == 'joinfail':
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
time.sleep(.01)
if n == 1 and scenario in ('ss', 'mixed'):
linger(78 if scenario == 'mixed' else 0)
# Successful second runtime: external SIGTERM must exit 0, not restart.
time.sleep(.03)
os.kill(os.getppid(), signal.SIGTERM)
linger()
elif command == 'resource-apply':
assert sys.argv[3:] == ['--resource-config', '/operator/resources.json']
assert os.environ['ZAPRET_AUTOTUNE_RESOURCE_APPLY'] == '1'
assert os.getsid(0) == os.getpid(), 'apply lacks owned session'
assert (root / 'private' / 'monitor-ready').read_text() == str(os.getpid()) + '\n'
events = (root / 'events').read_text().splitlines()
assert events[-2:] == ['reconcile', 'apply-launch'] and 'start' not in events
log('resource-apply')
n = increment('apply_runs')
if scenario == 'generic-apply-blocked' and n == 1:
sys.exit(78)
if scenario == 'generic-apply-abrupt' and n == 1:
sys.exit(23)
if scenario == 'generic-apply-cleanup' and n == 1:
log('resource-returned-zero')
sys.exit(0)
if scenario == 'generic-apply-signal' and n == 1:
os.kill(os.getpid(), signal.SIGKILL)
if scenario in ('generic-apply-dns-loss', 'generic-apply-ss-loss') and n == 1:
os.kill(int((root / ('dns-pid' if scenario.endswith('dns-loss') else 'ss-pid')).read_text()), signal.SIGKILL)
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
while True:
time.sleep(.01)
if scenario == 'generic-apply-cancel':
def restored(*_):
log('resource-restored')
sys.exit(0)
signal.signal(signal.SIGTERM, restored)
os.kill(os.getppid(), signal.SIGTERM)
while True:
time.sleep(.01)
log('resource-applied')
sys.exit(0)
'''
def run(scenario, expected, dns='', mode='native'):
with tempfile.TemporaryDirectory(prefix='supervision-') as tmp:
root = Path(tmp)
binary = root / 'bin'
binary.mkdir()
fake = binary / 'fake'
fake.write_text(FAKE)
fake.chmod(0o755)
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2', 'blockcheck', 'nfq'):
(binary / name).symlink_to(fake)
realtime = scenario.startswith('generic-budget-')
if realtime:
for name in ('date', 'timeout'):
(binary / name).unlink()
configs = root / 'private'
configs.mkdir()
if scenario == 'generic-apply-disabled':
(root / 'prepared').touch()
if scenario.startswith('dns-stale'):
if scenario == 'dns-stale-symlink':
(configs / 'dns-ready').symlink_to(configs / 'missing')
else:
(configs / 'dns-ready').write_text('foreign evidence\n')
text = (ROOT / 'managed-target-entrypoint.sh').read_text()
helper = ROOT / 'managed-target-supervision.sh'
if scenario == 'ackfail':
broken_helper = root / 'broken-helper.sh'
broken_helper.write_text(helper.read_text().replace('sleep "$seconds" & sleeper=$!', 'exit 78'))
helper = broken_helper
if scenario == 'generic-budget-ack':
delayed_helper = root / 'delayed-helper.sh'
delayed_helper.write_text(helper.read_text().replace(': > "$config_dir/join-ready"',
f'{shutil.which("sleep")} 300\n : > "$config_dir/join-ready"'))
helper = delayed_helper
text = text.replace('/opt/managed-target-supervision.sh', str(helper))
text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}')
text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune'))
text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2'))
if scenario.startswith('generic-apply-'):
launch = ' launch_controller managed resource-apply'
assert launch in text
text = text.replace(launch, f' {binary / "autotune"} apply-launch\n' + launch)
if scenario == 'generic-apply-launch-fail':
text = text.replace(' launch_controller managed resource-apply', ' false || return 78\n launch_controller managed resource-apply')
if scenario == 'generic-apply-ack-fail':
broken_helper = root / 'ack-helper.sh'
broken_helper.write_text(helper.read_text().replace(' ack_publish ||', ' false && ack_publish ||'))
text = text.replace(str(helper), str(broken_helper))
if scenario == 'generic-apply-cleanup':
text = text.replace('cleanup_session || { echo', "cleanup_session() { return 78; }\n cleanup_session || { echo", 1)
if scenario.endswith('-race'):
barrier = f'{binary / "autotune"} qualification-race-barrier "$monitor_pid"\n'
loop = 'while dns_alive && kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do'
assert loop in text
# The apply child has its own wait loop before this monitor loop.
before, after = text.rsplit(loop, 1)
text = before + barrier + loop + after
if realtime:
# Private fixture scaling only: production has no configurable grace.
text = text.replace('$(date +%s) + 80', '$(date +%s) + 12')
entry = root / 'entry.sh'
entry.write_text(text)
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
SCENARIO=scenario, SS_PASSWORD='ab' * 32, SS_DNS_SERVER=dns,
ZAPRET_CONFIG=str(root / 'config'), DNS_CLI=str(DNS_CLI),
REAL_SLEEP=shutil.which('sleep'))
if scenario.startswith('generic-'):
env['ZAPRET_AUTOTUNE_RESOURCE_CONFIG'] = '/operator/resources.json'
else:
env.pop('ZAPRET_AUTOTUNE_RESOURCE_CONFIG', None)
env.pop('ZAPRET_AUTOTUNE_QUALIFY_CONFIG', None)
env.pop('ZAPRET_AUTOTUNE_STRATEGY_CONFIG', None)
env.pop('ZAPRET_AUTOTUNE_RESOURCE_APPLY', None)
if scenario.startswith('generic-apply-') and scenario != 'generic-apply-disabled':
env['ZAPRET_AUTOTUNE_RESOURCE_APPLY'] = '1'
if scenario.startswith('qualify-'):
env['ZAPRET_AUTOTUNE_QUALIFY_CONFIG'] = '/operator/resources.json'
if scenario != 'qualify-missing':
env['ZAPRET_AUTOTUNE_STRATEGY_CONFIG'] = '/operator/strategies.json'
if mode is None:
env.pop('ZAPRET_AUTOTUNE_DNS_MODE', None)
else:
env['ZAPRET_AUTOTUNE_DNS_MODE'] = mode
peer = None
if scenario == 'orphans':
# Local fixture is not PID 1. Adopt orphan workers here so the test
# can reap their verified-dead zombies rather than burden host PID 1.
assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0
peer = subprocess.Popen([shutil.which('sleep'), '300'], start_new_session=True)
(root / 'peer').write_text(str(peer.pid))
try:
started = time.monotonic()
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=18 if realtime else 15)
if realtime:
assert time.monotonic() - started < 16, (scenario, 'aggregate cleanup exceeded grace')
if peer:
assert peer.poll() is None, 'unowned helper was killed'
finally:
if peer:
peer.terminate()
peer.wait(timeout=2)
events = (root / 'events').read_text().splitlines() if (root / 'events').exists() else []
assert result.returncode == expected, (scenario, result.returncode, result.stderr, events)
assert b'Illegal number' not in result.stderr, (scenario, result.stderr)
assert b'ab' * 32 not in result.stdout + result.stderr
if scenario in ('generic-apply-blocked', 'generic-apply-abrupt', 'generic-apply-cleanup'):
code = {'generic-apply-blocked': 78, 'generic-apply-abrupt': 23, 'generic-apply-cleanup': 0}[scenario]
assert f'controller-startup: apply_wait_exit={code}\n'.encode() in result.stderr, result.stderr
assert (b'controller-startup: apply_cleanup=failed\n' in result.stderr) == (scenario == 'generic-apply-cleanup'), result.stderr
for pid in ((root / 'children').read_text().splitlines() if (root / 'children').exists() else []):
assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid)
if scenario == 'orphans':
for pid in (root / 'workers').read_text().splitlines():
stat = Path(f'/proc/{pid}/stat')
assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('orphan survived', pid)
try:
reaped, _ = os.waitpid(int(pid), os.WNOHANG)
assert reaped == int(pid), ('worker not reaped', pid)
except ChildProcessError:
assert not stat.exists(), ('worker was not adopted/reaped', pid)
assert ctypes.CDLL(None).prctl(36, 0, 0, 0, 0) == 0
# Every activation and stop must immediately follow successful reconcile.
for i, event in enumerate(events):
if event in ('start', 'stop'):
assert events[i - 1] == 'reconcile' or event == 'start' and events[i - 1] == 'resource-applied', (scenario, events)
foreign = scenario in ('dns-symlink', 'dns-preack-foreign', 'dns-replace', 'dns-replace-symlink', 'dns-stale', 'dns-stale-symlink')
if foreign:
ready = configs / 'dns-ready'
assert ready.exists() or ready.is_symlink(), (scenario, 'foreign readiness deleted')
if scenario != 'dns-stale-symlink' and scenario != 'dns-symlink':
assert ready.read_text() == 'foreign evidence\n', (scenario, 'foreign bytes changed')
elif realtime and (configs / 'dns-ready').exists():
# Exhaustion cannot authorize unlinking without the owned Go helper.
assert (configs / 'dns-ready').read_text().strip() == (root / 'dns-pid').read_text()
else:
assert not (configs / 'dns-ready').exists(), (scenario, 'stale DNS readiness')
if mode == 'native':
assert 'dns-start' not in events
if scenario.startswith('generic-apply-') and 'blocked-window 1' in events:
assert events.count('apply-launch') == 1, (scenario, events)
assert int((root / 'elapsed').read_text()) == 180, (scenario, events)
assert events.count('dns-start') == events.count('ss-server') == events.count('ss-local') == 1, (scenario, events)
assert 'start' not in events and not any(e.startswith('monitor ') for e in events), (scenario, events)
assert 'cooldown 15' not in events and 'cooldown 30' not in events, (scenario, events)
assert events[events.index('blocked-window 1'):] == ['blocked-window 1', 'blocked-window 2', 'blocked-window 3'], (scenario, events)
if scenario != 'generic-apply-dns-loss':
assert events.index('dns-stop') < events.index('blocked-window 1'), (scenario, events)
return events
for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window', 'orphans'):
events = run(scenario, 0)
assert events.count('start') == 2, (scenario, events)
assert events.count('ss-server') == 2 and events.count('ss-local') == 2
assert events.count('stop') == 2
assert events.count('cooldown 15') == 1
for code in (64, 78, 1):
events = run(f'fatal{code}', code if code in (64, 78) else 78)
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
for scenario in ('mixed', 'unknown', 'unknown75', 'stopfail', 'joinfail', 'ackfail'):
events = run(scenario, 78)
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
if scenario.startswith('unknown'):
assert 'stop' not in events
events = run('budget', 78)
assert events.count('start') == 3
assert [e for e in events if e.startswith('cooldown')] == ['cooldown 15', 'cooldown 30']
events = run('cancel', 0)
assert events.count('start') == 1
events = run('prestartfail', 78)
assert 'start' not in events and 'stop' not in events
events = run('generic-ss', 0, mode='doh')
assert events.count('start') == 4 and events.count('stop') == 4, events
assert events.count('generic-pause') == 1, events
assert events.count('dns-start') == events.count('dns-stop') == 4, events
events = run('generic-unsafe', 0)
assert 'start' not in events and events.count('generic-pause') == 1, events
events = run('generic-apply-pass', 0, mode='doh')
assert events.count('resource-apply') == 1 and events.count('start') == 1, events
assert events.index('resource-applied') < events.index('start') < events.index('monitor 1'), events
events = run('generic-apply-cancel', 0, mode='doh')
assert 'resource-restored' in events and 'start' not in events and 'monitor 1' not in events, events
assert events.index('resource-restored') < events.index('stop') < events.index('dns-stop'), events
events = run('generic-apply-blocked', 0, mode='doh')
assert events.count('resource-apply') == 1 and 'blocked-window 3' in events, events
for scenario, code in (('generic-apply-abrupt', 0), ('generic-apply-cleanup', 78)):
events = run(scenario, code, mode='doh')
assert 'start' not in events and 'monitor 1' not in events and 'resource-applied' not in events, events
for scenario in ('signal', 'dns-loss', 'ss-loss', 'launch-fail', 'ack-fail'):
events = run('generic-apply-' + scenario, 78 if scenario == 'ack-fail' else 0, mode='doh')
assert 'blocked-window 3' in events and 'resource-applied' not in events, events
assert events.count('resource-apply') == (0 if scenario in ('launch-fail', 'ack-fail') else 1), events
for scenario in ('monitor-restart', 'ss-restart', 'dns-restart'):
events = run('generic-apply-' + scenario, 0, mode='doh')
assert events.count('resource-apply') == events.count('apply-launch') == 1, events
assert events.count('start') == events.count('stop') == 2 and 'monitor 2' in events, events
assert events.count('dns-start') == 2 and events.count('dns-stop') == (1 if scenario == 'dns-restart' else 2), events
# Independent fresh PID 1 invocations reset the RAM latch; flag=0 still repairs
# prepared bytes by reconcile, without launching apply.
events = run('generic-apply-pass', 0, mode='doh')
assert events.count('resource-apply') == 1, events
events = run('generic-apply-disabled', 0, mode='doh')
assert 'resource-apply' not in events and 'start' in events, events
for scenario in ('generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
events = run(scenario, 78, mode='doh')
assert events.count('start') == 1 and 'stop' not in events, (scenario, events)
for dns in ('8.8.8.8', '2001:4860:4860::8888'):
events = run('fatal64', 64, dns)
assert events.count('ss-server') == 1 and events.count('ss-local') == 1
for scenario in ('dns-outage', 'technical', 'ss', 'cancel', 'dns-death', 'dns-killed'):
events = run(scenario, 0, dns='8.8.8.8', mode=None)
assert events.index('dns-ready') < events.index('ss-server')
assert events.index('dns-ready') < events.index('ss-local')
if scenario not in ('dns-death', 'dns-killed'):
assert events.count('dns-start') == events.count('dns-stop')
assert events.index('stop') < events.index('dns-stop')
if scenario == 'dns-outage':
assert events.count('start') == 1 and 'cooldown 15' not in events
for scenario, code in (('dns-early', 78), ('dns-noack', 78), ('dns-wrongack', 78),
('dns-permissions', 78), ('dns-symlink', 78), ('dns-prestart', 78)):
events = run(scenario, code, mode='doh')
assert not any(e.startswith('monitor ') for e in events)
if scenario != 'dns-prestart':
assert 'ss-server' not in events and 'ss-local' not in events
for scenario in ('dns-stale', 'dns-stale-symlink', 'dns-preack-foreign', 'dns-replace', 'dns-replace-symlink'):
events = run(scenario, 78, mode='doh')
assert events.count('dns-start') <= 1 and 'cooldown 15' not in events
if scenario.startswith('dns-stale'):
assert not events
for scenario in ('dns-term', 'dns-int'):
events = run(scenario, 0, mode='doh')
assert events == ['dns-start', 'dns-stop'], events
for scenario in ('qualify-pass', 'qualify-negative', 'qualify-blocked', 'qualify-cancel'):
events = run(scenario, 0, mode='doh')
assert events.count('qualify') == 1 and not any(e.startswith('monitor ') for e in events), events
assert 'start' not in events and 'stop' not in events, events
assert events.index('dns-ready') < events.index('qualify') < events.index('dns-stop'), events
if scenario != 'qualify-blocked':
assert events.index('qualification-restored') < events.index('dns-stop'), events
events = run('qualify-missing', 64, mode='doh')
assert not events, events
for scenario, code in (('qualify-blocked-race', 0), ('qualify-unsafe-race', 78), ('qualify-unsafe', 78)):
events = run(scenario, code, mode='doh')
assert events.count('qualify') == 1 and 'start' not in events and 'stop' not in events, events
if scenario.endswith('-race'):
assert events.count('term-before-parent-reap') == 1, events
if code == 0:
assert events.index('term-before-parent-reap') < events.index('completion-checked') < events.index('dns-stop'), events
else:
assert 'completion-checked' not in events, events
def check_watchdog_traps(shell):
"""Same bounded_wait from normal flow, plain EXIT, and TERM -> EXIT traps."""
entrypoint = (ROOT / 'managed-target-entrypoint.sh').read_text()
bounded_wait = entrypoint[entrypoint.index('bounded_wait() {'):entrypoint.index('cleanup_cycle() {')]
bounded_wait = bounded_wait.replace('/opt/managed-target-supervision.sh', str(ROOT / 'managed-target-supervision.sh'))
for mode in ('normal', 'exit', 'term-exit'):
with tempfile.TemporaryDirectory(prefix='watchdog-traps-') as tmp:
directory = Path(tmp)
sleeper = directory / 'sleep'
sleeper.write_text('#!/bin/sh\necho $$ >> "$PIDS"\nexec "' + shutil.which('sleep') + '" "$@"\n')
sleeper.chmod(0o755)
script = directory / 'check.sh'
script.write_text(f'''set -eu
mode=$1
shift
config_dir={directory}
generic_mode=0
. {ROOT / 'managed-target-supervision.sh'}
{bounded_wait}
cleanup() {{
trap - EXIT
trap ':' INT TERM
kill "$owned" 2>/dev/null || true
bounded_wait "$owned" 5
[ "$join_timed_out" -eq 0 ]
}}
sleep 300 & owned=$!
case "$mode" in
normal) cleanup ;;
exit) trap cleanup EXIT; exit 0 ;;
term-exit) trap cleanup EXIT; trap 'exit 0' TERM; kill -TERM $$ ;;
esac
''')
env = dict(os.environ, PATH=f'{directory}:{os.environ["PATH"]}', PIDS=str(directory / 'pids'))
started = time.monotonic()
result = subprocess.run([*shell, str(script), mode], env=env, capture_output=True, timeout=10)
assert result.returncode == 0, (shell, mode, result.stderr)
assert time.monotonic() - started < 2, (shell, mode, 'watchdog cancellation delayed')
if (directory / 'pids').exists():
for pid in (directory / 'pids').read_text().splitlines():
assert not Path(f'/proc/{pid}').exists(), (shell, mode, 'orphan sleeper', pid)
def check_session_deadline(shell):
"""Startup sweep has no aggregate deadline; termination must keep its cap."""
entrypoint = (ROOT / 'managed-target-entrypoint.sh').read_text()
cleanup_session = entrypoint[entrypoint.index('cleanup_session() {'):entrypoint.index('capture_monitor_session() {')]
for deadline, live_until, expected, elapsed in (('', 102, 0, 2), ('180', 102, 0, 2), ('101', 110, 78, 1)):
with tempfile.TemporaryDirectory(prefix='session-deadline-') as tmp:
script = Path(tmp) / 'check.sh'
script.write_text(f'''set -eu
generic_mode=1
generic_cleanup_deadline='{deadline}'
monitor_sid=42
clock=100
date() {{ printf '%s\\n' "$clock"; }}
pause() {{ clock=$((clock + $1)); }}
signal_session() {{
[ "$1" = 42 ] || return 78
case "$2" in
TERM|KILL) ;;
CHECK) session_live=1; [ "$clock" -lt {live_until} ] || session_live=0 ;;
*) return 78 ;;
esac
}}
{cleanup_session}
code=0
cleanup_session || code=$?
[ "$code" -eq {expected} ]
[ "$clock" -eq {100 + elapsed} ]
[ "$generic_cleanup_deadline" = '{deadline}' ]
''')
result = subprocess.run([*shell, str(script)], capture_output=True, timeout=2)
assert result.returncode == 0, (shell, deadline, result.stderr)
expected_stderr = b'error: owned monitor session survived cleanup\n' if expected else b''
assert result.stderr == expected_stderr, (shell, deadline, result.stderr)
check_watchdog_traps(['/bin/sh'])
check_session_deadline(['/bin/sh'])
busybox = shutil.which('busybox')
if busybox:
check_watchdog_traps([busybox, 'ash'])
check_session_deadline([busybox, 'ash'])
for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')):
result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2)
assert result.returncode == 64
result = subprocess.run(['/bin/sh', '-c', 'helper=$1; shift; . "$helper"; proc_identity "$$"; own=$proc_sid; if signal_session "$own" KILL || signal_session 1 KILL; then exit 99; fi', 'check', str(ROOT / 'managed-target-supervision.sh')], capture_output=True, timeout=2)
assert result.returncode == 0, ('unsafe session accepted', result.returncode)
print('entrypoint supervision fixtures passed')