Add fixed startup branch labels and numeric apply exit status without relaxing startup guards or exposing private values. Cover shared launch/ACK and wrapper branches plus failed apply cleanup. Diagnostics only; the ACK publication race remains unchanged. Co-authored-by: GPT-6 <gpt-6@openai.com>
661 lines
32 KiB
Python
661 lines
32 KiB
Python
"""Isolated shell fixtures: python3 deploy/managed-target/check_supervision.py.
|
|
|
|
Only a temporary COPY of the entrypoint is redirected to fake executables;
|
|
production has no test switches, real network probes, or accelerated timers.
|
|
"""
|
|
import ctypes
|
|
import atexit
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
# Real readiness CLI checks root ownership. Use an isolated user namespace for
|
|
# offline fixtures when invoked as an ordinary user; no host-root requirement.
|
|
if os.geteuid() != 0:
|
|
os.execvp('unshare', ['unshare', '-Ur', 'python3', str(Path(__file__).resolve())])
|
|
CLI_TMP = tempfile.TemporaryDirectory(prefix='dns-readiness-cli-')
|
|
atexit.register(CLI_TMP.cleanup)
|
|
DNS_CLI = Path(CLI_TMP.name) / 'autotune'
|
|
subprocess.run(['go', 'build', '-o', str(DNS_CLI), './cmd/zapret-autotune'],
|
|
cwd=ROOT, env=dict(os.environ, GOPROXY='off', GOTOOLCHAIN='local'), check=True)
|
|
FAKE = r'''#!/usr/bin/env python3
|
|
import ctypes, json, os, pathlib, signal, subprocess, sys, time
|
|
root = pathlib.Path(os.environ['FIXTURE'])
|
|
name = pathlib.Path(sys.argv[0]).name
|
|
scenario = os.environ['SCENARIO']
|
|
def log(message):
|
|
with (root / 'events').open('a') as f:
|
|
f.write(message + '\n')
|
|
def increment(key):
|
|
path = root / key
|
|
n = int(path.read_text()) + 1 if path.exists() else 1
|
|
path.write_text(str(n))
|
|
return n
|
|
def linger(code=0):
|
|
def stop(*_):
|
|
if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh' and scenario not in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink', 'dns-prestart'):
|
|
os.kill(int((root / 'dns-pid').read_text()), 0)
|
|
sys.exit(code)
|
|
signal.signal(signal.SIGTERM, stop)
|
|
while True:
|
|
time.sleep(.01)
|
|
def assert_workers_gone():
|
|
if (root / 'workers').exists():
|
|
for pid in (root / 'workers').read_text().splitlines():
|
|
stat = pathlib.Path('/proc/' + pid + '/stat')
|
|
assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('old worker still live', pid)
|
|
if (root / 'peer').exists():
|
|
os.kill(int((root / 'peer').read_text()), 0)
|
|
if name in ('blockcheck', 'nfq'):
|
|
# Model Go's Setpgid check child and its NFQ worker, not a daemonized sysv
|
|
# process. Comm deliberately contains spaces/parentheses to test /proc parsing.
|
|
ctypes.CDLL(None).prctl(15, b'bc ) (fixture)', 0, 0, 0)
|
|
assert os.getsid(0) != os.getpgrp(), 'fixture did not split PGID from SID'
|
|
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
with (root / 'workers').open('a') as f:
|
|
f.write(str(os.getpid()) + '\n')
|
|
if name == 'blockcheck':
|
|
subprocess.Popen([str(root / 'bin' / 'nfq')])
|
|
while True:
|
|
time.sleep(.01)
|
|
if name == 'awk':
|
|
sys.exit(0)
|
|
if name == 'date':
|
|
n = increment('clock')
|
|
print(100000 + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0)))
|
|
sys.exit(0)
|
|
if name == 'sleep':
|
|
seconds = int(sys.argv[1])
|
|
if scenario.startswith('qualify-') and (root / 'qualified').exists():
|
|
log('safe-idle')
|
|
os.kill(os.getppid(), signal.SIGTERM)
|
|
time.sleep(.02)
|
|
sys.exit(0)
|
|
if scenario.startswith('generic-budget-') and seconds != 60:
|
|
os.execv(os.environ['REAL_SLEEP'], [os.environ['REAL_SLEEP'], *sys.argv[1:]])
|
|
if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack', 'generic-apply-blocked', 'generic-apply-abrupt', 'generic-apply-cleanup'):
|
|
log('generic-pause')
|
|
(root / 'budget-pause').touch()
|
|
if scenario != 'generic-ss':
|
|
os.kill(os.getppid(), signal.SIGTERM)
|
|
time.sleep(.01)
|
|
sys.exit(0)
|
|
if seconds in (15, 30):
|
|
log('cooldown ' + str(seconds))
|
|
if scenario == 'cancel':
|
|
os.kill(os.getppid(), signal.SIGTERM)
|
|
time.sleep(1)
|
|
else:
|
|
time.sleep(.01)
|
|
else:
|
|
time.sleep(.01 if seconds == 1 else 2)
|
|
sys.exit(0)
|
|
if name == 'timeout':
|
|
os.execv(sys.argv[4], sys.argv[4:])
|
|
if name in ('ss-server', 'ss-local'):
|
|
with (root / 'children').open('a') as f:
|
|
f.write(str(os.getpid()) + '\n')
|
|
assert 'SS_PASSWORD' not in os.environ
|
|
assert sys.argv[1] == '-c'
|
|
dns = ('127.0.0.1:5353' if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh'
|
|
else os.environ.get('SS_DNS_SERVER', '')) if name == 'ss-server' else ''
|
|
assert sys.argv[3:] == (['-d', dns] if dns else [])
|
|
if dns == '127.0.0.1:5353':
|
|
assert (root / 'private' / 'dns-ready').read_text().strip() == (root / 'dns-pid').read_text()
|
|
os.kill(int((root / 'dns-pid').read_text()), 0)
|
|
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
|
|
log(name)
|
|
if scenario.startswith('generic-budget-'):
|
|
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
while True:
|
|
time.sleep(.01)
|
|
if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail', 'generic-ss') and increment('ss_runs') <= (3 if scenario == 'generic-ss' else 1):
|
|
# Let startup/monitor begin before simulating a child crash.
|
|
while not (root / 'monitor_runs').exists() or (scenario == 'generic-ss' and int((root / 'monitor_runs').read_text()) < int((root / 'ss_runs').read_text())):
|
|
time.sleep(.01)
|
|
time.sleep(.03)
|
|
sys.exit(137)
|
|
linger()
|
|
if name == 'zapret2':
|
|
if scenario == 'orphans':
|
|
assert_workers_gone()
|
|
assert not (root / 'prepared').exists(), 'sourced prepared config'
|
|
log(sys.argv[1])
|
|
if scenario == 'stopfail' and sys.argv[1] == 'stop':
|
|
sys.exit(1)
|
|
sys.exit(0)
|
|
assert name == 'autotune'
|
|
if sys.argv[1] == 'qualification-race-barrier':
|
|
# Deterministic interleaving: child has exited, shell has not reached wait.
|
|
pid = int(sys.argv[2])
|
|
deadline = time.monotonic() + 5
|
|
while True:
|
|
stat = pathlib.Path('/proc/' + str(pid) + '/stat')
|
|
if not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z':
|
|
break
|
|
assert time.monotonic() < deadline, 'qualifier did not exit at barrier'
|
|
time.sleep(.001)
|
|
log('term-before-parent-reap')
|
|
os.kill(os.getppid(), signal.SIGTERM)
|
|
sys.exit(0)
|
|
if sys.argv[1] == 'qualification-completion-check':
|
|
ack = root / 'completion-ack'
|
|
if not ack.exists() or ack.is_symlink() or ack.stat().st_mode & 0o777 != 0o600:
|
|
sys.exit(78)
|
|
payload = json.loads(ack.read_text())
|
|
assert payload == {'pid': int(sys.argv[2]), 'code': int(sys.argv[3]), 'cleanup_complete': True}
|
|
log('completion-checked')
|
|
sys.exit(0)
|
|
if sys.argv[1:] == ['target-config-check']:
|
|
sys.exit(0)
|
|
if sys.argv[1] == 'dns-forwarder':
|
|
if sys.argv[2] in ('readiness-check', 'readiness-cleanup'):
|
|
# exec preserves PID1's actual inherited anchors; no simulated unlink.
|
|
os.execv(os.environ['DNS_CLI'], [os.environ['DNS_CLI'], *sys.argv[1:]])
|
|
mode = os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh')
|
|
if sys.argv[2] == 'mode':
|
|
print(mode)
|
|
sys.exit(0)
|
|
if sys.argv[2] == 'address':
|
|
print('127.0.0.1:5353' if mode == 'doh' else os.environ.get('SS_DNS_SERVER', ''))
|
|
sys.exit(0)
|
|
assert sys.argv[2:] == ['serve'] and mode == 'doh'
|
|
assert 'SS_PASSWORD' not in os.environ
|
|
with (root / 'children').open('a') as f:
|
|
f.write(str(os.getpid()) + '\n')
|
|
(root / 'dns-pid').write_text(str(os.getpid()))
|
|
log('dns-start')
|
|
if scenario == 'dns-early':
|
|
sys.exit(1)
|
|
ready = root / 'private' / 'dns-ready'
|
|
owned = None
|
|
if scenario == 'dns-preack-foreign':
|
|
ready.write_text('foreign evidence\n')
|
|
sys.exit(1)
|
|
if scenario not in ('dns-noack', 'dns-term', 'dns-int'):
|
|
with ready.open('x') as f:
|
|
os.chmod(ready, 0o600)
|
|
f.write(str(os.getpid() + (1 if scenario == 'dns-wrongack' else 0)) + '\n')
|
|
owned = ready.stat()
|
|
if scenario == 'dns-permissions':
|
|
ready.chmod(0o644)
|
|
if scenario == 'dns-symlink':
|
|
payload = ready.with_name('dns-payload')
|
|
ready.rename(payload)
|
|
ready.symlink_to(payload)
|
|
log('dns-ready')
|
|
def stop_dns(*_):
|
|
assert not (root / 'prepared').exists(), 'DNS stopped before restoration'
|
|
# SS children have been joined, not merely signalled.
|
|
for pid in (root / 'children').read_text().splitlines():
|
|
if int(pid) != os.getpid():
|
|
assert not pathlib.Path('/proc/' + pid).exists(), ('DNS stopped before child join', pid)
|
|
log('dns-stop')
|
|
if ready.exists() and not ready.is_symlink() and owned and os.path.samestat(owned, ready.stat()):
|
|
ready.unlink()
|
|
sys.exit(0)
|
|
signal.signal(signal.SIGTERM, stop_dns)
|
|
if scenario.startswith('generic-budget-'):
|
|
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
if scenario in ('dns-term', 'dns-int'):
|
|
os.kill(os.getppid(), signal.SIGTERM if scenario == 'dns-term' else signal.SIGINT)
|
|
if scenario in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink') and increment('dns_runs') == 1:
|
|
while not (root / 'monitor_runs').exists():
|
|
time.sleep(.01)
|
|
if scenario.startswith('dns-replace'):
|
|
# Retain old inode so a replacement cannot reuse its number.
|
|
ready.rename(ready.with_name('retired-ready'))
|
|
payload = ready.with_name('foreign-ready')
|
|
payload.write_text('foreign evidence\n')
|
|
payload.chmod(0o600)
|
|
if scenario == 'dns-replace-symlink':
|
|
ready.symlink_to(payload)
|
|
else:
|
|
payload.rename(ready)
|
|
os.kill(os.getpid(), signal.SIGKILL)
|
|
while True:
|
|
time.sleep(.01)
|
|
if sys.argv[1] == 'qualify':
|
|
assert sys.argv[2:] == ['--config', '/operator/resources.json', '--strategies', '/operator/strategies.json', '--isolated']
|
|
assert os.getsid(0) == os.getpid(), 'qualifier lacks owned session'
|
|
assert 'SS_PASSWORD' not in os.environ
|
|
events = (root / 'events').read_text().splitlines()
|
|
assert 'start' not in events and 'stop' not in events
|
|
assert events[-1] == 'reconcile', events
|
|
log('qualify')
|
|
assert increment('qualification_runs') == 1, 'qualification restarted'
|
|
def acknowledge(code):
|
|
ack = root / 'completion-ack'
|
|
ack.write_text(json.dumps({'pid': os.getpid(), 'code': code, 'cleanup_complete': True}))
|
|
ack.chmod(0o600)
|
|
if not scenario.endswith('-race'):
|
|
(root / 'qualified').touch()
|
|
if scenario in ('qualify-unsafe', 'qualify-unsafe-race'):
|
|
if not scenario.endswith('-race'):
|
|
(root / 'qualified').touch()
|
|
sys.exit(78)
|
|
if scenario in ('qualify-blocked', 'qualify-blocked-race'):
|
|
acknowledge(78)
|
|
sys.exit(78)
|
|
if scenario == 'qualify-cancel':
|
|
def restored(*_):
|
|
log('qualification-restored')
|
|
acknowledge(0)
|
|
sys.exit(0)
|
|
signal.signal(signal.SIGTERM, restored)
|
|
os.kill(os.getppid(), signal.SIGTERM)
|
|
while True:
|
|
time.sleep(.01)
|
|
log('qualification-restored')
|
|
acknowledge(0)
|
|
sys.exit(0)
|
|
command = sys.argv[2]
|
|
if command == 'init':
|
|
log('init')
|
|
(root / 'config').touch()
|
|
elif command == 'reconcile':
|
|
n = increment('reconciles')
|
|
log('reconcile')
|
|
if scenario.startswith('generic-budget-') and n == 2:
|
|
with (root / 'children').open('a') as f:
|
|
f.write(str(os.getpid()) + '\n')
|
|
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
while True:
|
|
time.sleep(.01)
|
|
if scenario == 'orphans':
|
|
assert_workers_gone()
|
|
if scenario in ('prestartfail', 'generic-unsafe'):
|
|
sys.exit(78)
|
|
if scenario == 'dns-prestart':
|
|
os.kill(int((root / 'dns-pid').read_text()), signal.SIGKILL)
|
|
time.sleep(.03)
|
|
if scenario in ('unknown', 'unknown75') and n == 2:
|
|
sys.exit(78)
|
|
(root / 'prepared').unlink(missing_ok=True)
|
|
elif command == 'monitor':
|
|
if scenario.startswith('generic-'):
|
|
assert os.environ['ZAPRET_AUTOTUNE_RESOURCE_CONFIG'] == '/operator/resources.json'
|
|
with (root / 'children').open('a') as f:
|
|
f.write(str(os.getpid()) + '\n')
|
|
n = increment('monitor_runs')
|
|
log('monitor ' + str(n))
|
|
assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session'
|
|
if scenario.startswith('generic-budget-'):
|
|
time.sleep(.03)
|
|
os.kill(os.getppid(), signal.SIGTERM)
|
|
if scenario == 'generic-budget-monitor':
|
|
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
while True:
|
|
time.sleep(.01)
|
|
linger()
|
|
if scenario == 'generic-ss' and n <= 3:
|
|
linger()
|
|
if scenario in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink') and n == 1:
|
|
linger()
|
|
if scenario == 'dns-outage':
|
|
# Upstream SERVFAIL is modelled by a live DNS child, not child failure.
|
|
assert (root / 'dns-pid').exists()
|
|
if n == 1 and scenario == 'orphans':
|
|
subprocess.Popen([str(root / 'bin' / 'blockcheck')], preexec_fn=os.setpgrp)
|
|
while not (root / 'workers').exists() or len((root / 'workers').read_text().splitlines()) != 2:
|
|
time.sleep(.01)
|
|
(root / 'prepared').touch()
|
|
os.kill(os.getpid(), signal.SIGKILL)
|
|
if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'):
|
|
sys.exit(75)
|
|
if scenario.startswith('fatal'):
|
|
sys.exit(int(scenario[5:]))
|
|
if n == 1 and scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'unknown', 'window'):
|
|
if scenario.startswith('crash') or scenario == 'unknown':
|
|
(root / 'prepared').touch()
|
|
code = int(scenario[5:]) if scenario.startswith('crash') and scenario != 'crash' else 2
|
|
sys.exit(code if scenario.startswith('crash') or scenario == 'unknown' else 75)
|
|
if scenario == 'joinfail':
|
|
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
while True:
|
|
time.sleep(.01)
|
|
if n == 1 and scenario in ('ss', 'mixed'):
|
|
linger(78 if scenario == 'mixed' else 0)
|
|
# Successful second runtime: external SIGTERM must exit 0, not restart.
|
|
time.sleep(.03)
|
|
os.kill(os.getppid(), signal.SIGTERM)
|
|
linger()
|
|
elif command == 'resource-apply':
|
|
assert sys.argv[3:] == ['--resource-config', '/operator/resources.json']
|
|
assert os.environ['ZAPRET_AUTOTUNE_RESOURCE_APPLY'] == '1'
|
|
assert os.getsid(0) == os.getpid(), 'apply lacks owned session'
|
|
assert (root / 'private' / 'monitor-ready').read_text() == str(os.getpid()) + '\n'
|
|
events = (root / 'events').read_text().splitlines()
|
|
assert events[-1] == 'reconcile' and 'start' not in events
|
|
log('resource-apply')
|
|
if scenario == 'generic-apply-blocked':
|
|
sys.exit(78)
|
|
if scenario in ('generic-apply-abrupt', 'generic-apply-cleanup'):
|
|
sys.exit(23)
|
|
if scenario == 'generic-apply-cancel':
|
|
def restored(*_):
|
|
log('resource-restored')
|
|
sys.exit(0)
|
|
signal.signal(signal.SIGTERM, restored)
|
|
os.kill(os.getppid(), signal.SIGTERM)
|
|
while True:
|
|
time.sleep(.01)
|
|
log('resource-applied')
|
|
sys.exit(0)
|
|
'''
|
|
|
|
|
|
def run(scenario, expected, dns='', mode='native'):
|
|
with tempfile.TemporaryDirectory(prefix='supervision-') as tmp:
|
|
root = Path(tmp)
|
|
binary = root / 'bin'
|
|
binary.mkdir()
|
|
fake = binary / 'fake'
|
|
fake.write_text(FAKE)
|
|
fake.chmod(0o755)
|
|
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2', 'blockcheck', 'nfq'):
|
|
(binary / name).symlink_to(fake)
|
|
realtime = scenario.startswith('generic-budget-')
|
|
if realtime:
|
|
for name in ('date', 'timeout'):
|
|
(binary / name).unlink()
|
|
configs = root / 'private'
|
|
configs.mkdir()
|
|
if scenario.startswith('dns-stale'):
|
|
if scenario == 'dns-stale-symlink':
|
|
(configs / 'dns-ready').symlink_to(configs / 'missing')
|
|
else:
|
|
(configs / 'dns-ready').write_text('foreign evidence\n')
|
|
text = (ROOT / 'managed-target-entrypoint.sh').read_text()
|
|
helper = ROOT / 'managed-target-supervision.sh'
|
|
if scenario == 'ackfail':
|
|
broken_helper = root / 'broken-helper.sh'
|
|
broken_helper.write_text(helper.read_text().replace('sleep "$seconds" & sleeper=$!', 'exit 78'))
|
|
helper = broken_helper
|
|
if scenario == 'generic-budget-ack':
|
|
delayed_helper = root / 'delayed-helper.sh'
|
|
delayed_helper.write_text(helper.read_text().replace(': > "$config_dir/join-ready"',
|
|
f'{shutil.which("sleep")} 300\n : > "$config_dir/join-ready"'))
|
|
helper = delayed_helper
|
|
text = text.replace('/opt/managed-target-supervision.sh', str(helper))
|
|
text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}')
|
|
text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune'))
|
|
text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2'))
|
|
if scenario == 'generic-apply-cleanup':
|
|
text = text.replace('cleanup_session || { echo', "cleanup_session() { return 78; }\n cleanup_session || { echo", 1)
|
|
if scenario.endswith('-race'):
|
|
barrier = f'{binary / "autotune"} qualification-race-barrier "$monitor_pid"\n'
|
|
loop = 'while dns_alive && kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do'
|
|
assert loop in text
|
|
# The apply child has its own wait loop before this monitor loop.
|
|
before, after = text.rsplit(loop, 1)
|
|
text = before + barrier + loop + after
|
|
if realtime:
|
|
# Private fixture scaling only: production has no configurable grace.
|
|
text = text.replace('$(date +%s) + 80', '$(date +%s) + 12')
|
|
entry = root / 'entry.sh'
|
|
entry.write_text(text)
|
|
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
|
|
SCENARIO=scenario, SS_PASSWORD='ab' * 32, SS_DNS_SERVER=dns,
|
|
ZAPRET_CONFIG=str(root / 'config'), DNS_CLI=str(DNS_CLI),
|
|
REAL_SLEEP=shutil.which('sleep'))
|
|
if scenario.startswith('generic-'):
|
|
env['ZAPRET_AUTOTUNE_RESOURCE_CONFIG'] = '/operator/resources.json'
|
|
else:
|
|
env.pop('ZAPRET_AUTOTUNE_RESOURCE_CONFIG', None)
|
|
env.pop('ZAPRET_AUTOTUNE_QUALIFY_CONFIG', None)
|
|
env.pop('ZAPRET_AUTOTUNE_STRATEGY_CONFIG', None)
|
|
env.pop('ZAPRET_AUTOTUNE_RESOURCE_APPLY', None)
|
|
if scenario.startswith('generic-apply-'):
|
|
env['ZAPRET_AUTOTUNE_RESOURCE_APPLY'] = '1'
|
|
if scenario.startswith('qualify-'):
|
|
env['ZAPRET_AUTOTUNE_QUALIFY_CONFIG'] = '/operator/resources.json'
|
|
if scenario != 'qualify-missing':
|
|
env['ZAPRET_AUTOTUNE_STRATEGY_CONFIG'] = '/operator/strategies.json'
|
|
if mode is None:
|
|
env.pop('ZAPRET_AUTOTUNE_DNS_MODE', None)
|
|
else:
|
|
env['ZAPRET_AUTOTUNE_DNS_MODE'] = mode
|
|
peer = None
|
|
if scenario == 'orphans':
|
|
# Local fixture is not PID 1. Adopt orphan workers here so the test
|
|
# can reap their verified-dead zombies rather than burden host PID 1.
|
|
assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0
|
|
peer = subprocess.Popen([shutil.which('sleep'), '300'], start_new_session=True)
|
|
(root / 'peer').write_text(str(peer.pid))
|
|
try:
|
|
started = time.monotonic()
|
|
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=18 if realtime else 15)
|
|
if realtime:
|
|
assert time.monotonic() - started < 16, (scenario, 'aggregate cleanup exceeded grace')
|
|
if peer:
|
|
assert peer.poll() is None, 'unowned helper was killed'
|
|
finally:
|
|
if peer:
|
|
peer.terminate()
|
|
peer.wait(timeout=2)
|
|
events = (root / 'events').read_text().splitlines() if (root / 'events').exists() else []
|
|
assert result.returncode == expected, (scenario, result.returncode, result.stderr, events)
|
|
assert b'Illegal number' not in result.stderr, (scenario, result.stderr)
|
|
assert b'ab' * 32 not in result.stdout + result.stderr
|
|
if scenario in ('generic-apply-blocked', 'generic-apply-abrupt', 'generic-apply-cleanup'):
|
|
code = 78 if scenario == 'generic-apply-blocked' else 23
|
|
assert f'controller-startup: apply_wait_exit={code}\n'.encode() in result.stderr, result.stderr
|
|
assert (b'controller-startup: apply_cleanup=failed\n' in result.stderr) == (scenario == 'generic-apply-cleanup'), result.stderr
|
|
for pid in ((root / 'children').read_text().splitlines() if (root / 'children').exists() else []):
|
|
assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid)
|
|
if scenario == 'orphans':
|
|
for pid in (root / 'workers').read_text().splitlines():
|
|
stat = Path(f'/proc/{pid}/stat')
|
|
assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('orphan survived', pid)
|
|
try:
|
|
reaped, _ = os.waitpid(int(pid), os.WNOHANG)
|
|
assert reaped == int(pid), ('worker not reaped', pid)
|
|
except ChildProcessError:
|
|
assert not stat.exists(), ('worker was not adopted/reaped', pid)
|
|
assert ctypes.CDLL(None).prctl(36, 0, 0, 0, 0) == 0
|
|
# Every activation and stop must immediately follow successful reconcile.
|
|
for i, event in enumerate(events):
|
|
if event in ('start', 'stop'):
|
|
assert events[i - 1] == 'reconcile' or event == 'start' and events[i - 1] == 'resource-applied', (scenario, events)
|
|
foreign = scenario in ('dns-symlink', 'dns-preack-foreign', 'dns-replace', 'dns-replace-symlink', 'dns-stale', 'dns-stale-symlink')
|
|
if foreign:
|
|
ready = configs / 'dns-ready'
|
|
assert ready.exists() or ready.is_symlink(), (scenario, 'foreign readiness deleted')
|
|
if scenario != 'dns-stale-symlink' and scenario != 'dns-symlink':
|
|
assert ready.read_text() == 'foreign evidence\n', (scenario, 'foreign bytes changed')
|
|
elif realtime and (configs / 'dns-ready').exists():
|
|
# Exhaustion cannot authorize unlinking without the owned Go helper.
|
|
assert (configs / 'dns-ready').read_text().strip() == (root / 'dns-pid').read_text()
|
|
else:
|
|
assert not (configs / 'dns-ready').exists(), (scenario, 'stale DNS readiness')
|
|
if mode == 'native':
|
|
assert 'dns-start' not in events
|
|
return events
|
|
|
|
|
|
for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window', 'orphans'):
|
|
events = run(scenario, 0)
|
|
assert events.count('start') == 2, (scenario, events)
|
|
assert events.count('ss-server') == 2 and events.count('ss-local') == 2
|
|
assert events.count('stop') == 2
|
|
assert events.count('cooldown 15') == 1
|
|
for code in (64, 78, 1):
|
|
events = run(f'fatal{code}', code if code in (64, 78) else 78)
|
|
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
|
|
for scenario in ('mixed', 'unknown', 'unknown75', 'stopfail', 'joinfail', 'ackfail'):
|
|
events = run(scenario, 78)
|
|
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
|
|
if scenario.startswith('unknown'):
|
|
assert 'stop' not in events
|
|
events = run('budget', 78)
|
|
assert events.count('start') == 3
|
|
assert [e for e in events if e.startswith('cooldown')] == ['cooldown 15', 'cooldown 30']
|
|
events = run('cancel', 0)
|
|
assert events.count('start') == 1
|
|
events = run('prestartfail', 78)
|
|
assert 'start' not in events and 'stop' not in events
|
|
events = run('generic-ss', 0, mode='doh')
|
|
assert events.count('start') == 4 and events.count('stop') == 4, events
|
|
assert events.count('generic-pause') == 1, events
|
|
assert events.count('dns-start') == events.count('dns-stop') == 4, events
|
|
events = run('generic-unsafe', 0)
|
|
assert 'start' not in events and events.count('generic-pause') == 1, events
|
|
events = run('generic-apply-pass', 0, mode='doh')
|
|
assert events.count('resource-apply') == 1 and events.count('start') == 1, events
|
|
assert events.index('resource-applied') < events.index('start') < events.index('monitor 1'), events
|
|
events = run('generic-apply-cancel', 0, mode='doh')
|
|
assert 'resource-restored' in events and 'start' not in events and 'monitor 1' not in events, events
|
|
assert events.index('resource-restored') < events.index('stop') < events.index('dns-stop'), events
|
|
events = run('generic-apply-blocked', 0, mode='doh')
|
|
assert 'start' not in events and 'monitor 1' not in events and 'generic-pause' in events, events
|
|
for scenario, code in (('generic-apply-abrupt', 0), ('generic-apply-cleanup', 78)):
|
|
events = run(scenario, code, mode='doh')
|
|
assert 'start' not in events and 'monitor 1' not in events and 'resource-applied' not in events, events
|
|
for scenario in ('generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
|
|
events = run(scenario, 78, mode='doh')
|
|
assert events.count('start') == 1 and 'stop' not in events, (scenario, events)
|
|
for dns in ('8.8.8.8', '2001:4860:4860::8888'):
|
|
events = run('fatal64', 64, dns)
|
|
assert events.count('ss-server') == 1 and events.count('ss-local') == 1
|
|
|
|
for scenario in ('dns-outage', 'technical', 'ss', 'cancel', 'dns-death', 'dns-killed'):
|
|
events = run(scenario, 0, dns='8.8.8.8', mode=None)
|
|
assert events.index('dns-ready') < events.index('ss-server')
|
|
assert events.index('dns-ready') < events.index('ss-local')
|
|
if scenario not in ('dns-death', 'dns-killed'):
|
|
assert events.count('dns-start') == events.count('dns-stop')
|
|
assert events.index('stop') < events.index('dns-stop')
|
|
if scenario == 'dns-outage':
|
|
assert events.count('start') == 1 and 'cooldown 15' not in events
|
|
for scenario, code in (('dns-early', 78), ('dns-noack', 78), ('dns-wrongack', 78),
|
|
('dns-permissions', 78), ('dns-symlink', 78), ('dns-prestart', 78)):
|
|
events = run(scenario, code, mode='doh')
|
|
assert not any(e.startswith('monitor ') for e in events)
|
|
if scenario != 'dns-prestart':
|
|
assert 'ss-server' not in events and 'ss-local' not in events
|
|
for scenario in ('dns-stale', 'dns-stale-symlink', 'dns-preack-foreign', 'dns-replace', 'dns-replace-symlink'):
|
|
events = run(scenario, 78, mode='doh')
|
|
assert events.count('dns-start') <= 1 and 'cooldown 15' not in events
|
|
if scenario.startswith('dns-stale'):
|
|
assert not events
|
|
for scenario in ('dns-term', 'dns-int'):
|
|
events = run(scenario, 0, mode='doh')
|
|
assert events == ['dns-start', 'dns-stop'], events
|
|
|
|
for scenario in ('qualify-pass', 'qualify-negative', 'qualify-blocked', 'qualify-cancel'):
|
|
events = run(scenario, 0, mode='doh')
|
|
assert events.count('qualify') == 1 and not any(e.startswith('monitor ') for e in events), events
|
|
assert 'start' not in events and 'stop' not in events, events
|
|
assert events.index('dns-ready') < events.index('qualify') < events.index('dns-stop'), events
|
|
if scenario != 'qualify-blocked':
|
|
assert events.index('qualification-restored') < events.index('dns-stop'), events
|
|
events = run('qualify-missing', 64, mode='doh')
|
|
assert not events, events
|
|
for scenario, code in (('qualify-blocked-race', 0), ('qualify-unsafe-race', 78), ('qualify-unsafe', 78)):
|
|
events = run(scenario, code, mode='doh')
|
|
assert events.count('qualify') == 1 and 'start' not in events and 'stop' not in events, events
|
|
if scenario.endswith('-race'):
|
|
assert events.count('term-before-parent-reap') == 1, events
|
|
if code == 0:
|
|
assert events.index('term-before-parent-reap') < events.index('completion-checked') < events.index('dns-stop'), events
|
|
else:
|
|
assert 'completion-checked' not in events, events
|
|
|
|
|
|
def check_watchdog_traps(shell):
|
|
"""Same bounded_wait from normal flow, plain EXIT, and TERM -> EXIT traps."""
|
|
entrypoint = (ROOT / 'managed-target-entrypoint.sh').read_text()
|
|
bounded_wait = entrypoint[entrypoint.index('bounded_wait() {'):entrypoint.index('cleanup_cycle() {')]
|
|
bounded_wait = bounded_wait.replace('/opt/managed-target-supervision.sh', str(ROOT / 'managed-target-supervision.sh'))
|
|
for mode in ('normal', 'exit', 'term-exit'):
|
|
with tempfile.TemporaryDirectory(prefix='watchdog-traps-') as tmp:
|
|
directory = Path(tmp)
|
|
sleeper = directory / 'sleep'
|
|
sleeper.write_text('#!/bin/sh\necho $$ >> "$PIDS"\nexec "' + shutil.which('sleep') + '" "$@"\n')
|
|
sleeper.chmod(0o755)
|
|
script = directory / 'check.sh'
|
|
script.write_text(f'''set -eu
|
|
mode=$1
|
|
shift
|
|
config_dir={directory}
|
|
generic_mode=0
|
|
. {ROOT / 'managed-target-supervision.sh'}
|
|
{bounded_wait}
|
|
cleanup() {{
|
|
trap - EXIT
|
|
trap ':' INT TERM
|
|
kill "$owned" 2>/dev/null || true
|
|
bounded_wait "$owned" 5
|
|
[ "$join_timed_out" -eq 0 ]
|
|
}}
|
|
sleep 300 & owned=$!
|
|
case "$mode" in
|
|
normal) cleanup ;;
|
|
exit) trap cleanup EXIT; exit 0 ;;
|
|
term-exit) trap cleanup EXIT; trap 'exit 0' TERM; kill -TERM $$ ;;
|
|
esac
|
|
''')
|
|
env = dict(os.environ, PATH=f'{directory}:{os.environ["PATH"]}', PIDS=str(directory / 'pids'))
|
|
started = time.monotonic()
|
|
result = subprocess.run([*shell, str(script), mode], env=env, capture_output=True, timeout=10)
|
|
assert result.returncode == 0, (shell, mode, result.stderr)
|
|
assert time.monotonic() - started < 2, (shell, mode, 'watchdog cancellation delayed')
|
|
if (directory / 'pids').exists():
|
|
for pid in (directory / 'pids').read_text().splitlines():
|
|
assert not Path(f'/proc/{pid}').exists(), (shell, mode, 'orphan sleeper', pid)
|
|
|
|
|
|
def check_session_deadline(shell):
|
|
"""Startup sweep has no aggregate deadline; termination must keep its cap."""
|
|
entrypoint = (ROOT / 'managed-target-entrypoint.sh').read_text()
|
|
cleanup_session = entrypoint[entrypoint.index('cleanup_session() {'):entrypoint.index('capture_monitor_session() {')]
|
|
for deadline, live_until, expected, elapsed in (('', 102, 0, 2), ('180', 102, 0, 2), ('101', 110, 78, 1)):
|
|
with tempfile.TemporaryDirectory(prefix='session-deadline-') as tmp:
|
|
script = Path(tmp) / 'check.sh'
|
|
script.write_text(f'''set -eu
|
|
generic_mode=1
|
|
generic_cleanup_deadline='{deadline}'
|
|
monitor_sid=42
|
|
clock=100
|
|
date() {{ printf '%s\\n' "$clock"; }}
|
|
pause() {{ clock=$((clock + $1)); }}
|
|
signal_session() {{
|
|
[ "$1" = 42 ] || return 78
|
|
case "$2" in
|
|
TERM|KILL) ;;
|
|
CHECK) session_live=1; [ "$clock" -lt {live_until} ] || session_live=0 ;;
|
|
*) return 78 ;;
|
|
esac
|
|
}}
|
|
{cleanup_session}
|
|
code=0
|
|
cleanup_session || code=$?
|
|
[ "$code" -eq {expected} ]
|
|
[ "$clock" -eq {100 + elapsed} ]
|
|
[ "$generic_cleanup_deadline" = '{deadline}' ]
|
|
''')
|
|
result = subprocess.run([*shell, str(script)], capture_output=True, timeout=2)
|
|
assert result.returncode == 0, (shell, deadline, result.stderr)
|
|
expected_stderr = b'error: owned monitor session survived cleanup\n' if expected else b''
|
|
assert result.stderr == expected_stderr, (shell, deadline, result.stderr)
|
|
|
|
|
|
check_watchdog_traps(['/bin/sh'])
|
|
check_session_deadline(['/bin/sh'])
|
|
busybox = shutil.which('busybox')
|
|
if busybox:
|
|
check_watchdog_traps([busybox, 'ash'])
|
|
check_session_deadline([busybox, 'ash'])
|
|
for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')):
|
|
result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2)
|
|
assert result.returncode == 64
|
|
result = subprocess.run(['/bin/sh', '-c', 'helper=$1; shift; . "$helper"; proc_identity "$$"; own=$proc_sid; if signal_session "$own" KILL || signal_session 1 KILL; then exit 99; fi', 'check', str(ROOT / 'managed-target-supervision.sh')], capture_output=True, timeout=2)
|
|
assert result.returncode == 0, ('unsafe session accepted', result.returncode)
|
|
print('entrypoint supervision fixtures passed')
|