Files
zapret-autotune/deploy/managed-target/check_supervision.py
T
ZeusinaandGPT-6 Bot 4efc066625 feat: add guarded periodic resource monitoring
Add checks-only scheduling, read-only status, private authenticated persistence and guarded PID1 lifecycle. Keep strategy selection/recovery and remote fault acceptance out of scope.

Co-authored-by: GPT-6 <gpt-6@openai.com>
2026-10-13 23:33:31 +03:00

489 lines
23 KiB
Python

"""Isolated shell fixtures: python3 deploy/managed-target/check_supervision.py.
Only a temporary COPY of the entrypoint is redirected to fake executables;
production has no test switches, real network probes, or accelerated timers.
"""
import ctypes
import atexit
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import time
ROOT = Path(__file__).resolve().parents[2]
# Real readiness CLI checks root ownership. Use an isolated user namespace for
# offline fixtures when invoked as an ordinary user; no host-root requirement.
if os.geteuid() != 0:
os.execvp('unshare', ['unshare', '-Ur', 'python3', str(Path(__file__).resolve())])
CLI_TMP = tempfile.TemporaryDirectory(prefix='dns-readiness-cli-')
atexit.register(CLI_TMP.cleanup)
DNS_CLI = Path(CLI_TMP.name) / 'autotune'
subprocess.run(['go', 'build', '-o', str(DNS_CLI), './cmd/zapret-autotune'],
cwd=ROOT, env=dict(os.environ, GOPROXY='off', GOTOOLCHAIN='local'), check=True)
FAKE = r'''#!/usr/bin/env python3
import ctypes, os, pathlib, signal, subprocess, sys, time
root = pathlib.Path(os.environ['FIXTURE'])
name = pathlib.Path(sys.argv[0]).name
scenario = os.environ['SCENARIO']
def log(message):
with (root / 'events').open('a') as f:
f.write(message + '\n')
def increment(key):
path = root / key
n = int(path.read_text()) + 1 if path.exists() else 1
path.write_text(str(n))
return n
def linger(code=0):
def stop(*_):
if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh' and scenario not in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink', 'dns-prestart'):
os.kill(int((root / 'dns-pid').read_text()), 0)
sys.exit(code)
signal.signal(signal.SIGTERM, stop)
while True:
time.sleep(.01)
def assert_workers_gone():
if (root / 'workers').exists():
for pid in (root / 'workers').read_text().splitlines():
stat = pathlib.Path('/proc/' + pid + '/stat')
assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('old worker still live', pid)
if (root / 'peer').exists():
os.kill(int((root / 'peer').read_text()), 0)
if name in ('blockcheck', 'nfq'):
# Model Go's Setpgid check child and its NFQ worker, not a daemonized sysv
# process. Comm deliberately contains spaces/parentheses to test /proc parsing.
ctypes.CDLL(None).prctl(15, b'bc ) (fixture)', 0, 0, 0)
assert os.getsid(0) != os.getpgrp(), 'fixture did not split PGID from SID'
signal.signal(signal.SIGTERM, signal.SIG_IGN)
with (root / 'workers').open('a') as f:
f.write(str(os.getpid()) + '\n')
if name == 'blockcheck':
subprocess.Popen([str(root / 'bin' / 'nfq')])
while True:
time.sleep(.01)
if name == 'awk':
sys.exit(0)
if name == 'date':
n = increment('clock')
print(100000 + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0)))
sys.exit(0)
if name == 'sleep':
seconds = int(sys.argv[1])
if scenario.startswith('generic-budget-') and seconds != 60:
os.execv(os.environ['REAL_SLEEP'], [os.environ['REAL_SLEEP'], *sys.argv[1:]])
if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
log('generic-pause')
(root / 'budget-pause').touch()
if scenario != 'generic-ss':
os.kill(os.getppid(), signal.SIGTERM)
time.sleep(.01)
sys.exit(0)
if seconds in (15, 30):
log('cooldown ' + str(seconds))
if scenario == 'cancel':
os.kill(os.getppid(), signal.SIGTERM)
time.sleep(1)
else:
time.sleep(.01)
else:
time.sleep(.01 if seconds == 1 else 2)
sys.exit(0)
if name == 'timeout':
os.execv(sys.argv[4], sys.argv[4:])
if name in ('ss-server', 'ss-local'):
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
assert 'SS_PASSWORD' not in os.environ
assert sys.argv[1] == '-c'
dns = ('127.0.0.1:5353' if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh'
else os.environ.get('SS_DNS_SERVER', '')) if name == 'ss-server' else ''
assert sys.argv[3:] == (['-d', dns] if dns else [])
if dns == '127.0.0.1:5353':
assert (root / 'private' / 'dns-ready').read_text().strip() == (root / 'dns-pid').read_text()
os.kill(int((root / 'dns-pid').read_text()), 0)
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
log(name)
if scenario.startswith('generic-budget-'):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
time.sleep(.01)
if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail', 'generic-ss') and increment('ss_runs') <= (3 if scenario == 'generic-ss' else 1):
# Let startup/monitor begin before simulating a child crash.
while not (root / 'monitor_runs').exists() or (scenario == 'generic-ss' and int((root / 'monitor_runs').read_text()) < int((root / 'ss_runs').read_text())):
time.sleep(.01)
time.sleep(.03)
sys.exit(137)
linger()
if name == 'zapret2':
if scenario == 'orphans':
assert_workers_gone()
assert not (root / 'prepared').exists(), 'sourced prepared config'
log(sys.argv[1])
if scenario == 'stopfail' and sys.argv[1] == 'stop':
sys.exit(1)
sys.exit(0)
assert name == 'autotune'
if sys.argv[1:] == ['target-config-check']:
sys.exit(0)
if sys.argv[1] == 'dns-forwarder':
if sys.argv[2] in ('readiness-check', 'readiness-cleanup'):
# exec preserves PID1's actual inherited anchors; no simulated unlink.
os.execv(os.environ['DNS_CLI'], [os.environ['DNS_CLI'], *sys.argv[1:]])
mode = os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh')
if sys.argv[2] == 'mode':
print(mode)
sys.exit(0)
if sys.argv[2] == 'address':
print('127.0.0.1:5353' if mode == 'doh' else os.environ.get('SS_DNS_SERVER', ''))
sys.exit(0)
assert sys.argv[2:] == ['serve'] and mode == 'doh'
assert 'SS_PASSWORD' not in os.environ
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
(root / 'dns-pid').write_text(str(os.getpid()))
log('dns-start')
if scenario == 'dns-early':
sys.exit(1)
ready = root / 'private' / 'dns-ready'
owned = None
if scenario == 'dns-preack-foreign':
ready.write_text('foreign evidence\n')
sys.exit(1)
if scenario not in ('dns-noack', 'dns-term', 'dns-int'):
with ready.open('x') as f:
os.chmod(ready, 0o600)
f.write(str(os.getpid() + (1 if scenario == 'dns-wrongack' else 0)) + '\n')
owned = ready.stat()
if scenario == 'dns-permissions':
ready.chmod(0o644)
if scenario == 'dns-symlink':
payload = ready.with_name('dns-payload')
ready.rename(payload)
ready.symlink_to(payload)
log('dns-ready')
def stop_dns(*_):
assert not (root / 'prepared').exists(), 'DNS stopped before restoration'
# SS children have been joined, not merely signalled.
for pid in (root / 'children').read_text().splitlines():
if int(pid) != os.getpid():
assert not pathlib.Path('/proc/' + pid).exists(), ('DNS stopped before child join', pid)
log('dns-stop')
if ready.exists() and not ready.is_symlink() and owned and os.path.samestat(owned, ready.stat()):
ready.unlink()
sys.exit(0)
signal.signal(signal.SIGTERM, stop_dns)
if scenario.startswith('generic-budget-'):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
if scenario in ('dns-term', 'dns-int'):
os.kill(os.getppid(), signal.SIGTERM if scenario == 'dns-term' else signal.SIGINT)
if scenario in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink') and increment('dns_runs') == 1:
while not (root / 'monitor_runs').exists():
time.sleep(.01)
if scenario.startswith('dns-replace'):
# Retain old inode so a replacement cannot reuse its number.
ready.rename(ready.with_name('retired-ready'))
payload = ready.with_name('foreign-ready')
payload.write_text('foreign evidence\n')
payload.chmod(0o600)
if scenario == 'dns-replace-symlink':
ready.symlink_to(payload)
else:
payload.rename(ready)
os.kill(os.getpid(), signal.SIGKILL)
while True:
time.sleep(.01)
command = sys.argv[2]
if command == 'init':
log('init')
(root / 'config').touch()
elif command == 'reconcile':
n = increment('reconciles')
log('reconcile')
if scenario.startswith('generic-budget-') and n == 2:
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
time.sleep(.01)
if scenario == 'orphans':
assert_workers_gone()
if scenario in ('prestartfail', 'generic-unsafe'):
sys.exit(78)
if scenario == 'dns-prestart':
os.kill(int((root / 'dns-pid').read_text()), signal.SIGKILL)
time.sleep(.03)
if scenario in ('unknown', 'unknown75') and n == 2:
sys.exit(78)
(root / 'prepared').unlink(missing_ok=True)
elif command == 'monitor':
if scenario.startswith('generic-'):
assert os.environ['ZAPRET_AUTOTUNE_RESOURCE_CONFIG'] == '/operator/resources.json'
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
n = increment('monitor_runs')
log('monitor ' + str(n))
assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session'
if scenario.startswith('generic-budget-'):
time.sleep(.03)
os.kill(os.getppid(), signal.SIGTERM)
if scenario == 'generic-budget-monitor':
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
time.sleep(.01)
linger()
if scenario == 'generic-ss' and n <= 3:
linger()
if scenario in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink') and n == 1:
linger()
if scenario == 'dns-outage':
# Upstream SERVFAIL is modelled by a live DNS child, not child failure.
assert (root / 'dns-pid').exists()
if n == 1 and scenario == 'orphans':
subprocess.Popen([str(root / 'bin' / 'blockcheck')], preexec_fn=os.setpgrp)
while not (root / 'workers').exists() or len((root / 'workers').read_text().splitlines()) != 2:
time.sleep(.01)
(root / 'prepared').touch()
os.kill(os.getpid(), signal.SIGKILL)
if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'):
sys.exit(75)
if scenario.startswith('fatal'):
sys.exit(int(scenario[5:]))
if n == 1 and scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'unknown', 'window'):
if scenario.startswith('crash') or scenario == 'unknown':
(root / 'prepared').touch()
code = int(scenario[5:]) if scenario.startswith('crash') and scenario != 'crash' else 2
sys.exit(code if scenario.startswith('crash') or scenario == 'unknown' else 75)
if scenario == 'joinfail':
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
time.sleep(.01)
if n == 1 and scenario in ('ss', 'mixed'):
linger(78 if scenario == 'mixed' else 0)
# Successful second runtime: external SIGTERM must exit 0, not restart.
time.sleep(.03)
os.kill(os.getppid(), signal.SIGTERM)
linger()
'''
def run(scenario, expected, dns='', mode='native'):
with tempfile.TemporaryDirectory(prefix='supervision-') as tmp:
root = Path(tmp)
binary = root / 'bin'
binary.mkdir()
fake = binary / 'fake'
fake.write_text(FAKE)
fake.chmod(0o755)
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2', 'blockcheck', 'nfq'):
(binary / name).symlink_to(fake)
realtime = scenario.startswith('generic-budget-')
if realtime:
for name in ('date', 'timeout'):
(binary / name).unlink()
configs = root / 'private'
configs.mkdir()
if scenario.startswith('dns-stale'):
if scenario == 'dns-stale-symlink':
(configs / 'dns-ready').symlink_to(configs / 'missing')
else:
(configs / 'dns-ready').write_text('foreign evidence\n')
text = (ROOT / 'managed-target-entrypoint.sh').read_text()
helper = ROOT / 'managed-target-supervision.sh'
if scenario == 'ackfail':
broken_helper = root / 'broken-helper.sh'
broken_helper.write_text(helper.read_text().replace('sleep "$seconds" & sleeper=$!', 'exit 78'))
helper = broken_helper
if scenario == 'generic-budget-ack':
delayed_helper = root / 'delayed-helper.sh'
delayed_helper.write_text(helper.read_text().replace(': > "$config_dir/join-ready"',
f'{shutil.which("sleep")} 300\n : > "$config_dir/join-ready"'))
helper = delayed_helper
text = text.replace('/opt/managed-target-supervision.sh', str(helper))
text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}')
text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune'))
text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2'))
if realtime:
# Private fixture scaling only: production has no configurable grace.
text = text.replace('$(date +%s) + 80', '$(date +%s) + 12')
entry = root / 'entry.sh'
entry.write_text(text)
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
SCENARIO=scenario, SS_PASSWORD='ab' * 32, SS_DNS_SERVER=dns,
ZAPRET_CONFIG=str(root / 'config'), DNS_CLI=str(DNS_CLI),
REAL_SLEEP=shutil.which('sleep'))
if scenario.startswith('generic-'):
env['ZAPRET_AUTOTUNE_RESOURCE_CONFIG'] = '/operator/resources.json'
else:
env.pop('ZAPRET_AUTOTUNE_RESOURCE_CONFIG', None)
if mode is None:
env.pop('ZAPRET_AUTOTUNE_DNS_MODE', None)
else:
env['ZAPRET_AUTOTUNE_DNS_MODE'] = mode
peer = None
if scenario == 'orphans':
# Local fixture is not PID 1. Adopt orphan workers here so the test
# can reap their verified-dead zombies rather than burden host PID 1.
assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0
peer = subprocess.Popen([shutil.which('sleep'), '300'], start_new_session=True)
(root / 'peer').write_text(str(peer.pid))
try:
started = time.monotonic()
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=18 if realtime else 15)
if realtime:
assert time.monotonic() - started < 16, (scenario, 'aggregate cleanup exceeded grace')
if peer:
assert peer.poll() is None, 'unowned helper was killed'
finally:
if peer:
peer.terminate()
peer.wait(timeout=2)
events = (root / 'events').read_text().splitlines() if (root / 'events').exists() else []
assert result.returncode == expected, (scenario, result.returncode, result.stderr, events)
assert b'ab' * 32 not in result.stdout + result.stderr
for pid in ((root / 'children').read_text().splitlines() if (root / 'children').exists() else []):
assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid)
if scenario == 'orphans':
for pid in (root / 'workers').read_text().splitlines():
stat = Path(f'/proc/{pid}/stat')
assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('orphan survived', pid)
try:
reaped, _ = os.waitpid(int(pid), os.WNOHANG)
assert reaped == int(pid), ('worker not reaped', pid)
except ChildProcessError:
assert not stat.exists(), ('worker was not adopted/reaped', pid)
assert ctypes.CDLL(None).prctl(36, 0, 0, 0, 0) == 0
# Every activation and stop must immediately follow successful reconcile.
for i, event in enumerate(events):
if event in ('start', 'stop'):
assert events[i - 1] == 'reconcile', (scenario, events)
foreign = scenario in ('dns-symlink', 'dns-preack-foreign', 'dns-replace', 'dns-replace-symlink', 'dns-stale', 'dns-stale-symlink')
if foreign:
ready = configs / 'dns-ready'
assert ready.exists() or ready.is_symlink(), (scenario, 'foreign readiness deleted')
if scenario != 'dns-stale-symlink' and scenario != 'dns-symlink':
assert ready.read_text() == 'foreign evidence\n', (scenario, 'foreign bytes changed')
elif realtime and (configs / 'dns-ready').exists():
# Exhaustion cannot authorize unlinking without the owned Go helper.
assert (configs / 'dns-ready').read_text().strip() == (root / 'dns-pid').read_text()
else:
assert not (configs / 'dns-ready').exists(), (scenario, 'stale DNS readiness')
if mode == 'native':
assert 'dns-start' not in events
return events
for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window', 'orphans'):
events = run(scenario, 0)
assert events.count('start') == 2, (scenario, events)
assert events.count('ss-server') == 2 and events.count('ss-local') == 2
assert events.count('stop') == 2
assert events.count('cooldown 15') == 1
for code in (64, 78, 1):
events = run(f'fatal{code}', code if code in (64, 78) else 78)
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
for scenario in ('mixed', 'unknown', 'unknown75', 'stopfail', 'joinfail', 'ackfail'):
events = run(scenario, 78)
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
if scenario.startswith('unknown'):
assert 'stop' not in events
events = run('budget', 78)
assert events.count('start') == 3
assert [e for e in events if e.startswith('cooldown')] == ['cooldown 15', 'cooldown 30']
events = run('cancel', 0)
assert events.count('start') == 1
events = run('prestartfail', 78)
assert 'start' not in events and 'stop' not in events
events = run('generic-ss', 0, mode='doh')
assert events.count('start') == 4 and events.count('stop') == 4, events
assert events.count('generic-pause') == 1, events
assert events.count('dns-start') == events.count('dns-stop') == 4, events
events = run('generic-unsafe', 0)
assert 'start' not in events and events.count('generic-pause') == 1, events
for scenario in ('generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
events = run(scenario, 78, mode='doh')
assert events.count('start') == 1 and 'stop' not in events, (scenario, events)
for dns in ('8.8.8.8', '2001:4860:4860::8888'):
events = run('fatal64', 64, dns)
assert events.count('ss-server') == 1 and events.count('ss-local') == 1
for scenario in ('dns-outage', 'technical', 'ss', 'cancel', 'dns-death', 'dns-killed'):
events = run(scenario, 0, dns='8.8.8.8', mode=None)
assert events.index('dns-ready') < events.index('ss-server')
assert events.index('dns-ready') < events.index('ss-local')
if scenario not in ('dns-death', 'dns-killed'):
assert events.count('dns-start') == events.count('dns-stop')
assert events.index('stop') < events.index('dns-stop')
if scenario == 'dns-outage':
assert events.count('start') == 1 and 'cooldown 15' not in events
for scenario, code in (('dns-early', 78), ('dns-noack', 78), ('dns-wrongack', 78),
('dns-permissions', 78), ('dns-symlink', 78), ('dns-prestart', 78)):
events = run(scenario, code, mode='doh')
assert not any(e.startswith('monitor ') for e in events)
if scenario != 'dns-prestart':
assert 'ss-server' not in events and 'ss-local' not in events
for scenario in ('dns-stale', 'dns-stale-symlink', 'dns-preack-foreign', 'dns-replace', 'dns-replace-symlink'):
events = run(scenario, 78, mode='doh')
assert events.count('dns-start') <= 1 and 'cooldown 15' not in events
if scenario.startswith('dns-stale'):
assert not events
for scenario in ('dns-term', 'dns-int'):
events = run(scenario, 0, mode='doh')
assert events == ['dns-start', 'dns-stop'], events
def check_watchdog_traps(shell):
"""Same bounded_wait from normal flow, plain EXIT, and TERM -> EXIT traps."""
entrypoint = (ROOT / 'managed-target-entrypoint.sh').read_text()
bounded_wait = entrypoint[entrypoint.index('bounded_wait() {'):entrypoint.index('cleanup_cycle() {')]
bounded_wait = bounded_wait.replace('/opt/managed-target-supervision.sh', str(ROOT / 'managed-target-supervision.sh'))
for mode in ('normal', 'exit', 'term-exit'):
with tempfile.TemporaryDirectory(prefix='watchdog-traps-') as tmp:
directory = Path(tmp)
sleeper = directory / 'sleep'
sleeper.write_text('#!/bin/sh\necho $$ >> "$PIDS"\nexec "' + shutil.which('sleep') + '" "$@"\n')
sleeper.chmod(0o755)
script = directory / 'check.sh'
script.write_text(f'''set -eu
mode=$1
shift
config_dir={directory}
generic_mode=0
. {ROOT / 'managed-target-supervision.sh'}
{bounded_wait}
cleanup() {{
trap - EXIT
trap ':' INT TERM
kill "$owned" 2>/dev/null || true
bounded_wait "$owned" 5
[ "$join_timed_out" -eq 0 ]
}}
sleep 300 & owned=$!
case "$mode" in
normal) cleanup ;;
exit) trap cleanup EXIT; exit 0 ;;
term-exit) trap cleanup EXIT; trap 'exit 0' TERM; kill -TERM $$ ;;
esac
''')
env = dict(os.environ, PATH=f'{directory}:{os.environ["PATH"]}', PIDS=str(directory / 'pids'))
started = time.monotonic()
result = subprocess.run([*shell, str(script), mode], env=env, capture_output=True, timeout=10)
assert result.returncode == 0, (shell, mode, result.stderr)
assert time.monotonic() - started < 2, (shell, mode, 'watchdog cancellation delayed')
if (directory / 'pids').exists():
for pid in (directory / 'pids').read_text().splitlines():
assert not Path(f'/proc/{pid}').exists(), (shell, mode, 'orphan sleeper', pid)
check_watchdog_traps(['/bin/sh'])
busybox = shutil.which('busybox')
if busybox:
check_watchdog_traps([busybox, 'ash'])
for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')):
result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2)
assert result.returncode == 64
result = subprocess.run(['/bin/sh', '-c', 'helper=$1; shift; . "$helper"; proc_identity "$$"; own=$proc_sid; if signal_session "$own" KILL || signal_session 1 KILL; then exit 99; fi', 'check', str(ROOT / 'managed-target-supervision.sh')], capture_output=True, timeout=2)
assert result.returncode == 0, ('unsafe session accepted', result.returncode)
print('entrypoint supervision fixtures passed')