fix(runtime): keep supervision and absence retries in container

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-11 09:01:52 +03:00
1 parent 226d93e2aa
commit 3f9fd2f36f
21 files changed
+1056 -137

No files matched your search

+223
View File
@@ -0,0 +1,223 @@
"""Isolated shell fixtures: python3 deploy/managed-target/check_supervision.py.
Only a temporary COPY of the entrypoint is redirected to fake executables;
production has no test switches, real network probes, or accelerated timers.
"""
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import time
ROOT = Path(__file__).resolve().parents[2]
FAKE = r'''#!/usr/bin/env python3
import os, pathlib, signal, sys, time
root = pathlib.Path(os.environ['FIXTURE'])
name = pathlib.Path(sys.argv[0]).name
scenario = os.environ['SCENARIO']
def log(message):
with (root / 'events').open('a') as f:
f.write(message + '\n')
def increment(key):
path = root / key
n = int(path.read_text()) + 1 if path.exists() else 1
path.write_text(str(n))
return n
def linger(code=0):
signal.signal(signal.SIGTERM, lambda *_: sys.exit(code))
while True:
time.sleep(.01)
if name == 'awk':
sys.exit(0)
if name == 'date':
n = increment('clock')
print(100000 + (n * 21601 if scenario == 'window' else n))
sys.exit(0)
if name == 'sleep':
seconds = int(sys.argv[1])
if seconds in (15, 30):
log('cooldown ' + str(seconds))
if scenario == 'cancel':
os.kill(os.getppid(), signal.SIGTERM)
time.sleep(1)
else:
time.sleep(.01)
else:
time.sleep(.01 if seconds == 1 else 2)
sys.exit(0)
if name == 'timeout':
os.execv(sys.argv[4], sys.argv[4:])
if name in ('ss-server', 'ss-local'):
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
assert 'SS_PASSWORD' not in os.environ
assert sys.argv[1] == '-c' and len(sys.argv) == 3
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
log(name)
if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail') and increment('ss_runs') == 1:
# Let startup/monitor begin before simulating a child crash.
while not (root / 'monitor_runs').exists():
time.sleep(.01)
time.sleep(.03)
sys.exit(137)
linger()
if name == 'zapret2':
assert not (root / 'prepared').exists(), 'sourced prepared config'
log(sys.argv[1])
if scenario == 'stopfail' and sys.argv[1] == 'stop':
sys.exit(1)
sys.exit(0)
assert name == 'autotune'
command = sys.argv[2]
if command == 'init':
log('init')
(root / 'config').touch()
elif command == 'reconcile':
n = increment('reconciles')
log('reconcile')
if scenario == 'prestartfail':
sys.exit(78)
if scenario in ('unknown', 'unknown75') and n == 2:
sys.exit(78)
(root / 'prepared').unlink(missing_ok=True)
elif command == 'monitor':
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
n = increment('monitor_runs')
log('monitor ' + str(n))
if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'):
sys.exit(75)
if scenario.startswith('fatal'):
sys.exit(int(scenario[5:]))
if n == 1 and scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'unknown', 'window'):
if scenario.startswith('crash') or scenario == 'unknown':
(root / 'prepared').touch()
code = int(scenario[5:]) if scenario.startswith('crash') and scenario != 'crash' else 2
sys.exit(code if scenario.startswith('crash') or scenario == 'unknown' else 75)
if scenario == 'joinfail':
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
time.sleep(.01)
if n == 1 and scenario in ('ss', 'mixed'):
linger(78 if scenario == 'mixed' else 0)
# Successful second runtime: external SIGTERM must exit 0, not restart.
time.sleep(.03)
os.kill(os.getppid(), signal.SIGTERM)
linger()
'''
def run(scenario, expected):
with tempfile.TemporaryDirectory(prefix='supervision-') as tmp:
root = Path(tmp)
binary = root / 'bin'
binary.mkdir()
fake = binary / 'fake'
fake.write_text(FAKE)
fake.chmod(0o755)
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2'):
(binary / name).symlink_to(fake)
configs = root / 'private'
configs.mkdir()
text = (ROOT / 'managed-target-entrypoint.sh').read_text()
helper = ROOT / 'managed-target-supervision.sh'
if scenario == 'ackfail':
broken_helper = root / 'broken-helper.sh'
broken_helper.write_text(helper.read_text().replace('sleep "$seconds" & sleeper=$!', 'exit 78'))
helper = broken_helper
text = text.replace('/opt/managed-target-supervision.sh', str(helper))
text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}')
text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune'))
text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2'))
entry = root / 'entry.sh'
entry.write_text(text)
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config'))
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15)
events = (root / 'events').read_text().splitlines()
assert result.returncode == expected, (scenario, result.returncode, result.stderr, events)
assert b'ab' * 32 not in result.stdout + result.stderr
for pid in (root / 'children').read_text().splitlines():
assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid)
# Every activation and stop must immediately follow successful reconcile.
for i, event in enumerate(events):
if event in ('start', 'stop'):
assert events[i - 1] == 'reconcile', (scenario, events)
return events
for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window'):
events = run(scenario, 0)
assert events.count('start') == 2, (scenario, events)
assert events.count('ss-server') == 2 and events.count('ss-local') == 2
assert events.count('stop') == 2
assert events.count('cooldown 15') == 1
for code in (64, 78, 1):
events = run(f'fatal{code}', code if code in (64, 78) else 78)
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
for scenario in ('mixed', 'unknown', 'unknown75', 'stopfail', 'joinfail', 'ackfail'):
events = run(scenario, 78)
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
if scenario.startswith('unknown'):
assert 'stop' not in events
events = run('budget', 78)
assert events.count('start') == 3
assert [e for e in events if e.startswith('cooldown')] == ['cooldown 15', 'cooldown 30']
events = run('cancel', 0)
assert events.count('start') == 1
events = run('prestartfail', 78)
assert 'start' not in events and 'stop' not in events
def check_watchdog_traps(shell):
"""Same bounded_wait from normal flow, plain EXIT, and TERM -> EXIT traps."""
entrypoint = (ROOT / 'managed-target-entrypoint.sh').read_text()
bounded_wait = entrypoint[entrypoint.index('bounded_wait() {'):entrypoint.index('cleanup_cycle() {')]
bounded_wait = bounded_wait.replace('/opt/managed-target-supervision.sh', str(ROOT / 'managed-target-supervision.sh'))
for mode in ('normal', 'exit', 'term-exit'):
with tempfile.TemporaryDirectory(prefix='watchdog-traps-') as tmp:
directory = Path(tmp)
sleeper = directory / 'sleep'
sleeper.write_text('#!/bin/sh\necho $$ >> "$PIDS"\nexec "' + shutil.which('sleep') + '" "$@"\n')
sleeper.chmod(0o755)
script = directory / 'check.sh'
script.write_text(f'''set -eu
mode=$1
shift
config_dir={directory}
. {ROOT / 'managed-target-supervision.sh'}
{bounded_wait}
cleanup() {{
trap - EXIT
trap ':' INT TERM
kill "$owned" 2>/dev/null || true
bounded_wait "$owned" 5
[ "$join_timed_out" -eq 0 ]
}}
sleep 300 & owned=$!
case "$mode" in
normal) cleanup ;;
exit) trap cleanup EXIT; exit 0 ;;
term-exit) trap cleanup EXIT; trap 'exit 0' TERM; kill -TERM $$ ;;
esac
''')
env = dict(os.environ, PATH=f'{directory}:{os.environ["PATH"]}', PIDS=str(directory / 'pids'))
started = time.monotonic()
result = subprocess.run([*shell, str(script), mode], env=env, capture_output=True, timeout=10)
assert result.returncode == 0, (shell, mode, result.stderr)
assert time.monotonic() - started < 2, (shell, mode, 'watchdog cancellation delayed')
if (directory / 'pids').exists():
for pid in (directory / 'pids').read_text().splitlines():
assert not Path(f'/proc/{pid}').exists(), (shell, mode, 'orphan sleeper', pid)
check_watchdog_traps(['/bin/sh'])
busybox = shutil.which('busybox')
if busybox:
check_watchdog_traps([busybox, 'ash'])
for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')):
result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2)
assert result.returncode == 64
print('entrypoint supervision fixtures passed')