fix(runtime): keep supervision and absence retries in container

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-11 09:01:52 +03:00
1 parent 226d93e2aa
commit 3f9fd2f36f
21 files changed
+1056 -137

No files matched your search

+22 -23
View File
@@ -34,36 +34,34 @@ in logs or shared terminals.
By default Compose uses the locally built `zapret-autotune-managed-target:local`.
Tags are only local build/test candidates, not production release references.
For production promotion, set `ZAPRET_AUTOTUNE_IMAGE=<registry>/<managed-target>@sha256:<digest>`
in the private `.env` and pull that immutable image before starting the unit.
Use a managed target image, not the generic image; the systemd command never
builds images. Do not enable the production service with a tag or local default.
in the private `.env` and pull that immutable image before starting the target.
Use a managed target image, not the generic image. Do not deploy production
with a tag or local default.
Offline static guard (from repository root):
`python3 deploy/managed-target/check_static.py`.
Isolated, accelerated shell supervision fixtures (no network):
`python3 deploy/managed-target/check_supervision.py`.
## Optional native systemd supervision
## In-container supervision
For unattended boot, place this checkout at `/opt/zapret2-autotune`, provision
`deploy/managed-target/.env` and state as above, and build the target image first.
For production, select and pull the immutable managed-target digest as above
instead of using a local build tag.
Install `deploy/managed-target/zapret-autotune.service` in `/etc/systemd/system/`,
then run `systemctl daemon-reload` and `systemctl enable --now zapret-autotune`.
Do not simultaneously start the same project with detached Compose.
Enabling the unit starts this isolated target on subsequent boots. `ExecStop`
only stops `target`; it does not replace the foreground process's failure code.
An explicit service stop suppresses restart, including during a shutdown race.
Compose has no automatic restart. The foreground systemd unit restarts **only
exit 75**, with at most three starts per six hours. Clean typed recovery timeouts,
Compose has no automatic restart. PID 1 restarts the entire owned runtime only
for technical failures, with three starts per rolling six hours (including initial
startup) and cancellable 15/30-second cooldowns. Clean typed recovery timeouts,
unexpected monitor exits/crashes, and Shadowsocks failures qualify only if
shutdown cleanup succeeds. Configuration (64), safety/database/rollback/cleanup
failures (78), and graceful SIGTERM (0) never request restart. Investigate and
failures (78), and graceful SIGTERM (0) never request restart. Exhausting the
restart budget exits 78. Investigate and
repair nonretry failures before manually starting again; do not clear state.
Monitor cancellation/join is bounded to 60 seconds, final target-local zapret
Monitor cancellation/join is bounded to 45 seconds, reconciliation gets 45 seconds
plus five to force kill, and final target-local zapret
stop gets 30 seconds then forced kill after five more, and each Shadowsocks join
gets five seconds. Exceeding a join bound fails closed. Compose grants 120
seconds; systemd grants 180 seconds.
gets five seconds. Exceeding a join bound fails closed. Compose grants 240
seconds. Reconcile after joining the monitor and before even stopping zapret:
stop also sources config, so unknown state is never sourced. The sequential
cleanup budget is 140 seconds; 240 seconds also covers an in-flight startup
command and scheduling grace. Watchdogs run in fresh shell interpreters and
cancel/join their own sleepers, including during PID 1's EXIT trap.
## Isolation and runtime contract
@@ -97,8 +95,9 @@ Any startup command failure/timeout exits 78, never the retry code 75. These
bounds do not shorten the monitor's 90-minute bootstrap/recovery scan budget.
SIGTERM/SIGINT cancels and waits for the monitor before stopping target-local
zapret and Shadowsocks children. Monitor fatal exits stop the
entrypoint. Monitor health is YouTube Player API status through the owned SOCKS,
zapret and Shadowsocks children. Nonretry monitor fatal exits stop the
entrypoint; technical exits and Shadowsocks failures restart from private configs
only after safe cleanup. Monitor health is YouTube Player API status through the owned SOCKS,
not video playback/SABR; GGC candidates are scoped to the exact discovered
hostname.
+9 -14
View File
@@ -7,7 +7,6 @@ directory = Path(__file__).resolve().parent
compose = (directory / "compose.yaml").read_text()
entrypoint = (directory.parents[1] / "managed-target-entrypoint.sh").read_text()
readme = (directory / "README.md").read_text()
unit = (directory / "zapret-autotune.service").read_text()
# ponytail: checks the committed Compose shape, not arbitrary YAML; use a YAML
# parser if this definition grows beyond its single target service.
@@ -29,19 +28,15 @@ assert "Docker administrators/root can still read container" in readme
assert "removed\nfrom the entrypoint environment" in readme
assert "No host networking" in readme
assert ' restart: "no"\n' in compose
assert " stop_grace_period: 120s\n" in compose
directives = dict(line.split("=", 1) for line in unit.splitlines() if "=" in line)
assert directives["Restart"] == "no"
assert directives["RestartForceExitStatus"] == "75"
assert "RestartPreventExitStatus" not in directives
assert directives["StartLimitIntervalSec"] == "6h"
assert directives["StartLimitBurst"] == "3"
assert directives["TimeoutStopSec"] == "180s"
assert directives["WorkingDirectory"] == "/opt/zapret2-autotune"
assert directives["WantedBy"] == "multi-user.target"
compose_command = "/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml"
assert directives["ExecStart"] == compose_command + " up --no-build --abort-on-container-exit --exit-code-from target target"
assert directives["ExecStop"] == compose_command + " stop target"
assert " stop_grace_period: 240s\n" in compose
assert 'bounded_wait "$monitor_pid" 45' in entrypoint
assert '/bin/sh /opt/managed-target-supervision.sh --watchdog' in entrypoint
assert 'COPY managed-target-supervision.sh /opt/managed-target-supervision.sh' in (directory.parents[1] / 'Dockerfile.target').read_text()
assert '!managed-target-supervision.sh' in (directory.parents[1] / '.dockerignore').read_text()
assert '"$count" -lt 3' in entrypoint
assert '"$((now - started))" -lt 21600' in entrypoint
assert "backoff=15" in entrypoint
assert "systemctl" not in entrypoint and "docker " not in entrypoint
assert "timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop" in entrypoint
for command, seconds in (("/usr/local/bin/zapret-autotune managed init", 45),
("/usr/local/bin/zapret-autotune managed reconcile", 45),
+223
View File
@@ -0,0 +1,223 @@
"""Isolated shell fixtures: python3 deploy/managed-target/check_supervision.py.
Only a temporary COPY of the entrypoint is redirected to fake executables;
production has no test switches, real network probes, or accelerated timers.
"""
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import time
ROOT = Path(__file__).resolve().parents[2]
FAKE = r'''#!/usr/bin/env python3
import os, pathlib, signal, sys, time
root = pathlib.Path(os.environ['FIXTURE'])
name = pathlib.Path(sys.argv[0]).name
scenario = os.environ['SCENARIO']
def log(message):
with (root / 'events').open('a') as f:
f.write(message + '\n')
def increment(key):
path = root / key
n = int(path.read_text()) + 1 if path.exists() else 1
path.write_text(str(n))
return n
def linger(code=0):
signal.signal(signal.SIGTERM, lambda *_: sys.exit(code))
while True:
time.sleep(.01)
if name == 'awk':
sys.exit(0)
if name == 'date':
n = increment('clock')
print(100000 + (n * 21601 if scenario == 'window' else n))
sys.exit(0)
if name == 'sleep':
seconds = int(sys.argv[1])
if seconds in (15, 30):
log('cooldown ' + str(seconds))
if scenario == 'cancel':
os.kill(os.getppid(), signal.SIGTERM)
time.sleep(1)
else:
time.sleep(.01)
else:
time.sleep(.01 if seconds == 1 else 2)
sys.exit(0)
if name == 'timeout':
os.execv(sys.argv[4], sys.argv[4:])
if name in ('ss-server', 'ss-local'):
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
assert 'SS_PASSWORD' not in os.environ
assert sys.argv[1] == '-c' and len(sys.argv) == 3
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
log(name)
if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail') and increment('ss_runs') == 1:
# Let startup/monitor begin before simulating a child crash.
while not (root / 'monitor_runs').exists():
time.sleep(.01)
time.sleep(.03)
sys.exit(137)
linger()
if name == 'zapret2':
assert not (root / 'prepared').exists(), 'sourced prepared config'
log(sys.argv[1])
if scenario == 'stopfail' and sys.argv[1] == 'stop':
sys.exit(1)
sys.exit(0)
assert name == 'autotune'
command = sys.argv[2]
if command == 'init':
log('init')
(root / 'config').touch()
elif command == 'reconcile':
n = increment('reconciles')
log('reconcile')
if scenario == 'prestartfail':
sys.exit(78)
if scenario in ('unknown', 'unknown75') and n == 2:
sys.exit(78)
(root / 'prepared').unlink(missing_ok=True)
elif command == 'monitor':
with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n')
n = increment('monitor_runs')
log('monitor ' + str(n))
if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'):
sys.exit(75)
if scenario.startswith('fatal'):
sys.exit(int(scenario[5:]))
if n == 1 and scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'unknown', 'window'):
if scenario.startswith('crash') or scenario == 'unknown':
(root / 'prepared').touch()
code = int(scenario[5:]) if scenario.startswith('crash') and scenario != 'crash' else 2
sys.exit(code if scenario.startswith('crash') or scenario == 'unknown' else 75)
if scenario == 'joinfail':
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
time.sleep(.01)
if n == 1 and scenario in ('ss', 'mixed'):
linger(78 if scenario == 'mixed' else 0)
# Successful second runtime: external SIGTERM must exit 0, not restart.
time.sleep(.03)
os.kill(os.getppid(), signal.SIGTERM)
linger()
'''
def run(scenario, expected):
with tempfile.TemporaryDirectory(prefix='supervision-') as tmp:
root = Path(tmp)
binary = root / 'bin'
binary.mkdir()
fake = binary / 'fake'
fake.write_text(FAKE)
fake.chmod(0o755)
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2'):
(binary / name).symlink_to(fake)
configs = root / 'private'
configs.mkdir()
text = (ROOT / 'managed-target-entrypoint.sh').read_text()
helper = ROOT / 'managed-target-supervision.sh'
if scenario == 'ackfail':
broken_helper = root / 'broken-helper.sh'
broken_helper.write_text(helper.read_text().replace('sleep "$seconds" & sleeper=$!', 'exit 78'))
helper = broken_helper
text = text.replace('/opt/managed-target-supervision.sh', str(helper))
text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}')
text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune'))
text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2'))
entry = root / 'entry.sh'
entry.write_text(text)
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config'))
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15)
events = (root / 'events').read_text().splitlines()
assert result.returncode == expected, (scenario, result.returncode, result.stderr, events)
assert b'ab' * 32 not in result.stdout + result.stderr
for pid in (root / 'children').read_text().splitlines():
assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid)
# Every activation and stop must immediately follow successful reconcile.
for i, event in enumerate(events):
if event in ('start', 'stop'):
assert events[i - 1] == 'reconcile', (scenario, events)
return events
for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window'):
events = run(scenario, 0)
assert events.count('start') == 2, (scenario, events)
assert events.count('ss-server') == 2 and events.count('ss-local') == 2
assert events.count('stop') == 2
assert events.count('cooldown 15') == 1
for code in (64, 78, 1):
events = run(f'fatal{code}', code if code in (64, 78) else 78)
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
for scenario in ('mixed', 'unknown', 'unknown75', 'stopfail', 'joinfail', 'ackfail'):
events = run(scenario, 78)
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
if scenario.startswith('unknown'):
assert 'stop' not in events
events = run('budget', 78)
assert events.count('start') == 3
assert [e for e in events if e.startswith('cooldown')] == ['cooldown 15', 'cooldown 30']
events = run('cancel', 0)
assert events.count('start') == 1
events = run('prestartfail', 78)
assert 'start' not in events and 'stop' not in events
def check_watchdog_traps(shell):
"""Same bounded_wait from normal flow, plain EXIT, and TERM -> EXIT traps."""
entrypoint = (ROOT / 'managed-target-entrypoint.sh').read_text()
bounded_wait = entrypoint[entrypoint.index('bounded_wait() {'):entrypoint.index('cleanup_cycle() {')]
bounded_wait = bounded_wait.replace('/opt/managed-target-supervision.sh', str(ROOT / 'managed-target-supervision.sh'))
for mode in ('normal', 'exit', 'term-exit'):
with tempfile.TemporaryDirectory(prefix='watchdog-traps-') as tmp:
directory = Path(tmp)
sleeper = directory / 'sleep'
sleeper.write_text('#!/bin/sh\necho $$ >> "$PIDS"\nexec "' + shutil.which('sleep') + '" "$@"\n')
sleeper.chmod(0o755)
script = directory / 'check.sh'
script.write_text(f'''set -eu
mode=$1
shift
config_dir={directory}
. {ROOT / 'managed-target-supervision.sh'}
{bounded_wait}
cleanup() {{
trap - EXIT
trap ':' INT TERM
kill "$owned" 2>/dev/null || true
bounded_wait "$owned" 5
[ "$join_timed_out" -eq 0 ]
}}
sleep 300 & owned=$!
case "$mode" in
normal) cleanup ;;
exit) trap cleanup EXIT; exit 0 ;;
term-exit) trap cleanup EXIT; trap 'exit 0' TERM; kill -TERM $$ ;;
esac
''')
env = dict(os.environ, PATH=f'{directory}:{os.environ["PATH"]}', PIDS=str(directory / 'pids'))
started = time.monotonic()
result = subprocess.run([*shell, str(script), mode], env=env, capture_output=True, timeout=10)
assert result.returncode == 0, (shell, mode, result.stderr)
assert time.monotonic() - started < 2, (shell, mode, 'watchdog cancellation delayed')
if (directory / 'pids').exists():
for pid in (directory / 'pids').read_text().splitlines():
assert not Path(f'/proc/{pid}').exists(), (shell, mode, 'orphan sleeper', pid)
check_watchdog_traps(['/bin/sh'])
busybox = shutil.which('busybox')
if busybox:
check_watchdog_traps([busybox, 'ash'])
for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')):
result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2)
assert result.returncode == 64
print('entrypoint supervision fixtures passed')
+1 -1
View File
@@ -31,7 +31,7 @@ services:
- /run/zapret-autotune-ss:rw,noexec,nosuid,nodev,size=1m,mode=0700,uid=0,gid=0
networks: [target]
restart: "no"
stop_grace_period: 120s
stop_grace_period: 240s
networks:
target:
driver: bridge
@@ -1,19 +0,0 @@
[Unit]
Description=Isolated zapret-autotune managed target
Requires=docker.service
After=docker.service
StartLimitIntervalSec=6h
StartLimitBurst=3
[Service]
Type=simple
WorkingDirectory=/opt/zapret2-autotune
ExecStart=/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml up --no-build --abort-on-container-exit --exit-code-from target target
ExecStop=/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml stop target
Restart=no
RestartForceExitStatus=75
RestartSec=30s
TimeoutStopSec=180s
[Install]
WantedBy=multi-user.target