diff --git a/RESOURCE_PLAN.md b/RESOURCE_PLAN.md index 7bfd4eb..6e36ed7 100644 --- a/RESOURCE_PLAN.md +++ b/RESOURCE_PLAN.md @@ -3,6 +3,7 @@ `plan` remains **proposal-only** and never changes runtime or config. Explicit generic startup apply is implemented and independently reviewed locally; remote acceptance is pending. This is not automatic recovery or production READY. +The per-PID-1 one-shot latch correction passed independent local review only. Explicit isolated qualification temporarily installs an owned NFQ worker and queue rules, but never changes legacy SQL groups, renderer or apply journals. Monitor HEALTHY, old SQL measurements and public check JSON are not proof. @@ -174,9 +175,22 @@ manager, new proof importer, SQL frontend surrogate or parallel journal exists. PID 1 starts owned SS/DoH, reconciles known bytes, launches the acknowledged `managed resource-apply --resource-config FILE` child before the checks-only monitor, and starts the native daemon only with verified committed config. -This is one finite apply invocation per startup cycle, not exactly once per -container: PID 1 can retry cycles under its existing restart budget. Identical -committed requests resume by authenticated refs without reapplying the transaction. +This is at most one apply launch per PID 1 lifetime. A RAM latch is marked failed +before launch and succeeds only after child wait=0 and owned-session cleanup. +Launch/ACK failure, nonzero/signal exit, SS/DoH loss, or incomplete cleanup leaves +it failed: after authoritative cleanup PID 1 remains cancellably BLOCKED, without +restarting SS/DoH, native zapret, apply or monitoring, even across former 60s retry +windows. Cleanup still stops SS/DoH (DNS last); BLOCKED does not keep them alive. +Successful apply is skipped on later monitor/SS restarts, but every runtime cycle +still reconciles committed refs before native start. Qualification and legacy +retry behavior are unchanged. + +**Ceiling:** this is not a durable manual-intent latch. A fresh PID 1 (including +an external Docker restart policy) resets authorization and, with flag=1, permits +one new attempt. Operators must disable unattended external restart/recreation +for this workflow. Set `ZAPRET_AUTOTUNE_RESOURCE_APPLY=0` for explicit repair-only +startup: prepared state is reconciled, never promoted or reapplied. Identical +committed requests still resume by authenticated refs without a new transaction. The command refuses standalone/docker-exec callers: actual PID 1 environment, entrypoint, namespace, direct-child executable/argv and held launch ACK are required, plus the existing controller singleton and no pre-existing NFQ worker. @@ -228,6 +242,10 @@ managed/resource status validates generic refs without inventing legacy groups. Local root fixture tests cover the production snapshot factory and shared engine; independent local review passed. Actual native/network/PID 1 apply acceptance is **not run**. The next gate is isolated bounded remote acceptance, not live rollout. +The per-PID-1 latch correction passed independent local review and deterministic +180-second simulated retry-window fixtures; no remote PASS is claimed for it. +Roadmap: authorized isolated acceptance; no remote deployment is authorized by +local fixtures. ## Bounded remote acceptance — source `5cdc0cd` diff --git a/deploy/managed-target/check_supervision.py b/deploy/managed-target/check_supervision.py index 3e7254f..a5ed100 100644 --- a/deploy/managed-target/check_supervision.py +++ b/deploy/managed-target/check_supervision.py @@ -38,7 +38,7 @@ def increment(key): return n def linger(code=0): def stop(*_): - if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh' and scenario not in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink', 'dns-prestart'): + if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh' and scenario not in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink', 'dns-prestart', 'generic-apply-dns-loss', 'generic-apply-dns-restart'): os.kill(int((root / 'dns-pid').read_text()), 0) sys.exit(code) signal.signal(signal.SIGTERM, stop) @@ -67,10 +67,21 @@ if name == 'awk': sys.exit(0) if name == 'date': n = increment('clock') - print(100000 + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0))) + elapsed = int((root / 'elapsed').read_text()) if (root / 'elapsed').exists() else 0 + print(100000 + elapsed + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0))) sys.exit(0) if name == 'sleep': seconds = int(sys.argv[1]) + if seconds == 60 and scenario.startswith('generic-apply-'): + # Three former 60s retry windows; the failed producer becomes healthy + # after its first call, so any unintended retry would reach native READY. + n = increment('blocked_windows') + (root / 'elapsed').write_text(str(n * 60)) + log('blocked-window ' + str(n)) + if n == 3: + os.kill(os.getppid(), signal.SIGTERM) + time.sleep(.01) + sys.exit(0) if scenario.startswith('qualify-') and (root / 'qualified').exists(): log('safe-idle') os.kill(os.getppid(), signal.SIGTERM) @@ -78,7 +89,7 @@ if name == 'sleep': sys.exit(0) if scenario.startswith('generic-budget-') and seconds != 60: os.execv(os.environ['REAL_SLEEP'], [os.environ['REAL_SLEEP'], *sys.argv[1:]]) - if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack', 'generic-apply-blocked', 'generic-apply-abrupt', 'generic-apply-cleanup'): + if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'): log('generic-pause') (root / 'budget-pause').touch() if scenario != 'generic-ss': @@ -110,6 +121,8 @@ if name in ('ss-server', 'ss-local'): os.kill(int((root / 'dns-pid').read_text()), 0) assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600 log(name) + if name == 'ss-local': + (root / 'ss-pid').write_text(str(os.getpid())) if scenario.startswith('generic-budget-'): signal.signal(signal.SIGTERM, signal.SIG_IGN) while True: @@ -153,6 +166,9 @@ if sys.argv[1] == 'qualification-completion-check': sys.exit(0) if sys.argv[1:] == ['target-config-check']: sys.exit(0) +if sys.argv[1:] == ['apply-launch']: + log('apply-launch') + sys.exit(0) if sys.argv[1] == 'dns-forwarder': if sys.argv[2] in ('readiness-check', 'readiness-cleanup'): # exec preserves PID1's actual inherited anchors; no simulated unlink. @@ -284,6 +300,14 @@ elif command == 'monitor': f.write(str(os.getpid()) + '\n') n = increment('monitor_runs') log('monitor ' + str(n)) + if n == 1 and scenario == 'generic-apply-monitor-restart': + sys.exit(75) + if n == 1 and scenario == 'generic-apply-ss-restart': + os.kill(int((root / 'ss-pid').read_text()), signal.SIGKILL) + linger() + if n == 1 and scenario == 'generic-apply-dns-restart': + os.kill(int((root / 'dns-pid').read_text()), signal.SIGKILL) + linger() assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session' if scenario.startswith('generic-budget-'): time.sleep(.03) @@ -331,12 +355,23 @@ elif command == 'resource-apply': assert os.getsid(0) == os.getpid(), 'apply lacks owned session' assert (root / 'private' / 'monitor-ready').read_text() == str(os.getpid()) + '\n' events = (root / 'events').read_text().splitlines() - assert events[-1] == 'reconcile' and 'start' not in events + assert events[-2:] == ['reconcile', 'apply-launch'] and 'start' not in events log('resource-apply') - if scenario == 'generic-apply-blocked': + n = increment('apply_runs') + if scenario == 'generic-apply-blocked' and n == 1: sys.exit(78) - if scenario in ('generic-apply-abrupt', 'generic-apply-cleanup'): + if scenario == 'generic-apply-abrupt' and n == 1: sys.exit(23) + if scenario == 'generic-apply-cleanup' and n == 1: + log('resource-returned-zero') + sys.exit(0) + if scenario == 'generic-apply-signal' and n == 1: + os.kill(os.getpid(), signal.SIGKILL) + if scenario in ('generic-apply-dns-loss', 'generic-apply-ss-loss') and n == 1: + os.kill(int((root / ('dns-pid' if scenario.endswith('dns-loss') else 'ss-pid')).read_text()), signal.SIGKILL) + signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) + while True: + time.sleep(.01) if scenario == 'generic-apply-cancel': def restored(*_): log('resource-restored') @@ -366,6 +401,8 @@ def run(scenario, expected, dns='', mode='native'): (binary / name).unlink() configs = root / 'private' configs.mkdir() + if scenario == 'generic-apply-disabled': + (root / 'prepared').touch() if scenario.startswith('dns-stale'): if scenario == 'dns-stale-symlink': (configs / 'dns-ready').symlink_to(configs / 'missing') @@ -386,6 +423,16 @@ def run(scenario, expected, dns='', mode='native'): text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}') text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune')) text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2')) + if scenario.startswith('generic-apply-'): + launch = ' launch_controller managed resource-apply' + assert launch in text + text = text.replace(launch, f' {binary / "autotune"} apply-launch\n' + launch) + if scenario == 'generic-apply-launch-fail': + text = text.replace(' launch_controller managed resource-apply', ' false || return 78\n launch_controller managed resource-apply') + if scenario == 'generic-apply-ack-fail': + broken_helper = root / 'ack-helper.sh' + broken_helper.write_text(helper.read_text().replace(' ack_publish ||', ' false && ack_publish ||')) + text = text.replace(str(helper), str(broken_helper)) if scenario == 'generic-apply-cleanup': text = text.replace('cleanup_session || { echo', "cleanup_session() { return 78; }\n cleanup_session || { echo", 1) if scenario.endswith('-race'): @@ -411,7 +458,7 @@ def run(scenario, expected, dns='', mode='native'): env.pop('ZAPRET_AUTOTUNE_QUALIFY_CONFIG', None) env.pop('ZAPRET_AUTOTUNE_STRATEGY_CONFIG', None) env.pop('ZAPRET_AUTOTUNE_RESOURCE_APPLY', None) - if scenario.startswith('generic-apply-'): + if scenario.startswith('generic-apply-') and scenario != 'generic-apply-disabled': env['ZAPRET_AUTOTUNE_RESOURCE_APPLY'] = '1' if scenario.startswith('qualify-'): env['ZAPRET_AUTOTUNE_QUALIFY_CONFIG'] = '/operator/resources.json' @@ -444,7 +491,7 @@ def run(scenario, expected, dns='', mode='native'): assert b'Illegal number' not in result.stderr, (scenario, result.stderr) assert b'ab' * 32 not in result.stdout + result.stderr if scenario in ('generic-apply-blocked', 'generic-apply-abrupt', 'generic-apply-cleanup'): - code = 78 if scenario == 'generic-apply-blocked' else 23 + code = {'generic-apply-blocked': 78, 'generic-apply-abrupt': 23, 'generic-apply-cleanup': 0}[scenario] assert f'controller-startup: apply_wait_exit={code}\n'.encode() in result.stderr, result.stderr assert (b'controller-startup: apply_cleanup=failed\n' in result.stderr) == (scenario == 'generic-apply-cleanup'), result.stderr for pid in ((root / 'children').read_text().splitlines() if (root / 'children').exists() else []): @@ -476,6 +523,15 @@ def run(scenario, expected, dns='', mode='native'): assert not (configs / 'dns-ready').exists(), (scenario, 'stale DNS readiness') if mode == 'native': assert 'dns-start' not in events + if scenario.startswith('generic-apply-') and 'blocked-window 1' in events: + assert events.count('apply-launch') == 1, (scenario, events) + assert int((root / 'elapsed').read_text()) == 180, (scenario, events) + assert events.count('dns-start') == events.count('ss-server') == events.count('ss-local') == 1, (scenario, events) + assert 'start' not in events and not any(e.startswith('monitor ') for e in events), (scenario, events) + assert 'cooldown 15' not in events and 'cooldown 30' not in events, (scenario, events) + assert events[events.index('blocked-window 1'):] == ['blocked-window 1', 'blocked-window 2', 'blocked-window 3'], (scenario, events) + if scenario != 'generic-apply-dns-loss': + assert events.index('dns-stop') < events.index('blocked-window 1'), (scenario, events) return events @@ -513,10 +569,25 @@ events = run('generic-apply-cancel', 0, mode='doh') assert 'resource-restored' in events and 'start' not in events and 'monitor 1' not in events, events assert events.index('resource-restored') < events.index('stop') < events.index('dns-stop'), events events = run('generic-apply-blocked', 0, mode='doh') -assert 'start' not in events and 'monitor 1' not in events and 'generic-pause' in events, events +assert events.count('resource-apply') == 1 and 'blocked-window 3' in events, events for scenario, code in (('generic-apply-abrupt', 0), ('generic-apply-cleanup', 78)): events = run(scenario, code, mode='doh') assert 'start' not in events and 'monitor 1' not in events and 'resource-applied' not in events, events +for scenario in ('signal', 'dns-loss', 'ss-loss', 'launch-fail', 'ack-fail'): + events = run('generic-apply-' + scenario, 78 if scenario == 'ack-fail' else 0, mode='doh') + assert 'blocked-window 3' in events and 'resource-applied' not in events, events + assert events.count('resource-apply') == (0 if scenario in ('launch-fail', 'ack-fail') else 1), events +for scenario in ('monitor-restart', 'ss-restart', 'dns-restart'): + events = run('generic-apply-' + scenario, 0, mode='doh') + assert events.count('resource-apply') == events.count('apply-launch') == 1, events + assert events.count('start') == events.count('stop') == 2 and 'monitor 2' in events, events + assert events.count('dns-start') == 2 and events.count('dns-stop') == (1 if scenario == 'dns-restart' else 2), events +# Independent fresh PID 1 invocations reset the RAM latch; flag=0 still repairs +# prepared bytes by reconcile, without launching apply. +events = run('generic-apply-pass', 0, mode='doh') +assert events.count('resource-apply') == 1, events +events = run('generic-apply-disabled', 0, mode='doh') +assert 'resource-apply' not in events and 'start' in events, events for scenario in ('generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'): events = run(scenario, 78, mode='doh') assert events.count('start') == 1 and 'stop' not in events, (scenario, events) diff --git a/managed-target-entrypoint.sh b/managed-target-entrypoint.sh index fd772ed..38e00f5 100644 --- a/managed-target-entrypoint.sh +++ b/managed-target-entrypoint.sh @@ -20,6 +20,8 @@ generic_mode=0 [ -z "${ZAPRET_AUTOTUNE_RESOURCE_CONFIG-}" ] || generic_mode=1 qualification_mode=0 qualification_attempted=0 +# RAM-only authorization: a new PID 1 may attempt again, never a runtime retry. +resource_apply_state=unattempted if [ -n "${ZAPRET_AUTOTUNE_QUALIFY_CONFIG-}${ZAPRET_AUTOTUNE_STRATEGY_CONFIG-}" ]; then [ -n "${ZAPRET_AUTOTUNE_QUALIFY_CONFIG-}" ] && [ -n "${ZAPRET_AUTOTUNE_STRATEGY_CONFIG-}" ] || { echo 'error: isolated qualification requires both config and strategy paths' >&2; exit 64; @@ -375,6 +377,7 @@ launch_controller() { fi } run_cycle() { +[ "$resource_apply_state" != failed ] || return 78 [ "$qualification_attempted" -eq 0 ] || return 78 start_dns || return $? set -- ss-server -c "$config_dir/server.json" @@ -406,7 +409,8 @@ fi timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile || return 78 if [ "$qualification_mode" -eq 0 ]; then zapret_started=1 -if [ "${ZAPRET_AUTOTUNE_RESOURCE_APPLY-0}" = 1 ]; then +if [ "${ZAPRET_AUTOTUNE_RESOURCE_APPLY-0}" = 1 ] && [ "$resource_apply_state" = unattempted ]; then + resource_apply_state=failed launch_controller managed resource-apply --resource-config "$ZAPRET_AUTOTUNE_RESOURCE_CONFIG" || return 78 while dns_alive && kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do pause 1; done { dns_alive && kill -0 "$server_pid" && kill -0 "$local_pid"; } || return 75 @@ -416,6 +420,7 @@ if [ "${ZAPRET_AUTOTUNE_RESOURCE_APPLY-0}" = 1 ]; then monitor_pid= cleanup_session || { echo 'controller-startup: apply_cleanup=failed' >&2; return 78; } [ "$apply_code" -eq 0 ] || return 78 + resource_apply_state=succeeded fi # Idempotent apply may have only verified the already committed config. timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || return 78 @@ -501,6 +506,10 @@ while :; do fi code=78 fi + if [ "$resource_apply_state" = failed ]; then + echo 'BLOCKED: resource apply failed/incomplete; operator repair/restart required' >&2 + while :; do pause 60; done + fi if [ "$generic_mode" -eq 1 ] && [ "$code" -eq 78 ]; then echo 'BLOCKED: generic target trust/startup failure; checks paused' >&2 pause 60