fix: prevent automatic retries of failed startup apply

Latch explicit resource apply once per PID 1 lifetime. Fail closed before launch and permanently pause after authoritative cleanup on failed or incomplete apply; retain reconciliation on successful monitor and service restarts.

Cover deterministic 180-second retry windows, launch/ACK/child/session/service failures, cancellation, successful restarts and fresh-PID1 flags. Document RAM-only reset limits and repair-only startup; no remote health acceptance is claimed.

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-15 08:05:27 +03:00
1 parent 36aace02da
commit 643d0c6220
3 files changed
+111 -13

No files matched your search

+80 -9
View File
@@ -38,7 +38,7 @@ def increment(key):
return n
def linger(code=0):
def stop(*_):
if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh' and scenario not in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink', 'dns-prestart'):
if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh' and scenario not in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink', 'dns-prestart', 'generic-apply-dns-loss', 'generic-apply-dns-restart'):
os.kill(int((root / 'dns-pid').read_text()), 0)
sys.exit(code)
signal.signal(signal.SIGTERM, stop)
@@ -67,10 +67,21 @@ if name == 'awk':
sys.exit(0)
if name == 'date':
n = increment('clock')
print(100000 + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0)))
elapsed = int((root / 'elapsed').read_text()) if (root / 'elapsed').exists() else 0
print(100000 + elapsed + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0)))
sys.exit(0)
if name == 'sleep':
seconds = int(sys.argv[1])
if seconds == 60 and scenario.startswith('generic-apply-'):
# Three former 60s retry windows; the failed producer becomes healthy
# after its first call, so any unintended retry would reach native READY.
n = increment('blocked_windows')
(root / 'elapsed').write_text(str(n * 60))
log('blocked-window ' + str(n))
if n == 3:
os.kill(os.getppid(), signal.SIGTERM)
time.sleep(.01)
sys.exit(0)
if scenario.startswith('qualify-') and (root / 'qualified').exists():
log('safe-idle')
os.kill(os.getppid(), signal.SIGTERM)
@@ -78,7 +89,7 @@ if name == 'sleep':
sys.exit(0)
if scenario.startswith('generic-budget-') and seconds != 60:
os.execv(os.environ['REAL_SLEEP'], [os.environ['REAL_SLEEP'], *sys.argv[1:]])
if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack', 'generic-apply-blocked', 'generic-apply-abrupt', 'generic-apply-cleanup'):
if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
log('generic-pause')
(root / 'budget-pause').touch()
if scenario != 'generic-ss':
@@ -110,6 +121,8 @@ if name in ('ss-server', 'ss-local'):
os.kill(int((root / 'dns-pid').read_text()), 0)
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
log(name)
if name == 'ss-local':
(root / 'ss-pid').write_text(str(os.getpid()))
if scenario.startswith('generic-budget-'):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
while True:
@@ -153,6 +166,9 @@ if sys.argv[1] == 'qualification-completion-check':
sys.exit(0)
if sys.argv[1:] == ['target-config-check']:
sys.exit(0)
if sys.argv[1:] == ['apply-launch']:
log('apply-launch')
sys.exit(0)
if sys.argv[1] == 'dns-forwarder':
if sys.argv[2] in ('readiness-check', 'readiness-cleanup'):
# exec preserves PID1's actual inherited anchors; no simulated unlink.
@@ -284,6 +300,14 @@ elif command == 'monitor':
f.write(str(os.getpid()) + '\n')
n = increment('monitor_runs')
log('monitor ' + str(n))
if n == 1 and scenario == 'generic-apply-monitor-restart':
sys.exit(75)
if n == 1 and scenario == 'generic-apply-ss-restart':
os.kill(int((root / 'ss-pid').read_text()), signal.SIGKILL)
linger()
if n == 1 and scenario == 'generic-apply-dns-restart':
os.kill(int((root / 'dns-pid').read_text()), signal.SIGKILL)
linger()
assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session'
if scenario.startswith('generic-budget-'):
time.sleep(.03)
@@ -331,12 +355,23 @@ elif command == 'resource-apply':
assert os.getsid(0) == os.getpid(), 'apply lacks owned session'
assert (root / 'private' / 'monitor-ready').read_text() == str(os.getpid()) + '\n'
events = (root / 'events').read_text().splitlines()
assert events[-1] == 'reconcile' and 'start' not in events
assert events[-2:] == ['reconcile', 'apply-launch'] and 'start' not in events
log('resource-apply')
if scenario == 'generic-apply-blocked':
n = increment('apply_runs')
if scenario == 'generic-apply-blocked' and n == 1:
sys.exit(78)
if scenario in ('generic-apply-abrupt', 'generic-apply-cleanup'):
if scenario == 'generic-apply-abrupt' and n == 1:
sys.exit(23)
if scenario == 'generic-apply-cleanup' and n == 1:
log('resource-returned-zero')
sys.exit(0)
if scenario == 'generic-apply-signal' and n == 1:
os.kill(os.getpid(), signal.SIGKILL)
if scenario in ('generic-apply-dns-loss', 'generic-apply-ss-loss') and n == 1:
os.kill(int((root / ('dns-pid' if scenario.endswith('dns-loss') else 'ss-pid')).read_text()), signal.SIGKILL)
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
while True:
time.sleep(.01)
if scenario == 'generic-apply-cancel':
def restored(*_):
log('resource-restored')
@@ -366,6 +401,8 @@ def run(scenario, expected, dns='', mode='native'):
(binary / name).unlink()
configs = root / 'private'
configs.mkdir()
if scenario == 'generic-apply-disabled':
(root / 'prepared').touch()
if scenario.startswith('dns-stale'):
if scenario == 'dns-stale-symlink':
(configs / 'dns-ready').symlink_to(configs / 'missing')
@@ -386,6 +423,16 @@ def run(scenario, expected, dns='', mode='native'):
text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}')
text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune'))
text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2'))
if scenario.startswith('generic-apply-'):
launch = ' launch_controller managed resource-apply'
assert launch in text
text = text.replace(launch, f' {binary / "autotune"} apply-launch\n' + launch)
if scenario == 'generic-apply-launch-fail':
text = text.replace(' launch_controller managed resource-apply', ' false || return 78\n launch_controller managed resource-apply')
if scenario == 'generic-apply-ack-fail':
broken_helper = root / 'ack-helper.sh'
broken_helper.write_text(helper.read_text().replace(' ack_publish ||', ' false && ack_publish ||'))
text = text.replace(str(helper), str(broken_helper))
if scenario == 'generic-apply-cleanup':
text = text.replace('cleanup_session || { echo', "cleanup_session() { return 78; }\n cleanup_session || { echo", 1)
if scenario.endswith('-race'):
@@ -411,7 +458,7 @@ def run(scenario, expected, dns='', mode='native'):
env.pop('ZAPRET_AUTOTUNE_QUALIFY_CONFIG', None)
env.pop('ZAPRET_AUTOTUNE_STRATEGY_CONFIG', None)
env.pop('ZAPRET_AUTOTUNE_RESOURCE_APPLY', None)
if scenario.startswith('generic-apply-'):
if scenario.startswith('generic-apply-') and scenario != 'generic-apply-disabled':
env['ZAPRET_AUTOTUNE_RESOURCE_APPLY'] = '1'
if scenario.startswith('qualify-'):
env['ZAPRET_AUTOTUNE_QUALIFY_CONFIG'] = '/operator/resources.json'
@@ -444,7 +491,7 @@ def run(scenario, expected, dns='', mode='native'):
assert b'Illegal number' not in result.stderr, (scenario, result.stderr)
assert b'ab' * 32 not in result.stdout + result.stderr
if scenario in ('generic-apply-blocked', 'generic-apply-abrupt', 'generic-apply-cleanup'):
code = 78 if scenario == 'generic-apply-blocked' else 23
code = {'generic-apply-blocked': 78, 'generic-apply-abrupt': 23, 'generic-apply-cleanup': 0}[scenario]
assert f'controller-startup: apply_wait_exit={code}\n'.encode() in result.stderr, result.stderr
assert (b'controller-startup: apply_cleanup=failed\n' in result.stderr) == (scenario == 'generic-apply-cleanup'), result.stderr
for pid in ((root / 'children').read_text().splitlines() if (root / 'children').exists() else []):
@@ -476,6 +523,15 @@ def run(scenario, expected, dns='', mode='native'):
assert not (configs / 'dns-ready').exists(), (scenario, 'stale DNS readiness')
if mode == 'native':
assert 'dns-start' not in events
if scenario.startswith('generic-apply-') and 'blocked-window 1' in events:
assert events.count('apply-launch') == 1, (scenario, events)
assert int((root / 'elapsed').read_text()) == 180, (scenario, events)
assert events.count('dns-start') == events.count('ss-server') == events.count('ss-local') == 1, (scenario, events)
assert 'start' not in events and not any(e.startswith('monitor ') for e in events), (scenario, events)
assert 'cooldown 15' not in events and 'cooldown 30' not in events, (scenario, events)
assert events[events.index('blocked-window 1'):] == ['blocked-window 1', 'blocked-window 2', 'blocked-window 3'], (scenario, events)
if scenario != 'generic-apply-dns-loss':
assert events.index('dns-stop') < events.index('blocked-window 1'), (scenario, events)
return events
@@ -513,10 +569,25 @@ events = run('generic-apply-cancel', 0, mode='doh')
assert 'resource-restored' in events and 'start' not in events and 'monitor 1' not in events, events
assert events.index('resource-restored') < events.index('stop') < events.index('dns-stop'), events
events = run('generic-apply-blocked', 0, mode='doh')
assert 'start' not in events and 'monitor 1' not in events and 'generic-pause' in events, events
assert events.count('resource-apply') == 1 and 'blocked-window 3' in events, events
for scenario, code in (('generic-apply-abrupt', 0), ('generic-apply-cleanup', 78)):
events = run(scenario, code, mode='doh')
assert 'start' not in events and 'monitor 1' not in events and 'resource-applied' not in events, events
for scenario in ('signal', 'dns-loss', 'ss-loss', 'launch-fail', 'ack-fail'):
events = run('generic-apply-' + scenario, 78 if scenario == 'ack-fail' else 0, mode='doh')
assert 'blocked-window 3' in events and 'resource-applied' not in events, events
assert events.count('resource-apply') == (0 if scenario in ('launch-fail', 'ack-fail') else 1), events
for scenario in ('monitor-restart', 'ss-restart', 'dns-restart'):
events = run('generic-apply-' + scenario, 0, mode='doh')
assert events.count('resource-apply') == events.count('apply-launch') == 1, events
assert events.count('start') == events.count('stop') == 2 and 'monitor 2' in events, events
assert events.count('dns-start') == 2 and events.count('dns-stop') == (1 if scenario == 'dns-restart' else 2), events
# Independent fresh PID 1 invocations reset the RAM latch; flag=0 still repairs
# prepared bytes by reconcile, without launching apply.
events = run('generic-apply-pass', 0, mode='doh')
assert events.count('resource-apply') == 1, events
events = run('generic-apply-disabled', 0, mode='doh')
assert 'resource-apply' not in events and 'start' in events, events
for scenario in ('generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
events = run(scenario, 78, mode='doh')
assert events.count('start') == 1 and 'stop' not in events, (scenario, events)