fix: prevent automatic retries of failed startup apply
Latch explicit resource apply once per PID 1 lifetime. Fail closed before launch and permanently pause after authoritative cleanup on failed or incomplete apply; retain reconciliation on successful monitor and service restarts. Cover deterministic 180-second retry windows, launch/ACK/child/session/service failures, cancellation, successful restarts and fresh-PID1 flags. Document RAM-only reset limits and repair-only startup; no remote health acceptance is claimed. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
36aace02da
commit
643d0c6220
3 files changed
+111
-13
No files matched your search
@@ -38,7 +38,7 @@ def increment(key):
|
||||
return n
|
||||
def linger(code=0):
|
||||
def stop(*_):
|
||||
if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh' and scenario not in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink', 'dns-prestart'):
|
||||
if os.environ.get('ZAPRET_AUTOTUNE_DNS_MODE', 'doh') == 'doh' and scenario not in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink', 'dns-prestart', 'generic-apply-dns-loss', 'generic-apply-dns-restart'):
|
||||
os.kill(int((root / 'dns-pid').read_text()), 0)
|
||||
sys.exit(code)
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
@@ -67,10 +67,21 @@ if name == 'awk':
|
||||
sys.exit(0)
|
||||
if name == 'date':
|
||||
n = increment('clock')
|
||||
print(100000 + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0)))
|
||||
elapsed = int((root / 'elapsed').read_text()) if (root / 'elapsed').exists() else 0
|
||||
print(100000 + elapsed + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0)))
|
||||
sys.exit(0)
|
||||
if name == 'sleep':
|
||||
seconds = int(sys.argv[1])
|
||||
if seconds == 60 and scenario.startswith('generic-apply-'):
|
||||
# Three former 60s retry windows; the failed producer becomes healthy
|
||||
# after its first call, so any unintended retry would reach native READY.
|
||||
n = increment('blocked_windows')
|
||||
(root / 'elapsed').write_text(str(n * 60))
|
||||
log('blocked-window ' + str(n))
|
||||
if n == 3:
|
||||
os.kill(os.getppid(), signal.SIGTERM)
|
||||
time.sleep(.01)
|
||||
sys.exit(0)
|
||||
if scenario.startswith('qualify-') and (root / 'qualified').exists():
|
||||
log('safe-idle')
|
||||
os.kill(os.getppid(), signal.SIGTERM)
|
||||
@@ -78,7 +89,7 @@ if name == 'sleep':
|
||||
sys.exit(0)
|
||||
if scenario.startswith('generic-budget-') and seconds != 60:
|
||||
os.execv(os.environ['REAL_SLEEP'], [os.environ['REAL_SLEEP'], *sys.argv[1:]])
|
||||
if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack', 'generic-apply-blocked', 'generic-apply-abrupt', 'generic-apply-cleanup'):
|
||||
if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
|
||||
log('generic-pause')
|
||||
(root / 'budget-pause').touch()
|
||||
if scenario != 'generic-ss':
|
||||
@@ -110,6 +121,8 @@ if name in ('ss-server', 'ss-local'):
|
||||
os.kill(int((root / 'dns-pid').read_text()), 0)
|
||||
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
|
||||
log(name)
|
||||
if name == 'ss-local':
|
||||
(root / 'ss-pid').write_text(str(os.getpid()))
|
||||
if scenario.startswith('generic-budget-'):
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
while True:
|
||||
@@ -153,6 +166,9 @@ if sys.argv[1] == 'qualification-completion-check':
|
||||
sys.exit(0)
|
||||
if sys.argv[1:] == ['target-config-check']:
|
||||
sys.exit(0)
|
||||
if sys.argv[1:] == ['apply-launch']:
|
||||
log('apply-launch')
|
||||
sys.exit(0)
|
||||
if sys.argv[1] == 'dns-forwarder':
|
||||
if sys.argv[2] in ('readiness-check', 'readiness-cleanup'):
|
||||
# exec preserves PID1's actual inherited anchors; no simulated unlink.
|
||||
@@ -284,6 +300,14 @@ elif command == 'monitor':
|
||||
f.write(str(os.getpid()) + '\n')
|
||||
n = increment('monitor_runs')
|
||||
log('monitor ' + str(n))
|
||||
if n == 1 and scenario == 'generic-apply-monitor-restart':
|
||||
sys.exit(75)
|
||||
if n == 1 and scenario == 'generic-apply-ss-restart':
|
||||
os.kill(int((root / 'ss-pid').read_text()), signal.SIGKILL)
|
||||
linger()
|
||||
if n == 1 and scenario == 'generic-apply-dns-restart':
|
||||
os.kill(int((root / 'dns-pid').read_text()), signal.SIGKILL)
|
||||
linger()
|
||||
assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session'
|
||||
if scenario.startswith('generic-budget-'):
|
||||
time.sleep(.03)
|
||||
@@ -331,12 +355,23 @@ elif command == 'resource-apply':
|
||||
assert os.getsid(0) == os.getpid(), 'apply lacks owned session'
|
||||
assert (root / 'private' / 'monitor-ready').read_text() == str(os.getpid()) + '\n'
|
||||
events = (root / 'events').read_text().splitlines()
|
||||
assert events[-1] == 'reconcile' and 'start' not in events
|
||||
assert events[-2:] == ['reconcile', 'apply-launch'] and 'start' not in events
|
||||
log('resource-apply')
|
||||
if scenario == 'generic-apply-blocked':
|
||||
n = increment('apply_runs')
|
||||
if scenario == 'generic-apply-blocked' and n == 1:
|
||||
sys.exit(78)
|
||||
if scenario in ('generic-apply-abrupt', 'generic-apply-cleanup'):
|
||||
if scenario == 'generic-apply-abrupt' and n == 1:
|
||||
sys.exit(23)
|
||||
if scenario == 'generic-apply-cleanup' and n == 1:
|
||||
log('resource-returned-zero')
|
||||
sys.exit(0)
|
||||
if scenario == 'generic-apply-signal' and n == 1:
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
if scenario in ('generic-apply-dns-loss', 'generic-apply-ss-loss') and n == 1:
|
||||
os.kill(int((root / ('dns-pid' if scenario.endswith('dns-loss') else 'ss-pid')).read_text()), signal.SIGKILL)
|
||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
|
||||
while True:
|
||||
time.sleep(.01)
|
||||
if scenario == 'generic-apply-cancel':
|
||||
def restored(*_):
|
||||
log('resource-restored')
|
||||
@@ -366,6 +401,8 @@ def run(scenario, expected, dns='', mode='native'):
|
||||
(binary / name).unlink()
|
||||
configs = root / 'private'
|
||||
configs.mkdir()
|
||||
if scenario == 'generic-apply-disabled':
|
||||
(root / 'prepared').touch()
|
||||
if scenario.startswith('dns-stale'):
|
||||
if scenario == 'dns-stale-symlink':
|
||||
(configs / 'dns-ready').symlink_to(configs / 'missing')
|
||||
@@ -386,6 +423,16 @@ def run(scenario, expected, dns='', mode='native'):
|
||||
text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}')
|
||||
text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune'))
|
||||
text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2'))
|
||||
if scenario.startswith('generic-apply-'):
|
||||
launch = ' launch_controller managed resource-apply'
|
||||
assert launch in text
|
||||
text = text.replace(launch, f' {binary / "autotune"} apply-launch\n' + launch)
|
||||
if scenario == 'generic-apply-launch-fail':
|
||||
text = text.replace(' launch_controller managed resource-apply', ' false || return 78\n launch_controller managed resource-apply')
|
||||
if scenario == 'generic-apply-ack-fail':
|
||||
broken_helper = root / 'ack-helper.sh'
|
||||
broken_helper.write_text(helper.read_text().replace(' ack_publish ||', ' false && ack_publish ||'))
|
||||
text = text.replace(str(helper), str(broken_helper))
|
||||
if scenario == 'generic-apply-cleanup':
|
||||
text = text.replace('cleanup_session || { echo', "cleanup_session() { return 78; }\n cleanup_session || { echo", 1)
|
||||
if scenario.endswith('-race'):
|
||||
@@ -411,7 +458,7 @@ def run(scenario, expected, dns='', mode='native'):
|
||||
env.pop('ZAPRET_AUTOTUNE_QUALIFY_CONFIG', None)
|
||||
env.pop('ZAPRET_AUTOTUNE_STRATEGY_CONFIG', None)
|
||||
env.pop('ZAPRET_AUTOTUNE_RESOURCE_APPLY', None)
|
||||
if scenario.startswith('generic-apply-'):
|
||||
if scenario.startswith('generic-apply-') and scenario != 'generic-apply-disabled':
|
||||
env['ZAPRET_AUTOTUNE_RESOURCE_APPLY'] = '1'
|
||||
if scenario.startswith('qualify-'):
|
||||
env['ZAPRET_AUTOTUNE_QUALIFY_CONFIG'] = '/operator/resources.json'
|
||||
@@ -444,7 +491,7 @@ def run(scenario, expected, dns='', mode='native'):
|
||||
assert b'Illegal number' not in result.stderr, (scenario, result.stderr)
|
||||
assert b'ab' * 32 not in result.stdout + result.stderr
|
||||
if scenario in ('generic-apply-blocked', 'generic-apply-abrupt', 'generic-apply-cleanup'):
|
||||
code = 78 if scenario == 'generic-apply-blocked' else 23
|
||||
code = {'generic-apply-blocked': 78, 'generic-apply-abrupt': 23, 'generic-apply-cleanup': 0}[scenario]
|
||||
assert f'controller-startup: apply_wait_exit={code}\n'.encode() in result.stderr, result.stderr
|
||||
assert (b'controller-startup: apply_cleanup=failed\n' in result.stderr) == (scenario == 'generic-apply-cleanup'), result.stderr
|
||||
for pid in ((root / 'children').read_text().splitlines() if (root / 'children').exists() else []):
|
||||
@@ -476,6 +523,15 @@ def run(scenario, expected, dns='', mode='native'):
|
||||
assert not (configs / 'dns-ready').exists(), (scenario, 'stale DNS readiness')
|
||||
if mode == 'native':
|
||||
assert 'dns-start' not in events
|
||||
if scenario.startswith('generic-apply-') and 'blocked-window 1' in events:
|
||||
assert events.count('apply-launch') == 1, (scenario, events)
|
||||
assert int((root / 'elapsed').read_text()) == 180, (scenario, events)
|
||||
assert events.count('dns-start') == events.count('ss-server') == events.count('ss-local') == 1, (scenario, events)
|
||||
assert 'start' not in events and not any(e.startswith('monitor ') for e in events), (scenario, events)
|
||||
assert 'cooldown 15' not in events and 'cooldown 30' not in events, (scenario, events)
|
||||
assert events[events.index('blocked-window 1'):] == ['blocked-window 1', 'blocked-window 2', 'blocked-window 3'], (scenario, events)
|
||||
if scenario != 'generic-apply-dns-loss':
|
||||
assert events.index('dns-stop') < events.index('blocked-window 1'), (scenario, events)
|
||||
return events
|
||||
|
||||
|
||||
@@ -513,10 +569,25 @@ events = run('generic-apply-cancel', 0, mode='doh')
|
||||
assert 'resource-restored' in events and 'start' not in events and 'monitor 1' not in events, events
|
||||
assert events.index('resource-restored') < events.index('stop') < events.index('dns-stop'), events
|
||||
events = run('generic-apply-blocked', 0, mode='doh')
|
||||
assert 'start' not in events and 'monitor 1' not in events and 'generic-pause' in events, events
|
||||
assert events.count('resource-apply') == 1 and 'blocked-window 3' in events, events
|
||||
for scenario, code in (('generic-apply-abrupt', 0), ('generic-apply-cleanup', 78)):
|
||||
events = run(scenario, code, mode='doh')
|
||||
assert 'start' not in events and 'monitor 1' not in events and 'resource-applied' not in events, events
|
||||
for scenario in ('signal', 'dns-loss', 'ss-loss', 'launch-fail', 'ack-fail'):
|
||||
events = run('generic-apply-' + scenario, 78 if scenario == 'ack-fail' else 0, mode='doh')
|
||||
assert 'blocked-window 3' in events and 'resource-applied' not in events, events
|
||||
assert events.count('resource-apply') == (0 if scenario in ('launch-fail', 'ack-fail') else 1), events
|
||||
for scenario in ('monitor-restart', 'ss-restart', 'dns-restart'):
|
||||
events = run('generic-apply-' + scenario, 0, mode='doh')
|
||||
assert events.count('resource-apply') == events.count('apply-launch') == 1, events
|
||||
assert events.count('start') == events.count('stop') == 2 and 'monitor 2' in events, events
|
||||
assert events.count('dns-start') == 2 and events.count('dns-stop') == (1 if scenario == 'dns-restart' else 2), events
|
||||
# Independent fresh PID 1 invocations reset the RAM latch; flag=0 still repairs
|
||||
# prepared bytes by reconcile, without launching apply.
|
||||
events = run('generic-apply-pass', 0, mode='doh')
|
||||
assert events.count('resource-apply') == 1, events
|
||||
events = run('generic-apply-disabled', 0, mode='doh')
|
||||
assert 'resource-apply' not in events and 'start' in events, events
|
||||
for scenario in ('generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
|
||||
events = run(scenario, 78, mode='doh')
|
||||
assert events.count('start') == 1 and 'stop' not in events, (scenario, events)
|
||||
|
||||
Reference in new issue
Block a user