feat: add guarded periodic resource monitoring
Add checks-only scheduling, read-only status, private authenticated persistence and guarded PID1 lifecycle. Keep strategy selection/recovery and remote fault acceptance out of scope. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
cfce8170d7
commit
4efc066625
16 files changed
+2307
-103
No files matched your search
@@ -59,21 +59,24 @@ assert " stop_grace_period: 240s\n" in compose
|
||||
assert 'bounded_wait "$monitor_pid" 45' in entrypoint
|
||||
assert '/bin/sh /opt/managed-target-supervision.sh --watchdog' in entrypoint
|
||||
assert 'setsid /bin/sh /opt/managed-target-supervision.sh --monitor' in entrypoint
|
||||
assert entrypoint.index('if ! cleanup_session; then') < entrypoint.index('if timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile; then')
|
||||
assert entrypoint.index('if ! cleanup_session; then') < entrypoint.index('if cleanup_command 45 /usr/local/bin/zapret-autotune managed reconcile; then')
|
||||
assert 'COPY managed-target-supervision.sh /opt/managed-target-supervision.sh' in (directory.parents[1] / 'Dockerfile.target').read_text()
|
||||
assert '!managed-target-supervision.sh' in (directory.parents[1] / '.dockerignore').read_text()
|
||||
assert '"$count" -lt 3' in entrypoint
|
||||
assert '"$count" -ge 3' in entrypoint
|
||||
assert 'BLOCKED: generic target restart budget; checks paused' in entrypoint
|
||||
assert '"$((now - started))" -lt 21600' in entrypoint
|
||||
assert "backoff=15" in entrypoint
|
||||
assert "systemctl" not in entrypoint and "docker " not in entrypoint
|
||||
assert "timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop" in entrypoint
|
||||
assert "cleanup_command 30 /opt/zapret2/init.d/sysv/zapret2 stop" in entrypoint
|
||||
assert 'generic_cleanup_deadline=$(($(date +%s) + 80))' in entrypoint
|
||||
assert 'timeout -k 1 "$cleanup_command_cap" "$@"' in entrypoint
|
||||
for command, seconds in (("/usr/local/bin/zapret-autotune managed init", 45),
|
||||
("/usr/local/bin/zapret-autotune managed reconcile", 45),
|
||||
("/opt/zapret2/init.d/sysv/zapret2 start", 30)):
|
||||
assert f"timeout -k 5 {seconds} {command} || exit 78" in entrypoint
|
||||
assert entrypoint.index("managed reconcile || exit 78") < entrypoint.index("zapret2 start || exit 78")
|
||||
assert f"timeout -k 5 {seconds} {command} || return 78" in entrypoint
|
||||
assert entrypoint.index("managed reconcile || return 78") < entrypoint.index("zapret2 start || return 78")
|
||||
experimental = 'managed apply --googlevideo-experiment native-circular --googlevideo-scope experimental_googlevideo_suffix --googlevideo-request startup || exit 78'
|
||||
assert entrypoint.index('zapret2 start || exit 78') < entrypoint.index(experimental)
|
||||
assert entrypoint.index('zapret2 start || return 78') < entrypoint.index(experimental)
|
||||
assert entrypoint.index(experimental) < entrypoint.index('setsid /bin/sh /opt/managed-target-supervision.sh --monitor')
|
||||
assert entrypoint.index('managed experimental-request-check || exit 78') < entrypoint.index('managed init || exit 78')
|
||||
assert entrypoint.index('managed experimental-request-check || return 78') < entrypoint.index('managed init || return 78')
|
||||
assert 'rm "${ZAPRET_CONFIG%/*}/.autotune-experimental-suffix-request"' not in entrypoint
|
||||
@@ -67,10 +67,19 @@ if name == 'awk':
|
||||
sys.exit(0)
|
||||
if name == 'date':
|
||||
n = increment('clock')
|
||||
print(100000 + (n * 21601 if scenario == 'window' else n))
|
||||
print(100000 + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0)))
|
||||
sys.exit(0)
|
||||
if name == 'sleep':
|
||||
seconds = int(sys.argv[1])
|
||||
if scenario.startswith('generic-budget-') and seconds != 60:
|
||||
os.execv(os.environ['REAL_SLEEP'], [os.environ['REAL_SLEEP'], *sys.argv[1:]])
|
||||
if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
|
||||
log('generic-pause')
|
||||
(root / 'budget-pause').touch()
|
||||
if scenario != 'generic-ss':
|
||||
os.kill(os.getppid(), signal.SIGTERM)
|
||||
time.sleep(.01)
|
||||
sys.exit(0)
|
||||
if seconds in (15, 30):
|
||||
log('cooldown ' + str(seconds))
|
||||
if scenario == 'cancel':
|
||||
@@ -96,9 +105,13 @@ if name in ('ss-server', 'ss-local'):
|
||||
os.kill(int((root / 'dns-pid').read_text()), 0)
|
||||
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
|
||||
log(name)
|
||||
if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail') and increment('ss_runs') == 1:
|
||||
if scenario.startswith('generic-budget-'):
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
while True:
|
||||
time.sleep(.01)
|
||||
if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail', 'generic-ss') and increment('ss_runs') <= (3 if scenario == 'generic-ss' else 1):
|
||||
# Let startup/monitor begin before simulating a child crash.
|
||||
while not (root / 'monitor_runs').exists():
|
||||
while not (root / 'monitor_runs').exists() or (scenario == 'generic-ss' and int((root / 'monitor_runs').read_text()) < int((root / 'ss_runs').read_text())):
|
||||
time.sleep(.01)
|
||||
time.sleep(.03)
|
||||
sys.exit(137)
|
||||
@@ -161,6 +174,8 @@ if sys.argv[1] == 'dns-forwarder':
|
||||
ready.unlink()
|
||||
sys.exit(0)
|
||||
signal.signal(signal.SIGTERM, stop_dns)
|
||||
if scenario.startswith('generic-budget-'):
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
if scenario in ('dns-term', 'dns-int'):
|
||||
os.kill(os.getppid(), signal.SIGTERM if scenario == 'dns-term' else signal.SIGINT)
|
||||
if scenario in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink') and increment('dns_runs') == 1:
|
||||
@@ -186,9 +201,15 @@ if command == 'init':
|
||||
elif command == 'reconcile':
|
||||
n = increment('reconciles')
|
||||
log('reconcile')
|
||||
if scenario.startswith('generic-budget-') and n == 2:
|
||||
with (root / 'children').open('a') as f:
|
||||
f.write(str(os.getpid()) + '\n')
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
while True:
|
||||
time.sleep(.01)
|
||||
if scenario == 'orphans':
|
||||
assert_workers_gone()
|
||||
if scenario == 'prestartfail':
|
||||
if scenario in ('prestartfail', 'generic-unsafe'):
|
||||
sys.exit(78)
|
||||
if scenario == 'dns-prestart':
|
||||
os.kill(int((root / 'dns-pid').read_text()), signal.SIGKILL)
|
||||
@@ -197,11 +218,23 @@ elif command == 'reconcile':
|
||||
sys.exit(78)
|
||||
(root / 'prepared').unlink(missing_ok=True)
|
||||
elif command == 'monitor':
|
||||
if scenario.startswith('generic-'):
|
||||
assert os.environ['ZAPRET_AUTOTUNE_RESOURCE_CONFIG'] == '/operator/resources.json'
|
||||
with (root / 'children').open('a') as f:
|
||||
f.write(str(os.getpid()) + '\n')
|
||||
n = increment('monitor_runs')
|
||||
log('monitor ' + str(n))
|
||||
assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session'
|
||||
if scenario.startswith('generic-budget-'):
|
||||
time.sleep(.03)
|
||||
os.kill(os.getppid(), signal.SIGTERM)
|
||||
if scenario == 'generic-budget-monitor':
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
while True:
|
||||
time.sleep(.01)
|
||||
linger()
|
||||
if scenario == 'generic-ss' and n <= 3:
|
||||
linger()
|
||||
if scenario in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink') and n == 1:
|
||||
linger()
|
||||
if scenario == 'dns-outage':
|
||||
@@ -245,6 +278,10 @@ def run(scenario, expected, dns='', mode='native'):
|
||||
fake.chmod(0o755)
|
||||
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2', 'blockcheck', 'nfq'):
|
||||
(binary / name).symlink_to(fake)
|
||||
realtime = scenario.startswith('generic-budget-')
|
||||
if realtime:
|
||||
for name in ('date', 'timeout'):
|
||||
(binary / name).unlink()
|
||||
configs = root / 'private'
|
||||
configs.mkdir()
|
||||
if scenario.startswith('dns-stale'):
|
||||
@@ -258,15 +295,28 @@ def run(scenario, expected, dns='', mode='native'):
|
||||
broken_helper = root / 'broken-helper.sh'
|
||||
broken_helper.write_text(helper.read_text().replace('sleep "$seconds" & sleeper=$!', 'exit 78'))
|
||||
helper = broken_helper
|
||||
if scenario == 'generic-budget-ack':
|
||||
delayed_helper = root / 'delayed-helper.sh'
|
||||
delayed_helper.write_text(helper.read_text().replace(': > "$config_dir/join-ready"',
|
||||
f'{shutil.which("sleep")} 300\n : > "$config_dir/join-ready"'))
|
||||
helper = delayed_helper
|
||||
text = text.replace('/opt/managed-target-supervision.sh', str(helper))
|
||||
text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}')
|
||||
text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune'))
|
||||
text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2'))
|
||||
if realtime:
|
||||
# Private fixture scaling only: production has no configurable grace.
|
||||
text = text.replace('$(date +%s) + 80', '$(date +%s) + 12')
|
||||
entry = root / 'entry.sh'
|
||||
entry.write_text(text)
|
||||
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
|
||||
SCENARIO=scenario, SS_PASSWORD='ab' * 32, SS_DNS_SERVER=dns,
|
||||
ZAPRET_CONFIG=str(root / 'config'), DNS_CLI=str(DNS_CLI))
|
||||
ZAPRET_CONFIG=str(root / 'config'), DNS_CLI=str(DNS_CLI),
|
||||
REAL_SLEEP=shutil.which('sleep'))
|
||||
if scenario.startswith('generic-'):
|
||||
env['ZAPRET_AUTOTUNE_RESOURCE_CONFIG'] = '/operator/resources.json'
|
||||
else:
|
||||
env.pop('ZAPRET_AUTOTUNE_RESOURCE_CONFIG', None)
|
||||
if mode is None:
|
||||
env.pop('ZAPRET_AUTOTUNE_DNS_MODE', None)
|
||||
else:
|
||||
@@ -279,7 +329,10 @@ def run(scenario, expected, dns='', mode='native'):
|
||||
peer = subprocess.Popen([shutil.which('sleep'), '300'], start_new_session=True)
|
||||
(root / 'peer').write_text(str(peer.pid))
|
||||
try:
|
||||
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15)
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=18 if realtime else 15)
|
||||
if realtime:
|
||||
assert time.monotonic() - started < 16, (scenario, 'aggregate cleanup exceeded grace')
|
||||
if peer:
|
||||
assert peer.poll() is None, 'unowned helper was killed'
|
||||
finally:
|
||||
@@ -311,6 +364,9 @@ def run(scenario, expected, dns='', mode='native'):
|
||||
assert ready.exists() or ready.is_symlink(), (scenario, 'foreign readiness deleted')
|
||||
if scenario != 'dns-stale-symlink' and scenario != 'dns-symlink':
|
||||
assert ready.read_text() == 'foreign evidence\n', (scenario, 'foreign bytes changed')
|
||||
elif realtime and (configs / 'dns-ready').exists():
|
||||
# Exhaustion cannot authorize unlinking without the owned Go helper.
|
||||
assert (configs / 'dns-ready').read_text().strip() == (root / 'dns-pid').read_text()
|
||||
else:
|
||||
assert not (configs / 'dns-ready').exists(), (scenario, 'stale DNS readiness')
|
||||
if mode == 'native':
|
||||
@@ -339,6 +395,15 @@ events = run('cancel', 0)
|
||||
assert events.count('start') == 1
|
||||
events = run('prestartfail', 78)
|
||||
assert 'start' not in events and 'stop' not in events
|
||||
events = run('generic-ss', 0, mode='doh')
|
||||
assert events.count('start') == 4 and events.count('stop') == 4, events
|
||||
assert events.count('generic-pause') == 1, events
|
||||
assert events.count('dns-start') == events.count('dns-stop') == 4, events
|
||||
events = run('generic-unsafe', 0)
|
||||
assert 'start' not in events and events.count('generic-pause') == 1, events
|
||||
for scenario in ('generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'):
|
||||
events = run(scenario, 78, mode='doh')
|
||||
assert events.count('start') == 1 and 'stop' not in events, (scenario, events)
|
||||
for dns in ('8.8.8.8', '2001:4860:4860::8888'):
|
||||
events = run('fatal64', 64, dns)
|
||||
assert events.count('ss-server') == 1 and events.count('ss-local') == 1
|
||||
@@ -384,6 +449,7 @@ def check_watchdog_traps(shell):
|
||||
mode=$1
|
||||
shift
|
||||
config_dir={directory}
|
||||
generic_mode=0
|
||||
. {ROOT / 'managed-target-supervision.sh'}
|
||||
{bounded_wait}
|
||||
cleanup() {{
|
||||
|
||||
@@ -18,6 +18,7 @@ services:
|
||||
ZAPRET_AUTOTUNE_DOH_BOOTSTRAP_IPS: ${ZAPRET_AUTOTUNE_DOH_BOOTSTRAP_IPS-}
|
||||
ZAPRET_AUTOTUNE_DOH_TIMEOUT_SEC: ${ZAPRET_AUTOTUNE_DOH_TIMEOUT_SEC-5}
|
||||
ZAPRET_AUTOTUNE_YOUTUBE_SERVICE: ${ZAPRET_AUTOTUNE_YOUTUBE_SERVICE-0}
|
||||
ZAPRET_AUTOTUNE_RESOURCE_CONFIG: ${ZAPRET_AUTOTUNE_RESOURCE_CONFIG-}
|
||||
ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE: ${ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE-cold}
|
||||
ZAPRET_AUTOTUNE_VIDEO_ID: ${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}
|
||||
ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS: '${ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS:-}'
|
||||
|
||||
Reference in new issue
Block a user