fix: publish controller startup acknowledgements atomically
Publish a complete PID acknowledgement by private temporary file and same-directory rename. Preserve unknown acknowledgement artifacts and remove only retained owned stale state. Keep malformed acknowledgement failures closed. Cover deterministic publication barriers, shared apply/monitor/qualify launch, foreign artifacts, owned replacement, and private-directory checks. This does not establish hostile root or same-UID writer no-clobber guarantees; the private mode-700 directory is the trust boundary. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
ade3b89137
commit
36aace02da
3 files changed
+139
-27
No files matched your search
@@ -1,7 +1,10 @@
|
||||
"""Offline deterministic startup guards: python3 deploy/managed-target/check_startup_diagnostics.py."""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
entry = (ROOT / 'managed-target-entrypoint.sh').read_text()
|
||||
@@ -10,10 +13,10 @@ launch = entry[entry.index('launch_controller() {'):entry.index('run_cycle() {')
|
||||
fixture_launch = launch.replace(' 8<&- 9<&- &', ' 8<&- 9<&-').replace(
|
||||
'monitor_ack_deadline=$(($(date +%s) + 5))', 'monitor_ack_deadline=100')
|
||||
secret = 'SECRET_SENTINEL'
|
||||
roles = ('managed resource-apply --resource-config', 'managed monitor', 'qualify --isolated')
|
||||
|
||||
# Run the actual shared launch for both callers. Only platform operations are
|
||||
# stubbed, so no daemon or monitor is started, and no deadline is extended.
|
||||
for role in ('managed resource-apply --resource-config', 'managed monitor'):
|
||||
# Injected empty/malformed ACKs must still fail closed, not retry the read.
|
||||
for role in roles:
|
||||
for scenario, diagnostic in (
|
||||
('empty', 'ack=read_failed'), ('malformed', 'ack=invalid_pid'),
|
||||
('wrong', 'ack=identity_mismatch'), ('session', 'launch=session_mismatch'),
|
||||
@@ -38,14 +41,11 @@ signal_tree() {{ :; }}
|
||||
proc_identity() {{ proc_sid=1; }}
|
||||
{capture}
|
||||
sleep 30 & seed=$!
|
||||
# The fake launch is synchronous, preserving $! as the seeded owned child.
|
||||
{fixture_launch}
|
||||
code=0
|
||||
launch_controller {role} '{secret}' || code=$?
|
||||
[ "$code" -eq 78 ]
|
||||
[ -z "$monitor_sid" ]
|
||||
# Model delayed publication after an empty ACK: failure already closed the gate.
|
||||
printf '%s\\n' "$seed" > "$config_dir/monitor-ready"
|
||||
command kill "$seed"
|
||||
wait "$seed" 2>/dev/null || true
|
||||
'''
|
||||
@@ -55,27 +55,59 @@ wait "$seed" 2>/dev/null || true
|
||||
assert secret.encode() not in result.stdout + result.stderr
|
||||
|
||||
helper = (ROOT / 'managed-target-supervision.sh').read_text()
|
||||
with tempfile.TemporaryDirectory(prefix='delayed-ack-') as tmp:
|
||||
wrapper_path = Path(tmp) / 'wrapper.sh'
|
||||
wrapper_path.write_text(helper.replace(' printf \'%s\\n\' "$$" > "$config_dir/monitor-ready"',
|
||||
' : > "$config_dir/monitor-ready"\n sleep 1\n printf \'%s\\n\' "$$" > "$config_dir/monitor-ready"'))
|
||||
real_launch = launch.replace('/opt/managed-target-supervision.sh', str(wrapper_path)).replace(
|
||||
'/usr/local/bin/zapret-autotune', '/bin/true')
|
||||
result = subprocess.run(['sh', '-c', f'''set -eu
|
||||
for role in roles:
|
||||
with tempfile.TemporaryDirectory(prefix='delayed-ack-') as tmp:
|
||||
root = Path(tmp)
|
||||
wrapper_path = root / 'wrapper.sh'
|
||||
write = ' printf \'%s\\n\' "$$" > "$ack_temp" || return 1'
|
||||
assert helper.count(write) == 1
|
||||
wrapper_path.write_text(helper.replace(write, ''' : > "$config_dir/barrier"
|
||||
while [ ! -e "$config_dir/release" ]; do sleep 0.01; done
|
||||
''' + write))
|
||||
real_launch = launch.replace('/opt/managed-target-supervision.sh', str(wrapper_path)).replace(
|
||||
'/usr/local/bin/zapret-autotune', '/bin/true')
|
||||
process = subprocess.Popen(['sh', '-c', f'''set -eu
|
||||
config_dir='{tmp}'
|
||||
monitor_sid=
|
||||
. '{ROOT / 'managed-target-supervision.sh'}'
|
||||
{capture}
|
||||
{real_launch}
|
||||
code=0
|
||||
launch_controller managed resource-apply '{secret}' || code=$?
|
||||
signal_tree "$monitor_pid" KILL
|
||||
wait "$monitor_pid" 2>/dev/null || true
|
||||
[ "$code" -eq 78 ]
|
||||
[ -z "$monitor_sid" ]
|
||||
'''], capture_output=True, timeout=3)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert result.stderr == b'controller-startup: ack=read_failed\n', result.stderr
|
||||
launch_controller {role} '{secret}'
|
||||
[ "$monitor_sid" = "$monitor_pid" ]
|
||||
wait "$monitor_pid"
|
||||
# An owned completed launch can be replaced for the next controller.
|
||||
monitor_sid=
|
||||
launch_controller {role} '{secret}'
|
||||
[ "$monitor_sid" = "$monitor_pid" ]
|
||||
wait "$monitor_pid"
|
||||
'''], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
try:
|
||||
deadline = time.monotonic() + 2
|
||||
while not (root / 'barrier').exists():
|
||||
assert process.poll() is None
|
||||
assert time.monotonic() < deadline
|
||||
time.sleep(0.01)
|
||||
assert not (root / 'monitor-ready').exists()
|
||||
pending, = root.glob('.monitor-ready.*')
|
||||
assert pending.read_bytes() == b''
|
||||
assert stat.S_IMODE(pending.stat().st_mode) == 0o600
|
||||
(root / 'release').touch()
|
||||
stdout, stderr = process.communicate(timeout=3)
|
||||
assert process.returncode == 0, stderr
|
||||
assert stderr == b'controller-startup: wrapper=exec_pending\n' * 2, stderr
|
||||
ready = root / 'monitor-ready'
|
||||
assert ready.read_text().strip().isdigit()
|
||||
assert ready.read_bytes().endswith(b'\n')
|
||||
assert ready.stat().st_nlink == 1
|
||||
assert ready.stat().st_uid == os.getuid()
|
||||
assert stat.S_IMODE(ready.stat().st_mode) == 0o600
|
||||
assert not list(root.glob('.monitor-ready.*'))
|
||||
assert secret.encode() not in stdout + stderr
|
||||
finally:
|
||||
(root / 'release').touch()
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.communicate(timeout=3)
|
||||
|
||||
wrapper = helper[helper.index('if [ "${1-}" = --monitor ]; then'):helper.index('elif [ "${1-}" = --watchdog ]; then')] + '\nfi\n'
|
||||
for identity, diagnostic, code in (
|
||||
@@ -85,10 +117,61 @@ for identity, diagnostic, code in (
|
||||
):
|
||||
with tempfile.TemporaryDirectory(prefix='wrapper-diagnostics-') as tmp:
|
||||
result = subprocess.run(['sh', '-c', f'proc_identity() {{ {identity}; }}\n{wrapper}',
|
||||
'fixture', '--monitor', tmp, '/bin/true', secret], capture_output=True, timeout=3)
|
||||
'fixture', '--monitor', tmp, '/bin/true', secret], capture_output=True, timeout=3)
|
||||
assert result.returncode == code, result.stderr
|
||||
assert result.stderr == f'controller-startup: wrapper={diagnostic}\n'.encode(), result.stderr
|
||||
ready = Path(tmp) / 'monitor-ready'
|
||||
assert ready.exists() == (code == 0)
|
||||
assert secret.encode() not in result.stdout + result.stderr
|
||||
print('startup diagnostic fixtures passed (shared apply/monitor launch and wrapper)')
|
||||
|
||||
# Both publication and parent cleanup preserve unknown artifacts, including links.
|
||||
for artifact in ('file', 'symlink', 'directory'):
|
||||
with tempfile.TemporaryDirectory(prefix='foreign-ack-') as tmp:
|
||||
root = Path(tmp)
|
||||
ready = root / 'monitor-ready'
|
||||
if artifact == 'file':
|
||||
ready.write_text(secret)
|
||||
elif artifact == 'symlink':
|
||||
ready.symlink_to(root / 'absent')
|
||||
else:
|
||||
ready.mkdir()
|
||||
before = ready.lstat()
|
||||
result = subprocess.run(['sh', '-c', f'proc_identity() {{ proc_sid=$$; }}\n{wrapper}',
|
||||
'fixture', '--monitor', tmp, '/bin/true', secret], capture_output=True, timeout=3)
|
||||
assert result.returncode == 78, result.stderr
|
||||
assert result.stderr == b'controller-startup: wrapper=ack_publish_failed\n', result.stderr
|
||||
result = subprocess.run(['sh', '-c', f'''set -eu
|
||||
config_dir='{tmp}'
|
||||
{launch}
|
||||
launch_controller managed monitor
|
||||
'''], capture_output=True, timeout=3)
|
||||
assert result.returncode == 78, result.stderr
|
||||
assert result.stderr == b'controller-startup: launch=ack_conflict\n', result.stderr
|
||||
assert ready.lstat() == before
|
||||
if artifact == 'file':
|
||||
assert ready.read_text() == secret
|
||||
assert not list(root.glob('.monitor-ready.*'))
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix='replaced-ack-') as tmp:
|
||||
result = subprocess.run(['sh', '-c', f'''set -eu
|
||||
config_dir='{tmp}'
|
||||
printf '123\n' > "$config_dir/monitor-ready"
|
||||
monitor_ready_identity=$(stat -c '%d:%i:%u:%a:%h' "$config_dir/monitor-ready")
|
||||
monitor_ready_pid=123
|
||||
printf '{secret}' > "$config_dir/replacement"
|
||||
mv "$config_dir/replacement" "$config_dir/monitor-ready"
|
||||
{launch}
|
||||
launch_controller managed monitor
|
||||
'''], capture_output=True, timeout=3)
|
||||
assert result.returncode == 78, result.stderr
|
||||
assert result.stderr == b'controller-startup: launch=ack_conflict\n'
|
||||
assert (Path(tmp) / 'monitor-ready').read_text() == secret
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix='unsafe-ack-dir-') as tmp:
|
||||
os.chmod(tmp, 0o755)
|
||||
result = subprocess.run(['sh', '-c', f'proc_identity() {{ proc_sid=$$; }}\n{wrapper}',
|
||||
'fixture', '--monitor', tmp, '/bin/true'], capture_output=True, timeout=3)
|
||||
assert result.returncode == 78, result.stderr
|
||||
assert result.stderr == b'controller-startup: wrapper=ack_publish_failed\n'
|
||||
assert not list(Path(tmp).iterdir())
|
||||
print('startup diagnostic fixtures passed (shared apply/monitor/qualify launch and atomic wrapper)')
|
||||
@@ -337,7 +337,20 @@ start_dns() {
|
||||
done
|
||||
}
|
||||
launch_controller() {
|
||||
rm -f "$config_dir/monitor-ready"
|
||||
# Only an ACK retained from our previous launch is owned stale state.
|
||||
# Unknown files (including dangling links) are evidence, not debris.
|
||||
if [ -e "$config_dir/monitor-ready" ] || [ -L "$config_dir/monitor-ready" ]; then
|
||||
if [ -z "${monitor_ready_identity-}" ] || [ -L "$config_dir/monitor-ready" ] ||
|
||||
[ ! -f "$config_dir/monitor-ready" ] ||
|
||||
[ "$(stat -c '%d:%i:%u:%a:%h' "$config_dir/monitor-ready")" != "$monitor_ready_identity" ] ||
|
||||
! { read -r stale_pid < "$config_dir/monitor-ready"; } 2>/dev/null ||
|
||||
[ "$stale_pid" != "${monitor_ready_pid-}" ]; then
|
||||
echo 'controller-startup: launch=ack_conflict' >&2
|
||||
return 78
|
||||
fi
|
||||
rm -f "$config_dir/monitor-ready" || return 78
|
||||
fi
|
||||
monitor_ready_identity=
|
||||
setsid /bin/sh /opt/managed-target-supervision.sh --monitor "$config_dir" /usr/local/bin/zapret-autotune "$@" 8<&- 9<&- &
|
||||
monitor_pid=$!
|
||||
monitor_ack_deadline=$(($(date +%s) + 5))
|
||||
@@ -353,6 +366,8 @@ launch_controller() {
|
||||
fi
|
||||
done
|
||||
capture_monitor_session || return 78
|
||||
monitor_ready_identity=$(stat -c '%d:%i:%u:%a:%h' "$config_dir/monitor-ready") || return 78
|
||||
monitor_ready_pid=$monitor_pid
|
||||
if proc_identity "$monitor_pid" && [ "$proc_sid" != "$monitor_sid" ]; then
|
||||
echo 'controller-startup: launch=session_mismatch' >&2
|
||||
monitor_sid=
|
||||
@@ -416,7 +431,6 @@ fi
|
||||
# fork a monitor we already need to cancel: an early TERM can race interpreter
|
||||
# startup before its cancellation handler exists. Clean this cycle directly.
|
||||
{ dns_alive && kill -0 "$server_pid" 2>/dev/null && kill -0 "$local_pid" 2>/dev/null; } || return 75
|
||||
rm -f "$config_dir/monitor-ready"
|
||||
if [ "$qualification_mode" -eq 1 ]; then
|
||||
qualification_attempted=1
|
||||
set -- qualify --config "$ZAPRET_AUTOTUNE_QUALIFY_CONFIG" --strategies "$ZAPRET_AUTOTUNE_STRATEGY_CONFIG" --isolated
|
||||
|
||||
@@ -68,7 +68,22 @@ if [ "${1-}" = --monitor ]; then
|
||||
# Acknowledge the private session before exec, so even an immediate monitor
|
||||
# crash leaves its ownership recorded. No worker can precede this handshake.
|
||||
trap 'exit 143' INT TERM
|
||||
printf '%s\n' "$$" > "$config_dir/monitor-ready"
|
||||
ack_temp=
|
||||
trap '[ -z "$ack_temp" ] || rm -f "$ack_temp"' EXIT
|
||||
ack_publish() {
|
||||
# Private same-UID directory: no untrusted writer can race the absent
|
||||
# destination check. Rename exposes complete bytes with a single link.
|
||||
[ -d "$config_dir" ] && [ ! -L "$config_dir" ] &&
|
||||
[ "$(stat -c '%u:%a' "$config_dir")" = "$(id -u):700" ] || return 1
|
||||
umask 077
|
||||
ack_temp=$(mktemp "$config_dir/.monitor-ready.XXXXXX") || return 1
|
||||
printf '%s\n' "$$" > "$ack_temp" || return 1
|
||||
[ ! -e "$config_dir/monitor-ready" ] && [ ! -L "$config_dir/monitor-ready" ] || return 1
|
||||
mv -T "$ack_temp" "$config_dir/monitor-ready" || return 1
|
||||
ack_temp=
|
||||
}
|
||||
ack_publish || { echo 'controller-startup: wrapper=ack_publish_failed' >&2; exit 78; }
|
||||
trap - EXIT
|
||||
echo 'controller-startup: wrapper=exec_pending' >&2
|
||||
exec "$@"
|
||||
elif [ "${1-}" = --watchdog ]; then
|
||||
|
||||
Reference in new issue
Block a user