fix: publish controller startup acknowledgements atomically
Publish a complete PID acknowledgement by private temporary file and same-directory rename. Preserve unknown acknowledgement artifacts and remove only retained owned stale state. Keep malformed acknowledgement failures closed. Cover deterministic publication barriers, shared apply/monitor/qualify launch, foreign artifacts, owned replacement, and private-directory checks. This does not establish hostile root or same-UID writer no-clobber guarantees; the private mode-700 directory is the trust boundary. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
ade3b89137
commit
36aace02da
3 files changed
+139
-27
No files matched your search
@@ -1,7 +1,10 @@
|
|||||||
"""Offline deterministic startup guards: python3 deploy/managed-target/check_startup_diagnostics.py."""
|
"""Offline deterministic startup guards: python3 deploy/managed-target/check_startup_diagnostics.py."""
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
entry = (ROOT / 'managed-target-entrypoint.sh').read_text()
|
entry = (ROOT / 'managed-target-entrypoint.sh').read_text()
|
||||||
@@ -10,10 +13,10 @@ launch = entry[entry.index('launch_controller() {'):entry.index('run_cycle() {')
|
|||||||
fixture_launch = launch.replace(' 8<&- 9<&- &', ' 8<&- 9<&-').replace(
|
fixture_launch = launch.replace(' 8<&- 9<&- &', ' 8<&- 9<&-').replace(
|
||||||
'monitor_ack_deadline=$(($(date +%s) + 5))', 'monitor_ack_deadline=100')
|
'monitor_ack_deadline=$(($(date +%s) + 5))', 'monitor_ack_deadline=100')
|
||||||
secret = 'SECRET_SENTINEL'
|
secret = 'SECRET_SENTINEL'
|
||||||
|
roles = ('managed resource-apply --resource-config', 'managed monitor', 'qualify --isolated')
|
||||||
|
|
||||||
# Run the actual shared launch for both callers. Only platform operations are
|
# Injected empty/malformed ACKs must still fail closed, not retry the read.
|
||||||
# stubbed, so no daemon or monitor is started, and no deadline is extended.
|
for role in roles:
|
||||||
for role in ('managed resource-apply --resource-config', 'managed monitor'):
|
|
||||||
for scenario, diagnostic in (
|
for scenario, diagnostic in (
|
||||||
('empty', 'ack=read_failed'), ('malformed', 'ack=invalid_pid'),
|
('empty', 'ack=read_failed'), ('malformed', 'ack=invalid_pid'),
|
||||||
('wrong', 'ack=identity_mismatch'), ('session', 'launch=session_mismatch'),
|
('wrong', 'ack=identity_mismatch'), ('session', 'launch=session_mismatch'),
|
||||||
@@ -38,14 +41,11 @@ signal_tree() {{ :; }}
|
|||||||
proc_identity() {{ proc_sid=1; }}
|
proc_identity() {{ proc_sid=1; }}
|
||||||
{capture}
|
{capture}
|
||||||
sleep 30 & seed=$!
|
sleep 30 & seed=$!
|
||||||
# The fake launch is synchronous, preserving $! as the seeded owned child.
|
|
||||||
{fixture_launch}
|
{fixture_launch}
|
||||||
code=0
|
code=0
|
||||||
launch_controller {role} '{secret}' || code=$?
|
launch_controller {role} '{secret}' || code=$?
|
||||||
[ "$code" -eq 78 ]
|
[ "$code" -eq 78 ]
|
||||||
[ -z "$monitor_sid" ]
|
[ -z "$monitor_sid" ]
|
||||||
# Model delayed publication after an empty ACK: failure already closed the gate.
|
|
||||||
printf '%s\\n' "$seed" > "$config_dir/monitor-ready"
|
|
||||||
command kill "$seed"
|
command kill "$seed"
|
||||||
wait "$seed" 2>/dev/null || true
|
wait "$seed" 2>/dev/null || true
|
||||||
'''
|
'''
|
||||||
@@ -55,27 +55,59 @@ wait "$seed" 2>/dev/null || true
|
|||||||
assert secret.encode() not in result.stdout + result.stderr
|
assert secret.encode() not in result.stdout + result.stderr
|
||||||
|
|
||||||
helper = (ROOT / 'managed-target-supervision.sh').read_text()
|
helper = (ROOT / 'managed-target-supervision.sh').read_text()
|
||||||
with tempfile.TemporaryDirectory(prefix='delayed-ack-') as tmp:
|
for role in roles:
|
||||||
wrapper_path = Path(tmp) / 'wrapper.sh'
|
with tempfile.TemporaryDirectory(prefix='delayed-ack-') as tmp:
|
||||||
wrapper_path.write_text(helper.replace(' printf \'%s\\n\' "$$" > "$config_dir/monitor-ready"',
|
root = Path(tmp)
|
||||||
' : > "$config_dir/monitor-ready"\n sleep 1\n printf \'%s\\n\' "$$" > "$config_dir/monitor-ready"'))
|
wrapper_path = root / 'wrapper.sh'
|
||||||
real_launch = launch.replace('/opt/managed-target-supervision.sh', str(wrapper_path)).replace(
|
write = ' printf \'%s\\n\' "$$" > "$ack_temp" || return 1'
|
||||||
'/usr/local/bin/zapret-autotune', '/bin/true')
|
assert helper.count(write) == 1
|
||||||
result = subprocess.run(['sh', '-c', f'''set -eu
|
wrapper_path.write_text(helper.replace(write, ''' : > "$config_dir/barrier"
|
||||||
|
while [ ! -e "$config_dir/release" ]; do sleep 0.01; done
|
||||||
|
''' + write))
|
||||||
|
real_launch = launch.replace('/opt/managed-target-supervision.sh', str(wrapper_path)).replace(
|
||||||
|
'/usr/local/bin/zapret-autotune', '/bin/true')
|
||||||
|
process = subprocess.Popen(['sh', '-c', f'''set -eu
|
||||||
config_dir='{tmp}'
|
config_dir='{tmp}'
|
||||||
monitor_sid=
|
monitor_sid=
|
||||||
. '{ROOT / 'managed-target-supervision.sh'}'
|
. '{ROOT / 'managed-target-supervision.sh'}'
|
||||||
{capture}
|
{capture}
|
||||||
{real_launch}
|
{real_launch}
|
||||||
code=0
|
launch_controller {role} '{secret}'
|
||||||
launch_controller managed resource-apply '{secret}' || code=$?
|
[ "$monitor_sid" = "$monitor_pid" ]
|
||||||
signal_tree "$monitor_pid" KILL
|
wait "$monitor_pid"
|
||||||
wait "$monitor_pid" 2>/dev/null || true
|
# An owned completed launch can be replaced for the next controller.
|
||||||
[ "$code" -eq 78 ]
|
monitor_sid=
|
||||||
[ -z "$monitor_sid" ]
|
launch_controller {role} '{secret}'
|
||||||
'''], capture_output=True, timeout=3)
|
[ "$monitor_sid" = "$monitor_pid" ]
|
||||||
assert result.returncode == 0, result.stderr
|
wait "$monitor_pid"
|
||||||
assert result.stderr == b'controller-startup: ack=read_failed\n', result.stderr
|
'''], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
try:
|
||||||
|
deadline = time.monotonic() + 2
|
||||||
|
while not (root / 'barrier').exists():
|
||||||
|
assert process.poll() is None
|
||||||
|
assert time.monotonic() < deadline
|
||||||
|
time.sleep(0.01)
|
||||||
|
assert not (root / 'monitor-ready').exists()
|
||||||
|
pending, = root.glob('.monitor-ready.*')
|
||||||
|
assert pending.read_bytes() == b''
|
||||||
|
assert stat.S_IMODE(pending.stat().st_mode) == 0o600
|
||||||
|
(root / 'release').touch()
|
||||||
|
stdout, stderr = process.communicate(timeout=3)
|
||||||
|
assert process.returncode == 0, stderr
|
||||||
|
assert stderr == b'controller-startup: wrapper=exec_pending\n' * 2, stderr
|
||||||
|
ready = root / 'monitor-ready'
|
||||||
|
assert ready.read_text().strip().isdigit()
|
||||||
|
assert ready.read_bytes().endswith(b'\n')
|
||||||
|
assert ready.stat().st_nlink == 1
|
||||||
|
assert ready.stat().st_uid == os.getuid()
|
||||||
|
assert stat.S_IMODE(ready.stat().st_mode) == 0o600
|
||||||
|
assert not list(root.glob('.monitor-ready.*'))
|
||||||
|
assert secret.encode() not in stdout + stderr
|
||||||
|
finally:
|
||||||
|
(root / 'release').touch()
|
||||||
|
if process.poll() is None:
|
||||||
|
process.kill()
|
||||||
|
process.communicate(timeout=3)
|
||||||
|
|
||||||
wrapper = helper[helper.index('if [ "${1-}" = --monitor ]; then'):helper.index('elif [ "${1-}" = --watchdog ]; then')] + '\nfi\n'
|
wrapper = helper[helper.index('if [ "${1-}" = --monitor ]; then'):helper.index('elif [ "${1-}" = --watchdog ]; then')] + '\nfi\n'
|
||||||
for identity, diagnostic, code in (
|
for identity, diagnostic, code in (
|
||||||
@@ -85,10 +117,61 @@ for identity, diagnostic, code in (
|
|||||||
):
|
):
|
||||||
with tempfile.TemporaryDirectory(prefix='wrapper-diagnostics-') as tmp:
|
with tempfile.TemporaryDirectory(prefix='wrapper-diagnostics-') as tmp:
|
||||||
result = subprocess.run(['sh', '-c', f'proc_identity() {{ {identity}; }}\n{wrapper}',
|
result = subprocess.run(['sh', '-c', f'proc_identity() {{ {identity}; }}\n{wrapper}',
|
||||||
'fixture', '--monitor', tmp, '/bin/true', secret], capture_output=True, timeout=3)
|
'fixture', '--monitor', tmp, '/bin/true', secret], capture_output=True, timeout=3)
|
||||||
assert result.returncode == code, result.stderr
|
assert result.returncode == code, result.stderr
|
||||||
assert result.stderr == f'controller-startup: wrapper={diagnostic}\n'.encode(), result.stderr
|
assert result.stderr == f'controller-startup: wrapper={diagnostic}\n'.encode(), result.stderr
|
||||||
ready = Path(tmp) / 'monitor-ready'
|
ready = Path(tmp) / 'monitor-ready'
|
||||||
assert ready.exists() == (code == 0)
|
assert ready.exists() == (code == 0)
|
||||||
assert secret.encode() not in result.stdout + result.stderr
|
assert secret.encode() not in result.stdout + result.stderr
|
||||||
print('startup diagnostic fixtures passed (shared apply/monitor launch and wrapper)')
|
|
||||||
|
# Both publication and parent cleanup preserve unknown artifacts, including links.
|
||||||
|
for artifact in ('file', 'symlink', 'directory'):
|
||||||
|
with tempfile.TemporaryDirectory(prefix='foreign-ack-') as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
ready = root / 'monitor-ready'
|
||||||
|
if artifact == 'file':
|
||||||
|
ready.write_text(secret)
|
||||||
|
elif artifact == 'symlink':
|
||||||
|
ready.symlink_to(root / 'absent')
|
||||||
|
else:
|
||||||
|
ready.mkdir()
|
||||||
|
before = ready.lstat()
|
||||||
|
result = subprocess.run(['sh', '-c', f'proc_identity() {{ proc_sid=$$; }}\n{wrapper}',
|
||||||
|
'fixture', '--monitor', tmp, '/bin/true', secret], capture_output=True, timeout=3)
|
||||||
|
assert result.returncode == 78, result.stderr
|
||||||
|
assert result.stderr == b'controller-startup: wrapper=ack_publish_failed\n', result.stderr
|
||||||
|
result = subprocess.run(['sh', '-c', f'''set -eu
|
||||||
|
config_dir='{tmp}'
|
||||||
|
{launch}
|
||||||
|
launch_controller managed monitor
|
||||||
|
'''], capture_output=True, timeout=3)
|
||||||
|
assert result.returncode == 78, result.stderr
|
||||||
|
assert result.stderr == b'controller-startup: launch=ack_conflict\n', result.stderr
|
||||||
|
assert ready.lstat() == before
|
||||||
|
if artifact == 'file':
|
||||||
|
assert ready.read_text() == secret
|
||||||
|
assert not list(root.glob('.monitor-ready.*'))
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory(prefix='replaced-ack-') as tmp:
|
||||||
|
result = subprocess.run(['sh', '-c', f'''set -eu
|
||||||
|
config_dir='{tmp}'
|
||||||
|
printf '123\n' > "$config_dir/monitor-ready"
|
||||||
|
monitor_ready_identity=$(stat -c '%d:%i:%u:%a:%h' "$config_dir/monitor-ready")
|
||||||
|
monitor_ready_pid=123
|
||||||
|
printf '{secret}' > "$config_dir/replacement"
|
||||||
|
mv "$config_dir/replacement" "$config_dir/monitor-ready"
|
||||||
|
{launch}
|
||||||
|
launch_controller managed monitor
|
||||||
|
'''], capture_output=True, timeout=3)
|
||||||
|
assert result.returncode == 78, result.stderr
|
||||||
|
assert result.stderr == b'controller-startup: launch=ack_conflict\n'
|
||||||
|
assert (Path(tmp) / 'monitor-ready').read_text() == secret
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory(prefix='unsafe-ack-dir-') as tmp:
|
||||||
|
os.chmod(tmp, 0o755)
|
||||||
|
result = subprocess.run(['sh', '-c', f'proc_identity() {{ proc_sid=$$; }}\n{wrapper}',
|
||||||
|
'fixture', '--monitor', tmp, '/bin/true'], capture_output=True, timeout=3)
|
||||||
|
assert result.returncode == 78, result.stderr
|
||||||
|
assert result.stderr == b'controller-startup: wrapper=ack_publish_failed\n'
|
||||||
|
assert not list(Path(tmp).iterdir())
|
||||||
|
print('startup diagnostic fixtures passed (shared apply/monitor/qualify launch and atomic wrapper)')
|
||||||
@@ -337,7 +337,20 @@ start_dns() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
launch_controller() {
|
launch_controller() {
|
||||||
rm -f "$config_dir/monitor-ready"
|
# Only an ACK retained from our previous launch is owned stale state.
|
||||||
|
# Unknown files (including dangling links) are evidence, not debris.
|
||||||
|
if [ -e "$config_dir/monitor-ready" ] || [ -L "$config_dir/monitor-ready" ]; then
|
||||||
|
if [ -z "${monitor_ready_identity-}" ] || [ -L "$config_dir/monitor-ready" ] ||
|
||||||
|
[ ! -f "$config_dir/monitor-ready" ] ||
|
||||||
|
[ "$(stat -c '%d:%i:%u:%a:%h' "$config_dir/monitor-ready")" != "$monitor_ready_identity" ] ||
|
||||||
|
! { read -r stale_pid < "$config_dir/monitor-ready"; } 2>/dev/null ||
|
||||||
|
[ "$stale_pid" != "${monitor_ready_pid-}" ]; then
|
||||||
|
echo 'controller-startup: launch=ack_conflict' >&2
|
||||||
|
return 78
|
||||||
|
fi
|
||||||
|
rm -f "$config_dir/monitor-ready" || return 78
|
||||||
|
fi
|
||||||
|
monitor_ready_identity=
|
||||||
setsid /bin/sh /opt/managed-target-supervision.sh --monitor "$config_dir" /usr/local/bin/zapret-autotune "$@" 8<&- 9<&- &
|
setsid /bin/sh /opt/managed-target-supervision.sh --monitor "$config_dir" /usr/local/bin/zapret-autotune "$@" 8<&- 9<&- &
|
||||||
monitor_pid=$!
|
monitor_pid=$!
|
||||||
monitor_ack_deadline=$(($(date +%s) + 5))
|
monitor_ack_deadline=$(($(date +%s) + 5))
|
||||||
@@ -353,6 +366,8 @@ launch_controller() {
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
capture_monitor_session || return 78
|
capture_monitor_session || return 78
|
||||||
|
monitor_ready_identity=$(stat -c '%d:%i:%u:%a:%h' "$config_dir/monitor-ready") || return 78
|
||||||
|
monitor_ready_pid=$monitor_pid
|
||||||
if proc_identity "$monitor_pid" && [ "$proc_sid" != "$monitor_sid" ]; then
|
if proc_identity "$monitor_pid" && [ "$proc_sid" != "$monitor_sid" ]; then
|
||||||
echo 'controller-startup: launch=session_mismatch' >&2
|
echo 'controller-startup: launch=session_mismatch' >&2
|
||||||
monitor_sid=
|
monitor_sid=
|
||||||
@@ -416,7 +431,6 @@ fi
|
|||||||
# fork a monitor we already need to cancel: an early TERM can race interpreter
|
# fork a monitor we already need to cancel: an early TERM can race interpreter
|
||||||
# startup before its cancellation handler exists. Clean this cycle directly.
|
# startup before its cancellation handler exists. Clean this cycle directly.
|
||||||
{ dns_alive && kill -0 "$server_pid" 2>/dev/null && kill -0 "$local_pid" 2>/dev/null; } || return 75
|
{ dns_alive && kill -0 "$server_pid" 2>/dev/null && kill -0 "$local_pid" 2>/dev/null; } || return 75
|
||||||
rm -f "$config_dir/monitor-ready"
|
|
||||||
if [ "$qualification_mode" -eq 1 ]; then
|
if [ "$qualification_mode" -eq 1 ]; then
|
||||||
qualification_attempted=1
|
qualification_attempted=1
|
||||||
set -- qualify --config "$ZAPRET_AUTOTUNE_QUALIFY_CONFIG" --strategies "$ZAPRET_AUTOTUNE_STRATEGY_CONFIG" --isolated
|
set -- qualify --config "$ZAPRET_AUTOTUNE_QUALIFY_CONFIG" --strategies "$ZAPRET_AUTOTUNE_STRATEGY_CONFIG" --isolated
|
||||||
|
|||||||
@@ -68,7 +68,22 @@ if [ "${1-}" = --monitor ]; then
|
|||||||
# Acknowledge the private session before exec, so even an immediate monitor
|
# Acknowledge the private session before exec, so even an immediate monitor
|
||||||
# crash leaves its ownership recorded. No worker can precede this handshake.
|
# crash leaves its ownership recorded. No worker can precede this handshake.
|
||||||
trap 'exit 143' INT TERM
|
trap 'exit 143' INT TERM
|
||||||
printf '%s\n' "$$" > "$config_dir/monitor-ready"
|
ack_temp=
|
||||||
|
trap '[ -z "$ack_temp" ] || rm -f "$ack_temp"' EXIT
|
||||||
|
ack_publish() {
|
||||||
|
# Private same-UID directory: no untrusted writer can race the absent
|
||||||
|
# destination check. Rename exposes complete bytes with a single link.
|
||||||
|
[ -d "$config_dir" ] && [ ! -L "$config_dir" ] &&
|
||||||
|
[ "$(stat -c '%u:%a' "$config_dir")" = "$(id -u):700" ] || return 1
|
||||||
|
umask 077
|
||||||
|
ack_temp=$(mktemp "$config_dir/.monitor-ready.XXXXXX") || return 1
|
||||||
|
printf '%s\n' "$$" > "$ack_temp" || return 1
|
||||||
|
[ ! -e "$config_dir/monitor-ready" ] && [ ! -L "$config_dir/monitor-ready" ] || return 1
|
||||||
|
mv -T "$ack_temp" "$config_dir/monitor-ready" || return 1
|
||||||
|
ack_temp=
|
||||||
|
}
|
||||||
|
ack_publish || { echo 'controller-startup: wrapper=ack_publish_failed' >&2; exit 78; }
|
||||||
|
trap - EXIT
|
||||||
echo 'controller-startup: wrapper=exec_pending' >&2
|
echo 'controller-startup: wrapper=exec_pending' >&2
|
||||||
exec "$@"
|
exec "$@"
|
||||||
elif [ "${1-}" = --watchdog ]; then
|
elif [ "${1-}" = --watchdog ]; then
|
||||||
|
|||||||
Reference in new issue
Block a user