diff --git a/Dockerfile.target b/Dockerfile.target index 200f524..06e7535 100644 --- a/Dockerfile.target +++ b/Dockerfile.target @@ -16,6 +16,7 @@ RUN if [ ! -e /opt/zapret2/lua ] && [ ! -L /opt/zapret2/lua ]; then \ cp -a /opt/zapret2/lua.dist /opt/zapret2/lua; \ fi \ && test -d /opt/zapret2/lua \ + && command -v setsid \ && mkdir -p /etc/zapret-autotune /var/lib/zapret-autotune/target \ && printf 'zapret-autotune-managed-v1\n' > /etc/zapret-autotune/target-id \ && chmod 700 /var/lib/zapret-autotune /var/lib/zapret-autotune/target diff --git a/cmd/zapret-autotune/managed_monitor_cli_test.go b/cmd/zapret-autotune/managed_monitor_cli_test.go index d452071..7c25243 100644 --- a/cmd/zapret-autotune/managed_monitor_cli_test.go +++ b/cmd/zapret-autotune/managed_monitor_cli_test.go @@ -668,18 +668,31 @@ func TestManagedEntrypointSupervision(t *testing.T) { cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "ZAPRET_CONFIG="+filepath.Join(root, "config"), "SS_PASSWORD="+strings.Repeat("a", 32)) cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} var shutdownSnapshot string + stopMonitorSession := func() { + ready, _ := os.ReadFile(filepath.Join(config, "monitor-ready")) + sid, err := strconv.Atoi(strings.TrimSpace(string(ready))) + if err != nil || sid <= 1 || sid == os.Getpid() || sid == cmd.Process.Pid { + return + } + // Deadline cleanup must cover the fixture's verified private SID, + // including separate check PGIDs, not just the entrypoint PGID. + _ = exec.Command("/bin/sh", "-c", `sid=$1; helper=$2; shift 2; . "$helper"; signal_session "$sid" KILL`, "cleanup", strconv.Itoa(sid), helper).Run() + } cmd.Cancel = func() error { // Snapshot only this fixture's process group before deadline cleanup; // no environment or private credential/config contents are collected. - processes, _ := exec.Command("ps", "-eo", "pid,ppid,pgid,stat,wchan:24,args").Output() + ready, _ := os.ReadFile(filepath.Join(config, "monitor-ready")) + ownedSID := strings.TrimSpace(string(ready)) + processes, _ := exec.Command("ps", "-eo", "pid,ppid,pgid,sid,stat,wchan:24,args").Output() for _, line := range strings.Split(string(processes), "\n") { fields := strings.Fields(line) - if len(fields) >= 3 && fields[2] == strconv.Itoa(cmd.Process.Pid) { + if len(fields) >= 4 && (fields[2] == strconv.Itoa(cmd.Process.Pid) || fields[3] == ownedSID) { shutdownSnapshot += line + "\n" } } fixtureLog, _ := os.ReadFile(log) shutdownSnapshot += "fixture log:\n" + string(fixtureLog) + stopMonitorSession() return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) } var output bytes.Buffer @@ -690,7 +703,16 @@ func TestManagedEntrypointSupervision(t *testing.T) { done := make(chan error, 1) exited := make(chan struct{}) go func() { done <- cmd.Wait(); close(exited) }() - t.Cleanup(func() { syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); <-exited }) + t.Cleanup(func() { + select { + case <-exited: + return + default: + stopMonitorSession() + syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + <-exited + } + }) if strings.HasPrefix(mode, "term") { for { b, _ := os.ReadFile(log) diff --git a/deploy/managed-target/README.md b/deploy/managed-target/README.md index 325f8e5..7c7fb23 100644 --- a/deploy/managed-target/README.md +++ b/deploy/managed-target/README.md @@ -59,9 +59,12 @@ stop gets 30 seconds then forced kill after five more, and each Shadowsocks join gets five seconds. Exceeding a join bound fails closed. Compose grants 240 seconds. Reconcile after joining the monitor and before even stopping zapret: stop also sources config, so unknown state is never sourced. The sequential -cleanup budget is 140 seconds; 240 seconds also covers an in-flight startup +cleanup budget is 146 seconds; 240 seconds also covers an in-flight startup command and scheduling grace. Watchdogs run in fresh shell interpreters and cancel/join their own sleepers, including during PID 1's EXIT trap. +The monitor has a verified private session. After a crash, remaining check/NFQ +workers in that session are terminated and verified gone before reconciliation +or reactivation; Shadowsocks, watchdogs and unrelated exec peers are excluded. ## Isolation and runtime contract diff --git a/deploy/managed-target/check_static.py b/deploy/managed-target/check_static.py index c996e55..d67d398 100644 --- a/deploy/managed-target/check_static.py +++ b/deploy/managed-target/check_static.py @@ -31,6 +31,8 @@ assert ' restart: "no"\n' in compose assert " stop_grace_period: 240s\n" in compose assert 'bounded_wait "$monitor_pid" 45' in entrypoint assert '/bin/sh /opt/managed-target-supervision.sh --watchdog' in entrypoint +assert 'setsid /bin/sh /opt/managed-target-supervision.sh --monitor' in entrypoint +assert entrypoint.index('if ! cleanup_session; then') < entrypoint.index('if timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile; then') assert 'COPY managed-target-supervision.sh /opt/managed-target-supervision.sh' in (directory.parents[1] / 'Dockerfile.target').read_text() assert '!managed-target-supervision.sh' in (directory.parents[1] / '.dockerignore').read_text() assert '"$count" -lt 3' in entrypoint diff --git a/deploy/managed-target/check_supervision.py b/deploy/managed-target/check_supervision.py index fa01512..61816f1 100644 --- a/deploy/managed-target/check_supervision.py +++ b/deploy/managed-target/check_supervision.py @@ -3,6 +3,7 @@ Only a temporary COPY of the entrypoint is redirected to fake executables; production has no test switches, real network probes, or accelerated timers. """ +import ctypes import os from pathlib import Path import shutil @@ -13,7 +14,7 @@ import time ROOT = Path(__file__).resolve().parents[2] FAKE = r'''#!/usr/bin/env python3 -import os, pathlib, signal, sys, time +import ctypes, os, pathlib, signal, subprocess, sys, time root = pathlib.Path(os.environ['FIXTURE']) name = pathlib.Path(sys.argv[0]).name scenario = os.environ['SCENARIO'] @@ -29,6 +30,25 @@ def linger(code=0): signal.signal(signal.SIGTERM, lambda *_: sys.exit(code)) while True: time.sleep(.01) +def assert_workers_gone(): + if (root / 'workers').exists(): + for pid in (root / 'workers').read_text().splitlines(): + stat = pathlib.Path('/proc/' + pid + '/stat') + assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('old worker still live', pid) + if (root / 'peer').exists(): + os.kill(int((root / 'peer').read_text()), 0) +if name in ('blockcheck', 'nfq'): + # Model Go's Setpgid check child and its NFQ worker, not a daemonized sysv + # process. Comm deliberately contains spaces/parentheses to test /proc parsing. + ctypes.CDLL(None).prctl(15, b'bc ) (fixture)', 0, 0, 0) + assert os.getsid(0) != os.getpgrp(), 'fixture did not split PGID from SID' + signal.signal(signal.SIGTERM, signal.SIG_IGN) + with (root / 'workers').open('a') as f: + f.write(str(os.getpid()) + '\n') + if name == 'blockcheck': + subprocess.Popen([str(root / 'bin' / 'nfq')]) + while True: + time.sleep(.01) if name == 'awk': sys.exit(0) if name == 'date': @@ -64,6 +84,8 @@ if name in ('ss-server', 'ss-local'): sys.exit(137) linger() if name == 'zapret2': + if scenario == 'orphans': + assert_workers_gone() assert not (root / 'prepared').exists(), 'sourced prepared config' log(sys.argv[1]) if scenario == 'stopfail' and sys.argv[1] == 'stop': @@ -77,6 +99,8 @@ if command == 'init': elif command == 'reconcile': n = increment('reconciles') log('reconcile') + if scenario == 'orphans': + assert_workers_gone() if scenario == 'prestartfail': sys.exit(78) if scenario in ('unknown', 'unknown75') and n == 2: @@ -87,6 +111,13 @@ elif command == 'monitor': f.write(str(os.getpid()) + '\n') n = increment('monitor_runs') log('monitor ' + str(n)) + assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session' + if n == 1 and scenario == 'orphans': + subprocess.Popen([str(root / 'bin' / 'blockcheck')], preexec_fn=os.setpgrp) + while not (root / 'workers').exists() or len((root / 'workers').read_text().splitlines()) != 2: + time.sleep(.01) + (root / 'prepared').touch() + os.kill(os.getpid(), signal.SIGKILL) if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'): sys.exit(75) if scenario.startswith('fatal'): @@ -117,7 +148,7 @@ def run(scenario, expected): fake = binary / 'fake' fake.write_text(FAKE) fake.chmod(0o755) - for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2'): + for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2', 'blockcheck', 'nfq'): (binary / name).symlink_to(fake) configs = root / 'private' configs.mkdir() @@ -135,12 +166,36 @@ def run(scenario, expected): entry.write_text(text) env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp, SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config')) - result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15) + peer = None + if scenario == 'orphans': + # Local fixture is not PID 1. Adopt orphan workers here so the test + # can reap their verified-dead zombies rather than burden host PID 1. + assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0 + peer = subprocess.Popen([shutil.which('sleep'), '300'], start_new_session=True) + (root / 'peer').write_text(str(peer.pid)) + try: + result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15) + if peer: + assert peer.poll() is None, 'unowned helper was killed' + finally: + if peer: + peer.terminate() + peer.wait(timeout=2) events = (root / 'events').read_text().splitlines() assert result.returncode == expected, (scenario, result.returncode, result.stderr, events) assert b'ab' * 32 not in result.stdout + result.stderr for pid in (root / 'children').read_text().splitlines(): assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid) + if scenario == 'orphans': + for pid in (root / 'workers').read_text().splitlines(): + stat = Path(f'/proc/{pid}/stat') + assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('orphan survived', pid) + try: + reaped, _ = os.waitpid(int(pid), os.WNOHANG) + assert reaped == int(pid), ('worker not reaped', pid) + except ChildProcessError: + assert not stat.exists(), ('worker was not adopted/reaped', pid) + assert ctypes.CDLL(None).prctl(36, 0, 0, 0, 0) == 0 # Every activation and stop must immediately follow successful reconcile. for i, event in enumerate(events): if event in ('start', 'stop'): @@ -148,7 +203,7 @@ def run(scenario, expected): return events -for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window'): +for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window', 'orphans'): events = run(scenario, 0) assert events.count('start') == 2, (scenario, events) assert events.count('ss-server') == 2 and events.count('ss-local') == 2 @@ -220,4 +275,6 @@ if busybox: for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')): result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2) assert result.returncode == 64 +result = subprocess.run(['/bin/sh', '-c', 'helper=$1; shift; . "$helper"; proc_identity "$$"; own=$proc_sid; if signal_session "$own" KILL || signal_session 1 KILL; then exit 99; fi', 'check', str(ROOT / 'managed-target-supervision.sh')], capture_output=True, timeout=2) +assert result.returncode == 0, ('unsafe session accepted', result.returncode) print('entrypoint supervision fixtures passed') diff --git a/managed-target-entrypoint.sh b/managed-target-entrypoint.sh index 7556b36..a9a44a8 100644 --- a/managed-target-entrypoint.sh +++ b/managed-target-entrypoint.sh @@ -29,6 +29,7 @@ unset ss_password server_pid= local_pid= monitor_pid= +monitor_sid= zapret_started=0 cooldown_pid= # Signal only descendants of an owned child, never the container/host namespace. @@ -40,6 +41,38 @@ pause() { wait "$cooldown_pid" cooldown_pid= } +cleanup_session() { + [ -n "$monitor_sid" ] || return 0 + # The verified SID survives leader death and Go's separate worker PGIDs. + # Never sweep all namespace children: Docker exec peers/SS/helpers are not + # in this session. Detached sysv daemons are reconciled/stopped separately. + for session_signal in TERM KILL; do + signal_session "$monitor_sid" "$session_signal" || return 78 + session_deadline=$(($(date +%s) + 3)) + while :; do + signal_session "$monitor_sid" CHECK || return 78 + if [ "$session_live" -eq 0 ]; then + monitor_sid= + return 0 + fi + [ "$(date +%s)" -lt "$session_deadline" ] || break + pause 1 + done + done + echo 'error: owned monitor session survived cleanup' >&2 + return 78 +} +capture_monitor_session() { + [ -z "$monitor_sid" ] || return 0 + if [ -e "$config_dir/monitor-ready" ]; then + read -r candidate_sid < "$config_dir/monitor-ready" || return 78 + case "$candidate_sid" in ''|*[!0-9]*) return 78 ;; esac + [ "$candidate_sid" = "$monitor_pid" ] && [ "$candidate_sid" -gt 1 ] && [ "$candidate_sid" != "$$" ] || return 78 + monitor_sid=$candidate_sid + elif proc_identity "$monitor_pid" && [ "$proc_sid" = "$monitor_pid" ] && [ "$proc_sid" -gt 1 ]; then + monitor_sid=$proc_sid + fi +} # Wait with a watchdog: never let an unresponsive child exceed Docker's grace. bounded_wait() { child=$1 @@ -71,13 +104,22 @@ cleanup_cycle() { cleanup_code=0 # Let the monitor cancel/join scans and restore committed bytes BEFORE stop. if [ -n "$monitor_pid" ]; then + capture_monitor_session || cleanup_code=78 kill -TERM "$monitor_pid" 2>/dev/null || true bounded_wait "$monitor_pid" 45 # Cancellation must restore safely even when SS failure requested retry. case "$waited_code" in 0|143) ;; *) echo "error: monitor cleanup status $waited_code" >&2; cleanup_code=78 ;; esac [ "$join_timed_out" -eq 0 ] || { echo 'error: monitor join timeout' >&2; cleanup_code=78; } + # SIGTERM may have arrived before the launch acknowledgement was read. + capture_monitor_session || cleanup_code=78 monitor_pid= fi + # Join/sweep scans even if the monitor was already reaped after SIGKILL. + # A failed sweep vetoes reconciliation and all daemon script sourcing. + if ! cleanup_session; then + cleanup_code=78 + zapret_started=0 + fi # Dedicated PID/network namespace: no host/other-target stop. if [ "$zapret_started" -eq 1 ]; then # Stop also sources config: clear trust before reconciliation, never @@ -147,8 +189,21 @@ timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || exit 78 # fork a monitor we already need to cancel: an early TERM can race interpreter # startup before its cancellation handler exists. Clean this cycle directly. { kill -0 "$server_pid" 2>/dev/null && kill -0 "$local_pid" 2>/dev/null; } || return 75 -/usr/local/bin/zapret-autotune managed monitor & +rm -f "$config_dir/monitor-ready" +setsid /bin/sh /opt/managed-target-supervision.sh --monitor "$config_dir" /usr/local/bin/zapret-autotune managed monitor & monitor_pid=$! +monitor_ack_deadline=$(($(date +%s) + 5)) +while [ ! -e "$config_dir/monitor-ready" ]; do + if ! kill -0 "$monitor_pid" 2>/dev/null || [ "$(date +%s)" -ge "$monitor_ack_deadline" ]; then + signal_tree "$monitor_pid" KILL + exit 78 + fi +done +capture_monitor_session || exit 78 +if proc_identity "$monitor_pid" && [ "$proc_sid" != "$monitor_sid" ]; then + monitor_sid= + exit 78 +fi while kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do pause 1 done diff --git a/managed-target-supervision.sh b/managed-target-supervision.sh index 9749b81..2884a5f 100644 --- a/managed-target-supervision.sh +++ b/managed-target-supervision.sh @@ -11,7 +11,63 @@ signal_tree() { kill "-$2" "$1" 2>/dev/null || true } -if [ "${1-}" = --watchdog ]; then +# /proc comm may contain spaces and closing parentheses: remove through the +# LAST ') ', then fields are state, PPID, PGID, SID (not whitespace-split comm). +proc_identity() { + proc_sid= + proc_state= + proc_line= + [ -r "/proc/$1/stat" ] || return 1 + read -r proc_line < "/proc/$1/stat" || return 1 + [ "${proc_line%% *}" = "$1" ] || return 1 + proc_tail=${proc_line##*) } + [ "$proc_tail" != "$proc_line" ] || return 1 + set -- $proc_tail + [ "$#" -ge 4 ] || return 1 + proc_state=$1 + proc_sid=$4 +} + +signal_session() { + case "$1" in ''|*[!0-9]*) return 1 ;; esac + [ "$1" -gt 1 ] && [ "$1" != "$$" ] || return 1 + proc_identity "$$" || return 1 + [ "$proc_sid" != "$1" ] || return 1 + session_live=0 + for session_stat in /proc/[0-9]*/stat; do + session_pid=${session_stat#/proc/} + session_pid=${session_pid%/stat} + if ! proc_identity "$session_pid"; then + # Disappearing processes are normal; an unreadable surviving stat + # is not evidence that the owned session is empty. + [ ! -e "$session_stat" ] || return 1 + continue + fi + [ "$proc_sid" = "$1" ] || continue + [ "$session_pid" -gt 1 ] && [ "$session_pid" != "$$" ] || return 1 + if [ "$proc_state" = Z ]; then + # PID 1 may have adopted this worker. Nonchildren return immediately; + # zombies never block. Shell SIGCHLD handling also reaps adoptees. + wait "$session_pid" 2>/dev/null || true + continue + fi + session_live=1 + [ "$2" = CHECK ] || kill "-$2" "$session_pid" 2>/dev/null || true + done +} + +if [ "${1-}" = --monitor ]; then + set -eu + [ "$#" -ge 3 ] || exit 64 + config_dir=$2 + shift 2 + proc_identity "$$" && [ "$proc_sid" = "$$" ] && [ "$$" -gt 1 ] || exit 78 + # Acknowledge the private session before exec, so even an immediate monitor + # crash leaves its ownership recorded. No worker can precede this handshake. + trap 'exit 143' INT TERM + printf '%s\n' "$$" > "$config_dir/monitor-ready" + exec "$@" +elif [ "${1-}" = --watchdog ]; then set -eu [ "$#" -eq 4 ] || exit 64 case "$2" in ''|*[!0-9]*) exit 64 ;; esac