fix(supervision): reap owned session workers after monitor crash
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
d7397ed6cf
commit
e9b70b00b6
7 files changed
+205
-9
No files matched your search
@@ -16,6 +16,7 @@ RUN if [ ! -e /opt/zapret2/lua ] && [ ! -L /opt/zapret2/lua ]; then \
|
|||||||
cp -a /opt/zapret2/lua.dist /opt/zapret2/lua; \
|
cp -a /opt/zapret2/lua.dist /opt/zapret2/lua; \
|
||||||
fi \
|
fi \
|
||||||
&& test -d /opt/zapret2/lua \
|
&& test -d /opt/zapret2/lua \
|
||||||
|
&& command -v setsid \
|
||||||
&& mkdir -p /etc/zapret-autotune /var/lib/zapret-autotune/target \
|
&& mkdir -p /etc/zapret-autotune /var/lib/zapret-autotune/target \
|
||||||
&& printf 'zapret-autotune-managed-v1\n' > /etc/zapret-autotune/target-id \
|
&& printf 'zapret-autotune-managed-v1\n' > /etc/zapret-autotune/target-id \
|
||||||
&& chmod 700 /var/lib/zapret-autotune /var/lib/zapret-autotune/target
|
&& chmod 700 /var/lib/zapret-autotune /var/lib/zapret-autotune/target
|
||||||
|
|||||||
@@ -668,18 +668,31 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
|||||||
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "ZAPRET_CONFIG="+filepath.Join(root, "config"), "SS_PASSWORD="+strings.Repeat("a", 32))
|
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "ZAPRET_CONFIG="+filepath.Join(root, "config"), "SS_PASSWORD="+strings.Repeat("a", 32))
|
||||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||||
var shutdownSnapshot string
|
var shutdownSnapshot string
|
||||||
|
stopMonitorSession := func() {
|
||||||
|
ready, _ := os.ReadFile(filepath.Join(config, "monitor-ready"))
|
||||||
|
sid, err := strconv.Atoi(strings.TrimSpace(string(ready)))
|
||||||
|
if err != nil || sid <= 1 || sid == os.Getpid() || sid == cmd.Process.Pid {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Deadline cleanup must cover the fixture's verified private SID,
|
||||||
|
// including separate check PGIDs, not just the entrypoint PGID.
|
||||||
|
_ = exec.Command("/bin/sh", "-c", `sid=$1; helper=$2; shift 2; . "$helper"; signal_session "$sid" KILL`, "cleanup", strconv.Itoa(sid), helper).Run()
|
||||||
|
}
|
||||||
cmd.Cancel = func() error {
|
cmd.Cancel = func() error {
|
||||||
// Snapshot only this fixture's process group before deadline cleanup;
|
// Snapshot only this fixture's process group before deadline cleanup;
|
||||||
// no environment or private credential/config contents are collected.
|
// no environment or private credential/config contents are collected.
|
||||||
processes, _ := exec.Command("ps", "-eo", "pid,ppid,pgid,stat,wchan:24,args").Output()
|
ready, _ := os.ReadFile(filepath.Join(config, "monitor-ready"))
|
||||||
|
ownedSID := strings.TrimSpace(string(ready))
|
||||||
|
processes, _ := exec.Command("ps", "-eo", "pid,ppid,pgid,sid,stat,wchan:24,args").Output()
|
||||||
for _, line := range strings.Split(string(processes), "\n") {
|
for _, line := range strings.Split(string(processes), "\n") {
|
||||||
fields := strings.Fields(line)
|
fields := strings.Fields(line)
|
||||||
if len(fields) >= 3 && fields[2] == strconv.Itoa(cmd.Process.Pid) {
|
if len(fields) >= 4 && (fields[2] == strconv.Itoa(cmd.Process.Pid) || fields[3] == ownedSID) {
|
||||||
shutdownSnapshot += line + "\n"
|
shutdownSnapshot += line + "\n"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
fixtureLog, _ := os.ReadFile(log)
|
fixtureLog, _ := os.ReadFile(log)
|
||||||
shutdownSnapshot += "fixture log:\n" + string(fixtureLog)
|
shutdownSnapshot += "fixture log:\n" + string(fixtureLog)
|
||||||
|
stopMonitorSession()
|
||||||
return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
||||||
}
|
}
|
||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
@@ -690,7 +703,16 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
|||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
exited := make(chan struct{})
|
exited := make(chan struct{})
|
||||||
go func() { done <- cmd.Wait(); close(exited) }()
|
go func() { done <- cmd.Wait(); close(exited) }()
|
||||||
t.Cleanup(func() { syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); <-exited })
|
t.Cleanup(func() {
|
||||||
|
select {
|
||||||
|
case <-exited:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
stopMonitorSession()
|
||||||
|
syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
||||||
|
<-exited
|
||||||
|
}
|
||||||
|
})
|
||||||
if strings.HasPrefix(mode, "term") {
|
if strings.HasPrefix(mode, "term") {
|
||||||
for {
|
for {
|
||||||
b, _ := os.ReadFile(log)
|
b, _ := os.ReadFile(log)
|
||||||
|
|||||||
@@ -59,9 +59,12 @@ stop gets 30 seconds then forced kill after five more, and each Shadowsocks join
|
|||||||
gets five seconds. Exceeding a join bound fails closed. Compose grants 240
|
gets five seconds. Exceeding a join bound fails closed. Compose grants 240
|
||||||
seconds. Reconcile after joining the monitor and before even stopping zapret:
|
seconds. Reconcile after joining the monitor and before even stopping zapret:
|
||||||
stop also sources config, so unknown state is never sourced. The sequential
|
stop also sources config, so unknown state is never sourced. The sequential
|
||||||
cleanup budget is 140 seconds; 240 seconds also covers an in-flight startup
|
cleanup budget is 146 seconds; 240 seconds also covers an in-flight startup
|
||||||
command and scheduling grace. Watchdogs run in fresh shell interpreters and
|
command and scheduling grace. Watchdogs run in fresh shell interpreters and
|
||||||
cancel/join their own sleepers, including during PID 1's EXIT trap.
|
cancel/join their own sleepers, including during PID 1's EXIT trap.
|
||||||
|
The monitor has a verified private session. After a crash, remaining check/NFQ
|
||||||
|
workers in that session are terminated and verified gone before reconciliation
|
||||||
|
or reactivation; Shadowsocks, watchdogs and unrelated exec peers are excluded.
|
||||||
|
|
||||||
## Isolation and runtime contract
|
## Isolation and runtime contract
|
||||||
|
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ assert ' restart: "no"\n' in compose
|
|||||||
assert " stop_grace_period: 240s\n" in compose
|
assert " stop_grace_period: 240s\n" in compose
|
||||||
assert 'bounded_wait "$monitor_pid" 45' in entrypoint
|
assert 'bounded_wait "$monitor_pid" 45' in entrypoint
|
||||||
assert '/bin/sh /opt/managed-target-supervision.sh --watchdog' in entrypoint
|
assert '/bin/sh /opt/managed-target-supervision.sh --watchdog' in entrypoint
|
||||||
|
assert 'setsid /bin/sh /opt/managed-target-supervision.sh --monitor' in entrypoint
|
||||||
|
assert entrypoint.index('if ! cleanup_session; then') < entrypoint.index('if timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile; then')
|
||||||
assert 'COPY managed-target-supervision.sh /opt/managed-target-supervision.sh' in (directory.parents[1] / 'Dockerfile.target').read_text()
|
assert 'COPY managed-target-supervision.sh /opt/managed-target-supervision.sh' in (directory.parents[1] / 'Dockerfile.target').read_text()
|
||||||
assert '!managed-target-supervision.sh' in (directory.parents[1] / '.dockerignore').read_text()
|
assert '!managed-target-supervision.sh' in (directory.parents[1] / '.dockerignore').read_text()
|
||||||
assert '"$count" -lt 3' in entrypoint
|
assert '"$count" -lt 3' in entrypoint
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
Only a temporary COPY of the entrypoint is redirected to fake executables;
|
Only a temporary COPY of the entrypoint is redirected to fake executables;
|
||||||
production has no test switches, real network probes, or accelerated timers.
|
production has no test switches, real network probes, or accelerated timers.
|
||||||
"""
|
"""
|
||||||
|
import ctypes
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import shutil
|
import shutil
|
||||||
@@ -13,7 +14,7 @@ import time
|
|||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
FAKE = r'''#!/usr/bin/env python3
|
FAKE = r'''#!/usr/bin/env python3
|
||||||
import os, pathlib, signal, sys, time
|
import ctypes, os, pathlib, signal, subprocess, sys, time
|
||||||
root = pathlib.Path(os.environ['FIXTURE'])
|
root = pathlib.Path(os.environ['FIXTURE'])
|
||||||
name = pathlib.Path(sys.argv[0]).name
|
name = pathlib.Path(sys.argv[0]).name
|
||||||
scenario = os.environ['SCENARIO']
|
scenario = os.environ['SCENARIO']
|
||||||
@@ -29,6 +30,25 @@ def linger(code=0):
|
|||||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(code))
|
signal.signal(signal.SIGTERM, lambda *_: sys.exit(code))
|
||||||
while True:
|
while True:
|
||||||
time.sleep(.01)
|
time.sleep(.01)
|
||||||
|
def assert_workers_gone():
|
||||||
|
if (root / 'workers').exists():
|
||||||
|
for pid in (root / 'workers').read_text().splitlines():
|
||||||
|
stat = pathlib.Path('/proc/' + pid + '/stat')
|
||||||
|
assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('old worker still live', pid)
|
||||||
|
if (root / 'peer').exists():
|
||||||
|
os.kill(int((root / 'peer').read_text()), 0)
|
||||||
|
if name in ('blockcheck', 'nfq'):
|
||||||
|
# Model Go's Setpgid check child and its NFQ worker, not a daemonized sysv
|
||||||
|
# process. Comm deliberately contains spaces/parentheses to test /proc parsing.
|
||||||
|
ctypes.CDLL(None).prctl(15, b'bc ) (fixture)', 0, 0, 0)
|
||||||
|
assert os.getsid(0) != os.getpgrp(), 'fixture did not split PGID from SID'
|
||||||
|
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||||
|
with (root / 'workers').open('a') as f:
|
||||||
|
f.write(str(os.getpid()) + '\n')
|
||||||
|
if name == 'blockcheck':
|
||||||
|
subprocess.Popen([str(root / 'bin' / 'nfq')])
|
||||||
|
while True:
|
||||||
|
time.sleep(.01)
|
||||||
if name == 'awk':
|
if name == 'awk':
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
if name == 'date':
|
if name == 'date':
|
||||||
@@ -64,6 +84,8 @@ if name in ('ss-server', 'ss-local'):
|
|||||||
sys.exit(137)
|
sys.exit(137)
|
||||||
linger()
|
linger()
|
||||||
if name == 'zapret2':
|
if name == 'zapret2':
|
||||||
|
if scenario == 'orphans':
|
||||||
|
assert_workers_gone()
|
||||||
assert not (root / 'prepared').exists(), 'sourced prepared config'
|
assert not (root / 'prepared').exists(), 'sourced prepared config'
|
||||||
log(sys.argv[1])
|
log(sys.argv[1])
|
||||||
if scenario == 'stopfail' and sys.argv[1] == 'stop':
|
if scenario == 'stopfail' and sys.argv[1] == 'stop':
|
||||||
@@ -77,6 +99,8 @@ if command == 'init':
|
|||||||
elif command == 'reconcile':
|
elif command == 'reconcile':
|
||||||
n = increment('reconciles')
|
n = increment('reconciles')
|
||||||
log('reconcile')
|
log('reconcile')
|
||||||
|
if scenario == 'orphans':
|
||||||
|
assert_workers_gone()
|
||||||
if scenario == 'prestartfail':
|
if scenario == 'prestartfail':
|
||||||
sys.exit(78)
|
sys.exit(78)
|
||||||
if scenario in ('unknown', 'unknown75') and n == 2:
|
if scenario in ('unknown', 'unknown75') and n == 2:
|
||||||
@@ -87,6 +111,13 @@ elif command == 'monitor':
|
|||||||
f.write(str(os.getpid()) + '\n')
|
f.write(str(os.getpid()) + '\n')
|
||||||
n = increment('monitor_runs')
|
n = increment('monitor_runs')
|
||||||
log('monitor ' + str(n))
|
log('monitor ' + str(n))
|
||||||
|
assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session'
|
||||||
|
if n == 1 and scenario == 'orphans':
|
||||||
|
subprocess.Popen([str(root / 'bin' / 'blockcheck')], preexec_fn=os.setpgrp)
|
||||||
|
while not (root / 'workers').exists() or len((root / 'workers').read_text().splitlines()) != 2:
|
||||||
|
time.sleep(.01)
|
||||||
|
(root / 'prepared').touch()
|
||||||
|
os.kill(os.getpid(), signal.SIGKILL)
|
||||||
if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'):
|
if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'):
|
||||||
sys.exit(75)
|
sys.exit(75)
|
||||||
if scenario.startswith('fatal'):
|
if scenario.startswith('fatal'):
|
||||||
@@ -117,7 +148,7 @@ def run(scenario, expected):
|
|||||||
fake = binary / 'fake'
|
fake = binary / 'fake'
|
||||||
fake.write_text(FAKE)
|
fake.write_text(FAKE)
|
||||||
fake.chmod(0o755)
|
fake.chmod(0o755)
|
||||||
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2'):
|
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2', 'blockcheck', 'nfq'):
|
||||||
(binary / name).symlink_to(fake)
|
(binary / name).symlink_to(fake)
|
||||||
configs = root / 'private'
|
configs = root / 'private'
|
||||||
configs.mkdir()
|
configs.mkdir()
|
||||||
@@ -135,12 +166,36 @@ def run(scenario, expected):
|
|||||||
entry.write_text(text)
|
entry.write_text(text)
|
||||||
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
|
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
|
||||||
SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config'))
|
SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config'))
|
||||||
|
peer = None
|
||||||
|
if scenario == 'orphans':
|
||||||
|
# Local fixture is not PID 1. Adopt orphan workers here so the test
|
||||||
|
# can reap their verified-dead zombies rather than burden host PID 1.
|
||||||
|
assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0
|
||||||
|
peer = subprocess.Popen([shutil.which('sleep'), '300'], start_new_session=True)
|
||||||
|
(root / 'peer').write_text(str(peer.pid))
|
||||||
|
try:
|
||||||
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15)
|
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15)
|
||||||
|
if peer:
|
||||||
|
assert peer.poll() is None, 'unowned helper was killed'
|
||||||
|
finally:
|
||||||
|
if peer:
|
||||||
|
peer.terminate()
|
||||||
|
peer.wait(timeout=2)
|
||||||
events = (root / 'events').read_text().splitlines()
|
events = (root / 'events').read_text().splitlines()
|
||||||
assert result.returncode == expected, (scenario, result.returncode, result.stderr, events)
|
assert result.returncode == expected, (scenario, result.returncode, result.stderr, events)
|
||||||
assert b'ab' * 32 not in result.stdout + result.stderr
|
assert b'ab' * 32 not in result.stdout + result.stderr
|
||||||
for pid in (root / 'children').read_text().splitlines():
|
for pid in (root / 'children').read_text().splitlines():
|
||||||
assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid)
|
assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid)
|
||||||
|
if scenario == 'orphans':
|
||||||
|
for pid in (root / 'workers').read_text().splitlines():
|
||||||
|
stat = Path(f'/proc/{pid}/stat')
|
||||||
|
assert not stat.exists() or stat.read_text().rsplit(') ', 1)[1].split()[0] == 'Z', ('orphan survived', pid)
|
||||||
|
try:
|
||||||
|
reaped, _ = os.waitpid(int(pid), os.WNOHANG)
|
||||||
|
assert reaped == int(pid), ('worker not reaped', pid)
|
||||||
|
except ChildProcessError:
|
||||||
|
assert not stat.exists(), ('worker was not adopted/reaped', pid)
|
||||||
|
assert ctypes.CDLL(None).prctl(36, 0, 0, 0, 0) == 0
|
||||||
# Every activation and stop must immediately follow successful reconcile.
|
# Every activation and stop must immediately follow successful reconcile.
|
||||||
for i, event in enumerate(events):
|
for i, event in enumerate(events):
|
||||||
if event in ('start', 'stop'):
|
if event in ('start', 'stop'):
|
||||||
@@ -148,7 +203,7 @@ def run(scenario, expected):
|
|||||||
return events
|
return events
|
||||||
|
|
||||||
|
|
||||||
for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window'):
|
for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window', 'orphans'):
|
||||||
events = run(scenario, 0)
|
events = run(scenario, 0)
|
||||||
assert events.count('start') == 2, (scenario, events)
|
assert events.count('start') == 2, (scenario, events)
|
||||||
assert events.count('ss-server') == 2 and events.count('ss-local') == 2
|
assert events.count('ss-server') == 2 and events.count('ss-local') == 2
|
||||||
@@ -220,4 +275,6 @@ if busybox:
|
|||||||
for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')):
|
for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')):
|
||||||
result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2)
|
result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2)
|
||||||
assert result.returncode == 64
|
assert result.returncode == 64
|
||||||
|
result = subprocess.run(['/bin/sh', '-c', 'helper=$1; shift; . "$helper"; proc_identity "$$"; own=$proc_sid; if signal_session "$own" KILL || signal_session 1 KILL; then exit 99; fi', 'check', str(ROOT / 'managed-target-supervision.sh')], capture_output=True, timeout=2)
|
||||||
|
assert result.returncode == 0, ('unsafe session accepted', result.returncode)
|
||||||
print('entrypoint supervision fixtures passed')
|
print('entrypoint supervision fixtures passed')
|
||||||
@@ -29,6 +29,7 @@ unset ss_password
|
|||||||
server_pid=
|
server_pid=
|
||||||
local_pid=
|
local_pid=
|
||||||
monitor_pid=
|
monitor_pid=
|
||||||
|
monitor_sid=
|
||||||
zapret_started=0
|
zapret_started=0
|
||||||
cooldown_pid=
|
cooldown_pid=
|
||||||
# Signal only descendants of an owned child, never the container/host namespace.
|
# Signal only descendants of an owned child, never the container/host namespace.
|
||||||
@@ -40,6 +41,38 @@ pause() {
|
|||||||
wait "$cooldown_pid"
|
wait "$cooldown_pid"
|
||||||
cooldown_pid=
|
cooldown_pid=
|
||||||
}
|
}
|
||||||
|
cleanup_session() {
|
||||||
|
[ -n "$monitor_sid" ] || return 0
|
||||||
|
# The verified SID survives leader death and Go's separate worker PGIDs.
|
||||||
|
# Never sweep all namespace children: Docker exec peers/SS/helpers are not
|
||||||
|
# in this session. Detached sysv daemons are reconciled/stopped separately.
|
||||||
|
for session_signal in TERM KILL; do
|
||||||
|
signal_session "$monitor_sid" "$session_signal" || return 78
|
||||||
|
session_deadline=$(($(date +%s) + 3))
|
||||||
|
while :; do
|
||||||
|
signal_session "$monitor_sid" CHECK || return 78
|
||||||
|
if [ "$session_live" -eq 0 ]; then
|
||||||
|
monitor_sid=
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
[ "$(date +%s)" -lt "$session_deadline" ] || break
|
||||||
|
pause 1
|
||||||
|
done
|
||||||
|
done
|
||||||
|
echo 'error: owned monitor session survived cleanup' >&2
|
||||||
|
return 78
|
||||||
|
}
|
||||||
|
capture_monitor_session() {
|
||||||
|
[ -z "$monitor_sid" ] || return 0
|
||||||
|
if [ -e "$config_dir/monitor-ready" ]; then
|
||||||
|
read -r candidate_sid < "$config_dir/monitor-ready" || return 78
|
||||||
|
case "$candidate_sid" in ''|*[!0-9]*) return 78 ;; esac
|
||||||
|
[ "$candidate_sid" = "$monitor_pid" ] && [ "$candidate_sid" -gt 1 ] && [ "$candidate_sid" != "$$" ] || return 78
|
||||||
|
monitor_sid=$candidate_sid
|
||||||
|
elif proc_identity "$monitor_pid" && [ "$proc_sid" = "$monitor_pid" ] && [ "$proc_sid" -gt 1 ]; then
|
||||||
|
monitor_sid=$proc_sid
|
||||||
|
fi
|
||||||
|
}
|
||||||
# Wait with a watchdog: never let an unresponsive child exceed Docker's grace.
|
# Wait with a watchdog: never let an unresponsive child exceed Docker's grace.
|
||||||
bounded_wait() {
|
bounded_wait() {
|
||||||
child=$1
|
child=$1
|
||||||
@@ -71,13 +104,22 @@ cleanup_cycle() {
|
|||||||
cleanup_code=0
|
cleanup_code=0
|
||||||
# Let the monitor cancel/join scans and restore committed bytes BEFORE stop.
|
# Let the monitor cancel/join scans and restore committed bytes BEFORE stop.
|
||||||
if [ -n "$monitor_pid" ]; then
|
if [ -n "$monitor_pid" ]; then
|
||||||
|
capture_monitor_session || cleanup_code=78
|
||||||
kill -TERM "$monitor_pid" 2>/dev/null || true
|
kill -TERM "$monitor_pid" 2>/dev/null || true
|
||||||
bounded_wait "$monitor_pid" 45
|
bounded_wait "$monitor_pid" 45
|
||||||
# Cancellation must restore safely even when SS failure requested retry.
|
# Cancellation must restore safely even when SS failure requested retry.
|
||||||
case "$waited_code" in 0|143) ;; *) echo "error: monitor cleanup status $waited_code" >&2; cleanup_code=78 ;; esac
|
case "$waited_code" in 0|143) ;; *) echo "error: monitor cleanup status $waited_code" >&2; cleanup_code=78 ;; esac
|
||||||
[ "$join_timed_out" -eq 0 ] || { echo 'error: monitor join timeout' >&2; cleanup_code=78; }
|
[ "$join_timed_out" -eq 0 ] || { echo 'error: monitor join timeout' >&2; cleanup_code=78; }
|
||||||
|
# SIGTERM may have arrived before the launch acknowledgement was read.
|
||||||
|
capture_monitor_session || cleanup_code=78
|
||||||
monitor_pid=
|
monitor_pid=
|
||||||
fi
|
fi
|
||||||
|
# Join/sweep scans even if the monitor was already reaped after SIGKILL.
|
||||||
|
# A failed sweep vetoes reconciliation and all daemon script sourcing.
|
||||||
|
if ! cleanup_session; then
|
||||||
|
cleanup_code=78
|
||||||
|
zapret_started=0
|
||||||
|
fi
|
||||||
# Dedicated PID/network namespace: no host/other-target stop.
|
# Dedicated PID/network namespace: no host/other-target stop.
|
||||||
if [ "$zapret_started" -eq 1 ]; then
|
if [ "$zapret_started" -eq 1 ]; then
|
||||||
# Stop also sources config: clear trust before reconciliation, never
|
# Stop also sources config: clear trust before reconciliation, never
|
||||||
@@ -147,8 +189,21 @@ timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || exit 78
|
|||||||
# fork a monitor we already need to cancel: an early TERM can race interpreter
|
# fork a monitor we already need to cancel: an early TERM can race interpreter
|
||||||
# startup before its cancellation handler exists. Clean this cycle directly.
|
# startup before its cancellation handler exists. Clean this cycle directly.
|
||||||
{ kill -0 "$server_pid" 2>/dev/null && kill -0 "$local_pid" 2>/dev/null; } || return 75
|
{ kill -0 "$server_pid" 2>/dev/null && kill -0 "$local_pid" 2>/dev/null; } || return 75
|
||||||
/usr/local/bin/zapret-autotune managed monitor &
|
rm -f "$config_dir/monitor-ready"
|
||||||
|
setsid /bin/sh /opt/managed-target-supervision.sh --monitor "$config_dir" /usr/local/bin/zapret-autotune managed monitor &
|
||||||
monitor_pid=$!
|
monitor_pid=$!
|
||||||
|
monitor_ack_deadline=$(($(date +%s) + 5))
|
||||||
|
while [ ! -e "$config_dir/monitor-ready" ]; do
|
||||||
|
if ! kill -0 "$monitor_pid" 2>/dev/null || [ "$(date +%s)" -ge "$monitor_ack_deadline" ]; then
|
||||||
|
signal_tree "$monitor_pid" KILL
|
||||||
|
exit 78
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
capture_monitor_session || exit 78
|
||||||
|
if proc_identity "$monitor_pid" && [ "$proc_sid" != "$monitor_sid" ]; then
|
||||||
|
monitor_sid=
|
||||||
|
exit 78
|
||||||
|
fi
|
||||||
while kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do
|
while kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do
|
||||||
pause 1
|
pause 1
|
||||||
done
|
done
|
||||||
|
|||||||
@@ -11,7 +11,63 @@ signal_tree() {
|
|||||||
kill "-$2" "$1" 2>/dev/null || true
|
kill "-$2" "$1" 2>/dev/null || true
|
||||||
}
|
}
|
||||||
|
|
||||||
if [ "${1-}" = --watchdog ]; then
|
# /proc comm may contain spaces and closing parentheses: remove through the
|
||||||
|
# LAST ') ', then fields are state, PPID, PGID, SID (not whitespace-split comm).
|
||||||
|
proc_identity() {
|
||||||
|
proc_sid=
|
||||||
|
proc_state=
|
||||||
|
proc_line=
|
||||||
|
[ -r "/proc/$1/stat" ] || return 1
|
||||||
|
read -r proc_line < "/proc/$1/stat" || return 1
|
||||||
|
[ "${proc_line%% *}" = "$1" ] || return 1
|
||||||
|
proc_tail=${proc_line##*) }
|
||||||
|
[ "$proc_tail" != "$proc_line" ] || return 1
|
||||||
|
set -- $proc_tail
|
||||||
|
[ "$#" -ge 4 ] || return 1
|
||||||
|
proc_state=$1
|
||||||
|
proc_sid=$4
|
||||||
|
}
|
||||||
|
|
||||||
|
signal_session() {
|
||||||
|
case "$1" in ''|*[!0-9]*) return 1 ;; esac
|
||||||
|
[ "$1" -gt 1 ] && [ "$1" != "$$" ] || return 1
|
||||||
|
proc_identity "$$" || return 1
|
||||||
|
[ "$proc_sid" != "$1" ] || return 1
|
||||||
|
session_live=0
|
||||||
|
for session_stat in /proc/[0-9]*/stat; do
|
||||||
|
session_pid=${session_stat#/proc/}
|
||||||
|
session_pid=${session_pid%/stat}
|
||||||
|
if ! proc_identity "$session_pid"; then
|
||||||
|
# Disappearing processes are normal; an unreadable surviving stat
|
||||||
|
# is not evidence that the owned session is empty.
|
||||||
|
[ ! -e "$session_stat" ] || return 1
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
[ "$proc_sid" = "$1" ] || continue
|
||||||
|
[ "$session_pid" -gt 1 ] && [ "$session_pid" != "$$" ] || return 1
|
||||||
|
if [ "$proc_state" = Z ]; then
|
||||||
|
# PID 1 may have adopted this worker. Nonchildren return immediately;
|
||||||
|
# zombies never block. Shell SIGCHLD handling also reaps adoptees.
|
||||||
|
wait "$session_pid" 2>/dev/null || true
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
session_live=1
|
||||||
|
[ "$2" = CHECK ] || kill "-$2" "$session_pid" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ "${1-}" = --monitor ]; then
|
||||||
|
set -eu
|
||||||
|
[ "$#" -ge 3 ] || exit 64
|
||||||
|
config_dir=$2
|
||||||
|
shift 2
|
||||||
|
proc_identity "$$" && [ "$proc_sid" = "$$" ] && [ "$$" -gt 1 ] || exit 78
|
||||||
|
# Acknowledge the private session before exec, so even an immediate monitor
|
||||||
|
# crash leaves its ownership recorded. No worker can precede this handshake.
|
||||||
|
trap 'exit 143' INT TERM
|
||||||
|
printf '%s\n' "$$" > "$config_dir/monitor-ready"
|
||||||
|
exec "$@"
|
||||||
|
elif [ "${1-}" = --watchdog ]; then
|
||||||
set -eu
|
set -eu
|
||||||
[ "$#" -eq 4 ] || exit 64
|
[ "$#" -eq 4 ] || exit 64
|
||||||
case "$2" in ''|*[!0-9]*) exit 64 ;; esac
|
case "$2" in ''|*[!0-9]*) exit 64 ;; esac
|
||||||
|
|||||||
Reference in new issue
Block a user