fix(supervision): reap owned session workers after monitor crash
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
d7397ed6cf
commit
e9b70b00b6
7 files changed
+206
-10
No files matched your search
@@ -29,6 +29,7 @@ unset ss_password
|
||||
server_pid=
|
||||
local_pid=
|
||||
monitor_pid=
|
||||
monitor_sid=
|
||||
zapret_started=0
|
||||
cooldown_pid=
|
||||
# Signal only descendants of an owned child, never the container/host namespace.
|
||||
@@ -40,6 +41,38 @@ pause() {
|
||||
wait "$cooldown_pid"
|
||||
cooldown_pid=
|
||||
}
|
||||
cleanup_session() {
|
||||
[ -n "$monitor_sid" ] || return 0
|
||||
# The verified SID survives leader death and Go's separate worker PGIDs.
|
||||
# Never sweep all namespace children: Docker exec peers/SS/helpers are not
|
||||
# in this session. Detached sysv daemons are reconciled/stopped separately.
|
||||
for session_signal in TERM KILL; do
|
||||
signal_session "$monitor_sid" "$session_signal" || return 78
|
||||
session_deadline=$(($(date +%s) + 3))
|
||||
while :; do
|
||||
signal_session "$monitor_sid" CHECK || return 78
|
||||
if [ "$session_live" -eq 0 ]; then
|
||||
monitor_sid=
|
||||
return 0
|
||||
fi
|
||||
[ "$(date +%s)" -lt "$session_deadline" ] || break
|
||||
pause 1
|
||||
done
|
||||
done
|
||||
echo 'error: owned monitor session survived cleanup' >&2
|
||||
return 78
|
||||
}
|
||||
capture_monitor_session() {
|
||||
[ -z "$monitor_sid" ] || return 0
|
||||
if [ -e "$config_dir/monitor-ready" ]; then
|
||||
read -r candidate_sid < "$config_dir/monitor-ready" || return 78
|
||||
case "$candidate_sid" in ''|*[!0-9]*) return 78 ;; esac
|
||||
[ "$candidate_sid" = "$monitor_pid" ] && [ "$candidate_sid" -gt 1 ] && [ "$candidate_sid" != "$$" ] || return 78
|
||||
monitor_sid=$candidate_sid
|
||||
elif proc_identity "$monitor_pid" && [ "$proc_sid" = "$monitor_pid" ] && [ "$proc_sid" -gt 1 ]; then
|
||||
monitor_sid=$proc_sid
|
||||
fi
|
||||
}
|
||||
# Wait with a watchdog: never let an unresponsive child exceed Docker's grace.
|
||||
bounded_wait() {
|
||||
child=$1
|
||||
@@ -71,13 +104,22 @@ cleanup_cycle() {
|
||||
cleanup_code=0
|
||||
# Let the monitor cancel/join scans and restore committed bytes BEFORE stop.
|
||||
if [ -n "$monitor_pid" ]; then
|
||||
capture_monitor_session || cleanup_code=78
|
||||
kill -TERM "$monitor_pid" 2>/dev/null || true
|
||||
bounded_wait "$monitor_pid" 45
|
||||
# Cancellation must restore safely even when SS failure requested retry.
|
||||
case "$waited_code" in 0|143) ;; *) echo "error: monitor cleanup status $waited_code" >&2; cleanup_code=78 ;; esac
|
||||
[ "$join_timed_out" -eq 0 ] || { echo 'error: monitor join timeout' >&2; cleanup_code=78; }
|
||||
# SIGTERM may have arrived before the launch acknowledgement was read.
|
||||
capture_monitor_session || cleanup_code=78
|
||||
monitor_pid=
|
||||
fi
|
||||
# Join/sweep scans even if the monitor was already reaped after SIGKILL.
|
||||
# A failed sweep vetoes reconciliation and all daemon script sourcing.
|
||||
if ! cleanup_session; then
|
||||
cleanup_code=78
|
||||
zapret_started=0
|
||||
fi
|
||||
# Dedicated PID/network namespace: no host/other-target stop.
|
||||
if [ "$zapret_started" -eq 1 ]; then
|
||||
# Stop also sources config: clear trust before reconciliation, never
|
||||
@@ -147,8 +189,21 @@ timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || exit 78
|
||||
# fork a monitor we already need to cancel: an early TERM can race interpreter
|
||||
# startup before its cancellation handler exists. Clean this cycle directly.
|
||||
{ kill -0 "$server_pid" 2>/dev/null && kill -0 "$local_pid" 2>/dev/null; } || return 75
|
||||
/usr/local/bin/zapret-autotune managed monitor &
|
||||
rm -f "$config_dir/monitor-ready"
|
||||
setsid /bin/sh /opt/managed-target-supervision.sh --monitor "$config_dir" /usr/local/bin/zapret-autotune managed monitor &
|
||||
monitor_pid=$!
|
||||
monitor_ack_deadline=$(($(date +%s) + 5))
|
||||
while [ ! -e "$config_dir/monitor-ready" ]; do
|
||||
if ! kill -0 "$monitor_pid" 2>/dev/null || [ "$(date +%s)" -ge "$monitor_ack_deadline" ]; then
|
||||
signal_tree "$monitor_pid" KILL
|
||||
exit 78
|
||||
fi
|
||||
done
|
||||
capture_monitor_session || exit 78
|
||||
if proc_identity "$monitor_pid" && [ "$proc_sid" != "$monitor_sid" ]; then
|
||||
monitor_sid=
|
||||
exit 78
|
||||
fi
|
||||
while kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do
|
||||
pause 1
|
||||
done
|
||||
|
||||
Reference in new issue
Block a user