fix(runtime): keep supervision and absence retries in container
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
226d93e2aa
commit
3f9fd2f36f
21 files changed
+1056
-137
No files matched your search
+100
-21
@@ -30,54 +30,97 @@ server_pid=
|
||||
local_pid=
|
||||
monitor_pid=
|
||||
zapret_started=0
|
||||
cooldown_pid=
|
||||
# Signal only descendants of an owned child, never the container/host namespace.
|
||||
# The monitor gets TERM itself first so it can join scans and restore bytes;
|
||||
# descendant signalling is reserved for SS shutdown and forced watchdog cleanup.
|
||||
. /opt/managed-target-supervision.sh
|
||||
pause() {
|
||||
sleep "$1" </dev/null >/dev/null 2>&1 & cooldown_pid=$!
|
||||
wait "$cooldown_pid"
|
||||
cooldown_pid=
|
||||
}
|
||||
# Wait with a watchdog: never let an unresponsive child exceed Docker's grace.
|
||||
bounded_wait() {
|
||||
child=$1
|
||||
seconds=$2
|
||||
(
|
||||
sleep "$seconds" & sleeper=$!
|
||||
trap 'kill "$sleeper" 2>/dev/null || true; exit 0' TERM INT
|
||||
wait "$sleeper"
|
||||
kill -KILL "$child" 2>/dev/null || true
|
||||
) </dev/null >/dev/null 2>&1 &
|
||||
rm -f "$config_dir/join-timeout" "$config_dir/join-ready"
|
||||
# Fresh interpreter: a forked shell can inherit a deferred EXIT/TERM trap
|
||||
# context and never run its cancellation handler during PID 1 shutdown.
|
||||
/bin/sh /opt/managed-target-supervision.sh --watchdog "$child" "$seconds" "$config_dir" </dev/null >/dev/null 2>&1 &
|
||||
watchdog=$!
|
||||
# A fast-exiting child must not cancel the watchdog before its trap exists.
|
||||
# Fail closed if exec/setup fails before acknowledgement; never wait forever.
|
||||
ack_deadline=$(($(date +%s) + 5))
|
||||
while [ ! -e "$config_dir/join-ready" ]; do
|
||||
if ! kill -0 "$watchdog" 2>/dev/null || [ "$(date +%s)" -ge "$ack_deadline" ]; then
|
||||
: > "$config_dir/join-timeout"
|
||||
signal_tree "$watchdog" KILL
|
||||
signal_tree "$child" KILL
|
||||
break
|
||||
fi
|
||||
done
|
||||
waited_code=0
|
||||
wait "$child" 2>/dev/null || waited_code=$?
|
||||
kill -TERM "$watchdog" 2>/dev/null || true
|
||||
wait "$watchdog" 2>/dev/null || true
|
||||
join_timed_out=0
|
||||
[ ! -e "$config_dir/join-timeout" ] || join_timed_out=1
|
||||
}
|
||||
cleanup() {
|
||||
final_code=$?
|
||||
case "$final_code" in 0|64|75|78) ;; *) final_code=78 ;; esac
|
||||
trap - EXIT INT TERM
|
||||
cleanup_cycle() {
|
||||
cleanup_code=0
|
||||
# Let the monitor cancel/join scans and restore committed bytes BEFORE stop.
|
||||
if [ -n "$monitor_pid" ]; then
|
||||
kill -TERM "$monitor_pid" 2>/dev/null || true
|
||||
bounded_wait "$monitor_pid" 60
|
||||
bounded_wait "$monitor_pid" 45
|
||||
# Cancellation must restore safely even when SS failure requested retry.
|
||||
case "$waited_code" in 0|143) ;; *) final_code=78 ;; esac
|
||||
case "$waited_code" in 0|143) ;; *) echo "error: monitor cleanup status $waited_code" >&2; cleanup_code=78 ;; esac
|
||||
[ "$join_timed_out" -eq 0 ] || { echo 'error: monitor join timeout' >&2; cleanup_code=78; }
|
||||
monitor_pid=
|
||||
fi
|
||||
# Dedicated PID/network namespace: no host/other-target stop.
|
||||
if [ "$zapret_started" -eq 1 ]; then
|
||||
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop || final_code=78
|
||||
# Stop also sources config: clear trust before reconciliation, never
|
||||
# source unknown bytes left by a crashed monitor, even for shutdown.
|
||||
zapret_started=0
|
||||
if timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile; then
|
||||
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop || cleanup_code=78
|
||||
else
|
||||
cleanup_code=78
|
||||
fi
|
||||
fi
|
||||
for pid in "$local_pid" "$server_pid"; do
|
||||
[ -z "$pid" ] || kill "$pid" 2>/dev/null || true
|
||||
[ -z "$pid" ] || signal_tree "$pid" TERM
|
||||
done
|
||||
for pid in "$local_pid" "$server_pid"; do
|
||||
if [ -n "$pid" ]; then
|
||||
bounded_wait "$pid" 5
|
||||
# Its original failure triggered supervision; only forced shutdown
|
||||
# is a cleanup failure, not the already-dead Shadowsocks status.
|
||||
[ "$waited_code" -ne 137 ] || final_code=78
|
||||
[ "$join_timed_out" -eq 0 ] || { echo 'error: Shadowsocks join timeout' >&2; cleanup_code=78; }
|
||||
fi
|
||||
done
|
||||
server_pid=
|
||||
local_pid=
|
||||
return "$cleanup_code"
|
||||
}
|
||||
cleanup() {
|
||||
final_code=$?
|
||||
trap - EXIT
|
||||
# A caught (not ignored) signal lets watchdog children install TERM traps.
|
||||
trap ':' INT TERM
|
||||
if [ -n "$cooldown_pid" ]; then
|
||||
kill "$cooldown_pid" 2>/dev/null || true
|
||||
wait "$cooldown_pid" 2>/dev/null || true
|
||||
fi
|
||||
cleanup_cycle || final_code=78
|
||||
exit "$final_code"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 0' INT TERM
|
||||
|
||||
# Never invoke /opt/entrypoint.sh: zapret must start AFTER reconciliation.
|
||||
run_cycle() {
|
||||
ss-server -c "$config_dir/server.json" &
|
||||
server_pid=$!
|
||||
ss-local -c "$config_dir/local.json" &
|
||||
@@ -86,10 +129,10 @@ local_pid=$!
|
||||
# Bounded readiness, no network probe. Managed commands verify socket ownership.
|
||||
attempt=0
|
||||
until awk '$2 == "00000000:0438" && $4 == "0A" { found=1 } END { exit !found }' /proc/net/tcp; do
|
||||
{ kill -0 "$server_pid" && kill -0 "$local_pid"; } || exit 75
|
||||
{ kill -0 "$server_pid" && kill -0 "$local_pid"; } || return 75
|
||||
attempt=$((attempt + 1))
|
||||
[ "$attempt" -lt 30 ] || { echo 'error: target SOCKS listener timeout' >&2; exit 75; }
|
||||
sleep 1
|
||||
[ "$attempt" -lt 30 ] || { echo 'error: target SOCKS listener timeout' >&2; return 75; }
|
||||
pause 1
|
||||
done
|
||||
if [ ! -e "$ZAPRET_CONFIG" ]; then
|
||||
timeout -k 5 45 /usr/local/bin/zapret-autotune managed init || exit 78
|
||||
@@ -100,10 +143,14 @@ timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile || exit 78
|
||||
zapret_started=1
|
||||
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || exit 78
|
||||
|
||||
# A replacement SS child can fail during reconciliation/activation. Do not
|
||||
# fork a monitor we already need to cancel: an early TERM can race interpreter
|
||||
# startup before its cancellation handler exists. Clean this cycle directly.
|
||||
{ kill -0 "$server_pid" 2>/dev/null && kill -0 "$local_pid" 2>/dev/null; } || return 75
|
||||
/usr/local/bin/zapret-autotune managed monitor &
|
||||
monitor_pid=$!
|
||||
while kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do
|
||||
sleep 1
|
||||
pause 1
|
||||
done
|
||||
if ! kill -0 "$monitor_pid" 2>/dev/null; then
|
||||
code=0
|
||||
@@ -115,7 +162,39 @@ if ! kill -0 "$monitor_pid" 2>/dev/null; then
|
||||
64|75|78) ;;
|
||||
*) code=78 ;;
|
||||
esac
|
||||
exit "$code"
|
||||
return "$code"
|
||||
fi
|
||||
echo 'error: target child process exited' >&2
|
||||
exit 75
|
||||
return 75
|
||||
}
|
||||
|
||||
# Three starts per rolling six hours, including initial startup. PID 1 owns
|
||||
# the entire runtime cycle; cooldown is cancellable without any external manager.
|
||||
starts=
|
||||
backoff=15
|
||||
while :; do
|
||||
now=$(date +%s)
|
||||
recent=
|
||||
count=0
|
||||
for started in $starts; do
|
||||
if [ "$((now - started))" -lt 21600 ]; then
|
||||
recent="$recent $started"
|
||||
count=$((count + 1))
|
||||
fi
|
||||
done
|
||||
[ "$count" -lt 3 ] || { echo 'error: target restart budget exhausted' >&2; exit 78; }
|
||||
starts="$recent $now"
|
||||
code=0
|
||||
run_cycle || code=$?
|
||||
cleanup_cycle || code=78
|
||||
[ "$code" -eq 75 ] || exit "$code"
|
||||
# A long-running monitor may outlive the window; recheck at failure time.
|
||||
now=$(date +%s)
|
||||
retry_count=0
|
||||
for started in $starts; do
|
||||
[ "$((now - started))" -ge 21600 ] || retry_count=$((retry_count + 1))
|
||||
done
|
||||
[ "$retry_count" -lt 3 ] || { echo 'error: target restart budget exhausted' >&2; exit 78; }
|
||||
pause "$backoff"
|
||||
[ "$backoff" -ge 60 ] || backoff=$((backoff * 2))
|
||||
done
|
||||
Reference in new issue
Block a user