feat: add guarded periodic resource monitoring
Add checks-only scheduling, read-only status, private authenticated persistence and guarded PID1 lifecycle. Keep strategy selection/recovery and remote fault acceptance out of scope. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
cfce8170d7
commit
4efc066625
16 files changed
+2307
-103
No files matched your search
+139
-20
@@ -16,6 +16,8 @@ esac
|
||||
/usr/local/bin/zapret-autotune target-config-check || exit 64
|
||||
dns_mode=$(/usr/local/bin/zapret-autotune dns-forwarder mode) || exit 64
|
||||
dns_address=$(/usr/local/bin/zapret-autotune dns-forwarder address) || exit 64
|
||||
generic_mode=0
|
||||
[ -z "${ZAPRET_AUTOTUNE_RESOURCE_CONFIG-}" ] || generic_mode=1
|
||||
case "$dns_mode" in doh|native) ;; *) exit 64 ;; esac
|
||||
config_dir=/run/zapret-autotune-ss
|
||||
awk '$2 == "/run/zapret-autotune-ss" && $3 == "tmpfs" { found=1 } END { exit !found }' /proc/mounts || {
|
||||
@@ -40,10 +42,45 @@ monitor_pid=
|
||||
monitor_sid=
|
||||
zapret_started=0
|
||||
cooldown_pid=
|
||||
generic_cleanup_deadline=
|
||||
generic_cleanup_failed=0
|
||||
# Signal only descendants of an owned child, never the container/host namespace.
|
||||
# The monitor gets TERM itself first so it can join scans and restore bytes;
|
||||
# descendant signalling is reserved for SS shutdown and forced watchdog cleanup.
|
||||
. /opt/managed-target-supervision.sh
|
||||
# One budget per cleanup cycle, including ACKs, kill escalation and helper joins.
|
||||
# Ten seconds of Docker's 90s grace remain for coarse clocks and final reaping.
|
||||
cleanup_remaining() {
|
||||
cleanup_left=$((generic_cleanup_deadline - $(date +%s)))
|
||||
[ "$cleanup_left" -ge 0 ] || cleanup_left=0
|
||||
}
|
||||
cleanup_command() {
|
||||
if [ "$generic_mode" -eq 0 ]; then
|
||||
timeout -k 5 "$@"
|
||||
return $?
|
||||
fi
|
||||
cleanup_command_cap=$1
|
||||
shift
|
||||
cleanup_remaining
|
||||
# timeout's KILL grace is part of, not additional to, the shared budget.
|
||||
[ "$cleanup_left" -gt 1 ] || return 78
|
||||
cleanup_command_left=$((cleanup_left - 1))
|
||||
[ "$cleanup_command_cap" -le "$cleanup_command_left" ] || cleanup_command_cap=$cleanup_command_left
|
||||
timeout -k 1 "$cleanup_command_cap" "$@"
|
||||
}
|
||||
cleanup_reap() {
|
||||
reap_pid=$1
|
||||
reap_deadline=$2
|
||||
[ "$reap_deadline" -le "$((generic_cleanup_deadline + 2))" ] || reap_deadline=$((generic_cleanup_deadline + 2))
|
||||
child_joined=0
|
||||
while proc_identity "$reap_pid" && [ "$proc_state" != Z ]; do
|
||||
[ "$(date +%s)" -lt "$reap_deadline" ] || return 0
|
||||
pause 1
|
||||
done
|
||||
waited_code=0
|
||||
wait "$reap_pid" 2>/dev/null || waited_code=$?
|
||||
child_joined=1
|
||||
}
|
||||
pause() {
|
||||
sleep "$1" 8<&- 9<&- </dev/null >/dev/null 2>&1 & cooldown_pid=$!
|
||||
wait "$cooldown_pid"
|
||||
@@ -57,6 +94,9 @@ cleanup_session() {
|
||||
for session_signal in TERM KILL; do
|
||||
signal_session "$monitor_sid" "$session_signal" || return 78
|
||||
session_deadline=$(($(date +%s) + 3))
|
||||
if [ "$generic_mode" -eq 1 ] && [ "$session_deadline" -gt "$generic_cleanup_deadline" ]; then
|
||||
session_deadline=$generic_cleanup_deadline
|
||||
fi
|
||||
while :; do
|
||||
signal_session "$monitor_sid" CHECK || return 78
|
||||
if [ "$session_live" -eq 0 ]; then
|
||||
@@ -85,6 +125,19 @@ capture_monitor_session() {
|
||||
bounded_wait() {
|
||||
child=$1
|
||||
seconds=$2
|
||||
if [ "$generic_mode" -eq 1 ]; then
|
||||
cleanup_remaining
|
||||
# Reserve ACK setup (5s) and post-KILL reaping (2s) before granting wait.
|
||||
if [ "$cleanup_left" -le 7 ]; then
|
||||
signal_tree "$child" KILL
|
||||
cleanup_reap "$child" "$(($(date +%s) + 2))"
|
||||
waited_code=${waited_code-137}
|
||||
join_timed_out=1
|
||||
return 0
|
||||
fi
|
||||
cleanup_wait_left=$((cleanup_left - 7))
|
||||
[ "$seconds" -le "$cleanup_wait_left" ] || seconds=$cleanup_wait_left
|
||||
fi
|
||||
rm -f "$config_dir/join-timeout" "$config_dir/join-ready"
|
||||
# Fresh interpreter: a forked shell can inherit a deferred EXIT/TERM trap
|
||||
# context and never run its cancellation handler during PID 1 shutdown.
|
||||
@@ -93,6 +146,9 @@ bounded_wait() {
|
||||
# A fast-exiting child must not cancel the watchdog before its trap exists.
|
||||
# Fail closed if exec/setup fails before acknowledgement; never wait forever.
|
||||
ack_deadline=$(($(date +%s) + 5))
|
||||
if [ "$generic_mode" -eq 1 ] && [ "$ack_deadline" -gt "$generic_cleanup_deadline" ]; then
|
||||
ack_deadline=$generic_cleanup_deadline
|
||||
fi
|
||||
while [ ! -e "$config_dir/join-ready" ]; do
|
||||
if ! kill -0 "$watchdog" 2>/dev/null || [ "$(date +%s)" -ge "$ack_deadline" ]; then
|
||||
: > "$config_dir/join-timeout"
|
||||
@@ -102,13 +158,39 @@ bounded_wait() {
|
||||
fi
|
||||
done
|
||||
waited_code=0
|
||||
wait "$child" 2>/dev/null || waited_code=$?
|
||||
if [ "$generic_mode" -eq 1 ]; then
|
||||
cleanup_join_deadline=$(($(date +%s) + seconds))
|
||||
[ "$cleanup_join_deadline" -le "$generic_cleanup_deadline" ] || cleanup_join_deadline=$generic_cleanup_deadline
|
||||
cleanup_reap "$child" "$cleanup_join_deadline"
|
||||
if [ "$child_joined" -eq 0 ]; then
|
||||
: > "$config_dir/join-timeout"
|
||||
signal_tree "$child" KILL
|
||||
cleanup_reap "$child" "$(($(date +%s) + 2))"
|
||||
fi
|
||||
joined_service=$child_joined
|
||||
else
|
||||
wait "$child" 2>/dev/null || waited_code=$?
|
||||
fi
|
||||
kill -TERM "$watchdog" 2>/dev/null || true
|
||||
wait "$watchdog" 2>/dev/null || true
|
||||
if [ "$generic_mode" -eq 1 ]; then
|
||||
service_waited_code=$waited_code
|
||||
cleanup_reap "$watchdog" "$(($(date +%s) + 2))"
|
||||
if [ "$child_joined" -eq 0 ]; then
|
||||
signal_tree "$watchdog" KILL
|
||||
cleanup_reap "$watchdog" "$(($(date +%s) + 2))"
|
||||
fi
|
||||
child_joined=$joined_service
|
||||
waited_code=$service_waited_code
|
||||
else
|
||||
wait "$watchdog" 2>/dev/null || true
|
||||
fi
|
||||
join_timed_out=0
|
||||
[ ! -e "$config_dir/join-timeout" ] || join_timed_out=1
|
||||
}
|
||||
cleanup_cycle() {
|
||||
if [ "$generic_mode" -eq 1 ] && [ -z "$generic_cleanup_deadline" ]; then
|
||||
generic_cleanup_deadline=$(($(date +%s) + 80))
|
||||
fi
|
||||
cleanup_code=0
|
||||
# Let the monitor cancel/join scans and restore committed bytes BEFORE stop.
|
||||
if [ -n "$monitor_pid" ]; then
|
||||
@@ -133,8 +215,8 @@ cleanup_cycle() {
|
||||
# Stop also sources config: clear trust before reconciliation, never
|
||||
# source unknown bytes left by a crashed monitor, even for shutdown.
|
||||
zapret_started=0
|
||||
if timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile; then
|
||||
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop || cleanup_code=78
|
||||
if cleanup_command 45 /usr/local/bin/zapret-autotune managed reconcile; then
|
||||
cleanup_command 30 /opt/zapret2/init.d/sysv/zapret2 stop || cleanup_code=78
|
||||
else
|
||||
cleanup_code=78
|
||||
fi
|
||||
@@ -142,9 +224,11 @@ cleanup_cycle() {
|
||||
for pid in "$local_pid" "$server_pid"; do
|
||||
[ -z "$pid" ] || signal_tree "$pid" TERM
|
||||
done
|
||||
ss_joined=1
|
||||
for pid in "$local_pid" "$server_pid"; do
|
||||
if [ -n "$pid" ]; then
|
||||
bounded_wait "$pid" 5
|
||||
if [ "$generic_mode" -eq 1 ] && [ "$child_joined" -eq 0 ]; then ss_joined=0; fi
|
||||
# Its original failure triggered supervision; only forced shutdown
|
||||
# is a cleanup failure, not the already-dead Shadowsocks status.
|
||||
[ "$join_timed_out" -eq 0 ] || { echo 'error: Shadowsocks join timeout' >&2; cleanup_code=78; }
|
||||
@@ -153,15 +237,27 @@ cleanup_cycle() {
|
||||
server_pid=
|
||||
local_pid=
|
||||
# DNS remains available through monitor rollback, zapret stop and SS joins.
|
||||
if [ -n "$dns_pid" ]; then
|
||||
if [ -n "$dns_pid" ] && [ "$ss_joined" -eq 1 ]; then
|
||||
kill -TERM "$dns_pid" 2>/dev/null || true
|
||||
bounded_wait "$dns_pid" 5
|
||||
[ "$join_timed_out" -eq 0 ] || { echo 'error: DNS join timeout' >&2; cleanup_code=78; }
|
||||
/usr/local/bin/zapret-autotune dns-forwarder readiness-cleanup "$dns_pid" || cleanup_code=78
|
||||
dns_pid=
|
||||
if [ "$generic_mode" -eq 1 ] && [ "$child_joined" -eq 0 ]; then
|
||||
cleanup_code=78
|
||||
elif [ "$generic_mode" -eq 1 ]; then
|
||||
cleanup_command 5 /usr/local/bin/zapret-autotune dns-forwarder readiness-cleanup "$dns_pid" || cleanup_code=78
|
||||
else
|
||||
/usr/local/bin/zapret-autotune dns-forwarder readiness-cleanup "$dns_pid" || cleanup_code=78
|
||||
fi
|
||||
if [ "$generic_mode" -eq 0 ] || [ "$child_joined" -eq 1 ]; then dns_pid=; fi
|
||||
fi
|
||||
if [ -z "$dns_pid" ]; then
|
||||
if [ "$dns_anchor_file" -eq 1 ]; then exec 9<&-; dns_anchor_file=0; fi
|
||||
if [ "$dns_anchor_dir" -eq 1 ]; then exec 8<&-; dns_anchor_dir=0; fi
|
||||
fi
|
||||
if [ "$generic_mode" -eq 1 ]; then
|
||||
[ "$cleanup_code" -eq 0 ] || generic_cleanup_failed=1
|
||||
generic_cleanup_deadline=
|
||||
fi
|
||||
if [ "$dns_anchor_file" -eq 1 ]; then exec 9<&-; dns_anchor_file=0; fi
|
||||
if [ "$dns_anchor_dir" -eq 1 ]; then exec 8<&-; dns_anchor_dir=0; fi
|
||||
return "$cleanup_code"
|
||||
}
|
||||
cleanup() {
|
||||
@@ -174,6 +270,7 @@ cleanup() {
|
||||
wait "$cooldown_pid" 2>/dev/null || true
|
||||
fi
|
||||
cleanup_cycle || final_code=78
|
||||
[ "$generic_cleanup_failed" -eq 0 ] || final_code=78
|
||||
exit "$final_code"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
@@ -232,19 +329,21 @@ until awk '$2 == "00000000:0438" && $4 == "0A" { found=1 } END { exit !found }'
|
||||
done
|
||||
dns_alive || return 75
|
||||
# Reject unsafe operator requests before init/reconcile can mutate owned state.
|
||||
timeout -k 5 45 /usr/local/bin/zapret-autotune managed experimental-request-check || exit 78
|
||||
if [ "$generic_mode" -eq 0 ]; then
|
||||
timeout -k 5 45 /usr/local/bin/zapret-autotune managed experimental-request-check || return 78
|
||||
fi
|
||||
if [ ! -e "$ZAPRET_CONFIG" ]; then
|
||||
timeout -k 5 45 /usr/local/bin/zapret-autotune managed init || exit 78
|
||||
timeout -k 5 45 /usr/local/bin/zapret-autotune managed init || return 78
|
||||
fi
|
||||
# Never activate pending or unknown bytes. Any reconciliation failure is fatal,
|
||||
# not a transient (75) restart request. Reload restores known bytes first.
|
||||
timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile || exit 78
|
||||
timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile || return 78
|
||||
zapret_started=1
|
||||
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || exit 78
|
||||
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || return 78
|
||||
|
||||
# Private operator request, consumed only after a successful guarded transaction.
|
||||
# Never suppress qualification gates globally or race the owned monitor.
|
||||
if [ -e "${ZAPRET_CONFIG%/*}/.autotune-experimental-suffix-request" ] || [ -L "${ZAPRET_CONFIG%/*}/.autotune-experimental-suffix-request" ]; then
|
||||
if [ "$generic_mode" -eq 0 ] && { [ -e "${ZAPRET_CONFIG%/*}/.autotune-experimental-suffix-request" ] || [ -L "${ZAPRET_CONFIG%/*}/.autotune-experimental-suffix-request" ]; }; then
|
||||
timeout -k 5 120 /usr/local/bin/zapret-autotune managed apply --googlevideo-experiment native-circular --googlevideo-scope experimental_googlevideo_suffix --googlevideo-request startup || exit 78
|
||||
fi
|
||||
|
||||
@@ -259,13 +358,13 @@ monitor_ack_deadline=$(($(date +%s) + 5))
|
||||
while [ ! -e "$config_dir/monitor-ready" ]; do
|
||||
if ! kill -0 "$monitor_pid" 2>/dev/null || [ "$(date +%s)" -ge "$monitor_ack_deadline" ]; then
|
||||
signal_tree "$monitor_pid" KILL
|
||||
exit 78
|
||||
return 78
|
||||
fi
|
||||
done
|
||||
capture_monitor_session || exit 78
|
||||
capture_monitor_session || return 78
|
||||
if proc_identity "$monitor_pid" && [ "$proc_sid" != "$monitor_sid" ]; then
|
||||
monitor_sid=
|
||||
exit 78
|
||||
return 78
|
||||
fi
|
||||
while dns_alive && kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do
|
||||
pause 1
|
||||
@@ -300,11 +399,29 @@ while :; do
|
||||
count=$((count + 1))
|
||||
fi
|
||||
done
|
||||
[ "$count" -lt 3 ] || { echo 'error: target restart budget exhausted' >&2; exit 78; }
|
||||
if [ "$count" -ge 3 ]; then
|
||||
if [ "$generic_mode" -eq 1 ]; then
|
||||
echo 'BLOCKED: generic target restart budget; checks paused' >&2
|
||||
pause 60
|
||||
continue
|
||||
fi
|
||||
echo 'error: target restart budget exhausted' >&2; exit 78
|
||||
fi
|
||||
starts="$recent $now"
|
||||
code=0
|
||||
run_cycle || code=$?
|
||||
cleanup_cycle || code=78
|
||||
if ! cleanup_cycle; then
|
||||
if [ "$generic_mode" -eq 1 ]; then
|
||||
echo 'BLOCKED: generic target unsafe cleanup; operator repair/restart required' >&2
|
||||
while :; do pause 60; done
|
||||
fi
|
||||
code=78
|
||||
fi
|
||||
if [ "$generic_mode" -eq 1 ] && [ "$code" -eq 78 ]; then
|
||||
echo 'BLOCKED: generic target trust/startup failure; checks paused' >&2
|
||||
pause 60
|
||||
continue
|
||||
fi
|
||||
[ "$code" -eq 75 ] || exit "$code"
|
||||
# A long-running monitor may outlive the window; recheck at failure time.
|
||||
now=$(date +%s)
|
||||
@@ -312,7 +429,9 @@ while :; do
|
||||
for started in $starts; do
|
||||
[ "$((now - started))" -ge 21600 ] || retry_count=$((retry_count + 1))
|
||||
done
|
||||
[ "$retry_count" -lt 3 ] || { echo 'error: target restart budget exhausted' >&2; exit 78; }
|
||||
if [ "$retry_count" -ge 3 ] && [ "$generic_mode" -eq 0 ]; then
|
||||
echo 'error: target restart budget exhausted' >&2; exit 78
|
||||
fi
|
||||
pause "$backoff"
|
||||
[ "$backoff" -ge 60 ] || backoff=$((backoff * 2))
|
||||
done
|
||||
Reference in new issue
Block a user