diff --git a/.dockerignore b/.dockerignore index a277402..cfd8f63 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,4 @@ !autotune-entrypoint.sh !Dockerfile.target !managed-target-entrypoint.sh +!managed-target-supervision.sh diff --git a/Dockerfile.target b/Dockerfile.target index 23ec1a6..200f524 100644 --- a/Dockerfile.target +++ b/Dockerfile.target @@ -10,6 +10,7 @@ FROM vernette/ss-zapret2@sha256:ba158180f27cd71a291ee521c291490374fd3fcf2f81eb14 USER root COPY --from=build /out/zapret-autotune /usr/local/bin/zapret-autotune COPY managed-target-entrypoint.sh /opt/managed-target-entrypoint.sh +COPY managed-target-supervision.sh /opt/managed-target-supervision.sh # Retain the pinned image's fakes and Lua; never mount host replacements. RUN if [ ! -e /opt/zapret2/lua ] && [ ! -L /opt/zapret2/lua ]; then \ cp -a /opt/zapret2/lua.dist /opt/zapret2/lua; \ diff --git a/MANAGED_TARGET.md b/MANAGED_TARGET.md index 82194e0..52c17ba 100644 --- a/MANAGED_TARGET.md +++ b/MANAGED_TARGET.md @@ -162,10 +162,10 @@ not rescanned merely because the process restarted. Entrypoint ordering is SS children → listener → fresh managed init if needed → managed reconcile → zapret start → one installed `/usr/local/bin/zapret-autotune managed monitor` child. It supervises -all three children; monitor exit is fatal even if it exits zero unexpectedly. +all three children; unexpected monitor exit is not a successful shutdown. SIGTERM/SIGINT cancels and waits the monitor before target-local zapret stop and SS child termination/wait. Existing targets and Xray remain unchanged. Deployment -allows 120s Compose shutdown grace and 180s systemd stop timeout for bounded +allows 240s Compose shutdown grace for bounded monitor cancellation/restoration and child joins. Code integration is not image/routing/signal live verification. @@ -185,7 +185,7 @@ This is a standalone **new isolated target** candidate, not a production-approve deployment and not an integration into existing zapret/Xray. Keep the exact `managed-test` role, ID, marker, mount, config, DB and owned-listener guards above. Do not change the role to `production`, adopt old state, share an old network -namespace, or relax a refusal. Provisioning and optional service installation +namespace, or relax a refusal. Container provisioning instructions are described in [deploy/managed-target/README.md](deploy/managed-target/README.md). ### Build once, promote the same image @@ -227,8 +227,8 @@ docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/co docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml pull target ``` -The supplied systemd unit uses this same `.env` and starts with `--no-build`; -no image-selection override or unit edit is needed. Keep the selected digest and +Start the selected container image with Compose `up --no-build`; +no host service installation is required. Keep the selected digest and private configuration together for upgrades and rollback. Never run a build against the production digest selector; build/tag separately as shown above. @@ -268,31 +268,55 @@ existing Xray deployment as part of bootstrap. ### Supervision and failure handling -Compose restart is disabled. Optional native systemd runs **foreground** Compose -with `--no-build --abort-on-container-exit --exit-code-from target`; do not also -run detached Compose for that project. The unit requests restart only for exit -75 via `Restart=no` plus `RestartForceExitStatus=75`, bounded by -`StartLimitBurst=3` in six hours (starts, including the initial -start, not three additional retries). Compose stop grace is 120s, systemd 180s. +The container entrypoint is PID 1 and owns all technical child restart control. +After a retryable monitor/SS failure it joins children, reconciles trusted state +and stops target-local zapret before starting a new runtime cycle in the **same +container**. Retries use cancellable backoff (15s, 30s, capped at 60s) and at most +three runtime starts per rolling six hours, including the initial start. Exhaustion +stops fail-closed with 78; configuration, safety, DB, journal, rollback or cleanup +errors never authorize retry. No new dependency, host service installation or +external process manager is required. SIGTERM/SIGINT cancels backoff or the running +monitor, waits for bounded restoration and joins children before exit. + +Compose restart remains disabled: a blanket container restart could loop over a +safety refusal or reset the in-container retry budget. Compose stop grace is 240s: +the sequential cleanup budget is 140s (45s monitor join, 45s reconciliation plus +5s kill grace, 30s zapret stop plus 5s kill grace, and two 5s SS joins), with room +for a deferred 45s startup command plus kill grace and scheduling overhead. +The join watchdog runs in a fresh shell and is cancelled when its child finishes; +graceful shutdown does not wait out the watchdog's full deadline. +Container-platform lifecycle, creation/recreation and host-boot policy belong to +the deployment operator, not an external manager supplied by this project. | Target exit | Operator meaning | | --- | --- | | `64` | Invalid configuration; fix settings before starting | | `78` | Safety/DB/journal/rollback/cleanup failure; investigate, no automatic restart | -| `75` | Technical failure eligible for bounded restart only after safe cleanup | +| `75` | Internal technical-cycle result; PID 1 retries only after safe cleanup, within its budget | | `0` | Graceful signal shutdown; no automatic restart | Bootstrap/recovery has a fixed **90-minute whole-job deadline**, plus an independent **30-second restoration reserve**. This is a catastrophic technical ceiling, not a normal candidate-check timeout and not an operator tunable. Valid -nonpassing candidates/unresolved GGC are normal recovery outcomes governed by -the persisted threshold/cooldown, not a reason to restart the container. Failed +nonpassing candidates/unresolved GGC are normal recovery outcomes, not a reason +to restart the container. Genuine **no qualified strategy** from validated scan/check +evidence persists a recovery wait of at least **two hours** in SQLite, including +empty bootstrap and frontend/GGC exhaustion. The same container stays alive and +continues light periodic Player API probes; it does not scan or apply during that +wait. Startup respects the durable expiry, and recovery resumes after expiry under +the normal failure-threshold policy. In both fresh bootstrap and periodic recovery, +the delay is the greater of two hours and the configured recovery cooldown; +a longer configured cooldown is not shortened. +Other ordinary recovery outcomes retain the configured cooldown (default **30m**). +Malformed/incomplete reports, runner errors and unsafe state are not strategy +absence and must not enter this wait. The two-hour minimum absence policy is not a +new operator setting, technical retry backoff, or the **90-minute watchdog**. Failed restoration or unsafe state vetoes retry: timeout exit 75 requires the job's own deadline and only timeout/cancellation-related errors, with no joined cleanup, DB or safety failure. Unexpected monitor/SS failures are separately technical -only if shutdown cleanup succeeds. Shutdown joins the monitor for up to 60s, -target-local zapret stop gets 30s then forced kill after five more, and SS children -get 5s each; failure to join/restore is fail-closed. The reserve is not extra +only if shutdown cleanup succeeds. Shutdown uses bounded monitor join, +guarded reconciliation, target-local zapret stop and SS child joins; +failure to join/restore is fail-closed. The reserve is not extra scanning time. The 90-minute budget is shared by the whole bootstrap/recovery job, not renewed per candidate or phase. @@ -304,7 +328,7 @@ be reported as complete playback health. ### Upgrade, rollback and operator recovery 1. Record the current image digest and private deployment configuration. Stop - the service (or foreground Compose), wait for complete cleanup, and verify + the target with Compose, wait for complete cleanup, and verify the target is stopped. If cleanup fails, investigate before upgrading. 2. With the target stopped, make a root-only backup of the **entire** state directory (including DB sidecars, journal, config, identity and lock file), @@ -329,16 +353,16 @@ is rolled back to qualified committed bytes before any reload/start; fresh baseline and known committed state are idempotent. Unknown/legacy state, drift, missing qualification or failed rollback blocks startup with a non-retry exit. -On drift, unreconcilable pending transaction, rollback failure, or exhausted systemd start -limit, leave state intact and service stopped. Diagnose using sanitized status +On drift, unreconcilable pending transaction, rollback failure, or exhausted PID 1 retry +budget, leave state intact and the container stopped. Diagnose using sanitized status and private artifacts; repair the actual provisioning/configuration issue or -restore a known consistent backup. Only then manually start (and reset the -systemd failed/start-limit state if needed). There is no automatic wipe or +restore a known consistent backup. Only then manually start the target with Compose. +There is no automatic wipe or automatic adoption path. Rollout acceptance remains pending real isolated-target restart/deadline tests: empty bootstrap, saved-first recovery, exact-host GGC rotation, image binding, NFQUEUE activation/rollback, SIGTERM during downtime, deadline restoration, -retry limits/exit propagation through foreground Compose/systemd, persistent +in-container retry limits/backoff, durable two-hour absence wait and resumption, persistent recreation and backup restore. Perform these on the new target only; local unit or static checks cannot establish production readiness. diff --git a/PROGRESS.md b/PROGRESS.md index 6d3d259..61fda00 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,8 +1,21 @@ # Прогресс реализации -Срез на **2026-10-10**. Этот файл описывает фактическое состояние кода, а не весь объём требований из `Requirements.md` и `Architecture.md`. +Срез на **2026-10-11**. Этот файл описывает фактическое состояние кода, а не весь объём требований из `Requirements.md` и `Architecture.md`. -## Готово +## Текущий статус + +- **2026-10-11 container-owned control (pending working-tree changes):** all technical child restart control belongs to the entrypoint/PID 1 inside the same container, not host systemd. Safe cleanup precedes each retry; three runtime starts per rolling six hours include the initial start, with cancellable 15s/30s backoff capped at 60s. Configuration/safety/DB/journal/rollback/cleanup failures and exhausted retry budget stop fail-closed. SIGTERM cancels and joins the monitor, restores safely, then stops target-local zapret and SS children. No additional dependency or host service installation is required. Compose keeps `restart: "no"` to avoid safety loops and now allows 240s stop grace: 140s sequential cleanup including 45s monitor join, plus room for a deferred 45s startup command/kill grace and overhead. A fresh-shell join watchdog is cancelled promptly on child completion; graceful shutdown never needs its full timeout. Container-platform lifecycle and host-boot policy remain deployment-operator responsibilities. +- **2026-10-11 strategy absence (pending working-tree changes):** genuine validated no-qualified-strategy outcomes in bootstrap/recovery persist a wait of at least two hours in SQLite. Both fresh bootstrap and periodic recovery use the greater of two hours and the configured recovery cooldown, preserving longer settings. The same container continues light probes without scans/apply during the wait; startup respects the persisted expiry, and normal threshold-driven recovery resumes after expiry. Other ordinary outcomes keep the default 30m cooldown. Malformed reports/runner/safety errors are not absence. The separate 90-minute whole-job watchdog plus 30-second restoration reserve remains an abnormal technical ceiling, not a normal no-strategy result or the absence wait. +- Root operator docs now describe this contract, preserving immutable tagged/digest promotion, BuildKit/pinned builds, private `.env`, whole stopped-state backup/restore and exact legacy `managed-test` guards for a new isolated target only. No old target adoption or Xray changes. New runtime changes still need their owners' focused checks and isolated live acceptance (retry/backoff, SIGTERM, durable absence expiry/resume); this documentation update does not claim those checks were run. + +## История реализации (не текущие инструкции) + +The dated facts below are historical snapshots. In particular, host-systemd unit +installation/restart/start-limit guidance and its acceptance tests are **superseded** +by container-owned control above; they impose no current deployment requirement. +Earlier “not deployed/not verified” snapshots do not override later recorded +isolated-target acceptance. Production acceptance of the new supervision/absence +policy remains pending. - **2026-10-11 production-candidate update (remote checks dated 2026-10-10 UTC):** `ed4298d` adds one 90-minute budget per managed bootstrap/recovery job, shared across phases. Watchdog expiration is abnormal/fatal, never ordinary nonpass/cooldown continuation; clean timeout permits technical exit 75 only after worker join and bounded 30-second restoration. Any guard/DB/rollback/cleanup or mixed unknown error vetoes restart (78); configuration errors use 64, graceful shutdown 0. Independent review reproduced and closed mixed-timeout, fatal-discovery and fatal-health/deadline races. Preferred frontend seed is now the layout-safe alternative verified in prior 60/60 requests; cached definitions are not rewritten. - **2026-10-10 restart/startup hardening:** optional `deploy/managed-target/zapret-autotune.service` uses foreground Compose, no blanket Docker restart, systemd allowlisted 75 with bounded starts (3 per 6h); shutdown grace 120s / unit stop budget 180s. `61e6e00` adds guarded fixed-DB `managed reconcile` before initial activation; init/reconcile/start are bounded (45/45/30s plus 5s kill grace), all startup refusals map to 78. Unknown state is not sourced even by cleanup. Actual systemd tests: technical 75 retries bounded, 78 does not retry; killing monitor in final image caused one whole-target restart, preserving both groups and Player API OK. Permanent production service was not installed/enabled; transient acceptance units were stopped. @@ -65,18 +78,18 @@ - На одном найденном GGC hostname текущая статическая стратегия `fakeddisorder:pos=midsld:tcp_flags_set=SYN` прошла data-transfer check 3/3 для TLS 1.2 и TLS 1.3. Frontend-комбинация на этом GGC hostname не прошла; frontend и GGC требуют отдельных профилей. - Отдельный Selectel speedtest с SNI `googlevideo` был проверен экспериментально, но **не используется** для ранжирования: принято решение доверять TLS/data-transfer результатам `blockcheckw`. -## Выполняется сейчас +## Исторические незавершённые работы (на момент записи) - Scan и shortlist check завершены, контейнер `zapret-autotune-googlevideo-scan` остановился с кодом 0. Проверка exact combined args прошла; `tune googlevideo` не запускался end-to-end и результаты вручную в SQLite не импортировались. - Новый отдельный `tune googlevideo` proposal-only и не меняет frontend-only `tune youtube`; удалённый end-to-end запуск команды ещё не выполнен. -## Не готово +## Исторические ограничения (на момент записи) - Проверка реального YouTube video-transfer/SABR. Прямой GET `serverAbrStreamingUrl` возвращает `sabr.malformed_config`; hostname discovery и `blockcheckw` data-transfer checks не равны проверке воспроизведения видео. - Guarded managed CLI/monitor интегрированы внутри designated target: transaction core, bootstrap/recovery callback и deployment definition есть, но нет proven image/runtime compatibility, выполненного container provisioning и реально запущенного managed target. Одноразовый Player API health сам не запускает live automation. Цель: новый отдельный prod-like zapret2-ss контейнер только в тестовой среде; существующий контейнер не трогать. Стабильный настраиваемый loopback SOCKS для Xray; Xray config вне scope. Перед runtime пройти route/rollback gates. - Полный сервис из требований: YAML-конфигурация сервисов, endpoint-история/покрытие, daemon/периодический мониторинг, API/метрики и multi-service режим. -## Открытые ограничения и следующие шаги +## Исторические следующие шаги (текущие gates указаны выше) 1. Проверить новый `tune googlevideo` и `recover googlevideo` end-to-end на удалённом сервере; отдельно решить, как импортировать или переиспользовать уже полученные scan результаты без повторного перебора. 2. Воспроизводимое описание отдельного prod-like target добавлено в `deploy/managed-target`; orchestrator должен проверить Compose syntax. Build/run отложены. До запуска проверить baseline-конфиг и NFQUEUE внутри container network namespace; текущие mounts, сеть, порты и конфиг не переиспользовать. diff --git a/README.md b/README.md index 8036f9f..e48e72c 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,27 @@ The standalone deployment candidate is a **new isolated target**, never an upgrade/adoption of an existing zapret or Xray service. The legacy `managed-test` role and fixed identity guards remain intentional, including for prod-like runs. See [release, configuration, upgrade and rollback guidance](MANAGED_TARGET.md#production-preparation) -and the [deployment/service instructions](deploy/managed-target/README.md). +and the [container deployment instructions](deploy/managed-target/README.md). Pinned builds and supervision are preparation, not production acceptance: real restart/deadline tests and end-to-end target validation remain rollout gates. Select an immutable managed-target digest with `ZAPRET_AUTOTUNE_IMAGE` in the private deployment `.env`; tags/local defaults are only build/test candidates. Bootstrap/recovery shares a 90-minute whole-job deadline plus a separate 30-second restoration reserve, an abnormal ceiling rather than normal candidate failure. -Timeout exit 75 requires clean timeout/cancellation only; cleanup/safety errors -veto retry. Optional foreground systemd uses `Restart=no` plus forced exit-75 -restart, limited to three starts per six hours; Compose stop grace is 120s and -systemd stop timeout 180s. +PID 1 handles technical child failures inside the same container, with safe +cleanup, bounded retries (three runtime starts per rolling six hours, including +the initial start) and backoff. Cleanup/safety/configuration errors veto retry; +SIGTERM gracefully cancels and joins children. Compose restart remains disabled +to prevent safety-failure loops; no host service installation or external process +manager is required. Container-platform lifecycle and host-boot policy belong to +the deployment operator. Compose stop grace is 240s; monitor join is bounded at +45s. Graceful shutdown cancels its watchdog rather than waiting out that deadline. +Genuine no-qualified-strategy outcomes instead persist a recovery wait of at least two hours +in SQLite while the same container continues light probes, then resume recovery +after expiry under the normal threshold policy. Both fresh bootstrap and periodic +recovery preserve a configured recovery cooldown longer than two hours. +This is not a watchdog timeout: +ordinary recovery cooldown stays 30m, and the abnormal whole-job ceiling stays 90m. ## Manual managed-test activation (not deployed) @@ -221,8 +231,12 @@ even after a newer acquisition starts. State survives reopening the database. The internal `runManagedMonitor` supervisor probes immediately and periodically, persists observations, and invokes injected recovery only at threshold outside cooldown. Unresolved GGC is reported without stopping monitoring; DB, safety, -runner, lock and finalization errors are fatal. Cooldown starts when recovery returns, including -failure or cancellation. Callbacks must honor cancellation and join their children. +runner, lock and finalization errors are fatal. Ordinary cooldown starts when recovery returns, +including failure or cancellation; validated strategy absence extends the durable +wait to at least two hours (or the longer configured recovery cooldown), in both +fresh bootstrap and periodic recovery, without restarting the container. Startup respects +the persisted expiry, and light probes continue without scans during the wait. +Callbacks must honor cancellation and join their children. Existing target guards run before a lifetime nonblocking `flock` in the private target state directory. With that exclusive singleton held, one-shot startup reconciliation uses schema-v5 PID/starttime identity: only a proven dead or diff --git a/cmd/zapret-autotune/managed_monitor_cli.go b/cmd/zapret-autotune/managed_monitor_cli.go index 0bf1090..f20108a 100644 --- a/cmd/zapret-autotune/managed_monitor_cli.go +++ b/cmd/zapret-autotune/managed_monitor_cli.go @@ -160,11 +160,11 @@ func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand return managedRecoveryDeps{}, fmt.Errorf("complete managed process adapters required") } // The blockcheck process-group runner cancels and waits descendants. - scan := func(ctx context.Context, _ string, host string, _, _ io.Writer) (string, error) { - return scanCommand(ctx, "/usr/bin/blockcheckw", host, io.Discard, io.Discard) + scan := func(ctx context.Context, _ string, host string, _ io.Writer, diagnostics io.Writer) (string, error) { + return scanCommand(ctx, "/usr/bin/blockcheckw", host, io.Discard, diagnostics) } - check := func(ctx context.Context, _ string, path, args, host string, _, _ io.Writer) error { - return checkCommand(ctx, "/usr/bin/blockcheckw", path, args, host, io.Discard, io.Discard) + check := func(ctx context.Context, _ string, path, args, host string, _ io.Writer, diagnostics io.Writer) error { + return checkCommand(ctx, "/usr/bin/blockcheckw", path, args, host, io.Discard, diagnostics) } run := func(ctx context.Context, path string, args ...string) error { if path != managedZapretScript || len(args) != 1 || (args[0] != "start" && args[0] != "stop" && args[0] != "restart") { diff --git a/cmd/zapret-autotune/managed_monitor_cli_test.go b/cmd/zapret-autotune/managed_monitor_cli_test.go index 687def5..d452071 100644 --- a/cmd/zapret-autotune/managed_monitor_cli_test.go +++ b/cmd/zapret-autotune/managed_monitor_cli_test.go @@ -3,11 +3,13 @@ package main import ( "bytes" "context" + "encoding/json" "errors" "io" "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "testing" @@ -123,7 +125,7 @@ func TestManagedEntrypointPrestartReconciliation(t *testing.T) { t.Fatal(err) } write("awk", "#!/bin/sh\nexit 0\n") - child := "#!/bin/sh\ntrap 'exit 0' TERM INT\nwhile :; do sleep 0.1 & wait $!; done\n" + child := "#!/bin/sh\nsleeper=\ntrap '[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; exit 0' TERM INT\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n" write("ss-server", child) write("ss-local", child) log := filepath.Join(root, "log") @@ -137,10 +139,16 @@ func TestManagedEntrypointPrestartReconciliation(t *testing.T) { if err != nil { t.Fatal(err) } - entry := write("entrypoint", strings.NewReplacer("/run/zapret-autotune-ss", ss, "/usr/local/bin/zapret-autotune", autotune, managedZapretScript, zapret, `"$ZAPRET_CONFIG"`, `"$FIXTURE_CONFIG"`).Replace(string(data))) + helper, err := filepath.Abs("../../managed-target-supervision.sh") + if err != nil { + t.Fatal(err) + } + entry := write("entrypoint", strings.NewReplacer("/opt/managed-target-supervision.sh", helper, "/run/zapret-autotune-ss", ss, "/usr/local/bin/zapret-autotune", autotune, managedZapretScript, zapret, `"$ZAPRET_CONFIG"`, `"$FIXTURE_CONFIG"`).Replace(string(data))) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() cmd := exec.CommandContext(ctx, "/bin/sh", entry) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) } cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "SS_PASSWORD="+strings.Repeat("a", 32), "MANAGED_PRESTART_HELPER=1", "FIXTURE_BINARY="+binary, "FIXTURE_TARGET="+d.directory, "FIXTURE_SCRIPT="+zapret) for key, value := range env { cmd.Env = append(cmd.Env, key+"="+value) @@ -354,6 +362,125 @@ func TestManagedMonitorProductionAdapterBinding(t *testing.T) { } } +func TestManagedMonitorProductionAbsenceDiagnostics(t *testing.T) { + for _, mode := range []string{"empty-nfq", "partial-nfq", "combined-nfq", "combined-capture-nfq", "success-scan-nfq", "success-combined-nfq", "artifact-nfq", "empty-timeout", "artifact-timeout", "malformed"} { + t.Run(mode, func(t *testing.T) { + recovery, _, _ := recoveryFixture(t, false) + monitor := monitorRuntimeFixture(t) + monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir + if err := os.Chmod(recovery.transaction.Dir, 0700); err != nil { + t.Fatal(err) + } + identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}` + if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil { + t.Fatal(err) + } + const infra = "FAIL 0/1 nfqws2 exited immediately (code 1) secret-never-print" + adapted, err := managedMonitorRecoveryAdapters(monitor.target, recovery.db, + func(_ context.Context, _, host string, out, diagnostics io.Writer) (string, error) { + if out != io.Discard || diagnostics == io.Discard { + t.Fatal("streams not privately isolated") + } + // Model exec's independent stdout/stderr writers without sharing a buffer. + done := make(chan struct{}) + go func() { io.WriteString(out, "secret-never-print"); close(done) }() + message := "FAIL 0/1 timeout" + if strings.Contains(mode, "nfq") || mode == "artifact-timeout" { + message = infra + } + if strings.Contains(mode, "combined") { + message = "FAIL 0/1 timeout" + } + io.WriteString(diagnostics, message) + <-done + data := []byte(`{"domain":"` + host + `","total":2,"working":0,"strategies":[]}`) + if mode == "partial-nfq" { + var report struct { + Strategies []json.RawMessage `json:"strategies"` + } + if err := json.Unmarshal([]byte(combinedReportFixture("--lua-desync=fake")), &report); err != nil { + t.Fatal(err) + } + report.Strategies = report.Strategies[:1] + partial, marshalErr := json.Marshal(report) + if marshalErr != nil { + t.Fatal(marshalErr) + } + data = partial + } + if strings.Contains(mode, "combined") || mode == "success-scan-nfq" { + data = []byte(combinedReportFixture("--lua-desync=fake")) + } + if mode == "malformed" { + data = []byte(`{"strategies":[]}`) + } + path := filepath.Join(t.TempDir(), "scan") + if mode == "artifact-nfq" || mode == "artifact-timeout" { + artifact := infra + if mode == "artifact-timeout" { + artifact = "FAIL 0/1 timeout" + } + if err := os.WriteFile(path+".check.json.stderr", []byte(artifact), 0600); err != nil { + t.Fatal(err) + } + } + return path, os.WriteFile(path+".check.json", data, 0600) + }, + func(_ context.Context, _, path, _, _ string, out, diagnostics io.Writer) error { + if out != io.Discard || diagnostics == io.Discard { + t.Fatal("check streams not privately isolated") + } + io.WriteString(diagnostics, infra) + if mode != "combined-capture-nfq" { + if err := os.WriteFile(path+".combined.json.stderr", []byte(infra), 0600); err != nil { + t.Fatal(err) + } + } + data := nonpassingManagedReport(t) + if mode == "success-combined-nfq" { + data = []byte(combinedReportFixture("--lua-desync=fake")) + } + return os.WriteFile(path+".combined.json", data, 0600) + }, recovery.run) + if err != nil { + t.Fatal(err) + } + if err := wireManagedRecovery(&monitor, adapted, func(error) { t.Fatal("unexpected GGC report") }); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + monitor.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() } + var reported error + monitor.reportRecoveryError = func(err error) { reported = err } + err = runManagedMonitor(ctx, monitor) + var fatal *managedRecoveryFatal + wantFatal := strings.Contains(mode, "nfq") + if errors.As(err, &fatal) != wantFatal || (!wantFatal && !errors.Is(err, context.Canceled)) { + t.Fatalf("mode=%s err=%v", mode, err) + } + if strings.Contains(errors.Join(err, reported).Error(), "secret-never-print") { + t.Fatal("diagnostics leaked") + } + state, stateErr := readMonitorState(monitor.db, monitor.serviceID) + if stateErr != nil { + t.Fatal(stateErr) + } + wantAbsence := mode == "empty-timeout" || mode == "artifact-timeout" + if errors.Is(reported, errNoQualifiedStrategy) != wantAbsence || errors.Is(err, errNoQualifiedStrategy) { + t.Fatalf("err=%v reported=%v", err, reported) + } + if wantAbsence { + if !state.CooldownUntil.Equal(monitorTestTime.Add(2 * time.Hour)) { + t.Fatalf("pause=%v", state.CooldownUntil) + } + } else if !state.CooldownUntil.IsZero() { + t.Fatalf("unexpected pause=%v", state.CooldownUntil) + } + }) + } +} + func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) { recovery, _, _ := recoveryFixture(t, false) monitor := monitorRuntimeFixture(t) @@ -468,7 +595,7 @@ func TestManagedMonitorBootstrapExistingFrontend(t *testing.T) { // Execute only rewritten target-local contracts and fake programs; no network, // Docker, real credentials, zapret or host firewall. Exercise the actual traps. func TestManagedEntrypointSupervision(t *testing.T) { - modes := []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"} + modes := []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-early", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"} for _, stage := range []string{"init", "reconcile", "start"} { for _, failure := range []string{"hang", "75", "124", "137", "7"} { modes = append(modes, "startup-"+stage+"-"+failure) @@ -499,21 +626,37 @@ func TestManagedEntrypointSupervision(t *testing.T) { write("timeout", "#!/bin/sh\nif [ \"$3\" = 45 ] || { [ \"$3\" = 30 ] && [ \"$5\" = start ]; }; then\nshift 3\nexec \"$REAL_TIMEOUT\" -k 1 1 \"$@\"\nfi\nexec \"$REAL_TIMEOUT\" \"$@\"\n") t.Setenv("REAL_TIMEOUT", timeout) } - startupFailure := "case \"$MODE\" in startup-$stage-hang) trap '' TERM INT; while :; do sleep 0.1; done;; startup-$stage-*) exit \"${MODE##*-}\";; esac\n" + // Accelerate only restart cooldowns; watchdog deadlines remain real. + realSleep, err := exec.LookPath("sleep") + if err != nil { + t.Fatal(err) + } + write("sleep", "#!/bin/sh\ncase \"$1\" in 15|30|60) exec \"$REAL_SLEEP\" 0.02;; *) exec \"$REAL_SLEEP\" \"$@\";; esac\n") + t.Setenv("REAL_SLEEP", realSleep) + startupFailure := "case \"$MODE\" in startup-$stage-hang) trap '' TERM INT; exec sleep 300;; startup-$stage-*) exit \"${MODE##*-}\";; esac\n" write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n") - child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & wait $!; done\n" + joinSleeper := "[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; " + child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\nsleeper=\ntrap '" + joinSleeper + "echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n" write("ss-server", child) write("ss-local", child) - zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nstage=$1\n"+startupFailure+"if [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; while :; do sleep 0.1; done; fi\n") - monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then\necho managed-$2 >> \"$LOG\"\nstage=$2\n"+startupFailure+"exit 0\nfi\ntrap 'echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; fi\necho managed-monitor >> \"$LOG\"\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & wait $!; done\n") + zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nstage=$1\n"+startupFailure+"if [ \"$MODE:$1\" = ss-early:start ] && [ \"$(grep -c zapret-start \"$LOG\")\" -gt 1 ]; then\nuntil [ \"$(grep -c 'stopped-.*/ss-local' \"$LOG\")\" -ge \"$(grep -c zapret-start \"$LOG\")\" ]; do sleep 0.02; done\nsleep 0.02\nfi\nif [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; exec sleep 300; fi\n") + monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then\necho managed-$2 >> \"$LOG\"\nstage=$2\n"+startupFailure+"exit 0\nfi\nsleeper=\ntrap '"+joinSleeper+"echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\necho managed-monitor >> \"$LOG\"\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; exec sleep 300; fi\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n") data, err := os.ReadFile("../../managed-target-entrypoint.sh") if err != nil { t.Fatal(err) } - text := strings.NewReplacer("/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data)) + helper, err := filepath.Abs("../../managed-target-supervision.sh") + if err != nil { + t.Fatal(err) + } + text := strings.NewReplacer("/opt/managed-target-supervision.sh", helper, "/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data)) if mode == "term-hang" { // Shorten only the fixture's watchdog, not the production contract. - text = strings.Replace(text, `bounded_wait "$monitor_pid" 60`, `bounded_wait "$monitor_pid" 1`, 1) + shortened := strings.Replace(text, `bounded_wait "$monitor_pid" 45`, `bounded_wait "$monitor_pid" 1`, 1) + if shortened == text { + t.Fatal("fixture did not shorten monitor watchdog") + } + text = shortened } if mode == "stop-hang" { text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1) @@ -524,6 +667,21 @@ func TestManagedEntrypointSupervision(t *testing.T) { cmd := exec.CommandContext(ctx, "/bin/sh", entry) cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "ZAPRET_CONFIG="+filepath.Join(root, "config"), "SS_PASSWORD="+strings.Repeat("a", 32)) cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + var shutdownSnapshot string + cmd.Cancel = func() error { + // Snapshot only this fixture's process group before deadline cleanup; + // no environment or private credential/config contents are collected. + processes, _ := exec.Command("ps", "-eo", "pid,ppid,pgid,stat,wchan:24,args").Output() + for _, line := range strings.Split(string(processes), "\n") { + fields := strings.Fields(line) + if len(fields) >= 3 && fields[2] == strconv.Itoa(cmd.Process.Pid) { + shutdownSnapshot += line + "\n" + } + } + fixtureLog, _ := os.ReadFile(log) + shutdownSnapshot += "fixture log:\n" + string(fixtureLog) + return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } var output bytes.Buffer cmd.Stdout, cmd.Stderr = &output, &output if err := cmd.Start(); err != nil { @@ -553,9 +711,11 @@ func TestManagedEntrypointSupervision(t *testing.T) { } err = <-done if ctx.Err() != nil { - t.Fatal("entrypoint did not join children", ctx.Err()) + t.Fatal("entrypoint did not join children", ctx.Err(), shutdownSnapshot, output.String()) } - want := 75 + // Repeated technical failures are supervised inside PID 1; after three + // starts the budget fails closed, rather than exporting exit 75. + want := 78 if strings.HasPrefix(mode, "startup-") { want = 78 } @@ -579,6 +739,14 @@ func TestManagedEntrypointSupervision(t *testing.T) { t.Fatal(err) } text = string(b) + retrying := mode == "technical" || mode == "panic" || mode == "killed" || mode == "segv" || mode == "zero" || mode == "ss" || mode == "ss-early" + starts := strings.Count(text, "zapret-start\n") + if retrying && starts != 3 || !retrying && !strings.HasPrefix(mode, "startup-") && starts != 1 { + t.Fatal("unexpected internal restart count", starts, text) + } + if mode == "ss-early" && strings.Count(text, "managed-monitor\n") != 1 { + t.Fatal("forked monitor after replacement SS child already exited", text) + } if strings.HasPrefix(mode, "startup-") { if strings.Contains(text, "managed-monitor") || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-server")) || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-local")) { t.Fatal("startup failure escaped cleanup", text) diff --git a/cmd/zapret-autotune/managed_recovery.go b/cmd/zapret-autotune/managed_recovery.go index 4b266ba..4a6b31c 100644 --- a/cmd/zapret-autotune/managed_recovery.go +++ b/cmd/zapret-autotune/managed_recovery.go @@ -107,6 +107,7 @@ func (e *managedRecoveryTimeout) Error() string { return e.Err.Error() } func (e *managedRecoveryTimeout) Unwrap() error { return e.Err } var ( + errNoQualifiedStrategy = errors.New("no fully passing check strategy for both tls12 and tls13") errManagedGGCUnresolved = errors.New("managed GGC scope unresolved") errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing") errManagedRecoveryDeadline = errors.New("managed recovery total deadline exceeded") @@ -420,6 +421,9 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma if ctx.Err() != nil { return result, e } + if errors.Is(e, errNoQualifiedStrategy) { + return result, e + } result.GGCError = fmt.Errorf("GGC validation or activation unresolved") return result, nil } @@ -474,7 +478,13 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos return 0, err } started := time.Now() - path, err := d.scan(ctx, "blockcheckw", host, io.Discard, io.Discard) + var diagnostics bytes.Buffer + path, err := d.scan(ctx, "blockcheckw", host, io.Discard, &diagnostics) + // Only validated report evidence, never a runner error, authorizes absence. + if errors.Is(err, errNoQualifiedStrategy) || errors.Is(err, errCandidateNonpass) { + err = fmt.Errorf("candidate scanner failed: %v", err) + } + err = managedCheckerError(err, path+".check.json.stderr", diagnostics.Bytes()) var report []byte var args, output string if err == nil { @@ -491,9 +501,16 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos } if err == nil { args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, host) + if err != nil && emptyManagedScanReport(report, host) { + err = errNoQualifiedStrategy + } } if err == nil { report, err = revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, d.check) + var fatal *managedRecoveryFatal + if !errors.As(err, &fatal) && errors.Is(err, errCandidateNonpass) { + err = errNoQualifiedStrategy + } } if err == nil { err = ctx.Err() @@ -524,6 +541,21 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos return id, nil } +// A successful owned scan may omit rejected rows. Missing/null metadata is not +// absence evidence; qualification still requires the usual full TLS checks. +func emptyManagedScanReport(raw []byte, host string) bool { + var report struct { + Domain string `json:"domain"` + Total *int `json:"total"` + Working *int `json:"working"` + Strategies *[]json.RawMessage `json:"strategies"` + } + d := json.NewDecoder(bytes.NewReader(raw)) + return d.Decode(&report) == nil && d.Decode(new(json.RawMessage)) == io.EOF && + report.Domain == host && report.Total != nil && *report.Total > 0 && + report.Working != nil && *report.Working == 0 && report.Strategies != nil && len(*report.Strategies) == 0 +} + // Installs callbacks. runManagedMonitor performs startup // under its singleton. Target guards are repeated before every recovery/probe. func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDeps, reportGGC func(error)) error { @@ -567,6 +599,14 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe if err := transaction.Reconcile(recovery.db); err != nil { return &managedRecoveryFatal{Err: err} } + state, err := readMonitorState(monitor.db, monitor.serviceID) + if err != nil { + return &managedRecoveryFatal{Err: err} + } + // Reconciliation and identity guards precede even a paused bootstrap. + if state.RecoveryRunning || monitor.now().Before(state.CooldownUntil) { + return nil + } ids, err := transaction.ActiveCandidates(recovery.db) if err != nil { return &managedRecoveryFatal{Err: err} diff --git a/cmd/zapret-autotune/managed_recovery_test.go b/cmd/zapret-autotune/managed_recovery_test.go index 798442b..b98ee52 100644 --- a/cmd/zapret-autotune/managed_recovery_test.go +++ b/cmd/zapret-autotune/managed_recovery_test.go @@ -86,6 +86,202 @@ func TestManagedRecoveryBootstrap(t *testing.T) { } } +func TestManagedSavedGGCInfrastructureCannotAuthorizeFallback(t *testing.T) { + for _, artifact := range []bool{false, true} { + t.Run(fmt.Sprint("artifact=", artifact), func(t *testing.T) { + d, front, _ := recoveryFixture(t, true) + managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true) + monitor := monitorRuntimeFixture(t) + monitor.db, monitor.target.directory = d.db, d.transaction.Dir + monitor.threshold = 1 + if err := os.Chmod(d.transaction.Dir, 0700); err != nil { + t.Fatal(err) + } + identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}` + if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil { + t.Fatal(err) + } + d.scan = func(context.Context, string, string, io.Writer, io.Writer) (string, error) { + t.Fatal("infrastructure failure authorized fallback scan") + return "", nil + } + checks := 0 + d.check = func(_ context.Context, _, path, _, host string, out, diagnostics io.Writer) error { + checks++ + if host != "fresh.googlevideo.com" || out != io.Discard { + t.Fatal("unexpected saved check") + } + const infra = "FAIL 0/1 timeout: nfqws2 exited immediately (code 1) private-secret" + if artifact { + if err := os.WriteFile(path+".combined.json.stderr", []byte(infra), 0600); err != nil { + return err + } + } else { + io.WriteString(diagnostics, infra) + } + return os.WriteFile(path+".combined.json", nonpassingManagedReport(t), 0600) + } + if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("infrastructure became unresolved GGC") }); err != nil { + t.Fatal(err) + } + monitor.probe = func(context.Context) error { return errMonitorTestUnhealthy } + monitor.reportRecoveryError = func(err error) { + var fatal *managedRecoveryFatal + if !errors.As(err, &fatal) || errors.Is(err, errNoQualifiedStrategy) { + t.Fatal(err) + } + } + err := runManagedMonitor(context.Background(), monitor) + var fatal *managedRecoveryFatal + if !errors.As(err, &fatal) || errors.Is(err, errCandidateNonpass) || strings.Contains(err.Error(), "private-secret") || checks != 1 { + t.Fatalf("checks=%d err=%v", checks, err) + } + state, stateErr := readMonitorState(d.db, monitor.serviceID) + if stateErr != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(monitor.cooldown)) { + t.Fatalf("%+v %v", state, stateErr) + } + assertActiveCandidates(t, d.transaction, d.db, []int64{front}) + }) + } +} + +func nonpassingManagedReport(t *testing.T) []byte { + t.Helper() + var report struct { + Strategies []map[string]any `json:"strategies"` + } + if err := json.Unmarshal([]byte(combinedReportFixture("--lua-desync=fake")), &report); err != nil { + t.Fatal(err) + } + for _, row := range report.Strategies { + row["passes_ok"], row["success_rate"] = 0, 0 + } + raw, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + return raw +} + +func TestManagedAbsenceClassification(t *testing.T) { + for _, mode := range []string{"shortlist", "combined", "malformed", "empty", "empty-valid", "empty-nfq", "empty-wrong-target", "runner"} { + t.Run(mode, func(t *testing.T) { + d, _, _ := recoveryFixture(t, false) + raw := nonpassingManagedReport(t) + d.scan = func(_ context.Context, _, host string, _, diagnostics io.Writer) (string, error) { + if mode == "runner" { + return "", errCandidateNonpass + } + data := raw + switch mode { + case "combined": + data = []byte(combinedReportFixture("--lua-desync=fake")) + case "malformed": + data = []byte(`{"strategies":[{"protocol":"tls12"}]}`) + case "empty": + data = []byte(`{"strategies":[]}`) + case "empty-valid", "empty-nfq", "empty-wrong-target": + if mode == "empty-nfq" { + fmt.Fprintln(diagnostics, "nfqws2 exited with status 1") + } + if mode == "empty-wrong-target" { + host = "untrusted.example" + } + data = []byte(fmt.Sprintf(`{"domain":%q,"total":10,"working":0,"strategies":[]}`, host)) + } + path := filepath.Join(t.TempDir(), "scan") + return path, os.WriteFile(path+".check.json", data, 0600) + } + d.check = func(_ context.Context, _, path, _, _ string, _, _ io.Writer) error { + return os.WriteFile(path+".combined.json", raw, 0600) + } + _, err := recoverManagedTarget(context.Background(), d) + want := mode == "shortlist" || mode == "combined" || mode == "empty-valid" + if err == nil || errors.Is(err, errNoQualifiedStrategy) != want { + t.Fatalf("classification: %v", err) + } + var fatal *managedRecoveryFatal + if errors.As(err, &fatal) && mode != "empty-nfq" { + t.Fatalf("unexpected fatal: %v", err) + } + }) + } +} + +func TestManagedBootstrapAbsencePauseRestartAndExpiry(t *testing.T) { + d, _, _ := recoveryFixture(t, false) + monitor := monitorRuntimeFixture(t) + monitor.db, monitor.target.directory = d.db, d.transaction.Dir + if err := os.Chmod(d.transaction.Dir, 0700); err != nil { + t.Fatal(err) + } + identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}` + if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil { + t.Fatal(err) + } + now := monitorTestTime + monitor.now = func() time.Time { return now } + scans := 0 + d.scan = func(_ context.Context, _, _ string, _, _ io.Writer) (string, error) { + scans++ + path := filepath.Join(t.TempDir(), "scan") + return path, os.WriteFile(path+".check.json", nonpassingManagedReport(t), 0600) + } + d.health = func(context.Context) (string, error) { return "UNHEALTHY", nil } + if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("unexpected GGC error") }); err != nil { + t.Fatal(err) + } + monitor.reportRecoveryError = func(err error) { + if !errors.Is(err, errNoQualifiedStrategy) { + t.Fatal(err) + } + } + for phase, elapsed := range []time.Duration{0, time.Hour, 2 * time.Hour} { + now = monitorTestTime.Add(elapsed) + ctx, cancel := context.WithCancel(context.Background()) + probes := 0 + monitor.wait = func(ctx context.Context, _ time.Duration) error { + probes++ + if probes == 4 { + cancel() + return ctx.Err() + } + return nil + } + if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + cancel() + wantScans := 1 + if phase == 2 { + wantScans = 2 + } + state, err := readMonitorState(monitor.db, monitor.serviceID) + if err != nil || scans != wantScans || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(time.Duration(wantScans)*2*time.Hour)) { + t.Fatalf("phase=%d scans=%d state=%+v err=%v", phase, scans, state, err) + } + // Reopen the same durable database before the next monitor instance. + var dbPath string + var seq int + var name string + if err := monitor.db.QueryRow(`PRAGMA database_list`).Scan(&seq, &name, &dbPath); err != nil { + t.Fatal(err) + } + if err := monitor.db.Close(); err != nil { + t.Fatal(err) + } + reopened, err := openStore(dbPath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { reopened.Close() }) + monitor.db, d.db = reopened, reopened + if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("unexpected GGC error") }); err != nil { + t.Fatal(err) + } + } +} + func TestManagedRecoveryJobDeadline(t *testing.T) { for _, mode := range []string{"bootstrap-scan", "bootstrap-saved", "periodic-saved", "periodic-apply", "periodic-ggc", "parent-cancel", "parent-deadline"} { t.Run(mode, func(t *testing.T) { diff --git a/cmd/zapret-autotune/monitor.go b/cmd/zapret-autotune/monitor.go index 40bb156..6e424aa 100644 --- a/cmd/zapret-autotune/monitor.go +++ b/cmd/zapret-autotune/monitor.go @@ -198,8 +198,8 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T if err := validateMonitorTime(serviceID, expires); err != nil { return err } - result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ? - WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), serviceID, token) + result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = MAX(COALESCE(cooldown_until, ?), ?) + WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), expires.UnixNano(), serviceID, token) if err != nil { return err } @@ -210,6 +210,23 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T return err } +// Extend bootstrap backoff without changing probes or any recovery ownership. +func pauseMonitorRecovery(db *sql.DB, serviceID string, now time.Time, cooldown time.Duration) error { + if err := validateMonitorTime(serviceID, now); err != nil { + return err + } + if cooldown <= 0 { + return fmt.Errorf("recovery cooldown must be positive") + } + expires := now.Add(cooldown) + if err := validateMonitorTime(serviceID, expires); err != nil { + return err + } + _, err := db.Exec(`INSERT INTO monitor_state (service_id, cooldown_until) VALUES (?, ?) + ON CONFLICT(service_id) DO UPDATE SET cooldown_until = MAX(COALESCE(monitor_state.cooldown_until, excluded.cooldown_until), excluded.cooldown_until)`, serviceID, expires.UnixNano()) + return err +} + // Startup policy only: call for each configured service before monitoring. // A legacy/indeterminate owner is an error, never permission to unlock. // The conditional write also protects against a finish/reacquire during the read. @@ -251,9 +268,9 @@ func reconcileMonitorRecovery(db *sql.DB, serviceID string, cooldown time.Durati } } result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, - recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ? + recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = MAX(COALESCE(cooldown_until, ?), ?) WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ? - AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), serviceID, token, pid.Int64, start.Int64) + AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), expires.UnixNano(), serviceID, token, pid.Int64, start.Int64) if err != nil { return false, err } diff --git a/cmd/zapret-autotune/monitor_runtime.go b/cmd/zapret-autotune/monitor_runtime.go index 037f46d..6369dbc 100644 --- a/cmd/zapret-autotune/monitor_runtime.go +++ b/cmd/zapret-autotune/monitor_runtime.go @@ -14,6 +14,16 @@ import ( const managedMonitorLockName = ".autotune-monitor-lock" +const noQualifiedStrategyCooldown = 2 * time.Hour + +func managedRecoveryCooldown(err error, configured time.Duration) time.Duration { + var fatal *managedRecoveryFatal + if !errors.As(err, &fatal) && errors.Is(err, errNoQualifiedStrategy) && configured < noQualifiedStrategyCooldown { + return noQualifiedStrategyCooldown + } + return configured +} + // Callbacks must honor ctx and join // their children before returning; the supervisor never detaches callbacks. type managedMonitorDeps struct { @@ -130,6 +140,11 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) { if ctx.Err() != nil { return ctx.Err() } + if errors.Is(err, errNoQualifiedStrategy) { + if err := pauseMonitorRecovery(d.db, d.serviceID, d.now(), managedRecoveryCooldown(err, d.cooldown)); err != nil { + return err + } + } // Bootstrap scans can validly find no qualified strategy. Keep the // supervisor alive so durable probes/cooldown govern the next attempt. d.reportRecoveryError(err) @@ -157,8 +172,9 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) { } if token != "" { recoveryErr := d.recover(ctx) + cooldown := managedRecoveryCooldown(recoveryErr, d.cooldown) // Finalize even on cancellation, using the non-context DB operation. - if err := finishMonitorRecovery(d.db, d.serviceID, token, d.now(), d.cooldown); err != nil { + if err := finishMonitorRecovery(d.db, d.serviceID, token, d.now(), cooldown); err != nil { return err } if recoveryErr != nil { diff --git a/cmd/zapret-autotune/monitor_runtime_test.go b/cmd/zapret-autotune/monitor_runtime_test.go index 725295b..21aea60 100644 --- a/cmd/zapret-autotune/monitor_runtime_test.go +++ b/cmd/zapret-autotune/monitor_runtime_test.go @@ -11,6 +11,95 @@ import ( var errMonitorTestUnhealthy = errors.New("unhealthy") +func TestManagedMonitorBootstrapConfiguredLongerPause(t *testing.T) { + d := monitorRuntimeFixture(t) + d.cooldown = 4 * time.Hour + d.bootstrap = func(context.Context) error { return errNoQualifiedStrategy } + d.reportRecoveryError = func(err error) { + if !errors.Is(err, errNoQualifiedStrategy) { + t.Fatal(err) + } + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() } + if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + state, err := readMonitorState(d.db, d.serviceID) + if err != nil || !state.CooldownUntil.Equal(monitorTestTime.Add(4*time.Hour)) { + t.Fatalf("%+v %v", state, err) + } +} + +func TestManagedMonitorPeriodicAbsenceCooldown(t *testing.T) { + d := monitorRuntimeFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + d.threshold = 1 + now := monitorTestTime + d.now = func() time.Time { return now } + d.probe = func(context.Context) error { return errMonitorTestUnhealthy } + recoveries := 0 + d.recover = func(context.Context) error { recoveries++; return errNoQualifiedStrategy } + d.reportRecoveryError = func(err error) { + if !errors.Is(err, errNoQualifiedStrategy) { + t.Fatal(err) + } + } + waits := 0 + d.wait = func(ctx context.Context, _ time.Duration) error { + waits++ + state, err := readMonitorState(d.db, d.serviceID) + want := monitorTestTime.Add(2 * time.Hour) + if waits == 3 { + want = monitorTestTime.Add(4 * time.Hour) + } + if err != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(want) { + t.Fatalf("%+v %v", state, err) + } + if waits == 3 { + cancel() + return ctx.Err() + } + now = now.Add(time.Hour) + return nil + } + if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if recoveries != 2 { + t.Fatalf("recoveries=%d", recoveries) + } +} + +func TestMonitorPausePreservesLongerCooldownAndLease(t *testing.T) { + d := monitorRuntimeFixture(t) + if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil { + t.Fatal(err) + } + token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime) + if err != nil || token == "" { + t.Fatalf("token=%s err=%v", token, err) + } + for _, duration := range []time.Duration{4 * time.Hour, 2 * time.Hour} { + if err := pauseMonitorRecovery(d.db, d.serviceID, monitorTestTime, duration); err != nil { + t.Fatal(err) + } + } + state, err := readMonitorState(d.db, d.serviceID) + if err != nil || !state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(4*time.Hour)) { + t.Fatalf("%+v %v", state, err) + } + if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, 30*time.Minute); err != nil { + t.Fatal(err) + } + state, err = readMonitorState(d.db, d.serviceID) + if err != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(4*time.Hour)) { + t.Fatalf("%+v %v", state, err) + } +} + // Exercise the production process reader under the real singleton. PID reuse // proves the old owner is gone and clears only that lease; a live owner stays // busy, while an owner without v5 identity makes startup fail closed. @@ -307,7 +396,7 @@ func TestManagedMonitorStartupLeaseAfterReopenStartsCrashCooldown(t *testing.T) if err != nil || oldToken == "" { t.Fatalf("token=%q err=%v", oldToken, err) } - expires := monitorTestTime.Add(d.cooldown) + expires := monitorTestTime.Add(time.Hour) if _, err := db.Exec(`UPDATE monitor_state SET cooldown_until = ? WHERE service_id = ?`, monitorTestTime.Add(time.Hour).UnixNano(), d.serviceID); err != nil { t.Fatal(err) } diff --git a/cmd/zapret-autotune/recover.go b/cmd/zapret-autotune/recover.go index 10cf81f..5888c25 100644 --- a/cmd/zapret-autotune/recover.go +++ b/cmd/zapret-autotune/recover.go @@ -61,10 +61,11 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa return nil, fmt.Errorf("candidate diagnostic artifact already exists or cannot be inspected: %v", err) } var raw bytes.Buffer - err := check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw) + err := check(ctx, "blockcheckw", path, c.Args, host, io.Discard, &raw) if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil { return nil, e } + err = managedCheckerError(err, path+".combined.json.stderr", raw.Bytes()) if err != nil { // Only our report parser may issue the nonpass authorization. if errors.Is(err, errCandidateNonpass) { @@ -91,6 +92,27 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa return report, err } +// Check infrastructure before interpreting any positive or negative evidence. +// Prefer the owned invocation's artifact over earlier scan/seed diagnostics; +// captured stderr is private, and stdout never shares its buffer. +func managedCheckerError(err error, artifact string, captured []byte) error { + var fatal *managedRecoveryFatal + if errors.As(err, &fatal) { + return err + } + diagnostics, readErr := os.ReadFile(artifact) + if readErr != nil { + if !errors.Is(readErr, os.ErrNotExist) { + return &managedRecoveryFatal{Err: fmt.Errorf("cannot inspect owned checker diagnostics")} + } + diagnostics = captured + } + if strings.Contains(strings.ToLower(string(diagnostics)), "nfqws2 exited") { + return &managedRecoveryFatal{Err: fmt.Errorf("checker nfqws2 worker exited unexpectedly")} + } + return err +} + var earlyTimeoutResult = regexp.MustCompile(`^ FAIL ([0-2])/([1-3]) timeout$`) var earlySuccessResult = regexp.MustCompile(`^ OK median [0-9]+ms, [0-9]+\.[0-9] KB/s$`) var earlyCheckPreamble = regexp.MustCompile("^(! --workers has no effect on `check` \\(it verifies strategies sequentially by design\\)\n)?" + diff --git a/cmd/zapret-autotune/tune.go b/cmd/zapret-autotune/tune.go index 978d36d..6af9c56 100644 --- a/cmd/zapret-autotune/tune.go +++ b/cmd/zapret-autotune/tune.go @@ -211,7 +211,7 @@ func selectBlockcheckEvidence(r io.Reader, expected, group, host string, recover for _, protocol := range []string{"tls12", "tls13"} { c, ok := best[protocol] if !ok { - return "", fmt.Errorf("no fully passing check strategy for both tls12 and tls13") + return "", errNoQualifiedStrategy } if protocol == "tls13" && c.Args == best["tls12"].Args { continue diff --git a/deploy/managed-target/README.md b/deploy/managed-target/README.md index 59812af..325f8e5 100644 --- a/deploy/managed-target/README.md +++ b/deploy/managed-target/README.md @@ -34,36 +34,34 @@ in logs or shared terminals. By default Compose uses the locally built `zapret-autotune-managed-target:local`. Tags are only local build/test candidates, not production release references. For production promotion, set `ZAPRET_AUTOTUNE_IMAGE=/@sha256:` -in the private `.env` and pull that immutable image before starting the unit. -Use a managed target image, not the generic image; the systemd command never -builds images. Do not enable the production service with a tag or local default. +in the private `.env` and pull that immutable image before starting the target. +Use a managed target image, not the generic image. Do not deploy production +with a tag or local default. Offline static guard (from repository root): `python3 deploy/managed-target/check_static.py`. +Isolated, accelerated shell supervision fixtures (no network): +`python3 deploy/managed-target/check_supervision.py`. -## Optional native systemd supervision +## In-container supervision -For unattended boot, place this checkout at `/opt/zapret2-autotune`, provision -`deploy/managed-target/.env` and state as above, and build the target image first. -For production, select and pull the immutable managed-target digest as above -instead of using a local build tag. -Install `deploy/managed-target/zapret-autotune.service` in `/etc/systemd/system/`, -then run `systemctl daemon-reload` and `systemctl enable --now zapret-autotune`. -Do not simultaneously start the same project with detached Compose. -Enabling the unit starts this isolated target on subsequent boots. `ExecStop` -only stops `target`; it does not replace the foreground process's failure code. -An explicit service stop suppresses restart, including during a shutdown race. - -Compose has no automatic restart. The foreground systemd unit restarts **only -exit 75**, with at most three starts per six hours. Clean typed recovery timeouts, +Compose has no automatic restart. PID 1 restarts the entire owned runtime only +for technical failures, with three starts per rolling six hours (including initial +startup) and cancellable 15/30-second cooldowns. Clean typed recovery timeouts, unexpected monitor exits/crashes, and Shadowsocks failures qualify only if shutdown cleanup succeeds. Configuration (64), safety/database/rollback/cleanup -failures (78), and graceful SIGTERM (0) never request restart. Investigate and +failures (78), and graceful SIGTERM (0) never request restart. Exhausting the +restart budget exits 78. Investigate and repair nonretry failures before manually starting again; do not clear state. -Monitor cancellation/join is bounded to 60 seconds, final target-local zapret +Monitor cancellation/join is bounded to 45 seconds, reconciliation gets 45 seconds +plus five to force kill, and final target-local zapret stop gets 30 seconds then forced kill after five more, and each Shadowsocks join -gets five seconds. Exceeding a join bound fails closed. Compose grants 120 -seconds; systemd grants 180 seconds. +gets five seconds. Exceeding a join bound fails closed. Compose grants 240 +seconds. Reconcile after joining the monitor and before even stopping zapret: +stop also sources config, so unknown state is never sourced. The sequential +cleanup budget is 140 seconds; 240 seconds also covers an in-flight startup +command and scheduling grace. Watchdogs run in fresh shell interpreters and +cancel/join their own sleepers, including during PID 1's EXIT trap. ## Isolation and runtime contract @@ -97,8 +95,9 @@ Any startup command failure/timeout exits 78, never the retry code 75. These bounds do not shorten the monitor's 90-minute bootstrap/recovery scan budget. SIGTERM/SIGINT cancels and waits for the monitor before stopping target-local -zapret and Shadowsocks children. Monitor fatal exits stop the -entrypoint. Monitor health is YouTube Player API status through the owned SOCKS, +zapret and Shadowsocks children. Nonretry monitor fatal exits stop the +entrypoint; technical exits and Shadowsocks failures restart from private configs +only after safe cleanup. Monitor health is YouTube Player API status through the owned SOCKS, not video playback/SABR; GGC candidates are scoped to the exact discovered hostname. diff --git a/deploy/managed-target/check_static.py b/deploy/managed-target/check_static.py index cc2298e..c996e55 100644 --- a/deploy/managed-target/check_static.py +++ b/deploy/managed-target/check_static.py @@ -7,7 +7,6 @@ directory = Path(__file__).resolve().parent compose = (directory / "compose.yaml").read_text() entrypoint = (directory.parents[1] / "managed-target-entrypoint.sh").read_text() readme = (directory / "README.md").read_text() -unit = (directory / "zapret-autotune.service").read_text() # ponytail: checks the committed Compose shape, not arbitrary YAML; use a YAML # parser if this definition grows beyond its single target service. @@ -29,19 +28,15 @@ assert "Docker administrators/root can still read container" in readme assert "removed\nfrom the entrypoint environment" in readme assert "No host networking" in readme assert ' restart: "no"\n' in compose -assert " stop_grace_period: 120s\n" in compose -directives = dict(line.split("=", 1) for line in unit.splitlines() if "=" in line) -assert directives["Restart"] == "no" -assert directives["RestartForceExitStatus"] == "75" -assert "RestartPreventExitStatus" not in directives -assert directives["StartLimitIntervalSec"] == "6h" -assert directives["StartLimitBurst"] == "3" -assert directives["TimeoutStopSec"] == "180s" -assert directives["WorkingDirectory"] == "/opt/zapret2-autotune" -assert directives["WantedBy"] == "multi-user.target" -compose_command = "/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml" -assert directives["ExecStart"] == compose_command + " up --no-build --abort-on-container-exit --exit-code-from target target" -assert directives["ExecStop"] == compose_command + " stop target" +assert " stop_grace_period: 240s\n" in compose +assert 'bounded_wait "$monitor_pid" 45' in entrypoint +assert '/bin/sh /opt/managed-target-supervision.sh --watchdog' in entrypoint +assert 'COPY managed-target-supervision.sh /opt/managed-target-supervision.sh' in (directory.parents[1] / 'Dockerfile.target').read_text() +assert '!managed-target-supervision.sh' in (directory.parents[1] / '.dockerignore').read_text() +assert '"$count" -lt 3' in entrypoint +assert '"$((now - started))" -lt 21600' in entrypoint +assert "backoff=15" in entrypoint +assert "systemctl" not in entrypoint and "docker " not in entrypoint assert "timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop" in entrypoint for command, seconds in (("/usr/local/bin/zapret-autotune managed init", 45), ("/usr/local/bin/zapret-autotune managed reconcile", 45), diff --git a/deploy/managed-target/check_supervision.py b/deploy/managed-target/check_supervision.py new file mode 100644 index 0000000..fa01512 --- /dev/null +++ b/deploy/managed-target/check_supervision.py @@ -0,0 +1,223 @@ +"""Isolated shell fixtures: python3 deploy/managed-target/check_supervision.py. + +Only a temporary COPY of the entrypoint is redirected to fake executables; +production has no test switches, real network probes, or accelerated timers. +""" +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import time + + +ROOT = Path(__file__).resolve().parents[2] +FAKE = r'''#!/usr/bin/env python3 +import os, pathlib, signal, sys, time +root = pathlib.Path(os.environ['FIXTURE']) +name = pathlib.Path(sys.argv[0]).name +scenario = os.environ['SCENARIO'] +def log(message): + with (root / 'events').open('a') as f: + f.write(message + '\n') +def increment(key): + path = root / key + n = int(path.read_text()) + 1 if path.exists() else 1 + path.write_text(str(n)) + return n +def linger(code=0): + signal.signal(signal.SIGTERM, lambda *_: sys.exit(code)) + while True: + time.sleep(.01) +if name == 'awk': + sys.exit(0) +if name == 'date': + n = increment('clock') + print(100000 + (n * 21601 if scenario == 'window' else n)) + sys.exit(0) +if name == 'sleep': + seconds = int(sys.argv[1]) + if seconds in (15, 30): + log('cooldown ' + str(seconds)) + if scenario == 'cancel': + os.kill(os.getppid(), signal.SIGTERM) + time.sleep(1) + else: + time.sleep(.01) + else: + time.sleep(.01 if seconds == 1 else 2) + sys.exit(0) +if name == 'timeout': + os.execv(sys.argv[4], sys.argv[4:]) +if name in ('ss-server', 'ss-local'): + with (root / 'children').open('a') as f: + f.write(str(os.getpid()) + '\n') + assert 'SS_PASSWORD' not in os.environ + assert sys.argv[1] == '-c' and len(sys.argv) == 3 + assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600 + log(name) + if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail') and increment('ss_runs') == 1: + # Let startup/monitor begin before simulating a child crash. + while not (root / 'monitor_runs').exists(): + time.sleep(.01) + time.sleep(.03) + sys.exit(137) + linger() +if name == 'zapret2': + assert not (root / 'prepared').exists(), 'sourced prepared config' + log(sys.argv[1]) + if scenario == 'stopfail' and sys.argv[1] == 'stop': + sys.exit(1) + sys.exit(0) +assert name == 'autotune' +command = sys.argv[2] +if command == 'init': + log('init') + (root / 'config').touch() +elif command == 'reconcile': + n = increment('reconciles') + log('reconcile') + if scenario == 'prestartfail': + sys.exit(78) + if scenario in ('unknown', 'unknown75') and n == 2: + sys.exit(78) + (root / 'prepared').unlink(missing_ok=True) +elif command == 'monitor': + with (root / 'children').open('a') as f: + f.write(str(os.getpid()) + '\n') + n = increment('monitor_runs') + log('monitor ' + str(n)) + if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'): + sys.exit(75) + if scenario.startswith('fatal'): + sys.exit(int(scenario[5:])) + if n == 1 and scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'unknown', 'window'): + if scenario.startswith('crash') or scenario == 'unknown': + (root / 'prepared').touch() + code = int(scenario[5:]) if scenario.startswith('crash') and scenario != 'crash' else 2 + sys.exit(code if scenario.startswith('crash') or scenario == 'unknown' else 75) + if scenario == 'joinfail': + signal.signal(signal.SIGTERM, signal.SIG_IGN) + while True: + time.sleep(.01) + if n == 1 and scenario in ('ss', 'mixed'): + linger(78 if scenario == 'mixed' else 0) + # Successful second runtime: external SIGTERM must exit 0, not restart. + time.sleep(.03) + os.kill(os.getppid(), signal.SIGTERM) + linger() +''' + + +def run(scenario, expected): + with tempfile.TemporaryDirectory(prefix='supervision-') as tmp: + root = Path(tmp) + binary = root / 'bin' + binary.mkdir() + fake = binary / 'fake' + fake.write_text(FAKE) + fake.chmod(0o755) + for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2'): + (binary / name).symlink_to(fake) + configs = root / 'private' + configs.mkdir() + text = (ROOT / 'managed-target-entrypoint.sh').read_text() + helper = ROOT / 'managed-target-supervision.sh' + if scenario == 'ackfail': + broken_helper = root / 'broken-helper.sh' + broken_helper.write_text(helper.read_text().replace('sleep "$seconds" & sleeper=$!', 'exit 78')) + helper = broken_helper + text = text.replace('/opt/managed-target-supervision.sh', str(helper)) + text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}') + text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune')) + text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2')) + entry = root / 'entry.sh' + entry.write_text(text) + env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp, + SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config')) + result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15) + events = (root / 'events').read_text().splitlines() + assert result.returncode == expected, (scenario, result.returncode, result.stderr, events) + assert b'ab' * 32 not in result.stdout + result.stderr + for pid in (root / 'children').read_text().splitlines(): + assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid) + # Every activation and stop must immediately follow successful reconcile. + for i, event in enumerate(events): + if event in ('start', 'stop'): + assert events[i - 1] == 'reconcile', (scenario, events) + return events + + +for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window'): + events = run(scenario, 0) + assert events.count('start') == 2, (scenario, events) + assert events.count('ss-server') == 2 and events.count('ss-local') == 2 + assert events.count('stop') == 2 + assert events.count('cooldown 15') == 1 +for code in (64, 78, 1): + events = run(f'fatal{code}', code if code in (64, 78) else 78) + assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events) +for scenario in ('mixed', 'unknown', 'unknown75', 'stopfail', 'joinfail', 'ackfail'): + events = run(scenario, 78) + assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events) + if scenario.startswith('unknown'): + assert 'stop' not in events +events = run('budget', 78) +assert events.count('start') == 3 +assert [e for e in events if e.startswith('cooldown')] == ['cooldown 15', 'cooldown 30'] +events = run('cancel', 0) +assert events.count('start') == 1 +events = run('prestartfail', 78) +assert 'start' not in events and 'stop' not in events + + +def check_watchdog_traps(shell): + """Same bounded_wait from normal flow, plain EXIT, and TERM -> EXIT traps.""" + entrypoint = (ROOT / 'managed-target-entrypoint.sh').read_text() + bounded_wait = entrypoint[entrypoint.index('bounded_wait() {'):entrypoint.index('cleanup_cycle() {')] + bounded_wait = bounded_wait.replace('/opt/managed-target-supervision.sh', str(ROOT / 'managed-target-supervision.sh')) + for mode in ('normal', 'exit', 'term-exit'): + with tempfile.TemporaryDirectory(prefix='watchdog-traps-') as tmp: + directory = Path(tmp) + sleeper = directory / 'sleep' + sleeper.write_text('#!/bin/sh\necho $$ >> "$PIDS"\nexec "' + shutil.which('sleep') + '" "$@"\n') + sleeper.chmod(0o755) + script = directory / 'check.sh' + script.write_text(f'''set -eu +mode=$1 +shift +config_dir={directory} +. {ROOT / 'managed-target-supervision.sh'} +{bounded_wait} +cleanup() {{ + trap - EXIT + trap ':' INT TERM + kill "$owned" 2>/dev/null || true + bounded_wait "$owned" 5 + [ "$join_timed_out" -eq 0 ] +}} +sleep 300 & owned=$! +case "$mode" in +normal) cleanup ;; +exit) trap cleanup EXIT; exit 0 ;; +term-exit) trap cleanup EXIT; trap 'exit 0' TERM; kill -TERM $$ ;; +esac +''') + env = dict(os.environ, PATH=f'{directory}:{os.environ["PATH"]}', PIDS=str(directory / 'pids')) + started = time.monotonic() + result = subprocess.run([*shell, str(script), mode], env=env, capture_output=True, timeout=10) + assert result.returncode == 0, (shell, mode, result.stderr) + assert time.monotonic() - started < 2, (shell, mode, 'watchdog cancellation delayed') + if (directory / 'pids').exists(): + for pid in (directory / 'pids').read_text().splitlines(): + assert not Path(f'/proc/{pid}').exists(), (shell, mode, 'orphan sleeper', pid) + + +check_watchdog_traps(['/bin/sh']) +busybox = shutil.which('busybox') +if busybox: + check_watchdog_traps([busybox, 'ash']) +for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')): + result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2) + assert result.returncode == 64 +print('entrypoint supervision fixtures passed') diff --git a/deploy/managed-target/compose.yaml b/deploy/managed-target/compose.yaml index 5c4dfc9..30b398c 100644 --- a/deploy/managed-target/compose.yaml +++ b/deploy/managed-target/compose.yaml @@ -31,7 +31,7 @@ services: - /run/zapret-autotune-ss:rw,noexec,nosuid,nodev,size=1m,mode=0700,uid=0,gid=0 networks: [target] restart: "no" - stop_grace_period: 120s + stop_grace_period: 240s networks: target: driver: bridge diff --git a/deploy/managed-target/zapret-autotune.service b/deploy/managed-target/zapret-autotune.service deleted file mode 100644 index 95b4250..0000000 --- a/deploy/managed-target/zapret-autotune.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Isolated zapret-autotune managed target -Requires=docker.service -After=docker.service -StartLimitIntervalSec=6h -StartLimitBurst=3 - -[Service] -Type=simple -WorkingDirectory=/opt/zapret2-autotune -ExecStart=/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml up --no-build --abort-on-container-exit --exit-code-from target target -ExecStop=/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml stop target -Restart=no -RestartForceExitStatus=75 -RestartSec=30s -TimeoutStopSec=180s - -[Install] -WantedBy=multi-user.target diff --git a/managed-target-entrypoint.sh b/managed-target-entrypoint.sh index 4827fa7..7556b36 100644 --- a/managed-target-entrypoint.sh +++ b/managed-target-entrypoint.sh @@ -30,54 +30,97 @@ server_pid= local_pid= monitor_pid= zapret_started=0 +cooldown_pid= +# Signal only descendants of an owned child, never the container/host namespace. +# The monitor gets TERM itself first so it can join scans and restore bytes; +# descendant signalling is reserved for SS shutdown and forced watchdog cleanup. +. /opt/managed-target-supervision.sh +pause() { + sleep "$1" /dev/null 2>&1 & cooldown_pid=$! + wait "$cooldown_pid" + cooldown_pid= +} # Wait with a watchdog: never let an unresponsive child exceed Docker's grace. bounded_wait() { child=$1 seconds=$2 - ( - sleep "$seconds" & sleeper=$! - trap 'kill "$sleeper" 2>/dev/null || true; exit 0' TERM INT - wait "$sleeper" - kill -KILL "$child" 2>/dev/null || true - ) /dev/null 2>&1 & + rm -f "$config_dir/join-timeout" "$config_dir/join-ready" + # Fresh interpreter: a forked shell can inherit a deferred EXIT/TERM trap + # context and never run its cancellation handler during PID 1 shutdown. + /bin/sh /opt/managed-target-supervision.sh --watchdog "$child" "$seconds" "$config_dir" /dev/null 2>&1 & watchdog=$! + # A fast-exiting child must not cancel the watchdog before its trap exists. + # Fail closed if exec/setup fails before acknowledgement; never wait forever. + ack_deadline=$(($(date +%s) + 5)) + while [ ! -e "$config_dir/join-ready" ]; do + if ! kill -0 "$watchdog" 2>/dev/null || [ "$(date +%s)" -ge "$ack_deadline" ]; then + : > "$config_dir/join-timeout" + signal_tree "$watchdog" KILL + signal_tree "$child" KILL + break + fi + done waited_code=0 wait "$child" 2>/dev/null || waited_code=$? kill -TERM "$watchdog" 2>/dev/null || true wait "$watchdog" 2>/dev/null || true + join_timed_out=0 + [ ! -e "$config_dir/join-timeout" ] || join_timed_out=1 } -cleanup() { - final_code=$? - case "$final_code" in 0|64|75|78) ;; *) final_code=78 ;; esac - trap - EXIT INT TERM +cleanup_cycle() { + cleanup_code=0 # Let the monitor cancel/join scans and restore committed bytes BEFORE stop. if [ -n "$monitor_pid" ]; then kill -TERM "$monitor_pid" 2>/dev/null || true - bounded_wait "$monitor_pid" 60 + bounded_wait "$monitor_pid" 45 # Cancellation must restore safely even when SS failure requested retry. - case "$waited_code" in 0|143) ;; *) final_code=78 ;; esac + case "$waited_code" in 0|143) ;; *) echo "error: monitor cleanup status $waited_code" >&2; cleanup_code=78 ;; esac + [ "$join_timed_out" -eq 0 ] || { echo 'error: monitor join timeout' >&2; cleanup_code=78; } + monitor_pid= fi # Dedicated PID/network namespace: no host/other-target stop. if [ "$zapret_started" -eq 1 ]; then - timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop || final_code=78 + # Stop also sources config: clear trust before reconciliation, never + # source unknown bytes left by a crashed monitor, even for shutdown. + zapret_started=0 + if timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile; then + timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop || cleanup_code=78 + else + cleanup_code=78 + fi fi for pid in "$local_pid" "$server_pid"; do - [ -z "$pid" ] || kill "$pid" 2>/dev/null || true + [ -z "$pid" ] || signal_tree "$pid" TERM done for pid in "$local_pid" "$server_pid"; do if [ -n "$pid" ]; then bounded_wait "$pid" 5 # Its original failure triggered supervision; only forced shutdown # is a cleanup failure, not the already-dead Shadowsocks status. - [ "$waited_code" -ne 137 ] || final_code=78 + [ "$join_timed_out" -eq 0 ] || { echo 'error: Shadowsocks join timeout' >&2; cleanup_code=78; } fi done + server_pid= + local_pid= + return "$cleanup_code" +} +cleanup() { + final_code=$? + trap - EXIT + # A caught (not ignored) signal lets watchdog children install TERM traps. + trap ':' INT TERM + if [ -n "$cooldown_pid" ]; then + kill "$cooldown_pid" 2>/dev/null || true + wait "$cooldown_pid" 2>/dev/null || true + fi + cleanup_cycle || final_code=78 exit "$final_code" } trap cleanup EXIT trap 'exit 0' INT TERM # Never invoke /opt/entrypoint.sh: zapret must start AFTER reconciliation. +run_cycle() { ss-server -c "$config_dir/server.json" & server_pid=$! ss-local -c "$config_dir/local.json" & @@ -86,10 +129,10 @@ local_pid=$! # Bounded readiness, no network probe. Managed commands verify socket ownership. attempt=0 until awk '$2 == "00000000:0438" && $4 == "0A" { found=1 } END { exit !found }' /proc/net/tcp; do - { kill -0 "$server_pid" && kill -0 "$local_pid"; } || exit 75 + { kill -0 "$server_pid" && kill -0 "$local_pid"; } || return 75 attempt=$((attempt + 1)) - [ "$attempt" -lt 30 ] || { echo 'error: target SOCKS listener timeout' >&2; exit 75; } - sleep 1 + [ "$attempt" -lt 30 ] || { echo 'error: target SOCKS listener timeout' >&2; return 75; } + pause 1 done if [ ! -e "$ZAPRET_CONFIG" ]; then timeout -k 5 45 /usr/local/bin/zapret-autotune managed init || exit 78 @@ -100,10 +143,14 @@ timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile || exit 78 zapret_started=1 timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || exit 78 +# A replacement SS child can fail during reconciliation/activation. Do not +# fork a monitor we already need to cancel: an early TERM can race interpreter +# startup before its cancellation handler exists. Clean this cycle directly. +{ kill -0 "$server_pid" 2>/dev/null && kill -0 "$local_pid" 2>/dev/null; } || return 75 /usr/local/bin/zapret-autotune managed monitor & monitor_pid=$! while kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do - sleep 1 + pause 1 done if ! kill -0 "$monitor_pid" 2>/dev/null; then code=0 @@ -115,7 +162,39 @@ if ! kill -0 "$monitor_pid" 2>/dev/null; then 64|75|78) ;; *) code=78 ;; esac - exit "$code" + return "$code" fi echo 'error: target child process exited' >&2 -exit 75 +return 75 +} + +# Three starts per rolling six hours, including initial startup. PID 1 owns +# the entire runtime cycle; cooldown is cancellable without any external manager. +starts= +backoff=15 +while :; do + now=$(date +%s) + recent= + count=0 + for started in $starts; do + if [ "$((now - started))" -lt 21600 ]; then + recent="$recent $started" + count=$((count + 1)) + fi + done + [ "$count" -lt 3 ] || { echo 'error: target restart budget exhausted' >&2; exit 78; } + starts="$recent $now" + code=0 + run_cycle || code=$? + cleanup_cycle || code=78 + [ "$code" -eq 75 ] || exit "$code" + # A long-running monitor may outlive the window; recheck at failure time. + now=$(date +%s) + retry_count=0 + for started in $starts; do + [ "$((now - started))" -ge 21600 ] || retry_count=$((retry_count + 1)) + done + [ "$retry_count" -lt 3 ] || { echo 'error: target restart budget exhausted' >&2; exit 78; } + pause "$backoff" + [ "$backoff" -ge 60 ] || backoff=$((backoff * 2)) +done diff --git a/managed-target-supervision.sh b/managed-target-supervision.sh new file mode 100644 index 0000000..9749b81 --- /dev/null +++ b/managed-target-supervision.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Shared owned-tree signalling; sourced by PID 1, executed afresh for watchdogs. +signal_tree() { + if [ -r "/proc/$1/task/$1/children" ]; then + descendants= + read -r descendants < "/proc/$1/task/$1/children" || true + for descendant in $descendants; do + signal_tree "$descendant" "$2" + done + fi + kill "-$2" "$1" 2>/dev/null || true +} + +if [ "${1-}" = --watchdog ]; then + set -eu + [ "$#" -eq 4 ] || exit 64 + case "$2" in ''|*[!0-9]*) exit 64 ;; esac + case "$3" in ''|*[!0-9]*) exit 64 ;; esac + [ "$2" -gt 1 ] && [ "$3" -gt 0 ] || exit 64 + child=$2 + seconds=$3 + config_dir=$4 + sleeper= + watchdog_cleanup() { + trap - EXIT + trap ':' INT TERM + if [ -n "$sleeper" ]; then + kill "$sleeper" 2>/dev/null || true + wait "$sleeper" 2>/dev/null || true + fi + } + trap watchdog_cleanup EXIT + trap 'exit 0' INT TERM + sleep "$seconds" & sleeper=$! + : > "$config_dir/join-ready" + wait "$sleeper" + : > "$config_dir/join-timeout" + signal_tree "$child" KILL +elif [ "$#" -ne 0 ]; then + exit 64 +fi