diff --git a/Dockerfile b/Dockerfile index 75cba62..c31b5f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.25.3 AS build +FROM golang:1.25.3@sha256:6d4e5e74f47db00f7f24da5f53c1b4198ae46862a47395e30477365458347bf2 AS build WORKDIR /src ENV CGO_ENABLED=0 GOTOOLCHAIN=local COPY go.mod go.sum ./ diff --git a/Dockerfile.target b/Dockerfile.target index 845cb9b..e9bebec 100644 --- a/Dockerfile.target +++ b/Dockerfile.target @@ -1,4 +1,4 @@ -FROM golang:1.25.3 AS build +FROM golang:1.25.3@sha256:6d4e5e74f47db00f7f24da5f53c1b4198ae46862a47395e30477365458347bf2 AS build WORKDIR /src ENV CGO_ENABLED=0 GOTOOLCHAIN=local COPY go.mod go.sum ./ diff --git a/MANAGED_TARGET.md b/MANAGED_TARGET.md index 36d0cf0..a307d99 100644 --- a/MANAGED_TARGET.md +++ b/MANAGED_TARGET.md @@ -135,7 +135,7 @@ identity/SOCKS guards. Fake-only checks are added, but tests/build are deferred to the orchestrator. `managed monitor` now binds those callbacks and only the dedicated target entrypoint starts it; no live/runtime validation has occurred. -## Dedicated monitor integration (verification deferred) +## Dedicated monitor integration (live acceptance pending) `zapret-autotune managed monitor [--interval 5m] [--failure-threshold 3] [--cooldown 30m]` uses the health environment defaults. Interval is positive and @@ -165,9 +165,9 @@ Entrypoint ordering is SS children → listener → managed init/status → zapr all three children; monitor exit is fatal even if it exits zero unexpectedly. SIGTERM/SIGINT cancels and waits the monitor before target-local zapret stop and SS child termination/wait. Existing targets and Xray remain unchanged. Deployment -must allow enough shutdown grace for independently bounded rollback/restoration -(potentially more than the currently defined 45s); this increment changes no Docker -configuration. Code integration is not image/routing/signal live verification. +allows 120s Compose shutdown grace and 180s systemd stop timeout for bounded +monitor cancellation/restoration and child joins. Code integration is not +image/routing/signal live verification. Injected unit tests cover exact identity/path/config, mountpoint refusal, old-container-like env, SOCKS port/process refusal, correct fresh-target success, @@ -178,3 +178,155 @@ deferred to the orchestrator; not run by this increment: go test ./cmd/zapret-autotune -run TestManaged go build ./cmd/zapret-autotune ``` + +## Production preparation + +This is a standalone **new isolated target** candidate, not a production-approved +deployment and not an integration into existing zapret/Xray. Keep the exact +`managed-test` role, ID, marker, mount, config, DB and owned-listener guards above. +Do not change the role to `production`, adopt old state, share an old network +namespace, or relax a refusal. Provisioning and optional service installation +are described in [deploy/managed-target/README.md](deploy/managed-target/README.md). + +### Build once, promote the same image + +Use `Dockerfile.target` for the supervised target; root `Dockerfile` is the +proposal CLI image, not the managed service. Both pin Go 1.25.3 and the runtime +by digest, use `GOTOOLCHAIN=local`, `CGO_ENABLED=0`, `go.sum`, and `-trimpath`. +Pins constrain build inputs; they do not establish live acceptance or guarantee +bit-for-bit builds on different platforms. Record the source commit, platform, +builder/runtime digests and resulting image digest for every release. + +Example (replace registry and release; never reuse a release tag): + +```sh +IMAGE=registry.example/team/zapret-autotune-managed-target:2026-10-10-r1 +docker build --pull -f Dockerfile.target -t "$IMAGE" . +docker push "$IMAGE" +docker image inspect --format '{{index .RepoDigests 0}}' "$IMAGE" +``` + +Promote the printed `registry/repository@sha256:...` reference unchanged from +validation to deployment; do not rebuild or use `latest`. Authenticate separately, +never put registry credentials in these commands. The supplied Compose file +selects `${ZAPRET_AUTOTUNE_IMAGE:-zapret-autotune-managed-target:local}`. Set +`ZAPRET_AUTOTUNE_IMAGE=registry.example/team/zapret-autotune-managed-target@sha256:` +in the same private deployment `.env` as `SS_PASSWORD` (mode `0600`), using the +actual printed digest. Tags and the local default are build/test candidates only; +an immutable managed-target digest is required for production promotion. From +the repository root, validate and pull without printing the rendered secrets: + +```sh +docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml config --quiet +docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml pull target +``` + +The supplied systemd unit uses this same `.env` and starts with `--no-build`; +no image-selection override or unit edit is needed. Keep the selected digest and +private configuration together for upgrades and rollback. Never run a build +against the production digest selector; build/tag separately as shown above. + +### Configuration and secrets + +Create fresh empty `target-state` owned by root/runtime UID 0, mode `0700`, and +the private deployment `.env` mode `0600`; rootless/remapped Docker needs the +mapped owner. Bootstrap is for empty provisioned state only; recreation retains +identity, journal, candidate DB, sidecars and lock file. Never automatically wipe +state to make startup succeed. The entrypoint creates Shadowsocks JSON in private +tmpfs (`0700` directory, `0600` files), unsets the password before children, and +does not put it in argv. Docker administrators/root can still read container env +and runtime config. Do not print `.env`, rendered Compose config, diagnostics or +signed URLs; use `docker compose ... config --quiet` for validation. + +| Existing setting | Default / accepted value for managed target | +| --- | --- | +| `ZAPRET_AUTOTUNE_IMAGE` | Local build/test default `zapret-autotune-managed-target:local`; production promotion requires a managed-target `repository@sha256:` | +| `SS_PASSWORD` | Required independent hex-only secret, at least 32 characters; deployment README generates 64 | +| `ZAPRET_AUTOTUNE_HOST_SOCKS_PORT` | `11080`; choose an available TCP/UDP port 1–65535, published only on `127.0.0.1` | +| `SOCKS_PORT` | Supplied deployment fixes internal listener to `1080`; not a host-port override | +| `ZAPRET_AUTOTUNE_SOCKS_PROXY` | Fixed `socks5://127.0.0.1:1080` inside this target, no credentials/direct fallback | +| `ZAPRET_AUTOTUNE_VIDEO_ID` | `dQw4w9WgXcQ`; exactly 11 ASCII letters, digits, `_` or `-` | +| `ZAPRET_AUTOTUNE_HEALTH_INTERVAL` | `5m`; duration in `(0,24h]` for monitor | +| `ZAPRET_AUTOTUNE_FAILURE_THRESHOLD` | `3`; integer 1–1000 for monitor | +| `ZAPRET_AUTOTUNE_RECOVERY_COOLDOWN` | `30m`; duration in `(0,168h]` for monitor | +| `ZAPRET_AUTOTUNE_DB` | Fixed `/var/lib/zapret-autotune/target/candidates.db` | + +The monitor's three policy flags override corresponding environment defaults; +explicit empty/invalid environment settings are rejected. The entrypoint accepts +no arguments. The supplied Compose environment does not forward the video/policy +settings from `.env`: add the desired existing variables to `target.environment` +in your deployment copy/override if needed. Keep all fixed identity/path settings +unchanged. No public SS listener is published; Xray configuration is external and +operator-owned, optionally using the host's loopback SOCKS port. Do not modify an +existing Xray deployment as part of bootstrap. + +### Supervision and failure handling + +Compose restart is disabled. Optional native systemd runs **foreground** Compose +with `--no-build --abort-on-container-exit --exit-code-from target`; do not also +run detached Compose for that project. The unit requests restart only for exit +75 via `Restart=no` plus `RestartForceExitStatus=75`, bounded by +`StartLimitBurst=3` in six hours (starts, including the initial +start, not three additional retries). Compose stop grace is 120s, systemd 180s. + +| Target exit | Operator meaning | +| --- | --- | +| `64` | Invalid configuration; fix settings before starting | +| `78` | Safety/DB/journal/rollback/cleanup failure; investigate, no automatic restart | +| `75` | Technical failure eligible for bounded restart only after safe cleanup | +| `0` | Graceful signal shutdown; no automatic restart | + +Bootstrap/recovery has a fixed **90-minute whole-job deadline**, plus an +independent **30-second restoration reserve**. This is a catastrophic technical +ceiling, not a normal candidate-check timeout and not an operator tunable. Valid +nonpassing candidates/unresolved GGC are normal recovery outcomes governed by +the persisted threshold/cooldown, not a reason to restart the container. Failed +restoration or unsafe state vetoes retry: timeout exit 75 requires the job's own +deadline and only timeout/cancellation-related errors, with no joined cleanup, +DB or safety failure. Unexpected monitor/SS failures are separately technical +only if shutdown cleanup succeeds. Shutdown joins the monitor for up to 60s, +target-local zapret stop gets 30s then forced kill after five more, and SS children +get 5s each; failure to join/restore is fail-closed. The reserve is not extra +scanning time. The 90-minute budget is shared by the whole bootstrap/recovery +job, not renewed per candidate or phase. + +Monitor health means Player API `OK` through owned SOCKS. GGC evidence covers +only the **exact discovered hostname** and tested TLS protocols, not wildcard +googlevideo coverage, video transfer, SABR or playback. Unresolved GGC must not +be reported as complete playback health. + +### Upgrade, rollback and operator recovery + +1. Record the current image digest and private deployment configuration. Stop + the service (or foreground Compose), wait for complete cleanup, and verify + the target is stopped. If cleanup fails, investigate before upgrading. +2. With the target stopped, make a root-only backup of the **entire** state + directory (including DB sidecars, journal, config, identity and lock file), + preserving ownership/modes. Store `.env` and image/config records privately + with it; do not print archive contents, diagnostics or signed URLs. A live + copy of only `candidates.db` is not a consistent rollback backup. +3. Select/pull the new immutable digest and start without building. Inspect + sanitized status/logs, verify identity/hash and expected active groups, owned + SOCKS, target-local queues, health scope and persistent recreation. Keep the + old image and backup until acceptance; DB migrations may prevent image-only + downgrade. +4. To roll back, stop and verify cleanup again. Preserve failed state separately + for investigation. Restore the matched stopped-state backup **as a whole** + into the same dedicated mount with private ownership/modes, select its old + digest/config, then start. Do not merge journals/DB generations or re-init. + A restored legacy/live/unverifiable recovery owner can fail closed; never + clear its lease or unlink the lock merely to force startup. + +On drift, pending transaction, rollback failure, or exhausted systemd start +limit, leave state intact and service stopped. Diagnose using sanitized status +and private artifacts; repair the actual provisioning/configuration issue or +restore a known consistent backup. Only then manually start (and reset the +systemd failed/start-limit state if needed). There is no automatic wipe or +automatic adoption path. + +Rollout acceptance remains pending real isolated-target restart/deadline tests: +empty bootstrap, saved-first recovery, exact-host GGC rotation, image binding, +NFQUEUE activation/rollback, SIGTERM during downtime, deadline restoration, +retry limits/exit propagation through foreground Compose/systemd, persistent +recreation and backup restore. Perform these on the new target only; local unit +or static checks cannot establish production readiness. diff --git a/README.md b/README.md index 56d49e2..dac317d 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,24 @@ Go CLI for scanning and checking YouTube TLS bypass strategies with `blockcheckw`. +## Production preparation (acceptance pending) + +The standalone deployment candidate is a **new isolated target**, never an +upgrade/adoption of an existing zapret or Xray service. The legacy `managed-test` +role and fixed identity guards remain intentional, including for prod-like runs. +See [release, configuration, upgrade and rollback guidance](MANAGED_TARGET.md#production-preparation) +and the [deployment/service instructions](deploy/managed-target/README.md). +Pinned builds and supervision are preparation, not production acceptance: real +restart/deadline tests and end-to-end target validation remain rollout gates. +Select an immutable managed-target digest with `ZAPRET_AUTOTUNE_IMAGE` in the +private deployment `.env`; tags/local defaults are only build/test candidates. +Bootstrap/recovery shares a 90-minute whole-job deadline plus a separate 30-second +restoration reserve, an abnormal ceiling rather than normal candidate failure. +Timeout exit 75 requires clean timeout/cancellation only; cleanup/safety errors +veto retry. Optional foreground systemd uses `Restart=no` plus forced exit-75 +restart, limited to three starts per six hours; Compose stop grace is 120s and +systemd stop timeout 180s. + ## Manual managed-test activation (not deployed) `managed init|status|active-ids|apply` requires exact role `managed-test`, target ID @@ -20,7 +38,7 @@ group, fixed target init executable and explicit runtime-config binding. It uses full target-local firewall/daemon restart and Player API post-check, with atomic rollback on failure. See [MANAGED_TARGET.md](MANAGED_TARGET.md) for guards and provisioning prerequisites. An isolated pinned deployment definition is in -[deploy/managed-target](deploy/managed-target/README.md); build/run are deferred. +[deploy/managed-target](deploy/managed-target/README.md); acceptance remains pending. An injected bootstrap/saved-first recovery adapter now connects the persisted monitor foundation to blockcheck, immutable candidates and transactional apply. `managed monitor` and the dedicated target entrypoint now wire these adapters; diff --git a/cmd/zapret-autotune/blockcheck.go b/cmd/zapret-autotune/blockcheck.go index 111c31f..cd44b45 100644 --- a/cmd/zapret-autotune/blockcheck.go +++ b/cmd/zapret-autotune/blockcheck.go @@ -21,7 +21,7 @@ func managedSeedArgs(host string) string { host = strings.ToLower(host) switch { case host == "www.youtube.com": - return "--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:ip_autottl=-3,3-20:repeats=1 --lua-desync=multidisorder:pos=1,midsld" + return "--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld" case host == "googlevideo.com" || strings.HasSuffix(host, ".googlevideo.com"): return "--payload=tls_client_hello --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN" default: diff --git a/cmd/zapret-autotune/blockcheck_test.go b/cmd/zapret-autotune/blockcheck_test.go index d7c2c23..ce23800 100644 --- a/cmd/zapret-autotune/blockcheck_test.go +++ b/cmd/zapret-autotune/blockcheck_test.go @@ -253,6 +253,13 @@ func TestRunManagedBlockcheckProtectsForeignWorkers(t *testing.T) { func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) { t.Setenv("TMPDIR", t.TempDir()) args := managedSeedArgs("www.youtube.com") + const wantSeed = "--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld" + if args != wantSeed { + t.Fatalf("frontend seed=%q, want %q", args, wantSeed) + } + if err := (&candidate{Args: args}).validateArgs(); err != nil { + t.Fatalf("invalid frontend seed grammar: %v", err) + } t.Setenv("ZAPRET_TEST_REPORT", combinedReportFixture(args)) argvPath := filepath.Join(t.TempDir(), "argv.jsonl") t.Setenv("ZAPRET_TEST_ARGV", argvPath) @@ -279,10 +286,23 @@ func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) { if err := decoder.Decode(&checkArgs); err != nil { t.Fatal(err) } - if !slices.Contains(checkArgs, "--from-file") || !slices.Contains(checkArgs, "--passes") || !slices.Contains(checkArgs, "--no-conflict-cleanup") { + wantCheck := []string{"--workers", "32", "check", "--from-file", path + ".input", "--domain", "www.youtube.com", "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", "--output", path + ".check.json"} + if !reflect.DeepEqual(checkArgs, wantCheck) { t.Fatalf("managed seeded check=%q", checkArgs) } - if seed := managedSeedArgs("rr3.googlevideo.com"); !strings.Contains(seed, "fakeddisorder") { + if err := decoder.Decode(&checkArgs); err != io.EOF { + t.Fatalf("unexpected extra blockcheck call: %q %v", checkArgs, err) + } + input, err := os.ReadFile(path + ".input") + if err != nil { + t.Fatal(err) + } + wantRows := "curl_test_https_tls12 ipv4 www.youtube.com : nfqws2 " + wantSeed + "\n" + + "curl_test_https_tls13 ipv4 www.youtube.com : nfqws2 " + wantSeed + "\n" + if string(input) != wantRows { + t.Fatalf("managed seed input=%q", input) + } + if seed := managedSeedArgs("rr3.googlevideo.com"); seed != "--payload=tls_client_hello --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN" { t.Fatalf("missing exact-host GGC seed: %q", seed) } } diff --git a/cmd/zapret-autotune/managed_monitor_cli.go b/cmd/zapret-autotune/managed_monitor_cli.go index aa1ab75..0bf1090 100644 --- a/cmd/zapret-autotune/managed_monitor_cli.go +++ b/cmd/zapret-autotune/managed_monitor_cli.go @@ -56,36 +56,41 @@ func managedMonitorCLI(args []string, stdout, stderr io.Writer, target managedDe return managedMonitorCLIContext(ctx, args, stdout, stderr, target, runManagedMonitor) } -func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) int { +func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) (code int) { cfg, err := managedMonitorConfig(args, target.lookup) if err != nil { fmt.Fprintln(stderr, "error: invalid managed monitor configuration") - return 2 + return 64 } - if code, err := managedRuntimeGuard(target); err != nil { + if _, err := managedRuntimeGuard(target); err != nil { fmt.Fprintln(stderr, "error: managed monitor target guard refused") - return code + return 78 } m := &managedTransaction{Dir: target.directory, Name: "config"} if err := managedDirectory(target.directory); err != nil { fmt.Fprintln(stderr, "error: unsafe managed monitor directory") - return 1 + return 78 } if err := managedCheckIdentity(m); err != nil { fmt.Fprintln(stderr, "error: managed monitor identity refused") - return 1 + return 78 } path, err := managedMonitorDatabase(target) if err != nil { fmt.Fprintln(stderr, "error: unsafe managed monitor database") - return 1 + return 78 } db, err := openStore(path) if err != nil { fmt.Fprintln(stderr, "error: managed monitor database unavailable") - return 1 + return 78 } - defer db.Close() + defer func() { + if err := db.Close(); err != nil { + fmt.Fprintln(stderr, "error: managed monitor database close failed") + code = 78 + } + }() recovery, err := managedMonitorRecoveryAdapters(target, db, runManagedBlockcheck, runManagedCombinedBlockcheckForHost, func(ctx context.Context, path string, args ...string) error { log, err := os.CreateTemp(target.directory, ".daemon-diagnostics-*.log") if err != nil { @@ -98,7 +103,7 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr }) if err != nil { fmt.Fprintln(stderr, "error: managed monitor adapters refused") - return 1 + return 78 } monitor := managedMonitorDeps{ target: target, db: db, serviceID: managedTargetID + "/youtube", @@ -110,7 +115,7 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr fmt.Fprintln(stderr, "managed GGC unresolved; monitoring continues (not playback health)") }); err != nil { fmt.Fprintln(stderr, "error: managed monitor wiring refused") - return 1 + return 78 } fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health") err = supervise(ctx, monitor) @@ -120,9 +125,34 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr return 0 } fmt.Fprintln(stderr, "error: managed monitor stopped; safety/database/runtime failure (details suppressed)") - return 1 + if managedMonitorTechnicalTimeout(err) { + return 75 + } + return 78 } - return 0 + if ctx.Err() != nil { + return 0 + } + return 75 // A supervisor must not return normally without cancellation. +} + +// Only clean typed timeouts may restart; a joined cleanup failure vetoes it. +func managedMonitorTechnicalTimeout(err error) bool { + if timeout, ok := err.(*managedRecoveryTimeout); ok { + return managedRecoveryTimeoutSafe(timeout.Err) + } + if joined, ok := err.(interface{ Unwrap() []error }); ok { + for _, e := range joined.Unwrap() { + if !managedMonitorTechnicalTimeout(e) { + return false + } + } + return len(joined.Unwrap()) != 0 + } + if wrapped, ok := err.(interface{ Unwrap() error }); ok { + return managedMonitorTechnicalTimeout(wrapped.Unwrap()) + } + return false } func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand cacheScan, checkCommand cacheCheck, command func(context.Context, string, ...string) error) (managedRecoveryDeps, error) { diff --git a/cmd/zapret-autotune/managed_monitor_cli_test.go b/cmd/zapret-autotune/managed_monitor_cli_test.go index 2aad4a7..a69961d 100644 --- a/cmd/zapret-autotune/managed_monitor_cli_test.go +++ b/cmd/zapret-autotune/managed_monitor_cli_test.go @@ -40,6 +40,64 @@ func TestManagedMonitorCancellationSafety(t *testing.T) { } } +func TestManagedMonitorExitClassification(t *testing.T) { + for _, mode := range []string{"timeout", "timeout-cleanup", "timeout-inner-cleanup", "fatal-timeout", "fatal-timeout-cleanup", "timeout-fatal", "job-timeout", "job-timeout-cleanup", "unknown", "zero", "cancel", "cancel-cleanup"} { + t.Run(mode, func(t *testing.T) { + d, env, _ := managedCLIFixture(t) + initManagedCLI(t, d) + env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + want := 78 + var out bytes.Buffer + code := managedMonitorCLIContext(ctx, nil, &out, &out, d, func(context.Context, managedMonitorDeps) error { + switch mode { + case "timeout": + want = 75 + return &managedRecoveryTimeout{Err: context.DeadlineExceeded} + case "timeout-cleanup": + return errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("diagnostic close failed")) + case "timeout-inner-cleanup": + return &managedRecoveryTimeout{Err: errors.Join(context.DeadlineExceeded, errors.New("diagnostic close failed"))} + case "fatal-timeout": + want = 75 // Fatal is also the monitor's stop signal for clean timeout. + return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: context.DeadlineExceeded}} + case "fatal-timeout-cleanup": + return &managedRecoveryFatal{Err: errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("cleanup"))} + case "timeout-fatal": + return &managedRecoveryTimeout{Err: &managedRecoveryFatal{Err: errors.New("cleanup")}} + case "job-timeout", "job-timeout-cleanup": + if mode == "job-timeout" { + want = 75 + } + return managedRecoveryJob(ctx, time.Millisecond, func(jobCtx context.Context) error { + <-jobCtx.Done() + if mode == "job-timeout-cleanup" { + return &managedRecoveryFatal{Err: errors.Join(jobCtx.Err(), errors.New("cleanup"))} + } + return jobCtx.Err() + }) + case "zero": + want = 75 + return nil + case "cancel": + want = 0 + cancel() + return ctx.Err() + case "cancel-cleanup": + cancel() + return errors.Join(ctx.Err(), errors.New("cleanup")) + default: + return errors.New("unknown") + } + }) + if code != want { + t.Fatal(code, want, out.String()) + } + }) + } +} + func TestManagedMonitorProductionAdapterBinding(t *testing.T) { recovery, _, _ := recoveryFixture(t, false) target, _, _ := managedCLIFixture(t) @@ -132,7 +190,11 @@ func TestManagedMonitorCLIGuardsBeforeLock(t *testing.T) { } var out bytes.Buffer code := managedMonitorCLIContext(context.Background(), nil, &out, &out, d, func(context.Context, managedMonitorDeps) error { t.Fatal("unsafe supervisor start"); return nil }) - if code == 0 || strings.Contains(out.String(), "password-do-not-print") { + want := 78 + if mode == "config" { + want = 64 + } + if code != want || strings.Contains(out.String(), "password-do-not-print") { t.Fatal(code, out.String()) } if _, err := os.Lstat(filepath.Join(d.directory, managedMonitorLockName)); !errors.Is(err, os.ErrNotExist) { @@ -199,7 +261,7 @@ func TestManagedMonitorBootstrapExistingFrontend(t *testing.T) { // Execute only rewritten target-local contracts and fake programs; no network, // Docker, real credentials, zapret or host firewall. Exercise the actual traps. func TestManagedEntrypointSupervision(t *testing.T) { - for _, mode := range []string{"term", "fatal"} { + for _, mode := range []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"} { t.Run(mode, func(t *testing.T) { root := t.TempDir() config := filepath.Join(root, "ss") @@ -215,16 +277,23 @@ func TestManagedEntrypointSupervision(t *testing.T) { } log := filepath.Join(root, "log") write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n") - child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do sleep 0.1 & wait $!; done\n" + child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & wait $!; done\n" write("ss-server", child) write("ss-local", child) - zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\n") - monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then echo managed-$2 >> \"$LOG\"; exit 0; fi\ntrap 'echo monitor-stopped >> \"$LOG\"; exit 0' TERM INT\necho managed-monitor >> \"$LOG\"\nif [ \"$MODE\" = fatal ]; then exit 7; fi\nwhile :; do sleep 0.1 & wait $!; done\n") + zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nif [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; while :; do sleep 0.1; done; fi\n") + monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then echo managed-$2 >> \"$LOG\"; exit 0; fi\ntrap 'echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; fi\necho managed-monitor >> \"$LOG\"\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & wait $!; done\n") data, err := os.ReadFile("../../managed-target-entrypoint.sh") if err != nil { t.Fatal(err) } text := strings.NewReplacer("/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data)) + if mode == "term-hang" { + // Shorten only the fixture's watchdog, not the production contract. + text = strings.Replace(text, `bounded_wait "$monitor_pid" 60`, `bounded_wait "$monitor_pid" 1`, 1) + } + if mode == "stop-hang" { + text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1) + } entry := write("entrypoint", text) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -240,7 +309,7 @@ func TestManagedEntrypointSupervision(t *testing.T) { exited := make(chan struct{}) go func() { done <- cmd.Wait(); close(exited) }() t.Cleanup(func() { syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); <-exited }) - if mode == "term" { + if strings.HasPrefix(mode, "term") { for { b, _ := os.ReadFile(log) if strings.Contains(string(b), "managed-monitor") { @@ -262,13 +331,20 @@ func TestManagedEntrypointSupervision(t *testing.T) { if ctx.Err() != nil { t.Fatal("entrypoint did not join children", ctx.Err()) } - if mode == "term" && err != nil { + want := 75 + switch mode { + case "term": + want = 0 + case "fatal", "unknown", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang": + want = 78 + } + if want == 0 && err != nil { t.Fatal(err, output.String()) } - if mode == "fatal" { + if want != 0 { var exit *exec.ExitError - if !errors.As(err, &exit) || exit.ExitCode() != 7 { - t.Fatal(err) + if !errors.As(err, &exit) || exit.ExitCode() != want { + t.Fatal(err, "want", want, output.String()) } } b, err := os.ReadFile(log) diff --git a/cmd/zapret-autotune/managed_recovery.go b/cmd/zapret-autotune/managed_recovery.go index 66f4545..4b266ba 100644 --- a/cmd/zapret-autotune/managed_recovery.go +++ b/cmd/zapret-autotune/managed_recovery.go @@ -9,12 +9,16 @@ import ( "fmt" "io" "os" + "os/exec" "strings" + "syscall" "time" ) const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2" +const managedRecoveryJobTimeout = 90 * time.Minute + // Caller owns startup reconciliation and the monitor singleton. Discover and // Health must bind to the guarded target SOCKS. No CLI parsing or remote work. type managedRecoveryDeps struct { @@ -25,6 +29,7 @@ type managedRecoveryDeps struct { discover func(context.Context) (string, error) health func(context.Context) (string, error) run func(context.Context, string, ...string) error + jobTimeout time.Duration // Zero uses the production budget; tests may shorten it. } // Target-local adapter: reuses bounded Player requests without invoking a CLI. @@ -55,7 +60,7 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, return target.player(bounded, args, out, diagnostics) }) if bounded.Err() != nil { - return nil, bounded.Err() + return nil, managedRecoveryContextError(ctx, errors.Join(err, bounded.Err())) } return body, err } @@ -95,11 +100,87 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, type managedRecoveryFatal struct{ Err error } +// Only clean job expiration is restartable; safety failures retain their type. +type managedRecoveryTimeout struct{ Err error } + +func (e *managedRecoveryTimeout) Error() string { return e.Err.Error() } +func (e *managedRecoveryTimeout) Unwrap() error { return e.Err } + var ( - errManagedGGCUnresolved = errors.New("managed GGC scope unresolved") - errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing") + errManagedGGCUnresolved = errors.New("managed GGC scope unresolved") + errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing") + errManagedRecoveryDeadline = errors.New("managed recovery total deadline exceeded") ) +// Validate every leaf, not just errors.Is: a joined cleanup failure must never +// authorize a restart. Callers must separately prove the job's own deadline. +func managedRecoveryTimeoutSafe(err error) bool { + if err == nil { + return true + } + switch e := err.(type) { + case *managedRecoveryFatal: + return false + case *exec.ExitError: + if e.ProcessState == nil { + return false + } + status, ok := e.Sys().(syscall.WaitStatus) + return ok && status.Signaled() && (status.Signal() == syscall.SIGTERM || status.Signal() == syscall.SIGKILL) + case interface{ Unwrap() []error }: + children := e.Unwrap() + if len(children) == 0 { + return false + } + for _, child := range children { + if !managedRecoveryTimeoutSafe(child) { + return false + } + } + return true + case interface{ Unwrap() error }: + child := e.Unwrap() + return child != nil && managedRecoveryTimeoutSafe(child) + default: + return err == context.Canceled || err == context.DeadlineExceeded || err == errManagedRecoveryDeadline + } +} + +// Cancellation must prevent late success without erasing a concurrent safety +// or cleanup failure. Request-local timeouts alone leave the job context live. +func managedRecoveryContextError(ctx context.Context, err error) error { + if ctx.Err() == nil { + return err + } + joined := errors.Join(err, ctx.Err()) + if !managedRecoveryTimeoutSafe(err) { + return &managedRecoveryFatal{Err: joined} + } + return joined +} + +// Synchronous callbacks own cancellation cleanup/join. Classify expiration only +// after they return, so the monitor cannot release ownership before restoration. +func managedRecoveryJob(ctx context.Context, budget time.Duration, job func(context.Context) error) error { + if budget == 0 { + budget = managedRecoveryJobTimeout + } + jobCtx, cancel := context.WithTimeoutCause(ctx, budget, errManagedRecoveryDeadline) + defer cancel() + err := job(jobCtx) + if errors.Is(context.Cause(jobCtx), errManagedRecoveryDeadline) { + joined := errors.Join(err, errManagedRecoveryDeadline, context.DeadlineExceeded) + if !managedRecoveryTimeoutSafe(err) { + return &managedRecoveryFatal{Err: joined} + } + return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: joined}} + } + if err == nil { + return jobCtx.Err() + } + return err +} + func (e *managedRecoveryFatal) Error() string { return "managed recovery safety failure: " + e.Err.Error() } @@ -119,15 +200,32 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma } m := *d.transaction // Rollback must restart even after job cancellation; independently bounded. + var cleanupCtx context.Context + var cleanupCancel context.CancelFunc + cleanupContext := func() context.Context { + if cleanupCtx == nil { + cleanupCtx, cleanupCancel = context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + } + return cleanupCtx + } + defer func() { + if cleanupCancel != nil { + cleanupCancel() + } + }() m.Reload = func() error { - cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) - defer cancel() - if err := d.run(cleanup, managedZapretScript, "restart"); err != nil { + if err := d.run(cleanupContext(), managedZapretScript, "restart"); err != nil { return &managedRecoveryFatal{Err: err} } return nil } - m.Health = func() (string, error) { return d.health(ctx) } + m.Health = func() (string, error) { + status, err := d.health(ctx) + if ctx.Err() != nil { + return "", managedRecoveryContextError(ctx, err) + } + return status, err + } ids, err := m.ActiveCandidates(d.db) if err != nil { return result, &managedRecoveryFatal{Err: err} @@ -150,9 +248,7 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma err = &managedRecoveryFatal{Err: errors.Join(err, e)} return } - cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) - defer cancel() - if e := d.run(cleanup, managedZapretScript, "start"); e != nil { + if e := d.run(cleanupContext(), managedZapretScript, "start"); e != nil { err = &managedRecoveryFatal{Err: errors.Join(err, e)} } } @@ -180,6 +276,14 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma return nil } defer func() { m.Reload = cleanupReload }() + defer func() { + // A rejected candidate's rollback is complete before the next attempt; + // that later attempt needs its own restoration reserve. + if ctx.Err() == nil && cleanupCancel != nil { + cleanupCancel() + cleanupCtx, cleanupCancel = nil, nil + } + }() e := m.Apply(d.db, ids) if e == nil { stopped = false @@ -197,12 +301,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma healthy := false if result.FrontendID != 0 { status, e := d.health(ctx) + e = managedRecoveryContextError(ctx, e) var fatal *managedRecoveryFatal if errors.As(e, &fatal) || errors.Is(e, context.Canceled) { return result, e } if err := ctx.Err(); err != nil { - return result, err + return result, e } // A bounded Player request timeout is unhealthy, not job cancellation. healthy = e == nil && status == "OK" @@ -272,8 +377,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma } } host, e := d.discover(ctx) + e = managedRecoveryContextError(ctx, e) + var discoveryFatal *managedRecoveryFatal + if errors.As(e, &discoveryFatal) { + return result, e + } if ctx.Err() != nil { - return result, ctx.Err() + return result, e } if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") { result.GGCError = fmt.Errorf("exact GGC discovery failed") @@ -302,12 +412,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma e = apply([]int64{result.FrontendID, cacheID}) } if e != nil { + e = managedRecoveryContextError(ctx, e) var fatal *managedRecoveryFatal if errors.As(e, &fatal) { return result, e } if ctx.Err() != nil { - return result, ctx.Err() + return result, e } result.GGCError = fmt.Errorf("GGC validation or activation unresolved") return result, nil @@ -441,7 +552,7 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe // are already typed managedRecoveryFatal by the transaction boundary. return err } - monitor.bootstrap = func(ctx context.Context) error { + bootstrap := func(ctx context.Context) error { if err := guard(); err != nil { return &managedRecoveryFatal{Err: err} } @@ -471,12 +582,18 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe } return callback(ctx) } - monitor.recover = callback + monitor.bootstrap = func(ctx context.Context) error { + return managedRecoveryJob(ctx, recovery.jobTimeout, bootstrap) + } + monitor.recover = func(ctx context.Context) error { + return managedRecoveryJob(ctx, recovery.jobTimeout, callback) + } monitor.probe = func(ctx context.Context) error { if err := guard(); err != nil { return &managedRecoveryFatal{Err: err} } status, err := recovery.health(ctx) + err = managedRecoveryContextError(ctx, err) if err != nil { return err } @@ -494,8 +611,9 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe // Keep Player API health separate from GGC scope (not playback health). // Two bounded Player requests reuse the existing health/discovery adapters. host, err := recovery.discover(ctx) + err = managedRecoveryContextError(ctx, err) if ctx.Err() != nil { - return ctx.Err() + return err } var fatal *managedRecoveryFatal if errors.As(err, &fatal) || errors.Is(err, context.Canceled) { diff --git a/cmd/zapret-autotune/managed_recovery_test.go b/cmd/zapret-autotune/managed_recovery_test.go index 0de524f..798442b 100644 --- a/cmd/zapret-autotune/managed_recovery_test.go +++ b/cmd/zapret-autotune/managed_recovery_test.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "os/exec" "path/filepath" "reflect" "strings" @@ -85,6 +86,367 @@ func TestManagedRecoveryBootstrap(t *testing.T) { } } +func TestManagedRecoveryJobDeadline(t *testing.T) { + for _, mode := range []string{"bootstrap-scan", "bootstrap-saved", "periodic-saved", "periodic-apply", "periodic-ggc", "parent-cancel", "parent-deadline"} { + t.Run(mode, func(t *testing.T) { + bootstrap := strings.HasPrefix(mode, "bootstrap") + d, front, events := recoveryFixture(t, !bootstrap) + if mode == "bootstrap-saved" || mode == "periodic-saved" || mode == "periodic-apply" { + managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true) + } + if err := os.Chmod(d.transaction.Dir, 0700); err != nil { + t.Fatal(err) + } + monitor := monitorRuntimeFixture(t) + monitor.db, monitor.target.directory = d.db, d.transaction.Dir + identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}` + if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil { + t.Fatal(err) + } + d.jobTimeout = 50 * time.Millisecond + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + if mode == "parent-deadline" { + var deadlineCancel context.CancelFunc + parent, deadlineCancel = context.WithTimeout(parent, 50*time.Millisecond) + defer deadlineCancel() + d.jobTimeout = time.Minute + } + joined := false + var jobDeadline time.Time + observe := func(ctx context.Context) { + deadline, ok := ctx.Deadline() + if !ok || (!jobDeadline.IsZero() && !deadline.Equal(jobDeadline)) { + t.Fatal("job deadline missing or reset between phases") + } + jobDeadline = deadline + } + block := func(ctx context.Context) error { + observe(ctx) + // Model the runner contract: cancel the owned worker and join before + // returning control to restoration and the supervisor. + done := make(chan struct{}) + go func() { <-ctx.Done(); close(done) }() + if mode == "parent-cancel" { + cancel() + } + <-done + joined = true + return ctx.Err() + } + baseScan, baseCheck := d.scan, d.check + d.scan = func(ctx context.Context, binary, host string, out, diagnostics io.Writer) (string, error) { + observe(ctx) + if mode == "bootstrap-scan" || mode == "parent-cancel" || mode == "parent-deadline" { + return "", block(ctx) + } + return baseScan(ctx, binary, host, out, diagnostics) + } + d.check = func(ctx context.Context, binary, path, args, host string, out, diagnostics io.Writer) error { + observe(ctx) + if mode != "periodic-apply" { + return block(ctx) + } + return baseCheck(ctx, binary, path, args, host, out, diagnostics) + } + healthCalls := 0 + d.health = func(ctx context.Context) (string, error) { + observe(ctx) + healthCalls++ + if mode == "periodic-apply" && healthCalls > 1 { + // Even a late OK must not commit a partially activated candidate. + _ = block(ctx) + return "OK", nil + } + if mode == "periodic-ggc" { + return "OK", nil + } + return "DEGRADED", nil + } + baseRun := d.run + d.run = func(ctx context.Context, path string, args ...string) error { + if args[0] == "start" || args[0] == "restart" && joined { + if !joined || ctx.Err() != nil { + t.Fatal("restoration preceded worker join or inherited cancellation") + } + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > 30*time.Second { + t.Fatal("restoration is not bounded") + } + } + return baseRun(ctx, path, args...) + } + if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("deadline became ordinary GGC failure") }); err != nil { + t.Fatal(err) + } + monitor.threshold = 1 + monitor.probe = func(context.Context) error { return errors.New("unhealthy") } + reports := 0 + monitor.reportRecoveryError = func(error) { reports++ } + monitor.wait = func(ctx context.Context, _ time.Duration) error { + t.Fatal("expired recovery entered ordinary cooldown retry") + return ctx.Err() + } + err := runManagedMonitor(parent, monitor) + var fatal *managedRecoveryFatal + var timeout *managedRecoveryTimeout + external := strings.HasPrefix(mode, "parent") + if errors.As(err, &fatal) == external || !joined { + t.Fatalf("err=%v joined=%v", err, joined) + } + if errors.As(err, &timeout) == external { + t.Fatalf("wrong restartable timeout classification: %v", err) + } + if !external && (!errors.Is(err, errManagedRecoveryDeadline) || !errors.Is(err, context.DeadlineExceeded)) { + t.Fatal(err) + } + if external && !errors.Is(err, parent.Err()) { + t.Fatal(err) + } + want := []int64(nil) + if front != 0 { + want = []int64{front} + } + assertActiveCandidates(t, d.transaction, d.db, want) // Also proves transaction lock release. + assertMonitorUnlocked(t, monitor) + if (*events)[len(*events)-1] != "start" || reports > 1 { + t.Fatalf("events=%v reports=%d", *events, reports) + } + if !bootstrap { + state, err := readMonitorState(d.db, monitor.serviceID) + if err != nil || state.RecoveryRunning { + t.Fatalf("recovery ownership leaked: %+v %v", state, err) + } + } + }) + } +} + +func TestManagedRecoveryJobSafetyFailureNotTimeout(t *testing.T) { + for _, mode := range []string{"cleanup", "rollback", "database", "guard"} { + t.Run(mode, func(t *testing.T) { + unsafe := errors.New(mode + " failed") + err := managedRecoveryJob(context.Background(), time.Nanosecond, func(ctx context.Context) error { + <-ctx.Done() + return &managedRecoveryFatal{Err: errors.Join(ctx.Err(), unsafe)} + }) + var fatal *managedRecoveryFatal + var timeout *managedRecoveryTimeout + if !errors.As(err, &fatal) || errors.As(err, &timeout) || !errors.Is(err, unsafe) || !errors.Is(err, errManagedRecoveryDeadline) { + t.Fatalf("unsafe deadline classified as restartable: %v", err) + } + }) + } +} + +func TestManagedRecoveryTimeoutLeaves(t *testing.T) { + processExit := func(command string) error { + t.Helper() + err := exec.Command("sh", "-c", command).Run() + var exit *exec.ExitError + if !errors.As(err, &exit) { + t.Fatalf("expected process exit: %v", err) + } + return err + } + closeErr := errors.New("diagnostic close failed") + for _, test := range []struct { + name string + err error + safe bool + }{ + {"nil", nil, true}, + {"deadline", context.DeadlineExceeded, true}, + {"wrapped-deadline", fmt.Errorf("worker: %w", context.DeadlineExceeded), true}, + {"canceled", context.Canceled, true}, + {"term", errors.Join(context.DeadlineExceeded, processExit("kill -TERM $$")), true}, + {"kill", errors.Join(context.Canceled, processExit("kill -KILL $$")), true}, + {"unknown-exit", errors.Join(context.DeadlineExceeded, processExit("exit 7")), false}, + {"unknown-leaf", closeErr, false}, + {"joined-close", errors.Join(context.DeadlineExceeded, closeErr), false}, + {"nested-close", fmt.Errorf("worker: %w", errors.Join(context.Canceled, fmt.Errorf("close: %w", closeErr))), false}, + {"empty-exit", &exec.ExitError{}, false}, + {"joined-fatal", errors.Join(context.DeadlineExceeded, &managedRecoveryFatal{Err: context.Canceled}), false}, + } { + t.Run(test.name, func(t *testing.T) { + if got := managedRecoveryTimeoutSafe(test.err); got != test.safe { + t.Fatalf("leaf validation=%v want=%v", got, test.safe) + } + err := managedRecoveryJob(context.Background(), time.Nanosecond, func(ctx context.Context) error { + <-ctx.Done() + return test.err + }) + var fatal *managedRecoveryFatal + var timeout *managedRecoveryTimeout + if !errors.As(err, &fatal) || errors.As(err, &timeout) != test.safe || !errors.Is(err, errManagedRecoveryDeadline) { + t.Fatalf("incorrect deadline classification: %v", err) + } + if test.safe && !managedRecoveryTimeoutSafe(timeout.Err) { + t.Fatal("timeout payload failed reusable validation") + } + }) + } +} + +func TestManagedRecoveryDiscoveryFatal(t *testing.T) { + for _, mode := range []string{"guard-fatal", "fatal-parent-cancel", "ordinary-unresolved"} { + t.Run(mode, func(t *testing.T) { + d, front, events := recoveryFixture(t, true) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + refusal := errors.New("discovery target guard refused") + d.discover = func(context.Context) (string, error) { + if mode == "ordinary-unresolved" { + return "", refusal + } + if mode == "fatal-parent-cancel" { + cancel() + return "", errors.Join(context.Canceled, &managedRecoveryFatal{Err: refusal}) + } + return "", &managedRecoveryFatal{Err: refusal} + } + result, err := recoverManagedTarget(ctx, d) + var fatal *managedRecoveryFatal + if mode == "ordinary-unresolved" { + if err != nil || result.GGCError == nil { + t.Fatalf("ordinary discovery error became fatal: %+v %v", result, err) + } + } else if !errors.As(err, &fatal) || !errors.Is(err, refusal) || result.GGCError != nil { + t.Fatalf("discovery safety refusal masked: %+v %v", result, err) + } + if mode == "fatal-parent-cancel" && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if len(*events) != 0 { + t.Fatal("discovery refusal triggered activation", *events) + } + assertActiveCandidates(t, d.transaction, d.db, []int64{front}) + }) + } +} + +func TestManagedRecoveryHealthDeadlinePreservesFailure(t *testing.T) { + for _, mode := range []string{"fatal-guard", "mixed-close", "late-ok"} { + t.Run(mode, func(t *testing.T) { + d, prior, events := recoveryFixture(t, true) + managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true) + priorBytes, err := d.transaction.read("config") + if err != nil { + t.Fatal(err) + } + failure := errors.New("concurrent health guard or diagnostic close failed") + calls := 0 + d.health = func(ctx context.Context) (string, error) { + calls++ + if calls == 1 { + return "DEGRADED", nil + } + <-ctx.Done() + switch mode { + case "fatal-guard": + return "", &managedRecoveryFatal{Err: failure} + case "mixed-close": + return "", errors.Join(ctx.Err(), failure) + default: + return "OK", nil + } + } + err = managedRecoveryJob(context.Background(), 50*time.Millisecond, func(ctx context.Context) error { + _, err := recoverManagedTarget(ctx, d) + return err + }) + var fatal *managedRecoveryFatal + var timeout *managedRecoveryTimeout + if !errors.As(err, &fatal) || errors.As(err, &timeout) != (mode == "late-ok") || !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("health deadline misclassified: %v", err) + } + if mode != "late-ok" && !errors.Is(err, failure) { + t.Fatalf("original safety failure erased: %v", err) + } + assertActiveCandidates(t, d.transaction, d.db, []int64{prior}) + assertManagedContent(t, d.transaction, string(priorBytes)) + if calls != 2 || strings.Count(strings.Join(*events, ","), "restart") != 2 || (*events)[len(*events)-1] != "start" { + t.Fatalf("rollback/restoration incomplete: calls=%d events=%v", calls, *events) + } + }) + } +} + +func TestManagedRecoveryContextError(t *testing.T) { + unsafe := errors.New("diagnostic close failed") + for _, original := range []error{nil, context.Canceled, context.DeadlineExceeded, unsafe, errors.Join(context.Canceled, unsafe), &managedRecoveryFatal{Err: unsafe}} { + ctx, cancel := context.WithCancel(context.Background()) + // A live job leaves request-local errors ordinary and unchanged. + if got := managedRecoveryContextError(ctx, original); got != original { + t.Fatalf("live context changed request-local error: %v", got) + } + cancel() + got := managedRecoveryContextError(ctx, original) + var fatal *managedRecoveryFatal + if !errors.Is(got, context.Canceled) || errors.As(got, &fatal) == managedRecoveryTimeoutSafe(original) { + t.Fatalf("cancellation lost safety classification: original=%v got=%v", original, got) + } + if original != nil && !errors.Is(got, original) { + t.Fatalf("original failure lost: %v", got) + } + } +} + +func TestManagedRecoveryBootstrapCleanupFailureSurvivesCancellation(t *testing.T) { + for _, mode := range []string{"deadline", "parent-cancel"} { + t.Run(mode, func(t *testing.T) { + d, _, events := recoveryFixture(t, false) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + joined := false + d.scan = func(jobCtx context.Context, _, _ string, _, _ io.Writer) (string, error) { + if mode == "parent-cancel" { + cancel() + } + <-jobCtx.Done() + joined = true + return "", jobCtx.Err() + } + unsafe := errors.New("last-good start failed") + baseRun := d.run + d.run = func(cleanup context.Context, path string, args ...string) error { + if err := baseRun(cleanup, path, args...); err != nil { + return err + } + if args[0] == "start" { + if !joined { + t.Fatal("cleanup before worker join") + } + return unsafe + } + return nil + } + monitor := monitorRuntimeFixture(t) + monitor.bootstrap = func(ctx context.Context) error { + return managedRecoveryJob(ctx, 20*time.Millisecond, func(jobCtx context.Context) error { + _, err := recoverManagedTarget(jobCtx, d) + return err + }) + } + err := runManagedMonitor(ctx, monitor) + var fatal *managedRecoveryFatal + var timeout *managedRecoveryTimeout + if !errors.As(err, &fatal) || errors.As(err, &timeout) || !errors.Is(err, unsafe) { + t.Fatalf("cleanup failure masked or restartable: %v", err) + } + if mode == "deadline" && !errors.Is(err, errManagedRecoveryDeadline) || mode == "parent-cancel" && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if !reflect.DeepEqual(*events, []string{"stop", "start"}) { + t.Fatal(*events) + } + assertActiveCandidates(t, d.transaction, d.db, nil) + assertManagedContent(t, d.transaction, managedBaseline) + assertMonitorUnlocked(t, monitor) + }) + } +} + func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) { d, _, events := recoveryFixture(t, false) savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true) diff --git a/cmd/zapret-autotune/monitor_runtime.go b/cmd/zapret-autotune/monitor_runtime.go index 10f5bdb..037f46d 100644 --- a/cmd/zapret-autotune/monitor_runtime.go +++ b/cmd/zapret-autotune/monitor_runtime.go @@ -123,13 +123,13 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) { } if d.bootstrap != nil { if err := d.bootstrap(ctx); err != nil { - if ctx.Err() != nil { - return ctx.Err() - } var fatal *managedRecoveryFatal if errors.As(err, &fatal) { return err } + if ctx.Err() != nil { + return ctx.Err() + } // Bootstrap scans can validly find no qualified strategy. Keep the // supervisor alive so durable probes/cooldown govern the next attempt. d.reportRecoveryError(err) diff --git a/deploy/managed-target/README.md b/deploy/managed-target/README.md index 4f348c3..78608f3 100644 --- a/deploy/managed-target/README.md +++ b/deploy/managed-target/README.md @@ -31,9 +31,40 @@ Then build and start only this Compose project. The rendered configuration contains the environment password, so never use plain `docker compose config` in logs or shared terminals. +By default Compose uses the locally built `zapret-autotune-managed-target:local`. +Tags are only local build/test candidates, not production release references. +For production promotion, set `ZAPRET_AUTOTUNE_IMAGE=/@sha256:` +in the private `.env` and pull that immutable image before starting the unit. +Use a managed target image, not the generic image; the systemd command never +builds images. Do not enable the production service with a tag or local default. + Offline static guard (from repository root): `python3 deploy/managed-target/check_static.py`. +## Optional native systemd supervision + +For unattended boot, place this checkout at `/opt/zapret2-autotune`, provision +`deploy/managed-target/.env` and state as above, and build the target image first. +For production, select and pull the immutable managed-target digest as above +instead of using a local build tag. +Install `deploy/managed-target/zapret-autotune.service` in `/etc/systemd/system/`, +then run `systemctl daemon-reload` and `systemctl enable --now zapret-autotune`. +Do not simultaneously start the same project with detached Compose. +Enabling the unit starts this isolated target on subsequent boots. `ExecStop` +only stops `target`; it does not replace the foreground process's failure code. +An explicit service stop suppresses restart, including during a shutdown race. + +Compose has no automatic restart. The foreground systemd unit restarts **only +exit 75**, with at most three starts per six hours. Clean typed recovery timeouts, +unexpected monitor exits/crashes, and Shadowsocks failures qualify only if +shutdown cleanup succeeds. Configuration (64), safety/database/rollback/cleanup +failures (78), and graceful SIGTERM (0) never request restart. Investigate and +repair nonretry failures before manually starting again; do not clear state. +Monitor cancellation/join is bounded to 60 seconds, final target-local zapret +stop gets 30 seconds then forced kill after five more, and each Shadowsocks join +gets five seconds. Exceeding a join bound fails closed. Compose grants 120 +seconds; systemd grants 180 seconds. + ## Isolation and runtime contract Only SOCKS TCP/UDP is published on host @@ -47,9 +78,10 @@ password before starting child processes. The dedicated bridge is not shared with existing services. No host networking, Docker socket, existing mounts, host Lua/fakes or privileged mode; all -capabilities are dropped except `NET_ADMIN`, `NET_RAW`, `SETUID`, `SETGID`. +capabilities are dropped except `NET_ADMIN`, `NET_RAW`, `SETUID`, `SETGID`, `KILL`. `NET_ADMIN` covers nft/NFQUEUE; `NET_RAW` covers nfqws2 raw sends. Zapret2's `nfqws2 --user=tpws` needs `SETUID`/`SETGID` for UID/GID switching. +`KILL` allows bounded cleanup of workers that have dropped their UID. Startup order: ss-server/ss-local → listener 1080 → guarded managed init on fresh state (status on recreation) → target-local `zapret2 start` → supervised diff --git a/deploy/managed-target/check_static.py b/deploy/managed-target/check_static.py index 5867b5a..4db1fd1 100644 --- a/deploy/managed-target/check_static.py +++ b/deploy/managed-target/check_static.py @@ -7,10 +7,12 @@ directory = Path(__file__).resolve().parent compose = (directory / "compose.yaml").read_text() entrypoint = (directory.parents[1] / "managed-target-entrypoint.sh").read_text() readme = (directory / "README.md").read_text() +unit = (directory / "zapret-autotune.service").read_text() # ponytail: checks the committed Compose shape, not arbitrary YAML; use a YAML # parser if this definition grows beyond its single target service. assert " privileged: false\n" in compose +assert " image: ${ZAPRET_AUTOTUNE_IMAGE:-zapret-autotune-managed-target:local}\n" in compose assert " cap_drop: [ALL]\n" in compose assert " cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID, KILL]\n" in compose assert compose.count(" - type: bind\n") == 1 @@ -26,3 +28,18 @@ assert "export SS_PASSWORD" not in entrypoint assert "Docker administrators/root can still read container" in readme assert "removed\nfrom the entrypoint environment" in readme assert "No host networking" in readme +assert ' restart: "no"\n' in compose +assert " stop_grace_period: 120s\n" in compose +directives = dict(line.split("=", 1) for line in unit.splitlines() if "=" in line) +assert directives["Restart"] == "no" +assert directives["RestartForceExitStatus"] == "75" +assert "RestartPreventExitStatus" not in directives +assert directives["StartLimitIntervalSec"] == "6h" +assert directives["StartLimitBurst"] == "3" +assert directives["TimeoutStopSec"] == "180s" +assert directives["WorkingDirectory"] == "/opt/zapret2-autotune" +assert directives["WantedBy"] == "multi-user.target" +compose_command = "/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml" +assert directives["ExecStart"] == compose_command + " up --no-build --abort-on-container-exit --exit-code-from target target" +assert directives["ExecStop"] == compose_command + " stop target" +assert "timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop" in entrypoint diff --git a/deploy/managed-target/compose.yaml b/deploy/managed-target/compose.yaml index 0134f01..5c4dfc9 100644 --- a/deploy/managed-target/compose.yaml +++ b/deploy/managed-target/compose.yaml @@ -1,7 +1,7 @@ name: zapret-autotune-managed-test services: target: - image: zapret-autotune-managed-target:local + image: ${ZAPRET_AUTOTUNE_IMAGE:-zapret-autotune-managed-target:local} build: context: ../.. dockerfile: Dockerfile.target @@ -30,7 +30,8 @@ services: tmpfs: - /run/zapret-autotune-ss:rw,noexec,nosuid,nodev,size=1m,mode=0700,uid=0,gid=0 networks: [target] - stop_grace_period: 45s + restart: "no" + stop_grace_period: 120s networks: target: driver: bridge diff --git a/deploy/managed-target/zapret-autotune.service b/deploy/managed-target/zapret-autotune.service new file mode 100644 index 0000000..95b4250 --- /dev/null +++ b/deploy/managed-target/zapret-autotune.service @@ -0,0 +1,19 @@ +[Unit] +Description=Isolated zapret-autotune managed target +Requires=docker.service +After=docker.service +StartLimitIntervalSec=6h +StartLimitBurst=3 + +[Service] +Type=simple +WorkingDirectory=/opt/zapret2-autotune +ExecStart=/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml up --no-build --abort-on-container-exit --exit-code-from target target +ExecStop=/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml stop target +Restart=no +RestartForceExitStatus=75 +RestartSec=30s +TimeoutStopSec=180s + +[Install] +WantedBy=multi-user.target diff --git a/managed-target-entrypoint.sh b/managed-target-entrypoint.sh index f8c8418..64c6ce0 100644 --- a/managed-target-entrypoint.sh +++ b/managed-target-entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh set -eu umask 077 -[ "$#" -eq 0 ] || { echo 'error: target entrypoint accepts no arguments' >&2; exit 1; } +[ "$#" -eq 0 ] || { echo 'error: target entrypoint accepts no arguments' >&2; exit 64; } # Credentials never enter argv or logs. Docker supplies SS_PASSWORD via env; # clear the exported value before starting any child processes. @@ -9,12 +9,12 @@ ss_password=${SS_PASSWORD-} unset SS_PASSWORD case "$ss_password" in ''|*[!0123456789abcdefABCDEF]*) - echo 'error: target secret must contain only hexadecimal characters' >&2; exit 1 ;; + echo 'error: target secret must contain only hexadecimal characters' >&2; exit 64 ;; esac -[ "${#ss_password}" -ge 32 ] || { echo 'error: target secret must be at least 32 hex characters' >&2; exit 1; } +[ "${#ss_password}" -ge 32 ] || { echo 'error: target secret must be at least 32 hex characters' >&2; exit 64; } config_dir=/run/zapret-autotune-ss awk '$2 == "/run/zapret-autotune-ss" && $3 == "tmpfs" { found=1 } END { exit !found }' /proc/mounts || { - echo 'error: target Shadowsocks config directory must be a dedicated tmpfs' >&2; exit 1; + echo 'error: target Shadowsocks config directory must be a dedicated tmpfs' >&2; exit 78; } chmod 0700 "$config_dir" cat > "$config_dir/server.json" </dev/null || true; exit 0' TERM INT + wait "$sleeper" + kill -KILL "$child" 2>/dev/null || true + ) /dev/null 2>&1 & + watchdog=$! + waited_code=0 + wait "$child" 2>/dev/null || waited_code=$? + kill -TERM "$watchdog" 2>/dev/null || true + wait "$watchdog" 2>/dev/null || true +} cleanup() { + final_code=$? + case "$final_code" in 0|64|75|78) ;; *) final_code=78 ;; esac trap - EXIT INT TERM # Let the monitor cancel/join scans and restore committed bytes BEFORE stop. if [ -n "$monitor_pid" ]; then kill -TERM "$monitor_pid" 2>/dev/null || true - wait "$monitor_pid" 2>/dev/null || true + bounded_wait "$monitor_pid" 60 + # Cancellation must restore safely even when SS failure requested retry. + case "$waited_code" in 0|143) ;; *) final_code=78 ;; esac fi # Dedicated PID/network namespace: no host/other-target stop. if [ "$zapret_started" -eq 1 ]; then - /opt/zapret2/init.d/sysv/zapret2 stop || true + timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop || final_code=78 fi for pid in "$local_pid" "$server_pid"; do [ -z "$pid" ] || kill "$pid" 2>/dev/null || true done for pid in "$local_pid" "$server_pid"; do - [ -z "$pid" ] || wait "$pid" 2>/dev/null || true + if [ -n "$pid" ]; then + bounded_wait "$pid" 5 + # Its original failure triggered supervision; only forced shutdown + # is a cleanup failure, not the already-dead Shadowsocks status. + [ "$waited_code" -ne 137 ] || final_code=78 + fi done + exit "$final_code" } trap cleanup EXIT trap 'exit 0' INT TERM @@ -60,9 +86,9 @@ local_pid=$! # Bounded readiness, no network probe. Init/status verifies socket ownership. attempt=0 until awk '$2 == "00000000:0438" && $4 == "0A" { found=1 } END { exit !found }' /proc/net/tcp; do - kill -0 "$server_pid" && kill -0 "$local_pid" + { kill -0 "$server_pid" && kill -0 "$local_pid"; } || exit 75 attempt=$((attempt + 1)) - [ "$attempt" -lt 30 ] || { echo 'error: target SOCKS listener timeout' >&2; exit 1; } + [ "$attempt" -lt 30 ] || { echo 'error: target SOCKS listener timeout' >&2; exit 75; } sleep 1 done if [ ! -e "$ZAPRET_CONFIG" ]; then @@ -84,8 +110,12 @@ if ! kill -0 "$monitor_pid" 2>/dev/null; then wait "$monitor_pid" || code=$? monitor_pid= echo 'error: target managed monitor exited' >&2 - [ "$code" -ne 0 ] || code=1 + case "$code" in + 0|2|137|139) code=75 ;; + 64|75|78) ;; + *) code=78 ;; + esac exit "$code" fi echo 'error: target child process exited' >&2 -exit 1 +exit 75