feat(runtime): bound recovery and add safe release supervision
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
780ca71e17
commit
b1e458a0a5
16 files changed
+941
-66
No files matched your search
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
FROM golang:1.25.3 AS build
|
FROM golang:1.25.3@sha256:6d4e5e74f47db00f7f24da5f53c1b4198ae46862a47395e30477365458347bf2 AS build
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
ENV CGO_ENABLED=0 GOTOOLCHAIN=local
|
ENV CGO_ENABLED=0 GOTOOLCHAIN=local
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
|
|||||||
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
FROM golang:1.25.3 AS build
|
FROM golang:1.25.3@sha256:6d4e5e74f47db00f7f24da5f53c1b4198ae46862a47395e30477365458347bf2 AS build
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
ENV CGO_ENABLED=0 GOTOOLCHAIN=local
|
ENV CGO_ENABLED=0 GOTOOLCHAIN=local
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
|
|||||||
+156
-4
@@ -135,7 +135,7 @@ identity/SOCKS guards. Fake-only checks are added, but tests/build are deferred
|
|||||||
to the orchestrator. `managed monitor` now binds those callbacks and only the
|
to the orchestrator. `managed monitor` now binds those callbacks and only the
|
||||||
dedicated target entrypoint starts it; no live/runtime validation has occurred.
|
dedicated target entrypoint starts it; no live/runtime validation has occurred.
|
||||||
|
|
||||||
## Dedicated monitor integration (verification deferred)
|
## Dedicated monitor integration (live acceptance pending)
|
||||||
|
|
||||||
`zapret-autotune managed monitor [--interval 5m] [--failure-threshold 3]
|
`zapret-autotune managed monitor [--interval 5m] [--failure-threshold 3]
|
||||||
[--cooldown 30m]` uses the health environment defaults. Interval is positive and
|
[--cooldown 30m]` uses the health environment defaults. Interval is positive and
|
||||||
@@ -165,9 +165,9 @@ Entrypoint ordering is SS children → listener → managed init/status → zapr
|
|||||||
all three children; monitor exit is fatal even if it exits zero unexpectedly.
|
all three children; monitor exit is fatal even if it exits zero unexpectedly.
|
||||||
SIGTERM/SIGINT cancels and waits the monitor before target-local zapret stop and
|
SIGTERM/SIGINT cancels and waits the monitor before target-local zapret stop and
|
||||||
SS child termination/wait. Existing targets and Xray remain unchanged. Deployment
|
SS child termination/wait. Existing targets and Xray remain unchanged. Deployment
|
||||||
must allow enough shutdown grace for independently bounded rollback/restoration
|
allows 120s Compose shutdown grace and 180s systemd stop timeout for bounded
|
||||||
(potentially more than the currently defined 45s); this increment changes no Docker
|
monitor cancellation/restoration and child joins. Code integration is not
|
||||||
configuration. Code integration is not image/routing/signal live verification.
|
image/routing/signal live verification.
|
||||||
|
|
||||||
Injected unit tests cover exact identity/path/config, mountpoint refusal,
|
Injected unit tests cover exact identity/path/config, mountpoint refusal,
|
||||||
old-container-like env, SOCKS port/process refusal, correct fresh-target success,
|
old-container-like env, SOCKS port/process refusal, correct fresh-target success,
|
||||||
@@ -178,3 +178,155 @@ deferred to the orchestrator; not run by this increment:
|
|||||||
go test ./cmd/zapret-autotune -run TestManaged
|
go test ./cmd/zapret-autotune -run TestManaged
|
||||||
go build ./cmd/zapret-autotune
|
go build ./cmd/zapret-autotune
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Production preparation
|
||||||
|
|
||||||
|
This is a standalone **new isolated target** candidate, not a production-approved
|
||||||
|
deployment and not an integration into existing zapret/Xray. Keep the exact
|
||||||
|
`managed-test` role, ID, marker, mount, config, DB and owned-listener guards above.
|
||||||
|
Do not change the role to `production`, adopt old state, share an old network
|
||||||
|
namespace, or relax a refusal. Provisioning and optional service installation
|
||||||
|
are described in [deploy/managed-target/README.md](deploy/managed-target/README.md).
|
||||||
|
|
||||||
|
### Build once, promote the same image
|
||||||
|
|
||||||
|
Use `Dockerfile.target` for the supervised target; root `Dockerfile` is the
|
||||||
|
proposal CLI image, not the managed service. Both pin Go 1.25.3 and the runtime
|
||||||
|
by digest, use `GOTOOLCHAIN=local`, `CGO_ENABLED=0`, `go.sum`, and `-trimpath`.
|
||||||
|
Pins constrain build inputs; they do not establish live acceptance or guarantee
|
||||||
|
bit-for-bit builds on different platforms. Record the source commit, platform,
|
||||||
|
builder/runtime digests and resulting image digest for every release.
|
||||||
|
|
||||||
|
Example (replace registry and release; never reuse a release tag):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
IMAGE=registry.example/team/zapret-autotune-managed-target:2026-10-10-r1
|
||||||
|
docker build --pull -f Dockerfile.target -t "$IMAGE" .
|
||||||
|
docker push "$IMAGE"
|
||||||
|
docker image inspect --format '{{index .RepoDigests 0}}' "$IMAGE"
|
||||||
|
```
|
||||||
|
|
||||||
|
Promote the printed `registry/repository@sha256:...` reference unchanged from
|
||||||
|
validation to deployment; do not rebuild or use `latest`. Authenticate separately,
|
||||||
|
never put registry credentials in these commands. The supplied Compose file
|
||||||
|
selects `${ZAPRET_AUTOTUNE_IMAGE:-zapret-autotune-managed-target:local}`. Set
|
||||||
|
`ZAPRET_AUTOTUNE_IMAGE=registry.example/team/zapret-autotune-managed-target@sha256:<digest>`
|
||||||
|
in the same private deployment `.env` as `SS_PASSWORD` (mode `0600`), using the
|
||||||
|
actual printed digest. Tags and the local default are build/test candidates only;
|
||||||
|
an immutable managed-target digest is required for production promotion. From
|
||||||
|
the repository root, validate and pull without printing the rendered secrets:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml config --quiet
|
||||||
|
docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml pull target
|
||||||
|
```
|
||||||
|
|
||||||
|
The supplied systemd unit uses this same `.env` and starts with `--no-build`;
|
||||||
|
no image-selection override or unit edit is needed. Keep the selected digest and
|
||||||
|
private configuration together for upgrades and rollback. Never run a build
|
||||||
|
against the production digest selector; build/tag separately as shown above.
|
||||||
|
|
||||||
|
### Configuration and secrets
|
||||||
|
|
||||||
|
Create fresh empty `target-state` owned by root/runtime UID 0, mode `0700`, and
|
||||||
|
the private deployment `.env` mode `0600`; rootless/remapped Docker needs the
|
||||||
|
mapped owner. Bootstrap is for empty provisioned state only; recreation retains
|
||||||
|
identity, journal, candidate DB, sidecars and lock file. Never automatically wipe
|
||||||
|
state to make startup succeed. The entrypoint creates Shadowsocks JSON in private
|
||||||
|
tmpfs (`0700` directory, `0600` files), unsets the password before children, and
|
||||||
|
does not put it in argv. Docker administrators/root can still read container env
|
||||||
|
and runtime config. Do not print `.env`, rendered Compose config, diagnostics or
|
||||||
|
signed URLs; use `docker compose ... config --quiet` for validation.
|
||||||
|
|
||||||
|
| Existing setting | Default / accepted value for managed target |
|
||||||
|
| --- | --- |
|
||||||
|
| `ZAPRET_AUTOTUNE_IMAGE` | Local build/test default `zapret-autotune-managed-target:local`; production promotion requires a managed-target `repository@sha256:<digest>` |
|
||||||
|
| `SS_PASSWORD` | Required independent hex-only secret, at least 32 characters; deployment README generates 64 |
|
||||||
|
| `ZAPRET_AUTOTUNE_HOST_SOCKS_PORT` | `11080`; choose an available TCP/UDP port 1–65535, published only on `127.0.0.1` |
|
||||||
|
| `SOCKS_PORT` | Supplied deployment fixes internal listener to `1080`; not a host-port override |
|
||||||
|
| `ZAPRET_AUTOTUNE_SOCKS_PROXY` | Fixed `socks5://127.0.0.1:1080` inside this target, no credentials/direct fallback |
|
||||||
|
| `ZAPRET_AUTOTUNE_VIDEO_ID` | `dQw4w9WgXcQ`; exactly 11 ASCII letters, digits, `_` or `-` |
|
||||||
|
| `ZAPRET_AUTOTUNE_HEALTH_INTERVAL` | `5m`; duration in `(0,24h]` for monitor |
|
||||||
|
| `ZAPRET_AUTOTUNE_FAILURE_THRESHOLD` | `3`; integer 1–1000 for monitor |
|
||||||
|
| `ZAPRET_AUTOTUNE_RECOVERY_COOLDOWN` | `30m`; duration in `(0,168h]` for monitor |
|
||||||
|
| `ZAPRET_AUTOTUNE_DB` | Fixed `/var/lib/zapret-autotune/target/candidates.db` |
|
||||||
|
|
||||||
|
The monitor's three policy flags override corresponding environment defaults;
|
||||||
|
explicit empty/invalid environment settings are rejected. The entrypoint accepts
|
||||||
|
no arguments. The supplied Compose environment does not forward the video/policy
|
||||||
|
settings from `.env`: add the desired existing variables to `target.environment`
|
||||||
|
in your deployment copy/override if needed. Keep all fixed identity/path settings
|
||||||
|
unchanged. No public SS listener is published; Xray configuration is external and
|
||||||
|
operator-owned, optionally using the host's loopback SOCKS port. Do not modify an
|
||||||
|
existing Xray deployment as part of bootstrap.
|
||||||
|
|
||||||
|
### Supervision and failure handling
|
||||||
|
|
||||||
|
Compose restart is disabled. Optional native systemd runs **foreground** Compose
|
||||||
|
with `--no-build --abort-on-container-exit --exit-code-from target`; do not also
|
||||||
|
run detached Compose for that project. The unit requests restart only for exit
|
||||||
|
75 via `Restart=no` plus `RestartForceExitStatus=75`, bounded by
|
||||||
|
`StartLimitBurst=3` in six hours (starts, including the initial
|
||||||
|
start, not three additional retries). Compose stop grace is 120s, systemd 180s.
|
||||||
|
|
||||||
|
| Target exit | Operator meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `64` | Invalid configuration; fix settings before starting |
|
||||||
|
| `78` | Safety/DB/journal/rollback/cleanup failure; investigate, no automatic restart |
|
||||||
|
| `75` | Technical failure eligible for bounded restart only after safe cleanup |
|
||||||
|
| `0` | Graceful signal shutdown; no automatic restart |
|
||||||
|
|
||||||
|
Bootstrap/recovery has a fixed **90-minute whole-job deadline**, plus an
|
||||||
|
independent **30-second restoration reserve**. This is a catastrophic technical
|
||||||
|
ceiling, not a normal candidate-check timeout and not an operator tunable. Valid
|
||||||
|
nonpassing candidates/unresolved GGC are normal recovery outcomes governed by
|
||||||
|
the persisted threshold/cooldown, not a reason to restart the container. Failed
|
||||||
|
restoration or unsafe state vetoes retry: timeout exit 75 requires the job's own
|
||||||
|
deadline and only timeout/cancellation-related errors, with no joined cleanup,
|
||||||
|
DB or safety failure. Unexpected monitor/SS failures are separately technical
|
||||||
|
only if shutdown cleanup succeeds. Shutdown joins the monitor for up to 60s,
|
||||||
|
target-local zapret stop gets 30s then forced kill after five more, and SS children
|
||||||
|
get 5s each; failure to join/restore is fail-closed. The reserve is not extra
|
||||||
|
scanning time. The 90-minute budget is shared by the whole bootstrap/recovery
|
||||||
|
job, not renewed per candidate or phase.
|
||||||
|
|
||||||
|
Monitor health means Player API `OK` through owned SOCKS. GGC evidence covers
|
||||||
|
only the **exact discovered hostname** and tested TLS protocols, not wildcard
|
||||||
|
googlevideo coverage, video transfer, SABR or playback. Unresolved GGC must not
|
||||||
|
be reported as complete playback health.
|
||||||
|
|
||||||
|
### Upgrade, rollback and operator recovery
|
||||||
|
|
||||||
|
1. Record the current image digest and private deployment configuration. Stop
|
||||||
|
the service (or foreground Compose), wait for complete cleanup, and verify
|
||||||
|
the target is stopped. If cleanup fails, investigate before upgrading.
|
||||||
|
2. With the target stopped, make a root-only backup of the **entire** state
|
||||||
|
directory (including DB sidecars, journal, config, identity and lock file),
|
||||||
|
preserving ownership/modes. Store `.env` and image/config records privately
|
||||||
|
with it; do not print archive contents, diagnostics or signed URLs. A live
|
||||||
|
copy of only `candidates.db` is not a consistent rollback backup.
|
||||||
|
3. Select/pull the new immutable digest and start without building. Inspect
|
||||||
|
sanitized status/logs, verify identity/hash and expected active groups, owned
|
||||||
|
SOCKS, target-local queues, health scope and persistent recreation. Keep the
|
||||||
|
old image and backup until acceptance; DB migrations may prevent image-only
|
||||||
|
downgrade.
|
||||||
|
4. To roll back, stop and verify cleanup again. Preserve failed state separately
|
||||||
|
for investigation. Restore the matched stopped-state backup **as a whole**
|
||||||
|
into the same dedicated mount with private ownership/modes, select its old
|
||||||
|
digest/config, then start. Do not merge journals/DB generations or re-init.
|
||||||
|
A restored legacy/live/unverifiable recovery owner can fail closed; never
|
||||||
|
clear its lease or unlink the lock merely to force startup.
|
||||||
|
|
||||||
|
On drift, pending transaction, rollback failure, or exhausted systemd start
|
||||||
|
limit, leave state intact and service stopped. Diagnose using sanitized status
|
||||||
|
and private artifacts; repair the actual provisioning/configuration issue or
|
||||||
|
restore a known consistent backup. Only then manually start (and reset the
|
||||||
|
systemd failed/start-limit state if needed). There is no automatic wipe or
|
||||||
|
automatic adoption path.
|
||||||
|
|
||||||
|
Rollout acceptance remains pending real isolated-target restart/deadline tests:
|
||||||
|
empty bootstrap, saved-first recovery, exact-host GGC rotation, image binding,
|
||||||
|
NFQUEUE activation/rollback, SIGTERM during downtime, deadline restoration,
|
||||||
|
retry limits/exit propagation through foreground Compose/systemd, persistent
|
||||||
|
recreation and backup restore. Perform these on the new target only; local unit
|
||||||
|
or static checks cannot establish production readiness.
|
||||||
@@ -2,6 +2,24 @@
|
|||||||
|
|
||||||
Go CLI for scanning and checking YouTube TLS bypass strategies with `blockcheckw`.
|
Go CLI for scanning and checking YouTube TLS bypass strategies with `blockcheckw`.
|
||||||
|
|
||||||
|
## Production preparation (acceptance pending)
|
||||||
|
|
||||||
|
The standalone deployment candidate is a **new isolated target**, never an
|
||||||
|
upgrade/adoption of an existing zapret or Xray service. The legacy `managed-test`
|
||||||
|
role and fixed identity guards remain intentional, including for prod-like runs.
|
||||||
|
See [release, configuration, upgrade and rollback guidance](MANAGED_TARGET.md#production-preparation)
|
||||||
|
and the [deployment/service instructions](deploy/managed-target/README.md).
|
||||||
|
Pinned builds and supervision are preparation, not production acceptance: real
|
||||||
|
restart/deadline tests and end-to-end target validation remain rollout gates.
|
||||||
|
Select an immutable managed-target digest with `ZAPRET_AUTOTUNE_IMAGE` in the
|
||||||
|
private deployment `.env`; tags/local defaults are only build/test candidates.
|
||||||
|
Bootstrap/recovery shares a 90-minute whole-job deadline plus a separate 30-second
|
||||||
|
restoration reserve, an abnormal ceiling rather than normal candidate failure.
|
||||||
|
Timeout exit 75 requires clean timeout/cancellation only; cleanup/safety errors
|
||||||
|
veto retry. Optional foreground systemd uses `Restart=no` plus forced exit-75
|
||||||
|
restart, limited to three starts per six hours; Compose stop grace is 120s and
|
||||||
|
systemd stop timeout 180s.
|
||||||
|
|
||||||
## Manual managed-test activation (not deployed)
|
## Manual managed-test activation (not deployed)
|
||||||
|
|
||||||
`managed init|status|active-ids|apply` requires exact role `managed-test`, target ID
|
`managed init|status|active-ids|apply` requires exact role `managed-test`, target ID
|
||||||
@@ -20,7 +38,7 @@ group, fixed target init executable and explicit runtime-config
|
|||||||
binding. It uses full target-local firewall/daemon restart and Player API post-check, with
|
binding. It uses full target-local firewall/daemon restart and Player API post-check, with
|
||||||
atomic rollback on failure. See [MANAGED_TARGET.md](MANAGED_TARGET.md) for guards
|
atomic rollback on failure. See [MANAGED_TARGET.md](MANAGED_TARGET.md) for guards
|
||||||
and provisioning prerequisites. An isolated pinned deployment definition is in
|
and provisioning prerequisites. An isolated pinned deployment definition is in
|
||||||
[deploy/managed-target](deploy/managed-target/README.md); build/run are deferred.
|
[deploy/managed-target](deploy/managed-target/README.md); acceptance remains pending.
|
||||||
An injected bootstrap/saved-first recovery adapter now connects the persisted
|
An injected bootstrap/saved-first recovery adapter now connects the persisted
|
||||||
monitor foundation to blockcheck, immutable candidates and transactional apply.
|
monitor foundation to blockcheck, immutable candidates and transactional apply.
|
||||||
`managed monitor` and the dedicated target entrypoint now wire these adapters;
|
`managed monitor` and the dedicated target entrypoint now wire these adapters;
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ func managedSeedArgs(host string) string {
|
|||||||
host = strings.ToLower(host)
|
host = strings.ToLower(host)
|
||||||
switch {
|
switch {
|
||||||
case host == "www.youtube.com":
|
case host == "www.youtube.com":
|
||||||
return "--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:ip_autottl=-3,3-20:repeats=1 --lua-desync=multidisorder:pos=1,midsld"
|
return "--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld"
|
||||||
case host == "googlevideo.com" || strings.HasSuffix(host, ".googlevideo.com"):
|
case host == "googlevideo.com" || strings.HasSuffix(host, ".googlevideo.com"):
|
||||||
return "--payload=tls_client_hello --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN"
|
return "--payload=tls_client_hello --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN"
|
||||||
default:
|
default:
|
||||||
|
|||||||
@@ -253,6 +253,13 @@ func TestRunManagedBlockcheckProtectsForeignWorkers(t *testing.T) {
|
|||||||
func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) {
|
func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) {
|
||||||
t.Setenv("TMPDIR", t.TempDir())
|
t.Setenv("TMPDIR", t.TempDir())
|
||||||
args := managedSeedArgs("www.youtube.com")
|
args := managedSeedArgs("www.youtube.com")
|
||||||
|
const wantSeed = "--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld"
|
||||||
|
if args != wantSeed {
|
||||||
|
t.Fatalf("frontend seed=%q, want %q", args, wantSeed)
|
||||||
|
}
|
||||||
|
if err := (&candidate{Args: args}).validateArgs(); err != nil {
|
||||||
|
t.Fatalf("invalid frontend seed grammar: %v", err)
|
||||||
|
}
|
||||||
t.Setenv("ZAPRET_TEST_REPORT", combinedReportFixture(args))
|
t.Setenv("ZAPRET_TEST_REPORT", combinedReportFixture(args))
|
||||||
argvPath := filepath.Join(t.TempDir(), "argv.jsonl")
|
argvPath := filepath.Join(t.TempDir(), "argv.jsonl")
|
||||||
t.Setenv("ZAPRET_TEST_ARGV", argvPath)
|
t.Setenv("ZAPRET_TEST_ARGV", argvPath)
|
||||||
@@ -279,10 +286,23 @@ func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) {
|
|||||||
if err := decoder.Decode(&checkArgs); err != nil {
|
if err := decoder.Decode(&checkArgs); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !slices.Contains(checkArgs, "--from-file") || !slices.Contains(checkArgs, "--passes") || !slices.Contains(checkArgs, "--no-conflict-cleanup") {
|
wantCheck := []string{"--workers", "32", "check", "--from-file", path + ".input", "--domain", "www.youtube.com", "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", "--output", path + ".check.json"}
|
||||||
|
if !reflect.DeepEqual(checkArgs, wantCheck) {
|
||||||
t.Fatalf("managed seeded check=%q", checkArgs)
|
t.Fatalf("managed seeded check=%q", checkArgs)
|
||||||
}
|
}
|
||||||
if seed := managedSeedArgs("rr3.googlevideo.com"); !strings.Contains(seed, "fakeddisorder") {
|
if err := decoder.Decode(&checkArgs); err != io.EOF {
|
||||||
|
t.Fatalf("unexpected extra blockcheck call: %q %v", checkArgs, err)
|
||||||
|
}
|
||||||
|
input, err := os.ReadFile(path + ".input")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantRows := "curl_test_https_tls12 ipv4 www.youtube.com : nfqws2 " + wantSeed + "\n" +
|
||||||
|
"curl_test_https_tls13 ipv4 www.youtube.com : nfqws2 " + wantSeed + "\n"
|
||||||
|
if string(input) != wantRows {
|
||||||
|
t.Fatalf("managed seed input=%q", input)
|
||||||
|
}
|
||||||
|
if seed := managedSeedArgs("rr3.googlevideo.com"); seed != "--payload=tls_client_hello --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN" {
|
||||||
t.Fatalf("missing exact-host GGC seed: %q", seed)
|
t.Fatalf("missing exact-host GGC seed: %q", seed)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,36 +56,41 @@ func managedMonitorCLI(args []string, stdout, stderr io.Writer, target managedDe
|
|||||||
return managedMonitorCLIContext(ctx, args, stdout, stderr, target, runManagedMonitor)
|
return managedMonitorCLIContext(ctx, args, stdout, stderr, target, runManagedMonitor)
|
||||||
}
|
}
|
||||||
|
|
||||||
func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) int {
|
func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) (code int) {
|
||||||
cfg, err := managedMonitorConfig(args, target.lookup)
|
cfg, err := managedMonitorConfig(args, target.lookup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, "error: invalid managed monitor configuration")
|
fmt.Fprintln(stderr, "error: invalid managed monitor configuration")
|
||||||
return 2
|
return 64
|
||||||
}
|
}
|
||||||
if code, err := managedRuntimeGuard(target); err != nil {
|
if _, err := managedRuntimeGuard(target); err != nil {
|
||||||
fmt.Fprintln(stderr, "error: managed monitor target guard refused")
|
fmt.Fprintln(stderr, "error: managed monitor target guard refused")
|
||||||
return code
|
return 78
|
||||||
}
|
}
|
||||||
m := &managedTransaction{Dir: target.directory, Name: "config"}
|
m := &managedTransaction{Dir: target.directory, Name: "config"}
|
||||||
if err := managedDirectory(target.directory); err != nil {
|
if err := managedDirectory(target.directory); err != nil {
|
||||||
fmt.Fprintln(stderr, "error: unsafe managed monitor directory")
|
fmt.Fprintln(stderr, "error: unsafe managed monitor directory")
|
||||||
return 1
|
return 78
|
||||||
}
|
}
|
||||||
if err := managedCheckIdentity(m); err != nil {
|
if err := managedCheckIdentity(m); err != nil {
|
||||||
fmt.Fprintln(stderr, "error: managed monitor identity refused")
|
fmt.Fprintln(stderr, "error: managed monitor identity refused")
|
||||||
return 1
|
return 78
|
||||||
}
|
}
|
||||||
path, err := managedMonitorDatabase(target)
|
path, err := managedMonitorDatabase(target)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, "error: unsafe managed monitor database")
|
fmt.Fprintln(stderr, "error: unsafe managed monitor database")
|
||||||
return 1
|
return 78
|
||||||
}
|
}
|
||||||
db, err := openStore(path)
|
db, err := openStore(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, "error: managed monitor database unavailable")
|
fmt.Fprintln(stderr, "error: managed monitor database unavailable")
|
||||||
return 1
|
return 78
|
||||||
}
|
}
|
||||||
defer db.Close()
|
defer func() {
|
||||||
|
if err := db.Close(); err != nil {
|
||||||
|
fmt.Fprintln(stderr, "error: managed monitor database close failed")
|
||||||
|
code = 78
|
||||||
|
}
|
||||||
|
}()
|
||||||
recovery, err := managedMonitorRecoveryAdapters(target, db, runManagedBlockcheck, runManagedCombinedBlockcheckForHost, func(ctx context.Context, path string, args ...string) error {
|
recovery, err := managedMonitorRecoveryAdapters(target, db, runManagedBlockcheck, runManagedCombinedBlockcheckForHost, func(ctx context.Context, path string, args ...string) error {
|
||||||
log, err := os.CreateTemp(target.directory, ".daemon-diagnostics-*.log")
|
log, err := os.CreateTemp(target.directory, ".daemon-diagnostics-*.log")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -98,7 +103,7 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr
|
|||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, "error: managed monitor adapters refused")
|
fmt.Fprintln(stderr, "error: managed monitor adapters refused")
|
||||||
return 1
|
return 78
|
||||||
}
|
}
|
||||||
monitor := managedMonitorDeps{
|
monitor := managedMonitorDeps{
|
||||||
target: target, db: db, serviceID: managedTargetID + "/youtube",
|
target: target, db: db, serviceID: managedTargetID + "/youtube",
|
||||||
@@ -110,7 +115,7 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr
|
|||||||
fmt.Fprintln(stderr, "managed GGC unresolved; monitoring continues (not playback health)")
|
fmt.Fprintln(stderr, "managed GGC unresolved; monitoring continues (not playback health)")
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
fmt.Fprintln(stderr, "error: managed monitor wiring refused")
|
fmt.Fprintln(stderr, "error: managed monitor wiring refused")
|
||||||
return 1
|
return 78
|
||||||
}
|
}
|
||||||
fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health")
|
fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health")
|
||||||
err = supervise(ctx, monitor)
|
err = supervise(ctx, monitor)
|
||||||
@@ -120,9 +125,34 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
fmt.Fprintln(stderr, "error: managed monitor stopped; safety/database/runtime failure (details suppressed)")
|
fmt.Fprintln(stderr, "error: managed monitor stopped; safety/database/runtime failure (details suppressed)")
|
||||||
return 1
|
if managedMonitorTechnicalTimeout(err) {
|
||||||
|
return 75
|
||||||
|
}
|
||||||
|
return 78
|
||||||
}
|
}
|
||||||
return 0
|
if ctx.Err() != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return 75 // A supervisor must not return normally without cancellation.
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only clean typed timeouts may restart; a joined cleanup failure vetoes it.
|
||||||
|
func managedMonitorTechnicalTimeout(err error) bool {
|
||||||
|
if timeout, ok := err.(*managedRecoveryTimeout); ok {
|
||||||
|
return managedRecoveryTimeoutSafe(timeout.Err)
|
||||||
|
}
|
||||||
|
if joined, ok := err.(interface{ Unwrap() []error }); ok {
|
||||||
|
for _, e := range joined.Unwrap() {
|
||||||
|
if !managedMonitorTechnicalTimeout(e) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(joined.Unwrap()) != 0
|
||||||
|
}
|
||||||
|
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
|
||||||
|
return managedMonitorTechnicalTimeout(wrapped.Unwrap())
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand cacheScan, checkCommand cacheCheck, command func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
|
func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand cacheScan, checkCommand cacheCheck, command func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
|
||||||
|
|||||||
@@ -40,6 +40,64 @@ func TestManagedMonitorCancellationSafety(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorExitClassification(t *testing.T) {
|
||||||
|
for _, mode := range []string{"timeout", "timeout-cleanup", "timeout-inner-cleanup", "fatal-timeout", "fatal-timeout-cleanup", "timeout-fatal", "job-timeout", "job-timeout-cleanup", "unknown", "zero", "cancel", "cancel-cleanup"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
d, env, _ := managedCLIFixture(t)
|
||||||
|
initManagedCLI(t, d)
|
||||||
|
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
want := 78
|
||||||
|
var out bytes.Buffer
|
||||||
|
code := managedMonitorCLIContext(ctx, nil, &out, &out, d, func(context.Context, managedMonitorDeps) error {
|
||||||
|
switch mode {
|
||||||
|
case "timeout":
|
||||||
|
want = 75
|
||||||
|
return &managedRecoveryTimeout{Err: context.DeadlineExceeded}
|
||||||
|
case "timeout-cleanup":
|
||||||
|
return errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("diagnostic close failed"))
|
||||||
|
case "timeout-inner-cleanup":
|
||||||
|
return &managedRecoveryTimeout{Err: errors.Join(context.DeadlineExceeded, errors.New("diagnostic close failed"))}
|
||||||
|
case "fatal-timeout":
|
||||||
|
want = 75 // Fatal is also the monitor's stop signal for clean timeout.
|
||||||
|
return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: context.DeadlineExceeded}}
|
||||||
|
case "fatal-timeout-cleanup":
|
||||||
|
return &managedRecoveryFatal{Err: errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("cleanup"))}
|
||||||
|
case "timeout-fatal":
|
||||||
|
return &managedRecoveryTimeout{Err: &managedRecoveryFatal{Err: errors.New("cleanup")}}
|
||||||
|
case "job-timeout", "job-timeout-cleanup":
|
||||||
|
if mode == "job-timeout" {
|
||||||
|
want = 75
|
||||||
|
}
|
||||||
|
return managedRecoveryJob(ctx, time.Millisecond, func(jobCtx context.Context) error {
|
||||||
|
<-jobCtx.Done()
|
||||||
|
if mode == "job-timeout-cleanup" {
|
||||||
|
return &managedRecoveryFatal{Err: errors.Join(jobCtx.Err(), errors.New("cleanup"))}
|
||||||
|
}
|
||||||
|
return jobCtx.Err()
|
||||||
|
})
|
||||||
|
case "zero":
|
||||||
|
want = 75
|
||||||
|
return nil
|
||||||
|
case "cancel":
|
||||||
|
want = 0
|
||||||
|
cancel()
|
||||||
|
return ctx.Err()
|
||||||
|
case "cancel-cleanup":
|
||||||
|
cancel()
|
||||||
|
return errors.Join(ctx.Err(), errors.New("cleanup"))
|
||||||
|
default:
|
||||||
|
return errors.New("unknown")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if code != want {
|
||||||
|
t.Fatal(code, want, out.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestManagedMonitorProductionAdapterBinding(t *testing.T) {
|
func TestManagedMonitorProductionAdapterBinding(t *testing.T) {
|
||||||
recovery, _, _ := recoveryFixture(t, false)
|
recovery, _, _ := recoveryFixture(t, false)
|
||||||
target, _, _ := managedCLIFixture(t)
|
target, _, _ := managedCLIFixture(t)
|
||||||
@@ -132,7 +190,11 @@ func TestManagedMonitorCLIGuardsBeforeLock(t *testing.T) {
|
|||||||
}
|
}
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
code := managedMonitorCLIContext(context.Background(), nil, &out, &out, d, func(context.Context, managedMonitorDeps) error { t.Fatal("unsafe supervisor start"); return nil })
|
code := managedMonitorCLIContext(context.Background(), nil, &out, &out, d, func(context.Context, managedMonitorDeps) error { t.Fatal("unsafe supervisor start"); return nil })
|
||||||
if code == 0 || strings.Contains(out.String(), "password-do-not-print") {
|
want := 78
|
||||||
|
if mode == "config" {
|
||||||
|
want = 64
|
||||||
|
}
|
||||||
|
if code != want || strings.Contains(out.String(), "password-do-not-print") {
|
||||||
t.Fatal(code, out.String())
|
t.Fatal(code, out.String())
|
||||||
}
|
}
|
||||||
if _, err := os.Lstat(filepath.Join(d.directory, managedMonitorLockName)); !errors.Is(err, os.ErrNotExist) {
|
if _, err := os.Lstat(filepath.Join(d.directory, managedMonitorLockName)); !errors.Is(err, os.ErrNotExist) {
|
||||||
@@ -199,7 +261,7 @@ func TestManagedMonitorBootstrapExistingFrontend(t *testing.T) {
|
|||||||
// Execute only rewritten target-local contracts and fake programs; no network,
|
// Execute only rewritten target-local contracts and fake programs; no network,
|
||||||
// Docker, real credentials, zapret or host firewall. Exercise the actual traps.
|
// Docker, real credentials, zapret or host firewall. Exercise the actual traps.
|
||||||
func TestManagedEntrypointSupervision(t *testing.T) {
|
func TestManagedEntrypointSupervision(t *testing.T) {
|
||||||
for _, mode := range []string{"term", "fatal"} {
|
for _, mode := range []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"} {
|
||||||
t.Run(mode, func(t *testing.T) {
|
t.Run(mode, func(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
config := filepath.Join(root, "ss")
|
config := filepath.Join(root, "ss")
|
||||||
@@ -215,16 +277,23 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
|||||||
}
|
}
|
||||||
log := filepath.Join(root, "log")
|
log := filepath.Join(root, "log")
|
||||||
write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n")
|
write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n")
|
||||||
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do sleep 0.1 & wait $!; done\n"
|
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & wait $!; done\n"
|
||||||
write("ss-server", child)
|
write("ss-server", child)
|
||||||
write("ss-local", child)
|
write("ss-local", child)
|
||||||
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\n")
|
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nif [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; while :; do sleep 0.1; done; fi\n")
|
||||||
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then echo managed-$2 >> \"$LOG\"; exit 0; fi\ntrap 'echo monitor-stopped >> \"$LOG\"; exit 0' TERM INT\necho managed-monitor >> \"$LOG\"\nif [ \"$MODE\" = fatal ]; then exit 7; fi\nwhile :; do sleep 0.1 & wait $!; done\n")
|
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then echo managed-$2 >> \"$LOG\"; exit 0; fi\ntrap 'echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; fi\necho managed-monitor >> \"$LOG\"\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & wait $!; done\n")
|
||||||
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
|
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
text := strings.NewReplacer("/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
|
text := strings.NewReplacer("/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
|
||||||
|
if mode == "term-hang" {
|
||||||
|
// Shorten only the fixture's watchdog, not the production contract.
|
||||||
|
text = strings.Replace(text, `bounded_wait "$monitor_pid" 60`, `bounded_wait "$monitor_pid" 1`, 1)
|
||||||
|
}
|
||||||
|
if mode == "stop-hang" {
|
||||||
|
text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1)
|
||||||
|
}
|
||||||
entry := write("entrypoint", text)
|
entry := write("entrypoint", text)
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -240,7 +309,7 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
|||||||
exited := make(chan struct{})
|
exited := make(chan struct{})
|
||||||
go func() { done <- cmd.Wait(); close(exited) }()
|
go func() { done <- cmd.Wait(); close(exited) }()
|
||||||
t.Cleanup(func() { syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); <-exited })
|
t.Cleanup(func() { syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); <-exited })
|
||||||
if mode == "term" {
|
if strings.HasPrefix(mode, "term") {
|
||||||
for {
|
for {
|
||||||
b, _ := os.ReadFile(log)
|
b, _ := os.ReadFile(log)
|
||||||
if strings.Contains(string(b), "managed-monitor") {
|
if strings.Contains(string(b), "managed-monitor") {
|
||||||
@@ -262,13 +331,20 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
|||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
t.Fatal("entrypoint did not join children", ctx.Err())
|
t.Fatal("entrypoint did not join children", ctx.Err())
|
||||||
}
|
}
|
||||||
if mode == "term" && err != nil {
|
want := 75
|
||||||
|
switch mode {
|
||||||
|
case "term":
|
||||||
|
want = 0
|
||||||
|
case "fatal", "unknown", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang":
|
||||||
|
want = 78
|
||||||
|
}
|
||||||
|
if want == 0 && err != nil {
|
||||||
t.Fatal(err, output.String())
|
t.Fatal(err, output.String())
|
||||||
}
|
}
|
||||||
if mode == "fatal" {
|
if want != 0 {
|
||||||
var exit *exec.ExitError
|
var exit *exec.ExitError
|
||||||
if !errors.As(err, &exit) || exit.ExitCode() != 7 {
|
if !errors.As(err, &exit) || exit.ExitCode() != want {
|
||||||
t.Fatal(err)
|
t.Fatal(err, "want", want, output.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
b, err := os.ReadFile(log)
|
b, err := os.ReadFile(log)
|
||||||
|
|||||||
@@ -9,12 +9,16 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2"
|
const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2"
|
||||||
|
|
||||||
|
const managedRecoveryJobTimeout = 90 * time.Minute
|
||||||
|
|
||||||
// Caller owns startup reconciliation and the monitor singleton. Discover and
|
// Caller owns startup reconciliation and the monitor singleton. Discover and
|
||||||
// Health must bind to the guarded target SOCKS. No CLI parsing or remote work.
|
// Health must bind to the guarded target SOCKS. No CLI parsing or remote work.
|
||||||
type managedRecoveryDeps struct {
|
type managedRecoveryDeps struct {
|
||||||
@@ -25,6 +29,7 @@ type managedRecoveryDeps struct {
|
|||||||
discover func(context.Context) (string, error)
|
discover func(context.Context) (string, error)
|
||||||
health func(context.Context) (string, error)
|
health func(context.Context) (string, error)
|
||||||
run func(context.Context, string, ...string) error
|
run func(context.Context, string, ...string) error
|
||||||
|
jobTimeout time.Duration // Zero uses the production budget; tests may shorten it.
|
||||||
}
|
}
|
||||||
|
|
||||||
// Target-local adapter: reuses bounded Player requests without invoking a CLI.
|
// Target-local adapter: reuses bounded Player requests without invoking a CLI.
|
||||||
@@ -55,7 +60,7 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
|
|||||||
return target.player(bounded, args, out, diagnostics)
|
return target.player(bounded, args, out, diagnostics)
|
||||||
})
|
})
|
||||||
if bounded.Err() != nil {
|
if bounded.Err() != nil {
|
||||||
return nil, bounded.Err()
|
return nil, managedRecoveryContextError(ctx, errors.Join(err, bounded.Err()))
|
||||||
}
|
}
|
||||||
return body, err
|
return body, err
|
||||||
}
|
}
|
||||||
@@ -95,11 +100,87 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
|
|||||||
|
|
||||||
type managedRecoveryFatal struct{ Err error }
|
type managedRecoveryFatal struct{ Err error }
|
||||||
|
|
||||||
|
// Only clean job expiration is restartable; safety failures retain their type.
|
||||||
|
type managedRecoveryTimeout struct{ Err error }
|
||||||
|
|
||||||
|
func (e *managedRecoveryTimeout) Error() string { return e.Err.Error() }
|
||||||
|
func (e *managedRecoveryTimeout) Unwrap() error { return e.Err }
|
||||||
|
|
||||||
var (
|
var (
|
||||||
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
|
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
|
||||||
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
|
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
|
||||||
|
errManagedRecoveryDeadline = errors.New("managed recovery total deadline exceeded")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Validate every leaf, not just errors.Is: a joined cleanup failure must never
|
||||||
|
// authorize a restart. Callers must separately prove the job's own deadline.
|
||||||
|
func managedRecoveryTimeoutSafe(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
switch e := err.(type) {
|
||||||
|
case *managedRecoveryFatal:
|
||||||
|
return false
|
||||||
|
case *exec.ExitError:
|
||||||
|
if e.ProcessState == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
status, ok := e.Sys().(syscall.WaitStatus)
|
||||||
|
return ok && status.Signaled() && (status.Signal() == syscall.SIGTERM || status.Signal() == syscall.SIGKILL)
|
||||||
|
case interface{ Unwrap() []error }:
|
||||||
|
children := e.Unwrap()
|
||||||
|
if len(children) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, child := range children {
|
||||||
|
if !managedRecoveryTimeoutSafe(child) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
case interface{ Unwrap() error }:
|
||||||
|
child := e.Unwrap()
|
||||||
|
return child != nil && managedRecoveryTimeoutSafe(child)
|
||||||
|
default:
|
||||||
|
return err == context.Canceled || err == context.DeadlineExceeded || err == errManagedRecoveryDeadline
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cancellation must prevent late success without erasing a concurrent safety
|
||||||
|
// or cleanup failure. Request-local timeouts alone leave the job context live.
|
||||||
|
func managedRecoveryContextError(ctx context.Context, err error) error {
|
||||||
|
if ctx.Err() == nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
joined := errors.Join(err, ctx.Err())
|
||||||
|
if !managedRecoveryTimeoutSafe(err) {
|
||||||
|
return &managedRecoveryFatal{Err: joined}
|
||||||
|
}
|
||||||
|
return joined
|
||||||
|
}
|
||||||
|
|
||||||
|
// Synchronous callbacks own cancellation cleanup/join. Classify expiration only
|
||||||
|
// after they return, so the monitor cannot release ownership before restoration.
|
||||||
|
func managedRecoveryJob(ctx context.Context, budget time.Duration, job func(context.Context) error) error {
|
||||||
|
if budget == 0 {
|
||||||
|
budget = managedRecoveryJobTimeout
|
||||||
|
}
|
||||||
|
jobCtx, cancel := context.WithTimeoutCause(ctx, budget, errManagedRecoveryDeadline)
|
||||||
|
defer cancel()
|
||||||
|
err := job(jobCtx)
|
||||||
|
if errors.Is(context.Cause(jobCtx), errManagedRecoveryDeadline) {
|
||||||
|
joined := errors.Join(err, errManagedRecoveryDeadline, context.DeadlineExceeded)
|
||||||
|
if !managedRecoveryTimeoutSafe(err) {
|
||||||
|
return &managedRecoveryFatal{Err: joined}
|
||||||
|
}
|
||||||
|
return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: joined}}
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
return jobCtx.Err()
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
func (e *managedRecoveryFatal) Error() string {
|
func (e *managedRecoveryFatal) Error() string {
|
||||||
return "managed recovery safety failure: " + e.Err.Error()
|
return "managed recovery safety failure: " + e.Err.Error()
|
||||||
}
|
}
|
||||||
@@ -119,15 +200,32 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
|||||||
}
|
}
|
||||||
m := *d.transaction
|
m := *d.transaction
|
||||||
// Rollback must restart even after job cancellation; independently bounded.
|
// Rollback must restart even after job cancellation; independently bounded.
|
||||||
|
var cleanupCtx context.Context
|
||||||
|
var cleanupCancel context.CancelFunc
|
||||||
|
cleanupContext := func() context.Context {
|
||||||
|
if cleanupCtx == nil {
|
||||||
|
cleanupCtx, cleanupCancel = context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||||
|
}
|
||||||
|
return cleanupCtx
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if cleanupCancel != nil {
|
||||||
|
cleanupCancel()
|
||||||
|
}
|
||||||
|
}()
|
||||||
m.Reload = func() error {
|
m.Reload = func() error {
|
||||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
if err := d.run(cleanupContext(), managedZapretScript, "restart"); err != nil {
|
||||||
defer cancel()
|
|
||||||
if err := d.run(cleanup, managedZapretScript, "restart"); err != nil {
|
|
||||||
return &managedRecoveryFatal{Err: err}
|
return &managedRecoveryFatal{Err: err}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
m.Health = func() (string, error) { return d.health(ctx) }
|
m.Health = func() (string, error) {
|
||||||
|
status, err := d.health(ctx)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return "", managedRecoveryContextError(ctx, err)
|
||||||
|
}
|
||||||
|
return status, err
|
||||||
|
}
|
||||||
ids, err := m.ActiveCandidates(d.db)
|
ids, err := m.ActiveCandidates(d.db)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, &managedRecoveryFatal{Err: err}
|
return result, &managedRecoveryFatal{Err: err}
|
||||||
@@ -150,9 +248,7 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
|||||||
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
if e := d.run(cleanupContext(), managedZapretScript, "start"); e != nil {
|
||||||
defer cancel()
|
|
||||||
if e := d.run(cleanup, managedZapretScript, "start"); e != nil {
|
|
||||||
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -180,6 +276,14 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
defer func() { m.Reload = cleanupReload }()
|
defer func() { m.Reload = cleanupReload }()
|
||||||
|
defer func() {
|
||||||
|
// A rejected candidate's rollback is complete before the next attempt;
|
||||||
|
// that later attempt needs its own restoration reserve.
|
||||||
|
if ctx.Err() == nil && cleanupCancel != nil {
|
||||||
|
cleanupCancel()
|
||||||
|
cleanupCtx, cleanupCancel = nil, nil
|
||||||
|
}
|
||||||
|
}()
|
||||||
e := m.Apply(d.db, ids)
|
e := m.Apply(d.db, ids)
|
||||||
if e == nil {
|
if e == nil {
|
||||||
stopped = false
|
stopped = false
|
||||||
@@ -197,12 +301,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
|||||||
healthy := false
|
healthy := false
|
||||||
if result.FrontendID != 0 {
|
if result.FrontendID != 0 {
|
||||||
status, e := d.health(ctx)
|
status, e := d.health(ctx)
|
||||||
|
e = managedRecoveryContextError(ctx, e)
|
||||||
var fatal *managedRecoveryFatal
|
var fatal *managedRecoveryFatal
|
||||||
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
|
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
|
||||||
return result, e
|
return result, e
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return result, err
|
return result, e
|
||||||
}
|
}
|
||||||
// A bounded Player request timeout is unhealthy, not job cancellation.
|
// A bounded Player request timeout is unhealthy, not job cancellation.
|
||||||
healthy = e == nil && status == "OK"
|
healthy = e == nil && status == "OK"
|
||||||
@@ -272,8 +377,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
host, e := d.discover(ctx)
|
host, e := d.discover(ctx)
|
||||||
|
e = managedRecoveryContextError(ctx, e)
|
||||||
|
var discoveryFatal *managedRecoveryFatal
|
||||||
|
if errors.As(e, &discoveryFatal) {
|
||||||
|
return result, e
|
||||||
|
}
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return result, ctx.Err()
|
return result, e
|
||||||
}
|
}
|
||||||
if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
|
if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
|
||||||
result.GGCError = fmt.Errorf("exact GGC discovery failed")
|
result.GGCError = fmt.Errorf("exact GGC discovery failed")
|
||||||
@@ -302,12 +412,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
|||||||
e = apply([]int64{result.FrontendID, cacheID})
|
e = apply([]int64{result.FrontendID, cacheID})
|
||||||
}
|
}
|
||||||
if e != nil {
|
if e != nil {
|
||||||
|
e = managedRecoveryContextError(ctx, e)
|
||||||
var fatal *managedRecoveryFatal
|
var fatal *managedRecoveryFatal
|
||||||
if errors.As(e, &fatal) {
|
if errors.As(e, &fatal) {
|
||||||
return result, e
|
return result, e
|
||||||
}
|
}
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return result, ctx.Err()
|
return result, e
|
||||||
}
|
}
|
||||||
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
|
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
|
||||||
return result, nil
|
return result, nil
|
||||||
@@ -441,7 +552,7 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
|||||||
// are already typed managedRecoveryFatal by the transaction boundary.
|
// are already typed managedRecoveryFatal by the transaction boundary.
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
monitor.bootstrap = func(ctx context.Context) error {
|
bootstrap := func(ctx context.Context) error {
|
||||||
if err := guard(); err != nil {
|
if err := guard(); err != nil {
|
||||||
return &managedRecoveryFatal{Err: err}
|
return &managedRecoveryFatal{Err: err}
|
||||||
}
|
}
|
||||||
@@ -471,12 +582,18 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
|||||||
}
|
}
|
||||||
return callback(ctx)
|
return callback(ctx)
|
||||||
}
|
}
|
||||||
monitor.recover = callback
|
monitor.bootstrap = func(ctx context.Context) error {
|
||||||
|
return managedRecoveryJob(ctx, recovery.jobTimeout, bootstrap)
|
||||||
|
}
|
||||||
|
monitor.recover = func(ctx context.Context) error {
|
||||||
|
return managedRecoveryJob(ctx, recovery.jobTimeout, callback)
|
||||||
|
}
|
||||||
monitor.probe = func(ctx context.Context) error {
|
monitor.probe = func(ctx context.Context) error {
|
||||||
if err := guard(); err != nil {
|
if err := guard(); err != nil {
|
||||||
return &managedRecoveryFatal{Err: err}
|
return &managedRecoveryFatal{Err: err}
|
||||||
}
|
}
|
||||||
status, err := recovery.health(ctx)
|
status, err := recovery.health(ctx)
|
||||||
|
err = managedRecoveryContextError(ctx, err)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -494,8 +611,9 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
|||||||
// Keep Player API health separate from GGC scope (not playback health).
|
// Keep Player API health separate from GGC scope (not playback health).
|
||||||
// Two bounded Player requests reuse the existing health/discovery adapters.
|
// Two bounded Player requests reuse the existing health/discovery adapters.
|
||||||
host, err := recovery.discover(ctx)
|
host, err := recovery.discover(ctx)
|
||||||
|
err = managedRecoveryContextError(ctx, err)
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return ctx.Err()
|
return err
|
||||||
}
|
}
|
||||||
var fatal *managedRecoveryFatal
|
var fatal *managedRecoveryFatal
|
||||||
if errors.As(err, &fatal) || errors.Is(err, context.Canceled) {
|
if errors.As(err, &fatal) || errors.Is(err, context.Canceled) {
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -85,6 +86,367 @@ func TestManagedRecoveryBootstrap(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryJobDeadline(t *testing.T) {
|
||||||
|
for _, mode := range []string{"bootstrap-scan", "bootstrap-saved", "periodic-saved", "periodic-apply", "periodic-ggc", "parent-cancel", "parent-deadline"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
bootstrap := strings.HasPrefix(mode, "bootstrap")
|
||||||
|
d, front, events := recoveryFixture(t, !bootstrap)
|
||||||
|
if mode == "bootstrap-saved" || mode == "periodic-saved" || mode == "periodic-apply" {
|
||||||
|
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(d.transaction.Dir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
monitor := monitorRuntimeFixture(t)
|
||||||
|
monitor.db, monitor.target.directory = d.db, d.transaction.Dir
|
||||||
|
identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||||
|
if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d.jobTimeout = 50 * time.Millisecond
|
||||||
|
parent, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
if mode == "parent-deadline" {
|
||||||
|
var deadlineCancel context.CancelFunc
|
||||||
|
parent, deadlineCancel = context.WithTimeout(parent, 50*time.Millisecond)
|
||||||
|
defer deadlineCancel()
|
||||||
|
d.jobTimeout = time.Minute
|
||||||
|
}
|
||||||
|
joined := false
|
||||||
|
var jobDeadline time.Time
|
||||||
|
observe := func(ctx context.Context) {
|
||||||
|
deadline, ok := ctx.Deadline()
|
||||||
|
if !ok || (!jobDeadline.IsZero() && !deadline.Equal(jobDeadline)) {
|
||||||
|
t.Fatal("job deadline missing or reset between phases")
|
||||||
|
}
|
||||||
|
jobDeadline = deadline
|
||||||
|
}
|
||||||
|
block := func(ctx context.Context) error {
|
||||||
|
observe(ctx)
|
||||||
|
// Model the runner contract: cancel the owned worker and join before
|
||||||
|
// returning control to restoration and the supervisor.
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { <-ctx.Done(); close(done) }()
|
||||||
|
if mode == "parent-cancel" {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
<-done
|
||||||
|
joined = true
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
baseScan, baseCheck := d.scan, d.check
|
||||||
|
d.scan = func(ctx context.Context, binary, host string, out, diagnostics io.Writer) (string, error) {
|
||||||
|
observe(ctx)
|
||||||
|
if mode == "bootstrap-scan" || mode == "parent-cancel" || mode == "parent-deadline" {
|
||||||
|
return "", block(ctx)
|
||||||
|
}
|
||||||
|
return baseScan(ctx, binary, host, out, diagnostics)
|
||||||
|
}
|
||||||
|
d.check = func(ctx context.Context, binary, path, args, host string, out, diagnostics io.Writer) error {
|
||||||
|
observe(ctx)
|
||||||
|
if mode != "periodic-apply" {
|
||||||
|
return block(ctx)
|
||||||
|
}
|
||||||
|
return baseCheck(ctx, binary, path, args, host, out, diagnostics)
|
||||||
|
}
|
||||||
|
healthCalls := 0
|
||||||
|
d.health = func(ctx context.Context) (string, error) {
|
||||||
|
observe(ctx)
|
||||||
|
healthCalls++
|
||||||
|
if mode == "periodic-apply" && healthCalls > 1 {
|
||||||
|
// Even a late OK must not commit a partially activated candidate.
|
||||||
|
_ = block(ctx)
|
||||||
|
return "OK", nil
|
||||||
|
}
|
||||||
|
if mode == "periodic-ggc" {
|
||||||
|
return "OK", nil
|
||||||
|
}
|
||||||
|
return "DEGRADED", nil
|
||||||
|
}
|
||||||
|
baseRun := d.run
|
||||||
|
d.run = func(ctx context.Context, path string, args ...string) error {
|
||||||
|
if args[0] == "start" || args[0] == "restart" && joined {
|
||||||
|
if !joined || ctx.Err() != nil {
|
||||||
|
t.Fatal("restoration preceded worker join or inherited cancellation")
|
||||||
|
}
|
||||||
|
deadline, ok := ctx.Deadline()
|
||||||
|
if !ok || time.Until(deadline) > 30*time.Second {
|
||||||
|
t.Fatal("restoration is not bounded")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return baseRun(ctx, path, args...)
|
||||||
|
}
|
||||||
|
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("deadline became ordinary GGC failure") }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
monitor.threshold = 1
|
||||||
|
monitor.probe = func(context.Context) error { return errors.New("unhealthy") }
|
||||||
|
reports := 0
|
||||||
|
monitor.reportRecoveryError = func(error) { reports++ }
|
||||||
|
monitor.wait = func(ctx context.Context, _ time.Duration) error {
|
||||||
|
t.Fatal("expired recovery entered ordinary cooldown retry")
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
err := runManagedMonitor(parent, monitor)
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
var timeout *managedRecoveryTimeout
|
||||||
|
external := strings.HasPrefix(mode, "parent")
|
||||||
|
if errors.As(err, &fatal) == external || !joined {
|
||||||
|
t.Fatalf("err=%v joined=%v", err, joined)
|
||||||
|
}
|
||||||
|
if errors.As(err, &timeout) == external {
|
||||||
|
t.Fatalf("wrong restartable timeout classification: %v", err)
|
||||||
|
}
|
||||||
|
if !external && (!errors.Is(err, errManagedRecoveryDeadline) || !errors.Is(err, context.DeadlineExceeded)) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if external && !errors.Is(err, parent.Err()) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []int64(nil)
|
||||||
|
if front != 0 {
|
||||||
|
want = []int64{front}
|
||||||
|
}
|
||||||
|
assertActiveCandidates(t, d.transaction, d.db, want) // Also proves transaction lock release.
|
||||||
|
assertMonitorUnlocked(t, monitor)
|
||||||
|
if (*events)[len(*events)-1] != "start" || reports > 1 {
|
||||||
|
t.Fatalf("events=%v reports=%d", *events, reports)
|
||||||
|
}
|
||||||
|
if !bootstrap {
|
||||||
|
state, err := readMonitorState(d.db, monitor.serviceID)
|
||||||
|
if err != nil || state.RecoveryRunning {
|
||||||
|
t.Fatalf("recovery ownership leaked: %+v %v", state, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryJobSafetyFailureNotTimeout(t *testing.T) {
|
||||||
|
for _, mode := range []string{"cleanup", "rollback", "database", "guard"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
unsafe := errors.New(mode + " failed")
|
||||||
|
err := managedRecoveryJob(context.Background(), time.Nanosecond, func(ctx context.Context) error {
|
||||||
|
<-ctx.Done()
|
||||||
|
return &managedRecoveryFatal{Err: errors.Join(ctx.Err(), unsafe)}
|
||||||
|
})
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
var timeout *managedRecoveryTimeout
|
||||||
|
if !errors.As(err, &fatal) || errors.As(err, &timeout) || !errors.Is(err, unsafe) || !errors.Is(err, errManagedRecoveryDeadline) {
|
||||||
|
t.Fatalf("unsafe deadline classified as restartable: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryTimeoutLeaves(t *testing.T) {
|
||||||
|
processExit := func(command string) error {
|
||||||
|
t.Helper()
|
||||||
|
err := exec.Command("sh", "-c", command).Run()
|
||||||
|
var exit *exec.ExitError
|
||||||
|
if !errors.As(err, &exit) {
|
||||||
|
t.Fatalf("expected process exit: %v", err)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
closeErr := errors.New("diagnostic close failed")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
safe bool
|
||||||
|
}{
|
||||||
|
{"nil", nil, true},
|
||||||
|
{"deadline", context.DeadlineExceeded, true},
|
||||||
|
{"wrapped-deadline", fmt.Errorf("worker: %w", context.DeadlineExceeded), true},
|
||||||
|
{"canceled", context.Canceled, true},
|
||||||
|
{"term", errors.Join(context.DeadlineExceeded, processExit("kill -TERM $$")), true},
|
||||||
|
{"kill", errors.Join(context.Canceled, processExit("kill -KILL $$")), true},
|
||||||
|
{"unknown-exit", errors.Join(context.DeadlineExceeded, processExit("exit 7")), false},
|
||||||
|
{"unknown-leaf", closeErr, false},
|
||||||
|
{"joined-close", errors.Join(context.DeadlineExceeded, closeErr), false},
|
||||||
|
{"nested-close", fmt.Errorf("worker: %w", errors.Join(context.Canceled, fmt.Errorf("close: %w", closeErr))), false},
|
||||||
|
{"empty-exit", &exec.ExitError{}, false},
|
||||||
|
{"joined-fatal", errors.Join(context.DeadlineExceeded, &managedRecoveryFatal{Err: context.Canceled}), false},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if got := managedRecoveryTimeoutSafe(test.err); got != test.safe {
|
||||||
|
t.Fatalf("leaf validation=%v want=%v", got, test.safe)
|
||||||
|
}
|
||||||
|
err := managedRecoveryJob(context.Background(), time.Nanosecond, func(ctx context.Context) error {
|
||||||
|
<-ctx.Done()
|
||||||
|
return test.err
|
||||||
|
})
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
var timeout *managedRecoveryTimeout
|
||||||
|
if !errors.As(err, &fatal) || errors.As(err, &timeout) != test.safe || !errors.Is(err, errManagedRecoveryDeadline) {
|
||||||
|
t.Fatalf("incorrect deadline classification: %v", err)
|
||||||
|
}
|
||||||
|
if test.safe && !managedRecoveryTimeoutSafe(timeout.Err) {
|
||||||
|
t.Fatal("timeout payload failed reusable validation")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryDiscoveryFatal(t *testing.T) {
|
||||||
|
for _, mode := range []string{"guard-fatal", "fatal-parent-cancel", "ordinary-unresolved"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
d, front, events := recoveryFixture(t, true)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
refusal := errors.New("discovery target guard refused")
|
||||||
|
d.discover = func(context.Context) (string, error) {
|
||||||
|
if mode == "ordinary-unresolved" {
|
||||||
|
return "", refusal
|
||||||
|
}
|
||||||
|
if mode == "fatal-parent-cancel" {
|
||||||
|
cancel()
|
||||||
|
return "", errors.Join(context.Canceled, &managedRecoveryFatal{Err: refusal})
|
||||||
|
}
|
||||||
|
return "", &managedRecoveryFatal{Err: refusal}
|
||||||
|
}
|
||||||
|
result, err := recoverManagedTarget(ctx, d)
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if mode == "ordinary-unresolved" {
|
||||||
|
if err != nil || result.GGCError == nil {
|
||||||
|
t.Fatalf("ordinary discovery error became fatal: %+v %v", result, err)
|
||||||
|
}
|
||||||
|
} else if !errors.As(err, &fatal) || !errors.Is(err, refusal) || result.GGCError != nil {
|
||||||
|
t.Fatalf("discovery safety refusal masked: %+v %v", result, err)
|
||||||
|
}
|
||||||
|
if mode == "fatal-parent-cancel" && !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(*events) != 0 {
|
||||||
|
t.Fatal("discovery refusal triggered activation", *events)
|
||||||
|
}
|
||||||
|
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryHealthDeadlinePreservesFailure(t *testing.T) {
|
||||||
|
for _, mode := range []string{"fatal-guard", "mixed-close", "late-ok"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
d, prior, events := recoveryFixture(t, true)
|
||||||
|
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
|
||||||
|
priorBytes, err := d.transaction.read("config")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
failure := errors.New("concurrent health guard or diagnostic close failed")
|
||||||
|
calls := 0
|
||||||
|
d.health = func(ctx context.Context) (string, error) {
|
||||||
|
calls++
|
||||||
|
if calls == 1 {
|
||||||
|
return "DEGRADED", nil
|
||||||
|
}
|
||||||
|
<-ctx.Done()
|
||||||
|
switch mode {
|
||||||
|
case "fatal-guard":
|
||||||
|
return "", &managedRecoveryFatal{Err: failure}
|
||||||
|
case "mixed-close":
|
||||||
|
return "", errors.Join(ctx.Err(), failure)
|
||||||
|
default:
|
||||||
|
return "OK", nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = managedRecoveryJob(context.Background(), 50*time.Millisecond, func(ctx context.Context) error {
|
||||||
|
_, err := recoverManagedTarget(ctx, d)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
var timeout *managedRecoveryTimeout
|
||||||
|
if !errors.As(err, &fatal) || errors.As(err, &timeout) != (mode == "late-ok") || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("health deadline misclassified: %v", err)
|
||||||
|
}
|
||||||
|
if mode != "late-ok" && !errors.Is(err, failure) {
|
||||||
|
t.Fatalf("original safety failure erased: %v", err)
|
||||||
|
}
|
||||||
|
assertActiveCandidates(t, d.transaction, d.db, []int64{prior})
|
||||||
|
assertManagedContent(t, d.transaction, string(priorBytes))
|
||||||
|
if calls != 2 || strings.Count(strings.Join(*events, ","), "restart") != 2 || (*events)[len(*events)-1] != "start" {
|
||||||
|
t.Fatalf("rollback/restoration incomplete: calls=%d events=%v", calls, *events)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryContextError(t *testing.T) {
|
||||||
|
unsafe := errors.New("diagnostic close failed")
|
||||||
|
for _, original := range []error{nil, context.Canceled, context.DeadlineExceeded, unsafe, errors.Join(context.Canceled, unsafe), &managedRecoveryFatal{Err: unsafe}} {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
// A live job leaves request-local errors ordinary and unchanged.
|
||||||
|
if got := managedRecoveryContextError(ctx, original); got != original {
|
||||||
|
t.Fatalf("live context changed request-local error: %v", got)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
got := managedRecoveryContextError(ctx, original)
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if !errors.Is(got, context.Canceled) || errors.As(got, &fatal) == managedRecoveryTimeoutSafe(original) {
|
||||||
|
t.Fatalf("cancellation lost safety classification: original=%v got=%v", original, got)
|
||||||
|
}
|
||||||
|
if original != nil && !errors.Is(got, original) {
|
||||||
|
t.Fatalf("original failure lost: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryBootstrapCleanupFailureSurvivesCancellation(t *testing.T) {
|
||||||
|
for _, mode := range []string{"deadline", "parent-cancel"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
d, _, events := recoveryFixture(t, false)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
joined := false
|
||||||
|
d.scan = func(jobCtx context.Context, _, _ string, _, _ io.Writer) (string, error) {
|
||||||
|
if mode == "parent-cancel" {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
<-jobCtx.Done()
|
||||||
|
joined = true
|
||||||
|
return "", jobCtx.Err()
|
||||||
|
}
|
||||||
|
unsafe := errors.New("last-good start failed")
|
||||||
|
baseRun := d.run
|
||||||
|
d.run = func(cleanup context.Context, path string, args ...string) error {
|
||||||
|
if err := baseRun(cleanup, path, args...); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if args[0] == "start" {
|
||||||
|
if !joined {
|
||||||
|
t.Fatal("cleanup before worker join")
|
||||||
|
}
|
||||||
|
return unsafe
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
monitor := monitorRuntimeFixture(t)
|
||||||
|
monitor.bootstrap = func(ctx context.Context) error {
|
||||||
|
return managedRecoveryJob(ctx, 20*time.Millisecond, func(jobCtx context.Context) error {
|
||||||
|
_, err := recoverManagedTarget(jobCtx, d)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
err := runManagedMonitor(ctx, monitor)
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
var timeout *managedRecoveryTimeout
|
||||||
|
if !errors.As(err, &fatal) || errors.As(err, &timeout) || !errors.Is(err, unsafe) {
|
||||||
|
t.Fatalf("cleanup failure masked or restartable: %v", err)
|
||||||
|
}
|
||||||
|
if mode == "deadline" && !errors.Is(err, errManagedRecoveryDeadline) || mode == "parent-cancel" && !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(*events, []string{"stop", "start"}) {
|
||||||
|
t.Fatal(*events)
|
||||||
|
}
|
||||||
|
assertActiveCandidates(t, d.transaction, d.db, nil)
|
||||||
|
assertManagedContent(t, d.transaction, managedBaseline)
|
||||||
|
assertMonitorUnlocked(t, monitor)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
|
func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
|
||||||
d, _, events := recoveryFixture(t, false)
|
d, _, events := recoveryFixture(t, false)
|
||||||
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
|
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
|
||||||
|
|||||||
@@ -123,13 +123,13 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
|||||||
}
|
}
|
||||||
if d.bootstrap != nil {
|
if d.bootstrap != nil {
|
||||||
if err := d.bootstrap(ctx); err != nil {
|
if err := d.bootstrap(ctx); err != nil {
|
||||||
if ctx.Err() != nil {
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
var fatal *managedRecoveryFatal
|
var fatal *managedRecoveryFatal
|
||||||
if errors.As(err, &fatal) {
|
if errors.As(err, &fatal) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
// Bootstrap scans can validly find no qualified strategy. Keep the
|
// Bootstrap scans can validly find no qualified strategy. Keep the
|
||||||
// supervisor alive so durable probes/cooldown govern the next attempt.
|
// supervisor alive so durable probes/cooldown govern the next attempt.
|
||||||
d.reportRecoveryError(err)
|
d.reportRecoveryError(err)
|
||||||
|
|||||||
@@ -31,9 +31,40 @@ Then build and start only this Compose project. The rendered configuration
|
|||||||
contains the environment password, so never use plain `docker compose config`
|
contains the environment password, so never use plain `docker compose config`
|
||||||
in logs or shared terminals.
|
in logs or shared terminals.
|
||||||
|
|
||||||
|
By default Compose uses the locally built `zapret-autotune-managed-target:local`.
|
||||||
|
Tags are only local build/test candidates, not production release references.
|
||||||
|
For production promotion, set `ZAPRET_AUTOTUNE_IMAGE=<registry>/<managed-target>@sha256:<digest>`
|
||||||
|
in the private `.env` and pull that immutable image before starting the unit.
|
||||||
|
Use a managed target image, not the generic image; the systemd command never
|
||||||
|
builds images. Do not enable the production service with a tag or local default.
|
||||||
|
|
||||||
Offline static guard (from repository root):
|
Offline static guard (from repository root):
|
||||||
`python3 deploy/managed-target/check_static.py`.
|
`python3 deploy/managed-target/check_static.py`.
|
||||||
|
|
||||||
|
## Optional native systemd supervision
|
||||||
|
|
||||||
|
For unattended boot, place this checkout at `/opt/zapret2-autotune`, provision
|
||||||
|
`deploy/managed-target/.env` and state as above, and build the target image first.
|
||||||
|
For production, select and pull the immutable managed-target digest as above
|
||||||
|
instead of using a local build tag.
|
||||||
|
Install `deploy/managed-target/zapret-autotune.service` in `/etc/systemd/system/`,
|
||||||
|
then run `systemctl daemon-reload` and `systemctl enable --now zapret-autotune`.
|
||||||
|
Do not simultaneously start the same project with detached Compose.
|
||||||
|
Enabling the unit starts this isolated target on subsequent boots. `ExecStop`
|
||||||
|
only stops `target`; it does not replace the foreground process's failure code.
|
||||||
|
An explicit service stop suppresses restart, including during a shutdown race.
|
||||||
|
|
||||||
|
Compose has no automatic restart. The foreground systemd unit restarts **only
|
||||||
|
exit 75**, with at most three starts per six hours. Clean typed recovery timeouts,
|
||||||
|
unexpected monitor exits/crashes, and Shadowsocks failures qualify only if
|
||||||
|
shutdown cleanup succeeds. Configuration (64), safety/database/rollback/cleanup
|
||||||
|
failures (78), and graceful SIGTERM (0) never request restart. Investigate and
|
||||||
|
repair nonretry failures before manually starting again; do not clear state.
|
||||||
|
Monitor cancellation/join is bounded to 60 seconds, final target-local zapret
|
||||||
|
stop gets 30 seconds then forced kill after five more, and each Shadowsocks join
|
||||||
|
gets five seconds. Exceeding a join bound fails closed. Compose grants 120
|
||||||
|
seconds; systemd grants 180 seconds.
|
||||||
|
|
||||||
## Isolation and runtime contract
|
## Isolation and runtime contract
|
||||||
|
|
||||||
Only SOCKS TCP/UDP is published on host
|
Only SOCKS TCP/UDP is published on host
|
||||||
@@ -47,9 +78,10 @@ password before starting child processes.
|
|||||||
|
|
||||||
The dedicated bridge is not shared with existing services. No host networking,
|
The dedicated bridge is not shared with existing services. No host networking,
|
||||||
Docker socket, existing mounts, host Lua/fakes or privileged mode; all
|
Docker socket, existing mounts, host Lua/fakes or privileged mode; all
|
||||||
capabilities are dropped except `NET_ADMIN`, `NET_RAW`, `SETUID`, `SETGID`.
|
capabilities are dropped except `NET_ADMIN`, `NET_RAW`, `SETUID`, `SETGID`, `KILL`.
|
||||||
`NET_ADMIN` covers nft/NFQUEUE; `NET_RAW` covers nfqws2 raw sends. Zapret2's
|
`NET_ADMIN` covers nft/NFQUEUE; `NET_RAW` covers nfqws2 raw sends. Zapret2's
|
||||||
`nfqws2 --user=tpws` needs `SETUID`/`SETGID` for UID/GID switching.
|
`nfqws2 --user=tpws` needs `SETUID`/`SETGID` for UID/GID switching.
|
||||||
|
`KILL` allows bounded cleanup of workers that have dropped their UID.
|
||||||
|
|
||||||
Startup order: ss-server/ss-local → listener 1080 → guarded managed init on
|
Startup order: ss-server/ss-local → listener 1080 → guarded managed init on
|
||||||
fresh state (status on recreation) → target-local `zapret2 start` → supervised
|
fresh state (status on recreation) → target-local `zapret2 start` → supervised
|
||||||
|
|||||||
@@ -7,10 +7,12 @@ directory = Path(__file__).resolve().parent
|
|||||||
compose = (directory / "compose.yaml").read_text()
|
compose = (directory / "compose.yaml").read_text()
|
||||||
entrypoint = (directory.parents[1] / "managed-target-entrypoint.sh").read_text()
|
entrypoint = (directory.parents[1] / "managed-target-entrypoint.sh").read_text()
|
||||||
readme = (directory / "README.md").read_text()
|
readme = (directory / "README.md").read_text()
|
||||||
|
unit = (directory / "zapret-autotune.service").read_text()
|
||||||
|
|
||||||
# ponytail: checks the committed Compose shape, not arbitrary YAML; use a YAML
|
# ponytail: checks the committed Compose shape, not arbitrary YAML; use a YAML
|
||||||
# parser if this definition grows beyond its single target service.
|
# parser if this definition grows beyond its single target service.
|
||||||
assert " privileged: false\n" in compose
|
assert " privileged: false\n" in compose
|
||||||
|
assert " image: ${ZAPRET_AUTOTUNE_IMAGE:-zapret-autotune-managed-target:local}\n" in compose
|
||||||
assert " cap_drop: [ALL]\n" in compose
|
assert " cap_drop: [ALL]\n" in compose
|
||||||
assert " cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID, KILL]\n" in compose
|
assert " cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID, KILL]\n" in compose
|
||||||
assert compose.count(" - type: bind\n") == 1
|
assert compose.count(" - type: bind\n") == 1
|
||||||
@@ -26,3 +28,18 @@ assert "export SS_PASSWORD" not in entrypoint
|
|||||||
assert "Docker administrators/root can still read container" in readme
|
assert "Docker administrators/root can still read container" in readme
|
||||||
assert "removed\nfrom the entrypoint environment" in readme
|
assert "removed\nfrom the entrypoint environment" in readme
|
||||||
assert "No host networking" in readme
|
assert "No host networking" in readme
|
||||||
|
assert ' restart: "no"\n' in compose
|
||||||
|
assert " stop_grace_period: 120s\n" in compose
|
||||||
|
directives = dict(line.split("=", 1) for line in unit.splitlines() if "=" in line)
|
||||||
|
assert directives["Restart"] == "no"
|
||||||
|
assert directives["RestartForceExitStatus"] == "75"
|
||||||
|
assert "RestartPreventExitStatus" not in directives
|
||||||
|
assert directives["StartLimitIntervalSec"] == "6h"
|
||||||
|
assert directives["StartLimitBurst"] == "3"
|
||||||
|
assert directives["TimeoutStopSec"] == "180s"
|
||||||
|
assert directives["WorkingDirectory"] == "/opt/zapret2-autotune"
|
||||||
|
assert directives["WantedBy"] == "multi-user.target"
|
||||||
|
compose_command = "/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml"
|
||||||
|
assert directives["ExecStart"] == compose_command + " up --no-build --abort-on-container-exit --exit-code-from target target"
|
||||||
|
assert directives["ExecStop"] == compose_command + " stop target"
|
||||||
|
assert "timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop" in entrypoint
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
name: zapret-autotune-managed-test
|
name: zapret-autotune-managed-test
|
||||||
services:
|
services:
|
||||||
target:
|
target:
|
||||||
image: zapret-autotune-managed-target:local
|
image: ${ZAPRET_AUTOTUNE_IMAGE:-zapret-autotune-managed-target:local}
|
||||||
build:
|
build:
|
||||||
context: ../..
|
context: ../..
|
||||||
dockerfile: Dockerfile.target
|
dockerfile: Dockerfile.target
|
||||||
@@ -30,7 +30,8 @@ services:
|
|||||||
tmpfs:
|
tmpfs:
|
||||||
- /run/zapret-autotune-ss:rw,noexec,nosuid,nodev,size=1m,mode=0700,uid=0,gid=0
|
- /run/zapret-autotune-ss:rw,noexec,nosuid,nodev,size=1m,mode=0700,uid=0,gid=0
|
||||||
networks: [target]
|
networks: [target]
|
||||||
stop_grace_period: 45s
|
restart: "no"
|
||||||
|
stop_grace_period: 120s
|
||||||
networks:
|
networks:
|
||||||
target:
|
target:
|
||||||
driver: bridge
|
driver: bridge
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Isolated zapret-autotune managed target
|
||||||
|
Requires=docker.service
|
||||||
|
After=docker.service
|
||||||
|
StartLimitIntervalSec=6h
|
||||||
|
StartLimitBurst=3
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
WorkingDirectory=/opt/zapret2-autotune
|
||||||
|
ExecStart=/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml up --no-build --abort-on-container-exit --exit-code-from target target
|
||||||
|
ExecStop=/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml stop target
|
||||||
|
Restart=no
|
||||||
|
RestartForceExitStatus=75
|
||||||
|
RestartSec=30s
|
||||||
|
TimeoutStopSec=180s
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
set -eu
|
set -eu
|
||||||
umask 077
|
umask 077
|
||||||
[ "$#" -eq 0 ] || { echo 'error: target entrypoint accepts no arguments' >&2; exit 1; }
|
[ "$#" -eq 0 ] || { echo 'error: target entrypoint accepts no arguments' >&2; exit 64; }
|
||||||
|
|
||||||
# Credentials never enter argv or logs. Docker supplies SS_PASSWORD via env;
|
# Credentials never enter argv or logs. Docker supplies SS_PASSWORD via env;
|
||||||
# clear the exported value before starting any child processes.
|
# clear the exported value before starting any child processes.
|
||||||
@@ -9,12 +9,12 @@ ss_password=${SS_PASSWORD-}
|
|||||||
unset SS_PASSWORD
|
unset SS_PASSWORD
|
||||||
case "$ss_password" in
|
case "$ss_password" in
|
||||||
''|*[!0123456789abcdefABCDEF]*)
|
''|*[!0123456789abcdefABCDEF]*)
|
||||||
echo 'error: target secret must contain only hexadecimal characters' >&2; exit 1 ;;
|
echo 'error: target secret must contain only hexadecimal characters' >&2; exit 64 ;;
|
||||||
esac
|
esac
|
||||||
[ "${#ss_password}" -ge 32 ] || { echo 'error: target secret must be at least 32 hex characters' >&2; exit 1; }
|
[ "${#ss_password}" -ge 32 ] || { echo 'error: target secret must be at least 32 hex characters' >&2; exit 64; }
|
||||||
config_dir=/run/zapret-autotune-ss
|
config_dir=/run/zapret-autotune-ss
|
||||||
awk '$2 == "/run/zapret-autotune-ss" && $3 == "tmpfs" { found=1 } END { exit !found }' /proc/mounts || {
|
awk '$2 == "/run/zapret-autotune-ss" && $3 == "tmpfs" { found=1 } END { exit !found }' /proc/mounts || {
|
||||||
echo 'error: target Shadowsocks config directory must be a dedicated tmpfs' >&2; exit 1;
|
echo 'error: target Shadowsocks config directory must be a dedicated tmpfs' >&2; exit 78;
|
||||||
}
|
}
|
||||||
chmod 0700 "$config_dir"
|
chmod 0700 "$config_dir"
|
||||||
cat > "$config_dir/server.json" <<EOF
|
cat > "$config_dir/server.json" <<EOF
|
||||||
@@ -30,23 +30,49 @@ server_pid=
|
|||||||
local_pid=
|
local_pid=
|
||||||
monitor_pid=
|
monitor_pid=
|
||||||
zapret_started=0
|
zapret_started=0
|
||||||
|
# Wait with a watchdog: never let an unresponsive child exceed Docker's grace.
|
||||||
|
bounded_wait() {
|
||||||
|
child=$1
|
||||||
|
seconds=$2
|
||||||
|
(
|
||||||
|
sleep "$seconds" & sleeper=$!
|
||||||
|
trap 'kill "$sleeper" 2>/dev/null || true; exit 0' TERM INT
|
||||||
|
wait "$sleeper"
|
||||||
|
kill -KILL "$child" 2>/dev/null || true
|
||||||
|
) </dev/null >/dev/null 2>&1 &
|
||||||
|
watchdog=$!
|
||||||
|
waited_code=0
|
||||||
|
wait "$child" 2>/dev/null || waited_code=$?
|
||||||
|
kill -TERM "$watchdog" 2>/dev/null || true
|
||||||
|
wait "$watchdog" 2>/dev/null || true
|
||||||
|
}
|
||||||
cleanup() {
|
cleanup() {
|
||||||
|
final_code=$?
|
||||||
|
case "$final_code" in 0|64|75|78) ;; *) final_code=78 ;; esac
|
||||||
trap - EXIT INT TERM
|
trap - EXIT INT TERM
|
||||||
# Let the monitor cancel/join scans and restore committed bytes BEFORE stop.
|
# Let the monitor cancel/join scans and restore committed bytes BEFORE stop.
|
||||||
if [ -n "$monitor_pid" ]; then
|
if [ -n "$monitor_pid" ]; then
|
||||||
kill -TERM "$monitor_pid" 2>/dev/null || true
|
kill -TERM "$monitor_pid" 2>/dev/null || true
|
||||||
wait "$monitor_pid" 2>/dev/null || true
|
bounded_wait "$monitor_pid" 60
|
||||||
|
# Cancellation must restore safely even when SS failure requested retry.
|
||||||
|
case "$waited_code" in 0|143) ;; *) final_code=78 ;; esac
|
||||||
fi
|
fi
|
||||||
# Dedicated PID/network namespace: no host/other-target stop.
|
# Dedicated PID/network namespace: no host/other-target stop.
|
||||||
if [ "$zapret_started" -eq 1 ]; then
|
if [ "$zapret_started" -eq 1 ]; then
|
||||||
/opt/zapret2/init.d/sysv/zapret2 stop || true
|
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop || final_code=78
|
||||||
fi
|
fi
|
||||||
for pid in "$local_pid" "$server_pid"; do
|
for pid in "$local_pid" "$server_pid"; do
|
||||||
[ -z "$pid" ] || kill "$pid" 2>/dev/null || true
|
[ -z "$pid" ] || kill "$pid" 2>/dev/null || true
|
||||||
done
|
done
|
||||||
for pid in "$local_pid" "$server_pid"; do
|
for pid in "$local_pid" "$server_pid"; do
|
||||||
[ -z "$pid" ] || wait "$pid" 2>/dev/null || true
|
if [ -n "$pid" ]; then
|
||||||
|
bounded_wait "$pid" 5
|
||||||
|
# Its original failure triggered supervision; only forced shutdown
|
||||||
|
# is a cleanup failure, not the already-dead Shadowsocks status.
|
||||||
|
[ "$waited_code" -ne 137 ] || final_code=78
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
|
exit "$final_code"
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
trap 'exit 0' INT TERM
|
trap 'exit 0' INT TERM
|
||||||
@@ -60,9 +86,9 @@ local_pid=$!
|
|||||||
# Bounded readiness, no network probe. Init/status verifies socket ownership.
|
# Bounded readiness, no network probe. Init/status verifies socket ownership.
|
||||||
attempt=0
|
attempt=0
|
||||||
until awk '$2 == "00000000:0438" && $4 == "0A" { found=1 } END { exit !found }' /proc/net/tcp; do
|
until awk '$2 == "00000000:0438" && $4 == "0A" { found=1 } END { exit !found }' /proc/net/tcp; do
|
||||||
kill -0 "$server_pid" && kill -0 "$local_pid"
|
{ kill -0 "$server_pid" && kill -0 "$local_pid"; } || exit 75
|
||||||
attempt=$((attempt + 1))
|
attempt=$((attempt + 1))
|
||||||
[ "$attempt" -lt 30 ] || { echo 'error: target SOCKS listener timeout' >&2; exit 1; }
|
[ "$attempt" -lt 30 ] || { echo 'error: target SOCKS listener timeout' >&2; exit 75; }
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
if [ ! -e "$ZAPRET_CONFIG" ]; then
|
if [ ! -e "$ZAPRET_CONFIG" ]; then
|
||||||
@@ -84,8 +110,12 @@ if ! kill -0 "$monitor_pid" 2>/dev/null; then
|
|||||||
wait "$monitor_pid" || code=$?
|
wait "$monitor_pid" || code=$?
|
||||||
monitor_pid=
|
monitor_pid=
|
||||||
echo 'error: target managed monitor exited' >&2
|
echo 'error: target managed monitor exited' >&2
|
||||||
[ "$code" -ne 0 ] || code=1
|
case "$code" in
|
||||||
|
0|2|137|139) code=75 ;;
|
||||||
|
64|75|78) ;;
|
||||||
|
*) code=78 ;;
|
||||||
|
esac
|
||||||
exit "$code"
|
exit "$code"
|
||||||
fi
|
fi
|
||||||
echo 'error: target child process exited' >&2
|
echo 'error: target child process exited' >&2
|
||||||
exit 1
|
exit 75
|
||||||
Reference in new issue
Block a user