diff --git a/RESOURCE_MONITOR.md b/RESOURCE_MONITOR.md new file mode 100644 index 0000000..53037e0 --- /dev/null +++ b/RESOURCE_MONITOR.md @@ -0,0 +1,66 @@ +# Periodic configured resource checks + +`managed monitor --resource-config FILE` runs resource-config v1 periodically. +PID 1 selects the same mode when `ZAPRET_AUTOTUNE_RESOURCE_CONFIG` is nonempty; +unset preserves the existing YouTube monitor. The modes share one singleton +lock. Supply an absolute container-visible config path (for example mount a +read-only operator config directory using a Compose override). Configuration +must be a regular root/runtime-owned, non-writable-by-others file without +symlink/hardlink aliases. Environment references are resolved on every reload. + +Each resource accepts `interval_ms` (default 300000) and `cooldown_ms` (default +30000), both integers in [100, 86400000]. Checks are sequential round robin +with independent due times; `tls: both` requires both passes. Ordinary DNS, +TLS, timeout or expectation failures mark only that resource DEGRADED and +retry after its cooldown. There is **no strategy selection, recovery or apply**, +and no no-strategy two-hour pause. HEALTHY means configured checks passed, +not playback health, generic recovery or whole-production READY: HEALTHY is +**CHECKS_ONLY**, never READY. The scheduling, status, persistence and lifecycle +base is unit/fixture-verified; isolated remote fault acceptance remains pending. + +Invalid resource JSON initially or after reload is BLOCKED, with no DNS or +transfers until repaired. Bad static target role, proxy or DNS settings still +fail startup validation; resource JSON is deliberately not part of +`target-config-check`. Ownership and fresh source snapshots are checked before +DNS and again before transfer; existing DNS/IP policy, pinned SOCKS transfers, +TLS verification and private stdin are unchanged. No cached addresses bypass +DNS policy. Existing zapret reconciliation remains mandatory and generic +startup never consumes experimental apply requests. + +State is independent of recovery SQL leases, fixed in the private target +directory as `.resource-monitor-state.json` and `.resource-monitor-key`. +Both are private 0600 files. Fingerprints HMAC the resolved configuration +(including secrets); state is also authenticated and never contains URLs, +headers, addresses or raw errors. Missing state initializes UNKNOWN. A missing +key with existing state, corrupt or unsafe file is preserved and BLOCKED, +never silently replaced. Restore the original key and a valid version-1 signed +state from a trusted backup to repair it. The controller holds directory, key +and state inode anchors: an identical signed replacement is still BLOCKED and +preserved. Repair on the original held inode is reloaded only after version/MAC +validation; replacing an inode requires deliberate operator repair/restart. +A persistence failure pauses probes; +deliberate repair requires a different valid authenticated snapshot, not just +retrying a failed write. Never chmod/adopt unknown files to bypass validation. +This is not isolation from a hostile same-UID actor: that actor can read the +HMAC key and race filesystem publication; root/container administrators can +also read private data. HMAC fingerprints protect stored secret values from +unkeyed guessing, not from an actor with access to the key. + +`managed resource-status` is read-only and reports BLOCKED or STALE explicitly +(a check overdue by >65s, unknown state age >65s, or changed configuration). +Idle reloads do not write a disk heartbeat: only configured state changes, +BLOCKED transitions and completed checks are flushed. Cancelled checks join +their children but do not change prior health, counts or completion/due times. +It reads the config path from +`ZAPRET_AUTOTUNE_RESOURCE_CONFIG`; use that environment setting also for manual +monitor/status invocations. Status rows report only configured-check health. + +TERM cancels/joins transfers before finalizing a completed snapshot and exits +0 on safe cleanup. PID 1 retains DNS through monitor/SS cleanup. In generic +mode transient child restart-budget exhaustion pauses until the rolling window +opens, rather than terminating the container. Unknown/unsafe cleanup stays +BLOCKED pending operator repair/restart; no unsafe daemon sourcing is allowed. +Generic cleanup shares an 80-second budget, including watchdog setup and +command kill escalation, with bounded final reaping; it is designed to fit +within a 90-second Docker stop limit (the supplied Compose grants 240 seconds). +These changes do not claim to fix legacy YouTube exit 78. diff --git a/cmd/zapret-autotune/main.go b/cmd/zapret-autotune/main.go index 2c0e5db..3ce759c 100644 --- a/cmd/zapret-autotune/main.go +++ b/cmd/zapret-autotune/main.go @@ -21,6 +21,10 @@ Commands: health youtube One-shot Player API health through configured loopback SOCKS managed init|reconcile|status|apply|monitor Guarded activation/monitoring in the dedicated managed-test target only + managed monitor --resource-config FILE + Periodic configured checks only (no recovery/apply) + managed resource-status + Read-only configured-check snapshot (explicit stale/blocked) check youtube Refuse checks until probe URLs/video IDs are configured and agreed check --config FILE Check configured HTTPS resources (checks only; no strategy selection) @@ -48,7 +52,18 @@ func runWithPlayer(args []string, stdout, stderr io.Writer, player playerRunner) return dnsForwarderCLI(args[1:], stdout, stderr) } if len(args) == 1 && args[0] == "target-config-check" { - if _, err := parseManagedHealthConfig(os.LookupEnv); err != nil { + path, _ := os.LookupEnv("ZAPRET_AUTOTUNE_RESOURCE_CONFIG") + if path != "" { + port, err := managedTargetSettings(os.LookupEnv) + if err != nil || port != 1080 { + fmt.Fprintln(stderr, "configerror: invalid generic target settings") + return 64 + } + if _, _, err := resourceStaticConfig(os.LookupEnv); err != nil { + fmt.Fprintln(stderr, "configerror: invalid generic DNS/proxy configuration") + return 64 + } + } else if _, err := parseManagedHealthConfig(os.LookupEnv); err != nil { fmt.Fprintln(stderr, "configerror: invalid target health/service configuration") return 64 } diff --git a/cmd/zapret-autotune/managed_cli.go b/cmd/zapret-autotune/managed_cli.go index 9f57121..5e92146 100644 --- a/cmd/zapret-autotune/managed_cli.go +++ b/cmd/zapret-autotune/managed_cli.go @@ -379,25 +379,9 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB, } func managedRuntimeGuard(d managedDeps) (int, error) { - role, _ := d.lookup("ZAPRET_AUTOTUNE_TARGET_ROLE") - if role != "managed-test" { - return 2, fmt.Errorf("ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test") - } - for key, expected := range map[string]string{ - "ZAPRET_AUTOTUNE_TARGET_ID": managedTargetID, - "ZAPRET_AUTOTUNE_TARGET_DIR": managedTargetDirectory, - "ZAPRET_CONFIG": managedTargetConfig, - } { - if value, _ := d.lookup(key); value != expected { - return 2, fmt.Errorf("%s does not match the fresh managed target contract", key) - } - } - portText, _ := d.lookup("SOCKS_PORT") - port, portErr := strconv.Atoi(portText) - proxy, _ := d.lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY") - proxy, proxyErr := managedProbeProxy(proxy) - if portErr != nil || port < 1 || port > 65535 || strconv.Itoa(port) != portText || proxyErr != nil || proxy != "socks5h://127.0.0.1:"+portText { - return 2, fmt.Errorf("managed SOCKS proxy must match the target SOCKS_PORT listener") + port, err := managedTargetSettings(d.lookup) + if err != nil { + return 2, err } if d.verifyRuntime == nil || d.verifySOCKS == nil { return 1, fmt.Errorf("managed runtime validators unavailable") @@ -411,7 +395,34 @@ func managedRuntimeGuard(d managedDeps) (int, error) { return 0, nil } +func managedTargetSettings(lookup func(string) (string, bool)) (int, error) { + role, _ := lookup("ZAPRET_AUTOTUNE_TARGET_ROLE") + if role != "managed-test" { + return 2, fmt.Errorf("ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test") + } + for key, expected := range map[string]string{ + "ZAPRET_AUTOTUNE_TARGET_ID": managedTargetID, + "ZAPRET_AUTOTUNE_TARGET_DIR": managedTargetDirectory, + "ZAPRET_CONFIG": managedTargetConfig, + } { + if value, _ := lookup(key); value != expected { + return 2, fmt.Errorf("%s does not match the fresh managed target contract", key) + } + } + portText, _ := lookup("SOCKS_PORT") + port, portErr := strconv.Atoi(portText) + proxy, _ := lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY") + proxy, proxyErr := managedProbeProxy(proxy) + if portErr != nil || port < 1 || port > 65535 || strconv.Itoa(port) != portText || proxyErr != nil || proxy != "socks5h://127.0.0.1:"+portText { + return 2, fmt.Errorf("managed SOCKS proxy must match the target SOCKS_PORT listener") + } + return port, nil +} + func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int { + if len(args) == 1 && args[0] == "resource-status" { + return resourceStatusCLI(stdout, d) + } if len(args) > 0 && args[0] == "monitor" { return managedMonitorCLI(args[1:], stdout, stderr, d) } diff --git a/cmd/zapret-autotune/managed_monitor_cli.go b/cmd/zapret-autotune/managed_monitor_cli.go index 48b777b..38cfebf 100644 --- a/cmd/zapret-autotune/managed_monitor_cli.go +++ b/cmd/zapret-autotune/managed_monitor_cli.go @@ -57,6 +57,17 @@ func managedMonitorCLI(args []string, stdout, stderr io.Writer, target managedDe } func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) (code int) { + resourcePath, _ := target.lookup("ZAPRET_AUTOTUNE_RESOURCE_CONFIG") + if len(args) == 2 && args[0] == "--resource-config" && args[1] != "" { + return resourceMonitorCLI(ctx, args[1], stdout, stderr, target) + } + if resourcePath != "" { + if len(args) != 0 { + fmt.Fprintln(stderr, "configerror: generic monitor accepts only --resource-config FILE") + return 64 + } + return resourceMonitorCLI(ctx, resourcePath, stdout, stderr, target) + } cfg, err := managedMonitorConfig(args, target.lookup) if err != nil { fmt.Fprintln(stderr, "error: invalid managed monitor configuration") diff --git a/cmd/zapret-autotune/managed_monitor_cli_test.go b/cmd/zapret-autotune/managed_monitor_cli_test.go index ff0fba2..2a6c1d0 100644 --- a/cmd/zapret-autotune/managed_monitor_cli_test.go +++ b/cmd/zapret-autotune/managed_monitor_cli_test.go @@ -702,7 +702,11 @@ func TestManagedEntrypointSupervision(t *testing.T) { text = shortened } if mode == "stop-hang" { - text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1) + shortened := strings.Replace(text, "cleanup_command 30 "+zapret+" stop", "timeout -k 1 1 "+zapret+" stop", 1) + if shortened == text { + t.Fatal("fixture did not shorten stop watchdog") + } + text = shortened } entry := write("entrypoint", text) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/cmd/zapret-autotune/resource_check.go b/cmd/zapret-autotune/resource_check.go index 69a3266..c8678e7 100644 --- a/cmd/zapret-autotune/resource_check.go +++ b/cmd/zapret-autotune/resource_check.go @@ -14,6 +14,7 @@ import ( "os" "os/signal" "regexp" + "sort" "strings" "syscall" "time" @@ -42,6 +43,8 @@ type resourceCheckSpec struct { Method string `json:"method"` TLS string `json:"tls"` TimeoutMS int `json:"timeout_ms"` + IntervalMS *int `json:"interval_ms,omitempty"` + CooldownMS *int `json:"cooldown_ms,omitempty"` Expect resourceCheckExpect `json:"expect"` HeadersEnv map[string]string `json:"headers_env,omitempty"` AllowPrivate bool `json:"allow_private,omitempty"` @@ -139,6 +142,18 @@ func parseResourceChecks(data []byte, lookup func(string) (string, bool)) (resou return fail() } ids[r.ID] = true + for _, scheduling := range []struct { + value **int + fallback int + }{{&r.IntervalMS, 300000}, {&r.CooldownMS, 30000}} { + if *scheduling.value == nil { + v := scheduling.fallback + *scheduling.value = &v + } + if **scheduling.value < 100 || **scheduling.value > 86400000 { + return fail() + } + } if (r.URL == "") == (r.URLEnv == "") { return fail() } @@ -202,7 +217,13 @@ func parseResourceChecks(data []byte, lookup func(string) (string, bool)) (resou return fail() } headers := map[string]bool{} - for name, env := range r.HeadersEnv { + names := make([]string, 0, len(r.HeadersEnv)) + for name := range r.HeadersEnv { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + env := r.HeadersEnv[name] lower := strings.ToLower(name) if !resourceHeader.MatchString(name) || headers[lower] || !resourceEnv.MatchString(env) { return fail() @@ -234,6 +255,10 @@ func parseResourceChecks(data []byte, lookup func(string) (string, bool)) (resou } func readResourceChecks(path string, lookup func(string) (string, bool)) (resourceCheckConfig, error) { + return readResourceChecksMode(path, lookup, false) +} + +func readResourceChecksMode(path string, lookup func(string) (string, bool), owned bool) (resourceCheckConfig, error) { fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK|syscall.O_CLOEXEC, 0) if err != nil { return resourceCheckConfig{}, errResourceConfig @@ -244,10 +269,22 @@ func readResourceChecks(path string, lookup func(string) (string, bool)) (resour if err != nil || !info.Mode().IsRegular() || info.Size() > 1<<20 { return resourceCheckConfig{}, errResourceConfig } + if owned { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok || (st.Uid != 0 && st.Uid != uint32(os.Geteuid())) || st.Nlink != 1 || info.Mode().Perm()&0022 != 0 { + return resourceCheckConfig{}, errResourceConfig + } + } data, err := io.ReadAll(io.LimitReader(f, (1<<20)+1)) if err != nil { return resourceCheckConfig{}, errResourceConfig } + if owned { + current, err := os.Lstat(path) + if err != nil || !os.SameFile(info, current) { + return resourceCheckConfig{}, errResourceConfig + } + } return parseResourceChecks(data, lookup) } @@ -329,6 +366,52 @@ type resourceCheckResult struct { Status int `json:"status"` } +// A synchronous runner: subprocess cancellation joins before this returns. +// The guard is also called after DNS, immediately before the pinned transfer. +func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpec, protocol, server, proxy string, d resourceCheckDeps) (resourceCheckResult, error) { + result := resourceCheckResult{Service: service, Resource: r.ID, Group: r.Group, TLS: protocol, Outcome: "dns_failed"} + if err := d.guard(); err != nil { + return result, err + } + probeCtx, cancel := context.WithTimeout(ctx, time.Duration(r.TimeoutMS)*time.Millisecond) + defer cancel() + ips, err := d.resolve(probeCtx, server, r.host) + if err == nil && len(ips) > 0 && len(ips) <= 64 { + approved := true + for _, ip := range ips { + if !approvedResourceIP(r, ip) { + approved = false + } + } + result.Outcome = "destination_denied" + if approved { + if err := d.guard(); err != nil { + return result, err + } + pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443") + options := resourceTransferOptions{generic: true, method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n"} + transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, false, d.runner, options) + result.Outcome = "transfer_failed" + if err == nil { + result.Bytes, result.TimeMS, result.Status = len(transfer.Body), transfer.LatencyMS, transfer.Status + result.Outcome = "expectation_failed" + for _, status := range r.Expect.Statuses { + if status == transfer.Status && len(transfer.Body) >= r.Expect.MinBytes && (r.Expect.ContentType == "" || transfer.ContentType == r.Expect.ContentType) { + result.Outcome = "pass" + } + } + } + } + } + if probeCtx.Err() != nil { + result.Outcome = "timeout" + if ctx.Err() != nil { + result.Outcome = "cancelled" + } + } + return result, nil +} + func resourceCheckCLI(ctx context.Context, path string, stdout, stderr io.Writer, d resourceCheckDeps) int { if d.lookup == nil { fmt.Fprintln(stderr, "configerror: invalid resource check configuration") @@ -357,57 +440,11 @@ func resourceCheckCLI(ctx context.Context, path string, stdout, stderr io.Writer if ctx.Err() != nil { return 1 } - if d.guard() != nil { + result, err := runResourceProtocol(ctx, s.ID, r, protocol, server, proxy, d) + if err != nil { fmt.Fprintln(stderr, "blocked: resource check ownership unavailable") return 2 } - result := resourceCheckResult{Service: s.ID, Resource: r.ID, Group: r.Group, TLS: protocol, Outcome: "dns_failed"} - probeCtx, cancel := context.WithTimeout(ctx, time.Duration(r.TimeoutMS)*time.Millisecond) - ips, err := d.resolve(probeCtx, server, r.host) - if err == nil && len(ips) > 0 && len(ips) <= 64 { - approved := true - for _, ip := range ips { - if !approvedResourceIP(r, ip) { - approved = false - } - } - result.Outcome = "destination_denied" - if approved { - if d.guard() != nil { - cancel() - fmt.Fprintln(stderr, "blocked: resource check ownership unavailable") - return 2 - } - // --connect-to changes the SOCKS CONNECT destination even for socks5h; - // URL authority, HTTP Host, SNI and certificate verification stay original. - pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443") - options := resourceTransferOptions{generic: true, method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n"} - transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, false, d.runner, options) - result.Outcome = "transfer_failed" - if err == nil { - result.Bytes, result.TimeMS, result.Status = len(transfer.Body), transfer.LatencyMS, transfer.Status - result.Outcome = "expectation_failed" - statusOK := false - for _, status := range r.Expect.Statuses { - if status == transfer.Status { - statusOK = true - } - } - if statusOK { - if len(transfer.Body) >= r.Expect.MinBytes && (r.Expect.ContentType == "" || transfer.ContentType == r.Expect.ContentType) { - result.Outcome = "pass" - } - } - } - } - } - if probeCtx.Err() != nil { - result.Outcome = "timeout" - if ctx.Err() != nil { - result.Outcome = "cancelled" - } - } - cancel() if result.Outcome != "pass" { code = 1 } diff --git a/cmd/zapret-autotune/resource_monitor.go b/cmd/zapret-autotune/resource_monitor.go new file mode 100644 index 0000000..f4e0006 --- /dev/null +++ b/cmd/zapret-autotune/resource_monitor.go @@ -0,0 +1,441 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "reflect" + "syscall" + "time" +) + +// HEALTHY means only that the configured checks passed, not production READY. +type resourceMonitorSource struct { + config resourceCheckConfig + server, proxy, fingerprint string +} + +func resourceStaticConfig(lookup func(string) (string, bool)) (string, string, error) { + dns, err := parseManagedDNSConfig(lookup) + raw, _ := lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY") + proxy, proxyErr := managedProbeProxy(raw) + if err != nil || proxyErr != nil || dns.Mode == "native" && dns.NativeServer == "" { + return "", "", errResourceConfig + } + server := dns.serverAddress() + if dns.Mode == "native" { + server = net.JoinHostPort(server, "53") + } + return server, proxy, nil +} +func readResourceMonitorSource(path string, lookup func(string) (string, bool), key []byte) (resourceMonitorSource, error) { + // Resolve each referenced environment value once in this immutable snapshot. + values := map[string]struct { + value string + ok bool + }{} + snapshot := func(name string) (string, bool) { + if v, ok := values[name]; ok { + return v.value, v.ok + } + v, ok := lookup(name) + if len(v) > 16384 { + return "", false + } + values[name] = struct { + value string + ok bool + }{v, ok} + return v, ok + } + cfg, err := readResourceChecksMode(path, snapshot, true) + server, proxy, staticErr := resourceStaticConfig(snapshot) + if err != nil || staticErr != nil { + return resourceMonitorSource{}, errResourceConfig + } + source := resourceMonitorSource{config: cfg, server: server, proxy: proxy} + b, _ := json.Marshal(cfg) + // Marshal omits the private curl headers; bind them explicitly as well. + parts := []string{string(b), server, proxy} + for _, s := range cfg.Services { + for _, r := range s.Resources { + parts = append(parts, r.privateConfig) + } + } + b, _ = json.Marshal(parts) + source.fingerprint = resourceMAC(key, b) + return source, nil +} +func resourceRowFingerprint(key []byte, source resourceMonitorSource, r resourceCheckSpec) string { + b, _ := json.Marshal(r) + b, _ = json.Marshal([]string{string(b), r.privateConfig, source.server, source.proxy}) + return resourceMAC(key, b) +} +func reconcileResourceRows(key []byte, source resourceMonitorSource, prior []resourceMonitorRow) []resourceMonitorRow { + old := map[string]resourceMonitorRow{} + for _, r := range prior { + old[r.Service+"/"+r.Resource] = r + } + rows := []resourceMonitorRow{} + for _, s := range source.config.Services { + for _, r := range s.Resources { + fp := resourceRowFingerprint(key, source, r) + row, ok := old[s.ID+"/"+r.ID] + if !ok || row.Fingerprint != fp { + row = resourceMonitorRow{Service: s.ID, Resource: r.ID, Fingerprint: fp, Health: "UNKNOWN", Outcome: "unknown"} + } + rows = append(rows, row) + } + } + return rows +} + +type resourceMonitorDeps struct { + load func() (resourceMonitorState, error) + save func(resourceMonitorState) error + source func() (resourceMonitorSource, error) + guard func() error + probe func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error) + key []byte + now func() time.Time + wait func(context.Context, time.Duration) error + report func(bool) +} + +// Sequential round robin with independent due times; reload at least every +// second while idle/blocked. No recovery leases or strategy mutations. +func runResourceMonitor(ctx context.Context, d resourceMonitorDeps) error { + cursor := 0 + persistenceBlocked, failedMAC, attemptedMAC := false, "", "" + var shutdownError error + for ctx.Err() == nil { + state, stateErr := d.load() + if persistenceBlocked && stateErr == nil && state.MAC != failedMAC && state.MAC != attemptedMAC { + persistenceBlocked = false + } + source, sourceErr := d.source() + blocked := persistenceBlocked || stateErr != nil || sourceErr != nil || d.guard() != nil + dirty := false + if !blocked { + rows := reconcileResourceRows(d.key, source, state.Rows) + dirty = !reflect.DeepEqual(rows, state.Rows) + state.Rows = rows + if len(state.Rows) > 0 { + cursor %= len(state.Rows) + for n := 0; n < len(state.Rows); n++ { + i := (cursor + n) % len(state.Rows) + row := &state.Rows[i] + if row.NextDue > d.now().UnixMilli() { + continue + } + var spec resourceCheckSpec + for _, s := range source.config.Services { + if s.ID == row.Service { + for _, r := range s.Resources { + if r.ID == row.Resource { + spec = r + } + } + } + } + guard := func() error { + if d.guard() != nil { + return errResourceConfig + } + fresh, err := d.source() + if err != nil || fresh.fingerprint != source.fingerprint { + return errResourceConfig + } + return nil + } + outcome, err := d.probe(ctx, row.Service, spec, source, guard) + cursor = (i + 1) % len(state.Rows) + if err != nil { + if ctx.Err() == nil || !managedCancellationOnly(err, ctx.Err()) { + blocked = true + if ctx.Err() != nil { + shutdownError = err + } + } + break + } + // Cancellation joins the probe but is not completed evidence. + // Keep the previous health, failure count and due time intact. + if ctx.Err() != nil || outcome == "cancelled" { + break + } + dirty = true + row.Outcome, row.LastCompleted = outcome, d.now().UnixMilli() + delay := *spec.CooldownMS + if outcome == "pass" { + row.Health, row.Failures, delay = "HEALTHY", 0, *spec.IntervalMS + } else { + row.Health = "DEGRADED" + if row.Failures < ^uint64(0) { + row.Failures++ + } + } + row.NextDue = row.LastCompleted + int64(delay) + break // persist each completed (and joined) check before another probe + } + } + } + dirty = dirty || state.Blocked != blocked + state.Blocked = blocked + if dirty { + state.Updated = d.now().UnixMilli() + } + if dirty && stateErr == nil && !persistenceBlocked { + if d.save(state) != nil { + blocked, persistenceBlocked, failedMAC, attemptedMAC = true, true, state.MAC, resourceStateMAC(d.key, state) + } + } + d.report(blocked) + if ctx.Err() != nil { + if persistenceBlocked { + return errors.Join(shutdownError, errResourceState) + } + return shutdownError + } + // Bound reloads, but don't delay an already due sibling by a cooldown. + delay := time.Second + if !blocked { + for _, r := range state.Rows { + milliseconds := r.NextDue - d.now().UnixMilli() + if milliseconds > 1000 { + milliseconds = 1000 + } + remaining := time.Duration(milliseconds) * time.Millisecond + if remaining < 0 { + remaining = 0 + } + if remaining < delay { + delay = remaining + } + } + } + if err := d.wait(ctx, delay); err != nil { + if ctx.Err() != nil { + if persistenceBlocked { + return errResourceState + } + return nil + } + return err + } + } + if persistenceBlocked { + return errResourceState + } + return nil +} + +func probeResourceMonitor(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error, check resourceCheckDeps) (string, error) { + check.guard = guard + protocols := []string{spec.TLS} + if spec.TLS == "both" { + protocols = []string{"tls12", "tls13"} + } + outcome := "pass" + for _, protocol := range protocols { + if ctx.Err() != nil { + return "cancelled", nil + } + result, err := runResourceProtocol(ctx, service, spec, protocol, source.server, source.proxy, check) + if err != nil { + return "", err + } + if result.Outcome != "pass" { + outcome = result.Outcome + } + } + return outcome, nil +} + +func resourceMonitorCLI(ctx context.Context, path string, stdout, stderr io.Writer, target managedDeps) (code int) { + return resourceMonitorCLIWithCheck(ctx, path, stdout, stderr, target, productionResourceCheckDeps()) +} + +func resourceMonitorCLIWithCheck(ctx context.Context, path string, stdout, stderr io.Writer, target managedDeps, check resourceCheckDeps) (code int) { + port, settingsErr := managedTargetSettings(target.lookup) + if settingsErr != nil || port != 1080 { + fmt.Fprintln(stderr, "configerror: invalid generic target settings") + return 64 + } + if _, _, err := resourceStaticConfig(target.lookup); err != nil { + fmt.Fprintln(stderr, "configerror: invalid generic DNS/proxy configuration") + return 64 + } + var lock *managedMonitorSingleton + var store *resourceMonitorStore + defer func() { + if store != nil && store.close() != nil { + code = 78 + } + if lock != nil && lock.close() != nil { + code = 78 + } + }() + lastBlocked := false + report := func(blocked bool) { + if blocked && !lastBlocked { + fmt.Fprintln(stderr, "BLOCKED: generic resource monitor guard/state unavailable") + } + lastBlocked = blocked + } + fmt.Fprintln(stdout, "generic resource monitor: configured checks only; no recovery/apply; HEALTHY is not production READY") + // Guard failures before store/lock acquisition remain cancellable and alive. + for ctx.Err() == nil { + if lock == nil { + var err error + lock, err = acquireManagedMonitorSingleton(target) + if err != nil { + if errors.Is(err, syscall.EWOULDBLOCK) { + fmt.Fprintln(stderr, "BLOCKED: monitor singleton already held") + return 78 + } + report(true) + if waitManagedMonitor(ctx, time.Second) != nil { + return 0 + } + continue + } + } + if store == nil { + var err error + store, err = openResourceMonitorStore(target.directory, true) + if err != nil { + report(true) + if waitManagedMonitor(ctx, time.Second) != nil { + return 0 + } + continue + } + } + err := runResourceMonitor(ctx, resourceMonitorDeps{ + load: store.load, save: store.save, key: store.key, + source: func() (resourceMonitorSource, error) { + return readResourceMonitorSource(path, target.lookup, store.key) + }, + guard: func() error { + if resourceSingletonGuard(lock, target.directory) != nil { + return errResourceState + } + // Probe guards validate the held snapshot, never reload/adopt a new + // authenticated operator edit while an older check is in flight. + if store.snapshotGuard() != nil { + return errResourceState + } + if resourceManagedIdentity(target.directory) != nil { + return errResourceState + } + return check.guard() + }, + probe: func(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error) (string, error) { + return probeResourceMonitor(ctx, service, spec, source, guard, check) + }, + now: time.Now, wait: waitManagedMonitor, report: report, + }) + if err != nil { + fmt.Fprintln(stderr, "BLOCKED: generic monitor safety/finalization failure") + return 78 + } + } + return 0 +} + +func resourceSingletonGuard(lock *managedMonitorSingleton, dir string) error { + path := filepath.Join(dir, managedMonitorLockName) + if managedPrivateFile(path) != nil { + return errResourceState + } + current, err := os.Lstat(path) + held, heldErr := lock.file.Stat() + if err != nil || heldErr != nil || !os.SameFile(current, held) { + return errResourceState + } + return nil +} + +func resourceStatusCLI(stdout io.Writer, target managedDeps) int { + blocked := func() int { fmt.Fprintln(stdout, `{"health":"BLOCKED","stale":true}`); return 1 } + store, err := openResourceMonitorStore(target.directory, false) + if err != nil { + return blocked() + } + defer store.close() + state, err := store.load() + if err != nil { + return blocked() + } + path, _ := target.lookup("ZAPRET_AUTOTUNE_RESOURCE_CONFIG") + source, err := readResourceMonitorSource(path, target.lookup, store.key) + check := productionResourceCheckDeps() + if err != nil || check.guard() != nil || resourceManagedIdentity(target.directory) != nil { + return blocked() + } + rows := reconcileResourceRows(store.key, source, state.Rows) + b1, _ := json.Marshal(rows) + b2, _ := json.Marshal(state.Rows) + stale := resourceSnapshotStale(state, time.Now().UnixMilli()) || string(b1) != string(b2) + health := "CHECKS_ONLY" + if stale { + health = "STALE" + } + if state.Blocked { + health = "BLOCKED" + } + if json.NewEncoder(stdout).Encode(struct { + Health string `json:"health"` + Stale bool `json:"stale"` + State resourceMonitorState `json:"state"` + }{health, stale, state}) != nil { + return 1 + } + if stale || state.Blocked { + return 1 + } + return 0 +} + +// Updated is a semantic-change timestamp, not a disk heartbeat. Evidence is +// stale only once its next scheduled check plus the maximum run budget is past. +func resourceSnapshotStale(state resourceMonitorState, now int64) bool { + if state.Updated > now { + return true + } + if len(state.Rows) == 0 { + return now-state.Updated > 65000 + } + for _, row := range state.Rows { + if row.NextDue == 0 { + if now-state.Updated > 65000 { + return true + } + } else if now > row.NextDue && now-row.NextDue > 65000 { + return true + } + } + return false +} + +func resourceManagedIdentity(dir string) error { + m := &managedTransaction{Dir: dir, Name: "config"} + if managedCheckIdentity(m) != nil { + return errResourceState + } + j, err := m.load() + if err != nil || j.State != "committed" { + return errResourceState + } + b, err := m.read("config") + if err != nil || managedHash(b) != j.Hash { + return errResourceState + } + return nil +} diff --git a/cmd/zapret-autotune/resource_monitor_boundary_test.go b/cmd/zapret-autotune/resource_monitor_boundary_test.go new file mode 100644 index 0000000..31e5c21 --- /dev/null +++ b/cmd/zapret-autotune/resource_monitor_boundary_test.go @@ -0,0 +1,326 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "os" + "path/filepath" + "reflect" + "testing" + "time" +) + +func resourceCompletedFixture(t *testing.T) (resourceMonitorSource, []byte, resourceMonitorState) { + t.Helper() + source, key := resourceMonitorFixture(t) + state := resourceMonitorState{Version: 1, Updated: 5000, Rows: reconcileResourceRows(key, source, nil)} + for i := range state.Rows { + state.Rows[i].Health, state.Rows[i].Outcome, state.Rows[i].LastCompleted = "HEALTHY", "pass", 5000 + state.Rows[i].NextDue = 9000 + } + return source, key, state +} + +func TestResourceMonitorCancelledRowsUnchangedAndJoined(t *testing.T) { + for _, mixed := range []bool{false, true} { + t.Run(map[bool]string{false: "curl-cancel", true: "cancel-and-guard"}[mixed], func(t *testing.T) { + source, key, state := resourceCompletedFixture(t) + prior := append([]resourceMonitorRow(nil), state.Rows...) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + joined, writes := false, 0 + check := resourceCheckDeps{resolve: func(context.Context, string, string) ([]net.IP, error) { + return []net.IP{net.ParseIP("93.184.216.34")}, nil + }, runner: func(ctx context.Context, _ []string, _ io.Reader, _ io.Writer, _ io.Writer) error { + cancel() + <-ctx.Done() + joined = true + return ctx.Err() + }} + d := resourceMonitorDeps{key: key, now: func() time.Time { return time.UnixMilli(10000) }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(s resourceMonitorState) error { + if !joined { + t.Error("persisted before joining curl") + } + writes++ + state = s + return nil + }, source: func() (resourceMonitorSource, error) { return source, nil }, guard: func() error { return nil }, report: func(bool) {}, wait: func(context.Context, time.Duration) error { t.Error("wait after cancellation"); return nil }} + d.probe = func(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error) (string, error) { + outcome, err := probeResourceMonitor(ctx, service, spec, source, guard, check) + if mixed { + return outcome, errors.Join(ctx.Err(), errResourceConfig) + } + return outcome, err + } + err := runResourceMonitor(ctx, d) + if mixed { + if !errors.Is(err, errResourceConfig) || !errors.Is(err, context.Canceled) || !state.Blocked || writes != 1 { + t.Fatal("lost mixed shutdown guard error", state, writes, err) + } + } else if err != nil || writes != 0 { + t.Fatal("cancel published evidence", writes, err) + } + if !joined || !reflect.DeepEqual(prior, state.Rows) { + t.Fatal("cancel changed completed rows or failed to join", state.Rows) + } + }) + } +} + +func TestResourceMonitorIdleNoPersistenceHeartbeat(t *testing.T) { + source, key, state := resourceCompletedFixture(t) + for i := range state.Rows { + state.Rows[i].NextDue = 305000 + } + now := time.UnixMilli(10000) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + writes, waits := 0, 0 + d := resourceMonitorDeps{key: key, now: func() time.Time { return now }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(s resourceMonitorState) error { writes++; state = s; return nil }, source: func() (resourceMonitorSource, error) { return source, nil }, guard: func() error { return nil }, report: func(bool) {}, probe: func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error) { + t.Error("probed resource before due") + return "pass", nil + }, wait: func(_ context.Context, d time.Duration) error { + now = now.Add(d) + waits++ + if waits == 5 { + cancel() + } + return nil + }} + if err := runResourceMonitor(ctx, d); err != nil { + t.Fatal(err) + } + if writes != 0 || state.Updated != 5000 { + t.Fatal("idle disk heartbeat", writes, state.Updated) + } + if resourceSnapshotStale(state, 100000) || !resourceSnapshotStale(state, 370001) { + t.Fatal("staleness still depends on idle heartbeat") + } +} + +func TestResourceMonitorBlockedTransitionsOnly(t *testing.T) { + source, key, state := resourceCompletedFixture(t) + for i := range state.Rows { + state.Rows[i].NextDue = 305000 + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tick, writes := 0, 0 + d := resourceMonitorDeps{key: key, now: func() time.Time { return time.UnixMilli(10000 + int64(tick)*1000) }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(s resourceMonitorState) error { writes++; state = s; return nil }, source: func() (resourceMonitorSource, error) { + if tick < 3 { + return resourceMonitorSource{}, errResourceConfig + } + return source, nil + }, guard: func() error { return nil }, report: func(bool) {}, probe: func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error) { + t.Error("unexpected probe") + return "pass", nil + }, wait: func(context.Context, time.Duration) error { + tick++ + if tick == 5 { + cancel() + } + return nil + }} + if err := runResourceMonitor(ctx, d); err != nil { + t.Fatal(err) + } + if writes != 2 || state.Blocked { + t.Fatal("blocked transition heartbeat or missing resume", writes, state) + } +} + +func resourceBoundaryStore(t *testing.T, createState bool) (*resourceMonitorStore, resourceMonitorState) { + t.Helper() + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + s, err := openResourceMonitorStore(dir, true) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.close() }) + v, err := s.load() + if err != nil { + t.Fatal(err) + } + if createState { + if err := s.save(v); err != nil { + t.Fatal(err) + } + } + return s, v +} + +func replaceResourceEntry(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.WriteFile(path+"-replacement", data, 0600); err != nil { + t.Fatal(err) + } + if err := os.Rename(path+"-replacement", path); err != nil { + t.Fatal(err) + } +} + +func TestResourceMonitorHeldEntriesRejectReplacement(t *testing.T) { + for _, name := range []string{resourceStateName, resourceKeyName} { + for _, point := range []string{"before-load", "loaded", "staged", "publish"} { + t.Run(name+"/"+point, func(t *testing.T) { + s, v := resourceBoundaryStore(t, true) + path := filepath.Join(s.path, name) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var replacement os.FileInfo + mutate := func() { + replaceResourceEntry(t, path, data) + replacement, err = os.Stat(path) + if err != nil { + t.Fatal(err) + } + } + if point == "before-load" { + mutate() + } else { + s.fault = func(p, _ string) { + if p == point { + mutate() + } + } + } + if point == "before-load" || point == "loaded" { + if _, err := s.load(); err == nil { + t.Fatal("load adopted identical replaced inode") + } + } + if err := s.save(v); err == nil { + t.Fatal("save clobbered identical replacement") + } + current, err := os.Stat(path) + if err != nil || replacement == nil || !os.SameFile(current, replacement) { + t.Fatal("foreign inode changed", err) + } + got, err := os.ReadFile(path) + if err != nil || !bytes.Equal(got, data) { + t.Fatal("foreign bytes changed", err) + } + }) + } + } +} + +func TestResourceMonitorMidStageCorruptionPreserved(t *testing.T) { + for _, name := range []string{resourceStateName, resourceKeyName, "stage", "stage-in-place"} { + t.Run(name, func(t *testing.T) { + s, v := resourceBoundaryStore(t, true) + bad := []byte("untrusted corrupt operator bytes") + path := "" + s.fault = func(point, stage string) { + if point != "staged" { + return + } + path = filepath.Join(s.path, name) + if name == "stage" || name == "stage-in-place" { + path = stage + } + if name == "stage-in-place" { + if err := os.WriteFile(path, bad, 0600); err != nil { + t.Fatal(err) + } + } else { + replaceResourceEntry(t, path, bad) + } + } + if err := s.save(v); err == nil { + t.Fatal("corruption overwritten") + } + got, err := os.ReadFile(path) + if err != nil || !bytes.Equal(got, bad) { + t.Fatal("unknown stage/file removed or clobbered", err) + } + }) + } +} + +func TestResourceMonitorInitialPublicationExclusive(t *testing.T) { + s, v := resourceBoundaryStore(t, false) + path := filepath.Join(s.path, resourceStateName) + bad := []byte("new foreign entry created just before initial publication") + s.fault = func(point, _ string) { + if point == "publish" { + if err := os.WriteFile(path, bad, 0600); err != nil { + t.Fatal(err) + } + } + } + if err := s.save(v); err == nil { + t.Fatal("exclusive initial publish overwrote raced entry") + } + got, err := os.ReadFile(path) + if err != nil || !bytes.Equal(got, bad) { + t.Fatal("new entry clobbered", err) + } + if _, err := s.load(); err == nil { + t.Fatal("new entry silently adopted") + } +} + +func TestResourceMonitorPublicationRetainsNewAnchor(t *testing.T) { + s, v := resourceBoundaryStore(t, true) + old := s.stateFile + v.Updated = 10000 + if err := s.save(v); err != nil { + t.Fatal(err) + } + if old == s.stateFile { + t.Fatal("old inode retained instead of published inode") + } + if _, err := old.Stat(); err == nil { + t.Fatal("old anchor not closed after verified publication") + } + if s.entryMatches(resourceStateName, s.stateFile) != nil { + t.Fatal("published state not anchored") + } + if loaded, err := s.load(); err != nil || loaded.Updated != 10000 { + t.Fatal(loaded, err) + } +} + +func TestResourceMonitorOriginalIdentityRepair(t *testing.T) { + s, v := resourceBoundaryStore(t, true) + path := filepath.Join(s.path, resourceStateName) + valid, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + original, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(`{"version":99}`), 0600); err != nil { + t.Fatal(err) + } + if _, err := s.load(); err == nil { + t.Fatal("corrupt original trusted") + } + if err := s.save(v); err == nil { + t.Fatal("corrupt original overwritten") + } + if err := os.WriteFile(path, valid, 0600); err != nil { + t.Fatal(err) + } + if _, err := s.load(); err != nil { + t.Fatal("valid original-identity repair blocked", err) + } + current, err := os.Stat(path) + if err != nil || !os.SameFile(current, original) { + t.Fatal("repair changed inode", err) + } + if err := s.save(v); err != nil { + t.Fatal("repaired original failed publish", err) + } +} diff --git a/cmd/zapret-autotune/resource_monitor_publication_test.go b/cmd/zapret-autotune/resource_monitor_publication_test.go new file mode 100644 index 0000000..cd7abf2 --- /dev/null +++ b/cmd/zapret-autotune/resource_monitor_publication_test.go @@ -0,0 +1,159 @@ +package main + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func TestResourceMonitorFinalPublicationStageGuard(t *testing.T) { + for _, existing := range []bool{false, true} { + for _, mutation := range []string{"byte", "truncate", "append", "growth", "path-replace", "key-replace", "key-in-place", "state-replace"} { + name := "initial/" + if existing { + name = "existing/" + } + t.Run(name+mutation, func(t *testing.T) { + s, v := resourceBoundaryStore(t, existing) + path := filepath.Join(s.path, resourceStateName) + oldBytes, oldErr := os.ReadFile(path) + oldInfo, _ := os.Stat(path) + if existing && oldErr != nil { + t.Fatal(oldErr) + } + v.Updated = 10000 + var changedPath string + var changedBytes []byte + var changedInfo os.FileInfo + s.fault = func(point, stage string) { + if point != "publish" { + return + } + data, err := os.ReadFile(stage) + if err != nil { + t.Fatal(err) + } + changedPath = stage + switch mutation { + case "byte": + data[0] = '[' + case "truncate": + data = data[:len(data)-1] + case "append": + data = append(data, ' ') + case "growth": + data = append(data, bytes.Repeat([]byte{'x'}, 65537)...) + case "key-replace", "key-in-place": + changedPath = filepath.Join(s.path, resourceKeyName) + data, err = os.ReadFile(changedPath) + if err != nil { + t.Fatal(err) + } + if mutation == "key-in-place" { + data[0] ^= 1 + } + case "state-replace": + changedPath = path + if existing { + data = append([]byte(nil), oldBytes...) + } + } + changedBytes = append([]byte(nil), data...) + if mutation == "path-replace" || mutation == "key-replace" || mutation == "state-replace" { + replaceResourceEntry(t, changedPath, data) + } else if err := os.WriteFile(changedPath, data, 0600); err != nil { + t.Fatal(err) + } + changedInfo, err = os.Stat(changedPath) + if err != nil { + t.Fatal(err) + } + } + if err := s.save(v); !errors.Is(err, errResourceState) { + t.Fatal("unsafe stage published", err) + } + current, err := os.ReadFile(changedPath) + info, infoErr := os.Stat(changedPath) + if err != nil || infoErr != nil || !bytes.Equal(current, changedBytes) || !os.SameFile(info, changedInfo) { + t.Fatal("unknown artifact changed/deleted", err, infoErr) + } + if mutation != "state-replace" { + if existing { + current, err := os.ReadFile(path) + info, infoErr := os.Stat(path) + if err != nil || infoErr != nil || !bytes.Equal(current, oldBytes) || !os.SameFile(info, oldInfo) { + t.Fatal("trusted old state lost before final guard", err, infoErr) + } + } else if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatal("initial corrupt state published", err) + } + } + if existing && s.entryMatches(resourceStateName, s.stateFile) != nil && mutation != "state-replace" { + t.Fatal("old trusted anchor lost") + } + }) + } + } +} + +func TestResourceMonitorPublicationFailureBlocksFurtherProbes(t *testing.T) { + source, _, _ := resourceCompletedFixture(t) + s, _ := resourceBoundaryStore(t, true) + state, err := s.load() + if err != nil { + t.Fatal(err) + } + state.Updated = 5000 + state.Rows = reconcileResourceRows(s.key, source, nil) + for i := range state.Rows { + state.Rows[i].Health, state.Rows[i].Outcome = "HEALTHY", "pass" + state.Rows[i].LastCompleted, state.Rows[i].NextDue = 5000, 9000 + } + if err := s.save(state); err != nil { + t.Fatal(err) + } + path := filepath.Join(s.path, resourceStateName) + old, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + oldInfo, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + s.fault = func(point, stage string) { + if point == "publish" { + if err := os.WriteFile(stage, []byte("unknown stage corruption"), 0600); err != nil { + t.Fatal(err) + } + } + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + probes, waits, blocked := 0, 0, false + d := resourceMonitorDeps{key: s.key, now: func() time.Time { return time.UnixMilli(10000) }, load: s.load, save: s.save, source: func() (resourceMonitorSource, error) { return source, nil }, guard: s.snapshotGuard, report: func(v bool) { blocked = v }, probe: func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error) { + probes++ + return "pass", nil + }, wait: func(context.Context, time.Duration) error { + waits++ + if waits == 3 { + cancel() + } + return nil + }} + if err := runResourceMonitor(ctx, d); !errors.Is(err, errResourceState) { + t.Fatal("lost unsafe publication failure", err) + } + if probes != 1 || !blocked { + t.Fatal("continued probing after failed publication", probes, blocked) + } + current, err := os.ReadFile(path) + currentInfo, infoErr := os.Stat(path) + if err != nil || infoErr != nil || !bytes.Equal(current, old) || !os.SameFile(currentInfo, oldInfo) { + t.Fatal("old completed snapshot lost", err, infoErr) + } +} diff --git a/cmd/zapret-autotune/resource_monitor_store.go b/cmd/zapret-autotune/resource_monitor_store.go new file mode 100644 index 0000000..ca8ac3f --- /dev/null +++ b/cmd/zapret-autotune/resource_monitor_store.go @@ -0,0 +1,402 @@ +package main + +import ( + "bytes" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strconv" + "syscall" +) + +const resourceStateName = ".resource-monitor-state.json" +const resourceKeyName = ".resource-monitor-key" + +var errResourceState = errors.New("resource state blocked") + +type resourceMonitorRow struct { + Service string `json:"service"` + Resource string `json:"resource"` + Fingerprint string `json:"fingerprint"` + Health string `json:"health"` + Outcome string `json:"outcome"` + Failures uint64 `json:"failures"` + LastCompleted int64 `json:"last_completed_ms"` + NextDue int64 `json:"next_due_ms"` +} +type resourceMonitorState struct { + Version int `json:"version"` + Updated int64 `json:"updated_ms"` + Blocked bool `json:"blocked"` + Rows []resourceMonitorRow `json:"rows"` + MAC string `json:"mac"` +} +type resourceMonitorStore struct { + dir *os.File + path string + key []byte + previous []byte + existed bool + keyFile, stateFile *os.File + // Deterministic filesystem interleavings in tests; nil in production. + fault func(point, stage string) +} + +func resourceMAC(key, data []byte) string { + m := hmac.New(sha256.New, key) + m.Write(data) + return hex.EncodeToString(m.Sum(nil)) +} +func resourceStateMAC(key []byte, s resourceMonitorState) string { + s.MAC = "" + b, _ := json.Marshal(s) + return resourceMAC(key, b) +} +func (s *resourceMonitorStore) guard() error { + if managedDirectory(s.path) != nil { + return errResourceState + } + current, err := os.Stat(s.path) + held, heldErr := s.dir.Stat() + if err != nil || heldErr != nil || !os.SameFile(current, held) { + return errResourceState + } + return nil +} +func (s *resourceMonitorStore) entryPath(name string) string { + return filepath.Join("/proc/self/fd", strconv.FormatUint(uint64(s.dir.Fd()), 10), name) +} + +func (s *resourceMonitorStore) close() error { + var errs []error + for _, f := range []*os.File{s.stateFile, s.keyFile, s.dir} { + if f != nil { + errs = append(errs, f.Close()) + } + } + s.stateFile, s.keyFile, s.dir = nil, nil, nil + return errors.Join(errs...) +} + +func resourcePrivateEntry(f *os.File) error { + info, err := f.Stat() + if err != nil { + return errResourceState + } + st, ok := info.Sys().(*syscall.Stat_t) + if !ok || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || st.Uid != uint32(os.Geteuid()) || st.Nlink != 1 || info.Size() > 65536 { + return errResourceState + } + return nil +} + +func (s *resourceMonitorStore) entryMatches(name string, held *os.File) error { + current, err := os.Lstat(s.entryPath(name)) + anchored, heldErr := held.Stat() + if err != nil || heldErr != nil || !os.SameFile(current, anchored) || resourcePrivateEntry(held) != nil { + return errResourceState + } + return nil +} + +func (s *resourceMonitorStore) openEntry(name string) (*os.File, []byte, error) { + if s.guard() != nil { + return nil, nil, errResourceState + } + fd, err := syscall.Openat(int(s.dir.Fd()), name, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK|syscall.O_CLOEXEC, 0) + if err != nil { + return nil, nil, err + } + f := os.NewFile(uintptr(fd), name) + if resourcePrivateEntry(f) != nil { + f.Close() + return nil, nil, errResourceState + } + b, err := io.ReadAll(io.LimitReader(f, 65537)) + if err != nil || len(b) > 65536 || s.entryMatches(name, f) != nil { + f.Close() + return nil, nil, errResourceState + } + return f, b, nil +} + +func (s *resourceMonitorStore) trustedRead(name string, held *os.File) ([]byte, error) { + f, b, err := s.openEntry(name) + if err != nil { + return nil, err + } + defer f.Close() + if held == nil || s.entryMatches(name, held) != nil { + return nil, errResourceState + } + readInfo, err := f.Stat() + heldInfo, heldErr := held.Stat() + if err != nil || heldErr != nil || !os.SameFile(readInfo, heldInfo) { + return nil, errResourceState + } + return b, nil +} + +// Both content and held inode identities must still match the trusted snapshot. +// In particular, an identical signed replacement is not silently adopted. +func (s *resourceMonitorStore) snapshotGuard() error { + key, err := s.trustedRead(resourceKeyName, s.keyFile) + if err != nil || !hmac.Equal(key, s.key) { + return errResourceState + } + if s.existed { + b, err := s.trustedRead(resourceStateName, s.stateFile) + if err != nil || !bytes.Equal(b, s.previous) { + return errResourceState + } + if _, err := s.decode(b); err != nil { + return errResourceState + } + } else { + f, _, err := s.openEntry(resourceStateName) + if f != nil { + f.Close() + } + if !errors.Is(err, os.ErrNotExist) { + return errResourceState + } + } + return s.guard() +} +func openResourceMonitorStore(path string, create bool) (*resourceMonitorStore, error) { + if managedDirectory(path) != nil { + return nil, errResourceState + } + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_DIRECTORY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0) + if err != nil { + return nil, errResourceState + } + s := &resourceMonitorStore{path: path, dir: os.NewFile(uintptr(fd), "resource state directory")} + fail := func() (*resourceMonitorStore, error) { s.close(); return nil, errResourceState } + stateFile, state, stateErr := s.openEntry(resourceStateName) + s.stateFile = stateFile + if stateErr != nil && !errors.Is(stateErr, os.ErrNotExist) { + return fail() + } + keyFile, key, err := s.openEntry(resourceKeyName) + s.keyFile = keyFile + if errors.Is(err, os.ErrNotExist) && errors.Is(stateErr, os.ErrNotExist) && create { + key = make([]byte, 32) + if _, err = rand.Read(key); err != nil { + return fail() + } + keyFD, err := syscall.Openat(fd, resourceKeyName, syscall.O_RDWR|syscall.O_CREAT|syscall.O_EXCL|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0600) + if err != nil { + return fail() + } + f := os.NewFile(uintptr(keyFD), "resource key") + s.keyFile = f + _, writeErr := f.Write(key) + syncErr := f.Sync() + if errors.Join(writeErr, syncErr, s.dir.Sync()) != nil { + return fail() + } + } else if err != nil { + return fail() + } + if len(key) != 32 { + return fail() + } + s.key = append([]byte(nil), key...) + s.previous, s.existed = state, stateErr == nil + if s.existed { + if _, err := s.decode(state); err != nil { + return fail() + } + } + if s.snapshotGuard() != nil { + return fail() + } + return s, nil +} +func (s *resourceMonitorStore) decode(b []byte) (resourceMonitorState, error) { + var v resourceMonitorState + d := json.NewDecoder(bytes.NewReader(b)) + if uniqueResourceJSON(d, 0) != nil { + return v, errResourceState + } + if _, err := d.Token(); err != io.EOF { + return v, errResourceState + } + d = json.NewDecoder(bytes.NewReader(b)) + d.DisallowUnknownFields() + if d.Decode(&v) != nil || v.Version != 1 || v.Updated < 0 || len(v.Rows) > 64 || !hmac.Equal([]byte(v.MAC), []byte(resourceStateMAC(s.key, v))) { + return v, errResourceState + } + seen := map[string]bool{} + services := map[string]bool{} + for _, r := range v.Rows { + id := r.Service + "/" + r.Resource + fp, err := hex.DecodeString(r.Fingerprint) + if !resourceID.MatchString(r.Service) || !resourceID.MatchString(r.Resource) || seen[id] || err != nil || len(fp) != 32 || r.LastCompleted < 0 || r.NextDue < 0 { + return v, errResourceState + } + seen[id] = true + services[r.Service] = true + if len(services) > 16 || r.LastCompleted > v.Updated || r.LastCompleted > 253402300799999 || (r.Health == "UNKNOWN" && r.NextDue != 0) || (r.Health != "UNKNOWN" && (r.NextDue < r.LastCompleted || r.NextDue-r.LastCompleted > 86400000)) { + return v, errResourceState + } + if r.Health != "UNKNOWN" && r.Health != "HEALTHY" && r.Health != "DEGRADED" { + return v, errResourceState + } + switch r.Outcome { + case "unknown", "pass", "dns_failed", "destination_denied", "transfer_failed", "expectation_failed", "timeout", "cancelled": + default: + return v, errResourceState + } + if (r.Health == "UNKNOWN" && (r.Outcome != "unknown" || r.LastCompleted != 0 || r.Failures != 0)) || (r.Health == "HEALTHY" && (r.Outcome != "pass" || r.Failures != 0)) || (r.Health == "DEGRADED" && (r.Outcome == "unknown" || r.Outcome == "pass" || r.Failures == 0)) { + return v, errResourceState + } + } + return v, nil +} +func (s *resourceMonitorStore) load() (resourceMonitorState, error) { + if s.guard() != nil { + return resourceMonitorState{}, errResourceState + } + key, err := s.trustedRead(resourceKeyName, s.keyFile) + if err != nil || !hmac.Equal(key, s.key) { + return resourceMonitorState{}, errResourceState + } + b, err := s.trustedRead(resourceStateName, s.stateFile) + if errors.Is(err, os.ErrNotExist) && !s.existed { + return resourceMonitorState{Version: 1, Rows: []resourceMonitorRow{}}, nil + } + if err != nil { + return resourceMonitorState{}, errResourceState + } + v, err := s.decode(b) + if err != nil { + return v, err + } + if s.fault != nil { + s.fault("loaded", "") + } + // Authenticated operator repair on the original held inode can be reloaded; + // replaced files remain preserved/BLOCKED until deliberately repaired. + key, err = s.trustedRead(resourceKeyName, s.keyFile) + if err != nil || !hmac.Equal(key, s.key) || s.entryMatches(resourceStateName, s.stateFile) != nil { + return v, errResourceState + } + current, err := s.trustedRead(resourceStateName, s.stateFile) + if err != nil || !bytes.Equal(current, b) { + return v, errResourceState + } + s.previous = b + return v, nil +} +func (s *resourceMonitorStore) save(v resourceMonitorState) error { + if s.snapshotGuard() != nil { + return errResourceState + } + v.MAC = resourceStateMAC(s.key, v) + b, err := json.Marshal(v) + if err != nil { + return errResourceState + } + if _, err := s.decode(b); err != nil { + return errResourceState + } + // Keep the staged inode open from creation through publication; reopening a + // path after rename must never adopt a foreign replacement as our own inode. + f, err := os.CreateTemp(s.entryPath(""), ".resource-monitor-stage-") + if err != nil { + return errResourceState + } + stage := f.Name() + stageGuard := func() bool { + if s.entryMatches(filepath.Base(stage), f) != nil { + return false + } + contents, err := io.ReadAll(io.NewSectionReader(f, 0, 65537)) + // Exact bytes bind the entire validated update, including its HMAC; + // truncation, append and in-place edits must fail before publication. + return err == nil && bytes.Equal(contents, b) && s.entryMatches(filepath.Base(stage), f) == nil + } + adopted := false + defer func() { + // Delete only the known staged inode, never an operator replacement. + info, err := os.Lstat(stage) + held, heldErr := f.Stat() + if err == nil && heldErr == nil && os.SameFile(info, held) && resourcePrivateEntry(f) == nil { + contents, readErr := io.ReadAll(io.NewSectionReader(f, 0, 65537)) + if readErr == nil && bytes.Equal(contents, b) { + os.Remove(stage) + } + } + if !adopted { + f.Close() + } + }() + if _, err := f.Write(b); err != nil { + return errResourceState + } + if f.Sync() != nil { + return errResourceState + } + if s.fault != nil { + s.fault("staged", stage) + } + if s.snapshotGuard() != nil || !stageGuard() { + return errResourceState + } + if s.fault != nil { + s.fault("publish", stage) + } + // Detect benign/operator replacements immediately before publication. An + // actively hostile same-UID actor racing between this final check and rename + // is outside the guarantee: POSIX rename has no inode compare-and-swap. + if s.existed { + if s.snapshotGuard() != nil || !stageGuard() { + return errResourceState + } + if os.Rename(stage, s.entryPath(resourceStateName)) != nil { + return errResourceState + } + } else { + // Exclusive initial publication cannot clobber a newly appeared entry. + key, err := s.trustedRead(resourceKeyName, s.keyFile) + if err != nil || !hmac.Equal(key, s.key) || s.guard() != nil || !stageGuard() { + return errResourceState + } + if os.Link(stage, s.entryPath(resourceStateName)) != nil { + return errResourceState + } + stageInfo, stageErr := os.Lstat(stage) + heldInfo, heldErr := f.Stat() + if stageErr != nil || heldErr != nil || !os.SameFile(stageInfo, heldInfo) { + return errResourceState + } + if os.Remove(stage) != nil { + return errResourceState + } + } + if s.entryMatches(resourceStateName, f) != nil { + return errResourceState + } + published, err := s.trustedRead(resourceStateName, f) + if err != nil || !bytes.Equal(published, b) { + return errResourceState + } + old := s.stateFile + s.stateFile, s.previous, s.existed, adopted = f, b, true, true + var closeErr error + if old != nil { + closeErr = old.Close() + } + if errors.Join(closeErr, s.dir.Sync()) != nil { + return errResourceState + } + return nil +} diff --git a/cmd/zapret-autotune/resource_monitor_test.go b/cmd/zapret-autotune/resource_monitor_test.go new file mode 100644 index 0000000..936cd14 --- /dev/null +++ b/cmd/zapret-autotune/resource_monitor_test.go @@ -0,0 +1,541 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "testing" + "time" +) + +func resourceMonitorFixture(t *testing.T) (resourceMonitorSource, []byte) { + t.Helper() + key := bytes.Repeat([]byte{7}, 32) + path := resourceCheckFile(t, resourceCheckExample) + s, err := readResourceMonitorSource(path, resourceCheckLookup, key) + if err != nil { + t.Fatal(err) + } + return s, key +} + +func TestResourceMonitorSchedule(t *testing.T) { + source, key := resourceMonitorFixture(t) + a, b, cooldown := 1000, 4000, 200 + source.config.Services[0].Resources[0].IntervalMS = &a + source.config.Services[0].Resources[0].CooldownMS = &cooldown + source.config.Services[1].Resources[0].IntervalMS = &b + state := resourceMonitorState{Version: 1} + now := time.UnixMilli(10000) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var visits []string + var elapsed []int64 + d := resourceMonitorDeps{key: key, now: func() time.Time { return now }, source: func() (resourceMonitorSource, error) { return source, nil }, guard: func() error { return nil }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(v resourceMonitorState) error { state = v; return nil }, report: func(bool) {}, wait: func(_ context.Context, d time.Duration) error { now = now.Add(d); return nil }} + d.probe = func(_ context.Context, service string, _ resourceCheckSpec, _ resourceMonitorSource, guard func() error) (string, error) { + if err := guard(); err != nil { + return "", err + } + visits = append(visits, service) + elapsed = append(elapsed, now.UnixMilli()-10000) + if len(visits) == 8 { + cancel() + } + if len(visits) == 1 { + return "timeout", nil + } + return "pass", nil + } + if err := runResourceMonitor(ctx, d); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(visits, []string{"first", "second", "first", "first", "first", "first", "second", "first"}) || !reflect.DeepEqual(elapsed, []int64{0, 0, 200, 1200, 2200, 3200, 4000, 4200}) { + t.Fatalf("unfair/due schedule: %v %v", visits, elapsed) + } + if state.Rows[0].Health != "HEALTHY" || state.Rows[0].Failures != 0 || state.Rows[1].NextDue != 18000 { + t.Fatal(state) + } +} + +func TestResourceMonitorBlockedRepairAndPersistenceFailure(t *testing.T) { + for _, kind := range []string{"source", "ownership", "storage", "save"} { + t.Run(kind, func(t *testing.T) { + source, key := resourceMonitorFixture(t) + state := resourceMonitorState{Version: 1, MAC: "initial"} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tick, probes := 0, 0 + d := resourceMonitorDeps{key: key, now: func() time.Time { return time.UnixMilli(10000 + int64(tick)*1000) }, report: func(bool) {}, wait: func(_ context.Context, _ time.Duration) error { + tick++ + if tick == 4 { + cancel() + } + return nil + }, source: func() (resourceMonitorSource, error) { + if kind == "source" && tick < 2 { + return resourceMonitorSource{}, errResourceConfig + } + return source, nil + }, guard: func() error { + if kind == "ownership" && tick < 2 { + return errResourceConfig + } + return nil + }, load: func() (resourceMonitorState, error) { + if kind == "storage" && tick < 2 { + return state, errResourceState + } + return state, nil + }, save: func(v resourceMonitorState) error { + if kind == "save" { + return errResourceState + } + state = v + return nil + }} + d.probe = func(_ context.Context, _ string, _ resourceCheckSpec, _ resourceMonitorSource, g func() error) (string, error) { + if err := g(); err != nil { + return "", err + } + probes++ + return "pass", nil + } + if err := runResourceMonitor(ctx, d); (err != nil) != (kind == "save") { + t.Fatal(err) + } + if kind == "save" { + if probes != 1 { + t.Fatalf("probed after persistence failure: %d", probes) + } + } else if probes != 2 { + t.Fatalf("did not resume safely: %d", probes) + } + }) + } +} + +func TestResourceMonitorConfigReconcilePrivacy(t *testing.T) { + source, key := resourceMonitorFixture(t) + rows := reconcileResourceRows(key, source, nil) + rows[0].Health, rows[0].Outcome, rows[0].LastCompleted = "HEALTHY", "pass", 10000 + rows[1].Health, rows[1].Outcome, rows[1].LastCompleted = "HEALTHY", "pass", 10000 + old := rows[1].Fingerprint + source.config.Services[0].Resources[0].URL = "https://example.com/changed-secret" + changed := reconcileResourceRows(key, source, rows) + if changed[0].Health != "UNKNOWN" || changed[1].Health != "HEALTHY" || changed[1].Fingerprint != old { + t.Fatal(changed) + } + source.config.Services = source.config.Services[1:] + if got := reconcileResourceRows(key, source, changed); len(got) != 1 || got[0].Service != "second" { + t.Fatal(got) + } +} + +func TestResourceMonitorStore(t *testing.T) { + source, key := resourceMonitorFixture(t) + dir := t.TempDir() + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + s, err := openResourceMonitorStore(dir, true) + if err != nil { + t.Fatal(err) + } + key = s.key + state, err := s.load() + if err != nil || len(state.Rows) != 0 { + t.Fatal(state, err) + } + state.Updated = 10000 + state.Rows = reconcileResourceRows(key, source, nil) + if err := s.save(state); err != nil { + t.Fatal(err) + } + s.close() + s, err = openResourceMonitorStore(dir, false) + if err != nil { + t.Fatal(err) + } + defer s.close() + loaded, err := s.load() + if err != nil || !reflect.DeepEqual(loaded.Rows, state.Rows) { + t.Fatal(loaded, err) + } + data, err := os.ReadFile(filepath.Join(dir, resourceStateName)) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"example.com", "private-token", "secret=value", "example.org", "Authorization"} { + if bytes.Contains(data, []byte(secret)) { + t.Fatal("secret serialized") + } + } + for _, name := range []string{resourceStateName, resourceKeyName} { + info, err := os.Stat(filepath.Join(dir, name)) + if err != nil || info.Mode().Perm() != 0600 { + t.Fatal(info, err) + } + } + // Corruption is preserved; restoring a valid signed version resumes loading. + bad := []byte(`{"version":99}`) + if err := os.WriteFile(filepath.Join(dir, resourceStateName), bad, 0600); err != nil { + t.Fatal(err) + } + if _, err := s.load(); err == nil { + t.Fatal("corrupt state trusted") + } + if err := s.save(state); err == nil { + t.Fatal("corrupt state overwritten") + } + got, _ := os.ReadFile(filepath.Join(dir, resourceStateName)) + if !bytes.Equal(got, bad) { + t.Fatal("evidence changed") + } + if err := os.WriteFile(filepath.Join(dir, resourceStateName), data, 0600); err != nil { + t.Fatal(err) + } + if _, err := s.load(); err != nil { + t.Fatal("valid repair refused", err) + } + if err := os.Remove(filepath.Join(dir, resourceKeyName)); err != nil { + t.Fatal(err) + } + if _, err := openResourceMonitorStore(dir, true); err == nil { + t.Fatal("missing key silently regenerated") + } +} + +func TestResourceMonitorUnsafeStorePreserved(t *testing.T) { + for _, kind := range []string{"symlink", "hardlink", "permissions", "wrong-owner", "directory-replaced"} { + t.Run(kind, func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "state") + if err := os.Mkdir(dir, 0700); err != nil { + t.Fatal(err) + } + s, err := openResourceMonitorStore(dir, true) + if err != nil { + t.Fatal(err) + } + defer s.close() + path := filepath.Join(dir, resourceKeyName) + original, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + switch kind { + case "symlink": + if err := os.Rename(path, path+"-original"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(path+"-original", path); err != nil { + t.Fatal(err) + } + case "hardlink": + if err := os.Link(path, path+"-alias"); err != nil { + t.Fatal(err) + } + case "permissions": + if err := os.Chmod(path, 0644); err != nil { + t.Fatal(err) + } + case "wrong-owner": + if os.Geteuid() != 0 { + t.Skip("requires chown capability") + } + if err := os.Chown(path, 12345, -1); err != nil { + t.Fatal(err) + } + case "directory-replaced": + if err := os.Rename(dir, dir+"-old"); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(dir, 0700); err != nil { + t.Fatal(err) + } + path = filepath.Join(dir+"-old", resourceKeyName) + } + if _, err := s.load(); err == nil { + t.Fatal("unsafe storage accepted") + } + if err := s.save(resourceMonitorState{Version: 1}); err == nil { + t.Fatal("unsafe storage overwritten") + } + data, err := os.ReadFile(path) + if err != nil || !bytes.Equal(data, original) { + t.Fatal("key changed", err) + } + }) + } +} + +func TestResourceMonitorReloadBeforeDNSAndTransfer(t *testing.T) { + for _, phase := range []string{"before-dns", "after-dns", "replace-after-dns", "env-after-dns", "owner-after-dns"} { + t.Run(phase, func(t *testing.T) { + key := bytes.Repeat([]byte{1}, 32) + path := resourceCheckFile(t, resourceCheckExample) + envChanged, ownerLost := false, false + lookup := func(k string) (string, bool) { + if envChanged && k == "CHECK_TOKEN" { + return "changed secret", true + } + return resourceCheckLookup(k) + } + source, err := readResourceMonitorSource(path, lookup, key) + if err != nil { + t.Fatal(err) + } + transfers, dnsCalls := 0, 0 + guard := func() error { + if ownerLost { + return errResourceConfig + } + fresh, err := readResourceMonitorSource(path, lookup, key) + if err != nil || fresh.fingerprint != source.fingerprint { + return errResourceConfig + } + return nil + } + mutate := func() { + switch phase { + case "env-after-dns": + envChanged = true + case "owner-after-dns": + ownerLost = true + case "replace-after-dns": + data := strings.Replace(resourceCheckExample, "secret=value", "secret=rotated", 1) + if err := os.WriteFile(path+"-next", []byte(data), 0600); err != nil { + t.Fatal(err) + } + if err := os.Rename(path+"-next", path); err != nil { + t.Fatal(err) + } + default: + if err := os.WriteFile(path, []byte(`{"version":1}`), 0600); err != nil { + t.Fatal(err) + } + } + } + if phase == "before-dns" { + mutate() + } + check := resourceCheckDeps{resolve: func(context.Context, string, string) ([]net.IP, error) { + dnsCalls++ + mutate() + return []net.IP{net.ParseIP("93.184.216.34")}, nil + }, runner: func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { transfers++; return nil }} + _, err = probeResourceMonitor(context.Background(), "first", source.config.Services[0].Resources[0], source, guard, check) + if err == nil || transfers != 0 || (phase == "before-dns" && dnsCalls != 0) { + t.Fatal("unsafe probe", transfers, dnsCalls, err) + } + }) + } +} + +func TestResourceMonitorBothTLSAndCancellationJoin(t *testing.T) { + source, _ := resourceMonitorFixture(t) + spec := source.config.Services[1].Resources[0] + check := resourceCheckDeps{resolve: func(context.Context, string, string) ([]net.IP, error) { + return []net.IP{net.ParseIP("93.184.216.34")}, nil + }} + calls := 0 + check.runner = func(_ context.Context, args []string, _ io.Reader, out, diag io.Writer) error { + calls++ + protocol := "tls12" + mode := "" + if strings.Contains(strings.Join(args, " "), "--tlsv1.3") { + protocol = "tls13" + mode = "verify" + } + configuredWire(spec.URL, spec.Method, protocol, mode, out, diag) + return nil + } + outcome, err := probeResourceMonitor(context.Background(), "second", spec, source, func() error { return nil }, check) + if err != nil || outcome == "pass" || calls != 2 { + t.Fatal(outcome, calls, err) + } + ctx, cancel := context.WithCancel(context.Background()) + joined := false + check.runner = func(ctx context.Context, _ []string, _ io.Reader, _ io.Writer, _ io.Writer) error { + cancel() + <-ctx.Done() + joined = true + return ctx.Err() + } + outcome, err = probeResourceMonitor(ctx, "second", spec, source, func() error { return nil }, check) + if err != nil || outcome != "cancelled" || !joined { + t.Fatal("did not join cancellation", outcome, err) + } +} + +func TestResourceMonitorSchemaScheduling(t *testing.T) { + for _, field := range []string{"interval_ms", "cooldown_ms"} { + for _, value := range []string{"0", "-1", "99", "86400001", "null", "\"1000\""} { + data := strings.Replace(resourceCheckExample, `"timeout_ms":1000`, `"timeout_ms":1000,"`+field+`":`+value, 1) + if _, err := parseResourceChecks([]byte(data), resourceCheckLookup); err == nil { + t.Fatal(field, value) + } + } + } +} + +func TestResourceMonitorSingletonAndGenericDispatch(t *testing.T) { + d, env, _ := managedCLIFixture(t) + lock, err := acquireManagedMonitorSingleton(d) + if err != nil { + t.Fatal(err) + } + defer lock.close() + if _, err := acquireManagedMonitorSingleton(d); !errors.Is(err, syscall.EWOULDBLOCK) { + t.Fatal("singleton conflict ignored", err) + } + // Generic dispatch precedes invalid YouTube/adaptive settings. A cancelled + // generic startup needs no runtime/network, database or valid source JSON. + env["SOCKS_PORT"] = "1080" + env["ZAPRET_AUTOTUNE_SOCKS_PROXY"] = "socks5h://127.0.0.1:1080" + env["ZAPRET_AUTOTUNE_RESOURCE_CONFIG"] = "/missing-invalid-config" + env["ZAPRET_AUTOTUNE_VIDEO_ID"] = "bad" + env["ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE"] = "invalid" + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var out, diag bytes.Buffer + code := managedMonitorCLIContext(ctx, nil, &out, &diag, d, func(context.Context, managedMonitorDeps) error { t.Fatal("YouTube supervisor invoked"); return nil }) + if code != 0 || !strings.Contains(out.String(), "generic resource monitor") { + t.Fatal(code, out.String(), diag.String()) + } + delete(env, "ZAPRET_AUTOTUNE_RESOURCE_CONFIG") + if code := managedMonitorCLIContext(ctx, nil, &out, &diag, d, nil); code != 64 { + t.Fatal("legacy invalid config no longer rejected", code) + } +} + +func TestResourceMonitorInvalidInitialConfigAliveAndTERM(t *testing.T) { + d, env, _ := managedCLIFixture(t) + env["SOCKS_PORT"] = "1080" + env["ZAPRET_AUTOTUNE_SOCKS_PROXY"] = "socks5h://127.0.0.1:1080" + d.verifySOCKS = func(int) error { return nil } + initManagedCLI(t, d) + path := resourceCheckFile(t, `{"invalid":true}`) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + started := make(chan struct{}, 1) + check := resourceCheckDeps{guard: func() error { + select { + case started <- struct{}{}: + default: + } + return nil + }, resolve: func(context.Context, string, string) ([]net.IP, error) { + t.Error("DNS with invalid source") + return nil, nil + }, runner: func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { + t.Error("transfer with invalid source") + return nil + }} + var out, diag bytes.Buffer + done := make(chan int, 1) + go func() { done <- resourceMonitorCLIWithCheck(ctx, path, &out, &diag, d, check) }() + // Source error short-circuits the runtime guard, so wait for authenticated + // BLOCKED persistence rather than relying on a diagnostic log race. + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + s, err := openResourceMonitorStore(d.directory, false) + if err == nil { + state, err := s.load() + s.close() + if err == nil && state.Blocked { + break + } + } + time.Sleep(10 * time.Millisecond) + } + select { + case code := <-done: + t.Fatalf("invalid source terminated controller: %d", code) + default: + } + cancel() + select { + case code := <-done: + if code != 0 { + t.Fatal(code, diag.String()) + } + case <-time.After(2 * time.Second): + t.Fatal("TERM did not join controller") + } + if !strings.Contains(diag.String(), "BLOCKED") { + t.Fatal("no blocked status", diag.String()) + } +} + +func TestResourceMonitorConcurrentEnvironmentReload(t *testing.T) { + path := resourceCheckFile(t, resourceCheckExample) + t.Setenv("CHECK_URL", "https://example.org/private") + t.Setenv("CHECK_TOKEN", "initial-secret") + t.Setenv("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5h://127.0.0.1:1080") + key := bytes.Repeat([]byte{3}, 32) + baseline, err := readResourceMonitorSource(path, os.LookupEnv, key) + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { + defer close(done) + for i := 0; i < 100; i++ { + os.Setenv("CHECK_TOKEN", "rotated-secret") + os.Setenv("CHECK_TOKEN", "initial-secret") + } + }() + for i := 0; i < 100; i++ { + s, err := readResourceMonitorSource(path, os.LookupEnv, key) + if err != nil { + t.Fatal(err) + } + if s.fingerprint == "" { + t.Fatal("empty fingerprint") + } + } + <-done + if err := os.Setenv("CHECK_TOKEN", "rotated-secret"); err != nil { + t.Fatal(err) + } + fresh, err := readResourceMonitorSource(path, os.LookupEnv, key) + if err != nil || fresh.fingerprint == baseline.fingerprint { + t.Fatal("secret rotation not bound", err) + } +} + +func TestResourceMonitorTargetConfigBoundary(t *testing.T) { + _, env, _ := managedCLIFixture(t) + for k, v := range env { + t.Setenv(k, v) + } + t.Setenv("SOCKS_PORT", "1080") + t.Setenv("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5h://127.0.0.1:1080") + t.Setenv("ZAPRET_AUTOTUNE_RESOURCE_CONFIG", resourceCheckFile(t, `{"invalid":true}`)) + t.Setenv("ZAPRET_AUTOTUNE_VIDEO_ID", "invalid") + t.Setenv("ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE", "invalid") + if code := run([]string{"target-config-check"}, io.Discard, io.Discard); code != 0 { + t.Fatal("bad resource JSON/YouTube blocked safe generic startup", code) + } + t.Setenv("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5h://foreign.example:1080") + if code := run([]string{"target-config-check"}, io.Discard, io.Discard); code != 64 { + t.Fatal("bad proxy allowed", code) + } + t.Setenv("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5h://127.0.0.1:1080") + t.Setenv("ZAPRET_AUTOTUNE_DNS_MODE", "invalid") + if code := run([]string{"target-config-check"}, io.Discard, io.Discard); code != 64 { + t.Fatal("bad DNS allowed", code) + } + t.Setenv("ZAPRET_AUTOTUNE_DNS_MODE", "doh") + t.Setenv("ZAPRET_AUTOTUNE_TARGET_ROLE", "foreign") + if code := run([]string{"target-config-check"}, io.Discard, io.Discard); code != 64 { + t.Fatal("bad target role allowed", code) + } +} diff --git a/deploy/managed-target/check_static.py b/deploy/managed-target/check_static.py index 22fd2a4..8b9c951 100644 --- a/deploy/managed-target/check_static.py +++ b/deploy/managed-target/check_static.py @@ -59,21 +59,24 @@ assert " stop_grace_period: 240s\n" in compose assert 'bounded_wait "$monitor_pid" 45' in entrypoint assert '/bin/sh /opt/managed-target-supervision.sh --watchdog' in entrypoint assert 'setsid /bin/sh /opt/managed-target-supervision.sh --monitor' in entrypoint -assert entrypoint.index('if ! cleanup_session; then') < entrypoint.index('if timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile; then') +assert entrypoint.index('if ! cleanup_session; then') < entrypoint.index('if cleanup_command 45 /usr/local/bin/zapret-autotune managed reconcile; then') assert 'COPY managed-target-supervision.sh /opt/managed-target-supervision.sh' in (directory.parents[1] / 'Dockerfile.target').read_text() assert '!managed-target-supervision.sh' in (directory.parents[1] / '.dockerignore').read_text() -assert '"$count" -lt 3' in entrypoint +assert '"$count" -ge 3' in entrypoint +assert 'BLOCKED: generic target restart budget; checks paused' in entrypoint assert '"$((now - started))" -lt 21600' in entrypoint assert "backoff=15" in entrypoint assert "systemctl" not in entrypoint and "docker " not in entrypoint -assert "timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop" in entrypoint +assert "cleanup_command 30 /opt/zapret2/init.d/sysv/zapret2 stop" in entrypoint +assert 'generic_cleanup_deadline=$(($(date +%s) + 80))' in entrypoint +assert 'timeout -k 1 "$cleanup_command_cap" "$@"' in entrypoint for command, seconds in (("/usr/local/bin/zapret-autotune managed init", 45), ("/usr/local/bin/zapret-autotune managed reconcile", 45), ("/opt/zapret2/init.d/sysv/zapret2 start", 30)): - assert f"timeout -k 5 {seconds} {command} || exit 78" in entrypoint -assert entrypoint.index("managed reconcile || exit 78") < entrypoint.index("zapret2 start || exit 78") + assert f"timeout -k 5 {seconds} {command} || return 78" in entrypoint +assert entrypoint.index("managed reconcile || return 78") < entrypoint.index("zapret2 start || return 78") experimental = 'managed apply --googlevideo-experiment native-circular --googlevideo-scope experimental_googlevideo_suffix --googlevideo-request startup || exit 78' -assert entrypoint.index('zapret2 start || exit 78') < entrypoint.index(experimental) +assert entrypoint.index('zapret2 start || return 78') < entrypoint.index(experimental) assert entrypoint.index(experimental) < entrypoint.index('setsid /bin/sh /opt/managed-target-supervision.sh --monitor') -assert entrypoint.index('managed experimental-request-check || exit 78') < entrypoint.index('managed init || exit 78') +assert entrypoint.index('managed experimental-request-check || return 78') < entrypoint.index('managed init || return 78') assert 'rm "${ZAPRET_CONFIG%/*}/.autotune-experimental-suffix-request"' not in entrypoint diff --git a/deploy/managed-target/check_supervision.py b/deploy/managed-target/check_supervision.py index 6a75683..480b460 100644 --- a/deploy/managed-target/check_supervision.py +++ b/deploy/managed-target/check_supervision.py @@ -67,10 +67,19 @@ if name == 'awk': sys.exit(0) if name == 'date': n = increment('clock') - print(100000 + (n * 21601 if scenario == 'window' else n)) + print(100000 + (n * 21601 if scenario == 'window' else n + (21601 if (root / 'budget-pause').exists() else 0))) sys.exit(0) if name == 'sleep': seconds = int(sys.argv[1]) + if scenario.startswith('generic-budget-') and seconds != 60: + os.execv(os.environ['REAL_SLEEP'], [os.environ['REAL_SLEEP'], *sys.argv[1:]]) + if seconds == 60 and scenario in ('generic-ss', 'generic-unsafe', 'generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'): + log('generic-pause') + (root / 'budget-pause').touch() + if scenario != 'generic-ss': + os.kill(os.getppid(), signal.SIGTERM) + time.sleep(.01) + sys.exit(0) if seconds in (15, 30): log('cooldown ' + str(seconds)) if scenario == 'cancel': @@ -96,9 +105,13 @@ if name in ('ss-server', 'ss-local'): os.kill(int((root / 'dns-pid').read_text()), 0) assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600 log(name) - if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail') and increment('ss_runs') == 1: + if scenario.startswith('generic-budget-'): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + while True: + time.sleep(.01) + if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail', 'generic-ss') and increment('ss_runs') <= (3 if scenario == 'generic-ss' else 1): # Let startup/monitor begin before simulating a child crash. - while not (root / 'monitor_runs').exists(): + while not (root / 'monitor_runs').exists() or (scenario == 'generic-ss' and int((root / 'monitor_runs').read_text()) < int((root / 'ss_runs').read_text())): time.sleep(.01) time.sleep(.03) sys.exit(137) @@ -161,6 +174,8 @@ if sys.argv[1] == 'dns-forwarder': ready.unlink() sys.exit(0) signal.signal(signal.SIGTERM, stop_dns) + if scenario.startswith('generic-budget-'): + signal.signal(signal.SIGTERM, signal.SIG_IGN) if scenario in ('dns-term', 'dns-int'): os.kill(os.getppid(), signal.SIGTERM if scenario == 'dns-term' else signal.SIGINT) if scenario in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink') and increment('dns_runs') == 1: @@ -186,9 +201,15 @@ if command == 'init': elif command == 'reconcile': n = increment('reconciles') log('reconcile') + if scenario.startswith('generic-budget-') and n == 2: + with (root / 'children').open('a') as f: + f.write(str(os.getpid()) + '\n') + signal.signal(signal.SIGTERM, signal.SIG_IGN) + while True: + time.sleep(.01) if scenario == 'orphans': assert_workers_gone() - if scenario == 'prestartfail': + if scenario in ('prestartfail', 'generic-unsafe'): sys.exit(78) if scenario == 'dns-prestart': os.kill(int((root / 'dns-pid').read_text()), signal.SIGKILL) @@ -197,11 +218,23 @@ elif command == 'reconcile': sys.exit(78) (root / 'prepared').unlink(missing_ok=True) elif command == 'monitor': + if scenario.startswith('generic-'): + assert os.environ['ZAPRET_AUTOTUNE_RESOURCE_CONFIG'] == '/operator/resources.json' with (root / 'children').open('a') as f: f.write(str(os.getpid()) + '\n') n = increment('monitor_runs') log('monitor ' + str(n)) assert os.getsid(0) == os.getpid(), 'monitor not in its own verified session' + if scenario.startswith('generic-budget-'): + time.sleep(.03) + os.kill(os.getppid(), signal.SIGTERM) + if scenario == 'generic-budget-monitor': + signal.signal(signal.SIGTERM, signal.SIG_IGN) + while True: + time.sleep(.01) + linger() + if scenario == 'generic-ss' and n <= 3: + linger() if scenario in ('dns-death', 'dns-killed', 'dns-replace', 'dns-replace-symlink') and n == 1: linger() if scenario == 'dns-outage': @@ -245,6 +278,10 @@ def run(scenario, expected, dns='', mode='native'): fake.chmod(0o755) for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2', 'blockcheck', 'nfq'): (binary / name).symlink_to(fake) + realtime = scenario.startswith('generic-budget-') + if realtime: + for name in ('date', 'timeout'): + (binary / name).unlink() configs = root / 'private' configs.mkdir() if scenario.startswith('dns-stale'): @@ -258,15 +295,28 @@ def run(scenario, expected, dns='', mode='native'): broken_helper = root / 'broken-helper.sh' broken_helper.write_text(helper.read_text().replace('sleep "$seconds" & sleeper=$!', 'exit 78')) helper = broken_helper + if scenario == 'generic-budget-ack': + delayed_helper = root / 'delayed-helper.sh' + delayed_helper.write_text(helper.read_text().replace(': > "$config_dir/join-ready"', + f'{shutil.which("sleep")} 300\n : > "$config_dir/join-ready"')) + helper = delayed_helper text = text.replace('/opt/managed-target-supervision.sh', str(helper)) text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}') text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune')) text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2')) + if realtime: + # Private fixture scaling only: production has no configurable grace. + text = text.replace('$(date +%s) + 80', '$(date +%s) + 12') entry = root / 'entry.sh' entry.write_text(text) env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp, SCENARIO=scenario, SS_PASSWORD='ab' * 32, SS_DNS_SERVER=dns, - ZAPRET_CONFIG=str(root / 'config'), DNS_CLI=str(DNS_CLI)) + ZAPRET_CONFIG=str(root / 'config'), DNS_CLI=str(DNS_CLI), + REAL_SLEEP=shutil.which('sleep')) + if scenario.startswith('generic-'): + env['ZAPRET_AUTOTUNE_RESOURCE_CONFIG'] = '/operator/resources.json' + else: + env.pop('ZAPRET_AUTOTUNE_RESOURCE_CONFIG', None) if mode is None: env.pop('ZAPRET_AUTOTUNE_DNS_MODE', None) else: @@ -279,7 +329,10 @@ def run(scenario, expected, dns='', mode='native'): peer = subprocess.Popen([shutil.which('sleep'), '300'], start_new_session=True) (root / 'peer').write_text(str(peer.pid)) try: - result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15) + started = time.monotonic() + result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=18 if realtime else 15) + if realtime: + assert time.monotonic() - started < 16, (scenario, 'aggregate cleanup exceeded grace') if peer: assert peer.poll() is None, 'unowned helper was killed' finally: @@ -311,6 +364,9 @@ def run(scenario, expected, dns='', mode='native'): assert ready.exists() or ready.is_symlink(), (scenario, 'foreign readiness deleted') if scenario != 'dns-stale-symlink' and scenario != 'dns-symlink': assert ready.read_text() == 'foreign evidence\n', (scenario, 'foreign bytes changed') + elif realtime and (configs / 'dns-ready').exists(): + # Exhaustion cannot authorize unlinking without the owned Go helper. + assert (configs / 'dns-ready').read_text().strip() == (root / 'dns-pid').read_text() else: assert not (configs / 'dns-ready').exists(), (scenario, 'stale DNS readiness') if mode == 'native': @@ -339,6 +395,15 @@ events = run('cancel', 0) assert events.count('start') == 1 events = run('prestartfail', 78) assert 'start' not in events and 'stop' not in events +events = run('generic-ss', 0, mode='doh') +assert events.count('start') == 4 and events.count('stop') == 4, events +assert events.count('generic-pause') == 1, events +assert events.count('dns-start') == events.count('dns-stop') == 4, events +events = run('generic-unsafe', 0) +assert 'start' not in events and events.count('generic-pause') == 1, events +for scenario in ('generic-budget-monitor', 'generic-budget-reconcile', 'generic-budget-ack'): + events = run(scenario, 78, mode='doh') + assert events.count('start') == 1 and 'stop' not in events, (scenario, events) for dns in ('8.8.8.8', '2001:4860:4860::8888'): events = run('fatal64', 64, dns) assert events.count('ss-server') == 1 and events.count('ss-local') == 1 @@ -384,6 +449,7 @@ def check_watchdog_traps(shell): mode=$1 shift config_dir={directory} +generic_mode=0 . {ROOT / 'managed-target-supervision.sh'} {bounded_wait} cleanup() {{ diff --git a/deploy/managed-target/compose.yaml b/deploy/managed-target/compose.yaml index 0dd7e2a..f25af3b 100644 --- a/deploy/managed-target/compose.yaml +++ b/deploy/managed-target/compose.yaml @@ -18,6 +18,7 @@ services: ZAPRET_AUTOTUNE_DOH_BOOTSTRAP_IPS: ${ZAPRET_AUTOTUNE_DOH_BOOTSTRAP_IPS-} ZAPRET_AUTOTUNE_DOH_TIMEOUT_SEC: ${ZAPRET_AUTOTUNE_DOH_TIMEOUT_SEC-5} ZAPRET_AUTOTUNE_YOUTUBE_SERVICE: ${ZAPRET_AUTOTUNE_YOUTUBE_SERVICE-0} + ZAPRET_AUTOTUNE_RESOURCE_CONFIG: ${ZAPRET_AUTOTUNE_RESOURCE_CONFIG-} ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE: ${ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE-cold} ZAPRET_AUTOTUNE_VIDEO_ID: ${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ} ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS: '${ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS:-}' diff --git a/managed-target-entrypoint.sh b/managed-target-entrypoint.sh index eda0324..d18a9cf 100644 --- a/managed-target-entrypoint.sh +++ b/managed-target-entrypoint.sh @@ -16,6 +16,8 @@ esac /usr/local/bin/zapret-autotune target-config-check || exit 64 dns_mode=$(/usr/local/bin/zapret-autotune dns-forwarder mode) || exit 64 dns_address=$(/usr/local/bin/zapret-autotune dns-forwarder address) || exit 64 +generic_mode=0 +[ -z "${ZAPRET_AUTOTUNE_RESOURCE_CONFIG-}" ] || generic_mode=1 case "$dns_mode" in doh|native) ;; *) exit 64 ;; esac config_dir=/run/zapret-autotune-ss awk '$2 == "/run/zapret-autotune-ss" && $3 == "tmpfs" { found=1 } END { exit !found }' /proc/mounts || { @@ -40,10 +42,45 @@ monitor_pid= monitor_sid= zapret_started=0 cooldown_pid= +generic_cleanup_deadline= +generic_cleanup_failed=0 # Signal only descendants of an owned child, never the container/host namespace. # The monitor gets TERM itself first so it can join scans and restore bytes; # descendant signalling is reserved for SS shutdown and forced watchdog cleanup. . /opt/managed-target-supervision.sh +# One budget per cleanup cycle, including ACKs, kill escalation and helper joins. +# Ten seconds of Docker's 90s grace remain for coarse clocks and final reaping. +cleanup_remaining() { + cleanup_left=$((generic_cleanup_deadline - $(date +%s))) + [ "$cleanup_left" -ge 0 ] || cleanup_left=0 +} +cleanup_command() { + if [ "$generic_mode" -eq 0 ]; then + timeout -k 5 "$@" + return $? + fi + cleanup_command_cap=$1 + shift + cleanup_remaining + # timeout's KILL grace is part of, not additional to, the shared budget. + [ "$cleanup_left" -gt 1 ] || return 78 + cleanup_command_left=$((cleanup_left - 1)) + [ "$cleanup_command_cap" -le "$cleanup_command_left" ] || cleanup_command_cap=$cleanup_command_left + timeout -k 1 "$cleanup_command_cap" "$@" +} +cleanup_reap() { + reap_pid=$1 + reap_deadline=$2 + [ "$reap_deadline" -le "$((generic_cleanup_deadline + 2))" ] || reap_deadline=$((generic_cleanup_deadline + 2)) + child_joined=0 + while proc_identity "$reap_pid" && [ "$proc_state" != Z ]; do + [ "$(date +%s)" -lt "$reap_deadline" ] || return 0 + pause 1 + done + waited_code=0 + wait "$reap_pid" 2>/dev/null || waited_code=$? + child_joined=1 +} pause() { sleep "$1" 8<&- 9<&- /dev/null 2>&1 & cooldown_pid=$! wait "$cooldown_pid" @@ -57,6 +94,9 @@ cleanup_session() { for session_signal in TERM KILL; do signal_session "$monitor_sid" "$session_signal" || return 78 session_deadline=$(($(date +%s) + 3)) + if [ "$generic_mode" -eq 1 ] && [ "$session_deadline" -gt "$generic_cleanup_deadline" ]; then + session_deadline=$generic_cleanup_deadline + fi while :; do signal_session "$monitor_sid" CHECK || return 78 if [ "$session_live" -eq 0 ]; then @@ -85,6 +125,19 @@ capture_monitor_session() { bounded_wait() { child=$1 seconds=$2 + if [ "$generic_mode" -eq 1 ]; then + cleanup_remaining + # Reserve ACK setup (5s) and post-KILL reaping (2s) before granting wait. + if [ "$cleanup_left" -le 7 ]; then + signal_tree "$child" KILL + cleanup_reap "$child" "$(($(date +%s) + 2))" + waited_code=${waited_code-137} + join_timed_out=1 + return 0 + fi + cleanup_wait_left=$((cleanup_left - 7)) + [ "$seconds" -le "$cleanup_wait_left" ] || seconds=$cleanup_wait_left + fi rm -f "$config_dir/join-timeout" "$config_dir/join-ready" # Fresh interpreter: a forked shell can inherit a deferred EXIT/TERM trap # context and never run its cancellation handler during PID 1 shutdown. @@ -93,6 +146,9 @@ bounded_wait() { # A fast-exiting child must not cancel the watchdog before its trap exists. # Fail closed if exec/setup fails before acknowledgement; never wait forever. ack_deadline=$(($(date +%s) + 5)) + if [ "$generic_mode" -eq 1 ] && [ "$ack_deadline" -gt "$generic_cleanup_deadline" ]; then + ack_deadline=$generic_cleanup_deadline + fi while [ ! -e "$config_dir/join-ready" ]; do if ! kill -0 "$watchdog" 2>/dev/null || [ "$(date +%s)" -ge "$ack_deadline" ]; then : > "$config_dir/join-timeout" @@ -102,13 +158,39 @@ bounded_wait() { fi done waited_code=0 - wait "$child" 2>/dev/null || waited_code=$? + if [ "$generic_mode" -eq 1 ]; then + cleanup_join_deadline=$(($(date +%s) + seconds)) + [ "$cleanup_join_deadline" -le "$generic_cleanup_deadline" ] || cleanup_join_deadline=$generic_cleanup_deadline + cleanup_reap "$child" "$cleanup_join_deadline" + if [ "$child_joined" -eq 0 ]; then + : > "$config_dir/join-timeout" + signal_tree "$child" KILL + cleanup_reap "$child" "$(($(date +%s) + 2))" + fi + joined_service=$child_joined + else + wait "$child" 2>/dev/null || waited_code=$? + fi kill -TERM "$watchdog" 2>/dev/null || true - wait "$watchdog" 2>/dev/null || true + if [ "$generic_mode" -eq 1 ]; then + service_waited_code=$waited_code + cleanup_reap "$watchdog" "$(($(date +%s) + 2))" + if [ "$child_joined" -eq 0 ]; then + signal_tree "$watchdog" KILL + cleanup_reap "$watchdog" "$(($(date +%s) + 2))" + fi + child_joined=$joined_service + waited_code=$service_waited_code + else + wait "$watchdog" 2>/dev/null || true + fi join_timed_out=0 [ ! -e "$config_dir/join-timeout" ] || join_timed_out=1 } cleanup_cycle() { + if [ "$generic_mode" -eq 1 ] && [ -z "$generic_cleanup_deadline" ]; then + generic_cleanup_deadline=$(($(date +%s) + 80)) + fi cleanup_code=0 # Let the monitor cancel/join scans and restore committed bytes BEFORE stop. if [ -n "$monitor_pid" ]; then @@ -133,8 +215,8 @@ cleanup_cycle() { # Stop also sources config: clear trust before reconciliation, never # source unknown bytes left by a crashed monitor, even for shutdown. zapret_started=0 - if timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile; then - timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop || cleanup_code=78 + if cleanup_command 45 /usr/local/bin/zapret-autotune managed reconcile; then + cleanup_command 30 /opt/zapret2/init.d/sysv/zapret2 stop || cleanup_code=78 else cleanup_code=78 fi @@ -142,9 +224,11 @@ cleanup_cycle() { for pid in "$local_pid" "$server_pid"; do [ -z "$pid" ] || signal_tree "$pid" TERM done + ss_joined=1 for pid in "$local_pid" "$server_pid"; do if [ -n "$pid" ]; then bounded_wait "$pid" 5 + if [ "$generic_mode" -eq 1 ] && [ "$child_joined" -eq 0 ]; then ss_joined=0; fi # Its original failure triggered supervision; only forced shutdown # is a cleanup failure, not the already-dead Shadowsocks status. [ "$join_timed_out" -eq 0 ] || { echo 'error: Shadowsocks join timeout' >&2; cleanup_code=78; } @@ -153,15 +237,27 @@ cleanup_cycle() { server_pid= local_pid= # DNS remains available through monitor rollback, zapret stop and SS joins. - if [ -n "$dns_pid" ]; then + if [ -n "$dns_pid" ] && [ "$ss_joined" -eq 1 ]; then kill -TERM "$dns_pid" 2>/dev/null || true bounded_wait "$dns_pid" 5 [ "$join_timed_out" -eq 0 ] || { echo 'error: DNS join timeout' >&2; cleanup_code=78; } - /usr/local/bin/zapret-autotune dns-forwarder readiness-cleanup "$dns_pid" || cleanup_code=78 - dns_pid= + if [ "$generic_mode" -eq 1 ] && [ "$child_joined" -eq 0 ]; then + cleanup_code=78 + elif [ "$generic_mode" -eq 1 ]; then + cleanup_command 5 /usr/local/bin/zapret-autotune dns-forwarder readiness-cleanup "$dns_pid" || cleanup_code=78 + else + /usr/local/bin/zapret-autotune dns-forwarder readiness-cleanup "$dns_pid" || cleanup_code=78 + fi + if [ "$generic_mode" -eq 0 ] || [ "$child_joined" -eq 1 ]; then dns_pid=; fi + fi + if [ -z "$dns_pid" ]; then + if [ "$dns_anchor_file" -eq 1 ]; then exec 9<&-; dns_anchor_file=0; fi + if [ "$dns_anchor_dir" -eq 1 ]; then exec 8<&-; dns_anchor_dir=0; fi + fi + if [ "$generic_mode" -eq 1 ]; then + [ "$cleanup_code" -eq 0 ] || generic_cleanup_failed=1 + generic_cleanup_deadline= fi - if [ "$dns_anchor_file" -eq 1 ]; then exec 9<&-; dns_anchor_file=0; fi - if [ "$dns_anchor_dir" -eq 1 ]; then exec 8<&-; dns_anchor_dir=0; fi return "$cleanup_code" } cleanup() { @@ -174,6 +270,7 @@ cleanup() { wait "$cooldown_pid" 2>/dev/null || true fi cleanup_cycle || final_code=78 + [ "$generic_cleanup_failed" -eq 0 ] || final_code=78 exit "$final_code" } trap cleanup EXIT @@ -232,19 +329,21 @@ until awk '$2 == "00000000:0438" && $4 == "0A" { found=1 } END { exit !found }' done dns_alive || return 75 # Reject unsafe operator requests before init/reconcile can mutate owned state. -timeout -k 5 45 /usr/local/bin/zapret-autotune managed experimental-request-check || exit 78 +if [ "$generic_mode" -eq 0 ]; then + timeout -k 5 45 /usr/local/bin/zapret-autotune managed experimental-request-check || return 78 +fi if [ ! -e "$ZAPRET_CONFIG" ]; then - timeout -k 5 45 /usr/local/bin/zapret-autotune managed init || exit 78 + timeout -k 5 45 /usr/local/bin/zapret-autotune managed init || return 78 fi # Never activate pending or unknown bytes. Any reconciliation failure is fatal, # not a transient (75) restart request. Reload restores known bytes first. -timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile || exit 78 +timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile || return 78 zapret_started=1 -timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || exit 78 +timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || return 78 # Private operator request, consumed only after a successful guarded transaction. # Never suppress qualification gates globally or race the owned monitor. -if [ -e "${ZAPRET_CONFIG%/*}/.autotune-experimental-suffix-request" ] || [ -L "${ZAPRET_CONFIG%/*}/.autotune-experimental-suffix-request" ]; then +if [ "$generic_mode" -eq 0 ] && { [ -e "${ZAPRET_CONFIG%/*}/.autotune-experimental-suffix-request" ] || [ -L "${ZAPRET_CONFIG%/*}/.autotune-experimental-suffix-request" ]; }; then timeout -k 5 120 /usr/local/bin/zapret-autotune managed apply --googlevideo-experiment native-circular --googlevideo-scope experimental_googlevideo_suffix --googlevideo-request startup || exit 78 fi @@ -259,13 +358,13 @@ monitor_ack_deadline=$(($(date +%s) + 5)) while [ ! -e "$config_dir/monitor-ready" ]; do if ! kill -0 "$monitor_pid" 2>/dev/null || [ "$(date +%s)" -ge "$monitor_ack_deadline" ]; then signal_tree "$monitor_pid" KILL - exit 78 + return 78 fi done -capture_monitor_session || exit 78 +capture_monitor_session || return 78 if proc_identity "$monitor_pid" && [ "$proc_sid" != "$monitor_sid" ]; then monitor_sid= - exit 78 + return 78 fi while dns_alive && kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do pause 1 @@ -300,11 +399,29 @@ while :; do count=$((count + 1)) fi done - [ "$count" -lt 3 ] || { echo 'error: target restart budget exhausted' >&2; exit 78; } + if [ "$count" -ge 3 ]; then + if [ "$generic_mode" -eq 1 ]; then + echo 'BLOCKED: generic target restart budget; checks paused' >&2 + pause 60 + continue + fi + echo 'error: target restart budget exhausted' >&2; exit 78 + fi starts="$recent $now" code=0 run_cycle || code=$? - cleanup_cycle || code=78 + if ! cleanup_cycle; then + if [ "$generic_mode" -eq 1 ]; then + echo 'BLOCKED: generic target unsafe cleanup; operator repair/restart required' >&2 + while :; do pause 60; done + fi + code=78 + fi + if [ "$generic_mode" -eq 1 ] && [ "$code" -eq 78 ]; then + echo 'BLOCKED: generic target trust/startup failure; checks paused' >&2 + pause 60 + continue + fi [ "$code" -eq 75 ] || exit "$code" # A long-running monitor may outlive the window; recheck at failure time. now=$(date +%s) @@ -312,7 +429,9 @@ while :; do for started in $starts; do [ "$((now - started))" -ge 21600 ] || retry_count=$((retry_count + 1)) done - [ "$retry_count" -lt 3 ] || { echo 'error: target restart budget exhausted' >&2; exit 78; } + if [ "$retry_count" -ge 3 ] && [ "$generic_mode" -eq 0 ]; then + echo 'error: target restart budget exhausted' >&2; exit 78 + fi pause "$backoff" [ "$backoff" -ge 60 ] || backoff=$((backoff * 2)) done diff --git a/resource-check.example.json b/resource-check.example.json index 3f95559..327d2e3 100644 --- a/resource-check.example.json +++ b/resource-check.example.json @@ -12,6 +12,8 @@ "method": "GET", "tls": "both", "timeout_ms": 10000, + "interval_ms": 300000, + "cooldown_ms": 30000, "expect": { "statuses": [200], "content_type": "text/html",