feat(runtime): bound recovery and add safe release supervision

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-10 22:51:58 +03:00
1 parent 780ca71e17
commit b1e458a0a5
16 files changed
+941 -66

No files matched your search

+1 -1
View File
@@ -21,7 +21,7 @@ func managedSeedArgs(host string) string {
host = strings.ToLower(host)
switch {
case host == "www.youtube.com":
return "--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:ip_autottl=-3,3-20:repeats=1 --lua-desync=multidisorder:pos=1,midsld"
return "--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld"
case host == "googlevideo.com" || strings.HasSuffix(host, ".googlevideo.com"):
return "--payload=tls_client_hello --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN"
default:
+22 -2
View File
@@ -253,6 +253,13 @@ func TestRunManagedBlockcheckProtectsForeignWorkers(t *testing.T) {
func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
args := managedSeedArgs("www.youtube.com")
const wantSeed = "--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld"
if args != wantSeed {
t.Fatalf("frontend seed=%q, want %q", args, wantSeed)
}
if err := (&candidate{Args: args}).validateArgs(); err != nil {
t.Fatalf("invalid frontend seed grammar: %v", err)
}
t.Setenv("ZAPRET_TEST_REPORT", combinedReportFixture(args))
argvPath := filepath.Join(t.TempDir(), "argv.jsonl")
t.Setenv("ZAPRET_TEST_ARGV", argvPath)
@@ -279,10 +286,23 @@ func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) {
if err := decoder.Decode(&checkArgs); err != nil {
t.Fatal(err)
}
if !slices.Contains(checkArgs, "--from-file") || !slices.Contains(checkArgs, "--passes") || !slices.Contains(checkArgs, "--no-conflict-cleanup") {
wantCheck := []string{"--workers", "32", "check", "--from-file", path + ".input", "--domain", "www.youtube.com", "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", "--output", path + ".check.json"}
if !reflect.DeepEqual(checkArgs, wantCheck) {
t.Fatalf("managed seeded check=%q", checkArgs)
}
if seed := managedSeedArgs("rr3.googlevideo.com"); !strings.Contains(seed, "fakeddisorder") {
if err := decoder.Decode(&checkArgs); err != io.EOF {
t.Fatalf("unexpected extra blockcheck call: %q %v", checkArgs, err)
}
input, err := os.ReadFile(path + ".input")
if err != nil {
t.Fatal(err)
}
wantRows := "curl_test_https_tls12 ipv4 www.youtube.com : nfqws2 " + wantSeed + "\n" +
"curl_test_https_tls13 ipv4 www.youtube.com : nfqws2 " + wantSeed + "\n"
if string(input) != wantRows {
t.Fatalf("managed seed input=%q", input)
}
if seed := managedSeedArgs("rr3.googlevideo.com"); seed != "--payload=tls_client_hello --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN" {
t.Fatalf("missing exact-host GGC seed: %q", seed)
}
}
+43 -13
View File
@@ -56,36 +56,41 @@ func managedMonitorCLI(args []string, stdout, stderr io.Writer, target managedDe
return managedMonitorCLIContext(ctx, args, stdout, stderr, target, runManagedMonitor)
}
func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) int {
func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) (code int) {
cfg, err := managedMonitorConfig(args, target.lookup)
if err != nil {
fmt.Fprintln(stderr, "error: invalid managed monitor configuration")
return 2
return 64
}
if code, err := managedRuntimeGuard(target); err != nil {
if _, err := managedRuntimeGuard(target); err != nil {
fmt.Fprintln(stderr, "error: managed monitor target guard refused")
return code
return 78
}
m := &managedTransaction{Dir: target.directory, Name: "config"}
if err := managedDirectory(target.directory); err != nil {
fmt.Fprintln(stderr, "error: unsafe managed monitor directory")
return 1
return 78
}
if err := managedCheckIdentity(m); err != nil {
fmt.Fprintln(stderr, "error: managed monitor identity refused")
return 1
return 78
}
path, err := managedMonitorDatabase(target)
if err != nil {
fmt.Fprintln(stderr, "error: unsafe managed monitor database")
return 1
return 78
}
db, err := openStore(path)
if err != nil {
fmt.Fprintln(stderr, "error: managed monitor database unavailable")
return 1
return 78
}
defer db.Close()
defer func() {
if err := db.Close(); err != nil {
fmt.Fprintln(stderr, "error: managed monitor database close failed")
code = 78
}
}()
recovery, err := managedMonitorRecoveryAdapters(target, db, runManagedBlockcheck, runManagedCombinedBlockcheckForHost, func(ctx context.Context, path string, args ...string) error {
log, err := os.CreateTemp(target.directory, ".daemon-diagnostics-*.log")
if err != nil {
@@ -98,7 +103,7 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr
})
if err != nil {
fmt.Fprintln(stderr, "error: managed monitor adapters refused")
return 1
return 78
}
monitor := managedMonitorDeps{
target: target, db: db, serviceID: managedTargetID + "/youtube",
@@ -110,7 +115,7 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr
fmt.Fprintln(stderr, "managed GGC unresolved; monitoring continues (not playback health)")
}); err != nil {
fmt.Fprintln(stderr, "error: managed monitor wiring refused")
return 1
return 78
}
fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health")
err = supervise(ctx, monitor)
@@ -120,9 +125,34 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr
return 0
}
fmt.Fprintln(stderr, "error: managed monitor stopped; safety/database/runtime failure (details suppressed)")
return 1
if managedMonitorTechnicalTimeout(err) {
return 75
}
return 78
}
return 0
if ctx.Err() != nil {
return 0
}
return 75 // A supervisor must not return normally without cancellation.
}
// Only clean typed timeouts may restart; a joined cleanup failure vetoes it.
func managedMonitorTechnicalTimeout(err error) bool {
if timeout, ok := err.(*managedRecoveryTimeout); ok {
return managedRecoveryTimeoutSafe(timeout.Err)
}
if joined, ok := err.(interface{ Unwrap() []error }); ok {
for _, e := range joined.Unwrap() {
if !managedMonitorTechnicalTimeout(e) {
return false
}
}
return len(joined.Unwrap()) != 0
}
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
return managedMonitorTechnicalTimeout(wrapped.Unwrap())
}
return false
}
func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand cacheScan, checkCommand cacheCheck, command func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
+86 -10
View File
@@ -40,6 +40,64 @@ func TestManagedMonitorCancellationSafety(t *testing.T) {
}
}
func TestManagedMonitorExitClassification(t *testing.T) {
for _, mode := range []string{"timeout", "timeout-cleanup", "timeout-inner-cleanup", "fatal-timeout", "fatal-timeout-cleanup", "timeout-fatal", "job-timeout", "job-timeout-cleanup", "unknown", "zero", "cancel", "cancel-cleanup"} {
t.Run(mode, func(t *testing.T) {
d, env, _ := managedCLIFixture(t)
initManagedCLI(t, d)
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
want := 78
var out bytes.Buffer
code := managedMonitorCLIContext(ctx, nil, &out, &out, d, func(context.Context, managedMonitorDeps) error {
switch mode {
case "timeout":
want = 75
return &managedRecoveryTimeout{Err: context.DeadlineExceeded}
case "timeout-cleanup":
return errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("diagnostic close failed"))
case "timeout-inner-cleanup":
return &managedRecoveryTimeout{Err: errors.Join(context.DeadlineExceeded, errors.New("diagnostic close failed"))}
case "fatal-timeout":
want = 75 // Fatal is also the monitor's stop signal for clean timeout.
return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: context.DeadlineExceeded}}
case "fatal-timeout-cleanup":
return &managedRecoveryFatal{Err: errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("cleanup"))}
case "timeout-fatal":
return &managedRecoveryTimeout{Err: &managedRecoveryFatal{Err: errors.New("cleanup")}}
case "job-timeout", "job-timeout-cleanup":
if mode == "job-timeout" {
want = 75
}
return managedRecoveryJob(ctx, time.Millisecond, func(jobCtx context.Context) error {
<-jobCtx.Done()
if mode == "job-timeout-cleanup" {
return &managedRecoveryFatal{Err: errors.Join(jobCtx.Err(), errors.New("cleanup"))}
}
return jobCtx.Err()
})
case "zero":
want = 75
return nil
case "cancel":
want = 0
cancel()
return ctx.Err()
case "cancel-cleanup":
cancel()
return errors.Join(ctx.Err(), errors.New("cleanup"))
default:
return errors.New("unknown")
}
})
if code != want {
t.Fatal(code, want, out.String())
}
})
}
}
func TestManagedMonitorProductionAdapterBinding(t *testing.T) {
recovery, _, _ := recoveryFixture(t, false)
target, _, _ := managedCLIFixture(t)
@@ -132,7 +190,11 @@ func TestManagedMonitorCLIGuardsBeforeLock(t *testing.T) {
}
var out bytes.Buffer
code := managedMonitorCLIContext(context.Background(), nil, &out, &out, d, func(context.Context, managedMonitorDeps) error { t.Fatal("unsafe supervisor start"); return nil })
if code == 0 || strings.Contains(out.String(), "password-do-not-print") {
want := 78
if mode == "config" {
want = 64
}
if code != want || strings.Contains(out.String(), "password-do-not-print") {
t.Fatal(code, out.String())
}
if _, err := os.Lstat(filepath.Join(d.directory, managedMonitorLockName)); !errors.Is(err, os.ErrNotExist) {
@@ -199,7 +261,7 @@ func TestManagedMonitorBootstrapExistingFrontend(t *testing.T) {
// Execute only rewritten target-local contracts and fake programs; no network,
// Docker, real credentials, zapret or host firewall. Exercise the actual traps.
func TestManagedEntrypointSupervision(t *testing.T) {
for _, mode := range []string{"term", "fatal"} {
for _, mode := range []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"} {
t.Run(mode, func(t *testing.T) {
root := t.TempDir()
config := filepath.Join(root, "ss")
@@ -215,16 +277,23 @@ func TestManagedEntrypointSupervision(t *testing.T) {
}
log := filepath.Join(root, "log")
write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n")
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do sleep 0.1 & wait $!; done\n"
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & wait $!; done\n"
write("ss-server", child)
write("ss-local", child)
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\n")
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then echo managed-$2 >> \"$LOG\"; exit 0; fi\ntrap 'echo monitor-stopped >> \"$LOG\"; exit 0' TERM INT\necho managed-monitor >> \"$LOG\"\nif [ \"$MODE\" = fatal ]; then exit 7; fi\nwhile :; do sleep 0.1 & wait $!; done\n")
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nif [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; while :; do sleep 0.1; done; fi\n")
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then echo managed-$2 >> \"$LOG\"; exit 0; fi\ntrap 'echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; fi\necho managed-monitor >> \"$LOG\"\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & wait $!; done\n")
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
if err != nil {
t.Fatal(err)
}
text := strings.NewReplacer("/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
if mode == "term-hang" {
// Shorten only the fixture's watchdog, not the production contract.
text = strings.Replace(text, `bounded_wait "$monitor_pid" 60`, `bounded_wait "$monitor_pid" 1`, 1)
}
if mode == "stop-hang" {
text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1)
}
entry := write("entrypoint", text)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -240,7 +309,7 @@ func TestManagedEntrypointSupervision(t *testing.T) {
exited := make(chan struct{})
go func() { done <- cmd.Wait(); close(exited) }()
t.Cleanup(func() { syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); <-exited })
if mode == "term" {
if strings.HasPrefix(mode, "term") {
for {
b, _ := os.ReadFile(log)
if strings.Contains(string(b), "managed-monitor") {
@@ -262,13 +331,20 @@ func TestManagedEntrypointSupervision(t *testing.T) {
if ctx.Err() != nil {
t.Fatal("entrypoint did not join children", ctx.Err())
}
if mode == "term" && err != nil {
want := 75
switch mode {
case "term":
want = 0
case "fatal", "unknown", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang":
want = 78
}
if want == 0 && err != nil {
t.Fatal(err, output.String())
}
if mode == "fatal" {
if want != 0 {
var exit *exec.ExitError
if !errors.As(err, &exit) || exit.ExitCode() != 7 {
t.Fatal(err)
if !errors.As(err, &exit) || exit.ExitCode() != want {
t.Fatal(err, "want", want, output.String())
}
}
b, err := os.ReadFile(log)
+134 -16
View File
@@ -9,12 +9,16 @@ import (
"fmt"
"io"
"os"
"os/exec"
"strings"
"syscall"
"time"
)
const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2"
const managedRecoveryJobTimeout = 90 * time.Minute
// Caller owns startup reconciliation and the monitor singleton. Discover and
// Health must bind to the guarded target SOCKS. No CLI parsing or remote work.
type managedRecoveryDeps struct {
@@ -25,6 +29,7 @@ type managedRecoveryDeps struct {
discover func(context.Context) (string, error)
health func(context.Context) (string, error)
run func(context.Context, string, ...string) error
jobTimeout time.Duration // Zero uses the production budget; tests may shorten it.
}
// Target-local adapter: reuses bounded Player requests without invoking a CLI.
@@ -55,7 +60,7 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
return target.player(bounded, args, out, diagnostics)
})
if bounded.Err() != nil {
return nil, bounded.Err()
return nil, managedRecoveryContextError(ctx, errors.Join(err, bounded.Err()))
}
return body, err
}
@@ -95,11 +100,87 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
type managedRecoveryFatal struct{ Err error }
// Only clean job expiration is restartable; safety failures retain their type.
type managedRecoveryTimeout struct{ Err error }
func (e *managedRecoveryTimeout) Error() string { return e.Err.Error() }
func (e *managedRecoveryTimeout) Unwrap() error { return e.Err }
var (
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
errManagedRecoveryDeadline = errors.New("managed recovery total deadline exceeded")
)
// Validate every leaf, not just errors.Is: a joined cleanup failure must never
// authorize a restart. Callers must separately prove the job's own deadline.
func managedRecoveryTimeoutSafe(err error) bool {
if err == nil {
return true
}
switch e := err.(type) {
case *managedRecoveryFatal:
return false
case *exec.ExitError:
if e.ProcessState == nil {
return false
}
status, ok := e.Sys().(syscall.WaitStatus)
return ok && status.Signaled() && (status.Signal() == syscall.SIGTERM || status.Signal() == syscall.SIGKILL)
case interface{ Unwrap() []error }:
children := e.Unwrap()
if len(children) == 0 {
return false
}
for _, child := range children {
if !managedRecoveryTimeoutSafe(child) {
return false
}
}
return true
case interface{ Unwrap() error }:
child := e.Unwrap()
return child != nil && managedRecoveryTimeoutSafe(child)
default:
return err == context.Canceled || err == context.DeadlineExceeded || err == errManagedRecoveryDeadline
}
}
// Cancellation must prevent late success without erasing a concurrent safety
// or cleanup failure. Request-local timeouts alone leave the job context live.
func managedRecoveryContextError(ctx context.Context, err error) error {
if ctx.Err() == nil {
return err
}
joined := errors.Join(err, ctx.Err())
if !managedRecoveryTimeoutSafe(err) {
return &managedRecoveryFatal{Err: joined}
}
return joined
}
// Synchronous callbacks own cancellation cleanup/join. Classify expiration only
// after they return, so the monitor cannot release ownership before restoration.
func managedRecoveryJob(ctx context.Context, budget time.Duration, job func(context.Context) error) error {
if budget == 0 {
budget = managedRecoveryJobTimeout
}
jobCtx, cancel := context.WithTimeoutCause(ctx, budget, errManagedRecoveryDeadline)
defer cancel()
err := job(jobCtx)
if errors.Is(context.Cause(jobCtx), errManagedRecoveryDeadline) {
joined := errors.Join(err, errManagedRecoveryDeadline, context.DeadlineExceeded)
if !managedRecoveryTimeoutSafe(err) {
return &managedRecoveryFatal{Err: joined}
}
return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: joined}}
}
if err == nil {
return jobCtx.Err()
}
return err
}
func (e *managedRecoveryFatal) Error() string {
return "managed recovery safety failure: " + e.Err.Error()
}
@@ -119,15 +200,32 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
}
m := *d.transaction
// Rollback must restart even after job cancellation; independently bounded.
var cleanupCtx context.Context
var cleanupCancel context.CancelFunc
cleanupContext := func() context.Context {
if cleanupCtx == nil {
cleanupCtx, cleanupCancel = context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
}
return cleanupCtx
}
defer func() {
if cleanupCancel != nil {
cleanupCancel()
}
}()
m.Reload = func() error {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
if err := d.run(cleanup, managedZapretScript, "restart"); err != nil {
if err := d.run(cleanupContext(), managedZapretScript, "restart"); err != nil {
return &managedRecoveryFatal{Err: err}
}
return nil
}
m.Health = func() (string, error) { return d.health(ctx) }
m.Health = func() (string, error) {
status, err := d.health(ctx)
if ctx.Err() != nil {
return "", managedRecoveryContextError(ctx, err)
}
return status, err
}
ids, err := m.ActiveCandidates(d.db)
if err != nil {
return result, &managedRecoveryFatal{Err: err}
@@ -150,9 +248,7 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
return
}
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
if e := d.run(cleanup, managedZapretScript, "start"); e != nil {
if e := d.run(cleanupContext(), managedZapretScript, "start"); e != nil {
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
}
}
@@ -180,6 +276,14 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
return nil
}
defer func() { m.Reload = cleanupReload }()
defer func() {
// A rejected candidate's rollback is complete before the next attempt;
// that later attempt needs its own restoration reserve.
if ctx.Err() == nil && cleanupCancel != nil {
cleanupCancel()
cleanupCtx, cleanupCancel = nil, nil
}
}()
e := m.Apply(d.db, ids)
if e == nil {
stopped = false
@@ -197,12 +301,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
healthy := false
if result.FrontendID != 0 {
status, e := d.health(ctx)
e = managedRecoveryContextError(ctx, e)
var fatal *managedRecoveryFatal
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
return result, e
}
if err := ctx.Err(); err != nil {
return result, err
return result, e
}
// A bounded Player request timeout is unhealthy, not job cancellation.
healthy = e == nil && status == "OK"
@@ -272,8 +377,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
}
}
host, e := d.discover(ctx)
e = managedRecoveryContextError(ctx, e)
var discoveryFatal *managedRecoveryFatal
if errors.As(e, &discoveryFatal) {
return result, e
}
if ctx.Err() != nil {
return result, ctx.Err()
return result, e
}
if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
result.GGCError = fmt.Errorf("exact GGC discovery failed")
@@ -302,12 +412,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
e = apply([]int64{result.FrontendID, cacheID})
}
if e != nil {
e = managedRecoveryContextError(ctx, e)
var fatal *managedRecoveryFatal
if errors.As(e, &fatal) {
return result, e
}
if ctx.Err() != nil {
return result, ctx.Err()
return result, e
}
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
return result, nil
@@ -441,7 +552,7 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
// are already typed managedRecoveryFatal by the transaction boundary.
return err
}
monitor.bootstrap = func(ctx context.Context) error {
bootstrap := func(ctx context.Context) error {
if err := guard(); err != nil {
return &managedRecoveryFatal{Err: err}
}
@@ -471,12 +582,18 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
}
return callback(ctx)
}
monitor.recover = callback
monitor.bootstrap = func(ctx context.Context) error {
return managedRecoveryJob(ctx, recovery.jobTimeout, bootstrap)
}
monitor.recover = func(ctx context.Context) error {
return managedRecoveryJob(ctx, recovery.jobTimeout, callback)
}
monitor.probe = func(ctx context.Context) error {
if err := guard(); err != nil {
return &managedRecoveryFatal{Err: err}
}
status, err := recovery.health(ctx)
err = managedRecoveryContextError(ctx, err)
if err != nil {
return err
}
@@ -494,8 +611,9 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
// Keep Player API health separate from GGC scope (not playback health).
// Two bounded Player requests reuse the existing health/discovery adapters.
host, err := recovery.discover(ctx)
err = managedRecoveryContextError(ctx, err)
if ctx.Err() != nil {
return ctx.Err()
return err
}
var fatal *managedRecoveryFatal
if errors.As(err, &fatal) || errors.Is(err, context.Canceled) {
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
@@ -85,6 +86,367 @@ func TestManagedRecoveryBootstrap(t *testing.T) {
}
}
func TestManagedRecoveryJobDeadline(t *testing.T) {
for _, mode := range []string{"bootstrap-scan", "bootstrap-saved", "periodic-saved", "periodic-apply", "periodic-ggc", "parent-cancel", "parent-deadline"} {
t.Run(mode, func(t *testing.T) {
bootstrap := strings.HasPrefix(mode, "bootstrap")
d, front, events := recoveryFixture(t, !bootstrap)
if mode == "bootstrap-saved" || mode == "periodic-saved" || mode == "periodic-apply" {
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
}
if err := os.Chmod(d.transaction.Dir, 0700); err != nil {
t.Fatal(err)
}
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = d.db, d.transaction.Dir
identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
d.jobTimeout = 50 * time.Millisecond
parent, cancel := context.WithCancel(context.Background())
defer cancel()
if mode == "parent-deadline" {
var deadlineCancel context.CancelFunc
parent, deadlineCancel = context.WithTimeout(parent, 50*time.Millisecond)
defer deadlineCancel()
d.jobTimeout = time.Minute
}
joined := false
var jobDeadline time.Time
observe := func(ctx context.Context) {
deadline, ok := ctx.Deadline()
if !ok || (!jobDeadline.IsZero() && !deadline.Equal(jobDeadline)) {
t.Fatal("job deadline missing or reset between phases")
}
jobDeadline = deadline
}
block := func(ctx context.Context) error {
observe(ctx)
// Model the runner contract: cancel the owned worker and join before
// returning control to restoration and the supervisor.
done := make(chan struct{})
go func() { <-ctx.Done(); close(done) }()
if mode == "parent-cancel" {
cancel()
}
<-done
joined = true
return ctx.Err()
}
baseScan, baseCheck := d.scan, d.check
d.scan = func(ctx context.Context, binary, host string, out, diagnostics io.Writer) (string, error) {
observe(ctx)
if mode == "bootstrap-scan" || mode == "parent-cancel" || mode == "parent-deadline" {
return "", block(ctx)
}
return baseScan(ctx, binary, host, out, diagnostics)
}
d.check = func(ctx context.Context, binary, path, args, host string, out, diagnostics io.Writer) error {
observe(ctx)
if mode != "periodic-apply" {
return block(ctx)
}
return baseCheck(ctx, binary, path, args, host, out, diagnostics)
}
healthCalls := 0
d.health = func(ctx context.Context) (string, error) {
observe(ctx)
healthCalls++
if mode == "periodic-apply" && healthCalls > 1 {
// Even a late OK must not commit a partially activated candidate.
_ = block(ctx)
return "OK", nil
}
if mode == "periodic-ggc" {
return "OK", nil
}
return "DEGRADED", nil
}
baseRun := d.run
d.run = func(ctx context.Context, path string, args ...string) error {
if args[0] == "start" || args[0] == "restart" && joined {
if !joined || ctx.Err() != nil {
t.Fatal("restoration preceded worker join or inherited cancellation")
}
deadline, ok := ctx.Deadline()
if !ok || time.Until(deadline) > 30*time.Second {
t.Fatal("restoration is not bounded")
}
}
return baseRun(ctx, path, args...)
}
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("deadline became ordinary GGC failure") }); err != nil {
t.Fatal(err)
}
monitor.threshold = 1
monitor.probe = func(context.Context) error { return errors.New("unhealthy") }
reports := 0
monitor.reportRecoveryError = func(error) { reports++ }
monitor.wait = func(ctx context.Context, _ time.Duration) error {
t.Fatal("expired recovery entered ordinary cooldown retry")
return ctx.Err()
}
err := runManagedMonitor(parent, monitor)
var fatal *managedRecoveryFatal
var timeout *managedRecoveryTimeout
external := strings.HasPrefix(mode, "parent")
if errors.As(err, &fatal) == external || !joined {
t.Fatalf("err=%v joined=%v", err, joined)
}
if errors.As(err, &timeout) == external {
t.Fatalf("wrong restartable timeout classification: %v", err)
}
if !external && (!errors.Is(err, errManagedRecoveryDeadline) || !errors.Is(err, context.DeadlineExceeded)) {
t.Fatal(err)
}
if external && !errors.Is(err, parent.Err()) {
t.Fatal(err)
}
want := []int64(nil)
if front != 0 {
want = []int64{front}
}
assertActiveCandidates(t, d.transaction, d.db, want) // Also proves transaction lock release.
assertMonitorUnlocked(t, monitor)
if (*events)[len(*events)-1] != "start" || reports > 1 {
t.Fatalf("events=%v reports=%d", *events, reports)
}
if !bootstrap {
state, err := readMonitorState(d.db, monitor.serviceID)
if err != nil || state.RecoveryRunning {
t.Fatalf("recovery ownership leaked: %+v %v", state, err)
}
}
})
}
}
func TestManagedRecoveryJobSafetyFailureNotTimeout(t *testing.T) {
for _, mode := range []string{"cleanup", "rollback", "database", "guard"} {
t.Run(mode, func(t *testing.T) {
unsafe := errors.New(mode + " failed")
err := managedRecoveryJob(context.Background(), time.Nanosecond, func(ctx context.Context) error {
<-ctx.Done()
return &managedRecoveryFatal{Err: errors.Join(ctx.Err(), unsafe)}
})
var fatal *managedRecoveryFatal
var timeout *managedRecoveryTimeout
if !errors.As(err, &fatal) || errors.As(err, &timeout) || !errors.Is(err, unsafe) || !errors.Is(err, errManagedRecoveryDeadline) {
t.Fatalf("unsafe deadline classified as restartable: %v", err)
}
})
}
}
func TestManagedRecoveryTimeoutLeaves(t *testing.T) {
processExit := func(command string) error {
t.Helper()
err := exec.Command("sh", "-c", command).Run()
var exit *exec.ExitError
if !errors.As(err, &exit) {
t.Fatalf("expected process exit: %v", err)
}
return err
}
closeErr := errors.New("diagnostic close failed")
for _, test := range []struct {
name string
err error
safe bool
}{
{"nil", nil, true},
{"deadline", context.DeadlineExceeded, true},
{"wrapped-deadline", fmt.Errorf("worker: %w", context.DeadlineExceeded), true},
{"canceled", context.Canceled, true},
{"term", errors.Join(context.DeadlineExceeded, processExit("kill -TERM $$")), true},
{"kill", errors.Join(context.Canceled, processExit("kill -KILL $$")), true},
{"unknown-exit", errors.Join(context.DeadlineExceeded, processExit("exit 7")), false},
{"unknown-leaf", closeErr, false},
{"joined-close", errors.Join(context.DeadlineExceeded, closeErr), false},
{"nested-close", fmt.Errorf("worker: %w", errors.Join(context.Canceled, fmt.Errorf("close: %w", closeErr))), false},
{"empty-exit", &exec.ExitError{}, false},
{"joined-fatal", errors.Join(context.DeadlineExceeded, &managedRecoveryFatal{Err: context.Canceled}), false},
} {
t.Run(test.name, func(t *testing.T) {
if got := managedRecoveryTimeoutSafe(test.err); got != test.safe {
t.Fatalf("leaf validation=%v want=%v", got, test.safe)
}
err := managedRecoveryJob(context.Background(), time.Nanosecond, func(ctx context.Context) error {
<-ctx.Done()
return test.err
})
var fatal *managedRecoveryFatal
var timeout *managedRecoveryTimeout
if !errors.As(err, &fatal) || errors.As(err, &timeout) != test.safe || !errors.Is(err, errManagedRecoveryDeadline) {
t.Fatalf("incorrect deadline classification: %v", err)
}
if test.safe && !managedRecoveryTimeoutSafe(timeout.Err) {
t.Fatal("timeout payload failed reusable validation")
}
})
}
}
func TestManagedRecoveryDiscoveryFatal(t *testing.T) {
for _, mode := range []string{"guard-fatal", "fatal-parent-cancel", "ordinary-unresolved"} {
t.Run(mode, func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
refusal := errors.New("discovery target guard refused")
d.discover = func(context.Context) (string, error) {
if mode == "ordinary-unresolved" {
return "", refusal
}
if mode == "fatal-parent-cancel" {
cancel()
return "", errors.Join(context.Canceled, &managedRecoveryFatal{Err: refusal})
}
return "", &managedRecoveryFatal{Err: refusal}
}
result, err := recoverManagedTarget(ctx, d)
var fatal *managedRecoveryFatal
if mode == "ordinary-unresolved" {
if err != nil || result.GGCError == nil {
t.Fatalf("ordinary discovery error became fatal: %+v %v", result, err)
}
} else if !errors.As(err, &fatal) || !errors.Is(err, refusal) || result.GGCError != nil {
t.Fatalf("discovery safety refusal masked: %+v %v", result, err)
}
if mode == "fatal-parent-cancel" && !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if len(*events) != 0 {
t.Fatal("discovery refusal triggered activation", *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
})
}
}
func TestManagedRecoveryHealthDeadlinePreservesFailure(t *testing.T) {
for _, mode := range []string{"fatal-guard", "mixed-close", "late-ok"} {
t.Run(mode, func(t *testing.T) {
d, prior, events := recoveryFixture(t, true)
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
priorBytes, err := d.transaction.read("config")
if err != nil {
t.Fatal(err)
}
failure := errors.New("concurrent health guard or diagnostic close failed")
calls := 0
d.health = func(ctx context.Context) (string, error) {
calls++
if calls == 1 {
return "DEGRADED", nil
}
<-ctx.Done()
switch mode {
case "fatal-guard":
return "", &managedRecoveryFatal{Err: failure}
case "mixed-close":
return "", errors.Join(ctx.Err(), failure)
default:
return "OK", nil
}
}
err = managedRecoveryJob(context.Background(), 50*time.Millisecond, func(ctx context.Context) error {
_, err := recoverManagedTarget(ctx, d)
return err
})
var fatal *managedRecoveryFatal
var timeout *managedRecoveryTimeout
if !errors.As(err, &fatal) || errors.As(err, &timeout) != (mode == "late-ok") || !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("health deadline misclassified: %v", err)
}
if mode != "late-ok" && !errors.Is(err, failure) {
t.Fatalf("original safety failure erased: %v", err)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{prior})
assertManagedContent(t, d.transaction, string(priorBytes))
if calls != 2 || strings.Count(strings.Join(*events, ","), "restart") != 2 || (*events)[len(*events)-1] != "start" {
t.Fatalf("rollback/restoration incomplete: calls=%d events=%v", calls, *events)
}
})
}
}
func TestManagedRecoveryContextError(t *testing.T) {
unsafe := errors.New("diagnostic close failed")
for _, original := range []error{nil, context.Canceled, context.DeadlineExceeded, unsafe, errors.Join(context.Canceled, unsafe), &managedRecoveryFatal{Err: unsafe}} {
ctx, cancel := context.WithCancel(context.Background())
// A live job leaves request-local errors ordinary and unchanged.
if got := managedRecoveryContextError(ctx, original); got != original {
t.Fatalf("live context changed request-local error: %v", got)
}
cancel()
got := managedRecoveryContextError(ctx, original)
var fatal *managedRecoveryFatal
if !errors.Is(got, context.Canceled) || errors.As(got, &fatal) == managedRecoveryTimeoutSafe(original) {
t.Fatalf("cancellation lost safety classification: original=%v got=%v", original, got)
}
if original != nil && !errors.Is(got, original) {
t.Fatalf("original failure lost: %v", got)
}
}
}
func TestManagedRecoveryBootstrapCleanupFailureSurvivesCancellation(t *testing.T) {
for _, mode := range []string{"deadline", "parent-cancel"} {
t.Run(mode, func(t *testing.T) {
d, _, events := recoveryFixture(t, false)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
joined := false
d.scan = func(jobCtx context.Context, _, _ string, _, _ io.Writer) (string, error) {
if mode == "parent-cancel" {
cancel()
}
<-jobCtx.Done()
joined = true
return "", jobCtx.Err()
}
unsafe := errors.New("last-good start failed")
baseRun := d.run
d.run = func(cleanup context.Context, path string, args ...string) error {
if err := baseRun(cleanup, path, args...); err != nil {
return err
}
if args[0] == "start" {
if !joined {
t.Fatal("cleanup before worker join")
}
return unsafe
}
return nil
}
monitor := monitorRuntimeFixture(t)
monitor.bootstrap = func(ctx context.Context) error {
return managedRecoveryJob(ctx, 20*time.Millisecond, func(jobCtx context.Context) error {
_, err := recoverManagedTarget(jobCtx, d)
return err
})
}
err := runManagedMonitor(ctx, monitor)
var fatal *managedRecoveryFatal
var timeout *managedRecoveryTimeout
if !errors.As(err, &fatal) || errors.As(err, &timeout) || !errors.Is(err, unsafe) {
t.Fatalf("cleanup failure masked or restartable: %v", err)
}
if mode == "deadline" && !errors.Is(err, errManagedRecoveryDeadline) || mode == "parent-cancel" && !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if !reflect.DeepEqual(*events, []string{"stop", "start"}) {
t.Fatal(*events)
}
assertActiveCandidates(t, d.transaction, d.db, nil)
assertManagedContent(t, d.transaction, managedBaseline)
assertMonitorUnlocked(t, monitor)
})
}
}
func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
d, _, events := recoveryFixture(t, false)
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
+3 -3
View File
@@ -123,13 +123,13 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
}
if d.bootstrap != nil {
if err := d.bootstrap(ctx); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
var fatal *managedRecoveryFatal
if errors.As(err, &fatal) {
return err
}
if ctx.Err() != nil {
return ctx.Err()
}
// Bootstrap scans can validly find no qualified strategy. Keep the
// supervisor alive so durable probes/cooldown govern the next attempt.
d.reportRecoveryError(err)