feat(runtime): bound recovery and add safe release supervision
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
780ca71e17
commit
b1e458a0a5
16 files changed
+941
-66
No files matched your search
@@ -21,7 +21,7 @@ func managedSeedArgs(host string) string {
|
||||
host = strings.ToLower(host)
|
||||
switch {
|
||||
case host == "www.youtube.com":
|
||||
return "--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:ip_autottl=-3,3-20:repeats=1 --lua-desync=multidisorder:pos=1,midsld"
|
||||
return "--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld"
|
||||
case host == "googlevideo.com" || strings.HasSuffix(host, ".googlevideo.com"):
|
||||
return "--payload=tls_client_hello --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN"
|
||||
default:
|
||||
|
||||
@@ -253,6 +253,13 @@ func TestRunManagedBlockcheckProtectsForeignWorkers(t *testing.T) {
|
||||
func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
args := managedSeedArgs("www.youtube.com")
|
||||
const wantSeed = "--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld"
|
||||
if args != wantSeed {
|
||||
t.Fatalf("frontend seed=%q, want %q", args, wantSeed)
|
||||
}
|
||||
if err := (&candidate{Args: args}).validateArgs(); err != nil {
|
||||
t.Fatalf("invalid frontend seed grammar: %v", err)
|
||||
}
|
||||
t.Setenv("ZAPRET_TEST_REPORT", combinedReportFixture(args))
|
||||
argvPath := filepath.Join(t.TempDir(), "argv.jsonl")
|
||||
t.Setenv("ZAPRET_TEST_ARGV", argvPath)
|
||||
@@ -279,10 +286,23 @@ func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) {
|
||||
if err := decoder.Decode(&checkArgs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(checkArgs, "--from-file") || !slices.Contains(checkArgs, "--passes") || !slices.Contains(checkArgs, "--no-conflict-cleanup") {
|
||||
wantCheck := []string{"--workers", "32", "check", "--from-file", path + ".input", "--domain", "www.youtube.com", "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", "--output", path + ".check.json"}
|
||||
if !reflect.DeepEqual(checkArgs, wantCheck) {
|
||||
t.Fatalf("managed seeded check=%q", checkArgs)
|
||||
}
|
||||
if seed := managedSeedArgs("rr3.googlevideo.com"); !strings.Contains(seed, "fakeddisorder") {
|
||||
if err := decoder.Decode(&checkArgs); err != io.EOF {
|
||||
t.Fatalf("unexpected extra blockcheck call: %q %v", checkArgs, err)
|
||||
}
|
||||
input, err := os.ReadFile(path + ".input")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantRows := "curl_test_https_tls12 ipv4 www.youtube.com : nfqws2 " + wantSeed + "\n" +
|
||||
"curl_test_https_tls13 ipv4 www.youtube.com : nfqws2 " + wantSeed + "\n"
|
||||
if string(input) != wantRows {
|
||||
t.Fatalf("managed seed input=%q", input)
|
||||
}
|
||||
if seed := managedSeedArgs("rr3.googlevideo.com"); seed != "--payload=tls_client_hello --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN" {
|
||||
t.Fatalf("missing exact-host GGC seed: %q", seed)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,36 +56,41 @@ func managedMonitorCLI(args []string, stdout, stderr io.Writer, target managedDe
|
||||
return managedMonitorCLIContext(ctx, args, stdout, stderr, target, runManagedMonitor)
|
||||
}
|
||||
|
||||
func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) int {
|
||||
func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) (code int) {
|
||||
cfg, err := managedMonitorConfig(args, target.lookup)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error: invalid managed monitor configuration")
|
||||
return 2
|
||||
return 64
|
||||
}
|
||||
if code, err := managedRuntimeGuard(target); err != nil {
|
||||
if _, err := managedRuntimeGuard(target); err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor target guard refused")
|
||||
return code
|
||||
return 78
|
||||
}
|
||||
m := &managedTransaction{Dir: target.directory, Name: "config"}
|
||||
if err := managedDirectory(target.directory); err != nil {
|
||||
fmt.Fprintln(stderr, "error: unsafe managed monitor directory")
|
||||
return 1
|
||||
return 78
|
||||
}
|
||||
if err := managedCheckIdentity(m); err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor identity refused")
|
||||
return 1
|
||||
return 78
|
||||
}
|
||||
path, err := managedMonitorDatabase(target)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error: unsafe managed monitor database")
|
||||
return 1
|
||||
return 78
|
||||
}
|
||||
db, err := openStore(path)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor database unavailable")
|
||||
return 1
|
||||
return 78
|
||||
}
|
||||
defer db.Close()
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor database close failed")
|
||||
code = 78
|
||||
}
|
||||
}()
|
||||
recovery, err := managedMonitorRecoveryAdapters(target, db, runManagedBlockcheck, runManagedCombinedBlockcheckForHost, func(ctx context.Context, path string, args ...string) error {
|
||||
log, err := os.CreateTemp(target.directory, ".daemon-diagnostics-*.log")
|
||||
if err != nil {
|
||||
@@ -98,7 +103,7 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor adapters refused")
|
||||
return 1
|
||||
return 78
|
||||
}
|
||||
monitor := managedMonitorDeps{
|
||||
target: target, db: db, serviceID: managedTargetID + "/youtube",
|
||||
@@ -110,7 +115,7 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr
|
||||
fmt.Fprintln(stderr, "managed GGC unresolved; monitoring continues (not playback health)")
|
||||
}); err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor wiring refused")
|
||||
return 1
|
||||
return 78
|
||||
}
|
||||
fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health")
|
||||
err = supervise(ctx, monitor)
|
||||
@@ -120,9 +125,34 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintln(stderr, "error: managed monitor stopped; safety/database/runtime failure (details suppressed)")
|
||||
return 1
|
||||
if managedMonitorTechnicalTimeout(err) {
|
||||
return 75
|
||||
}
|
||||
return 78
|
||||
}
|
||||
return 0
|
||||
if ctx.Err() != nil {
|
||||
return 0
|
||||
}
|
||||
return 75 // A supervisor must not return normally without cancellation.
|
||||
}
|
||||
|
||||
// Only clean typed timeouts may restart; a joined cleanup failure vetoes it.
|
||||
func managedMonitorTechnicalTimeout(err error) bool {
|
||||
if timeout, ok := err.(*managedRecoveryTimeout); ok {
|
||||
return managedRecoveryTimeoutSafe(timeout.Err)
|
||||
}
|
||||
if joined, ok := err.(interface{ Unwrap() []error }); ok {
|
||||
for _, e := range joined.Unwrap() {
|
||||
if !managedMonitorTechnicalTimeout(e) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(joined.Unwrap()) != 0
|
||||
}
|
||||
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
|
||||
return managedMonitorTechnicalTimeout(wrapped.Unwrap())
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand cacheScan, checkCommand cacheCheck, command func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
|
||||
|
||||
@@ -40,6 +40,64 @@ func TestManagedMonitorCancellationSafety(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorExitClassification(t *testing.T) {
|
||||
for _, mode := range []string{"timeout", "timeout-cleanup", "timeout-inner-cleanup", "fatal-timeout", "fatal-timeout-cleanup", "timeout-fatal", "job-timeout", "job-timeout-cleanup", "unknown", "zero", "cancel", "cancel-cleanup"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, env, _ := managedCLIFixture(t)
|
||||
initManagedCLI(t, d)
|
||||
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
want := 78
|
||||
var out bytes.Buffer
|
||||
code := managedMonitorCLIContext(ctx, nil, &out, &out, d, func(context.Context, managedMonitorDeps) error {
|
||||
switch mode {
|
||||
case "timeout":
|
||||
want = 75
|
||||
return &managedRecoveryTimeout{Err: context.DeadlineExceeded}
|
||||
case "timeout-cleanup":
|
||||
return errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("diagnostic close failed"))
|
||||
case "timeout-inner-cleanup":
|
||||
return &managedRecoveryTimeout{Err: errors.Join(context.DeadlineExceeded, errors.New("diagnostic close failed"))}
|
||||
case "fatal-timeout":
|
||||
want = 75 // Fatal is also the monitor's stop signal for clean timeout.
|
||||
return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: context.DeadlineExceeded}}
|
||||
case "fatal-timeout-cleanup":
|
||||
return &managedRecoveryFatal{Err: errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("cleanup"))}
|
||||
case "timeout-fatal":
|
||||
return &managedRecoveryTimeout{Err: &managedRecoveryFatal{Err: errors.New("cleanup")}}
|
||||
case "job-timeout", "job-timeout-cleanup":
|
||||
if mode == "job-timeout" {
|
||||
want = 75
|
||||
}
|
||||
return managedRecoveryJob(ctx, time.Millisecond, func(jobCtx context.Context) error {
|
||||
<-jobCtx.Done()
|
||||
if mode == "job-timeout-cleanup" {
|
||||
return &managedRecoveryFatal{Err: errors.Join(jobCtx.Err(), errors.New("cleanup"))}
|
||||
}
|
||||
return jobCtx.Err()
|
||||
})
|
||||
case "zero":
|
||||
want = 75
|
||||
return nil
|
||||
case "cancel":
|
||||
want = 0
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
case "cancel-cleanup":
|
||||
cancel()
|
||||
return errors.Join(ctx.Err(), errors.New("cleanup"))
|
||||
default:
|
||||
return errors.New("unknown")
|
||||
}
|
||||
})
|
||||
if code != want {
|
||||
t.Fatal(code, want, out.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorProductionAdapterBinding(t *testing.T) {
|
||||
recovery, _, _ := recoveryFixture(t, false)
|
||||
target, _, _ := managedCLIFixture(t)
|
||||
@@ -132,7 +190,11 @@ func TestManagedMonitorCLIGuardsBeforeLock(t *testing.T) {
|
||||
}
|
||||
var out bytes.Buffer
|
||||
code := managedMonitorCLIContext(context.Background(), nil, &out, &out, d, func(context.Context, managedMonitorDeps) error { t.Fatal("unsafe supervisor start"); return nil })
|
||||
if code == 0 || strings.Contains(out.String(), "password-do-not-print") {
|
||||
want := 78
|
||||
if mode == "config" {
|
||||
want = 64
|
||||
}
|
||||
if code != want || strings.Contains(out.String(), "password-do-not-print") {
|
||||
t.Fatal(code, out.String())
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(d.directory, managedMonitorLockName)); !errors.Is(err, os.ErrNotExist) {
|
||||
@@ -199,7 +261,7 @@ func TestManagedMonitorBootstrapExistingFrontend(t *testing.T) {
|
||||
// Execute only rewritten target-local contracts and fake programs; no network,
|
||||
// Docker, real credentials, zapret or host firewall. Exercise the actual traps.
|
||||
func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
for _, mode := range []string{"term", "fatal"} {
|
||||
for _, mode := range []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
config := filepath.Join(root, "ss")
|
||||
@@ -215,16 +277,23 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
}
|
||||
log := filepath.Join(root, "log")
|
||||
write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n")
|
||||
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do sleep 0.1 & wait $!; done\n"
|
||||
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & wait $!; done\n"
|
||||
write("ss-server", child)
|
||||
write("ss-local", child)
|
||||
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\n")
|
||||
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then echo managed-$2 >> \"$LOG\"; exit 0; fi\ntrap 'echo monitor-stopped >> \"$LOG\"; exit 0' TERM INT\necho managed-monitor >> \"$LOG\"\nif [ \"$MODE\" = fatal ]; then exit 7; fi\nwhile :; do sleep 0.1 & wait $!; done\n")
|
||||
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nif [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; while :; do sleep 0.1; done; fi\n")
|
||||
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then echo managed-$2 >> \"$LOG\"; exit 0; fi\ntrap 'echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; fi\necho managed-monitor >> \"$LOG\"\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & wait $!; done\n")
|
||||
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := strings.NewReplacer("/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
|
||||
if mode == "term-hang" {
|
||||
// Shorten only the fixture's watchdog, not the production contract.
|
||||
text = strings.Replace(text, `bounded_wait "$monitor_pid" 60`, `bounded_wait "$monitor_pid" 1`, 1)
|
||||
}
|
||||
if mode == "stop-hang" {
|
||||
text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1)
|
||||
}
|
||||
entry := write("entrypoint", text)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@@ -240,7 +309,7 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
exited := make(chan struct{})
|
||||
go func() { done <- cmd.Wait(); close(exited) }()
|
||||
t.Cleanup(func() { syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); <-exited })
|
||||
if mode == "term" {
|
||||
if strings.HasPrefix(mode, "term") {
|
||||
for {
|
||||
b, _ := os.ReadFile(log)
|
||||
if strings.Contains(string(b), "managed-monitor") {
|
||||
@@ -262,13 +331,20 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("entrypoint did not join children", ctx.Err())
|
||||
}
|
||||
if mode == "term" && err != nil {
|
||||
want := 75
|
||||
switch mode {
|
||||
case "term":
|
||||
want = 0
|
||||
case "fatal", "unknown", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang":
|
||||
want = 78
|
||||
}
|
||||
if want == 0 && err != nil {
|
||||
t.Fatal(err, output.String())
|
||||
}
|
||||
if mode == "fatal" {
|
||||
if want != 0 {
|
||||
var exit *exec.ExitError
|
||||
if !errors.As(err, &exit) || exit.ExitCode() != 7 {
|
||||
t.Fatal(err)
|
||||
if !errors.As(err, &exit) || exit.ExitCode() != want {
|
||||
t.Fatal(err, "want", want, output.String())
|
||||
}
|
||||
}
|
||||
b, err := os.ReadFile(log)
|
||||
|
||||
@@ -9,12 +9,16 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2"
|
||||
|
||||
const managedRecoveryJobTimeout = 90 * time.Minute
|
||||
|
||||
// Caller owns startup reconciliation and the monitor singleton. Discover and
|
||||
// Health must bind to the guarded target SOCKS. No CLI parsing or remote work.
|
||||
type managedRecoveryDeps struct {
|
||||
@@ -25,6 +29,7 @@ type managedRecoveryDeps struct {
|
||||
discover func(context.Context) (string, error)
|
||||
health func(context.Context) (string, error)
|
||||
run func(context.Context, string, ...string) error
|
||||
jobTimeout time.Duration // Zero uses the production budget; tests may shorten it.
|
||||
}
|
||||
|
||||
// Target-local adapter: reuses bounded Player requests without invoking a CLI.
|
||||
@@ -55,7 +60,7 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
|
||||
return target.player(bounded, args, out, diagnostics)
|
||||
})
|
||||
if bounded.Err() != nil {
|
||||
return nil, bounded.Err()
|
||||
return nil, managedRecoveryContextError(ctx, errors.Join(err, bounded.Err()))
|
||||
}
|
||||
return body, err
|
||||
}
|
||||
@@ -95,11 +100,87 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
|
||||
|
||||
type managedRecoveryFatal struct{ Err error }
|
||||
|
||||
// Only clean job expiration is restartable; safety failures retain their type.
|
||||
type managedRecoveryTimeout struct{ Err error }
|
||||
|
||||
func (e *managedRecoveryTimeout) Error() string { return e.Err.Error() }
|
||||
func (e *managedRecoveryTimeout) Unwrap() error { return e.Err }
|
||||
|
||||
var (
|
||||
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
|
||||
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
|
||||
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
|
||||
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
|
||||
errManagedRecoveryDeadline = errors.New("managed recovery total deadline exceeded")
|
||||
)
|
||||
|
||||
// Validate every leaf, not just errors.Is: a joined cleanup failure must never
|
||||
// authorize a restart. Callers must separately prove the job's own deadline.
|
||||
func managedRecoveryTimeoutSafe(err error) bool {
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
switch e := err.(type) {
|
||||
case *managedRecoveryFatal:
|
||||
return false
|
||||
case *exec.ExitError:
|
||||
if e.ProcessState == nil {
|
||||
return false
|
||||
}
|
||||
status, ok := e.Sys().(syscall.WaitStatus)
|
||||
return ok && status.Signaled() && (status.Signal() == syscall.SIGTERM || status.Signal() == syscall.SIGKILL)
|
||||
case interface{ Unwrap() []error }:
|
||||
children := e.Unwrap()
|
||||
if len(children) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, child := range children {
|
||||
if !managedRecoveryTimeoutSafe(child) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
case interface{ Unwrap() error }:
|
||||
child := e.Unwrap()
|
||||
return child != nil && managedRecoveryTimeoutSafe(child)
|
||||
default:
|
||||
return err == context.Canceled || err == context.DeadlineExceeded || err == errManagedRecoveryDeadline
|
||||
}
|
||||
}
|
||||
|
||||
// Cancellation must prevent late success without erasing a concurrent safety
|
||||
// or cleanup failure. Request-local timeouts alone leave the job context live.
|
||||
func managedRecoveryContextError(ctx context.Context, err error) error {
|
||||
if ctx.Err() == nil {
|
||||
return err
|
||||
}
|
||||
joined := errors.Join(err, ctx.Err())
|
||||
if !managedRecoveryTimeoutSafe(err) {
|
||||
return &managedRecoveryFatal{Err: joined}
|
||||
}
|
||||
return joined
|
||||
}
|
||||
|
||||
// Synchronous callbacks own cancellation cleanup/join. Classify expiration only
|
||||
// after they return, so the monitor cannot release ownership before restoration.
|
||||
func managedRecoveryJob(ctx context.Context, budget time.Duration, job func(context.Context) error) error {
|
||||
if budget == 0 {
|
||||
budget = managedRecoveryJobTimeout
|
||||
}
|
||||
jobCtx, cancel := context.WithTimeoutCause(ctx, budget, errManagedRecoveryDeadline)
|
||||
defer cancel()
|
||||
err := job(jobCtx)
|
||||
if errors.Is(context.Cause(jobCtx), errManagedRecoveryDeadline) {
|
||||
joined := errors.Join(err, errManagedRecoveryDeadline, context.DeadlineExceeded)
|
||||
if !managedRecoveryTimeoutSafe(err) {
|
||||
return &managedRecoveryFatal{Err: joined}
|
||||
}
|
||||
return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: joined}}
|
||||
}
|
||||
if err == nil {
|
||||
return jobCtx.Err()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *managedRecoveryFatal) Error() string {
|
||||
return "managed recovery safety failure: " + e.Err.Error()
|
||||
}
|
||||
@@ -119,15 +200,32 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
||||
}
|
||||
m := *d.transaction
|
||||
// Rollback must restart even after job cancellation; independently bounded.
|
||||
var cleanupCtx context.Context
|
||||
var cleanupCancel context.CancelFunc
|
||||
cleanupContext := func() context.Context {
|
||||
if cleanupCtx == nil {
|
||||
cleanupCtx, cleanupCancel = context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
}
|
||||
return cleanupCtx
|
||||
}
|
||||
defer func() {
|
||||
if cleanupCancel != nil {
|
||||
cleanupCancel()
|
||||
}
|
||||
}()
|
||||
m.Reload = func() error {
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := d.run(cleanup, managedZapretScript, "restart"); err != nil {
|
||||
if err := d.run(cleanupContext(), managedZapretScript, "restart"); err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
m.Health = func() (string, error) { return d.health(ctx) }
|
||||
m.Health = func() (string, error) {
|
||||
status, err := d.health(ctx)
|
||||
if ctx.Err() != nil {
|
||||
return "", managedRecoveryContextError(ctx, err)
|
||||
}
|
||||
return status, err
|
||||
}
|
||||
ids, err := m.ActiveCandidates(d.db)
|
||||
if err != nil {
|
||||
return result, &managedRecoveryFatal{Err: err}
|
||||
@@ -150,9 +248,7 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
||||
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
||||
return
|
||||
}
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cancel()
|
||||
if e := d.run(cleanup, managedZapretScript, "start"); e != nil {
|
||||
if e := d.run(cleanupContext(), managedZapretScript, "start"); e != nil {
|
||||
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
||||
}
|
||||
}
|
||||
@@ -180,6 +276,14 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
||||
return nil
|
||||
}
|
||||
defer func() { m.Reload = cleanupReload }()
|
||||
defer func() {
|
||||
// A rejected candidate's rollback is complete before the next attempt;
|
||||
// that later attempt needs its own restoration reserve.
|
||||
if ctx.Err() == nil && cleanupCancel != nil {
|
||||
cleanupCancel()
|
||||
cleanupCtx, cleanupCancel = nil, nil
|
||||
}
|
||||
}()
|
||||
e := m.Apply(d.db, ids)
|
||||
if e == nil {
|
||||
stopped = false
|
||||
@@ -197,12 +301,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
||||
healthy := false
|
||||
if result.FrontendID != 0 {
|
||||
status, e := d.health(ctx)
|
||||
e = managedRecoveryContextError(ctx, e)
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
|
||||
return result, e
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return result, err
|
||||
return result, e
|
||||
}
|
||||
// A bounded Player request timeout is unhealthy, not job cancellation.
|
||||
healthy = e == nil && status == "OK"
|
||||
@@ -272,8 +377,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
||||
}
|
||||
}
|
||||
host, e := d.discover(ctx)
|
||||
e = managedRecoveryContextError(ctx, e)
|
||||
var discoveryFatal *managedRecoveryFatal
|
||||
if errors.As(e, &discoveryFatal) {
|
||||
return result, e
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return result, ctx.Err()
|
||||
return result, e
|
||||
}
|
||||
if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
|
||||
result.GGCError = fmt.Errorf("exact GGC discovery failed")
|
||||
@@ -302,12 +412,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
||||
e = apply([]int64{result.FrontendID, cacheID})
|
||||
}
|
||||
if e != nil {
|
||||
e = managedRecoveryContextError(ctx, e)
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(e, &fatal) {
|
||||
return result, e
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return result, ctx.Err()
|
||||
return result, e
|
||||
}
|
||||
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
|
||||
return result, nil
|
||||
@@ -441,7 +552,7 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
||||
// are already typed managedRecoveryFatal by the transaction boundary.
|
||||
return err
|
||||
}
|
||||
monitor.bootstrap = func(ctx context.Context) error {
|
||||
bootstrap := func(ctx context.Context) error {
|
||||
if err := guard(); err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
@@ -471,12 +582,18 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
||||
}
|
||||
return callback(ctx)
|
||||
}
|
||||
monitor.recover = callback
|
||||
monitor.bootstrap = func(ctx context.Context) error {
|
||||
return managedRecoveryJob(ctx, recovery.jobTimeout, bootstrap)
|
||||
}
|
||||
monitor.recover = func(ctx context.Context) error {
|
||||
return managedRecoveryJob(ctx, recovery.jobTimeout, callback)
|
||||
}
|
||||
monitor.probe = func(ctx context.Context) error {
|
||||
if err := guard(); err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
status, err := recovery.health(ctx)
|
||||
err = managedRecoveryContextError(ctx, err)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -494,8 +611,9 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
||||
// Keep Player API health separate from GGC scope (not playback health).
|
||||
// Two bounded Player requests reuse the existing health/discovery adapters.
|
||||
host, err := recovery.discover(ctx)
|
||||
err = managedRecoveryContextError(ctx, err)
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
return err
|
||||
}
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(err, &fatal) || errors.Is(err, context.Canceled) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
@@ -85,6 +86,367 @@ func TestManagedRecoveryBootstrap(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryJobDeadline(t *testing.T) {
|
||||
for _, mode := range []string{"bootstrap-scan", "bootstrap-saved", "periodic-saved", "periodic-apply", "periodic-ggc", "parent-cancel", "parent-deadline"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
bootstrap := strings.HasPrefix(mode, "bootstrap")
|
||||
d, front, events := recoveryFixture(t, !bootstrap)
|
||||
if mode == "bootstrap-saved" || mode == "periodic-saved" || mode == "periodic-apply" {
|
||||
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
|
||||
}
|
||||
if err := os.Chmod(d.transaction.Dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target.directory = d.db, d.transaction.Dir
|
||||
identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||
if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.jobTimeout = 50 * time.Millisecond
|
||||
parent, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
if mode == "parent-deadline" {
|
||||
var deadlineCancel context.CancelFunc
|
||||
parent, deadlineCancel = context.WithTimeout(parent, 50*time.Millisecond)
|
||||
defer deadlineCancel()
|
||||
d.jobTimeout = time.Minute
|
||||
}
|
||||
joined := false
|
||||
var jobDeadline time.Time
|
||||
observe := func(ctx context.Context) {
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok || (!jobDeadline.IsZero() && !deadline.Equal(jobDeadline)) {
|
||||
t.Fatal("job deadline missing or reset between phases")
|
||||
}
|
||||
jobDeadline = deadline
|
||||
}
|
||||
block := func(ctx context.Context) error {
|
||||
observe(ctx)
|
||||
// Model the runner contract: cancel the owned worker and join before
|
||||
// returning control to restoration and the supervisor.
|
||||
done := make(chan struct{})
|
||||
go func() { <-ctx.Done(); close(done) }()
|
||||
if mode == "parent-cancel" {
|
||||
cancel()
|
||||
}
|
||||
<-done
|
||||
joined = true
|
||||
return ctx.Err()
|
||||
}
|
||||
baseScan, baseCheck := d.scan, d.check
|
||||
d.scan = func(ctx context.Context, binary, host string, out, diagnostics io.Writer) (string, error) {
|
||||
observe(ctx)
|
||||
if mode == "bootstrap-scan" || mode == "parent-cancel" || mode == "parent-deadline" {
|
||||
return "", block(ctx)
|
||||
}
|
||||
return baseScan(ctx, binary, host, out, diagnostics)
|
||||
}
|
||||
d.check = func(ctx context.Context, binary, path, args, host string, out, diagnostics io.Writer) error {
|
||||
observe(ctx)
|
||||
if mode != "periodic-apply" {
|
||||
return block(ctx)
|
||||
}
|
||||
return baseCheck(ctx, binary, path, args, host, out, diagnostics)
|
||||
}
|
||||
healthCalls := 0
|
||||
d.health = func(ctx context.Context) (string, error) {
|
||||
observe(ctx)
|
||||
healthCalls++
|
||||
if mode == "periodic-apply" && healthCalls > 1 {
|
||||
// Even a late OK must not commit a partially activated candidate.
|
||||
_ = block(ctx)
|
||||
return "OK", nil
|
||||
}
|
||||
if mode == "periodic-ggc" {
|
||||
return "OK", nil
|
||||
}
|
||||
return "DEGRADED", nil
|
||||
}
|
||||
baseRun := d.run
|
||||
d.run = func(ctx context.Context, path string, args ...string) error {
|
||||
if args[0] == "start" || args[0] == "restart" && joined {
|
||||
if !joined || ctx.Err() != nil {
|
||||
t.Fatal("restoration preceded worker join or inherited cancellation")
|
||||
}
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok || time.Until(deadline) > 30*time.Second {
|
||||
t.Fatal("restoration is not bounded")
|
||||
}
|
||||
}
|
||||
return baseRun(ctx, path, args...)
|
||||
}
|
||||
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("deadline became ordinary GGC failure") }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
monitor.threshold = 1
|
||||
monitor.probe = func(context.Context) error { return errors.New("unhealthy") }
|
||||
reports := 0
|
||||
monitor.reportRecoveryError = func(error) { reports++ }
|
||||
monitor.wait = func(ctx context.Context, _ time.Duration) error {
|
||||
t.Fatal("expired recovery entered ordinary cooldown retry")
|
||||
return ctx.Err()
|
||||
}
|
||||
err := runManagedMonitor(parent, monitor)
|
||||
var fatal *managedRecoveryFatal
|
||||
var timeout *managedRecoveryTimeout
|
||||
external := strings.HasPrefix(mode, "parent")
|
||||
if errors.As(err, &fatal) == external || !joined {
|
||||
t.Fatalf("err=%v joined=%v", err, joined)
|
||||
}
|
||||
if errors.As(err, &timeout) == external {
|
||||
t.Fatalf("wrong restartable timeout classification: %v", err)
|
||||
}
|
||||
if !external && (!errors.Is(err, errManagedRecoveryDeadline) || !errors.Is(err, context.DeadlineExceeded)) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if external && !errors.Is(err, parent.Err()) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []int64(nil)
|
||||
if front != 0 {
|
||||
want = []int64{front}
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, want) // Also proves transaction lock release.
|
||||
assertMonitorUnlocked(t, monitor)
|
||||
if (*events)[len(*events)-1] != "start" || reports > 1 {
|
||||
t.Fatalf("events=%v reports=%d", *events, reports)
|
||||
}
|
||||
if !bootstrap {
|
||||
state, err := readMonitorState(d.db, monitor.serviceID)
|
||||
if err != nil || state.RecoveryRunning {
|
||||
t.Fatalf("recovery ownership leaked: %+v %v", state, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryJobSafetyFailureNotTimeout(t *testing.T) {
|
||||
for _, mode := range []string{"cleanup", "rollback", "database", "guard"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
unsafe := errors.New(mode + " failed")
|
||||
err := managedRecoveryJob(context.Background(), time.Nanosecond, func(ctx context.Context) error {
|
||||
<-ctx.Done()
|
||||
return &managedRecoveryFatal{Err: errors.Join(ctx.Err(), unsafe)}
|
||||
})
|
||||
var fatal *managedRecoveryFatal
|
||||
var timeout *managedRecoveryTimeout
|
||||
if !errors.As(err, &fatal) || errors.As(err, &timeout) || !errors.Is(err, unsafe) || !errors.Is(err, errManagedRecoveryDeadline) {
|
||||
t.Fatalf("unsafe deadline classified as restartable: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryTimeoutLeaves(t *testing.T) {
|
||||
processExit := func(command string) error {
|
||||
t.Helper()
|
||||
err := exec.Command("sh", "-c", command).Run()
|
||||
var exit *exec.ExitError
|
||||
if !errors.As(err, &exit) {
|
||||
t.Fatalf("expected process exit: %v", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
closeErr := errors.New("diagnostic close failed")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
safe bool
|
||||
}{
|
||||
{"nil", nil, true},
|
||||
{"deadline", context.DeadlineExceeded, true},
|
||||
{"wrapped-deadline", fmt.Errorf("worker: %w", context.DeadlineExceeded), true},
|
||||
{"canceled", context.Canceled, true},
|
||||
{"term", errors.Join(context.DeadlineExceeded, processExit("kill -TERM $$")), true},
|
||||
{"kill", errors.Join(context.Canceled, processExit("kill -KILL $$")), true},
|
||||
{"unknown-exit", errors.Join(context.DeadlineExceeded, processExit("exit 7")), false},
|
||||
{"unknown-leaf", closeErr, false},
|
||||
{"joined-close", errors.Join(context.DeadlineExceeded, closeErr), false},
|
||||
{"nested-close", fmt.Errorf("worker: %w", errors.Join(context.Canceled, fmt.Errorf("close: %w", closeErr))), false},
|
||||
{"empty-exit", &exec.ExitError{}, false},
|
||||
{"joined-fatal", errors.Join(context.DeadlineExceeded, &managedRecoveryFatal{Err: context.Canceled}), false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := managedRecoveryTimeoutSafe(test.err); got != test.safe {
|
||||
t.Fatalf("leaf validation=%v want=%v", got, test.safe)
|
||||
}
|
||||
err := managedRecoveryJob(context.Background(), time.Nanosecond, func(ctx context.Context) error {
|
||||
<-ctx.Done()
|
||||
return test.err
|
||||
})
|
||||
var fatal *managedRecoveryFatal
|
||||
var timeout *managedRecoveryTimeout
|
||||
if !errors.As(err, &fatal) || errors.As(err, &timeout) != test.safe || !errors.Is(err, errManagedRecoveryDeadline) {
|
||||
t.Fatalf("incorrect deadline classification: %v", err)
|
||||
}
|
||||
if test.safe && !managedRecoveryTimeoutSafe(timeout.Err) {
|
||||
t.Fatal("timeout payload failed reusable validation")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryDiscoveryFatal(t *testing.T) {
|
||||
for _, mode := range []string{"guard-fatal", "fatal-parent-cancel", "ordinary-unresolved"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, front, events := recoveryFixture(t, true)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
refusal := errors.New("discovery target guard refused")
|
||||
d.discover = func(context.Context) (string, error) {
|
||||
if mode == "ordinary-unresolved" {
|
||||
return "", refusal
|
||||
}
|
||||
if mode == "fatal-parent-cancel" {
|
||||
cancel()
|
||||
return "", errors.Join(context.Canceled, &managedRecoveryFatal{Err: refusal})
|
||||
}
|
||||
return "", &managedRecoveryFatal{Err: refusal}
|
||||
}
|
||||
result, err := recoverManagedTarget(ctx, d)
|
||||
var fatal *managedRecoveryFatal
|
||||
if mode == "ordinary-unresolved" {
|
||||
if err != nil || result.GGCError == nil {
|
||||
t.Fatalf("ordinary discovery error became fatal: %+v %v", result, err)
|
||||
}
|
||||
} else if !errors.As(err, &fatal) || !errors.Is(err, refusal) || result.GGCError != nil {
|
||||
t.Fatalf("discovery safety refusal masked: %+v %v", result, err)
|
||||
}
|
||||
if mode == "fatal-parent-cancel" && !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*events) != 0 {
|
||||
t.Fatal("discovery refusal triggered activation", *events)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryHealthDeadlinePreservesFailure(t *testing.T) {
|
||||
for _, mode := range []string{"fatal-guard", "mixed-close", "late-ok"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, prior, events := recoveryFixture(t, true)
|
||||
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
|
||||
priorBytes, err := d.transaction.read("config")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failure := errors.New("concurrent health guard or diagnostic close failed")
|
||||
calls := 0
|
||||
d.health = func(ctx context.Context) (string, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return "DEGRADED", nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
switch mode {
|
||||
case "fatal-guard":
|
||||
return "", &managedRecoveryFatal{Err: failure}
|
||||
case "mixed-close":
|
||||
return "", errors.Join(ctx.Err(), failure)
|
||||
default:
|
||||
return "OK", nil
|
||||
}
|
||||
}
|
||||
err = managedRecoveryJob(context.Background(), 50*time.Millisecond, func(ctx context.Context) error {
|
||||
_, err := recoverManagedTarget(ctx, d)
|
||||
return err
|
||||
})
|
||||
var fatal *managedRecoveryFatal
|
||||
var timeout *managedRecoveryTimeout
|
||||
if !errors.As(err, &fatal) || errors.As(err, &timeout) != (mode == "late-ok") || !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("health deadline misclassified: %v", err)
|
||||
}
|
||||
if mode != "late-ok" && !errors.Is(err, failure) {
|
||||
t.Fatalf("original safety failure erased: %v", err)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{prior})
|
||||
assertManagedContent(t, d.transaction, string(priorBytes))
|
||||
if calls != 2 || strings.Count(strings.Join(*events, ","), "restart") != 2 || (*events)[len(*events)-1] != "start" {
|
||||
t.Fatalf("rollback/restoration incomplete: calls=%d events=%v", calls, *events)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryContextError(t *testing.T) {
|
||||
unsafe := errors.New("diagnostic close failed")
|
||||
for _, original := range []error{nil, context.Canceled, context.DeadlineExceeded, unsafe, errors.Join(context.Canceled, unsafe), &managedRecoveryFatal{Err: unsafe}} {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
// A live job leaves request-local errors ordinary and unchanged.
|
||||
if got := managedRecoveryContextError(ctx, original); got != original {
|
||||
t.Fatalf("live context changed request-local error: %v", got)
|
||||
}
|
||||
cancel()
|
||||
got := managedRecoveryContextError(ctx, original)
|
||||
var fatal *managedRecoveryFatal
|
||||
if !errors.Is(got, context.Canceled) || errors.As(got, &fatal) == managedRecoveryTimeoutSafe(original) {
|
||||
t.Fatalf("cancellation lost safety classification: original=%v got=%v", original, got)
|
||||
}
|
||||
if original != nil && !errors.Is(got, original) {
|
||||
t.Fatalf("original failure lost: %v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryBootstrapCleanupFailureSurvivesCancellation(t *testing.T) {
|
||||
for _, mode := range []string{"deadline", "parent-cancel"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, _, events := recoveryFixture(t, false)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
joined := false
|
||||
d.scan = func(jobCtx context.Context, _, _ string, _, _ io.Writer) (string, error) {
|
||||
if mode == "parent-cancel" {
|
||||
cancel()
|
||||
}
|
||||
<-jobCtx.Done()
|
||||
joined = true
|
||||
return "", jobCtx.Err()
|
||||
}
|
||||
unsafe := errors.New("last-good start failed")
|
||||
baseRun := d.run
|
||||
d.run = func(cleanup context.Context, path string, args ...string) error {
|
||||
if err := baseRun(cleanup, path, args...); err != nil {
|
||||
return err
|
||||
}
|
||||
if args[0] == "start" {
|
||||
if !joined {
|
||||
t.Fatal("cleanup before worker join")
|
||||
}
|
||||
return unsafe
|
||||
}
|
||||
return nil
|
||||
}
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.bootstrap = func(ctx context.Context) error {
|
||||
return managedRecoveryJob(ctx, 20*time.Millisecond, func(jobCtx context.Context) error {
|
||||
_, err := recoverManagedTarget(jobCtx, d)
|
||||
return err
|
||||
})
|
||||
}
|
||||
err := runManagedMonitor(ctx, monitor)
|
||||
var fatal *managedRecoveryFatal
|
||||
var timeout *managedRecoveryTimeout
|
||||
if !errors.As(err, &fatal) || errors.As(err, &timeout) || !errors.Is(err, unsafe) {
|
||||
t.Fatalf("cleanup failure masked or restartable: %v", err)
|
||||
}
|
||||
if mode == "deadline" && !errors.Is(err, errManagedRecoveryDeadline) || mode == "parent-cancel" && !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(*events, []string{"stop", "start"}) {
|
||||
t.Fatal(*events)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, nil)
|
||||
assertManagedContent(t, d.transaction, managedBaseline)
|
||||
assertMonitorUnlocked(t, monitor)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
|
||||
d, _, events := recoveryFixture(t, false)
|
||||
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
|
||||
|
||||
@@ -123,13 +123,13 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
||||
}
|
||||
if d.bootstrap != nil {
|
||||
if err := d.bootstrap(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(err, &fatal) {
|
||||
return err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
// Bootstrap scans can validly find no qualified strategy. Keep the
|
||||
// supervisor alive so durable probes/cooldown govern the next attempt.
|
||||
d.reportRecoveryError(err)
|
||||
|
||||
Reference in new issue
Block a user