Files
zapret-autotune/cmd/zapret-autotune/managed_recovery_test.go
T

1282 lines
46 KiB
Go

package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
func recoveryFixture(t *testing.T, active bool) (managedRecoveryDeps, int64, *[]string) {
t.Helper()
t.Setenv("TMPDIR", t.TempDir())
db := testStore(t)
m, _ := managedFixture(t)
if err := m.applyRendered([]byte(managedBaseline)); err != nil {
t.Fatal(err)
}
var front int64
if active {
front = managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
if err := m.Apply(db, []int64{front}); err != nil {
t.Fatal(err)
}
}
events := new([]string)
stopped := false
d := managedRecoveryDeps{
db: db, transaction: m,
health: func(context.Context) (string, error) { return "OK", nil },
discover: func(context.Context) (string, error) { return "fresh.googlevideo.com", nil },
run: func(ctx context.Context, path string, args ...string) error {
if path != managedZapretScript || len(args) != 1 {
t.Fatalf("unsafe command %s %v", path, args)
}
if err := ctx.Err(); err != nil {
t.Fatal("cleanup received cancelled context", err)
}
*events = append(*events, args[0])
stopped = args[0] == "stop"
return nil
},
scan: func(ctx context.Context, binary, host string, _, _ io.Writer) (string, error) {
if !stopped || binary != "blockcheckw" {
t.Fatal("scan without dedicated downtime")
}
*events = append(*events, "scan:"+host)
path := filepath.Join(t.TempDir(), "scan")
return path, os.WriteFile(path+".check.json", []byte(combinedReportFixture("--lua-desync=fake")), 0600)
},
check: func(ctx context.Context, binary, path, args, host string, _, _ io.Writer) error {
if !stopped || binary != "blockcheckw" {
t.Fatal("check without dedicated downtime")
}
*events = append(*events, "check:"+host+":"+args)
return os.WriteFile(path+".combined.json", []byte(combinedReportFixture(args)), 0600)
},
}
return d, front, events
}
func TestManagedRecoveryBootstrap(t *testing.T) {
d, _, events := recoveryFixture(t, false)
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || result.GGCError != nil || result.FrontendID == 0 || result.GooglevideoID == 0 {
t.Fatalf("%+v %v", result, err)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
frontend, err := readStrategyCandidate(d.db, result.FrontendID)
if err != nil || len(frontend.Endpoints) != 1 || frontend.Endpoints[0] != managedFrontendHost {
t.Fatalf("frontend scope=%v err=%v", frontend.Endpoints, err)
}
config, err := d.transaction.read("config")
if err != nil || !strings.Contains(string(config), "--hostlist-domains=www.youtube.com ") || !strings.Contains(string(config), "--hostlist-domains=^fresh.googlevideo.com ") || strings.Contains(string(config), "--hostlist-domains=googlevideo.com") {
t.Fatalf("%s %v", config, err)
}
if (*events)[0] != "stop" || strings.Count(strings.Join(*events, ","), "restart") != 2 {
t.Fatal(*events)
}
}
func TestManagedSavedGGCInfrastructureCannotAuthorizeFallback(t *testing.T) {
for _, artifact := range []bool{false, true} {
t.Run(fmt.Sprint("artifact=", artifact), func(t *testing.T) {
d, front, _ := recoveryFixture(t, true)
managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = d.db, d.transaction.Dir
monitor.threshold = 1
if err := os.Chmod(d.transaction.Dir, 0700); err != nil {
t.Fatal(err)
}
identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
d.scan = func(context.Context, string, string, io.Writer, io.Writer) (string, error) {
t.Fatal("infrastructure failure authorized fallback scan")
return "", nil
}
checks := 0
d.check = func(_ context.Context, _, path, _, host string, out, diagnostics io.Writer) error {
checks++
if host != "fresh.googlevideo.com" || out != io.Discard {
t.Fatal("unexpected saved check")
}
const infra = "FAIL 0/1 timeout: nfqws2 exited immediately (code 1) private-secret"
if artifact {
if err := os.WriteFile(path+".combined.json.stderr", []byte(infra), 0600); err != nil {
return err
}
} else {
io.WriteString(diagnostics, infra)
}
return os.WriteFile(path+".combined.json", nonpassingManagedReport(t), 0600)
}
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("infrastructure became unresolved GGC") }); err != nil {
t.Fatal(err)
}
monitor.probe = func(context.Context) error { return errMonitorTestUnhealthy }
monitor.reportRecoveryError = func(err error) {
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) || errors.Is(err, errNoQualifiedStrategy) {
t.Fatal(err)
}
}
err := runManagedMonitor(context.Background(), monitor)
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) || errors.Is(err, errCandidateNonpass) || strings.Contains(err.Error(), "private-secret") || checks != 1 {
t.Fatalf("checks=%d err=%v", checks, err)
}
state, stateErr := readMonitorState(d.db, monitor.serviceID)
if stateErr != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(monitor.cooldown)) {
t.Fatalf("%+v %v", state, stateErr)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
})
}
}
func nonpassingManagedReport(t *testing.T) []byte {
t.Helper()
var report struct {
Strategies []map[string]any `json:"strategies"`
}
if err := json.Unmarshal([]byte(combinedReportFixture("--lua-desync=fake")), &report); err != nil {
t.Fatal(err)
}
for _, row := range report.Strategies {
row["passes_ok"], row["success_rate"] = 0, 0
}
raw, err := json.Marshal(report)
if err != nil {
t.Fatal(err)
}
return raw
}
func TestManagedAbsenceClassification(t *testing.T) {
for _, mode := range []string{"shortlist", "combined", "malformed", "empty", "empty-valid", "empty-nfq", "empty-wrong-target", "runner"} {
t.Run(mode, func(t *testing.T) {
d, _, _ := recoveryFixture(t, false)
raw := nonpassingManagedReport(t)
d.scan = func(_ context.Context, _, host string, _, diagnostics io.Writer) (string, error) {
if mode == "runner" {
return "", errCandidateNonpass
}
data := raw
switch mode {
case "combined":
data = []byte(combinedReportFixture("--lua-desync=fake"))
case "malformed":
data = []byte(`{"strategies":[{"protocol":"tls12"}]}`)
case "empty":
data = []byte(`{"strategies":[]}`)
case "empty-valid", "empty-nfq", "empty-wrong-target":
if mode == "empty-nfq" {
fmt.Fprintln(diagnostics, "nfqws2 exited with status 1")
}
if mode == "empty-wrong-target" {
host = "untrusted.example"
}
data = []byte(fmt.Sprintf(`{"domain":%q,"total":10,"working":0,"strategies":[]}`, host))
}
path := filepath.Join(t.TempDir(), "scan")
return path, os.WriteFile(path+".check.json", data, 0600)
}
d.check = func(_ context.Context, _, path, _, _ string, _, _ io.Writer) error {
return os.WriteFile(path+".combined.json", raw, 0600)
}
_, err := recoverManagedTarget(context.Background(), d)
want := mode == "shortlist" || mode == "combined" || mode == "empty-valid"
if err == nil || errors.Is(err, errNoQualifiedStrategy) != want {
t.Fatalf("classification: %v", err)
}
var fatal *managedRecoveryFatal
if errors.As(err, &fatal) && mode != "empty-nfq" {
t.Fatalf("unexpected fatal: %v", err)
}
})
}
}
func TestManagedBootstrapAbsencePauseRestartAndExpiry(t *testing.T) {
d, _, _ := recoveryFixture(t, false)
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = d.db, d.transaction.Dir
if err := os.Chmod(d.transaction.Dir, 0700); err != nil {
t.Fatal(err)
}
identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
now := monitorTestTime
monitor.now = func() time.Time { return now }
scans := 0
d.scan = func(_ context.Context, _, _ string, _, _ io.Writer) (string, error) {
scans++
path := filepath.Join(t.TempDir(), "scan")
return path, os.WriteFile(path+".check.json", nonpassingManagedReport(t), 0600)
}
d.health = func(context.Context) (string, error) { return "UNHEALTHY", nil }
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("unexpected GGC error") }); err != nil {
t.Fatal(err)
}
monitor.reportRecoveryError = func(err error) {
if !errors.Is(err, errNoQualifiedStrategy) {
t.Fatal(err)
}
}
for phase, elapsed := range []time.Duration{0, time.Hour, 2 * time.Hour} {
now = monitorTestTime.Add(elapsed)
ctx, cancel := context.WithCancel(context.Background())
probes := 0
monitor.wait = func(ctx context.Context, _ time.Duration) error {
probes++
if probes == 4 {
cancel()
return ctx.Err()
}
return nil
}
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
cancel()
wantScans := 1
if phase == 2 {
wantScans = 2
}
state, err := readMonitorState(monitor.db, monitor.serviceID)
if err != nil || scans != wantScans || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(time.Duration(wantScans)*2*time.Hour)) {
t.Fatalf("phase=%d scans=%d state=%+v err=%v", phase, scans, state, err)
}
// Reopen the same durable database before the next monitor instance.
var dbPath string
var seq int
var name string
if err := monitor.db.QueryRow(`PRAGMA database_list`).Scan(&seq, &name, &dbPath); err != nil {
t.Fatal(err)
}
if err := monitor.db.Close(); err != nil {
t.Fatal(err)
}
reopened, err := openStore(dbPath)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { reopened.Close() })
monitor.db, d.db = reopened, reopened
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("unexpected GGC error") }); err != nil {
t.Fatal(err)
}
}
}
func TestManagedRecoveryJobDeadline(t *testing.T) {
for _, mode := range []string{"bootstrap-scan", "bootstrap-saved", "periodic-saved", "periodic-apply", "periodic-ggc", "parent-cancel", "parent-deadline"} {
t.Run(mode, func(t *testing.T) {
bootstrap := strings.HasPrefix(mode, "bootstrap")
d, front, events := recoveryFixture(t, !bootstrap)
if mode == "bootstrap-saved" || mode == "periodic-saved" || mode == "periodic-apply" {
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
}
if err := os.Chmod(d.transaction.Dir, 0700); err != nil {
t.Fatal(err)
}
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = d.db, d.transaction.Dir
identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
d.jobTimeout = 50 * time.Millisecond
parent, cancel := context.WithCancel(context.Background())
defer cancel()
if mode == "parent-deadline" {
var deadlineCancel context.CancelFunc
parent, deadlineCancel = context.WithTimeout(parent, 50*time.Millisecond)
defer deadlineCancel()
d.jobTimeout = time.Minute
}
joined := false
var jobDeadline time.Time
observe := func(ctx context.Context) {
deadline, ok := ctx.Deadline()
if !ok || (!jobDeadline.IsZero() && !deadline.Equal(jobDeadline)) {
t.Fatal("job deadline missing or reset between phases")
}
jobDeadline = deadline
}
block := func(ctx context.Context) error {
observe(ctx)
// Model the runner contract: cancel the owned worker and join before
// returning control to restoration and the supervisor.
done := make(chan struct{})
go func() { <-ctx.Done(); close(done) }()
if mode == "parent-cancel" {
cancel()
}
<-done
joined = true
return ctx.Err()
}
baseScan, baseCheck := d.scan, d.check
d.scan = func(ctx context.Context, binary, host string, out, diagnostics io.Writer) (string, error) {
observe(ctx)
if mode == "bootstrap-scan" || mode == "parent-cancel" || mode == "parent-deadline" {
return "", block(ctx)
}
return baseScan(ctx, binary, host, out, diagnostics)
}
d.check = func(ctx context.Context, binary, path, args, host string, out, diagnostics io.Writer) error {
observe(ctx)
if mode != "periodic-apply" {
return block(ctx)
}
return baseCheck(ctx, binary, path, args, host, out, diagnostics)
}
healthCalls := 0
d.health = func(ctx context.Context) (string, error) {
observe(ctx)
healthCalls++
if mode == "periodic-apply" && healthCalls > 1 {
// Even a late OK must not commit a partially activated candidate.
_ = block(ctx)
return "OK", nil
}
if mode == "periodic-ggc" {
return "OK", nil
}
return "DEGRADED", nil
}
baseRun := d.run
d.run = func(ctx context.Context, path string, args ...string) error {
if args[0] == "start" || args[0] == "restart" && joined {
if !joined || ctx.Err() != nil {
t.Fatal("restoration preceded worker join or inherited cancellation")
}
deadline, ok := ctx.Deadline()
if !ok || time.Until(deadline) > 30*time.Second {
t.Fatal("restoration is not bounded")
}
}
return baseRun(ctx, path, args...)
}
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("deadline became ordinary GGC failure") }); err != nil {
t.Fatal(err)
}
monitor.threshold = 1
monitor.probe = func(context.Context) error { return errors.New("unhealthy") }
reports := 0
monitor.reportRecoveryError = func(error) { reports++ }
monitor.wait = func(ctx context.Context, _ time.Duration) error {
t.Fatal("expired recovery entered ordinary cooldown retry")
return ctx.Err()
}
err := runManagedMonitor(parent, monitor)
var fatal *managedRecoveryFatal
var timeout *managedRecoveryTimeout
external := strings.HasPrefix(mode, "parent")
if errors.As(err, &fatal) == external || !joined {
t.Fatalf("err=%v joined=%v", err, joined)
}
if errors.As(err, &timeout) == external {
t.Fatalf("wrong restartable timeout classification: %v", err)
}
if !external && (!errors.Is(err, errManagedRecoveryDeadline) || !errors.Is(err, context.DeadlineExceeded)) {
t.Fatal(err)
}
if external && !errors.Is(err, parent.Err()) {
t.Fatal(err)
}
want := []int64(nil)
if front != 0 {
want = []int64{front}
}
assertActiveCandidates(t, d.transaction, d.db, want) // Also proves transaction lock release.
assertMonitorUnlocked(t, monitor)
if (*events)[len(*events)-1] != "start" || reports > 1 {
t.Fatalf("events=%v reports=%d", *events, reports)
}
if !bootstrap {
state, err := readMonitorState(d.db, monitor.serviceID)
if err != nil || state.RecoveryRunning {
t.Fatalf("recovery ownership leaked: %+v %v", state, err)
}
}
})
}
}
func TestManagedRecoveryJobSafetyFailureNotTimeout(t *testing.T) {
for _, mode := range []string{"cleanup", "rollback", "database", "guard"} {
t.Run(mode, func(t *testing.T) {
unsafe := errors.New(mode + " failed")
err := managedRecoveryJob(context.Background(), time.Nanosecond, func(ctx context.Context) error {
<-ctx.Done()
return &managedRecoveryFatal{Err: errors.Join(ctx.Err(), unsafe)}
})
var fatal *managedRecoveryFatal
var timeout *managedRecoveryTimeout
if !errors.As(err, &fatal) || errors.As(err, &timeout) || !errors.Is(err, unsafe) || !errors.Is(err, errManagedRecoveryDeadline) {
t.Fatalf("unsafe deadline classified as restartable: %v", err)
}
})
}
}
func TestManagedRecoveryTimeoutLeaves(t *testing.T) {
processExit := func(command string) error {
t.Helper()
err := exec.Command("sh", "-c", command).Run()
var exit *exec.ExitError
if !errors.As(err, &exit) {
t.Fatalf("expected process exit: %v", err)
}
return err
}
closeErr := errors.New("diagnostic close failed")
for _, test := range []struct {
name string
err error
safe bool
}{
{"nil", nil, true},
{"deadline", context.DeadlineExceeded, true},
{"wrapped-deadline", fmt.Errorf("worker: %w", context.DeadlineExceeded), true},
{"canceled", context.Canceled, true},
{"term", errors.Join(context.DeadlineExceeded, processExit("kill -TERM $$")), true},
{"kill", errors.Join(context.Canceled, processExit("kill -KILL $$")), true},
{"unknown-exit", errors.Join(context.DeadlineExceeded, processExit("exit 7")), false},
{"unknown-leaf", closeErr, false},
{"joined-close", errors.Join(context.DeadlineExceeded, closeErr), false},
{"nested-close", fmt.Errorf("worker: %w", errors.Join(context.Canceled, fmt.Errorf("close: %w", closeErr))), false},
{"empty-exit", &exec.ExitError{}, false},
{"joined-fatal", errors.Join(context.DeadlineExceeded, &managedRecoveryFatal{Err: context.Canceled}), false},
} {
t.Run(test.name, func(t *testing.T) {
if got := managedRecoveryTimeoutSafe(test.err); got != test.safe {
t.Fatalf("leaf validation=%v want=%v", got, test.safe)
}
err := managedRecoveryJob(context.Background(), time.Nanosecond, func(ctx context.Context) error {
<-ctx.Done()
return test.err
})
var fatal *managedRecoveryFatal
var timeout *managedRecoveryTimeout
if !errors.As(err, &fatal) || errors.As(err, &timeout) != test.safe || !errors.Is(err, errManagedRecoveryDeadline) {
t.Fatalf("incorrect deadline classification: %v", err)
}
if test.safe && !managedRecoveryTimeoutSafe(timeout.Err) {
t.Fatal("timeout payload failed reusable validation")
}
})
}
}
func TestManagedRecoveryDiscoveryFatal(t *testing.T) {
for _, mode := range []string{"guard-fatal", "fatal-parent-cancel", "ordinary-unresolved"} {
t.Run(mode, func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
refusal := errors.New("discovery target guard refused")
d.discover = func(context.Context) (string, error) {
if mode == "ordinary-unresolved" {
return "", refusal
}
if mode == "fatal-parent-cancel" {
cancel()
return "", errors.Join(context.Canceled, &managedRecoveryFatal{Err: refusal})
}
return "", &managedRecoveryFatal{Err: refusal}
}
result, err := recoverManagedTarget(ctx, d)
var fatal *managedRecoveryFatal
if mode == "ordinary-unresolved" {
if err != nil || result.GGCError == nil {
t.Fatalf("ordinary discovery error became fatal: %+v %v", result, err)
}
} else if !errors.As(err, &fatal) || !errors.Is(err, refusal) || result.GGCError != nil {
t.Fatalf("discovery safety refusal masked: %+v %v", result, err)
}
if mode == "fatal-parent-cancel" && !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if len(*events) != 0 {
t.Fatal("discovery refusal triggered activation", *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
})
}
}
func TestManagedRecoveryHealthDeadlinePreservesFailure(t *testing.T) {
for _, mode := range []string{"fatal-guard", "mixed-close", "late-ok"} {
t.Run(mode, func(t *testing.T) {
d, prior, events := recoveryFixture(t, true)
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
priorBytes, err := d.transaction.read("config")
if err != nil {
t.Fatal(err)
}
failure := errors.New("concurrent health guard or diagnostic close failed")
calls := 0
d.health = func(ctx context.Context) (string, error) {
calls++
if calls == 1 {
return "DEGRADED", nil
}
<-ctx.Done()
switch mode {
case "fatal-guard":
return "", &managedRecoveryFatal{Err: failure}
case "mixed-close":
return "", errors.Join(ctx.Err(), failure)
default:
return "OK", nil
}
}
err = managedRecoveryJob(context.Background(), 50*time.Millisecond, func(ctx context.Context) error {
_, err := recoverManagedTarget(ctx, d)
return err
})
var fatal *managedRecoveryFatal
var timeout *managedRecoveryTimeout
if !errors.As(err, &fatal) || errors.As(err, &timeout) != (mode == "late-ok") || !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("health deadline misclassified: %v", err)
}
if mode != "late-ok" && !errors.Is(err, failure) {
t.Fatalf("original safety failure erased: %v", err)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{prior})
assertManagedContent(t, d.transaction, string(priorBytes))
if calls != 2 || strings.Count(strings.Join(*events, ","), "restart") != 2 || (*events)[len(*events)-1] != "start" {
t.Fatalf("rollback/restoration incomplete: calls=%d events=%v", calls, *events)
}
})
}
}
func TestManagedRecoveryContextError(t *testing.T) {
unsafe := errors.New("diagnostic close failed")
for _, original := range []error{nil, context.Canceled, context.DeadlineExceeded, unsafe, errors.Join(context.Canceled, unsafe), &managedRecoveryFatal{Err: unsafe}} {
ctx, cancel := context.WithCancel(context.Background())
// A live job leaves request-local errors ordinary and unchanged.
if got := managedRecoveryContextError(ctx, original); got != original {
t.Fatalf("live context changed request-local error: %v", got)
}
cancel()
got := managedRecoveryContextError(ctx, original)
var fatal *managedRecoveryFatal
if !errors.Is(got, context.Canceled) || errors.As(got, &fatal) == managedRecoveryTimeoutSafe(original) {
t.Fatalf("cancellation lost safety classification: original=%v got=%v", original, got)
}
if original != nil && !errors.Is(got, original) {
t.Fatalf("original failure lost: %v", got)
}
}
}
func TestManagedRecoveryBootstrapCleanupFailureSurvivesCancellation(t *testing.T) {
for _, mode := range []string{"deadline", "parent-cancel"} {
t.Run(mode, func(t *testing.T) {
d, _, events := recoveryFixture(t, false)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
joined := false
d.scan = func(jobCtx context.Context, _, _ string, _, _ io.Writer) (string, error) {
if mode == "parent-cancel" {
cancel()
}
<-jobCtx.Done()
joined = true
return "", jobCtx.Err()
}
unsafe := errors.New("last-good start failed")
baseRun := d.run
d.run = func(cleanup context.Context, path string, args ...string) error {
if err := baseRun(cleanup, path, args...); err != nil {
return err
}
if args[0] == "start" {
if !joined {
t.Fatal("cleanup before worker join")
}
return unsafe
}
return nil
}
monitor := monitorRuntimeFixture(t)
monitor.bootstrap = func(ctx context.Context) error {
return managedRecoveryJob(ctx, 20*time.Millisecond, func(jobCtx context.Context) error {
_, err := recoverManagedTarget(jobCtx, d)
return err
})
}
err := runManagedMonitor(ctx, monitor)
var fatal *managedRecoveryFatal
var timeout *managedRecoveryTimeout
if !errors.As(err, &fatal) || errors.As(err, &timeout) || !errors.Is(err, unsafe) {
t.Fatalf("cleanup failure masked or restartable: %v", err)
}
if mode == "deadline" && !errors.Is(err, errManagedRecoveryDeadline) || mode == "parent-cancel" && !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if !reflect.DeepEqual(*events, []string{"stop", "start"}) {
t.Fatal(*events)
}
assertActiveCandidates(t, d.transaction, d.db, nil)
assertManagedContent(t, d.transaction, managedBaseline)
assertMonitorUnlocked(t, monitor)
})
}
}
func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
d, _, events := recoveryFixture(t, false)
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || result.GGCError != nil || result.FrontendID != savedID {
t.Fatalf("result=%+v err=%v", result, err)
}
for _, event := range *events {
if event == "scan:"+managedFrontendHost {
t.Fatalf("frontend scan ran despite a passing saved candidate: %v", *events)
}
}
if !strings.Contains(strings.Join(*events, ","), "check:"+managedFrontendHost+":--lua-desync=saved") {
t.Fatalf("saved candidate was not revalidated: %v", *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
}
func TestManagedRecoveryHealthTransportErrorReusesSavedFrontend(t *testing.T) {
for _, healthErr := range []error{errors.New("Player transport failed"), context.DeadlineExceeded} {
t.Run(healthErr.Error(), func(t *testing.T) {
d, _, events := recoveryFixture(t, true)
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
probes := 0
d.health = func(context.Context) (string, error) {
probes++
if probes == 1 {
return "", healthErr
}
return "OK", nil
}
d.discover = func(context.Context) (string, error) { return "", errors.New("unresolved") }
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || result.FrontendID != savedID || probes != 2 {
t.Fatalf("result=%+v err=%v probes=%d", result, err, probes)
}
want := []string{"stop", "check:" + managedFrontendHost + ":--lua-desync=saved", "restart"}
if !reflect.DeepEqual(*events, want) {
t.Fatalf("saved-first recovery events=%v want=%v", *events, want)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{savedID})
})
}
}
func TestManagedRecoveryInitialHealthFailsClosed(t *testing.T) {
for _, mode := range []string{"fatal", "canceled", "context-canceled"} {
t.Run(mode, func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var healthErr error = context.Canceled
if mode == "fatal" {
healthErr = &managedRecoveryFatal{Err: errors.New("unsafe target")}
}
d.health = func(context.Context) (string, error) {
if mode == "context-canceled" {
cancel()
return "OK", nil
}
return "", healthErr
}
_, err := recoverManagedTarget(ctx, d)
if !errors.Is(err, healthErr) || len(*events) != 0 {
t.Fatalf("err=%v events=%v", err, *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
})
}
}
func TestManagedRecoverySavedFirstAndFailClosed(t *testing.T) {
for _, mode := range []string{"pass", "nonpass", "malformed", "runtime", "checker-nonpass", "missing-report", "incomplete", "health-reject", "apply-infrastructure", "cancel", "cleanup-failure"} {
t.Run(mode, func(t *testing.T) {
d, old, events := recoveryFixture(t, true)
newID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=multisplit", true)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probes := 0
d.health = func(context.Context) (string, error) {
probes++
if mode == "apply-infrastructure" && probes == 2 {
return "", errors.New("probe infrastructure failed")
}
if probes == 1 || mode == "health-reject" && probes == 2 {
return "DEGRADED", nil
}
return "OK", nil
}
d.discover = func(context.Context) (string, error) { return "", errors.New("unresolved") }
checked := []string{}
baseCheck := d.check
d.check = func(ctx context.Context, binary, path, args, host string, out, stderr io.Writer) error {
checked = append(checked, args)
if mode == "checker-nonpass" {
return errCandidateNonpass
}
if mode == "runtime" || mode == "cleanup-failure" {
return errors.New("runner failed")
}
if mode == "cancel" {
cancel()
return ctx.Err()
}
if mode == "missing-report" {
return nil
}
if err := baseCheck(ctx, binary, path, args, host, out, stderr); err != nil {
return err
}
report := combinedReportFixture(args)
switch mode {
case "malformed":
report = `{}`
case "incomplete":
report = strings.ReplaceAll(report, "HTTPS/TLS1.3", "HTTPS/TLS1.2")
case "nonpass":
if !strings.Contains(path, "/scan") {
report = strings.Replace(report, `"passes_ok":3`, `"passes_ok":2`, 1)
}
}
return os.WriteFile(path+".combined.json", []byte(report), 0600)
}
if mode == "cleanup-failure" {
run := d.run
d.run = func(ctx context.Context, path string, args ...string) error {
if err := run(ctx, path, args...); err != nil {
return err
}
if args[0] == "start" {
return errors.New("cleanup failed")
}
return nil
}
}
result, err := recoverManagedTarget(ctx, d)
success := mode == "pass" || mode == "nonpass" || mode == "health-reject"
if (err == nil) != success {
t.Fatalf("result=%+v err=%v events=%v", result, err, *events)
}
if len(checked) == 0 || checked[0] != "--lua-desync=multisplit" {
t.Fatal(checked)
}
scanned := strings.Contains(strings.Join(*events, ","), "scan:")
if scanned != (mode == "nonpass") {
t.Fatal(*events)
}
want := old
if mode == "pass" {
want = newID
}
if mode == "nonpass" {
want = result.FrontendID
if want == old || want == newID {
t.Fatal("scan did not save fresh candidate")
}
}
assertActiveCandidates(t, d.transaction, d.db, []int64{want})
if !success && (*events)[len(*events)-1] != "start" {
t.Fatal("no cleanup", *events)
}
if mode == "health-reject" && (len(checked) != 2 || strings.Count(strings.Join(*events, ","), "restart") != 3) {
t.Fatal("health rejection did not rollback and try older", *events, checked)
}
if mode == "cleanup-failure" {
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) {
t.Fatal(err)
}
}
})
}
}
func TestManagedRecoveryScanArtifactsFailClosed(t *testing.T) {
for _, mode := range []string{"stale-combined", "remove-error", "missing-shortlist", "stale-shortlist", "malformed-combined", "cancel"} {
t.Run(mode, func(t *testing.T) {
d, _, events := recoveryFixture(t, false)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
path := filepath.Join(t.TempDir(), "scan")
d.scan = func(context.Context, string, string, io.Writer, io.Writer) (string, error) {
if mode != "missing-shortlist" {
if err := os.WriteFile(path+".check.json", []byte(combinedReportFixture("--lua-desync=fake")), 0600); err != nil {
t.Fatal(err)
}
}
if mode == "stale-shortlist" {
old := time.Now().Add(-time.Hour)
if err := os.Chtimes(path+".check.json", old, old); err != nil {
t.Fatal(err)
}
}
if mode == "remove-error" {
if err := os.Mkdir(path+".combined.json", 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(path+".combined.json", "keep"), nil, 0600); err != nil {
t.Fatal(err)
}
} else if err := os.WriteFile(path+".combined.json", []byte(combinedReportFixture("--lua-desync=fake")), 0600); err != nil {
t.Fatal(err)
}
if mode == "cancel" {
cancel()
}
return path, nil
}
checked := false
d.check = func(context.Context, string, string, string, string, io.Writer, io.Writer) error {
checked = true
if _, err := os.Stat(path + ".combined.json"); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("old combined artifact survived: %v", err)
}
if mode == "malformed-combined" {
return os.WriteFile(path+".combined.json", []byte(`{}`), 0600)
}
return nil // No fresh report: old passing evidence must not qualify.
}
result, err := recoverManagedTarget(ctx, d)
if err == nil || result.FrontendID != 0 || result.GooglevideoID != 0 {
t.Fatalf("artifact qualified: %+v %v", result, err)
}
wantCheck := mode == "stale-combined" || mode == "malformed-combined"
if checked != wantCheck {
t.Fatalf("checked=%v want=%v", checked, wantCheck)
}
if mode == "cancel" && !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if !reflect.DeepEqual(*events, []string{"stop", "start"}) {
t.Fatalf("target not restored: %v", *events)
}
assertActiveCandidates(t, d.transaction, d.db, nil)
assertManagedContent(t, d.transaction, managedBaseline)
ids, e := qualifiedCandidateIDs(d.db, "frontend")
if e != nil || len(ids) != 0 {
t.Fatalf("failed scan saved candidate: %v %v", ids, e)
}
})
}
}
func TestManagedRecoveryDiscoveryDropsOnlyOldGGC(t *testing.T) {
d, front, events := recoveryFixture(t, true)
cache := managedCandidateFixture(t, d.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
if err := d.transaction.Apply(d.db, []int64{front, cache}); err != nil {
t.Fatal(err)
}
d.discover = func(context.Context) (string, error) { return "", errors.New("discovery failed") }
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || result.GGCError == nil || result.FrontendID != front || result.GooglevideoID != 0 {
t.Fatalf("%+v %v", result, err)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
if !reflect.DeepEqual(*events, []string{"restart"}) {
t.Fatal(*events)
}
old, err := readStrategyCandidate(d.db, cache)
if err != nil || !old.Qualified || !reflect.DeepEqual(old.Endpoints, []string{"old.googlevideo.com"}) {
t.Fatal("immutable old GGC changed", old, err)
}
}
func TestManagedRecoveryRollbackFailureNeverTriesNext(t *testing.T) {
d, front, events := recoveryFixture(t, true)
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=multisplit", true)
d.health = func(context.Context) (string, error) { return "DEGRADED", nil }
run := d.run
restarts := 0
d.run = func(ctx context.Context, path string, args ...string) error {
if err := run(ctx, path, args...); err != nil {
return err
}
if args[0] == "restart" {
restarts++
if restarts == 2 {
return errors.New("rollback restart failed")
}
}
return nil
}
_, err := recoverManagedTarget(context.Background(), d)
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) {
t.Fatalf("rollback failure not fatal: %v", err)
}
joined := strings.Join(*events, ",")
if strings.Contains(joined, "scan:") || strings.Count(joined, "check:") != 1 {
t.Fatalf("rollback failure authorized another candidate: %v", *events)
}
if (*events)[len(*events)-1] != "start" {
t.Fatalf("reconciliation did not restore target: %v", *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
}
func TestManagedRecoveryFreshGGCAndPartialFailure(t *testing.T) {
for _, mode := range []string{"pass", "check-error", "health-reject", "discovery-error", "wildcard", "cancel"} {
t.Run(mode, func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
oldCache := managedCandidateFixture(t, d.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if mode == "discovery-error" {
d.discover = func(context.Context) (string, error) { return "", errors.New("secret URL") }
}
if mode == "wildcard" {
d.discover = func(context.Context) (string, error) { return "googlevideo.com", nil }
}
check := d.check
d.check = func(ctx context.Context, binary, path, args, host string, out, stderr io.Writer) error {
if host != "fresh.googlevideo.com" {
t.Fatal("old or wildcard scope", host)
}
if mode == "check-error" {
return errors.New("infrastructure")
}
if mode == "cancel" {
cancel()
return ctx.Err()
}
return check(ctx, binary, path, args, host, out, stderr)
}
probes := 0
d.health = func(context.Context) (string, error) {
probes++
if mode == "health-reject" && probes > 1 {
return "DEGRADED", nil
}
return "OK", nil
}
result, err := recoverManagedTarget(ctx, d)
if mode == "cancel" {
if !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
} else if err != nil {
t.Fatal(err)
}
if result.FrontendID != front {
t.Fatalf("%+v %v", result, err)
}
if strings.Contains(strings.Join(*events, ","), "scan:") {
t.Fatal("passing frontend/cache caused scan", *events)
}
want := []int64{front}
if mode == "pass" {
if result.GGCError != nil || result.GooglevideoID <= oldCache {
t.Fatalf("%+v", result)
}
want = append(want, result.GooglevideoID)
fresh, e := readStrategyCandidate(d.db, result.GooglevideoID)
old, oldErr := readStrategyCandidate(d.db, oldCache)
if e != nil || oldErr != nil || !reflect.DeepEqual(fresh.Endpoints, []string{"fresh.googlevideo.com"}) || !reflect.DeepEqual(old.Endpoints, []string{"old.googlevideo.com"}) {
t.Fatal(fresh, old, e, oldErr)
}
} else if mode != "cancel" && result.GGCError == nil {
t.Fatal("cache failure claimed success")
}
assertActiveCandidates(t, d.transaction, d.db, want)
})
}
}
func TestManagedQualifiedCandidateBound(t *testing.T) {
db := testStore(t)
var last int64
for i := 0; i < 12; i++ {
last = managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
}
managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", false)
ids, err := qualifiedCandidateIDs(db, "frontend")
if err != nil || len(ids) != 10 || ids[0] != last || ids[9] != last-9 {
t.Fatal(ids, err)
}
}
func TestManagedRecoveryMonitorCallbacks(t *testing.T) {
recovery, _, _ := recoveryFixture(t, false)
monitor := monitorRuntimeFixture(t)
monitor.db = recovery.db
monitor.target.directory = recovery.transaction.Dir
identity, err := json.Marshal(managedIdentity{Role: "managed-test", Directory: recovery.transaction.Dir, ID: strings.Repeat("01", 16)})
if err != nil {
t.Fatal(err)
}
if err := recovery.transaction.replace(managedMarkerName, identity); err != nil {
t.Fatal(err)
}
var reports []error
if err := wireManagedRecovery(&monitor, recovery, func(err error) { reports = append(reports, err) }); err != nil {
t.Fatal(err)
}
if err := monitor.bootstrap(context.Background()); err != nil {
t.Fatal(err)
}
if err := monitor.probe(context.Background()); err != nil {
t.Fatal(err)
}
if len(reports) != 0 {
t.Fatal(reports)
}
ids, err := recovery.transaction.ActiveCandidates(recovery.db)
if err != nil || len(ids) != 2 {
t.Fatal(ids, err)
}
// A drifted deployment is never adopted by startup/recovery callbacks.
if err := os.WriteFile(filepath.Join(recovery.transaction.Dir, "config"), []byte("external"), 0600); err != nil {
t.Fatal(err)
}
if err := monitor.recover(context.Background()); !errors.Is(err, errManagedDrift) {
t.Fatal(err)
}
}
func TestManagedRecoveryRuntimeFailures(t *testing.T) {
for _, mode := range []string{"stop", "restart", "health-infrastructure"} {
t.Run(mode, func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
d.health = func(context.Context) (string, error) {
if mode == "health-infrastructure" {
return "", errors.New("request infrastructure failed")
}
return "DEGRADED", nil
}
run := d.run
d.run = func(ctx context.Context, path string, args ...string) error {
if err := run(ctx, path, args...); err != nil {
return err
}
if args[0] == mode {
return errors.New("runtime failed")
}
return nil
}
_, err := recoverManagedTarget(context.Background(), d)
if err == nil {
t.Fatal("runtime failure accepted")
}
if strings.Contains(strings.Join(*events, ","), "scan:") {
t.Fatal("infrastructure triggered scan", *events)
}
if mode == "restart" {
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) {
t.Fatal(err)
}
// Failed rollback restart keeps a pending journal; do not start it.
if ids, err := d.transaction.ActiveCandidates(d.db); err == nil || len(ids) != 0 {
t.Fatal(ids, err)
}
} else {
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
if mode == "stop" && (*events)[len(*events)-1] != "start" {
t.Fatal(*events)
}
}
})
}
}
func TestManagedRecoveryPlayerAdapter(t *testing.T) {
target, _, _ := managedCLIFixture(t)
recovery, _, _ := recoveryFixture(t, false)
d, err := managedRecoveryDependencies(target, recovery.db, recovery.scan, recovery.check, recovery.run)
if err != nil {
t.Fatal(err)
}
status, err := d.health(context.Background())
if status != "OK" || err != nil {
t.Fatal(status, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := d.health(ctx); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if _, err := d.discover(context.Background()); err == nil {
t.Fatal("health-only response claimed GGC discovery")
}
if d.transaction.Dir != target.directory || d.transaction.Name != "config" {
t.Fatal("wrong target binding")
}
}
func TestManagedRecoveryProbeGGCScope(t *testing.T) {
for _, mode := range []string{"matching", "changed", "missing", "wildcard", "uppercase", "url", "foreign", "empty", "discovery-error", "fatal", "cancel", "drift", "identity", "health-reject"} {
t.Run(mode, func(t *testing.T) {
recovery, front, events := recoveryFixture(t, true)
if mode != "missing" {
cache := managedCandidateFixture(t, recovery.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=multisplit", true)
if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil {
t.Fatal(err)
}
}
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
discoveries := 0
recovery.discover = func(context.Context) (string, error) {
discoveries++
switch mode {
case "changed":
return "other.googlevideo.com", nil
case "wildcard":
return "googlevideo.com", nil
case "uppercase":
return "FRESH.googlevideo.com", nil
case "url":
return "https://fresh.googlevideo.com/video?secret=never-print", nil
case "foreign":
return "fresh.googlevideo.com.evil.example", nil
case "empty":
return "", nil
case "discovery-error":
return "", errors.New("secret=never-print")
case "fatal":
return "", &managedRecoveryFatal{Err: errors.New("unsafe target")}
case "cancel":
cancel()
}
return "fresh.googlevideo.com", nil
}
if mode == "health-reject" {
recovery.health = func(context.Context) (string, error) { return "DEGRADED", nil }
}
if err := wireManagedRecovery(&monitor, recovery, func(error) { t.Fatal("probe invoked recovery report") }); err != nil {
t.Fatal(err)
}
if mode == "drift" {
if err := os.WriteFile(filepath.Join(recovery.transaction.Dir, "config"), []byte("external"), 0600); err != nil {
t.Fatal(err)
}
}
if mode == "identity" {
if err := os.Remove(filepath.Join(recovery.transaction.Dir, managedMarkerName)); err != nil {
t.Fatal(err)
}
}
err := monitor.probe(ctx)
switch mode {
case "matching":
if err != nil {
t.Fatal(err)
}
case "changed", "missing":
if !errors.Is(err, errManagedGGCScopeChanged) {
t.Fatal(err)
}
case "fatal", "drift", "identity":
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) {
t.Fatal(err)
}
case "cancel":
if !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
case "health-reject":
var rejection *managedHealthRejection
if !errors.As(err, &rejection) {
t.Fatal(err)
}
default:
if !errors.Is(err, errManagedGGCUnresolved) || strings.Contains(err.Error(), "never-print") {
t.Fatal(err)
}
}
if len(*events) != 0 || (mode == "drift" || mode == "identity" || mode == "health-reject") && discoveries != 0 {
t.Fatalf("unsafe probe effects: events=%v discoveries=%d", *events, discoveries)
}
})
}
}
func TestManagedRecoveryGGCRotationPolicy(t *testing.T) {
for _, activeCache := range []bool{false, true} {
t.Run(fmt.Sprint(activeCache), func(t *testing.T) {
recovery, front, events := recoveryFixture(t, true)
if err := os.Chmod(recovery.transaction.Dir, 0700); err != nil {
t.Fatal(err)
}
if activeCache {
cache := managedCandidateFixture(t, recovery.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil {
t.Fatal(err)
}
}
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
if err := wireManagedRecovery(&monitor, recovery, func(err error) { t.Fatal(err) }); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probes := 0
probe := monitor.probe
monitor.probe = func(ctx context.Context) error { probes++; return probe(ctx) }
monitor.wait = func(ctx context.Context, _ time.Duration) error {
if probes < monitor.threshold && len(*events) != 0 {
t.Fatal("recovered before threshold", *events)
}
if probes == monitor.threshold+1 {
cancel()
return ctx.Err()
}
return nil
}
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
ids, err := recovery.transaction.ActiveCandidates(recovery.db)
if err != nil || len(ids) != 2 || ids[0] != front {
t.Fatal(ids, err)
}
cache, err := readStrategyCandidate(recovery.db, ids[1])
if err != nil || !reflect.DeepEqual(cache.Endpoints, []string{"fresh.googlevideo.com"}) {
t.Fatal(cache, err)
}
state, err := readMonitorState(monitor.db, monitor.serviceID)
if err != nil || state.CooldownUntil.IsZero() || strings.Contains(strings.Join(*events, ","), "scan:"+managedFrontendHost) {
t.Fatal(state, err, *events)
}
})
}
}