Files
zapret-autotune/cmd/zapret-autotune/managed_recovery_test.go
T
2026-10-10 16:37:59 +03:00

724 lines
25 KiB
Go

package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
func recoveryFixture(t *testing.T, active bool) (managedRecoveryDeps, int64, *[]string) {
t.Helper()
t.Setenv("TMPDIR", t.TempDir())
db := testStore(t)
m, _ := managedFixture(t)
if err := m.applyRendered([]byte(managedBaseline)); err != nil {
t.Fatal(err)
}
var front int64
if active {
front = managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
if err := m.Apply(db, []int64{front}); err != nil {
t.Fatal(err)
}
}
events := new([]string)
stopped := false
d := managedRecoveryDeps{
db: db, transaction: m,
health: func(context.Context) (string, error) { return "OK", nil },
discover: func(context.Context) (string, error) { return "fresh.googlevideo.com", nil },
run: func(ctx context.Context, path string, args ...string) error {
if path != managedZapretScript || len(args) != 1 {
t.Fatalf("unsafe command %s %v", path, args)
}
if err := ctx.Err(); err != nil {
t.Fatal("cleanup received cancelled context", err)
}
*events = append(*events, args[0])
stopped = args[0] == "stop"
return nil
},
scan: func(ctx context.Context, binary, host string, _, _ io.Writer) (string, error) {
if !stopped || binary != "blockcheckw" {
t.Fatal("scan without dedicated downtime")
}
*events = append(*events, "scan:"+host)
path := filepath.Join(t.TempDir(), "scan")
return path, os.WriteFile(path+".check.json", []byte(combinedReportFixture("--lua-desync=fake")), 0600)
},
check: func(ctx context.Context, binary, path, args, host string, _, _ io.Writer) error {
if !stopped || binary != "blockcheckw" {
t.Fatal("check without dedicated downtime")
}
*events = append(*events, "check:"+host+":"+args)
return os.WriteFile(path+".combined.json", []byte(combinedReportFixture(args)), 0600)
},
}
return d, front, events
}
func TestManagedRecoveryBootstrap(t *testing.T) {
d, _, events := recoveryFixture(t, false)
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || result.GGCError != nil || result.FrontendID == 0 || result.GooglevideoID == 0 {
t.Fatalf("%+v %v", result, err)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
frontend, err := readStrategyCandidate(d.db, result.FrontendID)
if err != nil || len(frontend.Endpoints) != 1 || frontend.Endpoints[0] != managedFrontendHost {
t.Fatalf("frontend scope=%v err=%v", frontend.Endpoints, err)
}
config, err := d.transaction.read("config")
if err != nil || !strings.Contains(string(config), "--hostlist-domains=www.youtube.com ") || !strings.Contains(string(config), "--hostlist-domains=^fresh.googlevideo.com ") || strings.Contains(string(config), "--hostlist-domains=googlevideo.com") {
t.Fatalf("%s %v", config, err)
}
if (*events)[0] != "stop" || strings.Count(strings.Join(*events, ","), "restart") != 2 {
t.Fatal(*events)
}
}
func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
d, _, events := recoveryFixture(t, false)
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || result.GGCError != nil || result.FrontendID != savedID {
t.Fatalf("result=%+v err=%v", result, err)
}
for _, event := range *events {
if event == "scan:"+managedFrontendHost {
t.Fatalf("frontend scan ran despite a passing saved candidate: %v", *events)
}
}
if !strings.Contains(strings.Join(*events, ","), "check:"+managedFrontendHost+":--lua-desync=saved") {
t.Fatalf("saved candidate was not revalidated: %v", *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
}
func TestManagedRecoveryHealthTransportErrorReusesSavedFrontend(t *testing.T) {
for _, healthErr := range []error{errors.New("Player transport failed"), context.DeadlineExceeded} {
t.Run(healthErr.Error(), func(t *testing.T) {
d, _, events := recoveryFixture(t, true)
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
probes := 0
d.health = func(context.Context) (string, error) {
probes++
if probes == 1 {
return "", healthErr
}
return "OK", nil
}
d.discover = func(context.Context) (string, error) { return "", errors.New("unresolved") }
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || result.FrontendID != savedID || probes != 2 {
t.Fatalf("result=%+v err=%v probes=%d", result, err, probes)
}
want := []string{"stop", "check:" + managedFrontendHost + ":--lua-desync=saved", "restart"}
if !reflect.DeepEqual(*events, want) {
t.Fatalf("saved-first recovery events=%v want=%v", *events, want)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{savedID})
})
}
}
func TestManagedRecoveryInitialHealthFailsClosed(t *testing.T) {
for _, mode := range []string{"fatal", "canceled", "context-canceled"} {
t.Run(mode, func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var healthErr error = context.Canceled
if mode == "fatal" {
healthErr = &managedRecoveryFatal{Err: errors.New("unsafe target")}
}
d.health = func(context.Context) (string, error) {
if mode == "context-canceled" {
cancel()
return "OK", nil
}
return "", healthErr
}
_, err := recoverManagedTarget(ctx, d)
if !errors.Is(err, healthErr) || len(*events) != 0 {
t.Fatalf("err=%v events=%v", err, *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
})
}
}
func TestManagedRecoverySavedFirstAndFailClosed(t *testing.T) {
for _, mode := range []string{"pass", "nonpass", "malformed", "runtime", "checker-nonpass", "missing-report", "incomplete", "health-reject", "apply-infrastructure", "cancel", "cleanup-failure"} {
t.Run(mode, func(t *testing.T) {
d, old, events := recoveryFixture(t, true)
newID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=multisplit", true)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probes := 0
d.health = func(context.Context) (string, error) {
probes++
if mode == "apply-infrastructure" && probes == 2 {
return "", errors.New("probe infrastructure failed")
}
if probes == 1 || mode == "health-reject" && probes == 2 {
return "DEGRADED", nil
}
return "OK", nil
}
d.discover = func(context.Context) (string, error) { return "", errors.New("unresolved") }
checked := []string{}
baseCheck := d.check
d.check = func(ctx context.Context, binary, path, args, host string, out, stderr io.Writer) error {
checked = append(checked, args)
if mode == "checker-nonpass" {
return errCandidateNonpass
}
if mode == "runtime" || mode == "cleanup-failure" {
return errors.New("runner failed")
}
if mode == "cancel" {
cancel()
return ctx.Err()
}
if mode == "missing-report" {
return nil
}
if err := baseCheck(ctx, binary, path, args, host, out, stderr); err != nil {
return err
}
report := combinedReportFixture(args)
switch mode {
case "malformed":
report = `{}`
case "incomplete":
report = strings.ReplaceAll(report, "HTTPS/TLS1.3", "HTTPS/TLS1.2")
case "nonpass":
if !strings.Contains(path, "/scan") {
report = strings.Replace(report, `"passes_ok":3`, `"passes_ok":2`, 1)
}
}
return os.WriteFile(path+".combined.json", []byte(report), 0600)
}
if mode == "cleanup-failure" {
run := d.run
d.run = func(ctx context.Context, path string, args ...string) error {
if err := run(ctx, path, args...); err != nil {
return err
}
if args[0] == "start" {
return errors.New("cleanup failed")
}
return nil
}
}
result, err := recoverManagedTarget(ctx, d)
success := mode == "pass" || mode == "nonpass" || mode == "health-reject"
if (err == nil) != success {
t.Fatalf("result=%+v err=%v events=%v", result, err, *events)
}
if len(checked) == 0 || checked[0] != "--lua-desync=multisplit" {
t.Fatal(checked)
}
scanned := strings.Contains(strings.Join(*events, ","), "scan:")
if scanned != (mode == "nonpass") {
t.Fatal(*events)
}
want := old
if mode == "pass" {
want = newID
}
if mode == "nonpass" {
want = result.FrontendID
if want == old || want == newID {
t.Fatal("scan did not save fresh candidate")
}
}
assertActiveCandidates(t, d.transaction, d.db, []int64{want})
if !success && (*events)[len(*events)-1] != "start" {
t.Fatal("no cleanup", *events)
}
if mode == "health-reject" && (len(checked) != 2 || strings.Count(strings.Join(*events, ","), "restart") != 3) {
t.Fatal("health rejection did not rollback and try older", *events, checked)
}
if mode == "cleanup-failure" {
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) {
t.Fatal(err)
}
}
})
}
}
func TestManagedRecoveryScanArtifactsFailClosed(t *testing.T) {
for _, mode := range []string{"stale-combined", "remove-error", "missing-shortlist", "stale-shortlist", "malformed-combined", "cancel"} {
t.Run(mode, func(t *testing.T) {
d, _, events := recoveryFixture(t, false)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
path := filepath.Join(t.TempDir(), "scan")
d.scan = func(context.Context, string, string, io.Writer, io.Writer) (string, error) {
if mode != "missing-shortlist" {
if err := os.WriteFile(path+".check.json", []byte(combinedReportFixture("--lua-desync=fake")), 0600); err != nil {
t.Fatal(err)
}
}
if mode == "stale-shortlist" {
old := time.Now().Add(-time.Hour)
if err := os.Chtimes(path+".check.json", old, old); err != nil {
t.Fatal(err)
}
}
if mode == "remove-error" {
if err := os.Mkdir(path+".combined.json", 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(path+".combined.json", "keep"), nil, 0600); err != nil {
t.Fatal(err)
}
} else if err := os.WriteFile(path+".combined.json", []byte(combinedReportFixture("--lua-desync=fake")), 0600); err != nil {
t.Fatal(err)
}
if mode == "cancel" {
cancel()
}
return path, nil
}
checked := false
d.check = func(context.Context, string, string, string, string, io.Writer, io.Writer) error {
checked = true
if _, err := os.Stat(path + ".combined.json"); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("old combined artifact survived: %v", err)
}
if mode == "malformed-combined" {
return os.WriteFile(path+".combined.json", []byte(`{}`), 0600)
}
return nil // No fresh report: old passing evidence must not qualify.
}
result, err := recoverManagedTarget(ctx, d)
if err == nil || result.FrontendID != 0 || result.GooglevideoID != 0 {
t.Fatalf("artifact qualified: %+v %v", result, err)
}
wantCheck := mode == "stale-combined" || mode == "malformed-combined"
if checked != wantCheck {
t.Fatalf("checked=%v want=%v", checked, wantCheck)
}
if mode == "cancel" && !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if !reflect.DeepEqual(*events, []string{"stop", "start"}) {
t.Fatalf("target not restored: %v", *events)
}
assertActiveCandidates(t, d.transaction, d.db, nil)
assertManagedContent(t, d.transaction, managedBaseline)
ids, e := qualifiedCandidateIDs(d.db, "frontend")
if e != nil || len(ids) != 0 {
t.Fatalf("failed scan saved candidate: %v %v", ids, e)
}
})
}
}
func TestManagedRecoveryDiscoveryDropsOnlyOldGGC(t *testing.T) {
d, front, events := recoveryFixture(t, true)
cache := managedCandidateFixture(t, d.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
if err := d.transaction.Apply(d.db, []int64{front, cache}); err != nil {
t.Fatal(err)
}
d.discover = func(context.Context) (string, error) { return "", errors.New("discovery failed") }
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || result.GGCError == nil || result.FrontendID != front || result.GooglevideoID != 0 {
t.Fatalf("%+v %v", result, err)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
if !reflect.DeepEqual(*events, []string{"restart"}) {
t.Fatal(*events)
}
old, err := readStrategyCandidate(d.db, cache)
if err != nil || !old.Qualified || !reflect.DeepEqual(old.Endpoints, []string{"old.googlevideo.com"}) {
t.Fatal("immutable old GGC changed", old, err)
}
}
func TestManagedRecoveryRollbackFailureNeverTriesNext(t *testing.T) {
d, front, events := recoveryFixture(t, true)
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=multisplit", true)
d.health = func(context.Context) (string, error) { return "DEGRADED", nil }
run := d.run
restarts := 0
d.run = func(ctx context.Context, path string, args ...string) error {
if err := run(ctx, path, args...); err != nil {
return err
}
if args[0] == "restart" {
restarts++
if restarts == 2 {
return errors.New("rollback restart failed")
}
}
return nil
}
_, err := recoverManagedTarget(context.Background(), d)
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) {
t.Fatalf("rollback failure not fatal: %v", err)
}
joined := strings.Join(*events, ",")
if strings.Contains(joined, "scan:") || strings.Count(joined, "check:") != 1 {
t.Fatalf("rollback failure authorized another candidate: %v", *events)
}
if (*events)[len(*events)-1] != "start" {
t.Fatalf("reconciliation did not restore target: %v", *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
}
func TestManagedRecoveryFreshGGCAndPartialFailure(t *testing.T) {
for _, mode := range []string{"pass", "check-error", "health-reject", "discovery-error", "wildcard", "cancel"} {
t.Run(mode, func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
oldCache := managedCandidateFixture(t, d.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if mode == "discovery-error" {
d.discover = func(context.Context) (string, error) { return "", errors.New("secret URL") }
}
if mode == "wildcard" {
d.discover = func(context.Context) (string, error) { return "googlevideo.com", nil }
}
check := d.check
d.check = func(ctx context.Context, binary, path, args, host string, out, stderr io.Writer) error {
if host != "fresh.googlevideo.com" {
t.Fatal("old or wildcard scope", host)
}
if mode == "check-error" {
return errors.New("infrastructure")
}
if mode == "cancel" {
cancel()
return ctx.Err()
}
return check(ctx, binary, path, args, host, out, stderr)
}
probes := 0
d.health = func(context.Context) (string, error) {
probes++
if mode == "health-reject" && probes > 1 {
return "DEGRADED", nil
}
return "OK", nil
}
result, err := recoverManagedTarget(ctx, d)
if mode == "cancel" {
if !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
} else if err != nil {
t.Fatal(err)
}
if result.FrontendID != front {
t.Fatalf("%+v %v", result, err)
}
if strings.Contains(strings.Join(*events, ","), "scan:") {
t.Fatal("passing frontend/cache caused scan", *events)
}
want := []int64{front}
if mode == "pass" {
if result.GGCError != nil || result.GooglevideoID <= oldCache {
t.Fatalf("%+v", result)
}
want = append(want, result.GooglevideoID)
fresh, e := readStrategyCandidate(d.db, result.GooglevideoID)
old, oldErr := readStrategyCandidate(d.db, oldCache)
if e != nil || oldErr != nil || !reflect.DeepEqual(fresh.Endpoints, []string{"fresh.googlevideo.com"}) || !reflect.DeepEqual(old.Endpoints, []string{"old.googlevideo.com"}) {
t.Fatal(fresh, old, e, oldErr)
}
} else if mode != "cancel" && result.GGCError == nil {
t.Fatal("cache failure claimed success")
}
assertActiveCandidates(t, d.transaction, d.db, want)
})
}
}
func TestManagedQualifiedCandidateBound(t *testing.T) {
db := testStore(t)
var last int64
for i := 0; i < 12; i++ {
last = managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
}
managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", false)
ids, err := qualifiedCandidateIDs(db, "frontend")
if err != nil || len(ids) != 10 || ids[0] != last || ids[9] != last-9 {
t.Fatal(ids, err)
}
}
func TestManagedRecoveryMonitorCallbacks(t *testing.T) {
recovery, _, _ := recoveryFixture(t, false)
monitor := monitorRuntimeFixture(t)
monitor.db = recovery.db
monitor.target.directory = recovery.transaction.Dir
identity, err := json.Marshal(managedIdentity{Role: "managed-test", Directory: recovery.transaction.Dir, ID: strings.Repeat("01", 16)})
if err != nil {
t.Fatal(err)
}
if err := recovery.transaction.replace(managedMarkerName, identity); err != nil {
t.Fatal(err)
}
var reports []error
if err := wireManagedRecovery(&monitor, recovery, func(err error) { reports = append(reports, err) }); err != nil {
t.Fatal(err)
}
if err := monitor.bootstrap(context.Background()); err != nil {
t.Fatal(err)
}
if err := monitor.probe(context.Background()); err != nil {
t.Fatal(err)
}
if len(reports) != 0 {
t.Fatal(reports)
}
ids, err := recovery.transaction.ActiveCandidates(recovery.db)
if err != nil || len(ids) != 2 {
t.Fatal(ids, err)
}
// A drifted deployment is never adopted by startup/recovery callbacks.
if err := os.WriteFile(filepath.Join(recovery.transaction.Dir, "config"), []byte("external"), 0600); err != nil {
t.Fatal(err)
}
if err := monitor.recover(context.Background()); !errors.Is(err, errManagedDrift) {
t.Fatal(err)
}
}
func TestManagedRecoveryRuntimeFailures(t *testing.T) {
for _, mode := range []string{"stop", "restart", "health-infrastructure"} {
t.Run(mode, func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
d.health = func(context.Context) (string, error) {
if mode == "health-infrastructure" {
return "", errors.New("request infrastructure failed")
}
return "DEGRADED", nil
}
run := d.run
d.run = func(ctx context.Context, path string, args ...string) error {
if err := run(ctx, path, args...); err != nil {
return err
}
if args[0] == mode {
return errors.New("runtime failed")
}
return nil
}
_, err := recoverManagedTarget(context.Background(), d)
if err == nil {
t.Fatal("runtime failure accepted")
}
if strings.Contains(strings.Join(*events, ","), "scan:") {
t.Fatal("infrastructure triggered scan", *events)
}
if mode == "restart" {
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) {
t.Fatal(err)
}
// Failed rollback restart keeps a pending journal; do not start it.
if ids, err := d.transaction.ActiveCandidates(d.db); err == nil || len(ids) != 0 {
t.Fatal(ids, err)
}
} else {
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
if mode == "stop" && (*events)[len(*events)-1] != "start" {
t.Fatal(*events)
}
}
})
}
}
func TestManagedRecoveryPlayerAdapter(t *testing.T) {
target, _, _ := managedCLIFixture(t)
recovery, _, _ := recoveryFixture(t, false)
d, err := managedRecoveryDependencies(target, recovery.db, recovery.scan, recovery.check, recovery.run)
if err != nil {
t.Fatal(err)
}
status, err := d.health(context.Background())
if status != "OK" || err != nil {
t.Fatal(status, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := d.health(ctx); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if _, err := d.discover(context.Background()); err == nil {
t.Fatal("health-only response claimed GGC discovery")
}
if d.transaction.Dir != target.directory || d.transaction.Name != "config" {
t.Fatal("wrong target binding")
}
}
func TestManagedRecoveryProbeGGCScope(t *testing.T) {
for _, mode := range []string{"matching", "changed", "missing", "wildcard", "uppercase", "url", "foreign", "empty", "discovery-error", "fatal", "cancel", "drift", "identity", "health-reject"} {
t.Run(mode, func(t *testing.T) {
recovery, front, events := recoveryFixture(t, true)
if mode != "missing" {
cache := managedCandidateFixture(t, recovery.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=multisplit", true)
if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil {
t.Fatal(err)
}
}
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
discoveries := 0
recovery.discover = func(context.Context) (string, error) {
discoveries++
switch mode {
case "changed":
return "other.googlevideo.com", nil
case "wildcard":
return "googlevideo.com", nil
case "uppercase":
return "FRESH.googlevideo.com", nil
case "url":
return "https://fresh.googlevideo.com/video?secret=never-print", nil
case "foreign":
return "fresh.googlevideo.com.evil.example", nil
case "empty":
return "", nil
case "discovery-error":
return "", errors.New("secret=never-print")
case "fatal":
return "", &managedRecoveryFatal{Err: errors.New("unsafe target")}
case "cancel":
cancel()
}
return "fresh.googlevideo.com", nil
}
if mode == "health-reject" {
recovery.health = func(context.Context) (string, error) { return "DEGRADED", nil }
}
if err := wireManagedRecovery(&monitor, recovery, func(error) { t.Fatal("probe invoked recovery report") }); err != nil {
t.Fatal(err)
}
if mode == "drift" {
if err := os.WriteFile(filepath.Join(recovery.transaction.Dir, "config"), []byte("external"), 0600); err != nil {
t.Fatal(err)
}
}
if mode == "identity" {
if err := os.Remove(filepath.Join(recovery.transaction.Dir, managedMarkerName)); err != nil {
t.Fatal(err)
}
}
err := monitor.probe(ctx)
switch mode {
case "matching":
if err != nil {
t.Fatal(err)
}
case "changed", "missing":
if !errors.Is(err, errManagedGGCScopeChanged) {
t.Fatal(err)
}
case "fatal", "drift", "identity":
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) {
t.Fatal(err)
}
case "cancel":
if !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
case "health-reject":
var rejection *managedHealthRejection
if !errors.As(err, &rejection) {
t.Fatal(err)
}
default:
if !errors.Is(err, errManagedGGCUnresolved) || strings.Contains(err.Error(), "never-print") {
t.Fatal(err)
}
}
if len(*events) != 0 || (mode == "drift" || mode == "identity" || mode == "health-reject") && discoveries != 0 {
t.Fatalf("unsafe probe effects: events=%v discoveries=%d", *events, discoveries)
}
})
}
}
func TestManagedRecoveryGGCRotationPolicy(t *testing.T) {
for _, activeCache := range []bool{false, true} {
t.Run(fmt.Sprint(activeCache), func(t *testing.T) {
recovery, front, events := recoveryFixture(t, true)
if err := os.Chmod(recovery.transaction.Dir, 0700); err != nil {
t.Fatal(err)
}
if activeCache {
cache := managedCandidateFixture(t, recovery.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil {
t.Fatal(err)
}
}
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
if err := wireManagedRecovery(&monitor, recovery, func(err error) { t.Fatal(err) }); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probes := 0
probe := monitor.probe
monitor.probe = func(ctx context.Context) error { probes++; return probe(ctx) }
monitor.wait = func(ctx context.Context, _ time.Duration) error {
if probes < monitor.threshold && len(*events) != 0 {
t.Fatal("recovered before threshold", *events)
}
if probes == monitor.threshold+1 {
cancel()
return ctx.Err()
}
return nil
}
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
ids, err := recovery.transaction.ActiveCandidates(recovery.db)
if err != nil || len(ids) != 2 || ids[0] != front {
t.Fatal(ids, err)
}
cache, err := readStrategyCandidate(recovery.db, ids[1])
if err != nil || !reflect.DeepEqual(cache.Endpoints, []string{"fresh.googlevideo.com"}) {
t.Fatal(cache, err)
}
state, err := readMonitorState(monitor.db, monitor.serviceID)
if err != nil || state.CooldownUntil.IsZero() || strings.Contains(strings.Join(*events, ","), "scan:"+managedFrontendHost) {
t.Fatal(state, err, *events)
}
})
}
}