diff --git a/cmd/zapret-autotune/managed_monitor_cli_test.go b/cmd/zapret-autotune/managed_monitor_cli_test.go index 5377d34..2aad4a7 100644 --- a/cmd/zapret-autotune/managed_monitor_cli_test.go +++ b/cmd/zapret-autotune/managed_monitor_cli_test.go @@ -105,8 +105,8 @@ func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) { if err := monitor.bootstrap(context.Background()); err != nil || reports != 1 { t.Fatal(err, reports) } - if err := monitor.probe(context.Background()); err != nil { - t.Fatal("unresolved GGC stopped API probe", err) + if err := monitor.probe(context.Background()); !errors.Is(err, errManagedGGCUnresolved) { + t.Fatal("unresolved GGC did not fail scope probe", err) } } @@ -167,10 +167,9 @@ func TestManagedMonitorCLIBindingAndCancellation(t *testing.T) { if m.bootstrap == nil || m.recover == nil || m.interval != 5*time.Minute || m.threshold != 3 || m.cooldown != 30*time.Minute { t.Fatal("missing wiring/defaults") } - if err := m.probe(ctx); err != nil { + if err := m.probe(ctx); !errors.Is(err, context.Canceled) { t.Fatal(err) } - _ = m.probe(ctx) return ctx.Err() }) if code != 0 || requests != 2 { diff --git a/cmd/zapret-autotune/managed_recovery.go b/cmd/zapret-autotune/managed_recovery.go index b6f6dc8..66f4545 100644 --- a/cmd/zapret-autotune/managed_recovery.go +++ b/cmd/zapret-autotune/managed_recovery.go @@ -95,6 +95,11 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, type managedRecoveryFatal struct{ Err error } +var ( + errManagedGGCUnresolved = errors.New("managed GGC scope unresolved") + errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing") +) + func (e *managedRecoveryFatal) Error() string { return "managed recovery safety failure: " + e.Err.Error() } @@ -478,7 +483,37 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe if status != "OK" { return &managedHealthRejection{Status: status} } - return nil + transaction := *recovery.transaction + transaction.Health = func() (string, error) { return recovery.health(ctx) } + // ActiveCandidates is read-only; never permit a probe to reload config. + transaction.Reload = func() error { return fmt.Errorf("probe cannot reload config") } + ids, err := transaction.ActiveCandidates(recovery.db) + if err != nil { + return &managedRecoveryFatal{Err: err} + } + // Keep Player API health separate from GGC scope (not playback health). + // Two bounded Player requests reuse the existing health/discovery adapters. + host, err := recovery.discover(ctx) + if ctx.Err() != nil { + return ctx.Err() + } + var fatal *managedRecoveryFatal + if errors.As(err, &fatal) || errors.Is(err, context.Canceled) { + return err + } + if err != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") { + return errManagedGGCUnresolved + } + for _, id := range ids { + candidate, err := readStrategyCandidate(recovery.db, id) + if err != nil { + return &managedRecoveryFatal{Err: err} + } + if candidate.Group == "googlevideo" && len(candidate.Endpoints) == 1 && candidate.Endpoints[0] == host { + return nil + } + } + return errManagedGGCScopeChanged } return nil } diff --git a/cmd/zapret-autotune/managed_recovery_test.go b/cmd/zapret-autotune/managed_recovery_test.go index 0a9a970..0de524f 100644 --- a/cmd/zapret-autotune/managed_recovery_test.go +++ b/cmd/zapret-autotune/managed_recovery_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "os" "path/filepath" @@ -569,3 +570,154 @@ func TestManagedRecoveryPlayerAdapter(t *testing.T) { t.Fatal("wrong target binding") } } + +func TestManagedRecoveryProbeGGCScope(t *testing.T) { + for _, mode := range []string{"matching", "changed", "missing", "wildcard", "uppercase", "url", "foreign", "empty", "discovery-error", "fatal", "cancel", "drift", "identity", "health-reject"} { + t.Run(mode, func(t *testing.T) { + recovery, front, events := recoveryFixture(t, true) + if mode != "missing" { + cache := managedCandidateFixture(t, recovery.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=multisplit", true) + if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil { + t.Fatal(err) + } + } + monitor := monitorRuntimeFixture(t) + monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir + identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}` + if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + discoveries := 0 + recovery.discover = func(context.Context) (string, error) { + discoveries++ + switch mode { + case "changed": + return "other.googlevideo.com", nil + case "wildcard": + return "googlevideo.com", nil + case "uppercase": + return "FRESH.googlevideo.com", nil + case "url": + return "https://fresh.googlevideo.com/video?secret=never-print", nil + case "foreign": + return "fresh.googlevideo.com.evil.example", nil + case "empty": + return "", nil + case "discovery-error": + return "", errors.New("secret=never-print") + case "fatal": + return "", &managedRecoveryFatal{Err: errors.New("unsafe target")} + case "cancel": + cancel() + } + return "fresh.googlevideo.com", nil + } + if mode == "health-reject" { + recovery.health = func(context.Context) (string, error) { return "DEGRADED", nil } + } + if err := wireManagedRecovery(&monitor, recovery, func(error) { t.Fatal("probe invoked recovery report") }); err != nil { + t.Fatal(err) + } + if mode == "drift" { + if err := os.WriteFile(filepath.Join(recovery.transaction.Dir, "config"), []byte("external"), 0600); err != nil { + t.Fatal(err) + } + } + if mode == "identity" { + if err := os.Remove(filepath.Join(recovery.transaction.Dir, managedMarkerName)); err != nil { + t.Fatal(err) + } + } + err := monitor.probe(ctx) + switch mode { + case "matching": + if err != nil { + t.Fatal(err) + } + case "changed", "missing": + if !errors.Is(err, errManagedGGCScopeChanged) { + t.Fatal(err) + } + case "fatal", "drift", "identity": + var fatal *managedRecoveryFatal + if !errors.As(err, &fatal) { + t.Fatal(err) + } + case "cancel": + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + case "health-reject": + var rejection *managedHealthRejection + if !errors.As(err, &rejection) { + t.Fatal(err) + } + default: + if !errors.Is(err, errManagedGGCUnresolved) || strings.Contains(err.Error(), "never-print") { + t.Fatal(err) + } + } + if len(*events) != 0 || (mode == "drift" || mode == "identity" || mode == "health-reject") && discoveries != 0 { + t.Fatalf("unsafe probe effects: events=%v discoveries=%d", *events, discoveries) + } + }) + } +} + +func TestManagedRecoveryGGCRotationPolicy(t *testing.T) { + for _, activeCache := range []bool{false, true} { + t.Run(fmt.Sprint(activeCache), func(t *testing.T) { + recovery, front, events := recoveryFixture(t, true) + if err := os.Chmod(recovery.transaction.Dir, 0700); err != nil { + t.Fatal(err) + } + if activeCache { + cache := managedCandidateFixture(t, recovery.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true) + if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil { + t.Fatal(err) + } + } + monitor := monitorRuntimeFixture(t) + monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir + identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}` + if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil { + t.Fatal(err) + } + if err := wireManagedRecovery(&monitor, recovery, func(err error) { t.Fatal(err) }); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + probes := 0 + probe := monitor.probe + monitor.probe = func(ctx context.Context) error { probes++; return probe(ctx) } + monitor.wait = func(ctx context.Context, _ time.Duration) error { + if probes < monitor.threshold && len(*events) != 0 { + t.Fatal("recovered before threshold", *events) + } + if probes == monitor.threshold+1 { + cancel() + return ctx.Err() + } + return nil + } + if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + ids, err := recovery.transaction.ActiveCandidates(recovery.db) + if err != nil || len(ids) != 2 || ids[0] != front { + t.Fatal(ids, err) + } + cache, err := readStrategyCandidate(recovery.db, ids[1]) + if err != nil || !reflect.DeepEqual(cache.Endpoints, []string{"fresh.googlevideo.com"}) { + t.Fatal(cache, err) + } + state, err := readMonitorState(monitor.db, monitor.serviceID) + if err != nil || state.CooldownUntil.IsZero() || strings.Contains(strings.Join(*events, ","), "scan:"+managedFrontendHost) { + t.Fatal(state, err, *events) + } + }) + } +} diff --git a/cmd/zapret-autotune/recover.go b/cmd/zapret-autotune/recover.go index 4018145..7475b1a 100644 --- a/cmd/zapret-autotune/recover.go +++ b/cmd/zapret-autotune/recover.go @@ -4,14 +4,17 @@ import ( "bytes" "context" "database/sql" + "encoding/json" "errors" "fmt" "io" "os" + "regexp" + "strconv" "strings" ) -// Only complete, well-formed non-passing measurements permit scan fallback. +// Only validated measurements or authenticated early timeouts permit fallback. var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols") const managedFrontendHost = "www.youtube.com" @@ -52,6 +55,10 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa if err := os.Remove(path + ".combined.json"); err != nil && !errors.Is(err, os.ErrNotExist) { return nil, err } + // The owned runner creates this with O_EXCL. Never accept stale diagnostics. + if _, err := os.Lstat(path + ".combined.json.stderr"); !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("candidate diagnostic artifact already exists or cannot be inspected: %v", err) + } var raw bytes.Buffer err := check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw) if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil { @@ -72,12 +79,117 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa return nil, err } _, err = selectBlockcheckEvidence(bytes.NewReader(report), c.Args, group, host, true) + if err != nil && !errors.Is(err, errCandidateNonpass) { + if diagnostics, readErr := os.ReadFile(path + ".combined.json.stderr"); readErr == nil && candidateEarlyTimeout(report, diagnostics, c.Args, group, host) { + err = errCandidateNonpass + } + } if contextErr := ctx.Err(); contextErr != nil { return nil, contextErr } return report, err } +var earlyTimeoutResult = regexp.MustCompile(`^ FAIL ([0-2])/([1-3]) timeout$`) +var earlySuccessResult = regexp.MustCompile(`^ OK median [0-9]+ms, [0-9]+\.[0-9] KB/s$`) +var earlyCheckPreamble = regexp.MustCompile("^(! --workers has no effect on `check` \\(it verifies strategies sequentially by design\\)\n)?" + + `=== Checking prerequisites ===\n OK nfqws2: /[^\s]+\n OK nft\n OK nft queue support\n` + + `check loaded 2 strategies from /[^\s]+, --take [0-9]+, --passes 3\n( ISP: AS[0-9]+ [^\n]+\n)?` + + `=== DNS resolve ===\n dns mode: auto\n ([a-z0-9.-]+) -> [0-9a-fA-F:.]+ \(via doh \(auto-fallback\)\)$`) +var earlyCheckSummary = regexp.MustCompile(`^ total: 2 \| working: ([01]) \| elapsed: [0-9]+\.[0-9]s\n OK JSON report → /[^\s]+$`) + +// This is rejection evidence only, never qualification evidence. Unknown CLI +// output fails closed; changes to the checker format require a new fixture. +func candidateEarlyTimeout(raw, diagnostics []byte, args, group, host string) bool { + var report struct { + Domain string `json:"domain"` + Total *int `json:"total"` + Working *int `json:"working"` + Strategies *[]json.RawMessage `json:"strategies"` + } + d := json.NewDecoder(bytes.NewReader(raw)) + if d.Decode(&report) != nil || d.Decode(new(json.RawMessage)) != io.EOF || report.Domain != host || report.Total == nil || *report.Total != 2 || report.Working == nil || *report.Working < 0 || *report.Working >= 2 || report.Strategies == nil || len(*report.Strategies) != *report.Working { + return false + } + success := "" + if *report.Working == 1 { + row := (*report.Strategies)[0] + var fields map[string]json.RawMessage + if json.Unmarshal(row, &fields) != nil || json.Unmarshal(fields["protocol"], &success) != nil || normalizeProtocol(success) == "" { + return false + } + other := "HTTPS/TLS1.2" + if success == other { + other = "HTTPS/TLS1.3" + } + fields["protocol"], _ = json.Marshal(other) + copyRow, err := json.Marshal(fields) + if err != nil { + return false + } + // Reuse every strict measurement/target/args check, including exactly + // 3/3, by supplying a protocol-flipped copy solely for validation. + complete, err := json.Marshal(map[string]any{"strategies": []json.RawMessage{row, copyRow}}) + if err != nil { + return false + } + if _, err := selectBlockcheckEvidence(bytes.NewReader(complete), args, group, host, false); err != nil { + return false + } + } + lines := strings.Split(strings.ReplaceAll(string(diagnostics), "\r\n", "\n"), "\n") + var evidence []string + for _, line := range lines { + if line != "" { + evidence = append(evidence, line) + } + } + // ponytail: pin the known CLI framing and metadata grammar; new checker + // formats need explicit fixtures, not permissive infrastructure filtering. + parts := strings.Split(strings.Join(evidence, "\n"), "\n=== Checking strategies (data transfer) ===\n") + if len(parts) != 2 { + return false + } + preamble := earlyCheckPreamble.FindStringSubmatch(parts[0]) + if preamble == nil || preamble[3] != host { + return false + } + parts = strings.Split(parts[1], "\n=== Check summary ===\n") + if len(parts) != 2 { + return false + } + summary := earlyCheckSummary.FindStringSubmatch(parts[1]) + if summary == nil || summary[1] != strconv.Itoa(*report.Working) { + return false + } + evidence = strings.Split(parts[0], "\n") + if len(evidence) != 7 || evidence[0] != " Tip: use --take 10 to stop after 10 verified per protocol" || evidence[1] != " Verifying 2 strategies (3 passes, early-exit on first fail)" || evidence[4] != " ────────────────────────────────────────────────────────────" { + return false + } + for i, protocol := range []string{"HTTPS/TLS1.2", "HTTPS/TLS1.3"} { + if evidence[2+3*i] != fmt.Sprintf(" [%d/2] %s nfqws2 %s", i+1, protocol, args) { + return false + } + result := evidence[3+3*i] + if protocol == success { + if !earlySuccessResult.MatchString(result) { + return false + } + } else { + match := earlyTimeoutResult.FindStringSubmatch(result) + if match == nil { + return false + } + n, _ := strconv.Atoi(match[1]) + m, _ := strconv.Atoi(match[2]) + if n != m-1 { + return false + } + } + } + return true +} + func revalidateGooglevideo(ctx context.Context, db *sql.DB, host string, stderr io.Writer, check cacheCheck) (string, []byte, error) { ids, err := qualifiedCandidateIDs(db, "googlevideo") if err != nil { diff --git a/cmd/zapret-autotune/recovery_primitives_test.go b/cmd/zapret-autotune/recovery_primitives_test.go index 4b22831..4e75603 100644 --- a/cmd/zapret-autotune/recovery_primitives_test.go +++ b/cmd/zapret-autotune/recovery_primitives_test.go @@ -210,3 +210,261 @@ func TestRevalidateCandidateScope(t *testing.T) { t.Fatalf("changed stored scope: %+v %v", stored, err) } } + +func earlyTimeoutDiagnosticsFixture(args, tls12, tls13 string, working int) string { + return fmt.Sprintf("! --workers has no effect on `check` (it verifies strategies sequentially by design)\n"+`=== Checking prerequisites === + OK nfqws2: /opt/zapret2/binaries/linux-x86_64/nfqws2 + OK nft + OK nft queue support +check loaded 2 strategies from /tmp/input.txt, --take 1, --passes 3 + ISP: AS3175 ... +=== DNS resolve === + dns mode: auto + www.youtube.com -> 142.251.156.4 (via doh (auto-fallback)) + +=== Checking strategies (data transfer) === + Tip: use --take 10 to stop after 10 verified per protocol + Verifying 2 strategies (3 passes, early-exit on first fail) + [1/2] HTTPS/TLS1.2 nfqws2 %s + %s + ──────────────────────────────────────────────────────────── + [2/2] HTTPS/TLS1.3 nfqws2 %s + %s + +=== Check summary === + total: 2 | working: %d | elapsed: 7.7s + OK JSON report → /tmp/input.txt.combined.json +`, args, tls12, args, tls13, working) +} + +func TestCandidateEarlyTimeoutRowValidation(t *testing.T) { + args := "--lua-desync=multisplit" + var passing struct { + Strategies []json.RawMessage `json:"strategies"` + } + if err := json.Unmarshal([]byte(combinedReportFixture(args)), &passing); err != nil { + t.Fatal(err) + } + diagnostics := []byte(earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", "OK median 242ms, 3728.2 KB/s", 1)) + check := func(t *testing.T, field string, value json.RawMessage) { + t.Helper() + var row map[string]json.RawMessage + if err := json.Unmarshal(passing.Strategies[1], &row); err != nil { + t.Fatal(err) + } + if value == nil { + delete(row, field) + } else { + row[field] = value + } + report, err := json.Marshal(map[string]any{"domain": "www.youtube.com", "total": 2, "working": 1, "strategies": []any{row}}) + if err != nil { + t.Fatal(err) + } + if candidateEarlyTimeout(report, diagnostics, args, "frontend", "www.youtube.com") { + t.Fatal("malformed passing row authorized fallback") + } + } + for _, field := range []string{"protocol", "args", "success_rate", "coverage", "passes_ok", "passes_total", "median_latency_ms", "median_speed_kbps"} { + for _, mutation := range []struct { + name string + value json.RawMessage + }{{"missing", nil}, {"null", json.RawMessage(`null`)}, {"wrong-type", json.RawMessage(`{}`)}} { + t.Run(field+"/"+mutation.name, func(t *testing.T) { check(t, field, mutation.value) }) + } + } + for _, mutation := range []struct{ field, value string }{ + {"protocol", `"HTTP"`}, {"args", `"--lua-desync=fake"`}, + {"group", `"googlevideo"`}, {"domains", `["other.example"]`}, {"domains", `[]`}, + {"success_rate", `0.9`}, {"success_rate", `1.1`}, {"success_rate", `-1`}, + {"coverage", `0.9`}, {"coverage", `1.1`}, {"coverage", `-1`}, + {"passes_ok", `2`}, {"passes_ok", `4`}, {"passes_ok", `-1`}, + {"passes_total", `2`}, {"passes_total", `4`}, {"passes_total", `-1`}, + {"median_latency_ms", `-1`}, {"median_speed_kbps", `-1`}, + } { + t.Run(mutation.field+"/"+mutation.value, func(t *testing.T) { check(t, mutation.field, json.RawMessage(mutation.value)) }) + } +} + +func TestRevalidateCandidateEarlyTimeout(t *testing.T) { + db := testStore(t) + args := "--lua-desync=multisplit" + saveProposal(t, db, args) + c, err := readStrategyCandidate(db, 1) + if err != nil { + t.Fatal(err) + } + var passing struct { + Strategies []json.RawMessage `json:"strategies"` + } + if err := json.Unmarshal([]byte(combinedReportFixture(args)), &passing); err != nil { + t.Fatal(err) + } + report := fmt.Sprintf(`{"domain":"www.youtube.com","total":2,"working":1,"strategies":[%s]}`, passing.Strategies[1]) + okResult := "OK median 242ms, 3728.2 KB/s" + diagnostics := earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", okResult, 1) + malformedOK := map[string]string{ + "ok-garbage": "OK garbage median garbage", + "ok-old-fixture": "OK 3/3 median 10ms 100kbps", + "ok-missing-latency": "OK median ms, 3728.2 KB/s", + "ok-negative-latency": "OK median -242ms, 3728.2 KB/s", + "ok-fractional-latency": "OK median 242.5ms, 3728.2 KB/s", + "ok-missing-speed": "OK median 242ms, KB/s", + "ok-negative-speed": "OK median 242ms, -3728.2 KB/s", + "ok-nan-speed": "OK median 242ms, NaN KB/s", + "ok-inf-speed": "OK median 242ms, Inf KB/s", + "ok-wrong-units": "OK median 242ms, 3728.2 kbps", + "ok-missing-comma": "OK median 242ms 3728.2 KB/s", + "ok-trailing-garbage": okResult + " garbage", + "ok-trailing-space": okResult + " ", + } + modes := []string{ + "timeout", "two-timeouts", "tls13-timeout", "wrong-domain", "wrong-row-args", "wrong-heading-args", "missing-artifact", "stale-artifact", "missing-report", "runner-failed", "runner-sentinel", "cancelled", + "nfq-exited", "extra-line", "missing-heading", "duplicate-heading", "wrong-protocol", "missing-fail", "success-not-ok", "fail-on-success", "wrong-total", "missing-total", "negative-working", "wrong-working", "working-two", "missing-working", "null-strategies", "trailing-json", "bad-json", + "zero-count", "equal-counts", "reversed-counts", "excess-count", "not-timeout", "partial-success", "missing-metric", "invalid-metric", "wrong-row-target", "empty-with-one-success", + "first-pass-timeout", "third-pass-timeout", "two-first-pass-timeouts", "unknown-section-line", "infra-section-line", "missing-section-marker", "missing-summary-marker", "duplicate-section", "wrong-verifying-count", "wrong-loaded-count", "wrong-summary-count", "wrong-summary-working", "unindented-heading", "extra-indent-heading", "wrong-dns-host", "unknown-preamble-line", + } + for mode := range malformedOK { + modes = append(modes, mode) + } + for _, mode := range modes { + t.Run(mode, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "check") + r, diagnostic := report, diagnostics + if result, ok := malformedOK[mode]; ok { + diagnostic = strings.Replace(diagnostic, okResult, result, 1) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + switch mode { + case "two-timeouts", "empty-with-one-success": + r = `{"domain":"www.youtube.com","total":2,"working":0,"strategies":[]}` + if mode == "two-timeouts" { + diagnostic = earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", "FAIL 2/3 timeout", 0) + } + case "two-first-pass-timeouts": + r = `{"domain":"www.youtube.com","total":2,"working":0,"strategies":[]}` + diagnostic = earlyTimeoutDiagnosticsFixture(args, "FAIL 0/1 timeout", "FAIL 0/1 timeout", 0) + case "first-pass-timeout": + diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", "FAIL 0/1 timeout", 1) + case "third-pass-timeout": + diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", "FAIL 2/3 timeout", 1) + case "tls13-timeout": + r = fmt.Sprintf(`{"domain":"www.youtube.com","total":2,"working":1,"strategies":[%s]}`, passing.Strategies[0]) + diagnostic = earlyTimeoutDiagnosticsFixture(args, okResult, "FAIL 1/2 timeout", 1) + case "wrong-domain": + r = strings.Replace(r, "www.youtube.com", "other.example", 1) + case "wrong-row-args": + r = strings.Replace(r, args, "--lua-desync=fake", 1) + case "wrong-heading-args": + diagnostic = strings.Replace(diagnostic, args, "--lua-desync=fake", 1) + case "nfq-exited": + diagnostic += "nfqws2 exited with status 1\n" + case "extra-line": + diagnostic += "unexpected infrastructure output\n" + case "missing-heading": + diagnostic = strings.Replace(diagnostic, " [1/2] HTTPS/TLS1.2 nfqws2 "+args+"\n", "", 1) + case "duplicate-heading": + diagnostic = strings.Replace(diagnostic, " [1/2]", " [1/2] HTTPS/TLS1.2 nfqws2 "+args+"\n [1/2]", 1) + case "unknown-section-line", "infra-section-line": + line := "unknown output" + if mode == "infra-section-line" { + line = "nfqws2 exited with status 1" + } + diagnostic = strings.Replace(diagnostic, " FAIL", line+"\n FAIL", 1) + case "missing-section-marker", "missing-summary-marker", "duplicate-section", "wrong-verifying-count", "wrong-loaded-count", "wrong-summary-count", "wrong-summary-working", "unindented-heading", "extra-indent-heading", "wrong-dns-host", "unknown-preamble-line": + replacements := map[string][2]string{ + "missing-section-marker": {"=== Checking strategies (data transfer) ===\n", ""}, + "missing-summary-marker": {"=== Check summary ===\n", ""}, + "duplicate-section": {"=== Checking strategies (data transfer) ===", "=== Checking strategies (data transfer) ===\n=== Checking strategies (data transfer) ==="}, + "wrong-verifying-count": {"Verifying 2 strategies", "Verifying 3 strategies"}, + "wrong-loaded-count": {"loaded 2 strategies", "loaded 3 strategies"}, + "wrong-summary-count": {"total: 2", "total: 3"}, + "wrong-summary-working": {"working: 1", "working: 0"}, + "unindented-heading": {" [1/2]", "[1/2]"}, + "extra-indent-heading": {" [1/2]", " [1/2]"}, + "wrong-dns-host": {"www.youtube.com ->", "other.example ->"}, + "unknown-preamble-line": {"=== DNS resolve ===", "unknown output\n=== DNS resolve ==="}, + } + pair := replacements[mode] + diagnostic = strings.Replace(diagnostic, pair[0], pair[1], 1) + case "wrong-protocol": + diagnostic = strings.Replace(diagnostic, "HTTPS/TLS1.2", "HTTPS/TLS1.3", 1) + case "missing-fail": + diagnostic = strings.Replace(diagnostic, " FAIL 1/2 timeout\n", "", 1) + case "success-not-ok": + diagnostic = strings.Replace(diagnostic, "OK median", "unknown median", 1) + case "fail-on-success": + diagnostic = strings.Replace(diagnostic, okResult, "FAIL 1/2 timeout", 1) + case "wrong-total", "missing-total", "negative-working", "wrong-working", "working-two", "missing-working", "null-strategies": + replacements := map[string][2]string{ + "wrong-total": {`"total":2`, `"total":3`}, "missing-total": {`"total":2,`, ""}, + "negative-working": {`"working":1`, `"working":-1`}, "wrong-working": {`"working":1`, `"working":0`}, + "working-two": {`"working":1`, `"working":2`}, "missing-working": {`"working":1,`, ""}, + "null-strategies": {`"strategies":[` + string(passing.Strategies[1]) + `]`, `"strategies":null`}, + } + pair := replacements[mode] + r = strings.Replace(r, pair[0], pair[1], 1) + case "trailing-json": + r += `{}` + case "bad-json": + r = `{` + case "zero-count", "equal-counts", "reversed-counts", "excess-count", "not-timeout": + replacements := map[string]string{"zero-count": "FAIL 0/2 timeout", "equal-counts": "FAIL 2/2 timeout", "reversed-counts": "FAIL 3/2 timeout", "excess-count": "FAIL 1/4 timeout", "not-timeout": "FAIL 1/2 connection refused"} + diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", replacements[mode], 1) + case "partial-success": + r = strings.Replace(r, `"passes_ok":3`, `"passes_ok":2`, 1) + case "missing-metric": + r = strings.Replace(r, `"coverage":1`, `"coverage":null`, 1) + case "invalid-metric": + r = strings.Replace(r, `"median_speed_kbps":100`, `"median_speed_kbps":-1`, 1) + case "wrong-row-target": + r = strings.Replace(r, `"protocol":`, `"domains":["other.example"],"protocol":`, 1) + } + if mode == "stale-artifact" { + if err := os.WriteFile(path+".combined.json.stderr", []byte(diagnostic), 0600); err != nil { + t.Fatal(err) + } + } + calls := 0 + check := func(_ context.Context, _, gotPath, gotArgs, host string, _, _ io.Writer) error { + calls++ + if gotPath != path || gotArgs != args || host != "www.youtube.com" { + t.Fatal("unexpected checker request") + } + if mode != "missing-report" { + if err := os.WriteFile(path+".combined.json", []byte(r), 0600); err != nil { + return err + } + } + if mode != "missing-artifact" { + if err := os.WriteFile(path+".combined.json.stderr", []byte(diagnostic), 0600); err != nil { + return err + } + } + switch mode { + case "runner-failed": + return fmt.Errorf("checker exited") + case "runner-sentinel": + return errCandidateNonpass + case "cancelled": + cancel() + } + return nil + } + _, err := revalidateCandidate(ctx, c, "frontend", "www.youtube.com", path, io.Discard, check) + nonpass := mode == "timeout" || mode == "two-timeouts" || mode == "tls13-timeout" || mode == "first-pass-timeout" || mode == "third-pass-timeout" || mode == "two-first-pass-timeouts" + if err == nil || errors.Is(err, errCandidateNonpass) != nonpass { + t.Fatalf("nonpass=%v err=%v", nonpass, err) + } + if nonpass { + if _, err := combinedBlockcheckOutputForHost(strings.NewReader(r), args, "www.youtube.com"); err == nil { + t.Fatal("early timeout report qualified for apply") + } + } + if mode == "stale-artifact" && calls != 0 { + t.Fatal("checker invoked with stale diagnostic evidence") + } + }) + } +}