fix(recovery): handle checker early timeouts and GGC rotation

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-10 16:37:59 +03:00
1 parent 741009afa7
commit beb4f16309
5 files changed
+562 -6

No files matched your search

@@ -105,8 +105,8 @@ func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) {
if err := monitor.bootstrap(context.Background()); err != nil || reports != 1 {
t.Fatal(err, reports)
}
if err := monitor.probe(context.Background()); err != nil {
t.Fatal("unresolved GGC stopped API probe", err)
if err := monitor.probe(context.Background()); !errors.Is(err, errManagedGGCUnresolved) {
t.Fatal("unresolved GGC did not fail scope probe", err)
}
}
@@ -167,10 +167,9 @@ func TestManagedMonitorCLIBindingAndCancellation(t *testing.T) {
if m.bootstrap == nil || m.recover == nil || m.interval != 5*time.Minute || m.threshold != 3 || m.cooldown != 30*time.Minute {
t.Fatal("missing wiring/defaults")
}
if err := m.probe(ctx); err != nil {
if err := m.probe(ctx); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
_ = m.probe(ctx)
return ctx.Err()
})
if code != 0 || requests != 2 {
+36 -1
View File
@@ -95,6 +95,11 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
type managedRecoveryFatal struct{ Err error }
var (
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
)
func (e *managedRecoveryFatal) Error() string {
return "managed recovery safety failure: " + e.Err.Error()
}
@@ -478,7 +483,37 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
if status != "OK" {
return &managedHealthRejection{Status: status}
}
return nil
transaction := *recovery.transaction
transaction.Health = func() (string, error) { return recovery.health(ctx) }
// ActiveCandidates is read-only; never permit a probe to reload config.
transaction.Reload = func() error { return fmt.Errorf("probe cannot reload config") }
ids, err := transaction.ActiveCandidates(recovery.db)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
// Keep Player API health separate from GGC scope (not playback health).
// Two bounded Player requests reuse the existing health/discovery adapters.
host, err := recovery.discover(ctx)
if ctx.Err() != nil {
return ctx.Err()
}
var fatal *managedRecoveryFatal
if errors.As(err, &fatal) || errors.Is(err, context.Canceled) {
return err
}
if err != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
return errManagedGGCUnresolved
}
for _, id := range ids {
candidate, err := readStrategyCandidate(recovery.db, id)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
if candidate.Group == "googlevideo" && len(candidate.Endpoints) == 1 && candidate.Endpoints[0] == host {
return nil
}
}
return errManagedGGCScopeChanged
}
return nil
}
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
@@ -569,3 +570,154 @@ func TestManagedRecoveryPlayerAdapter(t *testing.T) {
t.Fatal("wrong target binding")
}
}
func TestManagedRecoveryProbeGGCScope(t *testing.T) {
for _, mode := range []string{"matching", "changed", "missing", "wildcard", "uppercase", "url", "foreign", "empty", "discovery-error", "fatal", "cancel", "drift", "identity", "health-reject"} {
t.Run(mode, func(t *testing.T) {
recovery, front, events := recoveryFixture(t, true)
if mode != "missing" {
cache := managedCandidateFixture(t, recovery.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=multisplit", true)
if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil {
t.Fatal(err)
}
}
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
discoveries := 0
recovery.discover = func(context.Context) (string, error) {
discoveries++
switch mode {
case "changed":
return "other.googlevideo.com", nil
case "wildcard":
return "googlevideo.com", nil
case "uppercase":
return "FRESH.googlevideo.com", nil
case "url":
return "https://fresh.googlevideo.com/video?secret=never-print", nil
case "foreign":
return "fresh.googlevideo.com.evil.example", nil
case "empty":
return "", nil
case "discovery-error":
return "", errors.New("secret=never-print")
case "fatal":
return "", &managedRecoveryFatal{Err: errors.New("unsafe target")}
case "cancel":
cancel()
}
return "fresh.googlevideo.com", nil
}
if mode == "health-reject" {
recovery.health = func(context.Context) (string, error) { return "DEGRADED", nil }
}
if err := wireManagedRecovery(&monitor, recovery, func(error) { t.Fatal("probe invoked recovery report") }); err != nil {
t.Fatal(err)
}
if mode == "drift" {
if err := os.WriteFile(filepath.Join(recovery.transaction.Dir, "config"), []byte("external"), 0600); err != nil {
t.Fatal(err)
}
}
if mode == "identity" {
if err := os.Remove(filepath.Join(recovery.transaction.Dir, managedMarkerName)); err != nil {
t.Fatal(err)
}
}
err := monitor.probe(ctx)
switch mode {
case "matching":
if err != nil {
t.Fatal(err)
}
case "changed", "missing":
if !errors.Is(err, errManagedGGCScopeChanged) {
t.Fatal(err)
}
case "fatal", "drift", "identity":
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) {
t.Fatal(err)
}
case "cancel":
if !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
case "health-reject":
var rejection *managedHealthRejection
if !errors.As(err, &rejection) {
t.Fatal(err)
}
default:
if !errors.Is(err, errManagedGGCUnresolved) || strings.Contains(err.Error(), "never-print") {
t.Fatal(err)
}
}
if len(*events) != 0 || (mode == "drift" || mode == "identity" || mode == "health-reject") && discoveries != 0 {
t.Fatalf("unsafe probe effects: events=%v discoveries=%d", *events, discoveries)
}
})
}
}
func TestManagedRecoveryGGCRotationPolicy(t *testing.T) {
for _, activeCache := range []bool{false, true} {
t.Run(fmt.Sprint(activeCache), func(t *testing.T) {
recovery, front, events := recoveryFixture(t, true)
if err := os.Chmod(recovery.transaction.Dir, 0700); err != nil {
t.Fatal(err)
}
if activeCache {
cache := managedCandidateFixture(t, recovery.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil {
t.Fatal(err)
}
}
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
if err := wireManagedRecovery(&monitor, recovery, func(err error) { t.Fatal(err) }); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probes := 0
probe := monitor.probe
monitor.probe = func(ctx context.Context) error { probes++; return probe(ctx) }
monitor.wait = func(ctx context.Context, _ time.Duration) error {
if probes < monitor.threshold && len(*events) != 0 {
t.Fatal("recovered before threshold", *events)
}
if probes == monitor.threshold+1 {
cancel()
return ctx.Err()
}
return nil
}
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
ids, err := recovery.transaction.ActiveCandidates(recovery.db)
if err != nil || len(ids) != 2 || ids[0] != front {
t.Fatal(ids, err)
}
cache, err := readStrategyCandidate(recovery.db, ids[1])
if err != nil || !reflect.DeepEqual(cache.Endpoints, []string{"fresh.googlevideo.com"}) {
t.Fatal(cache, err)
}
state, err := readMonitorState(monitor.db, monitor.serviceID)
if err != nil || state.CooldownUntil.IsZero() || strings.Contains(strings.Join(*events, ","), "scan:"+managedFrontendHost) {
t.Fatal(state, err, *events)
}
})
}
}
+113 -1
View File
@@ -4,14 +4,17 @@ import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"regexp"
"strconv"
"strings"
)
// Only complete, well-formed non-passing measurements permit scan fallback.
// Only validated measurements or authenticated early timeouts permit fallback.
var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols")
const managedFrontendHost = "www.youtube.com"
@@ -52,6 +55,10 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
if err := os.Remove(path + ".combined.json"); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
// The owned runner creates this with O_EXCL. Never accept stale diagnostics.
if _, err := os.Lstat(path + ".combined.json.stderr"); !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("candidate diagnostic artifact already exists or cannot be inspected: %v", err)
}
var raw bytes.Buffer
err := check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw)
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
@@ -72,12 +79,117 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
return nil, err
}
_, err = selectBlockcheckEvidence(bytes.NewReader(report), c.Args, group, host, true)
if err != nil && !errors.Is(err, errCandidateNonpass) {
if diagnostics, readErr := os.ReadFile(path + ".combined.json.stderr"); readErr == nil && candidateEarlyTimeout(report, diagnostics, c.Args, group, host) {
err = errCandidateNonpass
}
}
if contextErr := ctx.Err(); contextErr != nil {
return nil, contextErr
}
return report, err
}
var earlyTimeoutResult = regexp.MustCompile(`^ FAIL ([0-2])/([1-3]) timeout$`)
var earlySuccessResult = regexp.MustCompile(`^ OK median [0-9]+ms, [0-9]+\.[0-9] KB/s$`)
var earlyCheckPreamble = regexp.MustCompile("^(! --workers has no effect on `check` \\(it verifies strategies sequentially by design\\)\n)?" +
`=== Checking prerequisites ===\n OK nfqws2: /[^\s]+\n OK nft\n OK nft queue support\n` +
`check loaded 2 strategies from /[^\s]+, --take [0-9]+, --passes 3\n( ISP: AS[0-9]+ [^\n]+\n)?` +
`=== DNS resolve ===\n dns mode: auto\n ([a-z0-9.-]+) -> [0-9a-fA-F:.]+ \(via doh \(auto-fallback\)\)$`)
var earlyCheckSummary = regexp.MustCompile(`^ total: 2 \| working: ([01]) \| elapsed: [0-9]+\.[0-9]s\n OK JSON report → /[^\s]+$`)
// This is rejection evidence only, never qualification evidence. Unknown CLI
// output fails closed; changes to the checker format require a new fixture.
func candidateEarlyTimeout(raw, diagnostics []byte, args, group, host string) bool {
var report struct {
Domain string `json:"domain"`
Total *int `json:"total"`
Working *int `json:"working"`
Strategies *[]json.RawMessage `json:"strategies"`
}
d := json.NewDecoder(bytes.NewReader(raw))
if d.Decode(&report) != nil || d.Decode(new(json.RawMessage)) != io.EOF || report.Domain != host || report.Total == nil || *report.Total != 2 || report.Working == nil || *report.Working < 0 || *report.Working >= 2 || report.Strategies == nil || len(*report.Strategies) != *report.Working {
return false
}
success := ""
if *report.Working == 1 {
row := (*report.Strategies)[0]
var fields map[string]json.RawMessage
if json.Unmarshal(row, &fields) != nil || json.Unmarshal(fields["protocol"], &success) != nil || normalizeProtocol(success) == "" {
return false
}
other := "HTTPS/TLS1.2"
if success == other {
other = "HTTPS/TLS1.3"
}
fields["protocol"], _ = json.Marshal(other)
copyRow, err := json.Marshal(fields)
if err != nil {
return false
}
// Reuse every strict measurement/target/args check, including exactly
// 3/3, by supplying a protocol-flipped copy solely for validation.
complete, err := json.Marshal(map[string]any{"strategies": []json.RawMessage{row, copyRow}})
if err != nil {
return false
}
if _, err := selectBlockcheckEvidence(bytes.NewReader(complete), args, group, host, false); err != nil {
return false
}
}
lines := strings.Split(strings.ReplaceAll(string(diagnostics), "\r\n", "\n"), "\n")
var evidence []string
for _, line := range lines {
if line != "" {
evidence = append(evidence, line)
}
}
// ponytail: pin the known CLI framing and metadata grammar; new checker
// formats need explicit fixtures, not permissive infrastructure filtering.
parts := strings.Split(strings.Join(evidence, "\n"), "\n=== Checking strategies (data transfer) ===\n")
if len(parts) != 2 {
return false
}
preamble := earlyCheckPreamble.FindStringSubmatch(parts[0])
if preamble == nil || preamble[3] != host {
return false
}
parts = strings.Split(parts[1], "\n=== Check summary ===\n")
if len(parts) != 2 {
return false
}
summary := earlyCheckSummary.FindStringSubmatch(parts[1])
if summary == nil || summary[1] != strconv.Itoa(*report.Working) {
return false
}
evidence = strings.Split(parts[0], "\n")
if len(evidence) != 7 || evidence[0] != " Tip: use --take 10 to stop after 10 verified per protocol" || evidence[1] != " Verifying 2 strategies (3 passes, early-exit on first fail)" || evidence[4] != " ────────────────────────────────────────────────────────────" {
return false
}
for i, protocol := range []string{"HTTPS/TLS1.2", "HTTPS/TLS1.3"} {
if evidence[2+3*i] != fmt.Sprintf(" [%d/2] %s nfqws2 %s", i+1, protocol, args) {
return false
}
result := evidence[3+3*i]
if protocol == success {
if !earlySuccessResult.MatchString(result) {
return false
}
} else {
match := earlyTimeoutResult.FindStringSubmatch(result)
if match == nil {
return false
}
n, _ := strconv.Atoi(match[1])
m, _ := strconv.Atoi(match[2])
if n != m-1 {
return false
}
}
}
return true
}
func revalidateGooglevideo(ctx context.Context, db *sql.DB, host string, stderr io.Writer, check cacheCheck) (string, []byte, error) {
ids, err := qualifiedCandidateIDs(db, "googlevideo")
if err != nil {
@@ -210,3 +210,261 @@ func TestRevalidateCandidateScope(t *testing.T) {
t.Fatalf("changed stored scope: %+v %v", stored, err)
}
}
func earlyTimeoutDiagnosticsFixture(args, tls12, tls13 string, working int) string {
return fmt.Sprintf("! --workers has no effect on `check` (it verifies strategies sequentially by design)\n"+`=== Checking prerequisites ===
OK nfqws2: /opt/zapret2/binaries/linux-x86_64/nfqws2
OK nft
OK nft queue support
check loaded 2 strategies from /tmp/input.txt, --take 1, --passes 3
ISP: AS3175 ...
=== DNS resolve ===
dns mode: auto
www.youtube.com -> 142.251.156.4 (via doh (auto-fallback))
=== Checking strategies (data transfer) ===
Tip: use --take 10 to stop after 10 verified per protocol
Verifying 2 strategies (3 passes, early-exit on first fail)
[1/2] HTTPS/TLS1.2 nfqws2 %s
%s
────────────────────────────────────────────────────────────
[2/2] HTTPS/TLS1.3 nfqws2 %s
%s
=== Check summary ===
total: 2 | working: %d | elapsed: 7.7s
OK JSON report → /tmp/input.txt.combined.json
`, args, tls12, args, tls13, working)
}
func TestCandidateEarlyTimeoutRowValidation(t *testing.T) {
args := "--lua-desync=multisplit"
var passing struct {
Strategies []json.RawMessage `json:"strategies"`
}
if err := json.Unmarshal([]byte(combinedReportFixture(args)), &passing); err != nil {
t.Fatal(err)
}
diagnostics := []byte(earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", "OK median 242ms, 3728.2 KB/s", 1))
check := func(t *testing.T, field string, value json.RawMessage) {
t.Helper()
var row map[string]json.RawMessage
if err := json.Unmarshal(passing.Strategies[1], &row); err != nil {
t.Fatal(err)
}
if value == nil {
delete(row, field)
} else {
row[field] = value
}
report, err := json.Marshal(map[string]any{"domain": "www.youtube.com", "total": 2, "working": 1, "strategies": []any{row}})
if err != nil {
t.Fatal(err)
}
if candidateEarlyTimeout(report, diagnostics, args, "frontend", "www.youtube.com") {
t.Fatal("malformed passing row authorized fallback")
}
}
for _, field := range []string{"protocol", "args", "success_rate", "coverage", "passes_ok", "passes_total", "median_latency_ms", "median_speed_kbps"} {
for _, mutation := range []struct {
name string
value json.RawMessage
}{{"missing", nil}, {"null", json.RawMessage(`null`)}, {"wrong-type", json.RawMessage(`{}`)}} {
t.Run(field+"/"+mutation.name, func(t *testing.T) { check(t, field, mutation.value) })
}
}
for _, mutation := range []struct{ field, value string }{
{"protocol", `"HTTP"`}, {"args", `"--lua-desync=fake"`},
{"group", `"googlevideo"`}, {"domains", `["other.example"]`}, {"domains", `[]`},
{"success_rate", `0.9`}, {"success_rate", `1.1`}, {"success_rate", `-1`},
{"coverage", `0.9`}, {"coverage", `1.1`}, {"coverage", `-1`},
{"passes_ok", `2`}, {"passes_ok", `4`}, {"passes_ok", `-1`},
{"passes_total", `2`}, {"passes_total", `4`}, {"passes_total", `-1`},
{"median_latency_ms", `-1`}, {"median_speed_kbps", `-1`},
} {
t.Run(mutation.field+"/"+mutation.value, func(t *testing.T) { check(t, mutation.field, json.RawMessage(mutation.value)) })
}
}
func TestRevalidateCandidateEarlyTimeout(t *testing.T) {
db := testStore(t)
args := "--lua-desync=multisplit"
saveProposal(t, db, args)
c, err := readStrategyCandidate(db, 1)
if err != nil {
t.Fatal(err)
}
var passing struct {
Strategies []json.RawMessage `json:"strategies"`
}
if err := json.Unmarshal([]byte(combinedReportFixture(args)), &passing); err != nil {
t.Fatal(err)
}
report := fmt.Sprintf(`{"domain":"www.youtube.com","total":2,"working":1,"strategies":[%s]}`, passing.Strategies[1])
okResult := "OK median 242ms, 3728.2 KB/s"
diagnostics := earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", okResult, 1)
malformedOK := map[string]string{
"ok-garbage": "OK garbage median garbage",
"ok-old-fixture": "OK 3/3 median 10ms 100kbps",
"ok-missing-latency": "OK median ms, 3728.2 KB/s",
"ok-negative-latency": "OK median -242ms, 3728.2 KB/s",
"ok-fractional-latency": "OK median 242.5ms, 3728.2 KB/s",
"ok-missing-speed": "OK median 242ms, KB/s",
"ok-negative-speed": "OK median 242ms, -3728.2 KB/s",
"ok-nan-speed": "OK median 242ms, NaN KB/s",
"ok-inf-speed": "OK median 242ms, Inf KB/s",
"ok-wrong-units": "OK median 242ms, 3728.2 kbps",
"ok-missing-comma": "OK median 242ms 3728.2 KB/s",
"ok-trailing-garbage": okResult + " garbage",
"ok-trailing-space": okResult + " ",
}
modes := []string{
"timeout", "two-timeouts", "tls13-timeout", "wrong-domain", "wrong-row-args", "wrong-heading-args", "missing-artifact", "stale-artifact", "missing-report", "runner-failed", "runner-sentinel", "cancelled",
"nfq-exited", "extra-line", "missing-heading", "duplicate-heading", "wrong-protocol", "missing-fail", "success-not-ok", "fail-on-success", "wrong-total", "missing-total", "negative-working", "wrong-working", "working-two", "missing-working", "null-strategies", "trailing-json", "bad-json",
"zero-count", "equal-counts", "reversed-counts", "excess-count", "not-timeout", "partial-success", "missing-metric", "invalid-metric", "wrong-row-target", "empty-with-one-success",
"first-pass-timeout", "third-pass-timeout", "two-first-pass-timeouts", "unknown-section-line", "infra-section-line", "missing-section-marker", "missing-summary-marker", "duplicate-section", "wrong-verifying-count", "wrong-loaded-count", "wrong-summary-count", "wrong-summary-working", "unindented-heading", "extra-indent-heading", "wrong-dns-host", "unknown-preamble-line",
}
for mode := range malformedOK {
modes = append(modes, mode)
}
for _, mode := range modes {
t.Run(mode, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "check")
r, diagnostic := report, diagnostics
if result, ok := malformedOK[mode]; ok {
diagnostic = strings.Replace(diagnostic, okResult, result, 1)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
switch mode {
case "two-timeouts", "empty-with-one-success":
r = `{"domain":"www.youtube.com","total":2,"working":0,"strategies":[]}`
if mode == "two-timeouts" {
diagnostic = earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", "FAIL 2/3 timeout", 0)
}
case "two-first-pass-timeouts":
r = `{"domain":"www.youtube.com","total":2,"working":0,"strategies":[]}`
diagnostic = earlyTimeoutDiagnosticsFixture(args, "FAIL 0/1 timeout", "FAIL 0/1 timeout", 0)
case "first-pass-timeout":
diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", "FAIL 0/1 timeout", 1)
case "third-pass-timeout":
diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", "FAIL 2/3 timeout", 1)
case "tls13-timeout":
r = fmt.Sprintf(`{"domain":"www.youtube.com","total":2,"working":1,"strategies":[%s]}`, passing.Strategies[0])
diagnostic = earlyTimeoutDiagnosticsFixture(args, okResult, "FAIL 1/2 timeout", 1)
case "wrong-domain":
r = strings.Replace(r, "www.youtube.com", "other.example", 1)
case "wrong-row-args":
r = strings.Replace(r, args, "--lua-desync=fake", 1)
case "wrong-heading-args":
diagnostic = strings.Replace(diagnostic, args, "--lua-desync=fake", 1)
case "nfq-exited":
diagnostic += "nfqws2 exited with status 1\n"
case "extra-line":
diagnostic += "unexpected infrastructure output\n"
case "missing-heading":
diagnostic = strings.Replace(diagnostic, " [1/2] HTTPS/TLS1.2 nfqws2 "+args+"\n", "", 1)
case "duplicate-heading":
diagnostic = strings.Replace(diagnostic, " [1/2]", " [1/2] HTTPS/TLS1.2 nfqws2 "+args+"\n [1/2]", 1)
case "unknown-section-line", "infra-section-line":
line := "unknown output"
if mode == "infra-section-line" {
line = "nfqws2 exited with status 1"
}
diagnostic = strings.Replace(diagnostic, " FAIL", line+"\n FAIL", 1)
case "missing-section-marker", "missing-summary-marker", "duplicate-section", "wrong-verifying-count", "wrong-loaded-count", "wrong-summary-count", "wrong-summary-working", "unindented-heading", "extra-indent-heading", "wrong-dns-host", "unknown-preamble-line":
replacements := map[string][2]string{
"missing-section-marker": {"=== Checking strategies (data transfer) ===\n", ""},
"missing-summary-marker": {"=== Check summary ===\n", ""},
"duplicate-section": {"=== Checking strategies (data transfer) ===", "=== Checking strategies (data transfer) ===\n=== Checking strategies (data transfer) ==="},
"wrong-verifying-count": {"Verifying 2 strategies", "Verifying 3 strategies"},
"wrong-loaded-count": {"loaded 2 strategies", "loaded 3 strategies"},
"wrong-summary-count": {"total: 2", "total: 3"},
"wrong-summary-working": {"working: 1", "working: 0"},
"unindented-heading": {" [1/2]", "[1/2]"},
"extra-indent-heading": {" [1/2]", " [1/2]"},
"wrong-dns-host": {"www.youtube.com ->", "other.example ->"},
"unknown-preamble-line": {"=== DNS resolve ===", "unknown output\n=== DNS resolve ==="},
}
pair := replacements[mode]
diagnostic = strings.Replace(diagnostic, pair[0], pair[1], 1)
case "wrong-protocol":
diagnostic = strings.Replace(diagnostic, "HTTPS/TLS1.2", "HTTPS/TLS1.3", 1)
case "missing-fail":
diagnostic = strings.Replace(diagnostic, " FAIL 1/2 timeout\n", "", 1)
case "success-not-ok":
diagnostic = strings.Replace(diagnostic, "OK median", "unknown median", 1)
case "fail-on-success":
diagnostic = strings.Replace(diagnostic, okResult, "FAIL 1/2 timeout", 1)
case "wrong-total", "missing-total", "negative-working", "wrong-working", "working-two", "missing-working", "null-strategies":
replacements := map[string][2]string{
"wrong-total": {`"total":2`, `"total":3`}, "missing-total": {`"total":2,`, ""},
"negative-working": {`"working":1`, `"working":-1`}, "wrong-working": {`"working":1`, `"working":0`},
"working-two": {`"working":1`, `"working":2`}, "missing-working": {`"working":1,`, ""},
"null-strategies": {`"strategies":[` + string(passing.Strategies[1]) + `]`, `"strategies":null`},
}
pair := replacements[mode]
r = strings.Replace(r, pair[0], pair[1], 1)
case "trailing-json":
r += `{}`
case "bad-json":
r = `{`
case "zero-count", "equal-counts", "reversed-counts", "excess-count", "not-timeout":
replacements := map[string]string{"zero-count": "FAIL 0/2 timeout", "equal-counts": "FAIL 2/2 timeout", "reversed-counts": "FAIL 3/2 timeout", "excess-count": "FAIL 1/4 timeout", "not-timeout": "FAIL 1/2 connection refused"}
diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", replacements[mode], 1)
case "partial-success":
r = strings.Replace(r, `"passes_ok":3`, `"passes_ok":2`, 1)
case "missing-metric":
r = strings.Replace(r, `"coverage":1`, `"coverage":null`, 1)
case "invalid-metric":
r = strings.Replace(r, `"median_speed_kbps":100`, `"median_speed_kbps":-1`, 1)
case "wrong-row-target":
r = strings.Replace(r, `"protocol":`, `"domains":["other.example"],"protocol":`, 1)
}
if mode == "stale-artifact" {
if err := os.WriteFile(path+".combined.json.stderr", []byte(diagnostic), 0600); err != nil {
t.Fatal(err)
}
}
calls := 0
check := func(_ context.Context, _, gotPath, gotArgs, host string, _, _ io.Writer) error {
calls++
if gotPath != path || gotArgs != args || host != "www.youtube.com" {
t.Fatal("unexpected checker request")
}
if mode != "missing-report" {
if err := os.WriteFile(path+".combined.json", []byte(r), 0600); err != nil {
return err
}
}
if mode != "missing-artifact" {
if err := os.WriteFile(path+".combined.json.stderr", []byte(diagnostic), 0600); err != nil {
return err
}
}
switch mode {
case "runner-failed":
return fmt.Errorf("checker exited")
case "runner-sentinel":
return errCandidateNonpass
case "cancelled":
cancel()
}
return nil
}
_, err := revalidateCandidate(ctx, c, "frontend", "www.youtube.com", path, io.Discard, check)
nonpass := mode == "timeout" || mode == "two-timeouts" || mode == "tls13-timeout" || mode == "first-pass-timeout" || mode == "third-pass-timeout" || mode == "two-first-pass-timeouts"
if err == nil || errors.Is(err, errCandidateNonpass) != nonpass {
t.Fatalf("nonpass=%v err=%v", nonpass, err)
}
if nonpass {
if _, err := combinedBlockcheckOutputForHost(strings.NewReader(r), args, "www.youtube.com"); err == nil {
t.Fatal("early timeout report qualified for apply")
}
}
if mode == "stale-artifact" && calls != 0 {
t.Fatal("checker invoked with stale diagnostic evidence")
}
})
}
}