fix(recovery): handle checker early timeouts and GGC rotation
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
741009afa7
commit
beb4f16309
5 files changed
+562
-6
No files matched your search
@@ -105,8 +105,8 @@ func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) {
|
||||
if err := monitor.bootstrap(context.Background()); err != nil || reports != 1 {
|
||||
t.Fatal(err, reports)
|
||||
}
|
||||
if err := monitor.probe(context.Background()); err != nil {
|
||||
t.Fatal("unresolved GGC stopped API probe", err)
|
||||
if err := monitor.probe(context.Background()); !errors.Is(err, errManagedGGCUnresolved) {
|
||||
t.Fatal("unresolved GGC did not fail scope probe", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,10 +167,9 @@ func TestManagedMonitorCLIBindingAndCancellation(t *testing.T) {
|
||||
if m.bootstrap == nil || m.recover == nil || m.interval != 5*time.Minute || m.threshold != 3 || m.cooldown != 30*time.Minute {
|
||||
t.Fatal("missing wiring/defaults")
|
||||
}
|
||||
if err := m.probe(ctx); err != nil {
|
||||
if err := m.probe(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = m.probe(ctx)
|
||||
return ctx.Err()
|
||||
})
|
||||
if code != 0 || requests != 2 {
|
||||
|
||||
@@ -95,6 +95,11 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
|
||||
|
||||
type managedRecoveryFatal struct{ Err error }
|
||||
|
||||
var (
|
||||
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
|
||||
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
|
||||
)
|
||||
|
||||
func (e *managedRecoveryFatal) Error() string {
|
||||
return "managed recovery safety failure: " + e.Err.Error()
|
||||
}
|
||||
@@ -478,7 +483,37 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
||||
if status != "OK" {
|
||||
return &managedHealthRejection{Status: status}
|
||||
}
|
||||
return nil
|
||||
transaction := *recovery.transaction
|
||||
transaction.Health = func() (string, error) { return recovery.health(ctx) }
|
||||
// ActiveCandidates is read-only; never permit a probe to reload config.
|
||||
transaction.Reload = func() error { return fmt.Errorf("probe cannot reload config") }
|
||||
ids, err := transaction.ActiveCandidates(recovery.db)
|
||||
if err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
// Keep Player API health separate from GGC scope (not playback health).
|
||||
// Two bounded Player requests reuse the existing health/discovery adapters.
|
||||
host, err := recovery.discover(ctx)
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(err, &fatal) || errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
if err != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
|
||||
return errManagedGGCUnresolved
|
||||
}
|
||||
for _, id := range ids {
|
||||
candidate, err := readStrategyCandidate(recovery.db, id)
|
||||
if err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
if candidate.Group == "googlevideo" && len(candidate.Endpoints) == 1 && candidate.Endpoints[0] == host {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errManagedGGCScopeChanged
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -569,3 +570,154 @@ func TestManagedRecoveryPlayerAdapter(t *testing.T) {
|
||||
t.Fatal("wrong target binding")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryProbeGGCScope(t *testing.T) {
|
||||
for _, mode := range []string{"matching", "changed", "missing", "wildcard", "uppercase", "url", "foreign", "empty", "discovery-error", "fatal", "cancel", "drift", "identity", "health-reject"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
recovery, front, events := recoveryFixture(t, true)
|
||||
if mode != "missing" {
|
||||
cache := managedCandidateFixture(t, recovery.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=multisplit", true)
|
||||
if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
||||
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
discoveries := 0
|
||||
recovery.discover = func(context.Context) (string, error) {
|
||||
discoveries++
|
||||
switch mode {
|
||||
case "changed":
|
||||
return "other.googlevideo.com", nil
|
||||
case "wildcard":
|
||||
return "googlevideo.com", nil
|
||||
case "uppercase":
|
||||
return "FRESH.googlevideo.com", nil
|
||||
case "url":
|
||||
return "https://fresh.googlevideo.com/video?secret=never-print", nil
|
||||
case "foreign":
|
||||
return "fresh.googlevideo.com.evil.example", nil
|
||||
case "empty":
|
||||
return "", nil
|
||||
case "discovery-error":
|
||||
return "", errors.New("secret=never-print")
|
||||
case "fatal":
|
||||
return "", &managedRecoveryFatal{Err: errors.New("unsafe target")}
|
||||
case "cancel":
|
||||
cancel()
|
||||
}
|
||||
return "fresh.googlevideo.com", nil
|
||||
}
|
||||
if mode == "health-reject" {
|
||||
recovery.health = func(context.Context) (string, error) { return "DEGRADED", nil }
|
||||
}
|
||||
if err := wireManagedRecovery(&monitor, recovery, func(error) { t.Fatal("probe invoked recovery report") }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mode == "drift" {
|
||||
if err := os.WriteFile(filepath.Join(recovery.transaction.Dir, "config"), []byte("external"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if mode == "identity" {
|
||||
if err := os.Remove(filepath.Join(recovery.transaction.Dir, managedMarkerName)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
err := monitor.probe(ctx)
|
||||
switch mode {
|
||||
case "matching":
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "changed", "missing":
|
||||
if !errors.Is(err, errManagedGGCScopeChanged) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "fatal", "drift", "identity":
|
||||
var fatal *managedRecoveryFatal
|
||||
if !errors.As(err, &fatal) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "cancel":
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "health-reject":
|
||||
var rejection *managedHealthRejection
|
||||
if !errors.As(err, &rejection) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
default:
|
||||
if !errors.Is(err, errManagedGGCUnresolved) || strings.Contains(err.Error(), "never-print") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(*events) != 0 || (mode == "drift" || mode == "identity" || mode == "health-reject") && discoveries != 0 {
|
||||
t.Fatalf("unsafe probe effects: events=%v discoveries=%d", *events, discoveries)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryGGCRotationPolicy(t *testing.T) {
|
||||
for _, activeCache := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(activeCache), func(t *testing.T) {
|
||||
recovery, front, events := recoveryFixture(t, true)
|
||||
if err := os.Chmod(recovery.transaction.Dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if activeCache {
|
||||
cache := managedCandidateFixture(t, recovery.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
|
||||
if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
||||
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := wireManagedRecovery(&monitor, recovery, func(err error) { t.Fatal(err) }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
probes := 0
|
||||
probe := monitor.probe
|
||||
monitor.probe = func(ctx context.Context) error { probes++; return probe(ctx) }
|
||||
monitor.wait = func(ctx context.Context, _ time.Duration) error {
|
||||
if probes < monitor.threshold && len(*events) != 0 {
|
||||
t.Fatal("recovered before threshold", *events)
|
||||
}
|
||||
if probes == monitor.threshold+1 {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids, err := recovery.transaction.ActiveCandidates(recovery.db)
|
||||
if err != nil || len(ids) != 2 || ids[0] != front {
|
||||
t.Fatal(ids, err)
|
||||
}
|
||||
cache, err := readStrategyCandidate(recovery.db, ids[1])
|
||||
if err != nil || !reflect.DeepEqual(cache.Endpoints, []string{"fresh.googlevideo.com"}) {
|
||||
t.Fatal(cache, err)
|
||||
}
|
||||
state, err := readMonitorState(monitor.db, monitor.serviceID)
|
||||
if err != nil || state.CooldownUntil.IsZero() || strings.Contains(strings.Join(*events, ","), "scan:"+managedFrontendHost) {
|
||||
t.Fatal(state, err, *events)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,14 +4,17 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Only complete, well-formed non-passing measurements permit scan fallback.
|
||||
// Only validated measurements or authenticated early timeouts permit fallback.
|
||||
var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols")
|
||||
|
||||
const managedFrontendHost = "www.youtube.com"
|
||||
@@ -52,6 +55,10 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
|
||||
if err := os.Remove(path + ".combined.json"); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, err
|
||||
}
|
||||
// The owned runner creates this with O_EXCL. Never accept stale diagnostics.
|
||||
if _, err := os.Lstat(path + ".combined.json.stderr"); !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("candidate diagnostic artifact already exists or cannot be inspected: %v", err)
|
||||
}
|
||||
var raw bytes.Buffer
|
||||
err := check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw)
|
||||
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
|
||||
@@ -72,12 +79,117 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
|
||||
return nil, err
|
||||
}
|
||||
_, err = selectBlockcheckEvidence(bytes.NewReader(report), c.Args, group, host, true)
|
||||
if err != nil && !errors.Is(err, errCandidateNonpass) {
|
||||
if diagnostics, readErr := os.ReadFile(path + ".combined.json.stderr"); readErr == nil && candidateEarlyTimeout(report, diagnostics, c.Args, group, host) {
|
||||
err = errCandidateNonpass
|
||||
}
|
||||
}
|
||||
if contextErr := ctx.Err(); contextErr != nil {
|
||||
return nil, contextErr
|
||||
}
|
||||
return report, err
|
||||
}
|
||||
|
||||
var earlyTimeoutResult = regexp.MustCompile(`^ FAIL ([0-2])/([1-3]) timeout$`)
|
||||
var earlySuccessResult = regexp.MustCompile(`^ OK median [0-9]+ms, [0-9]+\.[0-9] KB/s$`)
|
||||
var earlyCheckPreamble = regexp.MustCompile("^(! --workers has no effect on `check` \\(it verifies strategies sequentially by design\\)\n)?" +
|
||||
`=== Checking prerequisites ===\n OK nfqws2: /[^\s]+\n OK nft\n OK nft queue support\n` +
|
||||
`check loaded 2 strategies from /[^\s]+, --take [0-9]+, --passes 3\n( ISP: AS[0-9]+ [^\n]+\n)?` +
|
||||
`=== DNS resolve ===\n dns mode: auto\n ([a-z0-9.-]+) -> [0-9a-fA-F:.]+ \(via doh \(auto-fallback\)\)$`)
|
||||
var earlyCheckSummary = regexp.MustCompile(`^ total: 2 \| working: ([01]) \| elapsed: [0-9]+\.[0-9]s\n OK JSON report → /[^\s]+$`)
|
||||
|
||||
// This is rejection evidence only, never qualification evidence. Unknown CLI
|
||||
// output fails closed; changes to the checker format require a new fixture.
|
||||
func candidateEarlyTimeout(raw, diagnostics []byte, args, group, host string) bool {
|
||||
var report struct {
|
||||
Domain string `json:"domain"`
|
||||
Total *int `json:"total"`
|
||||
Working *int `json:"working"`
|
||||
Strategies *[]json.RawMessage `json:"strategies"`
|
||||
}
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
if d.Decode(&report) != nil || d.Decode(new(json.RawMessage)) != io.EOF || report.Domain != host || report.Total == nil || *report.Total != 2 || report.Working == nil || *report.Working < 0 || *report.Working >= 2 || report.Strategies == nil || len(*report.Strategies) != *report.Working {
|
||||
return false
|
||||
}
|
||||
success := ""
|
||||
if *report.Working == 1 {
|
||||
row := (*report.Strategies)[0]
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(row, &fields) != nil || json.Unmarshal(fields["protocol"], &success) != nil || normalizeProtocol(success) == "" {
|
||||
return false
|
||||
}
|
||||
other := "HTTPS/TLS1.2"
|
||||
if success == other {
|
||||
other = "HTTPS/TLS1.3"
|
||||
}
|
||||
fields["protocol"], _ = json.Marshal(other)
|
||||
copyRow, err := json.Marshal(fields)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
// Reuse every strict measurement/target/args check, including exactly
|
||||
// 3/3, by supplying a protocol-flipped copy solely for validation.
|
||||
complete, err := json.Marshal(map[string]any{"strategies": []json.RawMessage{row, copyRow}})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if _, err := selectBlockcheckEvidence(bytes.NewReader(complete), args, group, host, false); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
lines := strings.Split(strings.ReplaceAll(string(diagnostics), "\r\n", "\n"), "\n")
|
||||
var evidence []string
|
||||
for _, line := range lines {
|
||||
if line != "" {
|
||||
evidence = append(evidence, line)
|
||||
}
|
||||
}
|
||||
// ponytail: pin the known CLI framing and metadata grammar; new checker
|
||||
// formats need explicit fixtures, not permissive infrastructure filtering.
|
||||
parts := strings.Split(strings.Join(evidence, "\n"), "\n=== Checking strategies (data transfer) ===\n")
|
||||
if len(parts) != 2 {
|
||||
return false
|
||||
}
|
||||
preamble := earlyCheckPreamble.FindStringSubmatch(parts[0])
|
||||
if preamble == nil || preamble[3] != host {
|
||||
return false
|
||||
}
|
||||
parts = strings.Split(parts[1], "\n=== Check summary ===\n")
|
||||
if len(parts) != 2 {
|
||||
return false
|
||||
}
|
||||
summary := earlyCheckSummary.FindStringSubmatch(parts[1])
|
||||
if summary == nil || summary[1] != strconv.Itoa(*report.Working) {
|
||||
return false
|
||||
}
|
||||
evidence = strings.Split(parts[0], "\n")
|
||||
if len(evidence) != 7 || evidence[0] != " Tip: use --take 10 to stop after 10 verified per protocol" || evidence[1] != " Verifying 2 strategies (3 passes, early-exit on first fail)" || evidence[4] != " ────────────────────────────────────────────────────────────" {
|
||||
return false
|
||||
}
|
||||
for i, protocol := range []string{"HTTPS/TLS1.2", "HTTPS/TLS1.3"} {
|
||||
if evidence[2+3*i] != fmt.Sprintf(" [%d/2] %s nfqws2 %s", i+1, protocol, args) {
|
||||
return false
|
||||
}
|
||||
result := evidence[3+3*i]
|
||||
if protocol == success {
|
||||
if !earlySuccessResult.MatchString(result) {
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
match := earlyTimeoutResult.FindStringSubmatch(result)
|
||||
if match == nil {
|
||||
return false
|
||||
}
|
||||
n, _ := strconv.Atoi(match[1])
|
||||
m, _ := strconv.Atoi(match[2])
|
||||
if n != m-1 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func revalidateGooglevideo(ctx context.Context, db *sql.DB, host string, stderr io.Writer, check cacheCheck) (string, []byte, error) {
|
||||
ids, err := qualifiedCandidateIDs(db, "googlevideo")
|
||||
if err != nil {
|
||||
|
||||
@@ -210,3 +210,261 @@ func TestRevalidateCandidateScope(t *testing.T) {
|
||||
t.Fatalf("changed stored scope: %+v %v", stored, err)
|
||||
}
|
||||
}
|
||||
|
||||
func earlyTimeoutDiagnosticsFixture(args, tls12, tls13 string, working int) string {
|
||||
return fmt.Sprintf("! --workers has no effect on `check` (it verifies strategies sequentially by design)\n"+`=== Checking prerequisites ===
|
||||
OK nfqws2: /opt/zapret2/binaries/linux-x86_64/nfqws2
|
||||
OK nft
|
||||
OK nft queue support
|
||||
check loaded 2 strategies from /tmp/input.txt, --take 1, --passes 3
|
||||
ISP: AS3175 ...
|
||||
=== DNS resolve ===
|
||||
dns mode: auto
|
||||
www.youtube.com -> 142.251.156.4 (via doh (auto-fallback))
|
||||
|
||||
=== Checking strategies (data transfer) ===
|
||||
Tip: use --take 10 to stop after 10 verified per protocol
|
||||
Verifying 2 strategies (3 passes, early-exit on first fail)
|
||||
[1/2] HTTPS/TLS1.2 nfqws2 %s
|
||||
%s
|
||||
────────────────────────────────────────────────────────────
|
||||
[2/2] HTTPS/TLS1.3 nfqws2 %s
|
||||
%s
|
||||
|
||||
=== Check summary ===
|
||||
total: 2 | working: %d | elapsed: 7.7s
|
||||
OK JSON report → /tmp/input.txt.combined.json
|
||||
`, args, tls12, args, tls13, working)
|
||||
}
|
||||
|
||||
func TestCandidateEarlyTimeoutRowValidation(t *testing.T) {
|
||||
args := "--lua-desync=multisplit"
|
||||
var passing struct {
|
||||
Strategies []json.RawMessage `json:"strategies"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(combinedReportFixture(args)), &passing); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
diagnostics := []byte(earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", "OK median 242ms, 3728.2 KB/s", 1))
|
||||
check := func(t *testing.T, field string, value json.RawMessage) {
|
||||
t.Helper()
|
||||
var row map[string]json.RawMessage
|
||||
if err := json.Unmarshal(passing.Strategies[1], &row); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value == nil {
|
||||
delete(row, field)
|
||||
} else {
|
||||
row[field] = value
|
||||
}
|
||||
report, err := json.Marshal(map[string]any{"domain": "www.youtube.com", "total": 2, "working": 1, "strategies": []any{row}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if candidateEarlyTimeout(report, diagnostics, args, "frontend", "www.youtube.com") {
|
||||
t.Fatal("malformed passing row authorized fallback")
|
||||
}
|
||||
}
|
||||
for _, field := range []string{"protocol", "args", "success_rate", "coverage", "passes_ok", "passes_total", "median_latency_ms", "median_speed_kbps"} {
|
||||
for _, mutation := range []struct {
|
||||
name string
|
||||
value json.RawMessage
|
||||
}{{"missing", nil}, {"null", json.RawMessage(`null`)}, {"wrong-type", json.RawMessage(`{}`)}} {
|
||||
t.Run(field+"/"+mutation.name, func(t *testing.T) { check(t, field, mutation.value) })
|
||||
}
|
||||
}
|
||||
for _, mutation := range []struct{ field, value string }{
|
||||
{"protocol", `"HTTP"`}, {"args", `"--lua-desync=fake"`},
|
||||
{"group", `"googlevideo"`}, {"domains", `["other.example"]`}, {"domains", `[]`},
|
||||
{"success_rate", `0.9`}, {"success_rate", `1.1`}, {"success_rate", `-1`},
|
||||
{"coverage", `0.9`}, {"coverage", `1.1`}, {"coverage", `-1`},
|
||||
{"passes_ok", `2`}, {"passes_ok", `4`}, {"passes_ok", `-1`},
|
||||
{"passes_total", `2`}, {"passes_total", `4`}, {"passes_total", `-1`},
|
||||
{"median_latency_ms", `-1`}, {"median_speed_kbps", `-1`},
|
||||
} {
|
||||
t.Run(mutation.field+"/"+mutation.value, func(t *testing.T) { check(t, mutation.field, json.RawMessage(mutation.value)) })
|
||||
}
|
||||
}
|
||||
|
||||
func TestRevalidateCandidateEarlyTimeout(t *testing.T) {
|
||||
db := testStore(t)
|
||||
args := "--lua-desync=multisplit"
|
||||
saveProposal(t, db, args)
|
||||
c, err := readStrategyCandidate(db, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var passing struct {
|
||||
Strategies []json.RawMessage `json:"strategies"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(combinedReportFixture(args)), &passing); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report := fmt.Sprintf(`{"domain":"www.youtube.com","total":2,"working":1,"strategies":[%s]}`, passing.Strategies[1])
|
||||
okResult := "OK median 242ms, 3728.2 KB/s"
|
||||
diagnostics := earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", okResult, 1)
|
||||
malformedOK := map[string]string{
|
||||
"ok-garbage": "OK garbage median garbage",
|
||||
"ok-old-fixture": "OK 3/3 median 10ms 100kbps",
|
||||
"ok-missing-latency": "OK median ms, 3728.2 KB/s",
|
||||
"ok-negative-latency": "OK median -242ms, 3728.2 KB/s",
|
||||
"ok-fractional-latency": "OK median 242.5ms, 3728.2 KB/s",
|
||||
"ok-missing-speed": "OK median 242ms, KB/s",
|
||||
"ok-negative-speed": "OK median 242ms, -3728.2 KB/s",
|
||||
"ok-nan-speed": "OK median 242ms, NaN KB/s",
|
||||
"ok-inf-speed": "OK median 242ms, Inf KB/s",
|
||||
"ok-wrong-units": "OK median 242ms, 3728.2 kbps",
|
||||
"ok-missing-comma": "OK median 242ms 3728.2 KB/s",
|
||||
"ok-trailing-garbage": okResult + " garbage",
|
||||
"ok-trailing-space": okResult + " ",
|
||||
}
|
||||
modes := []string{
|
||||
"timeout", "two-timeouts", "tls13-timeout", "wrong-domain", "wrong-row-args", "wrong-heading-args", "missing-artifact", "stale-artifact", "missing-report", "runner-failed", "runner-sentinel", "cancelled",
|
||||
"nfq-exited", "extra-line", "missing-heading", "duplicate-heading", "wrong-protocol", "missing-fail", "success-not-ok", "fail-on-success", "wrong-total", "missing-total", "negative-working", "wrong-working", "working-two", "missing-working", "null-strategies", "trailing-json", "bad-json",
|
||||
"zero-count", "equal-counts", "reversed-counts", "excess-count", "not-timeout", "partial-success", "missing-metric", "invalid-metric", "wrong-row-target", "empty-with-one-success",
|
||||
"first-pass-timeout", "third-pass-timeout", "two-first-pass-timeouts", "unknown-section-line", "infra-section-line", "missing-section-marker", "missing-summary-marker", "duplicate-section", "wrong-verifying-count", "wrong-loaded-count", "wrong-summary-count", "wrong-summary-working", "unindented-heading", "extra-indent-heading", "wrong-dns-host", "unknown-preamble-line",
|
||||
}
|
||||
for mode := range malformedOK {
|
||||
modes = append(modes, mode)
|
||||
}
|
||||
for _, mode := range modes {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "check")
|
||||
r, diagnostic := report, diagnostics
|
||||
if result, ok := malformedOK[mode]; ok {
|
||||
diagnostic = strings.Replace(diagnostic, okResult, result, 1)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
switch mode {
|
||||
case "two-timeouts", "empty-with-one-success":
|
||||
r = `{"domain":"www.youtube.com","total":2,"working":0,"strategies":[]}`
|
||||
if mode == "two-timeouts" {
|
||||
diagnostic = earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", "FAIL 2/3 timeout", 0)
|
||||
}
|
||||
case "two-first-pass-timeouts":
|
||||
r = `{"domain":"www.youtube.com","total":2,"working":0,"strategies":[]}`
|
||||
diagnostic = earlyTimeoutDiagnosticsFixture(args, "FAIL 0/1 timeout", "FAIL 0/1 timeout", 0)
|
||||
case "first-pass-timeout":
|
||||
diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", "FAIL 0/1 timeout", 1)
|
||||
case "third-pass-timeout":
|
||||
diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", "FAIL 2/3 timeout", 1)
|
||||
case "tls13-timeout":
|
||||
r = fmt.Sprintf(`{"domain":"www.youtube.com","total":2,"working":1,"strategies":[%s]}`, passing.Strategies[0])
|
||||
diagnostic = earlyTimeoutDiagnosticsFixture(args, okResult, "FAIL 1/2 timeout", 1)
|
||||
case "wrong-domain":
|
||||
r = strings.Replace(r, "www.youtube.com", "other.example", 1)
|
||||
case "wrong-row-args":
|
||||
r = strings.Replace(r, args, "--lua-desync=fake", 1)
|
||||
case "wrong-heading-args":
|
||||
diagnostic = strings.Replace(diagnostic, args, "--lua-desync=fake", 1)
|
||||
case "nfq-exited":
|
||||
diagnostic += "nfqws2 exited with status 1\n"
|
||||
case "extra-line":
|
||||
diagnostic += "unexpected infrastructure output\n"
|
||||
case "missing-heading":
|
||||
diagnostic = strings.Replace(diagnostic, " [1/2] HTTPS/TLS1.2 nfqws2 "+args+"\n", "", 1)
|
||||
case "duplicate-heading":
|
||||
diagnostic = strings.Replace(diagnostic, " [1/2]", " [1/2] HTTPS/TLS1.2 nfqws2 "+args+"\n [1/2]", 1)
|
||||
case "unknown-section-line", "infra-section-line":
|
||||
line := "unknown output"
|
||||
if mode == "infra-section-line" {
|
||||
line = "nfqws2 exited with status 1"
|
||||
}
|
||||
diagnostic = strings.Replace(diagnostic, " FAIL", line+"\n FAIL", 1)
|
||||
case "missing-section-marker", "missing-summary-marker", "duplicate-section", "wrong-verifying-count", "wrong-loaded-count", "wrong-summary-count", "wrong-summary-working", "unindented-heading", "extra-indent-heading", "wrong-dns-host", "unknown-preamble-line":
|
||||
replacements := map[string][2]string{
|
||||
"missing-section-marker": {"=== Checking strategies (data transfer) ===\n", ""},
|
||||
"missing-summary-marker": {"=== Check summary ===\n", ""},
|
||||
"duplicate-section": {"=== Checking strategies (data transfer) ===", "=== Checking strategies (data transfer) ===\n=== Checking strategies (data transfer) ==="},
|
||||
"wrong-verifying-count": {"Verifying 2 strategies", "Verifying 3 strategies"},
|
||||
"wrong-loaded-count": {"loaded 2 strategies", "loaded 3 strategies"},
|
||||
"wrong-summary-count": {"total: 2", "total: 3"},
|
||||
"wrong-summary-working": {"working: 1", "working: 0"},
|
||||
"unindented-heading": {" [1/2]", "[1/2]"},
|
||||
"extra-indent-heading": {" [1/2]", " [1/2]"},
|
||||
"wrong-dns-host": {"www.youtube.com ->", "other.example ->"},
|
||||
"unknown-preamble-line": {"=== DNS resolve ===", "unknown output\n=== DNS resolve ==="},
|
||||
}
|
||||
pair := replacements[mode]
|
||||
diagnostic = strings.Replace(diagnostic, pair[0], pair[1], 1)
|
||||
case "wrong-protocol":
|
||||
diagnostic = strings.Replace(diagnostic, "HTTPS/TLS1.2", "HTTPS/TLS1.3", 1)
|
||||
case "missing-fail":
|
||||
diagnostic = strings.Replace(diagnostic, " FAIL 1/2 timeout\n", "", 1)
|
||||
case "success-not-ok":
|
||||
diagnostic = strings.Replace(diagnostic, "OK median", "unknown median", 1)
|
||||
case "fail-on-success":
|
||||
diagnostic = strings.Replace(diagnostic, okResult, "FAIL 1/2 timeout", 1)
|
||||
case "wrong-total", "missing-total", "negative-working", "wrong-working", "working-two", "missing-working", "null-strategies":
|
||||
replacements := map[string][2]string{
|
||||
"wrong-total": {`"total":2`, `"total":3`}, "missing-total": {`"total":2,`, ""},
|
||||
"negative-working": {`"working":1`, `"working":-1`}, "wrong-working": {`"working":1`, `"working":0`},
|
||||
"working-two": {`"working":1`, `"working":2`}, "missing-working": {`"working":1,`, ""},
|
||||
"null-strategies": {`"strategies":[` + string(passing.Strategies[1]) + `]`, `"strategies":null`},
|
||||
}
|
||||
pair := replacements[mode]
|
||||
r = strings.Replace(r, pair[0], pair[1], 1)
|
||||
case "trailing-json":
|
||||
r += `{}`
|
||||
case "bad-json":
|
||||
r = `{`
|
||||
case "zero-count", "equal-counts", "reversed-counts", "excess-count", "not-timeout":
|
||||
replacements := map[string]string{"zero-count": "FAIL 0/2 timeout", "equal-counts": "FAIL 2/2 timeout", "reversed-counts": "FAIL 3/2 timeout", "excess-count": "FAIL 1/4 timeout", "not-timeout": "FAIL 1/2 connection refused"}
|
||||
diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", replacements[mode], 1)
|
||||
case "partial-success":
|
||||
r = strings.Replace(r, `"passes_ok":3`, `"passes_ok":2`, 1)
|
||||
case "missing-metric":
|
||||
r = strings.Replace(r, `"coverage":1`, `"coverage":null`, 1)
|
||||
case "invalid-metric":
|
||||
r = strings.Replace(r, `"median_speed_kbps":100`, `"median_speed_kbps":-1`, 1)
|
||||
case "wrong-row-target":
|
||||
r = strings.Replace(r, `"protocol":`, `"domains":["other.example"],"protocol":`, 1)
|
||||
}
|
||||
if mode == "stale-artifact" {
|
||||
if err := os.WriteFile(path+".combined.json.stderr", []byte(diagnostic), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
calls := 0
|
||||
check := func(_ context.Context, _, gotPath, gotArgs, host string, _, _ io.Writer) error {
|
||||
calls++
|
||||
if gotPath != path || gotArgs != args || host != "www.youtube.com" {
|
||||
t.Fatal("unexpected checker request")
|
||||
}
|
||||
if mode != "missing-report" {
|
||||
if err := os.WriteFile(path+".combined.json", []byte(r), 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if mode != "missing-artifact" {
|
||||
if err := os.WriteFile(path+".combined.json.stderr", []byte(diagnostic), 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch mode {
|
||||
case "runner-failed":
|
||||
return fmt.Errorf("checker exited")
|
||||
case "runner-sentinel":
|
||||
return errCandidateNonpass
|
||||
case "cancelled":
|
||||
cancel()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, err := revalidateCandidate(ctx, c, "frontend", "www.youtube.com", path, io.Discard, check)
|
||||
nonpass := mode == "timeout" || mode == "two-timeouts" || mode == "tls13-timeout" || mode == "first-pass-timeout" || mode == "third-pass-timeout" || mode == "two-first-pass-timeouts"
|
||||
if err == nil || errors.Is(err, errCandidateNonpass) != nonpass {
|
||||
t.Fatalf("nonpass=%v err=%v", nonpass, err)
|
||||
}
|
||||
if nonpass {
|
||||
if _, err := combinedBlockcheckOutputForHost(strings.NewReader(r), args, "www.youtube.com"); err == nil {
|
||||
t.Fatal("early timeout report qualified for apply")
|
||||
}
|
||||
}
|
||||
if mode == "stale-artifact" && calls != 0 {
|
||||
t.Fatal("checker invoked with stale diagnostic evidence")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user