fix(recovery): handle checker early timeouts and GGC rotation
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
741009afa7
commit
beb4f16309
5 files changed
+561
-5
No files matched your search
@@ -105,8 +105,8 @@ func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) {
|
|||||||
if err := monitor.bootstrap(context.Background()); err != nil || reports != 1 {
|
if err := monitor.bootstrap(context.Background()); err != nil || reports != 1 {
|
||||||
t.Fatal(err, reports)
|
t.Fatal(err, reports)
|
||||||
}
|
}
|
||||||
if err := monitor.probe(context.Background()); err != nil {
|
if err := monitor.probe(context.Background()); !errors.Is(err, errManagedGGCUnresolved) {
|
||||||
t.Fatal("unresolved GGC stopped API probe", err)
|
t.Fatal("unresolved GGC did not fail scope probe", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,10 +167,9 @@ func TestManagedMonitorCLIBindingAndCancellation(t *testing.T) {
|
|||||||
if m.bootstrap == nil || m.recover == nil || m.interval != 5*time.Minute || m.threshold != 3 || m.cooldown != 30*time.Minute {
|
if m.bootstrap == nil || m.recover == nil || m.interval != 5*time.Minute || m.threshold != 3 || m.cooldown != 30*time.Minute {
|
||||||
t.Fatal("missing wiring/defaults")
|
t.Fatal("missing wiring/defaults")
|
||||||
}
|
}
|
||||||
if err := m.probe(ctx); err != nil {
|
if err := m.probe(ctx); !errors.Is(err, context.Canceled) {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
_ = m.probe(ctx)
|
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
})
|
})
|
||||||
if code != 0 || requests != 2 {
|
if code != 0 || requests != 2 {
|
||||||
|
|||||||
@@ -95,6 +95,11 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
|
|||||||
|
|
||||||
type managedRecoveryFatal struct{ Err error }
|
type managedRecoveryFatal struct{ Err error }
|
||||||
|
|
||||||
|
var (
|
||||||
|
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
|
||||||
|
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
|
||||||
|
)
|
||||||
|
|
||||||
func (e *managedRecoveryFatal) Error() string {
|
func (e *managedRecoveryFatal) Error() string {
|
||||||
return "managed recovery safety failure: " + e.Err.Error()
|
return "managed recovery safety failure: " + e.Err.Error()
|
||||||
}
|
}
|
||||||
@@ -478,7 +483,37 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
|||||||
if status != "OK" {
|
if status != "OK" {
|
||||||
return &managedHealthRejection{Status: status}
|
return &managedHealthRejection{Status: status}
|
||||||
}
|
}
|
||||||
|
transaction := *recovery.transaction
|
||||||
|
transaction.Health = func() (string, error) { return recovery.health(ctx) }
|
||||||
|
// ActiveCandidates is read-only; never permit a probe to reload config.
|
||||||
|
transaction.Reload = func() error { return fmt.Errorf("probe cannot reload config") }
|
||||||
|
ids, err := transaction.ActiveCandidates(recovery.db)
|
||||||
|
if err != nil {
|
||||||
|
return &managedRecoveryFatal{Err: err}
|
||||||
|
}
|
||||||
|
// Keep Player API health separate from GGC scope (not playback health).
|
||||||
|
// Two bounded Player requests reuse the existing health/discovery adapters.
|
||||||
|
host, err := recovery.discover(ctx)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if errors.As(err, &fatal) || errors.Is(err, context.Canceled) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
|
||||||
|
return errManagedGGCUnresolved
|
||||||
|
}
|
||||||
|
for _, id := range ids {
|
||||||
|
candidate, err := readStrategyCandidate(recovery.db, id)
|
||||||
|
if err != nil {
|
||||||
|
return &managedRecoveryFatal{Err: err}
|
||||||
|
}
|
||||||
|
if candidate.Group == "googlevideo" && len(candidate.Endpoints) == 1 && candidate.Endpoints[0] == host {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
return errManagedGGCScopeChanged
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -569,3 +570,154 @@ func TestManagedRecoveryPlayerAdapter(t *testing.T) {
|
|||||||
t.Fatal("wrong target binding")
|
t.Fatal("wrong target binding")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryProbeGGCScope(t *testing.T) {
|
||||||
|
for _, mode := range []string{"matching", "changed", "missing", "wildcard", "uppercase", "url", "foreign", "empty", "discovery-error", "fatal", "cancel", "drift", "identity", "health-reject"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
recovery, front, events := recoveryFixture(t, true)
|
||||||
|
if mode != "missing" {
|
||||||
|
cache := managedCandidateFixture(t, recovery.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=multisplit", true)
|
||||||
|
if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
monitor := monitorRuntimeFixture(t)
|
||||||
|
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
||||||
|
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||||
|
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
discoveries := 0
|
||||||
|
recovery.discover = func(context.Context) (string, error) {
|
||||||
|
discoveries++
|
||||||
|
switch mode {
|
||||||
|
case "changed":
|
||||||
|
return "other.googlevideo.com", nil
|
||||||
|
case "wildcard":
|
||||||
|
return "googlevideo.com", nil
|
||||||
|
case "uppercase":
|
||||||
|
return "FRESH.googlevideo.com", nil
|
||||||
|
case "url":
|
||||||
|
return "https://fresh.googlevideo.com/video?secret=never-print", nil
|
||||||
|
case "foreign":
|
||||||
|
return "fresh.googlevideo.com.evil.example", nil
|
||||||
|
case "empty":
|
||||||
|
return "", nil
|
||||||
|
case "discovery-error":
|
||||||
|
return "", errors.New("secret=never-print")
|
||||||
|
case "fatal":
|
||||||
|
return "", &managedRecoveryFatal{Err: errors.New("unsafe target")}
|
||||||
|
case "cancel":
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
return "fresh.googlevideo.com", nil
|
||||||
|
}
|
||||||
|
if mode == "health-reject" {
|
||||||
|
recovery.health = func(context.Context) (string, error) { return "DEGRADED", nil }
|
||||||
|
}
|
||||||
|
if err := wireManagedRecovery(&monitor, recovery, func(error) { t.Fatal("probe invoked recovery report") }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if mode == "drift" {
|
||||||
|
if err := os.WriteFile(filepath.Join(recovery.transaction.Dir, "config"), []byte("external"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mode == "identity" {
|
||||||
|
if err := os.Remove(filepath.Join(recovery.transaction.Dir, managedMarkerName)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err := monitor.probe(ctx)
|
||||||
|
switch mode {
|
||||||
|
case "matching":
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case "changed", "missing":
|
||||||
|
if !errors.Is(err, errManagedGGCScopeChanged) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case "fatal", "drift", "identity":
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if !errors.As(err, &fatal) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case "cancel":
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case "health-reject":
|
||||||
|
var rejection *managedHealthRejection
|
||||||
|
if !errors.As(err, &rejection) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
if !errors.Is(err, errManagedGGCUnresolved) || strings.Contains(err.Error(), "never-print") {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(*events) != 0 || (mode == "drift" || mode == "identity" || mode == "health-reject") && discoveries != 0 {
|
||||||
|
t.Fatalf("unsafe probe effects: events=%v discoveries=%d", *events, discoveries)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryGGCRotationPolicy(t *testing.T) {
|
||||||
|
for _, activeCache := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprint(activeCache), func(t *testing.T) {
|
||||||
|
recovery, front, events := recoveryFixture(t, true)
|
||||||
|
if err := os.Chmod(recovery.transaction.Dir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if activeCache {
|
||||||
|
cache := managedCandidateFixture(t, recovery.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
|
||||||
|
if err := recovery.transaction.Apply(recovery.db, []int64{front, cache}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
monitor := monitorRuntimeFixture(t)
|
||||||
|
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
||||||
|
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||||
|
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := wireManagedRecovery(&monitor, recovery, func(err error) { t.Fatal(err) }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
probes := 0
|
||||||
|
probe := monitor.probe
|
||||||
|
monitor.probe = func(ctx context.Context) error { probes++; return probe(ctx) }
|
||||||
|
monitor.wait = func(ctx context.Context, _ time.Duration) error {
|
||||||
|
if probes < monitor.threshold && len(*events) != 0 {
|
||||||
|
t.Fatal("recovered before threshold", *events)
|
||||||
|
}
|
||||||
|
if probes == monitor.threshold+1 {
|
||||||
|
cancel()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ids, err := recovery.transaction.ActiveCandidates(recovery.db)
|
||||||
|
if err != nil || len(ids) != 2 || ids[0] != front {
|
||||||
|
t.Fatal(ids, err)
|
||||||
|
}
|
||||||
|
cache, err := readStrategyCandidate(recovery.db, ids[1])
|
||||||
|
if err != nil || !reflect.DeepEqual(cache.Endpoints, []string{"fresh.googlevideo.com"}) {
|
||||||
|
t.Fatal(cache, err)
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(monitor.db, monitor.serviceID)
|
||||||
|
if err != nil || state.CooldownUntil.IsZero() || strings.Contains(strings.Join(*events, ","), "scan:"+managedFrontendHost) {
|
||||||
|
t.Fatal(state, err, *events)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,14 +4,17 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Only complete, well-formed non-passing measurements permit scan fallback.
|
// Only validated measurements or authenticated early timeouts permit fallback.
|
||||||
var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols")
|
var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols")
|
||||||
|
|
||||||
const managedFrontendHost = "www.youtube.com"
|
const managedFrontendHost = "www.youtube.com"
|
||||||
@@ -52,6 +55,10 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
|
|||||||
if err := os.Remove(path + ".combined.json"); err != nil && !errors.Is(err, os.ErrNotExist) {
|
if err := os.Remove(path + ".combined.json"); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// The owned runner creates this with O_EXCL. Never accept stale diagnostics.
|
||||||
|
if _, err := os.Lstat(path + ".combined.json.stderr"); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, fmt.Errorf("candidate diagnostic artifact already exists or cannot be inspected: %v", err)
|
||||||
|
}
|
||||||
var raw bytes.Buffer
|
var raw bytes.Buffer
|
||||||
err := check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw)
|
err := check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw)
|
||||||
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
|
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
|
||||||
@@ -72,12 +79,117 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
_, err = selectBlockcheckEvidence(bytes.NewReader(report), c.Args, group, host, true)
|
_, err = selectBlockcheckEvidence(bytes.NewReader(report), c.Args, group, host, true)
|
||||||
|
if err != nil && !errors.Is(err, errCandidateNonpass) {
|
||||||
|
if diagnostics, readErr := os.ReadFile(path + ".combined.json.stderr"); readErr == nil && candidateEarlyTimeout(report, diagnostics, c.Args, group, host) {
|
||||||
|
err = errCandidateNonpass
|
||||||
|
}
|
||||||
|
}
|
||||||
if contextErr := ctx.Err(); contextErr != nil {
|
if contextErr := ctx.Err(); contextErr != nil {
|
||||||
return nil, contextErr
|
return nil, contextErr
|
||||||
}
|
}
|
||||||
return report, err
|
return report, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var earlyTimeoutResult = regexp.MustCompile(`^ FAIL ([0-2])/([1-3]) timeout$`)
|
||||||
|
var earlySuccessResult = regexp.MustCompile(`^ OK median [0-9]+ms, [0-9]+\.[0-9] KB/s$`)
|
||||||
|
var earlyCheckPreamble = regexp.MustCompile("^(! --workers has no effect on `check` \\(it verifies strategies sequentially by design\\)\n)?" +
|
||||||
|
`=== Checking prerequisites ===\n OK nfqws2: /[^\s]+\n OK nft\n OK nft queue support\n` +
|
||||||
|
`check loaded 2 strategies from /[^\s]+, --take [0-9]+, --passes 3\n( ISP: AS[0-9]+ [^\n]+\n)?` +
|
||||||
|
`=== DNS resolve ===\n dns mode: auto\n ([a-z0-9.-]+) -> [0-9a-fA-F:.]+ \(via doh \(auto-fallback\)\)$`)
|
||||||
|
var earlyCheckSummary = regexp.MustCompile(`^ total: 2 \| working: ([01]) \| elapsed: [0-9]+\.[0-9]s\n OK JSON report → /[^\s]+$`)
|
||||||
|
|
||||||
|
// This is rejection evidence only, never qualification evidence. Unknown CLI
|
||||||
|
// output fails closed; changes to the checker format require a new fixture.
|
||||||
|
func candidateEarlyTimeout(raw, diagnostics []byte, args, group, host string) bool {
|
||||||
|
var report struct {
|
||||||
|
Domain string `json:"domain"`
|
||||||
|
Total *int `json:"total"`
|
||||||
|
Working *int `json:"working"`
|
||||||
|
Strategies *[]json.RawMessage `json:"strategies"`
|
||||||
|
}
|
||||||
|
d := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
if d.Decode(&report) != nil || d.Decode(new(json.RawMessage)) != io.EOF || report.Domain != host || report.Total == nil || *report.Total != 2 || report.Working == nil || *report.Working < 0 || *report.Working >= 2 || report.Strategies == nil || len(*report.Strategies) != *report.Working {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
success := ""
|
||||||
|
if *report.Working == 1 {
|
||||||
|
row := (*report.Strategies)[0]
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
if json.Unmarshal(row, &fields) != nil || json.Unmarshal(fields["protocol"], &success) != nil || normalizeProtocol(success) == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
other := "HTTPS/TLS1.2"
|
||||||
|
if success == other {
|
||||||
|
other = "HTTPS/TLS1.3"
|
||||||
|
}
|
||||||
|
fields["protocol"], _ = json.Marshal(other)
|
||||||
|
copyRow, err := json.Marshal(fields)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Reuse every strict measurement/target/args check, including exactly
|
||||||
|
// 3/3, by supplying a protocol-flipped copy solely for validation.
|
||||||
|
complete, err := json.Marshal(map[string]any{"strategies": []json.RawMessage{row, copyRow}})
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if _, err := selectBlockcheckEvidence(bytes.NewReader(complete), args, group, host, false); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines := strings.Split(strings.ReplaceAll(string(diagnostics), "\r\n", "\n"), "\n")
|
||||||
|
var evidence []string
|
||||||
|
for _, line := range lines {
|
||||||
|
if line != "" {
|
||||||
|
evidence = append(evidence, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// ponytail: pin the known CLI framing and metadata grammar; new checker
|
||||||
|
// formats need explicit fixtures, not permissive infrastructure filtering.
|
||||||
|
parts := strings.Split(strings.Join(evidence, "\n"), "\n=== Checking strategies (data transfer) ===\n")
|
||||||
|
if len(parts) != 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
preamble := earlyCheckPreamble.FindStringSubmatch(parts[0])
|
||||||
|
if preamble == nil || preamble[3] != host {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
parts = strings.Split(parts[1], "\n=== Check summary ===\n")
|
||||||
|
if len(parts) != 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
summary := earlyCheckSummary.FindStringSubmatch(parts[1])
|
||||||
|
if summary == nil || summary[1] != strconv.Itoa(*report.Working) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
evidence = strings.Split(parts[0], "\n")
|
||||||
|
if len(evidence) != 7 || evidence[0] != " Tip: use --take 10 to stop after 10 verified per protocol" || evidence[1] != " Verifying 2 strategies (3 passes, early-exit on first fail)" || evidence[4] != " ────────────────────────────────────────────────────────────" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i, protocol := range []string{"HTTPS/TLS1.2", "HTTPS/TLS1.3"} {
|
||||||
|
if evidence[2+3*i] != fmt.Sprintf(" [%d/2] %s nfqws2 %s", i+1, protocol, args) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
result := evidence[3+3*i]
|
||||||
|
if protocol == success {
|
||||||
|
if !earlySuccessResult.MatchString(result) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
match := earlyTimeoutResult.FindStringSubmatch(result)
|
||||||
|
if match == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
n, _ := strconv.Atoi(match[1])
|
||||||
|
m, _ := strconv.Atoi(match[2])
|
||||||
|
if n != m-1 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func revalidateGooglevideo(ctx context.Context, db *sql.DB, host string, stderr io.Writer, check cacheCheck) (string, []byte, error) {
|
func revalidateGooglevideo(ctx context.Context, db *sql.DB, host string, stderr io.Writer, check cacheCheck) (string, []byte, error) {
|
||||||
ids, err := qualifiedCandidateIDs(db, "googlevideo")
|
ids, err := qualifiedCandidateIDs(db, "googlevideo")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -210,3 +210,261 @@ func TestRevalidateCandidateScope(t *testing.T) {
|
|||||||
t.Fatalf("changed stored scope: %+v %v", stored, err)
|
t.Fatalf("changed stored scope: %+v %v", stored, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func earlyTimeoutDiagnosticsFixture(args, tls12, tls13 string, working int) string {
|
||||||
|
return fmt.Sprintf("! --workers has no effect on `check` (it verifies strategies sequentially by design)\n"+`=== Checking prerequisites ===
|
||||||
|
OK nfqws2: /opt/zapret2/binaries/linux-x86_64/nfqws2
|
||||||
|
OK nft
|
||||||
|
OK nft queue support
|
||||||
|
check loaded 2 strategies from /tmp/input.txt, --take 1, --passes 3
|
||||||
|
ISP: AS3175 ...
|
||||||
|
=== DNS resolve ===
|
||||||
|
dns mode: auto
|
||||||
|
www.youtube.com -> 142.251.156.4 (via doh (auto-fallback))
|
||||||
|
|
||||||
|
=== Checking strategies (data transfer) ===
|
||||||
|
Tip: use --take 10 to stop after 10 verified per protocol
|
||||||
|
Verifying 2 strategies (3 passes, early-exit on first fail)
|
||||||
|
[1/2] HTTPS/TLS1.2 nfqws2 %s
|
||||||
|
%s
|
||||||
|
────────────────────────────────────────────────────────────
|
||||||
|
[2/2] HTTPS/TLS1.3 nfqws2 %s
|
||||||
|
%s
|
||||||
|
|
||||||
|
=== Check summary ===
|
||||||
|
total: 2 | working: %d | elapsed: 7.7s
|
||||||
|
OK JSON report → /tmp/input.txt.combined.json
|
||||||
|
`, args, tls12, args, tls13, working)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCandidateEarlyTimeoutRowValidation(t *testing.T) {
|
||||||
|
args := "--lua-desync=multisplit"
|
||||||
|
var passing struct {
|
||||||
|
Strategies []json.RawMessage `json:"strategies"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(combinedReportFixture(args)), &passing); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
diagnostics := []byte(earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", "OK median 242ms, 3728.2 KB/s", 1))
|
||||||
|
check := func(t *testing.T, field string, value json.RawMessage) {
|
||||||
|
t.Helper()
|
||||||
|
var row map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(passing.Strategies[1], &row); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if value == nil {
|
||||||
|
delete(row, field)
|
||||||
|
} else {
|
||||||
|
row[field] = value
|
||||||
|
}
|
||||||
|
report, err := json.Marshal(map[string]any{"domain": "www.youtube.com", "total": 2, "working": 1, "strategies": []any{row}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if candidateEarlyTimeout(report, diagnostics, args, "frontend", "www.youtube.com") {
|
||||||
|
t.Fatal("malformed passing row authorized fallback")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, field := range []string{"protocol", "args", "success_rate", "coverage", "passes_ok", "passes_total", "median_latency_ms", "median_speed_kbps"} {
|
||||||
|
for _, mutation := range []struct {
|
||||||
|
name string
|
||||||
|
value json.RawMessage
|
||||||
|
}{{"missing", nil}, {"null", json.RawMessage(`null`)}, {"wrong-type", json.RawMessage(`{}`)}} {
|
||||||
|
t.Run(field+"/"+mutation.name, func(t *testing.T) { check(t, field, mutation.value) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, mutation := range []struct{ field, value string }{
|
||||||
|
{"protocol", `"HTTP"`}, {"args", `"--lua-desync=fake"`},
|
||||||
|
{"group", `"googlevideo"`}, {"domains", `["other.example"]`}, {"domains", `[]`},
|
||||||
|
{"success_rate", `0.9`}, {"success_rate", `1.1`}, {"success_rate", `-1`},
|
||||||
|
{"coverage", `0.9`}, {"coverage", `1.1`}, {"coverage", `-1`},
|
||||||
|
{"passes_ok", `2`}, {"passes_ok", `4`}, {"passes_ok", `-1`},
|
||||||
|
{"passes_total", `2`}, {"passes_total", `4`}, {"passes_total", `-1`},
|
||||||
|
{"median_latency_ms", `-1`}, {"median_speed_kbps", `-1`},
|
||||||
|
} {
|
||||||
|
t.Run(mutation.field+"/"+mutation.value, func(t *testing.T) { check(t, mutation.field, json.RawMessage(mutation.value)) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRevalidateCandidateEarlyTimeout(t *testing.T) {
|
||||||
|
db := testStore(t)
|
||||||
|
args := "--lua-desync=multisplit"
|
||||||
|
saveProposal(t, db, args)
|
||||||
|
c, err := readStrategyCandidate(db, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var passing struct {
|
||||||
|
Strategies []json.RawMessage `json:"strategies"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(combinedReportFixture(args)), &passing); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report := fmt.Sprintf(`{"domain":"www.youtube.com","total":2,"working":1,"strategies":[%s]}`, passing.Strategies[1])
|
||||||
|
okResult := "OK median 242ms, 3728.2 KB/s"
|
||||||
|
diagnostics := earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", okResult, 1)
|
||||||
|
malformedOK := map[string]string{
|
||||||
|
"ok-garbage": "OK garbage median garbage",
|
||||||
|
"ok-old-fixture": "OK 3/3 median 10ms 100kbps",
|
||||||
|
"ok-missing-latency": "OK median ms, 3728.2 KB/s",
|
||||||
|
"ok-negative-latency": "OK median -242ms, 3728.2 KB/s",
|
||||||
|
"ok-fractional-latency": "OK median 242.5ms, 3728.2 KB/s",
|
||||||
|
"ok-missing-speed": "OK median 242ms, KB/s",
|
||||||
|
"ok-negative-speed": "OK median 242ms, -3728.2 KB/s",
|
||||||
|
"ok-nan-speed": "OK median 242ms, NaN KB/s",
|
||||||
|
"ok-inf-speed": "OK median 242ms, Inf KB/s",
|
||||||
|
"ok-wrong-units": "OK median 242ms, 3728.2 kbps",
|
||||||
|
"ok-missing-comma": "OK median 242ms 3728.2 KB/s",
|
||||||
|
"ok-trailing-garbage": okResult + " garbage",
|
||||||
|
"ok-trailing-space": okResult + " ",
|
||||||
|
}
|
||||||
|
modes := []string{
|
||||||
|
"timeout", "two-timeouts", "tls13-timeout", "wrong-domain", "wrong-row-args", "wrong-heading-args", "missing-artifact", "stale-artifact", "missing-report", "runner-failed", "runner-sentinel", "cancelled",
|
||||||
|
"nfq-exited", "extra-line", "missing-heading", "duplicate-heading", "wrong-protocol", "missing-fail", "success-not-ok", "fail-on-success", "wrong-total", "missing-total", "negative-working", "wrong-working", "working-two", "missing-working", "null-strategies", "trailing-json", "bad-json",
|
||||||
|
"zero-count", "equal-counts", "reversed-counts", "excess-count", "not-timeout", "partial-success", "missing-metric", "invalid-metric", "wrong-row-target", "empty-with-one-success",
|
||||||
|
"first-pass-timeout", "third-pass-timeout", "two-first-pass-timeouts", "unknown-section-line", "infra-section-line", "missing-section-marker", "missing-summary-marker", "duplicate-section", "wrong-verifying-count", "wrong-loaded-count", "wrong-summary-count", "wrong-summary-working", "unindented-heading", "extra-indent-heading", "wrong-dns-host", "unknown-preamble-line",
|
||||||
|
}
|
||||||
|
for mode := range malformedOK {
|
||||||
|
modes = append(modes, mode)
|
||||||
|
}
|
||||||
|
for _, mode := range modes {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "check")
|
||||||
|
r, diagnostic := report, diagnostics
|
||||||
|
if result, ok := malformedOK[mode]; ok {
|
||||||
|
diagnostic = strings.Replace(diagnostic, okResult, result, 1)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
switch mode {
|
||||||
|
case "two-timeouts", "empty-with-one-success":
|
||||||
|
r = `{"domain":"www.youtube.com","total":2,"working":0,"strategies":[]}`
|
||||||
|
if mode == "two-timeouts" {
|
||||||
|
diagnostic = earlyTimeoutDiagnosticsFixture(args, "FAIL 1/2 timeout", "FAIL 2/3 timeout", 0)
|
||||||
|
}
|
||||||
|
case "two-first-pass-timeouts":
|
||||||
|
r = `{"domain":"www.youtube.com","total":2,"working":0,"strategies":[]}`
|
||||||
|
diagnostic = earlyTimeoutDiagnosticsFixture(args, "FAIL 0/1 timeout", "FAIL 0/1 timeout", 0)
|
||||||
|
case "first-pass-timeout":
|
||||||
|
diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", "FAIL 0/1 timeout", 1)
|
||||||
|
case "third-pass-timeout":
|
||||||
|
diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", "FAIL 2/3 timeout", 1)
|
||||||
|
case "tls13-timeout":
|
||||||
|
r = fmt.Sprintf(`{"domain":"www.youtube.com","total":2,"working":1,"strategies":[%s]}`, passing.Strategies[0])
|
||||||
|
diagnostic = earlyTimeoutDiagnosticsFixture(args, okResult, "FAIL 1/2 timeout", 1)
|
||||||
|
case "wrong-domain":
|
||||||
|
r = strings.Replace(r, "www.youtube.com", "other.example", 1)
|
||||||
|
case "wrong-row-args":
|
||||||
|
r = strings.Replace(r, args, "--lua-desync=fake", 1)
|
||||||
|
case "wrong-heading-args":
|
||||||
|
diagnostic = strings.Replace(diagnostic, args, "--lua-desync=fake", 1)
|
||||||
|
case "nfq-exited":
|
||||||
|
diagnostic += "nfqws2 exited with status 1\n"
|
||||||
|
case "extra-line":
|
||||||
|
diagnostic += "unexpected infrastructure output\n"
|
||||||
|
case "missing-heading":
|
||||||
|
diagnostic = strings.Replace(diagnostic, " [1/2] HTTPS/TLS1.2 nfqws2 "+args+"\n", "", 1)
|
||||||
|
case "duplicate-heading":
|
||||||
|
diagnostic = strings.Replace(diagnostic, " [1/2]", " [1/2] HTTPS/TLS1.2 nfqws2 "+args+"\n [1/2]", 1)
|
||||||
|
case "unknown-section-line", "infra-section-line":
|
||||||
|
line := "unknown output"
|
||||||
|
if mode == "infra-section-line" {
|
||||||
|
line = "nfqws2 exited with status 1"
|
||||||
|
}
|
||||||
|
diagnostic = strings.Replace(diagnostic, " FAIL", line+"\n FAIL", 1)
|
||||||
|
case "missing-section-marker", "missing-summary-marker", "duplicate-section", "wrong-verifying-count", "wrong-loaded-count", "wrong-summary-count", "wrong-summary-working", "unindented-heading", "extra-indent-heading", "wrong-dns-host", "unknown-preamble-line":
|
||||||
|
replacements := map[string][2]string{
|
||||||
|
"missing-section-marker": {"=== Checking strategies (data transfer) ===\n", ""},
|
||||||
|
"missing-summary-marker": {"=== Check summary ===\n", ""},
|
||||||
|
"duplicate-section": {"=== Checking strategies (data transfer) ===", "=== Checking strategies (data transfer) ===\n=== Checking strategies (data transfer) ==="},
|
||||||
|
"wrong-verifying-count": {"Verifying 2 strategies", "Verifying 3 strategies"},
|
||||||
|
"wrong-loaded-count": {"loaded 2 strategies", "loaded 3 strategies"},
|
||||||
|
"wrong-summary-count": {"total: 2", "total: 3"},
|
||||||
|
"wrong-summary-working": {"working: 1", "working: 0"},
|
||||||
|
"unindented-heading": {" [1/2]", "[1/2]"},
|
||||||
|
"extra-indent-heading": {" [1/2]", " [1/2]"},
|
||||||
|
"wrong-dns-host": {"www.youtube.com ->", "other.example ->"},
|
||||||
|
"unknown-preamble-line": {"=== DNS resolve ===", "unknown output\n=== DNS resolve ==="},
|
||||||
|
}
|
||||||
|
pair := replacements[mode]
|
||||||
|
diagnostic = strings.Replace(diagnostic, pair[0], pair[1], 1)
|
||||||
|
case "wrong-protocol":
|
||||||
|
diagnostic = strings.Replace(diagnostic, "HTTPS/TLS1.2", "HTTPS/TLS1.3", 1)
|
||||||
|
case "missing-fail":
|
||||||
|
diagnostic = strings.Replace(diagnostic, " FAIL 1/2 timeout\n", "", 1)
|
||||||
|
case "success-not-ok":
|
||||||
|
diagnostic = strings.Replace(diagnostic, "OK median", "unknown median", 1)
|
||||||
|
case "fail-on-success":
|
||||||
|
diagnostic = strings.Replace(diagnostic, okResult, "FAIL 1/2 timeout", 1)
|
||||||
|
case "wrong-total", "missing-total", "negative-working", "wrong-working", "working-two", "missing-working", "null-strategies":
|
||||||
|
replacements := map[string][2]string{
|
||||||
|
"wrong-total": {`"total":2`, `"total":3`}, "missing-total": {`"total":2,`, ""},
|
||||||
|
"negative-working": {`"working":1`, `"working":-1`}, "wrong-working": {`"working":1`, `"working":0`},
|
||||||
|
"working-two": {`"working":1`, `"working":2`}, "missing-working": {`"working":1,`, ""},
|
||||||
|
"null-strategies": {`"strategies":[` + string(passing.Strategies[1]) + `]`, `"strategies":null`},
|
||||||
|
}
|
||||||
|
pair := replacements[mode]
|
||||||
|
r = strings.Replace(r, pair[0], pair[1], 1)
|
||||||
|
case "trailing-json":
|
||||||
|
r += `{}`
|
||||||
|
case "bad-json":
|
||||||
|
r = `{`
|
||||||
|
case "zero-count", "equal-counts", "reversed-counts", "excess-count", "not-timeout":
|
||||||
|
replacements := map[string]string{"zero-count": "FAIL 0/2 timeout", "equal-counts": "FAIL 2/2 timeout", "reversed-counts": "FAIL 3/2 timeout", "excess-count": "FAIL 1/4 timeout", "not-timeout": "FAIL 1/2 connection refused"}
|
||||||
|
diagnostic = strings.Replace(diagnostic, "FAIL 1/2 timeout", replacements[mode], 1)
|
||||||
|
case "partial-success":
|
||||||
|
r = strings.Replace(r, `"passes_ok":3`, `"passes_ok":2`, 1)
|
||||||
|
case "missing-metric":
|
||||||
|
r = strings.Replace(r, `"coverage":1`, `"coverage":null`, 1)
|
||||||
|
case "invalid-metric":
|
||||||
|
r = strings.Replace(r, `"median_speed_kbps":100`, `"median_speed_kbps":-1`, 1)
|
||||||
|
case "wrong-row-target":
|
||||||
|
r = strings.Replace(r, `"protocol":`, `"domains":["other.example"],"protocol":`, 1)
|
||||||
|
}
|
||||||
|
if mode == "stale-artifact" {
|
||||||
|
if err := os.WriteFile(path+".combined.json.stderr", []byte(diagnostic), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
calls := 0
|
||||||
|
check := func(_ context.Context, _, gotPath, gotArgs, host string, _, _ io.Writer) error {
|
||||||
|
calls++
|
||||||
|
if gotPath != path || gotArgs != args || host != "www.youtube.com" {
|
||||||
|
t.Fatal("unexpected checker request")
|
||||||
|
}
|
||||||
|
if mode != "missing-report" {
|
||||||
|
if err := os.WriteFile(path+".combined.json", []byte(r), 0600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mode != "missing-artifact" {
|
||||||
|
if err := os.WriteFile(path+".combined.json.stderr", []byte(diagnostic), 0600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch mode {
|
||||||
|
case "runner-failed":
|
||||||
|
return fmt.Errorf("checker exited")
|
||||||
|
case "runner-sentinel":
|
||||||
|
return errCandidateNonpass
|
||||||
|
case "cancelled":
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err := revalidateCandidate(ctx, c, "frontend", "www.youtube.com", path, io.Discard, check)
|
||||||
|
nonpass := mode == "timeout" || mode == "two-timeouts" || mode == "tls13-timeout" || mode == "first-pass-timeout" || mode == "third-pass-timeout" || mode == "two-first-pass-timeouts"
|
||||||
|
if err == nil || errors.Is(err, errCandidateNonpass) != nonpass {
|
||||||
|
t.Fatalf("nonpass=%v err=%v", nonpass, err)
|
||||||
|
}
|
||||||
|
if nonpass {
|
||||||
|
if _, err := combinedBlockcheckOutputForHost(strings.NewReader(r), args, "www.youtube.com"); err == nil {
|
||||||
|
t.Fatal("early timeout report qualified for apply")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mode == "stale-artifact" && calls != 0 {
|
||||||
|
t.Fatal("checker invoked with stale diagnostic evidence")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user