Files
zapret-autotune/cmd/zapret-autotune/managed_transaction_test.go
T
2026-10-10 09:24:43 +03:00

640 lines
20 KiB
Go

package main
import (
"database/sql"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
)
func managedCandidateFixture(t *testing.T, db *sql.DB, group, host, args string, qualified bool) int64 {
t.Helper()
tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
id, err := insertStrategyCandidate(tx, group, []string{host}, args, nil)
if err != nil {
t.Fatal(err)
}
if qualified {
for _, protocol := range []string{"tls12", "tls13"} {
_, err := tx.Exec(`INSERT INTO candidate_evidence
(candidate_id, endpoint, protocol, passes_ok, passes_total, success_rate, coverage, median_latency_ms, median_speed_kbps, checked_at, raw_measurement)
VALUES (?, ?, ?, 3, 3, 1, 1, 1, 1, 'fixture', '{}')`, id, host, protocol)
if err != nil {
t.Fatal(err)
}
}
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
return id
}
func TestManagedRender(t *testing.T) {
db := testStore(t)
front := managedCandidateFixture(t, db, "frontend", "www.youtube.com", "--lua-desync=multisplit", true)
ggc := managedCandidateFixture(t, db, "googlevideo", "rr1---sn-test.googlevideo.com", "--lua-desync=fake", true)
b, err := renderManagedConfig(db, []int64{front, ggc})
if err != nil {
t.Fatal(err)
}
s := string(b)
assertManagedFirewallConfig(t, s, "1")
for _, want := range []string{"NFQWS2_ENABLE=1\n", "INIT_APPLY_FW=1\n", "NFQWS2_PORTS_TCP=80,443\n", "NFQWS2_PORTS_UDP=443\n", "--hostlist-domains=www.youtube.com --lua-desync=multisplit --new --filter-tcp=443 --filter-l7=tls --hostlist-domains=^rr1---sn-test.googlevideo.com --lua-desync=fake"} {
if !strings.Contains(s, want) {
t.Fatalf("missing %q in %s", want, s)
}
}
if strings.Contains(s, "QNUM=") {
t.Fatal("queue default overridden")
}
unqualified := managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", false)
wildcard := managedCandidateFixture(t, db, "googlevideo", "googlevideo.com", "--lua-desync=fake", true)
otherFront := managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
otherGGC := managedCandidateFixture(t, db, "googlevideo", "rr2.googlevideo.com", "--lua-desync=fake", true)
for _, ids := range [][]int64{{}, {unqualified, ggc}, {front, wildcard}, {front, otherFront}, {front, ggc, otherGGC}, {front, ggc, ggc}, {99999, ggc}, {0}, {-1}} {
if _, err := renderManagedConfig(db, ids); err == nil {
t.Fatalf("accepted %v", ids)
}
}
bootstrap, err := renderManagedConfig(db, []int64{front})
if err != nil || strings.Contains(string(bootstrap), "googlevideo") || strings.Contains(string(bootstrap), "--new") {
t.Fatalf("frontend-only config=%q err=%v", bootstrap, err)
}
cacheOnly, err := renderManagedConfig(db, []int64{ggc})
if err != nil || !strings.Contains(string(cacheOnly), "--hostlist-domains=^rr1---sn-test.googlevideo.com") || strings.Contains(string(cacheOnly), "youtube.com") {
t.Fatalf("GGC-only config=%q err=%v", cacheOnly, err)
}
}
func assertManagedFirewallConfig(t *testing.T, config, enabled string) {
t.Helper()
for _, line := range []string{
"NFQWS2_ENABLE=" + enabled, "INIT_APPLY_FW=1",
"NFQWS2_PORTS_TCP=80,443", "NFQWS2_PORTS_UDP=443",
"NFQWS2_TCP_PKT_OUT=15", "NFQWS2_TCP_PKT_IN=15",
"NFQWS2_UDP_PKT_OUT=15", "NFQWS2_UDP_PKT_IN=15",
} {
if strings.Count(config, "\n"+line+"\n") != 1 {
t.Fatalf("expected exactly one %q in %s", line, config)
}
}
if strings.Contains(config, "QNUM=") {
t.Fatal("queue default overridden")
}
}
func TestManagedRenderRejectsUnsafeArgs(t *testing.T) {
for _, args := range []string{"--lua-desync=fake'", "--lua-desync=fake\n", "--lua-desync=fake;touch /tmp/x", "--lua-desync=$(id)", "--lua-desync=fake`id`", "--lua-desync=fake\"", "--lua-desync=fake --new", "--hostlist-domains=googlevideo.com"} {
t.Run(args, func(t *testing.T) {
v := candidate{Args: args}
if err := v.validateArgs(); err == nil {
t.Fatal("unsafe strategy accepted by shared validator")
}
db := testStore(t)
// Exercise validation even if legacy fingerprint normalization accepts
// shell syntax: the renderer must never extend its inert args contract.
fingerprint, err := strategyFingerprint(args)
if err != nil {
return
}
result, err := db.Exec(`INSERT INTO strategy_candidates (strategy_group, endpoints, args, fingerprint) VALUES ('frontend', '["youtube.com"]', ?, ?)`, args, fingerprint)
if err != nil {
t.Fatal(err)
}
id, err := result.LastInsertId()
if err != nil {
t.Fatal(err)
}
for _, protocol := range []string{"tls12", "tls13"} {
if _, err := db.Exec(`INSERT INTO candidate_evidence VALUES (?, 'youtube.com', ?, 3, 3, 1, 1, 1, 1, 'fixture', '{}')`, id, protocol); err != nil {
t.Fatal(err)
}
}
ggc := managedCandidateFixture(t, db, "googlevideo", "rr1.googlevideo.com", "--lua-desync=fake", true)
if _, err := renderManagedConfig(db, []int64{id, ggc}); err == nil {
t.Fatal("unsafe config accepted")
}
})
}
}
func managedFixture(t *testing.T) (*managedTransaction, *int) {
t.Helper()
reloads := new(int)
m := &managedTransaction{Dir: t.TempDir(), Name: "config", Reload: func() error { *reloads++; return nil }, Health: func() (string, error) { return "OK", nil }}
if err := os.WriteFile(filepath.Join(m.Dir, m.Name), []byte("prior\n"), 0600); err != nil {
t.Fatal(err)
}
if err := m.Initialize(); err != nil {
t.Fatal(err)
}
return m, reloads
}
func assertManagedContent(t *testing.T, m *managedTransaction, want string) {
t.Helper()
b, err := os.ReadFile(filepath.Join(m.Dir, m.Name))
if err != nil || string(b) != want {
t.Fatalf("content=%q err=%v want=%q", b, err, want)
}
}
func TestManagedApplyDurableFlow(t *testing.T) {
m, reloads := managedFixture(t)
inode, err := os.Stat(filepath.Join(m.Dir, m.Name))
if err != nil {
t.Fatal(err)
}
m.fault = func(point string) error {
if point == "prepared" {
assertManagedContent(t, m, "prior\n")
backup, err := m.read(managedBackupName)
if err != nil || string(backup) != "prior\n" {
t.Fatalf("backup=%q err=%v", backup, err)
}
j, err := m.load()
if err != nil || j.State != "prepared" || j.Generation != 1 || j.Next != managedHash([]byte("next\n")) {
t.Fatalf("journal=%+v err=%v", j, err)
}
}
return nil
}
if err := m.applyRendered([]byte("next\n")); err != nil {
t.Fatal(err)
}
assertManagedContent(t, m, "next\n")
after, err := os.Stat(filepath.Join(m.Dir, m.Name))
if err != nil || os.SameFile(inode, after) {
t.Fatal("config was not atomically replaced")
}
j, err := m.load()
if err != nil || j.State != "committed" || j.Hash != managedHash([]byte("next\n")) || *reloads != 1 {
t.Fatalf("journal=%+v reloads=%d err=%v", j, *reloads, err)
}
if err := m.Reconcile(); err != nil || *reloads != 1 {
t.Fatalf("committed recovery reloaded: %v", err)
}
entries, err := filepath.Glob(filepath.Join(m.Dir, ".autotune-stage-*"))
if err != nil || len(entries) != 0 {
t.Fatalf("left stages: %v %v", entries, err)
}
}
func TestManagedApplyCandidateIDs(t *testing.T) {
db := testStore(t)
front := managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
ggc := managedCandidateFixture(t, db, "googlevideo", "rr1.googlevideo.com", "--lua-desync=multisplit", true)
m, reloads := managedFixture(t)
if err := m.Apply(db, []int64{ggc}); err != nil {
t.Fatal(err)
}
assertActiveCandidates(t, m, db, []int64{ggc})
if *reloads != 1 {
t.Fatal("GGC-only not activated")
}
if err := m.Apply(db, []int64{front}); err != nil {
t.Fatal(err)
}
assertActiveCandidates(t, m, db, []int64{front})
if err := m.Apply(db, []int64{front, ggc}); err != nil {
t.Fatal(err)
}
want, err := renderManagedConfig(db, []int64{front, ggc})
if err != nil {
t.Fatal(err)
}
assertManagedContent(t, m, string(want))
assertActiveCandidates(t, m, db, []int64{front, ggc})
replacement := managedCandidateFixture(t, db, "frontend", "www.youtube.com", "--lua-desync=multisplit", true)
if err := m.Apply(db, []int64{replacement, ggc}); err != nil {
t.Fatal(err)
}
assertActiveCandidates(t, m, db, []int64{replacement, ggc})
if err := m.Apply(db, []int64{replacement}); err != nil {
t.Fatal(err)
}
assertActiveCandidates(t, m, db, []int64{replacement})
b, err := m.read(m.Name)
if err != nil || strings.Contains(string(b), "googlevideo") {
t.Fatalf("GGC was not removed: %q %v", b, err)
}
}
func assertActiveCandidates(t *testing.T, m *managedTransaction, db *sql.DB, want []int64) {
t.Helper()
// Re-create the engine to ensure no process-local active state is consulted.
fresh := &managedTransaction{Dir: m.Dir, Name: m.Name, Reload: m.Reload, Health: m.Health}
got, err := fresh.ActiveCandidates(db)
if err != nil || !reflect.DeepEqual(got, want) {
t.Fatalf("active=%v want=%v err=%v", got, want, err)
}
}
func TestManagedMissingJournalNotAdopted(t *testing.T) {
m, reloads := managedFixture(t)
if err := os.Remove(filepath.Join(m.Dir, managedJournalName)); err != nil {
t.Fatal(err)
}
if err := m.applyRendered([]byte("next")); err == nil {
t.Fatal("missing journal adopted")
}
if err := m.Reconcile(); err == nil {
t.Fatal("missing journal reconciled")
}
assertManagedContent(t, m, "prior\n")
if *reloads != 0 {
t.Fatal("missing journal reloaded")
}
}
func TestManagedFailures(t *testing.T) {
for _, mode := range []string{"reload", "unhealthy", "probe-error", "rollback-reload"} {
t.Run(mode, func(t *testing.T) {
m, reloads := managedFixture(t)
m.Reload = func() error {
*reloads++
if mode == "reload" && *reloads == 1 || mode == "rollback-reload" && *reloads == 2 {
return errors.New("reload failed")
}
return nil
}
m.Health = func() (string, error) {
if mode == "probe-error" {
return "OK", errors.New("probe failed")
}
if mode != "reload" {
return "DEGRADED", nil
}
return "OK", nil
}
err := m.applyRendered([]byte("next\n"))
if err == nil {
t.Fatal("failure accepted")
}
assertManagedContent(t, m, "prior\n")
j, loadErr := m.load()
if loadErr != nil || *reloads != 2 {
t.Fatalf("journal=%+v err=%v reloads=%d", j, loadErr, *reloads)
}
var rollback *managedRollbackError
if mode == "rollback-reload" {
if !errors.As(err, &rollback) || j.State != "prepared" {
t.Fatalf("rollback failure not distinct: %v %+v", err, j)
}
m.Reload = func() error { return nil }
if err := m.Reconcile(); err != nil {
t.Fatal(err)
}
} else if j.State != "committed" {
t.Fatalf("rollback incomplete: %+v", j)
}
})
}
}
func TestManagedCrashReconciliation(t *testing.T) {
for _, point := range []string{"staged", "backup", "prepared", "replaced", "reloaded", "healthy"} {
t.Run(point, func(t *testing.T) {
m, reloads := managedFixture(t)
m.fault = func(p string) error {
if p == point {
return errors.New("simulated death")
}
return nil
}
if err := m.applyRendered([]byte("next\n")); err == nil {
t.Fatal("fault missed")
}
before := *reloads
m.fault = nil
if err := m.Reconcile(); err != nil {
t.Fatal(err)
}
assertManagedContent(t, m, "prior\n")
want := before + 1
if point == "staged" || point == "backup" {
want = before
}
if *reloads != want {
t.Fatalf("reloads=%d want=%d", *reloads, want)
}
if err := m.applyRendered([]byte("later\n")); err != nil {
t.Fatal(err)
}
})
}
}
func TestManagedRejectsPathsAndDrift(t *testing.T) {
for _, name := range []string{"../config", "/tmp/config", ".autotune-journal.json", ".", ""} {
m, _ := managedFixture(t)
m.Name = name
if err := m.applyRendered([]byte("next")); err == nil {
t.Fatalf("accepted name %q", name)
}
}
for _, entry := range []string{"config", managedBackupName, managedJournalName, ".autotune-lock"} {
m, _ := managedFixture(t)
outside := filepath.Join(t.TempDir(), "outside")
if err := os.WriteFile(outside, []byte("untouched"), 0600); err != nil {
t.Fatal(err)
}
path := filepath.Join(m.Dir, entry)
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, path); err != nil {
t.Fatal(err)
}
if err := m.applyRendered([]byte("next")); err == nil {
t.Fatalf("accepted symlink %s", entry)
}
b, err := os.ReadFile(outside)
if err != nil || string(b) != "untouched" {
t.Fatal("outside modified")
}
}
m, _ := managedFixture(t)
alias := filepath.Join(t.TempDir(), "alias")
if err := os.Symlink(m.Dir, alias); err != nil {
t.Fatal(err)
}
real := m.Dir
m.Dir = alias
if err := m.Reconcile(); err == nil {
t.Fatal("accepted symlink directory")
}
m.Dir = real
if err := os.WriteFile(filepath.Join(m.Dir, m.Name), []byte("external"), 0600); err != nil {
t.Fatal(err)
}
if err := m.applyRendered([]byte("next")); !errors.Is(err, errManagedDrift) {
t.Fatalf("drift accepted: %v", err)
}
if err := m.Reconcile(); !errors.Is(err, errManagedDrift) {
t.Fatalf("recovery drift accepted: %v", err)
}
assertManagedContent(t, m, "external")
}
func TestManagedRecoveryFailsClosed(t *testing.T) {
for _, damage := range []string{"drift", "backup", "reload", "journal", "nonregular"} {
t.Run(damage, func(t *testing.T) {
m, _ := managedFixture(t)
m.fault = func(p string) error {
if p == "replaced" {
return errors.New("death")
}
return nil
}
if err := m.applyRendered([]byte("next\n")); err == nil {
t.Fatal("fault missed")
}
switch damage {
case "drift":
if err := os.WriteFile(filepath.Join(m.Dir, m.Name), []byte("external"), 0600); err != nil {
t.Fatal(err)
}
case "backup", "journal":
name := managedBackupName
if damage == "journal" {
name = managedJournalName
}
if err := os.WriteFile(filepath.Join(m.Dir, name), []byte("bad"), 0600); err != nil {
t.Fatal(err)
}
case "reload":
m.Reload = func() error { return errors.New("unavailable") }
case "nonregular":
if err := os.Remove(filepath.Join(m.Dir, m.Name)); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(m.Dir, m.Name), 0700); err != nil {
t.Fatal(err)
}
}
m.fault = nil
if err := m.Reconcile(); err == nil {
t.Fatal("unsafe recovery accepted")
}
if err := m.applyRendered([]byte("later")); err == nil {
t.Fatal("apply accepted after failed recovery")
}
if damage == "drift" {
assertManagedContent(t, m, "external")
}
})
}
}
func TestManagedCandidateCrashRollback(t *testing.T) {
for _, point := range []string{"staged", "backup", "prepared", "replaced", "reloaded", "healthy", "unhealthy"} {
t.Run(point, func(t *testing.T) {
db := testStore(t)
front := managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
ggc := managedCandidateFixture(t, db, "googlevideo", "rr1.googlevideo.com", "--lua-desync=fake", true)
next := managedCandidateFixture(t, db, "frontend", "www.youtube.com", "--lua-desync=multisplit", true)
m, _ := managedFixture(t)
if err := m.Apply(db, []int64{front, ggc}); err != nil {
t.Fatal(err)
}
prior, err := m.read(m.Name)
if err != nil {
t.Fatal(err)
}
m.fault = func(p string) error {
if p == "prepared" {
j, err := m.load()
if err != nil || !reflect.DeepEqual(j.Candidates, []int64{front, ggc}) || !reflect.DeepEqual(j.PendingCandidates, []int64{next}) {
t.Fatalf("prepared refs=%+v err=%v", j, err)
}
}
if p == point {
return errors.New("death")
}
return nil
}
if point == "unhealthy" {
m.Health = func() (string, error) { return "DEGRADED", nil }
}
if err := m.Apply(db, []int64{next}); err == nil {
t.Fatal("failure missed")
}
if point != "staged" && point != "backup" && point != "unhealthy" {
if ids, err := m.ActiveCandidates(db); err == nil || len(ids) != 0 {
t.Fatalf("incomplete transaction exposed IDs: %v %v", ids, err)
}
}
m.fault = nil
if err := m.Reconcile(db); err != nil {
t.Fatal(err)
}
assertManagedContent(t, m, string(prior))
assertActiveCandidates(t, m, db, []int64{front, ggc})
j, err := m.load()
if err != nil || j.Version != 2 || len(j.PendingCandidates) != 0 {
t.Fatalf("rollback journal=%+v err=%v", j, err)
}
})
}
}
func TestManagedActiveCandidatesFailClosed(t *testing.T) {
for _, damage := range []string{"drift", "missing", "unqualified", "wrong-refs", "duplicate", "invalid-id", "duplicate-group", "ggc-only", "pending"} {
t.Run(damage, func(t *testing.T) {
db := testStore(t)
front := managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
other := managedCandidateFixture(t, db, "frontend", "www.youtube.com", "--lua-desync=multisplit", true)
ggc := managedCandidateFixture(t, db, "googlevideo", "rr1.googlevideo.com", "--lua-desync=fake", true)
m, _ := managedFixture(t)
if err := m.Apply(db, []int64{front, ggc}); err != nil {
t.Fatal(err)
}
j, err := m.load()
if err != nil {
t.Fatal(err)
}
switch damage {
case "drift":
if err := os.WriteFile(filepath.Join(m.Dir, m.Name), []byte("external"), 0600); err != nil {
t.Fatal(err)
}
case "missing":
j.Candidates = []int64{99999, ggc}
case "unqualified":
if _, err := db.Exec("DELETE FROM candidate_evidence WHERE candidate_id = ?", front); err != nil {
t.Fatal(err)
}
case "wrong-refs":
j.Candidates = []int64{other, ggc}
case "duplicate":
j.Candidates = []int64{front, front}
case "invalid-id":
j.Candidates = []int64{-1}
case "duplicate-group":
j.Candidates = []int64{front, other}
case "ggc-only":
j.Candidates = []int64{ggc}
case "pending":
j.PendingCandidates = []int64{other}
}
if err := m.journal(j); err != nil {
t.Fatal(err)
}
if ids, err := m.ActiveCandidates(db); err == nil || len(ids) != 0 {
t.Fatalf("unsafe active refs=%v err=%v", ids, err)
}
})
}
}
func TestManagedLegacyJournal(t *testing.T) {
db := testStore(t)
front := managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
m, _ := managedFixture(t)
j, err := m.load()
if err != nil {
t.Fatal(err)
}
j.Version = 1
if err := m.journal(j); err != nil {
t.Fatal(err)
}
if _, err := m.ActiveCandidates(db); err == nil {
t.Fatal("legacy IDs inferred")
}
m.fault = func(p string) error {
if p == "replaced" {
return errors.New("death")
}
return nil
}
if err := m.Apply(db, []int64{front}); err == nil {
t.Fatal("fault missed")
}
if err := m.Reconcile(db); err != nil {
t.Fatal(err)
}
assertManagedContent(t, m, "prior\n")
if _, err := m.ActiveCandidates(db); err == nil {
t.Fatal("rollback inferred legacy IDs")
}
m.fault = nil
if err := m.Apply(db, []int64{front}); err != nil {
t.Fatal(err)
}
assertActiveCandidates(t, m, db, []int64{front})
}
func TestManagedStaleTransactionCannotCommitIDs(t *testing.T) {
for _, point := range []string{"staged", "backup", "prepared", "healthy"} {
t.Run(point, func(t *testing.T) {
db := testStore(t)
front := managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
m, _ := managedFixture(t)
var newer managedJournal
m.fault = func(p string) error {
if p == point {
var err error
newer, err = m.load()
if err != nil {
return err
}
newer.Generation++
newer.ID = "newer-transaction"
return m.journal(newer)
}
return nil
}
if err := m.Apply(db, []int64{front}); err == nil || !strings.Contains(err.Error(), "stale") {
t.Fatalf("stale apply=%v", err)
}
got, err := m.load()
if err != nil || !reflect.DeepEqual(got, newer) || len(got.Candidates) != 0 {
t.Fatalf("stale transaction overwrote journal: %+v %v", got, err)
}
})
}
}
func TestManagedLegacyPreparedJournalRollback(t *testing.T) {
m, _ := managedFixture(t)
m.fault = func(point string) error {
if point == "replaced" {
return errors.New("death")
}
return nil
}
if err := m.applyRendered([]byte("legacy pending\n")); err == nil {
t.Fatal("fault missed")
}
j, err := m.load()
if err != nil {
t.Fatal(err)
}
j.Version = 1
if err := m.journal(j); err != nil {
t.Fatal(err)
}
m.fault = nil
if err := m.Reconcile(); err != nil {
t.Fatal(err)
}
assertManagedContent(t, m, "prior\n")
j, err = m.load()
if err != nil || j.Hash != managedHash([]byte("prior\n")) || j.State != "committed" || len(j.Candidates) != 0 {
t.Fatalf("legacy rollback: %+v %v", j, err)
}
if _, err := m.ActiveCandidates(nil); err == nil {
t.Fatal("legacy IDs inferred")
}
}