482 lines
16 KiB
Go
482 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
var monitorTestTime = time.Date(2026, 10, 10, 12, 0, 0, 123, time.UTC)
|
|
|
|
func requireMonitorBegin(t *testing.T, db *sql.DB, id string, now time.Time, want bool) string {
|
|
t.Helper()
|
|
got, err := beginMonitorRecovery(db, id, 3, now)
|
|
if err != nil || (got != "") != want {
|
|
t.Fatalf("begin %s: got=%v want=%v err=%v", id, got, want, err)
|
|
}
|
|
return got
|
|
}
|
|
|
|
func TestMonitorProbeStreakAndThreshold(t *testing.T) {
|
|
db := testStore(t)
|
|
now := monitorTestTime
|
|
state, err := readMonitorState(db, "new-test-service")
|
|
if err != nil || state.healthStatus() != "UNKNOWN" {
|
|
t.Fatalf("initial=%+v err=%v", state, err)
|
|
}
|
|
requireMonitorBegin(t, db, "new-test-service", now, false)
|
|
for i := 1; i <= 3; i++ {
|
|
if err := recordMonitorProbe(db, "new-test-service", false, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state, err := readMonitorState(db, "new-test-service")
|
|
if err != nil || state.ConsecutiveFailures != i || state.healthStatus() != "UNHEALTHY" || !state.LastProbeAt.Equal(now) {
|
|
t.Fatalf("probe %d: %+v err=%v", i, state, err)
|
|
}
|
|
if i < 3 {
|
|
requireMonitorBegin(t, db, "new-test-service", now, false)
|
|
}
|
|
}
|
|
requireMonitorBegin(t, db, "other-service", now, false)
|
|
token := requireMonitorBegin(t, db, "new-test-service", now, true)
|
|
requireMonitorBegin(t, db, "new-test-service", now, false)
|
|
if err := recordMonitorProbe(db, "new-test-service", true, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state, err = readMonitorState(db, "new-test-service")
|
|
if err != nil || state.ConsecutiveFailures != 0 || state.healthStatus() != "HEALTHY" || !state.RecoveryRunning {
|
|
t.Fatalf("success=%+v err=%v", state, err)
|
|
}
|
|
if err := finishMonitorRecovery(db, "new-test-service", token, now.Add(2*time.Minute), 30*time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
requireMonitorBegin(t, db, "new-test-service", now.Add(time.Hour), false)
|
|
if err := recordMonitorProbe(db, "new-test-service", false, now.Add(time.Hour)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state, err = readMonitorState(db, "new-test-service")
|
|
if err != nil || state.ConsecutiveFailures != 1 || state.CooldownUntil.IsZero() {
|
|
t.Fatalf("new streak=%+v err=%v", state, err)
|
|
}
|
|
}
|
|
|
|
func TestMonitorRecoveryDurableCooldownForEitherOutcome(t *testing.T) {
|
|
for _, outcome := range []string{"success", "failure"} {
|
|
t.Run(outcome, func(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "monitor.db")
|
|
db, err := openStore(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { db.Close() })
|
|
reopen := func() {
|
|
t.Helper()
|
|
if err := db.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db, err = openStore(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
now := monitorTestTime
|
|
for i := 0; i < 3; i++ {
|
|
if err := recordMonitorProbe(db, "service", false, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
token := requireMonitorBegin(t, db, "service", now, true)
|
|
reopen()
|
|
requireMonitorBegin(t, db, "service", now.Add(time.Hour), false)
|
|
// Outcome deliberately does not affect cooldown; completion time does.
|
|
completed := now.Add(10 * time.Minute)
|
|
if err := finishMonitorRecovery(db, "service", token, completed, 30*time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := finishMonitorRecovery(db, "service", token, completed.Add(time.Minute), 30*time.Minute); err == nil {
|
|
t.Fatal("repeated finish accepted")
|
|
}
|
|
reopen()
|
|
state, err := readMonitorState(db, "service")
|
|
expires := completed.Add(30 * time.Minute)
|
|
if err != nil || state.RecoveryRunning || state.ConsecutiveFailures != 3 || !state.CooldownUntil.Equal(expires) {
|
|
t.Fatalf("reloaded=%+v err=%v", state, err)
|
|
}
|
|
requireMonitorBegin(t, db, "service", expires.Add(-time.Nanosecond), false)
|
|
requireMonitorBegin(t, db, "service", expires, true)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMonitorStaleFinishCannotReleaseNewRecovery(t *testing.T) {
|
|
db := testStore(t)
|
|
now := monitorTestTime
|
|
for i := 0; i < 3; i++ {
|
|
if err := recordMonitorProbe(db, "service", false, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
tokenA := requireMonitorBegin(t, db, "service", now, true)
|
|
if err := finishMonitorRecovery(db, "service", tokenA, now, time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tokenB := requireMonitorBegin(t, db, "service", now.Add(time.Minute), true)
|
|
if tokenA == tokenB {
|
|
t.Fatal("new acquisition reused ownership token")
|
|
}
|
|
before, err := readMonitorState(db, "service")
|
|
if err != nil || !before.RecoveryRunning {
|
|
t.Fatalf("before stale finish: %+v err=%v", before, err)
|
|
}
|
|
for _, token := range []string{tokenA, ""} {
|
|
if err := finishMonitorRecovery(db, "service", token, now.Add(2*time.Minute), time.Hour); err == nil {
|
|
t.Fatal("unowned finish accepted")
|
|
}
|
|
after, err := readMonitorState(db, "service")
|
|
if err != nil || after != before {
|
|
t.Fatalf("unowned finish changed state: before=%+v after=%+v err=%v", before, after, err)
|
|
}
|
|
var storedToken string
|
|
if err := db.QueryRow("SELECT recovery_token FROM monitor_state WHERE service_id = ?", "service").Scan(&storedToken); err != nil || storedToken != tokenB {
|
|
t.Fatalf("new ownership changed: token=%q err=%v", storedToken, err)
|
|
}
|
|
requireMonitorBegin(t, db, "service", now.Add(24*time.Hour), false)
|
|
}
|
|
if err := finishMonitorRecovery(db, "service", tokenB, now.Add(2*time.Minute), time.Minute); err != nil {
|
|
t.Fatalf("current owner cannot finish: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestMonitorConcurrentWrites(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "monitor.db")
|
|
stores := make([]*sql.DB, 2)
|
|
for i := range stores {
|
|
db, err := openStore(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
stores[i] = db
|
|
}
|
|
const callers = 12
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < callers; i++ {
|
|
wg.Add(1)
|
|
go func(db *sql.DB) {
|
|
defer wg.Done()
|
|
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}(stores[i%2])
|
|
}
|
|
wg.Wait()
|
|
state, err := readMonitorState(stores[0], "service")
|
|
if err != nil || state.ConsecutiveFailures != callers {
|
|
t.Fatalf("increments lost: %+v err=%v", state, err)
|
|
}
|
|
results := make(chan bool, callers)
|
|
start := make(chan struct{})
|
|
for i := 0; i < callers; i++ {
|
|
wg.Add(1)
|
|
go func(db *sql.DB) {
|
|
defer wg.Done()
|
|
<-start
|
|
got, err := beginMonitorRecovery(db, "service", 3, monitorTestTime)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
results <- got != ""
|
|
}(stores[i%2])
|
|
}
|
|
close(start)
|
|
wg.Wait()
|
|
close(results)
|
|
winners := 0
|
|
for got := range results {
|
|
if got {
|
|
winners++
|
|
}
|
|
}
|
|
if winners != 1 {
|
|
t.Fatalf("recovery winners=%d", winners)
|
|
}
|
|
}
|
|
|
|
func TestMonitorValidationAndConstraints(t *testing.T) {
|
|
db := testStore(t)
|
|
for _, id := range []string{"", " ", " service", "service\x00"} {
|
|
if err := recordMonitorProbe(db, id, false, monitorTestTime); err == nil {
|
|
t.Fatalf("accepted ID %q", id)
|
|
}
|
|
}
|
|
for _, now := range []time.Time{{}, time.Date(2500, 1, 1, 0, 0, 0, 0, time.UTC)} {
|
|
if err := recordMonitorProbe(db, "service", false, now); err == nil {
|
|
t.Fatal("accepted invalid timestamp")
|
|
}
|
|
}
|
|
for _, threshold := range []int{0, -1} {
|
|
if _, err := beginMonitorRecovery(db, "service", threshold, monitorTestTime); err == nil {
|
|
t.Fatal("accepted invalid threshold")
|
|
}
|
|
}
|
|
for _, cooldown := range []time.Duration{0, -time.Minute} {
|
|
if err := finishMonitorRecovery(db, "service", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", monitorTestTime, cooldown); err == nil {
|
|
t.Fatal("accepted invalid cooldown")
|
|
}
|
|
}
|
|
if err := finishMonitorRecovery(db, "missing", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", monitorTestTime, time.Minute); err == nil {
|
|
t.Fatal("finished nonexistent recovery")
|
|
}
|
|
for _, statement := range []string{
|
|
`INSERT INTO monitor_state (service_id) VALUES ('')`,
|
|
`INSERT INTO monitor_state (service_id) VALUES (NULL)`,
|
|
`INSERT INTO monitor_state (service_id, consecutive_failures) VALUES ('x', -1)`,
|
|
`INSERT INTO monitor_state (service_id, consecutive_failures) VALUES ('x', 1)`,
|
|
`INSERT INTO monitor_state (service_id, recovery_running) VALUES ('x', 2)`,
|
|
`INSERT INTO monitor_state (service_id, recovery_running) VALUES ('x', 1)`,
|
|
`INSERT INTO monitor_state (service_id, recovery_token) VALUES ('x', 'bad')`,
|
|
`INSERT INTO monitor_state (service_id, recovery_token) VALUES ('x', '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef')`,
|
|
`INSERT INTO monitor_state (service_id, last_probe_at) VALUES ('x', 'bad')`,
|
|
`INSERT INTO monitor_state (service_id, cooldown_until) VALUES ('x', 'bad')`,
|
|
} {
|
|
if _, err := db.Exec(statement); err == nil {
|
|
t.Fatalf("constraint accepted %s", statement)
|
|
}
|
|
}
|
|
}
|
|
|
|
func monitorMigrationSnapshot(t *testing.T, db *sql.DB) string {
|
|
t.Helper()
|
|
snapshot := legacySnapshot(t, db)
|
|
for _, table := range []string{"strategy_candidates", "candidate_evidence", "googlevideo_tune_runs"} {
|
|
rows, err := db.Query("SELECT * FROM " + table + " ORDER BY 1, 2, 3")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
columns, err := rows.Columns()
|
|
if err != nil {
|
|
rows.Close()
|
|
t.Fatal(err)
|
|
}
|
|
values := make([]interface{}, len(columns))
|
|
pointers := make([]interface{}, len(columns))
|
|
for i := range values {
|
|
pointers[i] = &values[i]
|
|
}
|
|
for rows.Next() {
|
|
if err := rows.Scan(pointers...); err != nil {
|
|
rows.Close()
|
|
t.Fatal(err)
|
|
}
|
|
snapshot += fmt.Sprintf("%s:%#v;", table, values)
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return snapshot
|
|
}
|
|
|
|
func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
|
|
db := testStore(t)
|
|
saveProposal(t, db, "--lua-desync=fake")
|
|
if _, err := db.Exec(`INSERT INTO googlevideo_tune_runs (started_at, finished_at, state, error)
|
|
VALUES ('start', 'finish', 'FAILED', 'original failure');
|
|
DROP TABLE monitor_state; PRAGMA user_version = 3;`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := monitorMigrationSnapshot(t, db)
|
|
for i := 0; i < 2; i++ {
|
|
if err := migrate(db); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
var version, count int
|
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
|
t.Fatalf("version=%d err=%v", version, err)
|
|
}
|
|
if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 {
|
|
t.Fatalf("monitor count=%d err=%v", count, err)
|
|
}
|
|
if after := monitorMigrationSnapshot(t, db); before != after {
|
|
t.Fatal("v3 history/candidate data changed")
|
|
}
|
|
}
|
|
|
|
func TestMonitorCrashReconciliation(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
start int64
|
|
err error
|
|
clear, wantErr bool
|
|
}{
|
|
{"live", 100, nil, false, false},
|
|
{"missing", 0, os.ErrNotExist, true, false},
|
|
{"reused", 101, nil, true, false},
|
|
{"unreadable", 0, os.ErrPermission, false, true},
|
|
{"invalid", 0, nil, false, true},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "crash.db")
|
|
db, err := openStore(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { db.Close() })
|
|
for i := 0; i < 3; i++ {
|
|
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
token, err := beginMonitorRecoveryWithProcess(db, "service", 3, monitorTestTime, 42, func(pid int) (int64, error) {
|
|
if pid != 42 {
|
|
t.Fatalf("pid=%d", pid)
|
|
}
|
|
return 100, nil
|
|
})
|
|
if err != nil || token == "" {
|
|
t.Fatalf("acquire: %q %v", token, err)
|
|
}
|
|
var pid, start int64
|
|
if err := db.QueryRow(`SELECT recovery_owner_pid, recovery_owner_starttime FROM monitor_state WHERE service_id = 'service'`).Scan(&pid, &start); err != nil || pid != 42 || start != 100 {
|
|
t.Fatalf("owner=%d/%d err=%v", pid, start, err)
|
|
}
|
|
now := monitorTestTime.Add(time.Hour)
|
|
clear, err := reconcileMonitorRecovery(db, "service", 30*time.Minute, func(pid int) (int64, error) {
|
|
if pid != 42 {
|
|
t.Fatalf("pid=%d", pid)
|
|
}
|
|
return tc.start, tc.err
|
|
}, func() time.Time { return now })
|
|
if clear != tc.clear || (err != nil) != tc.wantErr {
|
|
t.Fatalf("clear=%v err=%v", clear, err)
|
|
}
|
|
if err := db.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db, err = openStore(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state, err := readMonitorState(db, "service")
|
|
if err != nil || state.RecoveryRunning == tc.clear {
|
|
t.Fatalf("state=%+v err=%v", state, err)
|
|
}
|
|
if tc.clear {
|
|
expires := now.Add(30 * time.Minute)
|
|
if !state.CooldownUntil.Equal(expires) {
|
|
t.Fatalf("cooldown=%v", state.CooldownUntil)
|
|
}
|
|
requireMonitorBegin(t, db, "service", expires.Add(-time.Nanosecond), false)
|
|
newToken := requireMonitorBegin(t, db, "service", expires, true)
|
|
if err := finishMonitorRecovery(db, "service", token, expires, time.Hour); err == nil {
|
|
t.Fatal("stale crash token accepted")
|
|
}
|
|
if err := finishMonitorRecovery(db, "service", newToken, expires, time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
} else if !state.CooldownUntil.IsZero() {
|
|
t.Fatal("retained lease changed cooldown")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMonitorV4MigrationPreservesUnverifiableLease(t *testing.T) {
|
|
db := testStore(t)
|
|
if _, err := db.Exec("DROP TABLE monitor_state;" + monitorSchema); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
token := strings.Repeat("a", 64)
|
|
if _, err := db.Exec(`INSERT INTO monitor_state(service_id, consecutive_failures, last_probe_at, recovery_running, recovery_token, cooldown_until) VALUES ('service', 3, ?, 1, ?, ?)`, monitorTestTime.UnixNano(), token, monitorTestTime.Add(time.Minute).UnixNano()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before, err := readMonitorState(db, "service")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 2; i++ {
|
|
if err := migrate(db); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
var version int
|
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
|
t.Fatalf("version=%d err=%v", version, err)
|
|
}
|
|
clear, err := reconcileMonitorRecovery(db, "service", time.Minute, func(int) (int64, error) {
|
|
t.Fatal("legacy owner must not be guessed")
|
|
return 0, os.ErrNotExist
|
|
}, func() time.Time { return monitorTestTime.Add(time.Hour) })
|
|
if clear || err == nil {
|
|
t.Fatalf("legacy clear=%v err=%v", clear, err)
|
|
}
|
|
after, err := readMonitorState(db, "service")
|
|
if err != nil || after != before {
|
|
t.Fatalf("migration changed state: %+v -> %+v err=%v", before, after, err)
|
|
}
|
|
var storedToken string
|
|
if err := db.QueryRow("SELECT recovery_token FROM monitor_state WHERE service_id = 'service'").Scan(&storedToken); err != nil || storedToken != token {
|
|
t.Fatalf("token=%q err=%v", storedToken, err)
|
|
}
|
|
}
|
|
|
|
func TestMonitorReconciliationCannotClearNewAcquisition(t *testing.T) {
|
|
db := testStore(t)
|
|
for i := 0; i < 3; i++ {
|
|
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
token := requireMonitorBegin(t, db, "service", monitorTestTime, true)
|
|
var newToken string
|
|
clear, err := reconcileMonitorRecovery(db, "service", time.Hour, func(int) (int64, error) {
|
|
if err := finishMonitorRecovery(db, "service", token, monitorTestTime, time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
newToken = requireMonitorBegin(t, db, "service", monitorTestTime.Add(time.Minute), true)
|
|
return 0, os.ErrNotExist
|
|
}, func() time.Time { return monitorTestTime.Add(time.Minute) })
|
|
if err != nil || clear {
|
|
t.Fatalf("clear=%v err=%v", clear, err)
|
|
}
|
|
if err := finishMonitorRecovery(db, "service", newToken, monitorTestTime.Add(time.Minute), time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestMonitorProcessStatParser(t *testing.T) {
|
|
stat := "42 (name with ) parentheses) S" + strings.Repeat(" 0", 18) + " 123 0"
|
|
if start, err := parseMonitorProcessStat(42, stat); err != nil || start != 123 {
|
|
t.Fatalf("start=%d err=%v", start, err)
|
|
}
|
|
for _, bad := range []string{"", "41 (name) S" + strings.Repeat(" 0", 18) + " 123", "42 (name) S", strings.Replace(stat, "123", "invalid", 1), strings.Replace(stat, "123", "0", 1)} {
|
|
if _, err := parseMonitorProcessStat(42, bad); err == nil {
|
|
t.Fatalf("accepted %q", bad)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMonitorV3MigrationRollback(t *testing.T) {
|
|
db := testStore(t)
|
|
saveProposal(t, db, "--lua-desync=fake")
|
|
if _, err := db.Exec(`DROP TABLE monitor_state; CREATE TABLE monitor_state (conflict INTEGER); PRAGMA user_version = 3;`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := monitorMigrationSnapshot(t, db)
|
|
if err := migrate(db); err == nil {
|
|
t.Fatal("expected conflicting v4 schema failure")
|
|
}
|
|
var version int
|
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 3 || before != monitorMigrationSnapshot(t, db) {
|
|
t.Fatalf("failed migration changed data/version: %d err=%v", version, err)
|
|
}
|
|
}
|