feat(monitor): persist health and recovery cooldown
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
5c540ce03b
commit
0c51fd0196
9 files changed
+497
-9
No files matched your search
@@ -4,6 +4,7 @@
|
|||||||
|
|
||||||
## Готово
|
## Готово
|
||||||
|
|
||||||
|
- Добавлена foundation schema v4: durable policy state по stable service ID, last probe/streak, recovery lock и cooldown. Внутренние операции с injected time атомарно записывают probe и захватывают recovery только при threshold/истёкшем cooldown/свободном lock; завершение любого исхода запускает cooldown от completion time и требует точный ownership token, возвращённый acquisition. Stale/duplicate finish не меняет новую lease или cooldown; добавлен regression A → finish A → acquire B → stale finish A. Crash сохраняет lock (reconciliation пока не реализован). Добавлены unit-тесты streak/reset, threshold, concurrent writes/acquisition, reload/cooldown и сохранения v3 history/candidates при миграции. Проверки tests/build отложены оркестратору; здесь не запускались. Это только foundation: CLI health/recover не подключены, daemon/apply/network/container/Xray не затронуты.
|
||||||
- Добавлен одноразовый `health youtube`: bounded Player API-запрос только через явно заданный loopback SOCKS (`ZAPRET_AUTOTUNE_SOCKS_PROXY`, default `socks5://127.0.0.1:11080`), без ambient proxy/direct fallback. Health требует только `playabilityStatus.status=OK`, не streaming URL и не playback. Config валидирует video ID (default `dQw4w9WgXcQ`) и будущую policy: interval `5m`, failure threshold `3`, cooldown `30m`. Добавлены injectable unit-тесты config/health/CLI; проверки этого increment отложены оркестратору, тесты/build/network здесь не запускались. Autonomous monitor/apply/rollback не добавлены, контейнеры не изменены.
|
- Добавлен одноразовый `health youtube`: bounded Player API-запрос только через явно заданный loopback SOCKS (`ZAPRET_AUTOTUNE_SOCKS_PROXY`, default `socks5://127.0.0.1:11080`), без ambient proxy/direct fallback. Health требует только `playabilityStatus.status=OK`, не streaming URL и не playback. Config валидирует video ID (default `dQw4w9WgXcQ`) и будущую policy: interval `5m`, failure threshold `3`, cooldown `30m`. Добавлены injectable unit-тесты config/health/CLI; проверки этого increment отложены оркестратору, тесты/build/network здесь не запускались. Autonomous monitor/apply/rollback не добавлены, контейнеры не изменены.
|
||||||
- Завершён удалённый ручной GGC scan для `rr16---sn-n8v7znsz.googlevideo.com`: TLS 1.2 — 1 525 рабочих стратегий, TLS 1.3 — 800. Shortlist check нашёл 20 проходящих вариантов; точная объединённая стратегия `--lua-init=fake_default_tls=tls_mod(fake_default_tls,'rnd') --payload=tls_client_hello --lua-desync=multidisorder:pos=2,midsld:seqovl=1:seqovl_pattern=fake_default_tls --in-range=-s1 --lua-desync=oob:urp=midsld` прошла по 3/3 для TLS 1.2 и TLS 1.3. Результат относится только к указанному hostname; не означает playback/SABR и не применялся.
|
- Завершён удалённый ручной GGC scan для `rr16---sn-n8v7znsz.googlevideo.com`: TLS 1.2 — 1 525 рабочих стратегий, TLS 1.3 — 800. Shortlist check нашёл 20 проходящих вариантов; точная объединённая стратегия `--lua-init=fake_default_tls=tls_mod(fake_default_tls,'rnd') --payload=tls_client_hello --lua-desync=multidisorder:pos=2,midsld:seqovl=1:seqovl_pattern=fake_default_tls --in-range=-s1 --lua-desync=oob:urp=midsld` прошла по 3/3 для TLS 1.2 и TLS 1.3. Результат относится только к указанному hostname; не означает playback/SABR и не применялся.
|
||||||
- Добавлена proposal-only команда `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`: discovery одного свежего exact host → qualified schema-v3 сценарии newest-first → combined TLS1.2/1.3 3/3 точных сохранённых args. Первый успешный сценарий сохраняется новым immutable candidate/run и выдаётся как NOT ACTIVE без scan. Только отсутствие eligible candidates или валидные non-pass результаты разрешают обычный scan на уже найденном host; infrastructure/malformed-report ошибки fail closed. Apply/monitor и удалённый scan не затронуты.
|
- Добавлена proposal-only команда `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`: discovery одного свежего exact host → qualified schema-v3 сценарии newest-first → combined TLS1.2/1.3 3/3 точных сохранённых args. Первый успешный сценарий сохраняется новым immutable candidate/run и выдаётся как NOT ACTIVE без scan. Только отсутствие eligible candidates или валидные non-pass результаты разрешают обычный scan на уже найденном host; infrastructure/malformed-report ошибки fail closed. Apply/monitor и удалённый scan не затронуты.
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ threshold and cooldown are validated future recovery policy settings only:
|
|||||||
The future managed target is a **brand-new prod-like zapret2-ss container in the
|
The future managed target is a **brand-new prod-like zapret2-ss container in the
|
||||||
test environment only**, not the existing container. Its stable configurable
|
test environment only**, not the existing container. Its stable configurable
|
||||||
SOCKS endpoint must be loopback-accessible to this probe and same-host Xray.
|
SOCKS endpoint must be loopback-accessible to this probe and same-host Xray.
|
||||||
This increment does not create or change any container, persist health, apply
|
The one-shot command does not persist health. This increment does not create or change any container, apply
|
||||||
configuration, restart services or roll back anything. Xray configuration is
|
configuration, restart services or roll back anything. Xray configuration is
|
||||||
out of scope and will be configured separately by the user. Run this command in
|
out of scope and will be configured separately by the user. Run this command in
|
||||||
the host network context where that loopback endpoint is reachable; Docker
|
the host network context where that loopback endpoint is reachable; Docker
|
||||||
@@ -142,9 +142,24 @@ it must be writable. In Docker, mount that directory to retain history:
|
|||||||
docker run --rm -v autotune-data:/var/lib/zapret-autotune zapret-autotune status
|
docker run --rm -v autotune-data:/var/lib/zapret-autotune zapret-autotune status
|
||||||
```
|
```
|
||||||
|
|
||||||
Schema version 3 is initialized transactionally on `tune` and `status`; subsequent
|
Schema version 4 is initialized transactionally on `tune` and `status`; subsequent
|
||||||
opens are idempotent and newer, unsupported schema versions are rejected.
|
opens are idempotent and newer, unsupported schema versions are rejected.
|
||||||
|
|
||||||
|
Schema v4 adds a policy-state foundation keyed by stable service ID (not a cache
|
||||||
|
hostname or SOCKS address): last probe time, consecutive failures, recovery lock
|
||||||
|
and cooldown expiry. Internal operations accept injected time: success resets the
|
||||||
|
streak, failures increment it, and an atomic recovery acquisition requires the
|
||||||
|
configured threshold, an elapsed cooldown and no running recovery. Completion,
|
||||||
|
successful or failed, requires the exact per-acquisition ownership token returned
|
||||||
|
by acquisition, clears the lock and starts cooldown from completion time. Stale
|
||||||
|
or duplicate completion is rejected without changing the lock, token or cooldown,
|
||||||
|
even after a newer acquisition starts. A crash still leaves the durable lock held;
|
||||||
|
the token does not introduce automatic expiry or crash reconciliation.
|
||||||
|
State survives reopening the database; a crash during recovery leaves the lock
|
||||||
|
held, pending future explicit crash reconciliation. Existing history/candidates
|
||||||
|
are unchanged. These operations are not wired to `health youtube` or recovery:
|
||||||
|
**no daemon, autonomous probe, apply or container/Xray changes are added**.
|
||||||
|
|
||||||
Schema v3 adds immutable strategy candidate definitions (group, declared exact
|
Schema v3 adds immutable strategy candidate definitions (group, declared exact
|
||||||
endpoints, args and normalized-args fingerprint) and endpoint/protocol evidence.
|
endpoints, args and normalized-args fingerprint) and endpoint/protocol evidence.
|
||||||
Each newly successful tune saves its exact verified combined frontend profile for
|
Each newly successful tune saves its exact verified combined frontend profile for
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ func populatedV2Store(t *testing.T) *sql.DB {
|
|||||||
if err := migrate(db); err != nil {
|
if err := migrate(db); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, err := db.Exec(`DROP TABLE candidate_evidence; DROP TABLE strategy_candidates; PRAGMA user_version = 2;`); err != nil {
|
if _, err := db.Exec(`DROP TABLE monitor_state; DROP TABLE candidate_evidence; DROP TABLE strategy_candidates; PRAGMA user_version = 2;`); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
return db
|
return db
|
||||||
@@ -99,7 +99,7 @@ func TestPopulatedV2CandidateMigration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
var version int
|
var version int
|
||||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 3 {
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
||||||
t.Fatalf("version=%d err=%v", version, err)
|
t.Fatalf("version=%d err=%v", version, err)
|
||||||
}
|
}
|
||||||
if before != legacySnapshot(t, db) {
|
if before != legacySnapshot(t, db) {
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ func TestPopulatedV1FingerprintMigration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
assertRowCounts(t, db, 3, 4)
|
assertRowCounts(t, db, 3, 4)
|
||||||
var version int
|
var version int
|
||||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 3 {
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
||||||
t.Fatalf("version=%d err=%v", version, err)
|
t.Fatalf("version=%d err=%v", version, err)
|
||||||
}
|
}
|
||||||
for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} {
|
for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} {
|
||||||
|
|||||||
@@ -154,7 +154,7 @@ fi
|
|||||||
if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
|
if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if evidence != 2 || legacy != 0 || version != 3 {
|
if evidence != 2 || legacy != 0 || version != 4 {
|
||||||
t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version)
|
t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version)
|
||||||
}
|
}
|
||||||
for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} {
|
for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} {
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Policy foundation only; service_id identifies a service, not a rotating host
|
||||||
|
// or proxy address. No probe, scheduler or configuration changes happen here.
|
||||||
|
const monitorSchema = `
|
||||||
|
CREATE TABLE monitor_state (
|
||||||
|
service_id TEXT PRIMARY KEY NOT NULL CHECK (typeof(service_id) = 'text' AND length(trim(service_id)) > 0),
|
||||||
|
consecutive_failures INTEGER NOT NULL DEFAULT 0 CHECK (typeof(consecutive_failures) = 'integer' AND consecutive_failures >= 0),
|
||||||
|
last_probe_at INTEGER CHECK (last_probe_at IS NULL OR typeof(last_probe_at) = 'integer'),
|
||||||
|
recovery_running INTEGER NOT NULL DEFAULT 0 CHECK (typeof(recovery_running) = 'integer' AND recovery_running IN (0, 1)),
|
||||||
|
recovery_token TEXT CHECK (recovery_token IS NULL OR (typeof(recovery_token) = 'text' AND length(recovery_token) = 64)),
|
||||||
|
cooldown_until INTEGER CHECK (cooldown_until IS NULL OR typeof(cooldown_until) = 'integer'),
|
||||||
|
CHECK ((recovery_running = 0 AND recovery_token IS NULL) OR (recovery_running = 1 AND recovery_token IS NOT NULL)),
|
||||||
|
CHECK (last_probe_at IS NOT NULL OR consecutive_failures = 0)
|
||||||
|
);
|
||||||
|
PRAGMA user_version = 4;`
|
||||||
|
|
||||||
|
type monitorState struct {
|
||||||
|
ConsecutiveFailures int
|
||||||
|
LastProbeAt time.Time
|
||||||
|
RecoveryRunning bool
|
||||||
|
CooldownUntil time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s monitorState) healthStatus() string {
|
||||||
|
if s.LastProbeAt.IsZero() {
|
||||||
|
return "UNKNOWN"
|
||||||
|
}
|
||||||
|
if s.ConsecutiveFailures > 0 {
|
||||||
|
return "UNHEALTHY"
|
||||||
|
}
|
||||||
|
return "HEALTHY"
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateMonitorTime(serviceID string, now time.Time) error {
|
||||||
|
if strings.TrimSpace(serviceID) == "" || strings.TrimSpace(serviceID) != serviceID || strings.ContainsRune(serviceID, '\x00') {
|
||||||
|
return fmt.Errorf("invalid monitor service ID")
|
||||||
|
}
|
||||||
|
// Reject zero/overflowing timestamps rather than silently wrapping UnixNano.
|
||||||
|
if now.IsZero() || !time.Unix(0, now.UnixNano()).Equal(now) {
|
||||||
|
return fmt.Errorf("invalid monitor timestamp")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordMonitorProbe(db *sql.DB, serviceID string, success bool, now time.Time) error {
|
||||||
|
if err := validateMonitorTime(serviceID, now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
failures := 1
|
||||||
|
if success {
|
||||||
|
failures = 0
|
||||||
|
}
|
||||||
|
// One UPSERT serializes increments and leaves the recovery lock/cooldown intact.
|
||||||
|
_, err := db.Exec(`INSERT INTO monitor_state (service_id, consecutive_failures, last_probe_at)
|
||||||
|
VALUES (?, ?, ?) ON CONFLICT(service_id) DO UPDATE SET
|
||||||
|
consecutive_failures = CASE WHEN excluded.consecutive_failures = 0 THEN 0 ELSE monitor_state.consecutive_failures + 1 END,
|
||||||
|
last_probe_at = excluded.last_probe_at`, serviceID, failures, now.UnixNano())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func readMonitorState(db *sql.DB, serviceID string) (monitorState, error) {
|
||||||
|
var state monitorState
|
||||||
|
var probe, cooldown sql.NullInt64
|
||||||
|
err := db.QueryRow(`SELECT consecutive_failures, last_probe_at, recovery_running, cooldown_until
|
||||||
|
FROM monitor_state WHERE service_id = ?`, serviceID).Scan(&state.ConsecutiveFailures, &probe, &state.RecoveryRunning, &cooldown)
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return state, nil // A service without recorded probes has unknown health.
|
||||||
|
}
|
||||||
|
if probe.Valid {
|
||||||
|
state.LastProbeAt = time.Unix(0, probe.Int64).UTC()
|
||||||
|
}
|
||||||
|
if cooldown.Valid {
|
||||||
|
state.CooldownUntil = time.Unix(0, cooldown.Int64).UTC()
|
||||||
|
}
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// A nonempty token identifies this acquisition only; an empty token means busy
|
||||||
|
// or ineligible. The caller must retain it for finalization, including after reopen.
|
||||||
|
func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.Time) (string, error) {
|
||||||
|
if err := validateMonitorTime(serviceID, now); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if threshold <= 0 {
|
||||||
|
return "", fmt.Errorf("failure threshold must be positive")
|
||||||
|
}
|
||||||
|
var ownership [32]byte
|
||||||
|
if _, err := rand.Read(ownership[:]); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
token := hex.EncodeToString(ownership[:])
|
||||||
|
// Compare-and-set in one SQLite write, including across independent processes.
|
||||||
|
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 1, recovery_token = ?
|
||||||
|
WHERE service_id = ? AND consecutive_failures >= ? AND recovery_running = 0
|
||||||
|
AND (cooldown_until IS NULL OR cooldown_until <= ?)`, token, serviceID, threshold, now.UnixNano())
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
count, err := result.RowsAffected()
|
||||||
|
if err != nil || count != 1 {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return token, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Call on either success or failure. A crash leaves the durable lock held;
|
||||||
|
// automatic crash reconciliation is deliberately not part of this foundation.
|
||||||
|
func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.Time, cooldown time.Duration) error {
|
||||||
|
if err := validateMonitorTime(serviceID, completed); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if cooldown <= 0 {
|
||||||
|
return fmt.Errorf("recovery cooldown must be positive")
|
||||||
|
}
|
||||||
|
if len(token) != 64 {
|
||||||
|
return fmt.Errorf("invalid recovery ownership token")
|
||||||
|
}
|
||||||
|
expires := completed.Add(cooldown)
|
||||||
|
if err := validateMonitorTime(serviceID, expires); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, cooldown_until = ?
|
||||||
|
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), serviceID, token)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
count, err := result.RowsAffected()
|
||||||
|
if err == nil && count != 1 {
|
||||||
|
return fmt.Errorf("no recovery owned by this token for service %q", serviceID)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -0,0 +1,325 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var monitorTestTime = time.Date(2026, 10, 10, 12, 0, 0, 123, time.UTC)
|
||||||
|
|
||||||
|
func requireMonitorBegin(t *testing.T, db *sql.DB, id string, now time.Time, want bool) string {
|
||||||
|
t.Helper()
|
||||||
|
got, err := beginMonitorRecovery(db, id, 3, now)
|
||||||
|
if err != nil || (got != "") != want {
|
||||||
|
t.Fatalf("begin %s: got=%v want=%v err=%v", id, got, want, err)
|
||||||
|
}
|
||||||
|
return got
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorProbeStreakAndThreshold(t *testing.T) {
|
||||||
|
db := testStore(t)
|
||||||
|
now := monitorTestTime
|
||||||
|
state, err := readMonitorState(db, "new-test-service")
|
||||||
|
if err != nil || state.healthStatus() != "UNKNOWN" {
|
||||||
|
t.Fatalf("initial=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
requireMonitorBegin(t, db, "new-test-service", now, false)
|
||||||
|
for i := 1; i <= 3; i++ {
|
||||||
|
if err := recordMonitorProbe(db, "new-test-service", false, now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(db, "new-test-service")
|
||||||
|
if err != nil || state.ConsecutiveFailures != i || state.healthStatus() != "UNHEALTHY" || !state.LastProbeAt.Equal(now) {
|
||||||
|
t.Fatalf("probe %d: %+v err=%v", i, state, err)
|
||||||
|
}
|
||||||
|
if i < 3 {
|
||||||
|
requireMonitorBegin(t, db, "new-test-service", now, false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
requireMonitorBegin(t, db, "other-service", now, false)
|
||||||
|
token := requireMonitorBegin(t, db, "new-test-service", now, true)
|
||||||
|
requireMonitorBegin(t, db, "new-test-service", now, false)
|
||||||
|
if err := recordMonitorProbe(db, "new-test-service", true, now.Add(time.Minute)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err = readMonitorState(db, "new-test-service")
|
||||||
|
if err != nil || state.ConsecutiveFailures != 0 || state.healthStatus() != "HEALTHY" || !state.RecoveryRunning {
|
||||||
|
t.Fatalf("success=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(db, "new-test-service", token, now.Add(2*time.Minute), 30*time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
requireMonitorBegin(t, db, "new-test-service", now.Add(time.Hour), false)
|
||||||
|
if err := recordMonitorProbe(db, "new-test-service", false, now.Add(time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err = readMonitorState(db, "new-test-service")
|
||||||
|
if err != nil || state.ConsecutiveFailures != 1 || state.CooldownUntil.IsZero() {
|
||||||
|
t.Fatalf("new streak=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorRecoveryDurableCooldownForEitherOutcome(t *testing.T) {
|
||||||
|
for _, outcome := range []string{"success", "failure"} {
|
||||||
|
t.Run(outcome, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "monitor.db")
|
||||||
|
db, err := openStore(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { db.Close() })
|
||||||
|
reopen := func() {
|
||||||
|
t.Helper()
|
||||||
|
if err := db.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
db, err = openStore(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
now := monitorTestTime
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if err := recordMonitorProbe(db, "service", false, now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
token := requireMonitorBegin(t, db, "service", now, true)
|
||||||
|
reopen()
|
||||||
|
requireMonitorBegin(t, db, "service", now.Add(time.Hour), false)
|
||||||
|
// Outcome deliberately does not affect cooldown; completion time does.
|
||||||
|
completed := now.Add(10 * time.Minute)
|
||||||
|
if err := finishMonitorRecovery(db, "service", token, completed, 30*time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(db, "service", token, completed.Add(time.Minute), 30*time.Minute); err == nil {
|
||||||
|
t.Fatal("repeated finish accepted")
|
||||||
|
}
|
||||||
|
reopen()
|
||||||
|
state, err := readMonitorState(db, "service")
|
||||||
|
expires := completed.Add(30 * time.Minute)
|
||||||
|
if err != nil || state.RecoveryRunning || state.ConsecutiveFailures != 3 || !state.CooldownUntil.Equal(expires) {
|
||||||
|
t.Fatalf("reloaded=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
requireMonitorBegin(t, db, "service", expires.Add(-time.Nanosecond), false)
|
||||||
|
requireMonitorBegin(t, db, "service", expires, true)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorStaleFinishCannotReleaseNewRecovery(t *testing.T) {
|
||||||
|
db := testStore(t)
|
||||||
|
now := monitorTestTime
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if err := recordMonitorProbe(db, "service", false, now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tokenA := requireMonitorBegin(t, db, "service", now, true)
|
||||||
|
if err := finishMonitorRecovery(db, "service", tokenA, now, time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tokenB := requireMonitorBegin(t, db, "service", now.Add(time.Minute), true)
|
||||||
|
if tokenA == tokenB {
|
||||||
|
t.Fatal("new acquisition reused ownership token")
|
||||||
|
}
|
||||||
|
before, err := readMonitorState(db, "service")
|
||||||
|
if err != nil || !before.RecoveryRunning {
|
||||||
|
t.Fatalf("before stale finish: %+v err=%v", before, err)
|
||||||
|
}
|
||||||
|
for _, token := range []string{tokenA, ""} {
|
||||||
|
if err := finishMonitorRecovery(db, "service", token, now.Add(2*time.Minute), time.Hour); err == nil {
|
||||||
|
t.Fatal("unowned finish accepted")
|
||||||
|
}
|
||||||
|
after, err := readMonitorState(db, "service")
|
||||||
|
if err != nil || after != before {
|
||||||
|
t.Fatalf("unowned finish changed state: before=%+v after=%+v err=%v", before, after, err)
|
||||||
|
}
|
||||||
|
var storedToken string
|
||||||
|
if err := db.QueryRow("SELECT recovery_token FROM monitor_state WHERE service_id = ?", "service").Scan(&storedToken); err != nil || storedToken != tokenB {
|
||||||
|
t.Fatalf("new ownership changed: token=%q err=%v", storedToken, err)
|
||||||
|
}
|
||||||
|
requireMonitorBegin(t, db, "service", now.Add(24*time.Hour), false)
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(db, "service", tokenB, now.Add(2*time.Minute), time.Minute); err != nil {
|
||||||
|
t.Fatalf("current owner cannot finish: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorConcurrentWrites(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "monitor.db")
|
||||||
|
stores := make([]*sql.DB, 2)
|
||||||
|
for i := range stores {
|
||||||
|
db, err := openStore(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
stores[i] = db
|
||||||
|
}
|
||||||
|
const callers = 12
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < callers; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(db *sql.DB) {
|
||||||
|
defer wg.Done()
|
||||||
|
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}(stores[i%2])
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
state, err := readMonitorState(stores[0], "service")
|
||||||
|
if err != nil || state.ConsecutiveFailures != callers {
|
||||||
|
t.Fatalf("increments lost: %+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
results := make(chan bool, callers)
|
||||||
|
start := make(chan struct{})
|
||||||
|
for i := 0; i < callers; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(db *sql.DB) {
|
||||||
|
defer wg.Done()
|
||||||
|
<-start
|
||||||
|
got, err := beginMonitorRecovery(db, "service", 3, monitorTestTime)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
results <- got != ""
|
||||||
|
}(stores[i%2])
|
||||||
|
}
|
||||||
|
close(start)
|
||||||
|
wg.Wait()
|
||||||
|
close(results)
|
||||||
|
winners := 0
|
||||||
|
for got := range results {
|
||||||
|
if got {
|
||||||
|
winners++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if winners != 1 {
|
||||||
|
t.Fatalf("recovery winners=%d", winners)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorValidationAndConstraints(t *testing.T) {
|
||||||
|
db := testStore(t)
|
||||||
|
for _, id := range []string{"", " ", " service", "service\x00"} {
|
||||||
|
if err := recordMonitorProbe(db, id, false, monitorTestTime); err == nil {
|
||||||
|
t.Fatalf("accepted ID %q", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, now := range []time.Time{{}, time.Date(2500, 1, 1, 0, 0, 0, 0, time.UTC)} {
|
||||||
|
if err := recordMonitorProbe(db, "service", false, now); err == nil {
|
||||||
|
t.Fatal("accepted invalid timestamp")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, threshold := range []int{0, -1} {
|
||||||
|
if _, err := beginMonitorRecovery(db, "service", threshold, monitorTestTime); err == nil {
|
||||||
|
t.Fatal("accepted invalid threshold")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, cooldown := range []time.Duration{0, -time.Minute} {
|
||||||
|
if err := finishMonitorRecovery(db, "service", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", monitorTestTime, cooldown); err == nil {
|
||||||
|
t.Fatal("accepted invalid cooldown")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(db, "missing", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", monitorTestTime, time.Minute); err == nil {
|
||||||
|
t.Fatal("finished nonexistent recovery")
|
||||||
|
}
|
||||||
|
for _, statement := range []string{
|
||||||
|
`INSERT INTO monitor_state (service_id) VALUES ('')`,
|
||||||
|
`INSERT INTO monitor_state (service_id) VALUES (NULL)`,
|
||||||
|
`INSERT INTO monitor_state (service_id, consecutive_failures) VALUES ('x', -1)`,
|
||||||
|
`INSERT INTO monitor_state (service_id, consecutive_failures) VALUES ('x', 1)`,
|
||||||
|
`INSERT INTO monitor_state (service_id, recovery_running) VALUES ('x', 2)`,
|
||||||
|
`INSERT INTO monitor_state (service_id, recovery_running) VALUES ('x', 1)`,
|
||||||
|
`INSERT INTO monitor_state (service_id, recovery_token) VALUES ('x', 'bad')`,
|
||||||
|
`INSERT INTO monitor_state (service_id, recovery_token) VALUES ('x', '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef')`,
|
||||||
|
`INSERT INTO monitor_state (service_id, last_probe_at) VALUES ('x', 'bad')`,
|
||||||
|
`INSERT INTO monitor_state (service_id, cooldown_until) VALUES ('x', 'bad')`,
|
||||||
|
} {
|
||||||
|
if _, err := db.Exec(statement); err == nil {
|
||||||
|
t.Fatalf("constraint accepted %s", statement)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitorMigrationSnapshot(t *testing.T, db *sql.DB) string {
|
||||||
|
t.Helper()
|
||||||
|
snapshot := legacySnapshot(t, db)
|
||||||
|
for _, table := range []string{"strategy_candidates", "candidate_evidence", "googlevideo_tune_runs"} {
|
||||||
|
rows, err := db.Query("SELECT * FROM " + table + " ORDER BY 1, 2, 3")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
columns, err := rows.Columns()
|
||||||
|
if err != nil {
|
||||||
|
rows.Close()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
values := make([]interface{}, len(columns))
|
||||||
|
pointers := make([]interface{}, len(columns))
|
||||||
|
for i := range values {
|
||||||
|
pointers[i] = &values[i]
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
if err := rows.Scan(pointers...); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
snapshot += fmt.Sprintf("%s:%#v;", table, values)
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
|
||||||
|
db := testStore(t)
|
||||||
|
saveProposal(t, db, "--lua-desync=fake")
|
||||||
|
if _, err := db.Exec(`INSERT INTO googlevideo_tune_runs (started_at, finished_at, state, error)
|
||||||
|
VALUES ('start', 'finish', 'FAILED', 'original failure');
|
||||||
|
DROP TABLE monitor_state; PRAGMA user_version = 3;`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before := monitorMigrationSnapshot(t, db)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
if err := migrate(db); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var version, count int
|
||||||
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
||||||
|
t.Fatalf("version=%d err=%v", version, err)
|
||||||
|
}
|
||||||
|
if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 {
|
||||||
|
t.Fatalf("monitor count=%d err=%v", count, err)
|
||||||
|
}
|
||||||
|
if after := monitorMigrationSnapshot(t, db); before != after {
|
||||||
|
t.Fatal("v3 history/candidate data changed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorV3MigrationRollback(t *testing.T) {
|
||||||
|
db := testStore(t)
|
||||||
|
saveProposal(t, db, "--lua-desync=fake")
|
||||||
|
if _, err := db.Exec(`DROP TABLE monitor_state; CREATE TABLE monitor_state (conflict INTEGER); PRAGMA user_version = 3;`); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before := monitorMigrationSnapshot(t, db)
|
||||||
|
if err := migrate(db); err == nil {
|
||||||
|
t.Fatal("expected conflicting v4 schema failure")
|
||||||
|
}
|
||||||
|
var version int
|
||||||
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 3 || before != monitorMigrationSnapshot(t, db) {
|
||||||
|
t.Fatalf("failed migration changed data/version: %d err=%v", version, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -114,7 +114,7 @@ func migrate(db *sql.DB) error {
|
|||||||
if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil {
|
if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if version > 3 {
|
if version > 4 {
|
||||||
return fmt.Errorf("unsupported database schema version %d", version)
|
return fmt.Errorf("unsupported database schema version %d", version)
|
||||||
}
|
}
|
||||||
if version == 0 {
|
if version == 0 {
|
||||||
@@ -231,6 +231,11 @@ PRAGMA user_version = 1;`)
|
|||||||
if _, err := conn.ExecContext(ctx, googlevideoRunSchema); err != nil {
|
if _, err := conn.ExecContext(ctx, googlevideoRunSchema); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if version < 4 {
|
||||||
|
if _, err := conn.ExecContext(ctx, monitorSchema); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
_, err = conn.ExecContext(ctx, "COMMIT")
|
_, err = conn.ExecContext(ctx, "COMMIT")
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -224,7 +224,7 @@ func TestConcurrentFirstOpens(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, db := range stores {
|
for _, db := range stores {
|
||||||
var version, objects int
|
var version, objects int
|
||||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 3 {
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
||||||
t.Fatalf("version=%d err=%v", version, err)
|
t.Fatalf("version=%d err=%v", version, err)
|
||||||
}
|
}
|
||||||
if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE
|
if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE
|
||||||
@@ -377,7 +377,7 @@ func TestSchemaConstraintsAndIncompleteEvidence(t *testing.T) {
|
|||||||
|
|
||||||
func TestUnsupportedSchemaVersion(t *testing.T) {
|
func TestUnsupportedSchemaVersion(t *testing.T) {
|
||||||
db := testStore(t)
|
db := testStore(t)
|
||||||
if _, err := db.Exec("PRAGMA user_version = 4"); err != nil {
|
if _, err := db.Exec("PRAGMA user_version = 5"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") {
|
if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") {
|
||||||
|
|||||||
Reference in new issue
Block a user