feat(monitor): add crash-safe supervisor loop

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-10 09:16:58 +03:00
1 parent 54a0fb79f8
commit 468532dc8f
12 files changed
+971 -44

No files matched your search

+1 -1
View File
@@ -99,7 +99,7 @@ func TestPopulatedV2CandidateMigration(t *testing.T) {
}
}
var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d err=%v", version, err)
}
if before != legacySnapshot(t, db) {
+1 -1
View File
@@ -117,7 +117,7 @@ func TestPopulatedV1FingerprintMigration(t *testing.T) {
}
assertRowCounts(t, db, 3, 4)
var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d err=%v", version, err)
}
for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} {
+1 -1
View File
@@ -154,7 +154,7 @@ fi
if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
t.Fatal(err)
}
if evidence != 2 || legacy != 0 || version != 4 {
if evidence != 2 || legacy != 0 || version != 5 {
t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version)
}
for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} {
+19 -17
View File
@@ -218,16 +218,10 @@ func managedStatus(m *managedTransaction, out io.Writer) error {
})
}
func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
command, ids, err := managedArguments(args)
if err != nil {
fmt.Fprintln(stderr, "error:", err)
return 2
}
func managedRuntimeGuard(d managedDeps) (int, error) {
role, _ := d.lookup("ZAPRET_AUTOTUNE_TARGET_ROLE")
if role != "managed-test" {
fmt.Fprintln(stderr, "error: ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test")
return 2
return 2, fmt.Errorf("ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test")
}
for key, expected := range map[string]string{
"ZAPRET_AUTOTUNE_TARGET_ID": managedTargetID,
@@ -235,28 +229,36 @@ func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
"ZAPRET_CONFIG": managedTargetConfig,
} {
if value, _ := d.lookup(key); value != expected {
fmt.Fprintln(stderr, "error:", key, "does not match the fresh managed target contract")
return 2
return 2, fmt.Errorf("%s does not match the fresh managed target contract", key)
}
}
portText, _ := d.lookup("SOCKS_PORT")
port, portErr := strconv.Atoi(portText)
proxy, _ := d.lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
if portErr != nil || port < 1 || port > 65535 || strconv.Itoa(port) != portText || proxy != "socks5://127.0.0.1:"+portText {
fmt.Fprintln(stderr, "error: managed SOCKS proxy must match the target SOCKS_PORT listener")
return 2
return 2, fmt.Errorf("managed SOCKS proxy must match the target SOCKS_PORT listener")
}
if d.verifyRuntime == nil || d.verifySOCKS == nil {
fmt.Fprintln(stderr, "error: managed runtime validators unavailable")
return 1
return 1, fmt.Errorf("managed runtime validators unavailable")
}
if err := d.verifyRuntime(managedTargetDirectory); err != nil {
fmt.Fprintln(stderr, "error:", err)
return 1
return 1, err
}
if err := d.verifySOCKS(port); err != nil {
return 1, err
}
return 0, nil
}
func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
command, ids, err := managedArguments(args)
if err != nil {
fmt.Fprintln(stderr, "error:", err)
return 1
return 2
}
if code, err := managedRuntimeGuard(d); err != nil {
fmt.Fprintln(stderr, "error:", err)
return code
}
dir := d.directory
if err := managedDirectory(dir); err != nil {
+125 -5
View File
@@ -4,7 +4,10 @@ import (
"crypto/rand"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"os"
"strconv"
"strings"
"time"
)
@@ -24,6 +27,59 @@ CREATE TABLE monitor_state (
);
PRAGMA user_version = 4;`
// Legacy v4 leases have no provable owner: preserve them and fail closed.
const monitorOwnerSchema = `
ALTER TABLE monitor_state ADD COLUMN recovery_owner_pid INTEGER
CHECK (recovery_owner_pid IS NULL OR (typeof(recovery_owner_pid) = 'integer' AND recovery_owner_pid > 0 AND recovery_running = 1));
ALTER TABLE monitor_state ADD COLUMN recovery_owner_starttime INTEGER
CHECK ((recovery_owner_pid IS NULL AND recovery_owner_starttime IS NULL) OR
(recovery_owner_pid IS NOT NULL AND typeof(recovery_owner_starttime) = 'integer' AND recovery_owner_starttime > 0));
PRAGMA user_version = 5;`
// Only os.ErrNotExist proves absence; all other errors are indeterminate.
type monitorProcessReader func(pid int) (int64, error)
func readMonitorProcessStarttime(pid int) (int64, error) {
if pid <= 0 {
return 0, fmt.Errorf("invalid recovery owner PID")
}
data, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
if err != nil {
if errors.Is(err, os.ErrNotExist) {
// A missing stat file alone does not prove a missing process (or
// even a mounted procfs). Confirm absence and a readable procfs.
_, dirErr := os.Stat(fmt.Sprintf("/proc/%d", pid))
self, selfErr := os.ReadFile("/proc/self/stat")
if selfErr == nil {
_, selfErr = parseMonitorProcessStat(os.Getpid(), string(self))
}
if !errors.Is(dirErr, os.ErrNotExist) || selfErr != nil {
return 0, fmt.Errorf("cannot establish process absence for PID %d", pid)
}
}
return 0, err
}
return parseMonitorProcessStat(pid, string(data))
}
func parseMonitorProcessStat(pid int, stat string) (int64, error) {
// comm can contain spaces and parentheses. Field 22 is starttime;
// fields following the final ')' begin with field 3 (state).
open, close := strings.IndexByte(stat, '('), strings.LastIndexByte(stat, ')')
if open < 0 || close <= open || strings.TrimSpace(stat[:open]) != strconv.Itoa(pid) {
return 0, fmt.Errorf("invalid process stat identity")
}
fields := strings.Fields(stat[close+1:])
if len(fields) < 20 || len(fields[0]) != 1 || !strings.Contains("RSDZTtXxKWPI", fields[0]) {
return 0, fmt.Errorf("invalid process stat fields")
}
start, err := strconv.ParseInt(fields[19], 10, 64)
if err != nil || start <= 0 {
return 0, fmt.Errorf("invalid process starttime")
}
return start, nil
}
type monitorState struct {
ConsecutiveFailures int
LastProbeAt time.Time
@@ -88,21 +144,35 @@ func readMonitorState(db *sql.DB, serviceID string) (monitorState, error) {
// A nonempty token identifies this acquisition only; an empty token means busy
// or ineligible. The caller must retain it for finalization, including after reopen.
func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.Time) (string, error) {
return beginMonitorRecoveryWithProcess(db, serviceID, threshold, now, os.Getpid(), readMonitorProcessStarttime)
}
func beginMonitorRecoveryWithProcess(db *sql.DB, serviceID string, threshold int, now time.Time, pid int, read monitorProcessReader) (string, error) {
if err := validateMonitorTime(serviceID, now); err != nil {
return "", err
}
if threshold <= 0 {
return "", fmt.Errorf("failure threshold must be positive")
}
if pid <= 0 || read == nil {
return "", fmt.Errorf("invalid recovery process reader/owner")
}
start, err := read(pid)
if err != nil {
return "", fmt.Errorf("read recovery owner: %w", err)
}
if start <= 0 {
return "", fmt.Errorf("invalid recovery owner starttime")
}
var ownership [32]byte
if _, err := rand.Read(ownership[:]); err != nil {
return "", err
}
token := hex.EncodeToString(ownership[:])
// Compare-and-set in one SQLite write, including across independent processes.
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 1, recovery_token = ?
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 1, recovery_token = ?, recovery_owner_pid = ?, recovery_owner_starttime = ?
WHERE service_id = ? AND consecutive_failures >= ? AND recovery_running = 0
AND (cooldown_until IS NULL OR cooldown_until <= ?)`, token, serviceID, threshold, now.UnixNano())
AND (cooldown_until IS NULL OR cooldown_until <= ?)`, token, pid, start, serviceID, threshold, now.UnixNano())
if err != nil {
return "", err
}
@@ -113,8 +183,7 @@ func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.
return token, nil
}
// Call on either success or failure. A crash leaves the durable lock held;
// automatic crash reconciliation is deliberately not part of this foundation.
// Call on either success or failure. Only this acquisition's token can finish.
func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.Time, cooldown time.Duration) error {
if err := validateMonitorTime(serviceID, completed); err != nil {
return err
@@ -129,7 +198,7 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
if err := validateMonitorTime(serviceID, expires); err != nil {
return err
}
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, cooldown_until = ?
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), serviceID, token)
if err != nil {
return err
@@ -140,3 +209,54 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
}
return err
}
// Startup policy only: call for each configured service before monitoring.
// A legacy/indeterminate owner is an error, never permission to unlock.
// The conditional write also protects against a finish/reacquire during the read.
func reconcileMonitorRecovery(db *sql.DB, serviceID string, cooldown time.Duration, read monitorProcessReader, clock func() time.Time) (bool, error) {
if read == nil || clock == nil || cooldown <= 0 {
return false, fmt.Errorf("invalid reconciliation dependencies/cooldown")
}
now := clock()
if err := validateMonitorTime(serviceID, now); err != nil {
return false, err
}
expires := now.Add(cooldown)
if err := validateMonitorTime(serviceID, expires); err != nil {
return false, err
}
var token string
var pid, start sql.NullInt64
err := db.QueryRow(`SELECT recovery_token, recovery_owner_pid, recovery_owner_starttime
FROM monitor_state WHERE service_id = ? AND recovery_running = 1`, serviceID).Scan(&token, &pid, &start)
if err == sql.ErrNoRows {
return false, nil
}
if err != nil {
return false, err
}
if !pid.Valid || !start.Valid || pid.Int64 <= 0 || int64(int(pid.Int64)) != pid.Int64 || start.Int64 <= 0 || len(token) != 64 {
return false, fmt.Errorf("unverifiable recovery owner for service %q", serviceID)
}
current, err := read(int(pid.Int64))
if err != nil && !errors.Is(err, os.ErrNotExist) {
return false, fmt.Errorf("read recovery owner: %w", err)
}
if err == nil {
if current <= 0 {
return false, fmt.Errorf("invalid recovery owner starttime")
}
if current == start.Int64 {
return false, nil
}
}
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL,
recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?
AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), serviceID, token, pid.Int64, start.Int64)
if err != nil {
return false, err
}
count, err := result.RowsAffected()
return count == 1, err
}
+157
View File
@@ -0,0 +1,157 @@
package main
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"syscall"
"time"
)
const managedMonitorLockName = ".autotune-monitor-lock"
// No production adapter or entrypoint yet. Callbacks must honor ctx and join
// their children before returning; the supervisor never detaches callbacks.
type managedMonitorDeps struct {
target managedDeps
db *sql.DB
serviceID string
interval time.Duration
threshold int
cooldown time.Duration
probe func(context.Context) error
recover func(context.Context) error
now func() time.Time
wait func(context.Context, time.Duration) error
reportRecoveryError func(error)
}
// A startup reconciliation can be consumed once, never during live recovery.
type managedMonitorSingleton struct {
mu sync.Mutex
file *os.File
reconciled bool
}
func acquireManagedMonitorSingleton(d managedDeps) (*managedMonitorSingleton, error) {
if d.lookup == nil {
return nil, fmt.Errorf("managed runtime lookup unavailable")
}
if _, err := managedRuntimeGuard(d); err != nil {
return nil, err
}
if err := managedDirectory(d.directory); err != nil {
return nil, err
}
path := filepath.Join(d.directory, managedMonitorLockName)
fd, err := syscall.Open(path, syscall.O_CREAT|syscall.O_RDWR|syscall.O_NOFOLLOW|syscall.O_NONBLOCK|syscall.O_CLOEXEC, 0600)
if err != nil {
return nil, err
}
f := os.NewFile(uintptr(fd), "managed monitor singleton")
if err := managedPrivateFile(path); err != nil {
f.Close()
return nil, err
}
if err := syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
f.Close()
return nil, fmt.Errorf("managed monitor singleton: %w", err)
}
return &managedMonitorSingleton{file: f}, nil
}
func (s *managedMonitorSingleton) close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.file == nil {
return nil
}
err := s.file.Close() // Releases flock; never unlink the lock inode.
s.file = nil
return err
}
// Singleton serialization does not prove a recovery owner is dead. Delegate
// that decision to the v5 PID/starttime policy, including its crash cooldown.
func reconcileManagedMonitorRecovery(db *sql.DB, serviceID string, s *managedMonitorSingleton, cooldown time.Duration, read monitorProcessReader, now func() time.Time) error {
if s == nil {
return fmt.Errorf("monitor singleton required")
}
s.mu.Lock()
defer s.mu.Unlock()
if s.file == nil || s.reconciled {
return fmt.Errorf("startup singleton context unavailable")
}
s.reconciled = true
_, err := reconcileMonitorRecovery(db, serviceID, cooldown, read, now)
return err
}
func waitManagedMonitor(ctx context.Context, d time.Duration) error {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
if d.db == nil || d.interval <= 0 || d.threshold <= 0 || d.cooldown <= 0 || d.probe == nil || d.recover == nil || d.now == nil || d.wait == nil || d.reportRecoveryError == nil {
return fmt.Errorf("complete managed monitor dependencies and positive policy required")
}
if err := validateMonitorTime(d.serviceID, d.now()); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
singleton, err := acquireManagedMonitorSingleton(d.target)
if err != nil {
return err
}
defer func() { err = errors.Join(err, singleton.close()) }()
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, singleton, d.cooldown, readMonitorProcessStarttime, d.now); err != nil {
return err
}
for {
if err := ctx.Err(); err != nil {
return err
}
probeErr := d.probe(ctx)
if err := ctx.Err(); err != nil {
return err
}
if err := recordMonitorProbe(d.db, d.serviceID, probeErr == nil, d.now()); err != nil {
return err
}
if probeErr != nil {
token, err := beginMonitorRecovery(d.db, d.serviceID, d.threshold, d.now())
if err != nil {
return err
}
if token != "" {
recoveryErr := d.recover(ctx)
// Finalize even on cancellation, using the non-context DB operation.
if err := finishMonitorRecovery(d.db, d.serviceID, token, d.now(), d.cooldown); err != nil {
return err
}
if recoveryErr != nil {
d.reportRecoveryError(recoveryErr)
}
}
}
if err := ctx.Err(); err != nil {
return err
}
if err := d.wait(ctx, d.interval); err != nil {
return err
}
}
}
+471
View File
@@ -0,0 +1,471 @@
package main
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
)
var errMonitorTestUnhealthy = errors.New("unhealthy")
// Exercise the production process reader under the real singleton. PID reuse
// proves the old owner is gone and clears only that lease; a live owner stays
// busy, while an owner without v5 identity makes startup fail closed.
func TestManagedMonitorStartupOwnerPolicy(t *testing.T) {
for _, owner := range []string{"live", "reused", "legacy"} {
t.Run(owner, func(t *testing.T) {
d := monitorRuntimeFixture(t)
for i := 0; i < d.threshold; i++ {
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
t.Fatal(err)
}
}
token := requireMonitorBegin(t, d.db, d.serviceID, monitorTestTime, true)
if owner == "reused" {
if _, err := d.db.Exec(`UPDATE monitor_state SET recovery_owner_starttime = recovery_owner_starttime + 1 WHERE service_id = ?`, d.serviceID); err != nil {
t.Fatal(err)
}
}
if owner == "legacy" {
if _, err := d.db.Exec(`UPDATE monitor_state SET recovery_owner_pid = NULL, recovery_owner_starttime = NULL WHERE service_id = ?`, d.serviceID); err != nil {
t.Fatal(err)
}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probes := 0
d.probe = func(context.Context) error { probes++; return errMonitorTestUnhealthy }
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
err := runManagedMonitor(ctx, d)
if owner == "legacy" {
if err == nil || errors.Is(err, context.Canceled) || probes != 0 {
t.Fatalf("legacy startup: probes=%d err=%v", probes, err)
}
} else if !errors.Is(err, context.Canceled) || probes != 1 {
t.Fatalf("startup: probes=%d err=%v", probes, err)
}
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.RecoveryRunning != (owner != "reused") {
t.Fatalf("owner=%s state=%+v err=%v", owner, state, err)
}
if owner == "reused" {
if !state.CooldownUntil.Equal(monitorTestTime.Add(d.cooldown)) {
t.Fatalf("crash cooldown=%v", state.CooldownUntil)
}
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err == nil {
t.Fatal("old owner token accepted")
}
} else {
var stored string
if err := d.db.QueryRow(`SELECT recovery_token FROM monitor_state WHERE service_id = ?`, d.serviceID).Scan(&stored); err != nil || stored != token {
t.Fatalf("lease changed: %q %v", stored, err)
}
if !state.CooldownUntil.IsZero() {
t.Fatal("retained owner cooldown changed")
}
}
assertMonitorUnlocked(t, d)
})
}
}
func TestManagedMonitorUnverifiableOwnerUnderSingleton(t *testing.T) {
d := monitorRuntimeFixture(t)
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
t.Fatal(err)
}
token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime)
if err != nil || token == "" {
t.Fatalf("token=%q err=%v", token, err)
}
s, err := acquireManagedMonitorSingleton(d.target)
if err != nil {
t.Fatal(err)
}
defer s.close()
reader := func(int) (int64, error) { return 0, os.ErrPermission }
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, reader, d.now); !errors.Is(err, os.ErrPermission) {
t.Fatalf("unverifiable owner: %v", err)
}
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || !state.RecoveryRunning || !state.CooldownUntil.IsZero() {
t.Fatalf("state=%+v err=%v", state, err)
}
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err != nil {
t.Fatal(err)
}
if err := s.close(); err != nil {
t.Fatal(err)
}
assertMonitorUnlocked(t, d)
}
func monitorRuntimeFixture(t *testing.T) managedMonitorDeps {
t.Helper()
target, _, _ := managedCLIFixture(t)
if err := os.Chmod(target.directory, 0700); err != nil {
t.Fatal(err)
}
db, err := openStore(filepath.Join(t.TempDir(), "monitor.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
return managedMonitorDeps{
target: target, db: db, serviceID: "managed-youtube",
interval: 5 * time.Minute, threshold: 3, cooldown: 30 * time.Minute,
now: func() time.Time { return monitorTestTime },
probe: func(context.Context) error { return nil },
recover: func(context.Context) error { t.Fatal("unexpected recovery"); return nil },
wait: waitManagedMonitor,
reportRecoveryError: func(error) { t.Fatal("unexpected recovery error") },
}
}
func assertMonitorUnlocked(t *testing.T, d managedMonitorDeps) {
t.Helper()
s, err := acquireManagedMonitorSingleton(d.target)
if err != nil {
t.Fatalf("singleton leaked: %v", err)
}
if err := s.close(); err != nil {
t.Fatal(err)
}
}
func TestManagedMonitorImmediatePeriodicAndReset(t *testing.T) {
d := monitorRuntimeFixture(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
now := monitorTestTime
d.now = func() time.Time { return now }
observations := []bool{false, false, true, false, false, false}
wantStreak := []int{1, 2, 0, 1, 2, 3}
probes, waits, recoveries := 0, 0, 0
d.probe = func(context.Context) error {
if probes != waits || !now.Equal(monitorTestTime.Add(time.Duration(probes)*d.interval)) {
t.Fatal("probe not immediate/periodic")
}
healthy := observations[probes]
probes++
if healthy {
return nil
}
return errMonitorTestUnhealthy
}
d.recover = func(context.Context) error {
recoveries++
if probes != 6 {
t.Fatalf("recovery at probe %d", probes)
}
return nil
}
d.wait = func(ctx context.Context, interval time.Duration) error {
if interval != d.interval {
t.Fatal("wrong interval")
}
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.ConsecutiveFailures != wantStreak[probes-1] || !state.LastProbeAt.Equal(now) {
t.Fatalf("state=%+v err=%v", state, err)
}
waits++
if probes == len(observations) {
cancel()
return ctx.Err()
}
now = now.Add(interval)
return nil
}
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if probes != 6 || recoveries != 1 {
t.Fatalf("probes=%d recoveries=%d", probes, recoveries)
}
assertMonitorUnlocked(t, d)
}
func TestManagedMonitorCooldownFromCompletionEitherOutcome(t *testing.T) {
for _, failed := range []bool{false, true} {
t.Run(map[bool]string{false: "success", true: "failure"}[failed], func(t *testing.T) {
d := monitorRuntimeFixture(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
now := monitorTestTime
d.now = func() time.Time { return now }
probes, recoveries, reports := 0, 0, 0
d.probe = func(context.Context) error { probes++; return errMonitorTestUnhealthy }
d.recover = func(context.Context) error {
recoveries++
want := monitorTestTime.Add(10 * time.Minute)
if recoveries == 2 {
want = monitorTestTime.Add(50 * time.Minute)
}
if recoveries > 2 || !now.Equal(want) {
t.Fatalf("recovery %d at %v", recoveries, now)
}
if token := requireMonitorBegin(t, d.db, d.serviceID, now, false); token != "" {
t.Fatal("double start")
}
now = now.Add(10 * time.Minute)
if failed {
return errMonitorTestUnhealthy
}
return nil
}
d.reportRecoveryError = func(err error) {
if !errors.Is(err, errMonitorTestUnhealthy) {
t.Fatal(err)
}
reports++
}
d.wait = func(ctx context.Context, interval time.Duration) error {
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.RecoveryRunning {
t.Fatalf("state=%+v err=%v", state, err)
}
if recoveries > 0 {
want := monitorTestTime.Add(50 * time.Minute)
if recoveries == 2 {
want = monitorTestTime.Add(90 * time.Minute)
}
if !state.CooldownUntil.Equal(want) {
t.Fatalf("cooldown=%v want=%v", state.CooldownUntil, want)
}
}
if probes == 9 {
cancel()
return ctx.Err()
}
now = now.Add(interval)
return nil
}
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if recoveries != 2 || (failed && reports != 2) || (!failed && reports != 0) {
t.Fatalf("recoveries=%d reports=%d", recoveries, reports)
}
assertMonitorUnlocked(t, d)
})
}
}
func TestManagedMonitorStartupLeaseAfterReopenStartsCrashCooldown(t *testing.T) {
d := monitorRuntimeFixture(t)
path := filepath.Join(t.TempDir(), "monitor.db")
db, err := openStore(path)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 3; i++ {
if err := recordMonitorProbe(db, d.serviceID, false, monitorTestTime); err != nil {
t.Fatal(err)
}
}
// An impossible Linux PID proves a dead owner without spawning a child.
oldToken, err := beginMonitorRecoveryWithProcess(db, d.serviceID, 3, monitorTestTime, 1<<30, func(int) (int64, error) { return 100, nil })
if err != nil || oldToken == "" {
t.Fatalf("token=%q err=%v", oldToken, err)
}
expires := monitorTestTime.Add(d.cooldown)
if _, err := db.Exec(`UPDATE monitor_state SET cooldown_until = ? WHERE service_id = ?`, monitorTestTime.Add(time.Hour).UnixNano(), d.serviceID); err != nil {
t.Fatal(err)
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
d.db, err = openStore(path)
if err != nil {
t.Fatal(err)
}
defer d.db.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
d.probe = func(context.Context) error {
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(expires) || state.ConsecutiveFailures != 3 {
t.Fatalf("startup=%+v err=%v", state, err)
}
if err := finishMonitorRecovery(d.db, d.serviceID, oldToken, monitorTestTime, d.cooldown); err == nil {
t.Fatal("stale finish accepted")
}
return errMonitorTestUnhealthy
}
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.ConsecutiveFailures != 4 || !state.CooldownUntil.Equal(expires) {
t.Fatalf("state=%+v err=%v", state, err)
}
assertMonitorUnlocked(t, d)
}
func TestManagedMonitorConcurrentStartAndCancellation(t *testing.T) {
for _, phase := range []string{"probe", "recovery", "wait"} {
t.Run(phase, func(t *testing.T) {
d := monitorRuntimeFixture(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
entered := make(chan struct{})
childExited := make(chan struct{})
block := func(ctx context.Context) error {
go func() { <-ctx.Done(); close(childExited) }()
close(entered)
<-childExited // Callback joins its child before the supervisor releases the lock.
return ctx.Err()
}
switch phase {
case "probe":
d.probe = block
case "recovery":
d.threshold = 1
d.probe = func(context.Context) error { return errMonitorTestUnhealthy }
d.recover = block
d.reportRecoveryError = func(error) {}
case "wait":
d.wait = func(ctx context.Context, _ time.Duration) error { return block(ctx) }
}
done := make(chan error, 1)
finished := false
t.Cleanup(func() {
cancel()
if !finished {
select {
case <-done:
case <-time.After(5 * time.Second):
t.Error("supervisor leaked during cleanup")
}
}
})
go func() { done <- runManagedMonitor(ctx, d) }()
select {
case <-entered:
case err := <-done:
finished = true
t.Fatalf("early exit: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("monitor did not start")
}
if err := runManagedMonitor(context.Background(), d); err == nil {
t.Fatal("parallel monitor accepted")
}
cancel()
select {
case err := <-done:
finished = true
if !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("cancellation blocked")
}
select {
case <-childExited:
default:
t.Fatal("callback child leaked")
}
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.RecoveryRunning {
t.Fatalf("state=%+v err=%v", state, err)
}
if phase == "recovery" && !state.CooldownUntil.Equal(monitorTestTime.Add(d.cooldown)) {
t.Fatal("cancelled recovery not finalized")
}
assertMonitorUnlocked(t, d)
})
}
}
func TestManagedMonitorReconciliationCapabilityAndGuards(t *testing.T) {
d := monitorRuntimeFixture(t)
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, nil, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
t.Fatal("missing singleton accepted")
}
s, err := acquireManagedMonitorSingleton(d.target)
if err != nil {
t.Fatal(err)
}
defer s.close()
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err != nil {
t.Fatal(err)
}
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
t.Fatal(err)
}
token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime)
if err != nil || token == "" {
t.Fatalf("token=%q err=%v", token, err)
}
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
t.Fatal("live lease cleared by repeated startup")
}
if err := s.close(); err != nil {
t.Fatal(err)
}
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
t.Fatal("closed capability accepted")
}
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err != nil {
t.Fatal(err)
}
d.target.directory = t.TempDir()
d.target.lookup = func(string) (string, bool) { return "", false }
if _, err := acquireManagedMonitorSingleton(d.target); err == nil {
t.Fatal("unguarded singleton accepted")
}
if _, err := os.Lstat(filepath.Join(d.target.directory, managedMonitorLockName)); !os.IsNotExist(err) {
t.Fatalf("guard touched lock: %v", err)
}
}
func TestManagedMonitorFatalStoreAndFinalizationErrors(t *testing.T) {
for _, phase := range []string{"startup", "record", "begin", "finish"} {
t.Run(phase, func(t *testing.T) {
d := monitorRuntimeFixture(t)
d.threshold = 1
if phase == "startup" {
if err := d.db.Close(); err != nil {
t.Fatal(err)
}
}
d.probe = func(context.Context) error {
if phase == "startup" {
t.Fatal("probe ran after startup DB error")
}
if phase == "begin" {
_, err := d.db.Exec(`CREATE TRIGGER reject_monitor_begin BEFORE UPDATE OF recovery_running ON monitor_state WHEN NEW.recovery_running = 1 BEGIN SELECT RAISE(ABORT, 'begin rejected'); END`)
if err != nil {
t.Fatal(err)
}
}
if phase == "record" {
if err := d.db.Close(); err != nil {
t.Fatal(err)
}
}
return errMonitorTestUnhealthy
}
d.recover = func(context.Context) error {
if phase != "finish" {
t.Fatal("recovery ran after fatal DB error")
}
_, err := d.db.Exec(`CREATE TRIGGER reject_monitor_finish BEFORE UPDATE OF recovery_running ON monitor_state WHEN NEW.recovery_running = 0 BEGIN SELECT RAISE(ABORT, 'finish rejected'); END`)
if err != nil {
t.Fatal(err)
}
return nil
}
d.wait = func(context.Context, time.Duration) error { t.Fatal("continued after fatal DB error"); return nil }
if err := runManagedMonitor(context.Background(), d); err == nil {
t.Fatal("DB error ignored")
}
assertMonitorUnlocked(t, d)
})
}
}
+157 -1
View File
@@ -3,7 +3,9 @@ package main
import (
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
@@ -297,7 +299,7 @@ func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
}
}
var version, count int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d err=%v", version, err)
}
if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 {
@@ -308,6 +310,160 @@ func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
}
}
func TestMonitorCrashReconciliation(t *testing.T) {
for _, tc := range []struct {
name string
start int64
err error
clear, wantErr bool
}{
{"live", 100, nil, false, false},
{"missing", 0, os.ErrNotExist, true, false},
{"reused", 101, nil, true, false},
{"unreadable", 0, os.ErrPermission, false, true},
{"invalid", 0, nil, false, true},
} {
t.Run(tc.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "crash.db")
db, err := openStore(path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
for i := 0; i < 3; i++ {
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
t.Fatal(err)
}
}
token, err := beginMonitorRecoveryWithProcess(db, "service", 3, monitorTestTime, 42, func(pid int) (int64, error) {
if pid != 42 {
t.Fatalf("pid=%d", pid)
}
return 100, nil
})
if err != nil || token == "" {
t.Fatalf("acquire: %q %v", token, err)
}
var pid, start int64
if err := db.QueryRow(`SELECT recovery_owner_pid, recovery_owner_starttime FROM monitor_state WHERE service_id = 'service'`).Scan(&pid, &start); err != nil || pid != 42 || start != 100 {
t.Fatalf("owner=%d/%d err=%v", pid, start, err)
}
now := monitorTestTime.Add(time.Hour)
clear, err := reconcileMonitorRecovery(db, "service", 30*time.Minute, func(pid int) (int64, error) {
if pid != 42 {
t.Fatalf("pid=%d", pid)
}
return tc.start, tc.err
}, func() time.Time { return now })
if clear != tc.clear || (err != nil) != tc.wantErr {
t.Fatalf("clear=%v err=%v", clear, err)
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
db, err = openStore(path)
if err != nil {
t.Fatal(err)
}
state, err := readMonitorState(db, "service")
if err != nil || state.RecoveryRunning == tc.clear {
t.Fatalf("state=%+v err=%v", state, err)
}
if tc.clear {
expires := now.Add(30 * time.Minute)
if !state.CooldownUntil.Equal(expires) {
t.Fatalf("cooldown=%v", state.CooldownUntil)
}
requireMonitorBegin(t, db, "service", expires.Add(-time.Nanosecond), false)
newToken := requireMonitorBegin(t, db, "service", expires, true)
if err := finishMonitorRecovery(db, "service", token, expires, time.Hour); err == nil {
t.Fatal("stale crash token accepted")
}
if err := finishMonitorRecovery(db, "service", newToken, expires, time.Minute); err != nil {
t.Fatal(err)
}
} else if !state.CooldownUntil.IsZero() {
t.Fatal("retained lease changed cooldown")
}
})
}
}
func TestMonitorV4MigrationPreservesUnverifiableLease(t *testing.T) {
db := testStore(t)
if _, err := db.Exec("DROP TABLE monitor_state;" + monitorSchema); err != nil {
t.Fatal(err)
}
token := strings.Repeat("a", 64)
if _, err := db.Exec(`INSERT INTO monitor_state(service_id, consecutive_failures, last_probe_at, recovery_running, recovery_token, cooldown_until) VALUES ('service', 3, ?, 1, ?, ?)`, monitorTestTime.UnixNano(), token, monitorTestTime.Add(time.Minute).UnixNano()); err != nil {
t.Fatal(err)
}
before, err := readMonitorState(db, "service")
if err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
if err := migrate(db); err != nil {
t.Fatal(err)
}
}
var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d err=%v", version, err)
}
clear, err := reconcileMonitorRecovery(db, "service", time.Minute, func(int) (int64, error) {
t.Fatal("legacy owner must not be guessed")
return 0, os.ErrNotExist
}, func() time.Time { return monitorTestTime.Add(time.Hour) })
if clear || err == nil {
t.Fatalf("legacy clear=%v err=%v", clear, err)
}
after, err := readMonitorState(db, "service")
if err != nil || after != before {
t.Fatalf("migration changed state: %+v -> %+v err=%v", before, after, err)
}
var storedToken string
if err := db.QueryRow("SELECT recovery_token FROM monitor_state WHERE service_id = 'service'").Scan(&storedToken); err != nil || storedToken != token {
t.Fatalf("token=%q err=%v", storedToken, err)
}
}
func TestMonitorReconciliationCannotClearNewAcquisition(t *testing.T) {
db := testStore(t)
for i := 0; i < 3; i++ {
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
t.Fatal(err)
}
}
token := requireMonitorBegin(t, db, "service", monitorTestTime, true)
var newToken string
clear, err := reconcileMonitorRecovery(db, "service", time.Hour, func(int) (int64, error) {
if err := finishMonitorRecovery(db, "service", token, monitorTestTime, time.Minute); err != nil {
t.Fatal(err)
}
newToken = requireMonitorBegin(t, db, "service", monitorTestTime.Add(time.Minute), true)
return 0, os.ErrNotExist
}, func() time.Time { return monitorTestTime.Add(time.Minute) })
if err != nil || clear {
t.Fatalf("clear=%v err=%v", clear, err)
}
if err := finishMonitorRecovery(db, "service", newToken, monitorTestTime.Add(time.Minute), time.Minute); err != nil {
t.Fatal(err)
}
}
func TestMonitorProcessStatParser(t *testing.T) {
stat := "42 (name with ) parentheses) S" + strings.Repeat(" 0", 18) + " 123 0"
if start, err := parseMonitorProcessStat(42, stat); err != nil || start != 123 {
t.Fatalf("start=%d err=%v", start, err)
}
for _, bad := range []string{"", "41 (name) S" + strings.Repeat(" 0", 18) + " 123", "42 (name) S", strings.Replace(stat, "123", "invalid", 1), strings.Replace(stat, "123", "0", 1)} {
if _, err := parseMonitorProcessStat(42, bad); err == nil {
t.Fatalf("accepted %q", bad)
}
}
}
func TestMonitorV3MigrationRollback(t *testing.T) {
db := testStore(t)
saveProposal(t, db, "--lua-desync=fake")
+6 -1
View File
@@ -114,7 +114,7 @@ func migrate(db *sql.DB) error {
if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil {
return err
}
if version > 4 {
if version > 5 {
return fmt.Errorf("unsupported database schema version %d", version)
}
if version == 0 {
@@ -236,6 +236,11 @@ PRAGMA user_version = 1;`)
return err
}
}
if version < 5 {
if _, err := conn.ExecContext(ctx, monitorOwnerSchema); err != nil {
return err
}
}
_, err = conn.ExecContext(ctx, "COMMIT")
return err
}
+2 -2
View File
@@ -224,7 +224,7 @@ func TestConcurrentFirstOpens(t *testing.T) {
}
for _, db := range stores {
var version, objects int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d err=%v", version, err)
}
if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE
@@ -377,7 +377,7 @@ func TestSchemaConstraintsAndIncompleteEvidence(t *testing.T) {
func TestUnsupportedSchemaVersion(t *testing.T) {
db := testStore(t)
if _, err := db.Exec("PRAGMA user_version = 5"); err != nil {
if _, err := db.Exec("PRAGMA user_version = 6"); err != nil {
t.Fatal(err)
}
if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") {