feat(monitor): add crash-safe supervisor loop
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
54a0fb79f8
commit
468532dc8f
12 files changed
+971
-44
No files matched your search
@@ -99,7 +99,7 @@ func TestPopulatedV2CandidateMigration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
var version int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
||||
t.Fatalf("version=%d err=%v", version, err)
|
||||
}
|
||||
if before != legacySnapshot(t, db) {
|
||||
|
||||
@@ -117,7 +117,7 @@ func TestPopulatedV1FingerprintMigration(t *testing.T) {
|
||||
}
|
||||
assertRowCounts(t, db, 3, 4)
|
||||
var version int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
||||
t.Fatalf("version=%d err=%v", version, err)
|
||||
}
|
||||
for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} {
|
||||
|
||||
@@ -154,7 +154,7 @@ fi
|
||||
if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if evidence != 2 || legacy != 0 || version != 4 {
|
||||
if evidence != 2 || legacy != 0 || version != 5 {
|
||||
t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version)
|
||||
}
|
||||
for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} {
|
||||
|
||||
@@ -218,16 +218,10 @@ func managedStatus(m *managedTransaction, out io.Writer) error {
|
||||
})
|
||||
}
|
||||
|
||||
func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
|
||||
command, ids, err := managedArguments(args)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error:", err)
|
||||
return 2
|
||||
}
|
||||
func managedRuntimeGuard(d managedDeps) (int, error) {
|
||||
role, _ := d.lookup("ZAPRET_AUTOTUNE_TARGET_ROLE")
|
||||
if role != "managed-test" {
|
||||
fmt.Fprintln(stderr, "error: ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test")
|
||||
return 2
|
||||
return 2, fmt.Errorf("ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test")
|
||||
}
|
||||
for key, expected := range map[string]string{
|
||||
"ZAPRET_AUTOTUNE_TARGET_ID": managedTargetID,
|
||||
@@ -235,28 +229,36 @@ func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
|
||||
"ZAPRET_CONFIG": managedTargetConfig,
|
||||
} {
|
||||
if value, _ := d.lookup(key); value != expected {
|
||||
fmt.Fprintln(stderr, "error:", key, "does not match the fresh managed target contract")
|
||||
return 2
|
||||
return 2, fmt.Errorf("%s does not match the fresh managed target contract", key)
|
||||
}
|
||||
}
|
||||
portText, _ := d.lookup("SOCKS_PORT")
|
||||
port, portErr := strconv.Atoi(portText)
|
||||
proxy, _ := d.lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
|
||||
if portErr != nil || port < 1 || port > 65535 || strconv.Itoa(port) != portText || proxy != "socks5://127.0.0.1:"+portText {
|
||||
fmt.Fprintln(stderr, "error: managed SOCKS proxy must match the target SOCKS_PORT listener")
|
||||
return 2
|
||||
return 2, fmt.Errorf("managed SOCKS proxy must match the target SOCKS_PORT listener")
|
||||
}
|
||||
if d.verifyRuntime == nil || d.verifySOCKS == nil {
|
||||
fmt.Fprintln(stderr, "error: managed runtime validators unavailable")
|
||||
return 1
|
||||
return 1, fmt.Errorf("managed runtime validators unavailable")
|
||||
}
|
||||
if err := d.verifyRuntime(managedTargetDirectory); err != nil {
|
||||
fmt.Fprintln(stderr, "error:", err)
|
||||
return 1
|
||||
return 1, err
|
||||
}
|
||||
if err := d.verifySOCKS(port); err != nil {
|
||||
return 1, err
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
|
||||
command, ids, err := managedArguments(args)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error:", err)
|
||||
return 1
|
||||
return 2
|
||||
}
|
||||
if code, err := managedRuntimeGuard(d); err != nil {
|
||||
fmt.Fprintln(stderr, "error:", err)
|
||||
return code
|
||||
}
|
||||
dir := d.directory
|
||||
if err := managedDirectory(dir); err != nil {
|
||||
|
||||
@@ -4,7 +4,10 @@ import (
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -24,6 +27,59 @@ CREATE TABLE monitor_state (
|
||||
);
|
||||
PRAGMA user_version = 4;`
|
||||
|
||||
// Legacy v4 leases have no provable owner: preserve them and fail closed.
|
||||
const monitorOwnerSchema = `
|
||||
ALTER TABLE monitor_state ADD COLUMN recovery_owner_pid INTEGER
|
||||
CHECK (recovery_owner_pid IS NULL OR (typeof(recovery_owner_pid) = 'integer' AND recovery_owner_pid > 0 AND recovery_running = 1));
|
||||
ALTER TABLE monitor_state ADD COLUMN recovery_owner_starttime INTEGER
|
||||
CHECK ((recovery_owner_pid IS NULL AND recovery_owner_starttime IS NULL) OR
|
||||
(recovery_owner_pid IS NOT NULL AND typeof(recovery_owner_starttime) = 'integer' AND recovery_owner_starttime > 0));
|
||||
PRAGMA user_version = 5;`
|
||||
|
||||
// Only os.ErrNotExist proves absence; all other errors are indeterminate.
|
||||
type monitorProcessReader func(pid int) (int64, error)
|
||||
|
||||
func readMonitorProcessStarttime(pid int) (int64, error) {
|
||||
if pid <= 0 {
|
||||
return 0, fmt.Errorf("invalid recovery owner PID")
|
||||
}
|
||||
data, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
// A missing stat file alone does not prove a missing process (or
|
||||
// even a mounted procfs). Confirm absence and a readable procfs.
|
||||
_, dirErr := os.Stat(fmt.Sprintf("/proc/%d", pid))
|
||||
self, selfErr := os.ReadFile("/proc/self/stat")
|
||||
if selfErr == nil {
|
||||
_, selfErr = parseMonitorProcessStat(os.Getpid(), string(self))
|
||||
}
|
||||
if !errors.Is(dirErr, os.ErrNotExist) || selfErr != nil {
|
||||
return 0, fmt.Errorf("cannot establish process absence for PID %d", pid)
|
||||
}
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
return parseMonitorProcessStat(pid, string(data))
|
||||
}
|
||||
|
||||
func parseMonitorProcessStat(pid int, stat string) (int64, error) {
|
||||
// comm can contain spaces and parentheses. Field 22 is starttime;
|
||||
// fields following the final ')' begin with field 3 (state).
|
||||
open, close := strings.IndexByte(stat, '('), strings.LastIndexByte(stat, ')')
|
||||
if open < 0 || close <= open || strings.TrimSpace(stat[:open]) != strconv.Itoa(pid) {
|
||||
return 0, fmt.Errorf("invalid process stat identity")
|
||||
}
|
||||
fields := strings.Fields(stat[close+1:])
|
||||
if len(fields) < 20 || len(fields[0]) != 1 || !strings.Contains("RSDZTtXxKWPI", fields[0]) {
|
||||
return 0, fmt.Errorf("invalid process stat fields")
|
||||
}
|
||||
start, err := strconv.ParseInt(fields[19], 10, 64)
|
||||
if err != nil || start <= 0 {
|
||||
return 0, fmt.Errorf("invalid process starttime")
|
||||
}
|
||||
return start, nil
|
||||
}
|
||||
|
||||
type monitorState struct {
|
||||
ConsecutiveFailures int
|
||||
LastProbeAt time.Time
|
||||
@@ -88,21 +144,35 @@ func readMonitorState(db *sql.DB, serviceID string) (monitorState, error) {
|
||||
// A nonempty token identifies this acquisition only; an empty token means busy
|
||||
// or ineligible. The caller must retain it for finalization, including after reopen.
|
||||
func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.Time) (string, error) {
|
||||
return beginMonitorRecoveryWithProcess(db, serviceID, threshold, now, os.Getpid(), readMonitorProcessStarttime)
|
||||
}
|
||||
|
||||
func beginMonitorRecoveryWithProcess(db *sql.DB, serviceID string, threshold int, now time.Time, pid int, read monitorProcessReader) (string, error) {
|
||||
if err := validateMonitorTime(serviceID, now); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if threshold <= 0 {
|
||||
return "", fmt.Errorf("failure threshold must be positive")
|
||||
}
|
||||
if pid <= 0 || read == nil {
|
||||
return "", fmt.Errorf("invalid recovery process reader/owner")
|
||||
}
|
||||
start, err := read(pid)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read recovery owner: %w", err)
|
||||
}
|
||||
if start <= 0 {
|
||||
return "", fmt.Errorf("invalid recovery owner starttime")
|
||||
}
|
||||
var ownership [32]byte
|
||||
if _, err := rand.Read(ownership[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
token := hex.EncodeToString(ownership[:])
|
||||
// Compare-and-set in one SQLite write, including across independent processes.
|
||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 1, recovery_token = ?
|
||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 1, recovery_token = ?, recovery_owner_pid = ?, recovery_owner_starttime = ?
|
||||
WHERE service_id = ? AND consecutive_failures >= ? AND recovery_running = 0
|
||||
AND (cooldown_until IS NULL OR cooldown_until <= ?)`, token, serviceID, threshold, now.UnixNano())
|
||||
AND (cooldown_until IS NULL OR cooldown_until <= ?)`, token, pid, start, serviceID, threshold, now.UnixNano())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -113,8 +183,7 @@ func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.
|
||||
return token, nil
|
||||
}
|
||||
|
||||
// Call on either success or failure. A crash leaves the durable lock held;
|
||||
// automatic crash reconciliation is deliberately not part of this foundation.
|
||||
// Call on either success or failure. Only this acquisition's token can finish.
|
||||
func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.Time, cooldown time.Duration) error {
|
||||
if err := validateMonitorTime(serviceID, completed); err != nil {
|
||||
return err
|
||||
@@ -129,7 +198,7 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
|
||||
if err := validateMonitorTime(serviceID, expires); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, cooldown_until = ?
|
||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
|
||||
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), serviceID, token)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -140,3 +209,54 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Startup policy only: call for each configured service before monitoring.
|
||||
// A legacy/indeterminate owner is an error, never permission to unlock.
|
||||
// The conditional write also protects against a finish/reacquire during the read.
|
||||
func reconcileMonitorRecovery(db *sql.DB, serviceID string, cooldown time.Duration, read monitorProcessReader, clock func() time.Time) (bool, error) {
|
||||
if read == nil || clock == nil || cooldown <= 0 {
|
||||
return false, fmt.Errorf("invalid reconciliation dependencies/cooldown")
|
||||
}
|
||||
now := clock()
|
||||
if err := validateMonitorTime(serviceID, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
expires := now.Add(cooldown)
|
||||
if err := validateMonitorTime(serviceID, expires); err != nil {
|
||||
return false, err
|
||||
}
|
||||
var token string
|
||||
var pid, start sql.NullInt64
|
||||
err := db.QueryRow(`SELECT recovery_token, recovery_owner_pid, recovery_owner_starttime
|
||||
FROM monitor_state WHERE service_id = ? AND recovery_running = 1`, serviceID).Scan(&token, &pid, &start)
|
||||
if err == sql.ErrNoRows {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !pid.Valid || !start.Valid || pid.Int64 <= 0 || int64(int(pid.Int64)) != pid.Int64 || start.Int64 <= 0 || len(token) != 64 {
|
||||
return false, fmt.Errorf("unverifiable recovery owner for service %q", serviceID)
|
||||
}
|
||||
current, err := read(int(pid.Int64))
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return false, fmt.Errorf("read recovery owner: %w", err)
|
||||
}
|
||||
if err == nil {
|
||||
if current <= 0 {
|
||||
return false, fmt.Errorf("invalid recovery owner starttime")
|
||||
}
|
||||
if current == start.Int64 {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL,
|
||||
recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
|
||||
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?
|
||||
AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), serviceID, token, pid.Int64, start.Int64)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
count, err := result.RowsAffected()
|
||||
return count == 1, err
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
const managedMonitorLockName = ".autotune-monitor-lock"
|
||||
|
||||
// No production adapter or entrypoint yet. Callbacks must honor ctx and join
|
||||
// their children before returning; the supervisor never detaches callbacks.
|
||||
type managedMonitorDeps struct {
|
||||
target managedDeps
|
||||
db *sql.DB
|
||||
serviceID string
|
||||
interval time.Duration
|
||||
threshold int
|
||||
cooldown time.Duration
|
||||
probe func(context.Context) error
|
||||
recover func(context.Context) error
|
||||
now func() time.Time
|
||||
wait func(context.Context, time.Duration) error
|
||||
reportRecoveryError func(error)
|
||||
}
|
||||
|
||||
// A startup reconciliation can be consumed once, never during live recovery.
|
||||
type managedMonitorSingleton struct {
|
||||
mu sync.Mutex
|
||||
file *os.File
|
||||
reconciled bool
|
||||
}
|
||||
|
||||
func acquireManagedMonitorSingleton(d managedDeps) (*managedMonitorSingleton, error) {
|
||||
if d.lookup == nil {
|
||||
return nil, fmt.Errorf("managed runtime lookup unavailable")
|
||||
}
|
||||
if _, err := managedRuntimeGuard(d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := managedDirectory(d.directory); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
path := filepath.Join(d.directory, managedMonitorLockName)
|
||||
fd, err := syscall.Open(path, syscall.O_CREAT|syscall.O_RDWR|syscall.O_NOFOLLOW|syscall.O_NONBLOCK|syscall.O_CLOEXEC, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := os.NewFile(uintptr(fd), "managed monitor singleton")
|
||||
if err := managedPrivateFile(path); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("managed monitor singleton: %w", err)
|
||||
}
|
||||
return &managedMonitorSingleton{file: f}, nil
|
||||
}
|
||||
|
||||
func (s *managedMonitorSingleton) close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.file == nil {
|
||||
return nil
|
||||
}
|
||||
err := s.file.Close() // Releases flock; never unlink the lock inode.
|
||||
s.file = nil
|
||||
return err
|
||||
}
|
||||
|
||||
// Singleton serialization does not prove a recovery owner is dead. Delegate
|
||||
// that decision to the v5 PID/starttime policy, including its crash cooldown.
|
||||
func reconcileManagedMonitorRecovery(db *sql.DB, serviceID string, s *managedMonitorSingleton, cooldown time.Duration, read monitorProcessReader, now func() time.Time) error {
|
||||
if s == nil {
|
||||
return fmt.Errorf("monitor singleton required")
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.file == nil || s.reconciled {
|
||||
return fmt.Errorf("startup singleton context unavailable")
|
||||
}
|
||||
s.reconciled = true
|
||||
_, err := reconcileMonitorRecovery(db, serviceID, cooldown, read, now)
|
||||
return err
|
||||
}
|
||||
|
||||
func waitManagedMonitor(ctx context.Context, d time.Duration) error {
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
||||
if d.db == nil || d.interval <= 0 || d.threshold <= 0 || d.cooldown <= 0 || d.probe == nil || d.recover == nil || d.now == nil || d.wait == nil || d.reportRecoveryError == nil {
|
||||
return fmt.Errorf("complete managed monitor dependencies and positive policy required")
|
||||
}
|
||||
if err := validateMonitorTime(d.serviceID, d.now()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
singleton, err := acquireManagedMonitorSingleton(d.target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { err = errors.Join(err, singleton.close()) }()
|
||||
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, singleton, d.cooldown, readMonitorProcessStarttime, d.now); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
probeErr := d.probe(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := recordMonitorProbe(d.db, d.serviceID, probeErr == nil, d.now()); err != nil {
|
||||
return err
|
||||
}
|
||||
if probeErr != nil {
|
||||
token, err := beginMonitorRecovery(d.db, d.serviceID, d.threshold, d.now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if token != "" {
|
||||
recoveryErr := d.recover(ctx)
|
||||
// Finalize even on cancellation, using the non-context DB operation.
|
||||
if err := finishMonitorRecovery(d.db, d.serviceID, token, d.now(), d.cooldown); err != nil {
|
||||
return err
|
||||
}
|
||||
if recoveryErr != nil {
|
||||
d.reportRecoveryError(recoveryErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.wait(ctx, d.interval); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,471 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var errMonitorTestUnhealthy = errors.New("unhealthy")
|
||||
|
||||
// Exercise the production process reader under the real singleton. PID reuse
|
||||
// proves the old owner is gone and clears only that lease; a live owner stays
|
||||
// busy, while an owner without v5 identity makes startup fail closed.
|
||||
func TestManagedMonitorStartupOwnerPolicy(t *testing.T) {
|
||||
for _, owner := range []string{"live", "reused", "legacy"} {
|
||||
t.Run(owner, func(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
for i := 0; i < d.threshold; i++ {
|
||||
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
token := requireMonitorBegin(t, d.db, d.serviceID, monitorTestTime, true)
|
||||
if owner == "reused" {
|
||||
if _, err := d.db.Exec(`UPDATE monitor_state SET recovery_owner_starttime = recovery_owner_starttime + 1 WHERE service_id = ?`, d.serviceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if owner == "legacy" {
|
||||
if _, err := d.db.Exec(`UPDATE monitor_state SET recovery_owner_pid = NULL, recovery_owner_starttime = NULL WHERE service_id = ?`, d.serviceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
probes := 0
|
||||
d.probe = func(context.Context) error { probes++; return errMonitorTestUnhealthy }
|
||||
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||
err := runManagedMonitor(ctx, d)
|
||||
if owner == "legacy" {
|
||||
if err == nil || errors.Is(err, context.Canceled) || probes != 0 {
|
||||
t.Fatalf("legacy startup: probes=%d err=%v", probes, err)
|
||||
}
|
||||
} else if !errors.Is(err, context.Canceled) || probes != 1 {
|
||||
t.Fatalf("startup: probes=%d err=%v", probes, err)
|
||||
}
|
||||
state, err := readMonitorState(d.db, d.serviceID)
|
||||
if err != nil || state.RecoveryRunning != (owner != "reused") {
|
||||
t.Fatalf("owner=%s state=%+v err=%v", owner, state, err)
|
||||
}
|
||||
if owner == "reused" {
|
||||
if !state.CooldownUntil.Equal(monitorTestTime.Add(d.cooldown)) {
|
||||
t.Fatalf("crash cooldown=%v", state.CooldownUntil)
|
||||
}
|
||||
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err == nil {
|
||||
t.Fatal("old owner token accepted")
|
||||
}
|
||||
} else {
|
||||
var stored string
|
||||
if err := d.db.QueryRow(`SELECT recovery_token FROM monitor_state WHERE service_id = ?`, d.serviceID).Scan(&stored); err != nil || stored != token {
|
||||
t.Fatalf("lease changed: %q %v", stored, err)
|
||||
}
|
||||
if !state.CooldownUntil.IsZero() {
|
||||
t.Fatal("retained owner cooldown changed")
|
||||
}
|
||||
}
|
||||
assertMonitorUnlocked(t, d)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorUnverifiableOwnerUnderSingleton(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime)
|
||||
if err != nil || token == "" {
|
||||
t.Fatalf("token=%q err=%v", token, err)
|
||||
}
|
||||
s, err := acquireManagedMonitorSingleton(d.target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.close()
|
||||
reader := func(int) (int64, error) { return 0, os.ErrPermission }
|
||||
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, reader, d.now); !errors.Is(err, os.ErrPermission) {
|
||||
t.Fatalf("unverifiable owner: %v", err)
|
||||
}
|
||||
state, err := readMonitorState(d.db, d.serviceID)
|
||||
if err != nil || !state.RecoveryRunning || !state.CooldownUntil.IsZero() {
|
||||
t.Fatalf("state=%+v err=%v", state, err)
|
||||
}
|
||||
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertMonitorUnlocked(t, d)
|
||||
}
|
||||
|
||||
func monitorRuntimeFixture(t *testing.T) managedMonitorDeps {
|
||||
t.Helper()
|
||||
target, _, _ := managedCLIFixture(t)
|
||||
if err := os.Chmod(target.directory, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db, err := openStore(filepath.Join(t.TempDir(), "monitor.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return managedMonitorDeps{
|
||||
target: target, db: db, serviceID: "managed-youtube",
|
||||
interval: 5 * time.Minute, threshold: 3, cooldown: 30 * time.Minute,
|
||||
now: func() time.Time { return monitorTestTime },
|
||||
probe: func(context.Context) error { return nil },
|
||||
recover: func(context.Context) error { t.Fatal("unexpected recovery"); return nil },
|
||||
wait: waitManagedMonitor,
|
||||
reportRecoveryError: func(error) { t.Fatal("unexpected recovery error") },
|
||||
}
|
||||
}
|
||||
|
||||
func assertMonitorUnlocked(t *testing.T, d managedMonitorDeps) {
|
||||
t.Helper()
|
||||
s, err := acquireManagedMonitorSingleton(d.target)
|
||||
if err != nil {
|
||||
t.Fatalf("singleton leaked: %v", err)
|
||||
}
|
||||
if err := s.close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorImmediatePeriodicAndReset(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
now := monitorTestTime
|
||||
d.now = func() time.Time { return now }
|
||||
observations := []bool{false, false, true, false, false, false}
|
||||
wantStreak := []int{1, 2, 0, 1, 2, 3}
|
||||
probes, waits, recoveries := 0, 0, 0
|
||||
d.probe = func(context.Context) error {
|
||||
if probes != waits || !now.Equal(monitorTestTime.Add(time.Duration(probes)*d.interval)) {
|
||||
t.Fatal("probe not immediate/periodic")
|
||||
}
|
||||
healthy := observations[probes]
|
||||
probes++
|
||||
if healthy {
|
||||
return nil
|
||||
}
|
||||
return errMonitorTestUnhealthy
|
||||
}
|
||||
d.recover = func(context.Context) error {
|
||||
recoveries++
|
||||
if probes != 6 {
|
||||
t.Fatalf("recovery at probe %d", probes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
d.wait = func(ctx context.Context, interval time.Duration) error {
|
||||
if interval != d.interval {
|
||||
t.Fatal("wrong interval")
|
||||
}
|
||||
state, err := readMonitorState(d.db, d.serviceID)
|
||||
if err != nil || state.ConsecutiveFailures != wantStreak[probes-1] || !state.LastProbeAt.Equal(now) {
|
||||
t.Fatalf("state=%+v err=%v", state, err)
|
||||
}
|
||||
waits++
|
||||
if probes == len(observations) {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
now = now.Add(interval)
|
||||
return nil
|
||||
}
|
||||
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if probes != 6 || recoveries != 1 {
|
||||
t.Fatalf("probes=%d recoveries=%d", probes, recoveries)
|
||||
}
|
||||
assertMonitorUnlocked(t, d)
|
||||
}
|
||||
|
||||
func TestManagedMonitorCooldownFromCompletionEitherOutcome(t *testing.T) {
|
||||
for _, failed := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "success", true: "failure"}[failed], func(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
now := monitorTestTime
|
||||
d.now = func() time.Time { return now }
|
||||
probes, recoveries, reports := 0, 0, 0
|
||||
d.probe = func(context.Context) error { probes++; return errMonitorTestUnhealthy }
|
||||
d.recover = func(context.Context) error {
|
||||
recoveries++
|
||||
want := monitorTestTime.Add(10 * time.Minute)
|
||||
if recoveries == 2 {
|
||||
want = monitorTestTime.Add(50 * time.Minute)
|
||||
}
|
||||
if recoveries > 2 || !now.Equal(want) {
|
||||
t.Fatalf("recovery %d at %v", recoveries, now)
|
||||
}
|
||||
if token := requireMonitorBegin(t, d.db, d.serviceID, now, false); token != "" {
|
||||
t.Fatal("double start")
|
||||
}
|
||||
now = now.Add(10 * time.Minute)
|
||||
if failed {
|
||||
return errMonitorTestUnhealthy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
d.reportRecoveryError = func(err error) {
|
||||
if !errors.Is(err, errMonitorTestUnhealthy) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reports++
|
||||
}
|
||||
d.wait = func(ctx context.Context, interval time.Duration) error {
|
||||
state, err := readMonitorState(d.db, d.serviceID)
|
||||
if err != nil || state.RecoveryRunning {
|
||||
t.Fatalf("state=%+v err=%v", state, err)
|
||||
}
|
||||
if recoveries > 0 {
|
||||
want := monitorTestTime.Add(50 * time.Minute)
|
||||
if recoveries == 2 {
|
||||
want = monitorTestTime.Add(90 * time.Minute)
|
||||
}
|
||||
if !state.CooldownUntil.Equal(want) {
|
||||
t.Fatalf("cooldown=%v want=%v", state.CooldownUntil, want)
|
||||
}
|
||||
}
|
||||
if probes == 9 {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
now = now.Add(interval)
|
||||
return nil
|
||||
}
|
||||
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recoveries != 2 || (failed && reports != 2) || (!failed && reports != 0) {
|
||||
t.Fatalf("recoveries=%d reports=%d", recoveries, reports)
|
||||
}
|
||||
assertMonitorUnlocked(t, d)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorStartupLeaseAfterReopenStartsCrashCooldown(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
path := filepath.Join(t.TempDir(), "monitor.db")
|
||||
db, err := openStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := recordMonitorProbe(db, d.serviceID, false, monitorTestTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// An impossible Linux PID proves a dead owner without spawning a child.
|
||||
oldToken, err := beginMonitorRecoveryWithProcess(db, d.serviceID, 3, monitorTestTime, 1<<30, func(int) (int64, error) { return 100, nil })
|
||||
if err != nil || oldToken == "" {
|
||||
t.Fatalf("token=%q err=%v", oldToken, err)
|
||||
}
|
||||
expires := monitorTestTime.Add(d.cooldown)
|
||||
if _, err := db.Exec(`UPDATE monitor_state SET cooldown_until = ? WHERE service_id = ?`, monitorTestTime.Add(time.Hour).UnixNano(), d.serviceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.db, err = openStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.db.Close()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
d.probe = func(context.Context) error {
|
||||
state, err := readMonitorState(d.db, d.serviceID)
|
||||
if err != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(expires) || state.ConsecutiveFailures != 3 {
|
||||
t.Fatalf("startup=%+v err=%v", state, err)
|
||||
}
|
||||
if err := finishMonitorRecovery(d.db, d.serviceID, oldToken, monitorTestTime, d.cooldown); err == nil {
|
||||
t.Fatal("stale finish accepted")
|
||||
}
|
||||
return errMonitorTestUnhealthy
|
||||
}
|
||||
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := readMonitorState(d.db, d.serviceID)
|
||||
if err != nil || state.ConsecutiveFailures != 4 || !state.CooldownUntil.Equal(expires) {
|
||||
t.Fatalf("state=%+v err=%v", state, err)
|
||||
}
|
||||
assertMonitorUnlocked(t, d)
|
||||
}
|
||||
|
||||
func TestManagedMonitorConcurrentStartAndCancellation(t *testing.T) {
|
||||
for _, phase := range []string{"probe", "recovery", "wait"} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
entered := make(chan struct{})
|
||||
childExited := make(chan struct{})
|
||||
block := func(ctx context.Context) error {
|
||||
go func() { <-ctx.Done(); close(childExited) }()
|
||||
close(entered)
|
||||
<-childExited // Callback joins its child before the supervisor releases the lock.
|
||||
return ctx.Err()
|
||||
}
|
||||
switch phase {
|
||||
case "probe":
|
||||
d.probe = block
|
||||
case "recovery":
|
||||
d.threshold = 1
|
||||
d.probe = func(context.Context) error { return errMonitorTestUnhealthy }
|
||||
d.recover = block
|
||||
d.reportRecoveryError = func(error) {}
|
||||
case "wait":
|
||||
d.wait = func(ctx context.Context, _ time.Duration) error { return block(ctx) }
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
finished := false
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
if !finished {
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("supervisor leaked during cleanup")
|
||||
}
|
||||
}
|
||||
})
|
||||
go func() { done <- runManagedMonitor(ctx, d) }()
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-done:
|
||||
finished = true
|
||||
t.Fatalf("early exit: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("monitor did not start")
|
||||
}
|
||||
if err := runManagedMonitor(context.Background(), d); err == nil {
|
||||
t.Fatal("parallel monitor accepted")
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
finished = true
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cancellation blocked")
|
||||
}
|
||||
select {
|
||||
case <-childExited:
|
||||
default:
|
||||
t.Fatal("callback child leaked")
|
||||
}
|
||||
state, err := readMonitorState(d.db, d.serviceID)
|
||||
if err != nil || state.RecoveryRunning {
|
||||
t.Fatalf("state=%+v err=%v", state, err)
|
||||
}
|
||||
if phase == "recovery" && !state.CooldownUntil.Equal(monitorTestTime.Add(d.cooldown)) {
|
||||
t.Fatal("cancelled recovery not finalized")
|
||||
}
|
||||
assertMonitorUnlocked(t, d)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorReconciliationCapabilityAndGuards(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, nil, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
|
||||
t.Fatal("missing singleton accepted")
|
||||
}
|
||||
s, err := acquireManagedMonitorSingleton(d.target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.close()
|
||||
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime)
|
||||
if err != nil || token == "" {
|
||||
t.Fatalf("token=%q err=%v", token, err)
|
||||
}
|
||||
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
|
||||
t.Fatal("live lease cleared by repeated startup")
|
||||
}
|
||||
if err := s.close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
|
||||
t.Fatal("closed capability accepted")
|
||||
}
|
||||
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.target.directory = t.TempDir()
|
||||
d.target.lookup = func(string) (string, bool) { return "", false }
|
||||
if _, err := acquireManagedMonitorSingleton(d.target); err == nil {
|
||||
t.Fatal("unguarded singleton accepted")
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(d.target.directory, managedMonitorLockName)); !os.IsNotExist(err) {
|
||||
t.Fatalf("guard touched lock: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorFatalStoreAndFinalizationErrors(t *testing.T) {
|
||||
for _, phase := range []string{"startup", "record", "begin", "finish"} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
d.threshold = 1
|
||||
if phase == "startup" {
|
||||
if err := d.db.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d.probe = func(context.Context) error {
|
||||
if phase == "startup" {
|
||||
t.Fatal("probe ran after startup DB error")
|
||||
}
|
||||
if phase == "begin" {
|
||||
_, err := d.db.Exec(`CREATE TRIGGER reject_monitor_begin BEFORE UPDATE OF recovery_running ON monitor_state WHEN NEW.recovery_running = 1 BEGIN SELECT RAISE(ABORT, 'begin rejected'); END`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if phase == "record" {
|
||||
if err := d.db.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return errMonitorTestUnhealthy
|
||||
}
|
||||
d.recover = func(context.Context) error {
|
||||
if phase != "finish" {
|
||||
t.Fatal("recovery ran after fatal DB error")
|
||||
}
|
||||
_, err := d.db.Exec(`CREATE TRIGGER reject_monitor_finish BEFORE UPDATE OF recovery_running ON monitor_state WHEN NEW.recovery_running = 0 BEGIN SELECT RAISE(ABORT, 'finish rejected'); END`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
d.wait = func(context.Context, time.Duration) error { t.Fatal("continued after fatal DB error"); return nil }
|
||||
if err := runManagedMonitor(context.Background(), d); err == nil {
|
||||
t.Fatal("DB error ignored")
|
||||
}
|
||||
assertMonitorUnlocked(t, d)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,9 @@ package main
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -297,7 +299,7 @@ func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
var version, count int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
||||
t.Fatalf("version=%d err=%v", version, err)
|
||||
}
|
||||
if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 {
|
||||
@@ -308,6 +310,160 @@ func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorCrashReconciliation(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
start int64
|
||||
err error
|
||||
clear, wantErr bool
|
||||
}{
|
||||
{"live", 100, nil, false, false},
|
||||
{"missing", 0, os.ErrNotExist, true, false},
|
||||
{"reused", 101, nil, true, false},
|
||||
{"unreadable", 0, os.ErrPermission, false, true},
|
||||
{"invalid", 0, nil, false, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "crash.db")
|
||||
db, err := openStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
token, err := beginMonitorRecoveryWithProcess(db, "service", 3, monitorTestTime, 42, func(pid int) (int64, error) {
|
||||
if pid != 42 {
|
||||
t.Fatalf("pid=%d", pid)
|
||||
}
|
||||
return 100, nil
|
||||
})
|
||||
if err != nil || token == "" {
|
||||
t.Fatalf("acquire: %q %v", token, err)
|
||||
}
|
||||
var pid, start int64
|
||||
if err := db.QueryRow(`SELECT recovery_owner_pid, recovery_owner_starttime FROM monitor_state WHERE service_id = 'service'`).Scan(&pid, &start); err != nil || pid != 42 || start != 100 {
|
||||
t.Fatalf("owner=%d/%d err=%v", pid, start, err)
|
||||
}
|
||||
now := monitorTestTime.Add(time.Hour)
|
||||
clear, err := reconcileMonitorRecovery(db, "service", 30*time.Minute, func(pid int) (int64, error) {
|
||||
if pid != 42 {
|
||||
t.Fatalf("pid=%d", pid)
|
||||
}
|
||||
return tc.start, tc.err
|
||||
}, func() time.Time { return now })
|
||||
if clear != tc.clear || (err != nil) != tc.wantErr {
|
||||
t.Fatalf("clear=%v err=%v", clear, err)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db, err = openStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := readMonitorState(db, "service")
|
||||
if err != nil || state.RecoveryRunning == tc.clear {
|
||||
t.Fatalf("state=%+v err=%v", state, err)
|
||||
}
|
||||
if tc.clear {
|
||||
expires := now.Add(30 * time.Minute)
|
||||
if !state.CooldownUntil.Equal(expires) {
|
||||
t.Fatalf("cooldown=%v", state.CooldownUntil)
|
||||
}
|
||||
requireMonitorBegin(t, db, "service", expires.Add(-time.Nanosecond), false)
|
||||
newToken := requireMonitorBegin(t, db, "service", expires, true)
|
||||
if err := finishMonitorRecovery(db, "service", token, expires, time.Hour); err == nil {
|
||||
t.Fatal("stale crash token accepted")
|
||||
}
|
||||
if err := finishMonitorRecovery(db, "service", newToken, expires, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else if !state.CooldownUntil.IsZero() {
|
||||
t.Fatal("retained lease changed cooldown")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorV4MigrationPreservesUnverifiableLease(t *testing.T) {
|
||||
db := testStore(t)
|
||||
if _, err := db.Exec("DROP TABLE monitor_state;" + monitorSchema); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
token := strings.Repeat("a", 64)
|
||||
if _, err := db.Exec(`INSERT INTO monitor_state(service_id, consecutive_failures, last_probe_at, recovery_running, recovery_token, cooldown_until) VALUES ('service', 3, ?, 1, ?, ?)`, monitorTestTime.UnixNano(), token, monitorTestTime.Add(time.Minute).UnixNano()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := readMonitorState(db, "service")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := migrate(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var version int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
||||
t.Fatalf("version=%d err=%v", version, err)
|
||||
}
|
||||
clear, err := reconcileMonitorRecovery(db, "service", time.Minute, func(int) (int64, error) {
|
||||
t.Fatal("legacy owner must not be guessed")
|
||||
return 0, os.ErrNotExist
|
||||
}, func() time.Time { return monitorTestTime.Add(time.Hour) })
|
||||
if clear || err == nil {
|
||||
t.Fatalf("legacy clear=%v err=%v", clear, err)
|
||||
}
|
||||
after, err := readMonitorState(db, "service")
|
||||
if err != nil || after != before {
|
||||
t.Fatalf("migration changed state: %+v -> %+v err=%v", before, after, err)
|
||||
}
|
||||
var storedToken string
|
||||
if err := db.QueryRow("SELECT recovery_token FROM monitor_state WHERE service_id = 'service'").Scan(&storedToken); err != nil || storedToken != token {
|
||||
t.Fatalf("token=%q err=%v", storedToken, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorReconciliationCannotClearNewAcquisition(t *testing.T) {
|
||||
db := testStore(t)
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
token := requireMonitorBegin(t, db, "service", monitorTestTime, true)
|
||||
var newToken string
|
||||
clear, err := reconcileMonitorRecovery(db, "service", time.Hour, func(int) (int64, error) {
|
||||
if err := finishMonitorRecovery(db, "service", token, monitorTestTime, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newToken = requireMonitorBegin(t, db, "service", monitorTestTime.Add(time.Minute), true)
|
||||
return 0, os.ErrNotExist
|
||||
}, func() time.Time { return monitorTestTime.Add(time.Minute) })
|
||||
if err != nil || clear {
|
||||
t.Fatalf("clear=%v err=%v", clear, err)
|
||||
}
|
||||
if err := finishMonitorRecovery(db, "service", newToken, monitorTestTime.Add(time.Minute), time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorProcessStatParser(t *testing.T) {
|
||||
stat := "42 (name with ) parentheses) S" + strings.Repeat(" 0", 18) + " 123 0"
|
||||
if start, err := parseMonitorProcessStat(42, stat); err != nil || start != 123 {
|
||||
t.Fatalf("start=%d err=%v", start, err)
|
||||
}
|
||||
for _, bad := range []string{"", "41 (name) S" + strings.Repeat(" 0", 18) + " 123", "42 (name) S", strings.Replace(stat, "123", "invalid", 1), strings.Replace(stat, "123", "0", 1)} {
|
||||
if _, err := parseMonitorProcessStat(42, bad); err == nil {
|
||||
t.Fatalf("accepted %q", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorV3MigrationRollback(t *testing.T) {
|
||||
db := testStore(t)
|
||||
saveProposal(t, db, "--lua-desync=fake")
|
||||
|
||||
@@ -114,7 +114,7 @@ func migrate(db *sql.DB) error {
|
||||
if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil {
|
||||
return err
|
||||
}
|
||||
if version > 4 {
|
||||
if version > 5 {
|
||||
return fmt.Errorf("unsupported database schema version %d", version)
|
||||
}
|
||||
if version == 0 {
|
||||
@@ -236,6 +236,11 @@ PRAGMA user_version = 1;`)
|
||||
return err
|
||||
}
|
||||
}
|
||||
if version < 5 {
|
||||
if _, err := conn.ExecContext(ctx, monitorOwnerSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = conn.ExecContext(ctx, "COMMIT")
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -224,7 +224,7 @@ func TestConcurrentFirstOpens(t *testing.T) {
|
||||
}
|
||||
for _, db := range stores {
|
||||
var version, objects int
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
||||
t.Fatalf("version=%d err=%v", version, err)
|
||||
}
|
||||
if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE
|
||||
@@ -377,7 +377,7 @@ func TestSchemaConstraintsAndIncompleteEvidence(t *testing.T) {
|
||||
|
||||
func TestUnsupportedSchemaVersion(t *testing.T) {
|
||||
db := testStore(t)
|
||||
if _, err := db.Exec("PRAGMA user_version = 5"); err != nil {
|
||||
if _, err := db.Exec("PRAGMA user_version = 6"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") {
|
||||
|
||||
Reference in new issue
Block a user