280 lines
11 KiB
Go
280 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"crypto/rand"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Policy foundation only; service_id identifies a service, not a rotating host
|
|
// or proxy address. No probe, scheduler or configuration changes happen here.
|
|
const monitorSchema = `
|
|
CREATE TABLE monitor_state (
|
|
service_id TEXT PRIMARY KEY NOT NULL CHECK (typeof(service_id) = 'text' AND length(trim(service_id)) > 0),
|
|
consecutive_failures INTEGER NOT NULL DEFAULT 0 CHECK (typeof(consecutive_failures) = 'integer' AND consecutive_failures >= 0),
|
|
last_probe_at INTEGER CHECK (last_probe_at IS NULL OR typeof(last_probe_at) = 'integer'),
|
|
recovery_running INTEGER NOT NULL DEFAULT 0 CHECK (typeof(recovery_running) = 'integer' AND recovery_running IN (0, 1)),
|
|
recovery_token TEXT CHECK (recovery_token IS NULL OR (typeof(recovery_token) = 'text' AND length(recovery_token) = 64)),
|
|
cooldown_until INTEGER CHECK (cooldown_until IS NULL OR typeof(cooldown_until) = 'integer'),
|
|
CHECK ((recovery_running = 0 AND recovery_token IS NULL) OR (recovery_running = 1 AND recovery_token IS NOT NULL)),
|
|
CHECK (last_probe_at IS NOT NULL OR consecutive_failures = 0)
|
|
);
|
|
PRAGMA user_version = 4;`
|
|
|
|
// Legacy v4 leases have no provable owner: preserve them and fail closed.
|
|
const monitorOwnerSchema = `
|
|
ALTER TABLE monitor_state ADD COLUMN recovery_owner_pid INTEGER
|
|
CHECK (recovery_owner_pid IS NULL OR (typeof(recovery_owner_pid) = 'integer' AND recovery_owner_pid > 0 AND recovery_running = 1));
|
|
ALTER TABLE monitor_state ADD COLUMN recovery_owner_starttime INTEGER
|
|
CHECK ((recovery_owner_pid IS NULL AND recovery_owner_starttime IS NULL) OR
|
|
(recovery_owner_pid IS NOT NULL AND typeof(recovery_owner_starttime) = 'integer' AND recovery_owner_starttime > 0));
|
|
PRAGMA user_version = 5;`
|
|
|
|
// Only os.ErrNotExist proves absence; all other errors are indeterminate.
|
|
type monitorProcessReader func(pid int) (int64, error)
|
|
|
|
func readMonitorProcessStarttime(pid int) (int64, error) {
|
|
if pid <= 0 {
|
|
return 0, fmt.Errorf("invalid recovery owner PID")
|
|
}
|
|
data, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
// A missing stat file alone does not prove a missing process (or
|
|
// even a mounted procfs). Confirm absence and a readable procfs.
|
|
_, dirErr := os.Stat(fmt.Sprintf("/proc/%d", pid))
|
|
self, selfErr := os.ReadFile("/proc/self/stat")
|
|
if selfErr == nil {
|
|
_, selfErr = parseMonitorProcessStat(os.Getpid(), string(self))
|
|
}
|
|
if !errors.Is(dirErr, os.ErrNotExist) || selfErr != nil {
|
|
return 0, fmt.Errorf("cannot establish process absence for PID %d", pid)
|
|
}
|
|
}
|
|
return 0, err
|
|
}
|
|
return parseMonitorProcessStat(pid, string(data))
|
|
}
|
|
|
|
func parseMonitorProcessStat(pid int, stat string) (int64, error) {
|
|
// comm can contain spaces and parentheses. Field 22 is starttime;
|
|
// fields following the final ')' begin with field 3 (state).
|
|
open, close := strings.IndexByte(stat, '('), strings.LastIndexByte(stat, ')')
|
|
if open < 0 || close <= open || strings.TrimSpace(stat[:open]) != strconv.Itoa(pid) {
|
|
return 0, fmt.Errorf("invalid process stat identity")
|
|
}
|
|
fields := strings.Fields(stat[close+1:])
|
|
if len(fields) < 20 || len(fields[0]) != 1 || !strings.Contains("RSDZTtXxKWPI", fields[0]) {
|
|
return 0, fmt.Errorf("invalid process stat fields")
|
|
}
|
|
start, err := strconv.ParseInt(fields[19], 10, 64)
|
|
if err != nil || start <= 0 {
|
|
return 0, fmt.Errorf("invalid process starttime")
|
|
}
|
|
return start, nil
|
|
}
|
|
|
|
type monitorState struct {
|
|
ConsecutiveFailures int
|
|
LastProbeAt time.Time
|
|
RecoveryRunning bool
|
|
CooldownUntil time.Time
|
|
}
|
|
|
|
func (s monitorState) healthStatus() string {
|
|
if s.LastProbeAt.IsZero() {
|
|
return "UNKNOWN"
|
|
}
|
|
if s.ConsecutiveFailures > 0 {
|
|
return "UNHEALTHY"
|
|
}
|
|
return "HEALTHY"
|
|
}
|
|
|
|
func validateMonitorTime(serviceID string, now time.Time) error {
|
|
if strings.TrimSpace(serviceID) == "" || strings.TrimSpace(serviceID) != serviceID || strings.ContainsRune(serviceID, '\x00') {
|
|
return fmt.Errorf("invalid monitor service ID")
|
|
}
|
|
// Reject zero/overflowing timestamps rather than silently wrapping UnixNano.
|
|
if now.IsZero() || !time.Unix(0, now.UnixNano()).Equal(now) {
|
|
return fmt.Errorf("invalid monitor timestamp")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func recordMonitorProbe(db *sql.DB, serviceID string, success bool, now time.Time) error {
|
|
if err := validateMonitorTime(serviceID, now); err != nil {
|
|
return err
|
|
}
|
|
failures := 1
|
|
if success {
|
|
failures = 0
|
|
}
|
|
// One UPSERT serializes increments and leaves the recovery lock/cooldown intact.
|
|
_, err := db.Exec(`INSERT INTO monitor_state (service_id, consecutive_failures, last_probe_at)
|
|
VALUES (?, ?, ?) ON CONFLICT(service_id) DO UPDATE SET
|
|
consecutive_failures = CASE WHEN excluded.consecutive_failures = 0 THEN 0 ELSE monitor_state.consecutive_failures + 1 END,
|
|
last_probe_at = excluded.last_probe_at`, serviceID, failures, now.UnixNano())
|
|
return err
|
|
}
|
|
|
|
func readMonitorState(db *sql.DB, serviceID string) (monitorState, error) {
|
|
var state monitorState
|
|
var probe, cooldown sql.NullInt64
|
|
err := db.QueryRow(`SELECT consecutive_failures, last_probe_at, recovery_running, cooldown_until
|
|
FROM monitor_state WHERE service_id = ?`, serviceID).Scan(&state.ConsecutiveFailures, &probe, &state.RecoveryRunning, &cooldown)
|
|
if err == sql.ErrNoRows {
|
|
return state, nil // A service without recorded probes has unknown health.
|
|
}
|
|
if probe.Valid {
|
|
state.LastProbeAt = time.Unix(0, probe.Int64).UTC()
|
|
}
|
|
if cooldown.Valid {
|
|
state.CooldownUntil = time.Unix(0, cooldown.Int64).UTC()
|
|
}
|
|
return state, err
|
|
}
|
|
|
|
// A nonempty token identifies this acquisition only; an empty token means busy
|
|
// or ineligible. The caller must retain it for finalization, including after reopen.
|
|
func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.Time) (string, error) {
|
|
return beginMonitorRecoveryWithProcess(db, serviceID, threshold, now, os.Getpid(), readMonitorProcessStarttime)
|
|
}
|
|
|
|
func beginMonitorRecoveryWithProcess(db *sql.DB, serviceID string, threshold int, now time.Time, pid int, read monitorProcessReader) (string, error) {
|
|
if err := validateMonitorTime(serviceID, now); err != nil {
|
|
return "", err
|
|
}
|
|
if threshold <= 0 {
|
|
return "", fmt.Errorf("failure threshold must be positive")
|
|
}
|
|
if pid <= 0 || read == nil {
|
|
return "", fmt.Errorf("invalid recovery process reader/owner")
|
|
}
|
|
start, err := read(pid)
|
|
if err != nil {
|
|
return "", fmt.Errorf("read recovery owner: %w", err)
|
|
}
|
|
if start <= 0 {
|
|
return "", fmt.Errorf("invalid recovery owner starttime")
|
|
}
|
|
var ownership [32]byte
|
|
if _, err := rand.Read(ownership[:]); err != nil {
|
|
return "", err
|
|
}
|
|
token := hex.EncodeToString(ownership[:])
|
|
// Compare-and-set in one SQLite write, including across independent processes.
|
|
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 1, recovery_token = ?, recovery_owner_pid = ?, recovery_owner_starttime = ?
|
|
WHERE service_id = ? AND consecutive_failures >= ? AND recovery_running = 0
|
|
AND (cooldown_until IS NULL OR cooldown_until <= ?)`, token, pid, start, serviceID, threshold, now.UnixNano())
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
count, err := result.RowsAffected()
|
|
if err != nil || count != 1 {
|
|
return "", err
|
|
}
|
|
return token, nil
|
|
}
|
|
|
|
// Call on either success or failure. Only this acquisition's token can finish.
|
|
func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.Time, cooldown time.Duration) error {
|
|
if err := validateMonitorTime(serviceID, completed); err != nil {
|
|
return err
|
|
}
|
|
if cooldown <= 0 {
|
|
return fmt.Errorf("recovery cooldown must be positive")
|
|
}
|
|
if len(token) != 64 {
|
|
return fmt.Errorf("invalid recovery ownership token")
|
|
}
|
|
expires := completed.Add(cooldown)
|
|
if err := validateMonitorTime(serviceID, expires); err != nil {
|
|
return err
|
|
}
|
|
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = MAX(COALESCE(cooldown_until, ?), ?)
|
|
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), expires.UnixNano(), serviceID, token)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
count, err := result.RowsAffected()
|
|
if err == nil && count != 1 {
|
|
return fmt.Errorf("no recovery owned by this token for service %q", serviceID)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Extend bootstrap backoff without changing probes or any recovery ownership.
|
|
func pauseMonitorRecovery(db *sql.DB, serviceID string, now time.Time, cooldown time.Duration) error {
|
|
if err := validateMonitorTime(serviceID, now); err != nil {
|
|
return err
|
|
}
|
|
if cooldown <= 0 {
|
|
return fmt.Errorf("recovery cooldown must be positive")
|
|
}
|
|
expires := now.Add(cooldown)
|
|
if err := validateMonitorTime(serviceID, expires); err != nil {
|
|
return err
|
|
}
|
|
_, err := db.Exec(`INSERT INTO monitor_state (service_id, cooldown_until) VALUES (?, ?)
|
|
ON CONFLICT(service_id) DO UPDATE SET cooldown_until = MAX(COALESCE(monitor_state.cooldown_until, excluded.cooldown_until), excluded.cooldown_until)`, serviceID, expires.UnixNano())
|
|
return err
|
|
}
|
|
|
|
// Startup policy only: call for each configured service before monitoring.
|
|
// A legacy/indeterminate owner is an error, never permission to unlock.
|
|
// The conditional write also protects against a finish/reacquire during the read.
|
|
func reconcileMonitorRecovery(db *sql.DB, serviceID string, cooldown time.Duration, read monitorProcessReader, clock func() time.Time) (bool, error) {
|
|
if read == nil || clock == nil || cooldown <= 0 {
|
|
return false, fmt.Errorf("invalid reconciliation dependencies/cooldown")
|
|
}
|
|
now := clock()
|
|
if err := validateMonitorTime(serviceID, now); err != nil {
|
|
return false, err
|
|
}
|
|
expires := now.Add(cooldown)
|
|
if err := validateMonitorTime(serviceID, expires); err != nil {
|
|
return false, err
|
|
}
|
|
var token string
|
|
var pid, start sql.NullInt64
|
|
err := db.QueryRow(`SELECT recovery_token, recovery_owner_pid, recovery_owner_starttime
|
|
FROM monitor_state WHERE service_id = ? AND recovery_running = 1`, serviceID).Scan(&token, &pid, &start)
|
|
if err == sql.ErrNoRows {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !pid.Valid || !start.Valid || pid.Int64 <= 0 || int64(int(pid.Int64)) != pid.Int64 || start.Int64 <= 0 || len(token) != 64 {
|
|
return false, fmt.Errorf("unverifiable recovery owner for service %q", serviceID)
|
|
}
|
|
current, err := read(int(pid.Int64))
|
|
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return false, fmt.Errorf("read recovery owner: %w", err)
|
|
}
|
|
if err == nil {
|
|
if current <= 0 {
|
|
return false, fmt.Errorf("invalid recovery owner starttime")
|
|
}
|
|
if current == start.Int64 {
|
|
return false, nil
|
|
}
|
|
}
|
|
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL,
|
|
recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = MAX(COALESCE(cooldown_until, ?), ?)
|
|
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?
|
|
AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), expires.UnixNano(), serviceID, token, pid.Int64, start.Int64)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
count, err := result.RowsAffected()
|
|
return count == 1, err
|
|
}
|