Add checks-only scheduling, read-only status, private authenticated persistence and guarded PID1 lifecycle. Keep strategy selection/recovery and remote fault acceptance out of scope. Co-authored-by: GPT-6 <gpt-6@openai.com>
442 lines
13 KiB
Go
442 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
// HEALTHY means only that the configured checks passed, not production READY.
|
|
type resourceMonitorSource struct {
|
|
config resourceCheckConfig
|
|
server, proxy, fingerprint string
|
|
}
|
|
|
|
func resourceStaticConfig(lookup func(string) (string, bool)) (string, string, error) {
|
|
dns, err := parseManagedDNSConfig(lookup)
|
|
raw, _ := lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
|
|
proxy, proxyErr := managedProbeProxy(raw)
|
|
if err != nil || proxyErr != nil || dns.Mode == "native" && dns.NativeServer == "" {
|
|
return "", "", errResourceConfig
|
|
}
|
|
server := dns.serverAddress()
|
|
if dns.Mode == "native" {
|
|
server = net.JoinHostPort(server, "53")
|
|
}
|
|
return server, proxy, nil
|
|
}
|
|
func readResourceMonitorSource(path string, lookup func(string) (string, bool), key []byte) (resourceMonitorSource, error) {
|
|
// Resolve each referenced environment value once in this immutable snapshot.
|
|
values := map[string]struct {
|
|
value string
|
|
ok bool
|
|
}{}
|
|
snapshot := func(name string) (string, bool) {
|
|
if v, ok := values[name]; ok {
|
|
return v.value, v.ok
|
|
}
|
|
v, ok := lookup(name)
|
|
if len(v) > 16384 {
|
|
return "", false
|
|
}
|
|
values[name] = struct {
|
|
value string
|
|
ok bool
|
|
}{v, ok}
|
|
return v, ok
|
|
}
|
|
cfg, err := readResourceChecksMode(path, snapshot, true)
|
|
server, proxy, staticErr := resourceStaticConfig(snapshot)
|
|
if err != nil || staticErr != nil {
|
|
return resourceMonitorSource{}, errResourceConfig
|
|
}
|
|
source := resourceMonitorSource{config: cfg, server: server, proxy: proxy}
|
|
b, _ := json.Marshal(cfg)
|
|
// Marshal omits the private curl headers; bind them explicitly as well.
|
|
parts := []string{string(b), server, proxy}
|
|
for _, s := range cfg.Services {
|
|
for _, r := range s.Resources {
|
|
parts = append(parts, r.privateConfig)
|
|
}
|
|
}
|
|
b, _ = json.Marshal(parts)
|
|
source.fingerprint = resourceMAC(key, b)
|
|
return source, nil
|
|
}
|
|
func resourceRowFingerprint(key []byte, source resourceMonitorSource, r resourceCheckSpec) string {
|
|
b, _ := json.Marshal(r)
|
|
b, _ = json.Marshal([]string{string(b), r.privateConfig, source.server, source.proxy})
|
|
return resourceMAC(key, b)
|
|
}
|
|
func reconcileResourceRows(key []byte, source resourceMonitorSource, prior []resourceMonitorRow) []resourceMonitorRow {
|
|
old := map[string]resourceMonitorRow{}
|
|
for _, r := range prior {
|
|
old[r.Service+"/"+r.Resource] = r
|
|
}
|
|
rows := []resourceMonitorRow{}
|
|
for _, s := range source.config.Services {
|
|
for _, r := range s.Resources {
|
|
fp := resourceRowFingerprint(key, source, r)
|
|
row, ok := old[s.ID+"/"+r.ID]
|
|
if !ok || row.Fingerprint != fp {
|
|
row = resourceMonitorRow{Service: s.ID, Resource: r.ID, Fingerprint: fp, Health: "UNKNOWN", Outcome: "unknown"}
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
}
|
|
return rows
|
|
}
|
|
|
|
type resourceMonitorDeps struct {
|
|
load func() (resourceMonitorState, error)
|
|
save func(resourceMonitorState) error
|
|
source func() (resourceMonitorSource, error)
|
|
guard func() error
|
|
probe func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error)
|
|
key []byte
|
|
now func() time.Time
|
|
wait func(context.Context, time.Duration) error
|
|
report func(bool)
|
|
}
|
|
|
|
// Sequential round robin with independent due times; reload at least every
|
|
// second while idle/blocked. No recovery leases or strategy mutations.
|
|
func runResourceMonitor(ctx context.Context, d resourceMonitorDeps) error {
|
|
cursor := 0
|
|
persistenceBlocked, failedMAC, attemptedMAC := false, "", ""
|
|
var shutdownError error
|
|
for ctx.Err() == nil {
|
|
state, stateErr := d.load()
|
|
if persistenceBlocked && stateErr == nil && state.MAC != failedMAC && state.MAC != attemptedMAC {
|
|
persistenceBlocked = false
|
|
}
|
|
source, sourceErr := d.source()
|
|
blocked := persistenceBlocked || stateErr != nil || sourceErr != nil || d.guard() != nil
|
|
dirty := false
|
|
if !blocked {
|
|
rows := reconcileResourceRows(d.key, source, state.Rows)
|
|
dirty = !reflect.DeepEqual(rows, state.Rows)
|
|
state.Rows = rows
|
|
if len(state.Rows) > 0 {
|
|
cursor %= len(state.Rows)
|
|
for n := 0; n < len(state.Rows); n++ {
|
|
i := (cursor + n) % len(state.Rows)
|
|
row := &state.Rows[i]
|
|
if row.NextDue > d.now().UnixMilli() {
|
|
continue
|
|
}
|
|
var spec resourceCheckSpec
|
|
for _, s := range source.config.Services {
|
|
if s.ID == row.Service {
|
|
for _, r := range s.Resources {
|
|
if r.ID == row.Resource {
|
|
spec = r
|
|
}
|
|
}
|
|
}
|
|
}
|
|
guard := func() error {
|
|
if d.guard() != nil {
|
|
return errResourceConfig
|
|
}
|
|
fresh, err := d.source()
|
|
if err != nil || fresh.fingerprint != source.fingerprint {
|
|
return errResourceConfig
|
|
}
|
|
return nil
|
|
}
|
|
outcome, err := d.probe(ctx, row.Service, spec, source, guard)
|
|
cursor = (i + 1) % len(state.Rows)
|
|
if err != nil {
|
|
if ctx.Err() == nil || !managedCancellationOnly(err, ctx.Err()) {
|
|
blocked = true
|
|
if ctx.Err() != nil {
|
|
shutdownError = err
|
|
}
|
|
}
|
|
break
|
|
}
|
|
// Cancellation joins the probe but is not completed evidence.
|
|
// Keep the previous health, failure count and due time intact.
|
|
if ctx.Err() != nil || outcome == "cancelled" {
|
|
break
|
|
}
|
|
dirty = true
|
|
row.Outcome, row.LastCompleted = outcome, d.now().UnixMilli()
|
|
delay := *spec.CooldownMS
|
|
if outcome == "pass" {
|
|
row.Health, row.Failures, delay = "HEALTHY", 0, *spec.IntervalMS
|
|
} else {
|
|
row.Health = "DEGRADED"
|
|
if row.Failures < ^uint64(0) {
|
|
row.Failures++
|
|
}
|
|
}
|
|
row.NextDue = row.LastCompleted + int64(delay)
|
|
break // persist each completed (and joined) check before another probe
|
|
}
|
|
}
|
|
}
|
|
dirty = dirty || state.Blocked != blocked
|
|
state.Blocked = blocked
|
|
if dirty {
|
|
state.Updated = d.now().UnixMilli()
|
|
}
|
|
if dirty && stateErr == nil && !persistenceBlocked {
|
|
if d.save(state) != nil {
|
|
blocked, persistenceBlocked, failedMAC, attemptedMAC = true, true, state.MAC, resourceStateMAC(d.key, state)
|
|
}
|
|
}
|
|
d.report(blocked)
|
|
if ctx.Err() != nil {
|
|
if persistenceBlocked {
|
|
return errors.Join(shutdownError, errResourceState)
|
|
}
|
|
return shutdownError
|
|
}
|
|
// Bound reloads, but don't delay an already due sibling by a cooldown.
|
|
delay := time.Second
|
|
if !blocked {
|
|
for _, r := range state.Rows {
|
|
milliseconds := r.NextDue - d.now().UnixMilli()
|
|
if milliseconds > 1000 {
|
|
milliseconds = 1000
|
|
}
|
|
remaining := time.Duration(milliseconds) * time.Millisecond
|
|
if remaining < 0 {
|
|
remaining = 0
|
|
}
|
|
if remaining < delay {
|
|
delay = remaining
|
|
}
|
|
}
|
|
}
|
|
if err := d.wait(ctx, delay); err != nil {
|
|
if ctx.Err() != nil {
|
|
if persistenceBlocked {
|
|
return errResourceState
|
|
}
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
if persistenceBlocked {
|
|
return errResourceState
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func probeResourceMonitor(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error, check resourceCheckDeps) (string, error) {
|
|
check.guard = guard
|
|
protocols := []string{spec.TLS}
|
|
if spec.TLS == "both" {
|
|
protocols = []string{"tls12", "tls13"}
|
|
}
|
|
outcome := "pass"
|
|
for _, protocol := range protocols {
|
|
if ctx.Err() != nil {
|
|
return "cancelled", nil
|
|
}
|
|
result, err := runResourceProtocol(ctx, service, spec, protocol, source.server, source.proxy, check)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if result.Outcome != "pass" {
|
|
outcome = result.Outcome
|
|
}
|
|
}
|
|
return outcome, nil
|
|
}
|
|
|
|
func resourceMonitorCLI(ctx context.Context, path string, stdout, stderr io.Writer, target managedDeps) (code int) {
|
|
return resourceMonitorCLIWithCheck(ctx, path, stdout, stderr, target, productionResourceCheckDeps())
|
|
}
|
|
|
|
func resourceMonitorCLIWithCheck(ctx context.Context, path string, stdout, stderr io.Writer, target managedDeps, check resourceCheckDeps) (code int) {
|
|
port, settingsErr := managedTargetSettings(target.lookup)
|
|
if settingsErr != nil || port != 1080 {
|
|
fmt.Fprintln(stderr, "configerror: invalid generic target settings")
|
|
return 64
|
|
}
|
|
if _, _, err := resourceStaticConfig(target.lookup); err != nil {
|
|
fmt.Fprintln(stderr, "configerror: invalid generic DNS/proxy configuration")
|
|
return 64
|
|
}
|
|
var lock *managedMonitorSingleton
|
|
var store *resourceMonitorStore
|
|
defer func() {
|
|
if store != nil && store.close() != nil {
|
|
code = 78
|
|
}
|
|
if lock != nil && lock.close() != nil {
|
|
code = 78
|
|
}
|
|
}()
|
|
lastBlocked := false
|
|
report := func(blocked bool) {
|
|
if blocked && !lastBlocked {
|
|
fmt.Fprintln(stderr, "BLOCKED: generic resource monitor guard/state unavailable")
|
|
}
|
|
lastBlocked = blocked
|
|
}
|
|
fmt.Fprintln(stdout, "generic resource monitor: configured checks only; no recovery/apply; HEALTHY is not production READY")
|
|
// Guard failures before store/lock acquisition remain cancellable and alive.
|
|
for ctx.Err() == nil {
|
|
if lock == nil {
|
|
var err error
|
|
lock, err = acquireManagedMonitorSingleton(target)
|
|
if err != nil {
|
|
if errors.Is(err, syscall.EWOULDBLOCK) {
|
|
fmt.Fprintln(stderr, "BLOCKED: monitor singleton already held")
|
|
return 78
|
|
}
|
|
report(true)
|
|
if waitManagedMonitor(ctx, time.Second) != nil {
|
|
return 0
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
if store == nil {
|
|
var err error
|
|
store, err = openResourceMonitorStore(target.directory, true)
|
|
if err != nil {
|
|
report(true)
|
|
if waitManagedMonitor(ctx, time.Second) != nil {
|
|
return 0
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
err := runResourceMonitor(ctx, resourceMonitorDeps{
|
|
load: store.load, save: store.save, key: store.key,
|
|
source: func() (resourceMonitorSource, error) {
|
|
return readResourceMonitorSource(path, target.lookup, store.key)
|
|
},
|
|
guard: func() error {
|
|
if resourceSingletonGuard(lock, target.directory) != nil {
|
|
return errResourceState
|
|
}
|
|
// Probe guards validate the held snapshot, never reload/adopt a new
|
|
// authenticated operator edit while an older check is in flight.
|
|
if store.snapshotGuard() != nil {
|
|
return errResourceState
|
|
}
|
|
if resourceManagedIdentity(target.directory) != nil {
|
|
return errResourceState
|
|
}
|
|
return check.guard()
|
|
},
|
|
probe: func(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error) (string, error) {
|
|
return probeResourceMonitor(ctx, service, spec, source, guard, check)
|
|
},
|
|
now: time.Now, wait: waitManagedMonitor, report: report,
|
|
})
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "BLOCKED: generic monitor safety/finalization failure")
|
|
return 78
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func resourceSingletonGuard(lock *managedMonitorSingleton, dir string) error {
|
|
path := filepath.Join(dir, managedMonitorLockName)
|
|
if managedPrivateFile(path) != nil {
|
|
return errResourceState
|
|
}
|
|
current, err := os.Lstat(path)
|
|
held, heldErr := lock.file.Stat()
|
|
if err != nil || heldErr != nil || !os.SameFile(current, held) {
|
|
return errResourceState
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func resourceStatusCLI(stdout io.Writer, target managedDeps) int {
|
|
blocked := func() int { fmt.Fprintln(stdout, `{"health":"BLOCKED","stale":true}`); return 1 }
|
|
store, err := openResourceMonitorStore(target.directory, false)
|
|
if err != nil {
|
|
return blocked()
|
|
}
|
|
defer store.close()
|
|
state, err := store.load()
|
|
if err != nil {
|
|
return blocked()
|
|
}
|
|
path, _ := target.lookup("ZAPRET_AUTOTUNE_RESOURCE_CONFIG")
|
|
source, err := readResourceMonitorSource(path, target.lookup, store.key)
|
|
check := productionResourceCheckDeps()
|
|
if err != nil || check.guard() != nil || resourceManagedIdentity(target.directory) != nil {
|
|
return blocked()
|
|
}
|
|
rows := reconcileResourceRows(store.key, source, state.Rows)
|
|
b1, _ := json.Marshal(rows)
|
|
b2, _ := json.Marshal(state.Rows)
|
|
stale := resourceSnapshotStale(state, time.Now().UnixMilli()) || string(b1) != string(b2)
|
|
health := "CHECKS_ONLY"
|
|
if stale {
|
|
health = "STALE"
|
|
}
|
|
if state.Blocked {
|
|
health = "BLOCKED"
|
|
}
|
|
if json.NewEncoder(stdout).Encode(struct {
|
|
Health string `json:"health"`
|
|
Stale bool `json:"stale"`
|
|
State resourceMonitorState `json:"state"`
|
|
}{health, stale, state}) != nil {
|
|
return 1
|
|
}
|
|
if stale || state.Blocked {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// Updated is a semantic-change timestamp, not a disk heartbeat. Evidence is
|
|
// stale only once its next scheduled check plus the maximum run budget is past.
|
|
func resourceSnapshotStale(state resourceMonitorState, now int64) bool {
|
|
if state.Updated > now {
|
|
return true
|
|
}
|
|
if len(state.Rows) == 0 {
|
|
return now-state.Updated > 65000
|
|
}
|
|
for _, row := range state.Rows {
|
|
if row.NextDue == 0 {
|
|
if now-state.Updated > 65000 {
|
|
return true
|
|
}
|
|
} else if now > row.NextDue && now-row.NextDue > 65000 {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func resourceManagedIdentity(dir string) error {
|
|
m := &managedTransaction{Dir: dir, Name: "config"}
|
|
if managedCheckIdentity(m) != nil {
|
|
return errResourceState
|
|
}
|
|
j, err := m.load()
|
|
if err != nil || j.State != "committed" {
|
|
return errResourceState
|
|
}
|
|
b, err := m.read("config")
|
|
if err != nil || managedHash(b) != j.Hash {
|
|
return errResourceState
|
|
}
|
|
return nil
|
|
}
|