Files
zapret-autotune/cmd/zapret-autotune/resource_monitor.go
T
ZeusinaandGPT-6 Bot 4efc066625 feat: add guarded periodic resource monitoring
Add checks-only scheduling, read-only status, private authenticated persistence and guarded PID1 lifecycle. Keep strategy selection/recovery and remote fault acceptance out of scope.

Co-authored-by: GPT-6 <gpt-6@openai.com>
2026-10-13 23:33:31 +03:00

442 lines
13 KiB
Go

package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"os"
"path/filepath"
"reflect"
"syscall"
"time"
)
// HEALTHY means only that the configured checks passed, not production READY.
type resourceMonitorSource struct {
config resourceCheckConfig
server, proxy, fingerprint string
}
func resourceStaticConfig(lookup func(string) (string, bool)) (string, string, error) {
dns, err := parseManagedDNSConfig(lookup)
raw, _ := lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
proxy, proxyErr := managedProbeProxy(raw)
if err != nil || proxyErr != nil || dns.Mode == "native" && dns.NativeServer == "" {
return "", "", errResourceConfig
}
server := dns.serverAddress()
if dns.Mode == "native" {
server = net.JoinHostPort(server, "53")
}
return server, proxy, nil
}
func readResourceMonitorSource(path string, lookup func(string) (string, bool), key []byte) (resourceMonitorSource, error) {
// Resolve each referenced environment value once in this immutable snapshot.
values := map[string]struct {
value string
ok bool
}{}
snapshot := func(name string) (string, bool) {
if v, ok := values[name]; ok {
return v.value, v.ok
}
v, ok := lookup(name)
if len(v) > 16384 {
return "", false
}
values[name] = struct {
value string
ok bool
}{v, ok}
return v, ok
}
cfg, err := readResourceChecksMode(path, snapshot, true)
server, proxy, staticErr := resourceStaticConfig(snapshot)
if err != nil || staticErr != nil {
return resourceMonitorSource{}, errResourceConfig
}
source := resourceMonitorSource{config: cfg, server: server, proxy: proxy}
b, _ := json.Marshal(cfg)
// Marshal omits the private curl headers; bind them explicitly as well.
parts := []string{string(b), server, proxy}
for _, s := range cfg.Services {
for _, r := range s.Resources {
parts = append(parts, r.privateConfig)
}
}
b, _ = json.Marshal(parts)
source.fingerprint = resourceMAC(key, b)
return source, nil
}
func resourceRowFingerprint(key []byte, source resourceMonitorSource, r resourceCheckSpec) string {
b, _ := json.Marshal(r)
b, _ = json.Marshal([]string{string(b), r.privateConfig, source.server, source.proxy})
return resourceMAC(key, b)
}
func reconcileResourceRows(key []byte, source resourceMonitorSource, prior []resourceMonitorRow) []resourceMonitorRow {
old := map[string]resourceMonitorRow{}
for _, r := range prior {
old[r.Service+"/"+r.Resource] = r
}
rows := []resourceMonitorRow{}
for _, s := range source.config.Services {
for _, r := range s.Resources {
fp := resourceRowFingerprint(key, source, r)
row, ok := old[s.ID+"/"+r.ID]
if !ok || row.Fingerprint != fp {
row = resourceMonitorRow{Service: s.ID, Resource: r.ID, Fingerprint: fp, Health: "UNKNOWN", Outcome: "unknown"}
}
rows = append(rows, row)
}
}
return rows
}
type resourceMonitorDeps struct {
load func() (resourceMonitorState, error)
save func(resourceMonitorState) error
source func() (resourceMonitorSource, error)
guard func() error
probe func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error)
key []byte
now func() time.Time
wait func(context.Context, time.Duration) error
report func(bool)
}
// Sequential round robin with independent due times; reload at least every
// second while idle/blocked. No recovery leases or strategy mutations.
func runResourceMonitor(ctx context.Context, d resourceMonitorDeps) error {
cursor := 0
persistenceBlocked, failedMAC, attemptedMAC := false, "", ""
var shutdownError error
for ctx.Err() == nil {
state, stateErr := d.load()
if persistenceBlocked && stateErr == nil && state.MAC != failedMAC && state.MAC != attemptedMAC {
persistenceBlocked = false
}
source, sourceErr := d.source()
blocked := persistenceBlocked || stateErr != nil || sourceErr != nil || d.guard() != nil
dirty := false
if !blocked {
rows := reconcileResourceRows(d.key, source, state.Rows)
dirty = !reflect.DeepEqual(rows, state.Rows)
state.Rows = rows
if len(state.Rows) > 0 {
cursor %= len(state.Rows)
for n := 0; n < len(state.Rows); n++ {
i := (cursor + n) % len(state.Rows)
row := &state.Rows[i]
if row.NextDue > d.now().UnixMilli() {
continue
}
var spec resourceCheckSpec
for _, s := range source.config.Services {
if s.ID == row.Service {
for _, r := range s.Resources {
if r.ID == row.Resource {
spec = r
}
}
}
}
guard := func() error {
if d.guard() != nil {
return errResourceConfig
}
fresh, err := d.source()
if err != nil || fresh.fingerprint != source.fingerprint {
return errResourceConfig
}
return nil
}
outcome, err := d.probe(ctx, row.Service, spec, source, guard)
cursor = (i + 1) % len(state.Rows)
if err != nil {
if ctx.Err() == nil || !managedCancellationOnly(err, ctx.Err()) {
blocked = true
if ctx.Err() != nil {
shutdownError = err
}
}
break
}
// Cancellation joins the probe but is not completed evidence.
// Keep the previous health, failure count and due time intact.
if ctx.Err() != nil || outcome == "cancelled" {
break
}
dirty = true
row.Outcome, row.LastCompleted = outcome, d.now().UnixMilli()
delay := *spec.CooldownMS
if outcome == "pass" {
row.Health, row.Failures, delay = "HEALTHY", 0, *spec.IntervalMS
} else {
row.Health = "DEGRADED"
if row.Failures < ^uint64(0) {
row.Failures++
}
}
row.NextDue = row.LastCompleted + int64(delay)
break // persist each completed (and joined) check before another probe
}
}
}
dirty = dirty || state.Blocked != blocked
state.Blocked = blocked
if dirty {
state.Updated = d.now().UnixMilli()
}
if dirty && stateErr == nil && !persistenceBlocked {
if d.save(state) != nil {
blocked, persistenceBlocked, failedMAC, attemptedMAC = true, true, state.MAC, resourceStateMAC(d.key, state)
}
}
d.report(blocked)
if ctx.Err() != nil {
if persistenceBlocked {
return errors.Join(shutdownError, errResourceState)
}
return shutdownError
}
// Bound reloads, but don't delay an already due sibling by a cooldown.
delay := time.Second
if !blocked {
for _, r := range state.Rows {
milliseconds := r.NextDue - d.now().UnixMilli()
if milliseconds > 1000 {
milliseconds = 1000
}
remaining := time.Duration(milliseconds) * time.Millisecond
if remaining < 0 {
remaining = 0
}
if remaining < delay {
delay = remaining
}
}
}
if err := d.wait(ctx, delay); err != nil {
if ctx.Err() != nil {
if persistenceBlocked {
return errResourceState
}
return nil
}
return err
}
}
if persistenceBlocked {
return errResourceState
}
return nil
}
func probeResourceMonitor(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error, check resourceCheckDeps) (string, error) {
check.guard = guard
protocols := []string{spec.TLS}
if spec.TLS == "both" {
protocols = []string{"tls12", "tls13"}
}
outcome := "pass"
for _, protocol := range protocols {
if ctx.Err() != nil {
return "cancelled", nil
}
result, err := runResourceProtocol(ctx, service, spec, protocol, source.server, source.proxy, check)
if err != nil {
return "", err
}
if result.Outcome != "pass" {
outcome = result.Outcome
}
}
return outcome, nil
}
func resourceMonitorCLI(ctx context.Context, path string, stdout, stderr io.Writer, target managedDeps) (code int) {
return resourceMonitorCLIWithCheck(ctx, path, stdout, stderr, target, productionResourceCheckDeps())
}
func resourceMonitorCLIWithCheck(ctx context.Context, path string, stdout, stderr io.Writer, target managedDeps, check resourceCheckDeps) (code int) {
port, settingsErr := managedTargetSettings(target.lookup)
if settingsErr != nil || port != 1080 {
fmt.Fprintln(stderr, "configerror: invalid generic target settings")
return 64
}
if _, _, err := resourceStaticConfig(target.lookup); err != nil {
fmt.Fprintln(stderr, "configerror: invalid generic DNS/proxy configuration")
return 64
}
var lock *managedMonitorSingleton
var store *resourceMonitorStore
defer func() {
if store != nil && store.close() != nil {
code = 78
}
if lock != nil && lock.close() != nil {
code = 78
}
}()
lastBlocked := false
report := func(blocked bool) {
if blocked && !lastBlocked {
fmt.Fprintln(stderr, "BLOCKED: generic resource monitor guard/state unavailable")
}
lastBlocked = blocked
}
fmt.Fprintln(stdout, "generic resource monitor: configured checks only; no recovery/apply; HEALTHY is not production READY")
// Guard failures before store/lock acquisition remain cancellable and alive.
for ctx.Err() == nil {
if lock == nil {
var err error
lock, err = acquireManagedMonitorSingleton(target)
if err != nil {
if errors.Is(err, syscall.EWOULDBLOCK) {
fmt.Fprintln(stderr, "BLOCKED: monitor singleton already held")
return 78
}
report(true)
if waitManagedMonitor(ctx, time.Second) != nil {
return 0
}
continue
}
}
if store == nil {
var err error
store, err = openResourceMonitorStore(target.directory, true)
if err != nil {
report(true)
if waitManagedMonitor(ctx, time.Second) != nil {
return 0
}
continue
}
}
err := runResourceMonitor(ctx, resourceMonitorDeps{
load: store.load, save: store.save, key: store.key,
source: func() (resourceMonitorSource, error) {
return readResourceMonitorSource(path, target.lookup, store.key)
},
guard: func() error {
if resourceSingletonGuard(lock, target.directory) != nil {
return errResourceState
}
// Probe guards validate the held snapshot, never reload/adopt a new
// authenticated operator edit while an older check is in flight.
if store.snapshotGuard() != nil {
return errResourceState
}
if resourceManagedIdentity(target.directory) != nil {
return errResourceState
}
return check.guard()
},
probe: func(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error) (string, error) {
return probeResourceMonitor(ctx, service, spec, source, guard, check)
},
now: time.Now, wait: waitManagedMonitor, report: report,
})
if err != nil {
fmt.Fprintln(stderr, "BLOCKED: generic monitor safety/finalization failure")
return 78
}
}
return 0
}
func resourceSingletonGuard(lock *managedMonitorSingleton, dir string) error {
path := filepath.Join(dir, managedMonitorLockName)
if managedPrivateFile(path) != nil {
return errResourceState
}
current, err := os.Lstat(path)
held, heldErr := lock.file.Stat()
if err != nil || heldErr != nil || !os.SameFile(current, held) {
return errResourceState
}
return nil
}
func resourceStatusCLI(stdout io.Writer, target managedDeps) int {
blocked := func() int { fmt.Fprintln(stdout, `{"health":"BLOCKED","stale":true}`); return 1 }
store, err := openResourceMonitorStore(target.directory, false)
if err != nil {
return blocked()
}
defer store.close()
state, err := store.load()
if err != nil {
return blocked()
}
path, _ := target.lookup("ZAPRET_AUTOTUNE_RESOURCE_CONFIG")
source, err := readResourceMonitorSource(path, target.lookup, store.key)
check := productionResourceCheckDeps()
if err != nil || check.guard() != nil || resourceManagedIdentity(target.directory) != nil {
return blocked()
}
rows := reconcileResourceRows(store.key, source, state.Rows)
b1, _ := json.Marshal(rows)
b2, _ := json.Marshal(state.Rows)
stale := resourceSnapshotStale(state, time.Now().UnixMilli()) || string(b1) != string(b2)
health := "CHECKS_ONLY"
if stale {
health = "STALE"
}
if state.Blocked {
health = "BLOCKED"
}
if json.NewEncoder(stdout).Encode(struct {
Health string `json:"health"`
Stale bool `json:"stale"`
State resourceMonitorState `json:"state"`
}{health, stale, state}) != nil {
return 1
}
if stale || state.Blocked {
return 1
}
return 0
}
// Updated is a semantic-change timestamp, not a disk heartbeat. Evidence is
// stale only once its next scheduled check plus the maximum run budget is past.
func resourceSnapshotStale(state resourceMonitorState, now int64) bool {
if state.Updated > now {
return true
}
if len(state.Rows) == 0 {
return now-state.Updated > 65000
}
for _, row := range state.Rows {
if row.NextDue == 0 {
if now-state.Updated > 65000 {
return true
}
} else if now > row.NextDue && now-row.NextDue > 65000 {
return true
}
}
return false
}
func resourceManagedIdentity(dir string) error {
m := &managedTransaction{Dir: dir, Name: "config"}
if managedCheckIdentity(m) != nil {
return errResourceState
}
j, err := m.load()
if err != nil || j.State != "committed" {
return errResourceState
}
b, err := m.read("config")
if err != nil || managedHash(b) != j.Hash {
return errResourceState
}
return nil
}