Files
zapret-autotune/cmd/zapret-autotune/managed_monitor_cli.go
T

205 lines
7.7 KiB
Go

package main
import (
"context"
"database/sql"
"errors"
"flag"
"fmt"
"io"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
)
// Provisioning owns executable, database, proxy and target paths; no path flags.
func managedMonitorConfig(args []string, lookup func(string) (string, bool)) (healthConfig, error) {
cfg, err := parseHealthConfig(lookup)
if err != nil {
return cfg, err
}
f := flag.NewFlagSet("managed monitor", flag.ContinueOnError)
f.SetOutput(io.Discard)
f.DurationVar(&cfg.HealthInterval, "interval", cfg.HealthInterval, "")
f.IntVar(&cfg.FailureThreshold, "failure-threshold", cfg.FailureThreshold, "")
f.DurationVar(&cfg.RecoveryCooldown, "cooldown", cfg.RecoveryCooldown, "")
if f.Parse(args) != nil || f.NArg() != 0 || cfg.HealthInterval <= 0 || cfg.HealthInterval > 24*time.Hour || cfg.RecoveryCooldown <= 0 || cfg.RecoveryCooldown > 7*24*time.Hour || cfg.FailureThreshold < 1 || cfg.FailureThreshold > 1000 {
return healthConfig{}, fmt.Errorf("monitor requires --interval in (0,24h], --failure-threshold in [1,1000], --cooldown in (0,168h]; no path/command flags")
}
return cfg, nil
}
func managedMonitorDatabase(d managedDeps) (string, error) {
path, _ := d.lookup("ZAPRET_AUTOTUNE_DB")
if path != managedTargetDirectory+"/candidates.db" {
return "", fmt.Errorf("monitor database must be target-owned candidates.db")
}
for _, suffix := range []string{"", "-wal", "-shm", "-journal"} {
p := filepath.Join(d.directory, "candidates.db"+suffix)
if _, err := os.Lstat(p); errors.Is(err, os.ErrNotExist) {
continue
} else if err != nil {
return "", err
}
if err := managedPrivateFile(p); err != nil {
return "", err
}
}
return filepath.Join(d.directory, "candidates.db"), nil
}
func managedMonitorCLI(args []string, stdout, stderr io.Writer, target managedDeps) int {
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
return managedMonitorCLIContext(ctx, args, stdout, stderr, target, runManagedMonitor)
}
func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) (code int) {
cfg, err := managedMonitorConfig(args, target.lookup)
if err != nil {
fmt.Fprintln(stderr, "error: invalid managed monitor configuration")
return 64
}
if _, err := managedRuntimeGuard(target); err != nil {
fmt.Fprintln(stderr, "error: managed monitor target guard refused")
return 78
}
m := &managedTransaction{Dir: target.directory, Name: "config"}
if err := managedDirectory(target.directory); err != nil {
fmt.Fprintln(stderr, "error: unsafe managed monitor directory")
return 78
}
if err := managedCheckIdentity(m); err != nil {
fmt.Fprintln(stderr, "error: managed monitor identity refused")
return 78
}
path, err := managedMonitorDatabase(target)
if err != nil {
fmt.Fprintln(stderr, "error: unsafe managed monitor database")
return 78
}
db, err := openStore(path)
if err != nil {
fmt.Fprintln(stderr, "error: managed monitor database unavailable")
return 78
}
defer func() {
if err := db.Close(); err != nil {
fmt.Fprintln(stderr, "error: managed monitor database close failed")
code = 78
}
}()
recovery, err := managedMonitorRecoveryAdapters(target, db, runManagedBlockcheck, runManagedCombinedBlockcheckForHost, func(ctx context.Context, path string, args ...string) error {
log, err := os.CreateTemp(target.directory, ".daemon-diagnostics-*.log")
if err != nil {
return err
}
// Target-owned 0600 artifact, never public stdout/stderr. Daemon startup
// diagnostics are necessary to distinguish activation from scan failure.
err = runBlockcheckCommand(ctx, path, "", args, log, log)
return errors.Join(err, log.Close())
})
if err != nil {
fmt.Fprintln(stderr, "error: managed monitor adapters refused")
return 78
}
monitor := managedMonitorDeps{
target: target, db: db, serviceID: managedTargetID + "/youtube",
interval: cfg.HealthInterval, threshold: cfg.FailureThreshold, cooldown: cfg.RecoveryCooldown,
now: time.Now, wait: waitManagedMonitor,
reportRecoveryError: func(error) { fmt.Fprintln(stderr, "managed recovery failed (details suppressed)") },
}
if recovery.service {
monitor.reportProbeError = func(err error) {
var failure *managedResourceFailure
if errors.As(err, &failure) {
fmt.Fprintln(stderr, failure.PublicError())
} else {
fmt.Fprintln(stderr, "DEGRADED: service scope unresolved")
}
}
}
if err := wireManagedRecovery(&monitor, recovery, func(error) {
fmt.Fprintln(stderr, "managed GGC unresolved; monitoring continues (not playback health)")
}); err != nil {
fmt.Fprintln(stderr, "error: managed monitor wiring refused")
return 78
}
if recovery.service {
fmt.Fprintln(stdout, "managed monitor: service resource checks; SABR unknown; owned temporary asset verifier")
} else {
fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health")
}
err = supervise(ctx, monitor)
if err != nil {
var fatal *managedRecoveryFatal
if ctx.Err() != nil && managedCancellationOnly(err, ctx.Err()) && !errors.As(err, &fatal) {
return 0
}
fmt.Fprintln(stderr, "error: managed monitor stopped; safety/database/runtime failure (details suppressed)")
if managedMonitorTechnicalTimeout(err) {
return 75
}
return 78
}
if ctx.Err() != nil {
return 0
}
return 75 // A supervisor must not return normally without cancellation.
}
// Only clean typed timeouts may restart; a joined cleanup failure vetoes it.
func managedMonitorTechnicalTimeout(err error) bool {
if timeout, ok := err.(*managedRecoveryTimeout); ok {
return managedRecoveryTimeoutSafe(timeout.Err)
}
if joined, ok := err.(interface{ Unwrap() []error }); ok {
for _, e := range joined.Unwrap() {
if !managedMonitorTechnicalTimeout(e) {
return false
}
}
return len(joined.Unwrap()) != 0
}
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
return managedMonitorTechnicalTimeout(wrapped.Unwrap())
}
return false
}
func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand cacheScan, checkCommand cacheCheck, command func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
if scanCommand == nil || checkCommand == nil || command == nil {
return managedRecoveryDeps{}, fmt.Errorf("complete managed process adapters required")
}
// The blockcheck process-group runner cancels and waits descendants.
scan := func(ctx context.Context, _ string, host string, _ io.Writer, diagnostics io.Writer) (string, error) {
return scanCommand(ctx, "/usr/bin/blockcheckw", host, io.Discard, diagnostics)
}
check := func(ctx context.Context, _ string, path, args, host string, _ io.Writer, diagnostics io.Writer) error {
return checkCommand(ctx, "/usr/bin/blockcheckw", path, args, host, io.Discard, diagnostics)
}
run := func(ctx context.Context, path string, args ...string) error {
if path != managedZapretScript || len(args) != 1 || (args[0] != "start" && args[0] != "stop" && args[0] != "restart") {
return fmt.Errorf("invalid target-local daemon operation")
}
bounded, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
return command(bounded, path, args...)
}
d, err := managedRecoveryDependencies(target, db, scan, check, run)
if err == nil && d.service {
d.verifyAssets = newManagedAssetVerifier(target)
if d.adaptive {
d.verifyGooglevideo = newManagedGooglevideoVerifier(target)
}
}
return d, err
}
// Joined cancellation plus failed cleanup/DB/lock close is not a graceful exit.
func managedCancellationOnly(err, cancellation error) bool {
return err != nil && errors.Is(err, cancellation) && managedRecoveryTimeoutSafe(err)
}