Files
zapret-autotune/cmd/zapret-autotune/managed_recovery.go
T

485 lines
15 KiB
Go

package main
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"
"time"
)
const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2"
// Caller owns startup reconciliation and the monitor singleton. Discover and
// Health must bind to the guarded target SOCKS. No CLI parsing or remote work.
type managedRecoveryDeps struct {
db *sql.DB
transaction *managedTransaction
scan cacheScan
check cacheCheck
discover func(context.Context) (string, error)
health func(context.Context) (string, error)
run func(context.Context, string, ...string) error
}
// Target-local adapter: reuses bounded Player requests without invoking a CLI.
// The injected command runner receives only fixed script/argv from orchestration.
func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, check cacheCheck, run func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
d := managedRecoveryDeps{db: db, scan: scan, check: check, run: run}
if target.lookup == nil || target.player == nil || db == nil || scan == nil || check == nil || run == nil {
return d, fmt.Errorf("complete target-local recovery adapter dependencies required")
}
if _, err := managedRuntimeGuard(target); err != nil {
return d, err
}
cfg, err := parseHealthConfig(target.lookup)
if err != nil {
return d, err
}
d.transaction = &managedTransaction{Dir: target.directory, Name: "config"}
request := func(ctx context.Context) ([]byte, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if _, err := managedRuntimeGuard(target); err != nil {
return nil, &managedRecoveryFatal{Err: err}
}
bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
defer cancel()
body, err := requestPlayer(cfg.VideoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error {
return target.player(bounded, args, out, diagnostics)
})
if bounded.Err() != nil {
return nil, bounded.Err()
}
return body, err
}
d.health = func(ctx context.Context) (string, error) {
body, err := request(ctx)
if err != nil {
return "", err
}
var response struct {
Playability struct {
Status string `json:"status"`
} `json:"playabilityStatus"`
}
if err := json.Unmarshal(body, &response); err != nil {
return "", fmt.Errorf("malformed Player API health response")
}
if response.Playability.Status == "" {
return "", fmt.Errorf("missing Player API health status")
}
if response.Playability.Status != "OK" {
return response.Playability.Status, nil
}
if err := playerHealth(body); err != nil {
return "", err
}
return "OK", nil
}
d.discover = func(ctx context.Context) (string, error) {
body, err := request(ctx)
if err != nil {
return "", err
}
return playerHostname(body)
}
return d, nil
}
type managedRecoveryFatal struct{ Err error }
func (e *managedRecoveryFatal) Error() string {
return "managed recovery safety failure: " + e.Err.Error()
}
func (e *managedRecoveryFatal) Unwrap() error { return e.Err }
type managedRecoveryResult struct {
FrontendID, GooglevideoID int64
GGCError error // Unresolved; never a playback/Player API health claim.
}
func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result managedRecoveryResult, err error) {
if d.db == nil || d.transaction == nil || d.scan == nil || d.check == nil || d.discover == nil || d.health == nil || d.run == nil {
return result, fmt.Errorf("complete managed recovery dependencies required")
}
if err := ctx.Err(); err != nil {
return result, err
}
m := *d.transaction
// Rollback must restart even after job cancellation; independently bounded.
m.Reload = func() error {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
if err := d.run(cleanup, managedZapretScript, "restart"); err != nil {
return &managedRecoveryFatal{Err: err}
}
return nil
}
m.Health = func() (string, error) { return d.health(ctx) }
ids, err := m.ActiveCandidates(d.db)
if err != nil {
return result, &managedRecoveryFatal{Err: err}
}
for _, id := range ids {
c, e := readStrategyCandidate(d.db, id)
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
if c.Group == "frontend" {
result.FrontendID = id
}
}
stopped := false
defer func() {
if stopped {
// Never start bytes left pending by a failed rollback. Reconcile restores
// the last committed set or fails closed without activating unknown bytes.
if e := m.Reconcile(d.db); e != nil {
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
return
}
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
if e := d.run(cleanup, managedZapretScript, "start"); e != nil {
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
}
}
}()
stop := func() error {
stopped = true // A partially failed stop also needs restoration.
return d.run(ctx, managedZapretScript, "stop")
}
apply := func(ids []int64) error {
if err := ctx.Err(); err != nil {
return err
}
cleanupReload := m.Reload
first := true
m.Reload = func() error {
if !first {
return cleanupReload()
}
first = false
activation, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if err := d.run(activation, managedZapretScript, "restart"); err != nil {
return &managedRecoveryFatal{Err: err}
}
return nil
}
defer func() { m.Reload = cleanupReload }()
e := m.Apply(d.db, ids)
if e == nil {
stopped = false
}
var rollback *managedRollbackError
if errors.As(e, &rollback) {
return &managedRecoveryFatal{Err: e}
}
var rejection *managedHealthRejection
if e != nil && !errors.As(e, &rejection) && !errors.Is(e, context.Canceled) && !errors.Is(e, context.DeadlineExceeded) {
return &managedRecoveryFatal{Err: e}
}
return e
}
healthy := false
if result.FrontendID != 0 {
status, e := d.health(ctx)
var fatal *managedRecoveryFatal
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
return result, e
}
if err := ctx.Err(); err != nil {
return result, err
}
// A bounded Player request timeout is unhealthy, not job cancellation.
healthy = e == nil && status == "OK"
}
if !healthy {
if e := stop(); e != nil {
return result, e
}
// Both bootstrap and live recovery check saved definitions before scanning.
saved, e := qualifiedCandidateIDs(d.db, "frontend")
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
for _, id := range saved {
c, e := readStrategyCandidate(d.db, id)
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
inScope := false
for _, endpoint := range c.Endpoints {
inScope = inScope || endpoint == managedFrontendHost || strings.HasSuffix(managedFrontendHost, "."+endpoint)
}
if !inScope {
return result, &managedRecoveryFatal{Err: fmt.Errorf("saved frontend scope excludes Player API host")}
}
_, e = managedRecheck(ctx, d.check, c.Args, "frontend", managedFrontendHost)
if errors.Is(e, errCandidateNonpass) {
continue
}
if e != nil {
return result, e
}
e = apply([]int64{id})
if e == nil {
result.FrontendID, healthy = id, true
break
}
var fatal *managedRecoveryFatal
var rejection *managedHealthRejection
if errors.As(e, &fatal) || !errors.As(e, &rejection) {
return result, e
}
// Health rollback restarted the prior config: stop before rechecking.
if e := stop(); e != nil {
return result, e
}
}
if !healthy {
id, e := managedScanCandidate(ctx, d, "frontend", managedFrontendHost)
if e != nil {
return result, e
}
if e := apply([]int64{id}); e != nil {
return result, e
}
result.FrontendID = id
}
}
// A frontend-only rollback boundary prevents cache failures undoing bootstrap.
current, e := m.ActiveCandidates(d.db)
if e != nil {
return result, e
}
if len(current) != 1 || current[0] != result.FrontendID {
if e := apply([]int64{result.FrontendID}); e != nil {
return result, e
}
}
host, e := d.discover(ctx)
if ctx.Err() != nil {
return result, ctx.Err()
}
if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
result.GGCError = fmt.Errorf("exact GGC discovery failed")
return result, nil
}
if e := stop(); e != nil {
return result, e
}
args, report, e := revalidateGooglevideo(ctx, d.db, host, io.Discard, d.check)
var cacheID int64
if e == nil && args != "" {
e = saveGooglevideoRun(d.db, time.Now(), host, args, report, nil)
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
if e == nil {
cacheID, e = newestManagedCandidateForScope(d.db, "googlevideo", host, args)
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
}
} else if e == nil {
cacheID, e = managedScanCandidate(ctx, d, "googlevideo", host)
}
if e == nil {
e = apply([]int64{result.FrontendID, cacheID})
}
if e != nil {
var fatal *managedRecoveryFatal
if errors.As(e, &fatal) {
return result, e
}
if ctx.Err() != nil {
return result, ctx.Err()
}
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
return result, nil
}
result.GooglevideoID = cacheID
return result, nil
}
func newestManagedCandidate(db *sql.DB, group string) (int64, error) {
ids, err := qualifiedCandidateIDs(db, group)
if err != nil {
return 0, err
}
if len(ids) == 0 {
return 0, fmt.Errorf("no qualified candidate")
}
return ids[0], nil
}
func newestManagedCandidateForScope(db *sql.DB, group, host, args string) (int64, error) {
id, err := newestManagedCandidate(db, group)
if err != nil {
return 0, err
}
c, err := readStrategyCandidate(db, id)
if err != nil {
return 0, err
}
if c.Args != args || len(c.Endpoints) != 1 || c.Endpoints[0] != host {
return 0, fmt.Errorf("latest saved candidate scope changed")
}
return id, nil
}
func managedRecheck(ctx context.Context, check cacheCheck, args, group, host string) ([]byte, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
f, err := os.CreateTemp("", "blockcheckw-managed-*.txt")
if err != nil {
return nil, err
}
path := f.Name()
if err := f.Close(); err != nil {
return nil, err
}
return revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, check)
}
// Reuses proposal selection and transactional persistence with caller context.
func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, host string) (int64, error) {
if err := ctx.Err(); err != nil {
return 0, err
}
started := time.Now()
path, err := d.scan(ctx, "blockcheckw", host, io.Discard, io.Discard)
var report []byte
var args, output string
if err == nil {
var info os.FileInfo
info, err = os.Stat(path + ".check.json")
// ponytail: whole-second mtime permits same-second artifacts from injected
// scans; production uses unique paths. Require that contract if precision grows.
if err == nil && (!info.Mode().IsRegular() || info.ModTime().Before(started.Truncate(time.Second))) {
err = fmt.Errorf("missing fresh scan shortlist")
}
}
if err == nil {
report, err = os.ReadFile(path + ".check.json")
}
if err == nil {
args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, host)
}
if err == nil {
report, err = revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, d.check)
}
if err == nil {
err = ctx.Err()
}
if group == "frontend" {
if err == nil {
output, err = combinedBlockcheckOutputForHost(bytes.NewReader(report), args, host)
}
if e := saveFrontendRunForHost(d.db, started, args, output, report, err, host); e != nil {
return 0, &managedRecoveryFatal{Err: e}
}
} else {
var safe error
if err != nil {
safe = fmt.Errorf("cache scan or verification failed")
}
if e := saveGooglevideoRun(d.db, started, host, args, report, safe); e != nil {
return 0, &managedRecoveryFatal{Err: e}
}
}
if err != nil {
return 0, err
}
id, err := newestManagedCandidateForScope(d.db, group, host, args)
if err != nil {
return 0, &managedRecoveryFatal{Err: err}
}
return id, nil
}
// Installs callbacks. runManagedMonitor performs startup
// under its singleton. Target guards are repeated before every recovery/probe.
func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDeps, reportGGC func(error)) error {
if monitor == nil || monitor.db == nil || monitor.db != recovery.db || recovery.transaction == nil || reportGGC == nil || recovery.health == nil || recovery.run == nil || recovery.scan == nil || recovery.check == nil || recovery.discover == nil {
return fmt.Errorf("matching monitor/recovery database and hooks required")
}
if recovery.transaction.Dir != monitor.target.directory || recovery.transaction.Name != "config" {
return fmt.Errorf("recovery transaction must own the monitor target config")
}
guard := func() error {
if _, err := managedRuntimeGuard(monitor.target); err != nil {
return err
}
return managedCheckIdentity(recovery.transaction)
}
callback := func(ctx context.Context) error {
if err := guard(); err != nil {
return &managedRecoveryFatal{Err: err}
}
result, err := recoverManagedTarget(ctx, recovery)
if result.GGCError != nil {
reportGGC(result.GGCError)
}
// The supervisor reports ordinary recovery failures and retries only
// after its durable cooldown. Safety failures (rollback/cleanup/drift)
// are already typed managedRecoveryFatal by the transaction boundary.
return err
}
monitor.bootstrap = func(ctx context.Context) error {
if err := guard(); err != nil {
return &managedRecoveryFatal{Err: err}
}
// Reconcile once at startup, never adopt a pending live deployment.
transaction := *recovery.transaction
transaction.Health = func() (string, error) { return recovery.health(ctx) }
transaction.Reload = func() error {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
return recovery.run(cleanup, managedZapretScript, "restart")
}
if err := transaction.Reconcile(recovery.db); err != nil {
return &managedRecoveryFatal{Err: err}
}
ids, err := transaction.ActiveCandidates(recovery.db)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
for _, id := range ids {
candidate, err := readStrategyCandidate(recovery.db, id)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
if candidate.Group == "frontend" && candidate.Qualified {
return nil
}
}
return callback(ctx)
}
monitor.recover = callback
monitor.probe = func(ctx context.Context) error {
if err := guard(); err != nil {
return &managedRecoveryFatal{Err: err}
}
status, err := recovery.health(ctx)
if err != nil {
return err
}
if status != "OK" {
return &managedHealthRejection{Status: status}
}
return nil
}
return nil
}