485 lines
15 KiB
Go
485 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2"
|
|
|
|
// Caller owns startup reconciliation and the monitor singleton. Discover and
|
|
// Health must bind to the guarded target SOCKS. No CLI parsing or remote work.
|
|
type managedRecoveryDeps struct {
|
|
db *sql.DB
|
|
transaction *managedTransaction
|
|
scan cacheScan
|
|
check cacheCheck
|
|
discover func(context.Context) (string, error)
|
|
health func(context.Context) (string, error)
|
|
run func(context.Context, string, ...string) error
|
|
}
|
|
|
|
// Target-local adapter: reuses bounded Player requests without invoking a CLI.
|
|
// The injected command runner receives only fixed script/argv from orchestration.
|
|
func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, check cacheCheck, run func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
|
|
d := managedRecoveryDeps{db: db, scan: scan, check: check, run: run}
|
|
if target.lookup == nil || target.player == nil || db == nil || scan == nil || check == nil || run == nil {
|
|
return d, fmt.Errorf("complete target-local recovery adapter dependencies required")
|
|
}
|
|
if _, err := managedRuntimeGuard(target); err != nil {
|
|
return d, err
|
|
}
|
|
cfg, err := parseHealthConfig(target.lookup)
|
|
if err != nil {
|
|
return d, err
|
|
}
|
|
d.transaction = &managedTransaction{Dir: target.directory, Name: "config"}
|
|
request := func(ctx context.Context) ([]byte, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := managedRuntimeGuard(target); err != nil {
|
|
return nil, &managedRecoveryFatal{Err: err}
|
|
}
|
|
bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
|
|
defer cancel()
|
|
body, err := requestPlayer(cfg.VideoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error {
|
|
return target.player(bounded, args, out, diagnostics)
|
|
})
|
|
if bounded.Err() != nil {
|
|
return nil, bounded.Err()
|
|
}
|
|
return body, err
|
|
}
|
|
d.health = func(ctx context.Context) (string, error) {
|
|
body, err := request(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var response struct {
|
|
Playability struct {
|
|
Status string `json:"status"`
|
|
} `json:"playabilityStatus"`
|
|
}
|
|
if err := json.Unmarshal(body, &response); err != nil {
|
|
return "", fmt.Errorf("malformed Player API health response")
|
|
}
|
|
if response.Playability.Status == "" {
|
|
return "", fmt.Errorf("missing Player API health status")
|
|
}
|
|
if response.Playability.Status != "OK" {
|
|
return response.Playability.Status, nil
|
|
}
|
|
if err := playerHealth(body); err != nil {
|
|
return "", err
|
|
}
|
|
return "OK", nil
|
|
}
|
|
d.discover = func(ctx context.Context) (string, error) {
|
|
body, err := request(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return playerHostname(body)
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
type managedRecoveryFatal struct{ Err error }
|
|
|
|
func (e *managedRecoveryFatal) Error() string {
|
|
return "managed recovery safety failure: " + e.Err.Error()
|
|
}
|
|
func (e *managedRecoveryFatal) Unwrap() error { return e.Err }
|
|
|
|
type managedRecoveryResult struct {
|
|
FrontendID, GooglevideoID int64
|
|
GGCError error // Unresolved; never a playback/Player API health claim.
|
|
}
|
|
|
|
func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result managedRecoveryResult, err error) {
|
|
if d.db == nil || d.transaction == nil || d.scan == nil || d.check == nil || d.discover == nil || d.health == nil || d.run == nil {
|
|
return result, fmt.Errorf("complete managed recovery dependencies required")
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return result, err
|
|
}
|
|
m := *d.transaction
|
|
// Rollback must restart even after job cancellation; independently bounded.
|
|
m.Reload = func() error {
|
|
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
|
defer cancel()
|
|
if err := d.run(cleanup, managedZapretScript, "restart"); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
return nil
|
|
}
|
|
m.Health = func() (string, error) { return d.health(ctx) }
|
|
ids, err := m.ActiveCandidates(d.db)
|
|
if err != nil {
|
|
return result, &managedRecoveryFatal{Err: err}
|
|
}
|
|
for _, id := range ids {
|
|
c, e := readStrategyCandidate(d.db, id)
|
|
if e != nil {
|
|
return result, &managedRecoveryFatal{Err: e}
|
|
}
|
|
if c.Group == "frontend" {
|
|
result.FrontendID = id
|
|
}
|
|
}
|
|
stopped := false
|
|
defer func() {
|
|
if stopped {
|
|
// Never start bytes left pending by a failed rollback. Reconcile restores
|
|
// the last committed set or fails closed without activating unknown bytes.
|
|
if e := m.Reconcile(d.db); e != nil {
|
|
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
|
return
|
|
}
|
|
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
|
defer cancel()
|
|
if e := d.run(cleanup, managedZapretScript, "start"); e != nil {
|
|
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
|
}
|
|
}
|
|
}()
|
|
stop := func() error {
|
|
stopped = true // A partially failed stop also needs restoration.
|
|
return d.run(ctx, managedZapretScript, "stop")
|
|
}
|
|
apply := func(ids []int64) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
cleanupReload := m.Reload
|
|
first := true
|
|
m.Reload = func() error {
|
|
if !first {
|
|
return cleanupReload()
|
|
}
|
|
first = false
|
|
activation, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
if err := d.run(activation, managedZapretScript, "restart"); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
return nil
|
|
}
|
|
defer func() { m.Reload = cleanupReload }()
|
|
e := m.Apply(d.db, ids)
|
|
if e == nil {
|
|
stopped = false
|
|
}
|
|
var rollback *managedRollbackError
|
|
if errors.As(e, &rollback) {
|
|
return &managedRecoveryFatal{Err: e}
|
|
}
|
|
var rejection *managedHealthRejection
|
|
if e != nil && !errors.As(e, &rejection) && !errors.Is(e, context.Canceled) && !errors.Is(e, context.DeadlineExceeded) {
|
|
return &managedRecoveryFatal{Err: e}
|
|
}
|
|
return e
|
|
}
|
|
healthy := false
|
|
if result.FrontendID != 0 {
|
|
status, e := d.health(ctx)
|
|
var fatal *managedRecoveryFatal
|
|
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
|
|
return result, e
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return result, err
|
|
}
|
|
// A bounded Player request timeout is unhealthy, not job cancellation.
|
|
healthy = e == nil && status == "OK"
|
|
}
|
|
if !healthy {
|
|
if e := stop(); e != nil {
|
|
return result, e
|
|
}
|
|
// Both bootstrap and live recovery check saved definitions before scanning.
|
|
saved, e := qualifiedCandidateIDs(d.db, "frontend")
|
|
if e != nil {
|
|
return result, &managedRecoveryFatal{Err: e}
|
|
}
|
|
for _, id := range saved {
|
|
c, e := readStrategyCandidate(d.db, id)
|
|
if e != nil {
|
|
return result, &managedRecoveryFatal{Err: e}
|
|
}
|
|
inScope := false
|
|
for _, endpoint := range c.Endpoints {
|
|
inScope = inScope || endpoint == managedFrontendHost || strings.HasSuffix(managedFrontendHost, "."+endpoint)
|
|
}
|
|
if !inScope {
|
|
return result, &managedRecoveryFatal{Err: fmt.Errorf("saved frontend scope excludes Player API host")}
|
|
}
|
|
_, e = managedRecheck(ctx, d.check, c.Args, "frontend", managedFrontendHost)
|
|
if errors.Is(e, errCandidateNonpass) {
|
|
continue
|
|
}
|
|
if e != nil {
|
|
return result, e
|
|
}
|
|
e = apply([]int64{id})
|
|
if e == nil {
|
|
result.FrontendID, healthy = id, true
|
|
break
|
|
}
|
|
var fatal *managedRecoveryFatal
|
|
var rejection *managedHealthRejection
|
|
if errors.As(e, &fatal) || !errors.As(e, &rejection) {
|
|
return result, e
|
|
}
|
|
// Health rollback restarted the prior config: stop before rechecking.
|
|
if e := stop(); e != nil {
|
|
return result, e
|
|
}
|
|
}
|
|
if !healthy {
|
|
id, e := managedScanCandidate(ctx, d, "frontend", managedFrontendHost)
|
|
if e != nil {
|
|
return result, e
|
|
}
|
|
if e := apply([]int64{id}); e != nil {
|
|
return result, e
|
|
}
|
|
result.FrontendID = id
|
|
}
|
|
}
|
|
// A frontend-only rollback boundary prevents cache failures undoing bootstrap.
|
|
current, e := m.ActiveCandidates(d.db)
|
|
if e != nil {
|
|
return result, e
|
|
}
|
|
if len(current) != 1 || current[0] != result.FrontendID {
|
|
if e := apply([]int64{result.FrontendID}); e != nil {
|
|
return result, e
|
|
}
|
|
}
|
|
host, e := d.discover(ctx)
|
|
if ctx.Err() != nil {
|
|
return result, ctx.Err()
|
|
}
|
|
if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
|
|
result.GGCError = fmt.Errorf("exact GGC discovery failed")
|
|
return result, nil
|
|
}
|
|
if e := stop(); e != nil {
|
|
return result, e
|
|
}
|
|
args, report, e := revalidateGooglevideo(ctx, d.db, host, io.Discard, d.check)
|
|
var cacheID int64
|
|
if e == nil && args != "" {
|
|
e = saveGooglevideoRun(d.db, time.Now(), host, args, report, nil)
|
|
if e != nil {
|
|
return result, &managedRecoveryFatal{Err: e}
|
|
}
|
|
if e == nil {
|
|
cacheID, e = newestManagedCandidateForScope(d.db, "googlevideo", host, args)
|
|
if e != nil {
|
|
return result, &managedRecoveryFatal{Err: e}
|
|
}
|
|
}
|
|
} else if e == nil {
|
|
cacheID, e = managedScanCandidate(ctx, d, "googlevideo", host)
|
|
}
|
|
if e == nil {
|
|
e = apply([]int64{result.FrontendID, cacheID})
|
|
}
|
|
if e != nil {
|
|
var fatal *managedRecoveryFatal
|
|
if errors.As(e, &fatal) {
|
|
return result, e
|
|
}
|
|
if ctx.Err() != nil {
|
|
return result, ctx.Err()
|
|
}
|
|
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
|
|
return result, nil
|
|
}
|
|
result.GooglevideoID = cacheID
|
|
return result, nil
|
|
}
|
|
|
|
func newestManagedCandidate(db *sql.DB, group string) (int64, error) {
|
|
ids, err := qualifiedCandidateIDs(db, group)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if len(ids) == 0 {
|
|
return 0, fmt.Errorf("no qualified candidate")
|
|
}
|
|
return ids[0], nil
|
|
}
|
|
|
|
func newestManagedCandidateForScope(db *sql.DB, group, host, args string) (int64, error) {
|
|
id, err := newestManagedCandidate(db, group)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
c, err := readStrategyCandidate(db, id)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if c.Args != args || len(c.Endpoints) != 1 || c.Endpoints[0] != host {
|
|
return 0, fmt.Errorf("latest saved candidate scope changed")
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
func managedRecheck(ctx context.Context, check cacheCheck, args, group, host string) ([]byte, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
f, err := os.CreateTemp("", "blockcheckw-managed-*.txt")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
path := f.Name()
|
|
if err := f.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
return revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, check)
|
|
}
|
|
|
|
// Reuses proposal selection and transactional persistence with caller context.
|
|
func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, host string) (int64, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
started := time.Now()
|
|
path, err := d.scan(ctx, "blockcheckw", host, io.Discard, io.Discard)
|
|
var report []byte
|
|
var args, output string
|
|
if err == nil {
|
|
var info os.FileInfo
|
|
info, err = os.Stat(path + ".check.json")
|
|
// ponytail: whole-second mtime permits same-second artifacts from injected
|
|
// scans; production uses unique paths. Require that contract if precision grows.
|
|
if err == nil && (!info.Mode().IsRegular() || info.ModTime().Before(started.Truncate(time.Second))) {
|
|
err = fmt.Errorf("missing fresh scan shortlist")
|
|
}
|
|
}
|
|
if err == nil {
|
|
report, err = os.ReadFile(path + ".check.json")
|
|
}
|
|
if err == nil {
|
|
args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, host)
|
|
}
|
|
if err == nil {
|
|
report, err = revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, d.check)
|
|
}
|
|
if err == nil {
|
|
err = ctx.Err()
|
|
}
|
|
if group == "frontend" {
|
|
if err == nil {
|
|
output, err = combinedBlockcheckOutputForHost(bytes.NewReader(report), args, host)
|
|
}
|
|
if e := saveFrontendRunForHost(d.db, started, args, output, report, err, host); e != nil {
|
|
return 0, &managedRecoveryFatal{Err: e}
|
|
}
|
|
} else {
|
|
var safe error
|
|
if err != nil {
|
|
safe = fmt.Errorf("cache scan or verification failed")
|
|
}
|
|
if e := saveGooglevideoRun(d.db, started, host, args, report, safe); e != nil {
|
|
return 0, &managedRecoveryFatal{Err: e}
|
|
}
|
|
}
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
id, err := newestManagedCandidateForScope(d.db, group, host, args)
|
|
if err != nil {
|
|
return 0, &managedRecoveryFatal{Err: err}
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
// Installs callbacks. runManagedMonitor performs startup
|
|
// under its singleton. Target guards are repeated before every recovery/probe.
|
|
func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDeps, reportGGC func(error)) error {
|
|
if monitor == nil || monitor.db == nil || monitor.db != recovery.db || recovery.transaction == nil || reportGGC == nil || recovery.health == nil || recovery.run == nil || recovery.scan == nil || recovery.check == nil || recovery.discover == nil {
|
|
return fmt.Errorf("matching monitor/recovery database and hooks required")
|
|
}
|
|
if recovery.transaction.Dir != monitor.target.directory || recovery.transaction.Name != "config" {
|
|
return fmt.Errorf("recovery transaction must own the monitor target config")
|
|
}
|
|
guard := func() error {
|
|
if _, err := managedRuntimeGuard(monitor.target); err != nil {
|
|
return err
|
|
}
|
|
return managedCheckIdentity(recovery.transaction)
|
|
}
|
|
callback := func(ctx context.Context) error {
|
|
if err := guard(); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
result, err := recoverManagedTarget(ctx, recovery)
|
|
if result.GGCError != nil {
|
|
reportGGC(result.GGCError)
|
|
}
|
|
// The supervisor reports ordinary recovery failures and retries only
|
|
// after its durable cooldown. Safety failures (rollback/cleanup/drift)
|
|
// are already typed managedRecoveryFatal by the transaction boundary.
|
|
return err
|
|
}
|
|
monitor.bootstrap = func(ctx context.Context) error {
|
|
if err := guard(); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
// Reconcile once at startup, never adopt a pending live deployment.
|
|
transaction := *recovery.transaction
|
|
transaction.Health = func() (string, error) { return recovery.health(ctx) }
|
|
transaction.Reload = func() error {
|
|
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
|
defer cancel()
|
|
return recovery.run(cleanup, managedZapretScript, "restart")
|
|
}
|
|
if err := transaction.Reconcile(recovery.db); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
ids, err := transaction.ActiveCandidates(recovery.db)
|
|
if err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
for _, id := range ids {
|
|
candidate, err := readStrategyCandidate(recovery.db, id)
|
|
if err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
if candidate.Group == "frontend" && candidate.Qualified {
|
|
return nil
|
|
}
|
|
}
|
|
return callback(ctx)
|
|
}
|
|
monitor.recover = callback
|
|
monitor.probe = func(ctx context.Context) error {
|
|
if err := guard(); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
status, err := recovery.health(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if status != "OK" {
|
|
return &managedHealthRejection{Status: status}
|
|
}
|
|
return nil
|
|
}
|
|
return nil
|
|
}
|