package main import ( "bytes" "context" "database/sql" "encoding/json" "errors" "fmt" "io" "os" "strings" "time" ) const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2" // Caller owns startup reconciliation and the monitor singleton. Discover and // Health must bind to the guarded target SOCKS. No CLI parsing or remote work. type managedRecoveryDeps struct { db *sql.DB transaction *managedTransaction scan cacheScan check cacheCheck discover func(context.Context) (string, error) health func(context.Context) (string, error) run func(context.Context, string, ...string) error } // Target-local adapter: reuses bounded Player requests without invoking a CLI. // The injected command runner receives only fixed script/argv from orchestration. func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, check cacheCheck, run func(context.Context, string, ...string) error) (managedRecoveryDeps, error) { d := managedRecoveryDeps{db: db, scan: scan, check: check, run: run} if target.lookup == nil || target.player == nil || db == nil || scan == nil || check == nil || run == nil { return d, fmt.Errorf("complete target-local recovery adapter dependencies required") } if _, err := managedRuntimeGuard(target); err != nil { return d, err } cfg, err := parseHealthConfig(target.lookup) if err != nil { return d, err } d.transaction = &managedTransaction{Dir: target.directory, Name: "config"} request := func(ctx context.Context) ([]byte, error) { if err := ctx.Err(); err != nil { return nil, err } if _, err := managedRuntimeGuard(target); err != nil { return nil, &managedRecoveryFatal{Err: err} } bounded, cancel := context.WithTimeout(ctx, 35*time.Second) defer cancel() body, err := requestPlayer(cfg.VideoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error { return target.player(bounded, args, out, diagnostics) }) if bounded.Err() != nil { return nil, bounded.Err() } return body, err } d.health = func(ctx context.Context) (string, error) { body, err := request(ctx) if err != nil { return "", err } var response struct { Playability struct { Status string `json:"status"` } `json:"playabilityStatus"` } if err := json.Unmarshal(body, &response); err != nil { return "", fmt.Errorf("malformed Player API health response") } if response.Playability.Status == "" { return "", fmt.Errorf("missing Player API health status") } if response.Playability.Status != "OK" { return response.Playability.Status, nil } if err := playerHealth(body); err != nil { return "", err } return "OK", nil } d.discover = func(ctx context.Context) (string, error) { body, err := request(ctx) if err != nil { return "", err } return playerHostname(body) } return d, nil } type managedRecoveryFatal struct{ Err error } func (e *managedRecoveryFatal) Error() string { return "managed recovery safety failure: " + e.Err.Error() } func (e *managedRecoveryFatal) Unwrap() error { return e.Err } type managedRecoveryResult struct { FrontendID, GooglevideoID int64 GGCError error // Unresolved; never a playback/Player API health claim. } func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result managedRecoveryResult, err error) { if d.db == nil || d.transaction == nil || d.scan == nil || d.check == nil || d.discover == nil || d.health == nil || d.run == nil { return result, fmt.Errorf("complete managed recovery dependencies required") } if err := ctx.Err(); err != nil { return result, err } m := *d.transaction // Rollback must restart even after job cancellation; independently bounded. m.Reload = func() error { cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() if err := d.run(cleanup, managedZapretScript, "restart"); err != nil { return &managedRecoveryFatal{Err: err} } return nil } m.Health = func() (string, error) { return d.health(ctx) } ids, err := m.ActiveCandidates(d.db) if err != nil { return result, &managedRecoveryFatal{Err: err} } for _, id := range ids { c, e := readStrategyCandidate(d.db, id) if e != nil { return result, &managedRecoveryFatal{Err: e} } if c.Group == "frontend" { result.FrontendID = id } } stopped := false defer func() { if stopped { // Never start bytes left pending by a failed rollback. Reconcile restores // the last committed set or fails closed without activating unknown bytes. if e := m.Reconcile(d.db); e != nil { err = &managedRecoveryFatal{Err: errors.Join(err, e)} return } cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() if e := d.run(cleanup, managedZapretScript, "start"); e != nil { err = &managedRecoveryFatal{Err: errors.Join(err, e)} } } }() stop := func() error { stopped = true // A partially failed stop also needs restoration. return d.run(ctx, managedZapretScript, "stop") } apply := func(ids []int64) error { if err := ctx.Err(); err != nil { return err } cleanupReload := m.Reload first := true m.Reload = func() error { if !first { return cleanupReload() } first = false activation, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() if err := d.run(activation, managedZapretScript, "restart"); err != nil { return &managedRecoveryFatal{Err: err} } return nil } defer func() { m.Reload = cleanupReload }() e := m.Apply(d.db, ids) if e == nil { stopped = false } var rollback *managedRollbackError if errors.As(e, &rollback) { return &managedRecoveryFatal{Err: e} } var rejection *managedHealthRejection if e != nil && !errors.As(e, &rejection) && !errors.Is(e, context.Canceled) && !errors.Is(e, context.DeadlineExceeded) { return &managedRecoveryFatal{Err: e} } return e } healthy := false if result.FrontendID != 0 { status, e := d.health(ctx) var fatal *managedRecoveryFatal if errors.As(e, &fatal) || errors.Is(e, context.Canceled) { return result, e } if err := ctx.Err(); err != nil { return result, err } // A bounded Player request timeout is unhealthy, not job cancellation. healthy = e == nil && status == "OK" } if !healthy { if e := stop(); e != nil { return result, e } // Both bootstrap and live recovery check saved definitions before scanning. saved, e := qualifiedCandidateIDs(d.db, "frontend") if e != nil { return result, &managedRecoveryFatal{Err: e} } for _, id := range saved { c, e := readStrategyCandidate(d.db, id) if e != nil { return result, &managedRecoveryFatal{Err: e} } inScope := false for _, endpoint := range c.Endpoints { inScope = inScope || endpoint == managedFrontendHost || strings.HasSuffix(managedFrontendHost, "."+endpoint) } if !inScope { return result, &managedRecoveryFatal{Err: fmt.Errorf("saved frontend scope excludes Player API host")} } _, e = managedRecheck(ctx, d.check, c.Args, "frontend", managedFrontendHost) if errors.Is(e, errCandidateNonpass) { continue } if e != nil { return result, e } e = apply([]int64{id}) if e == nil { result.FrontendID, healthy = id, true break } var fatal *managedRecoveryFatal var rejection *managedHealthRejection if errors.As(e, &fatal) || !errors.As(e, &rejection) { return result, e } // Health rollback restarted the prior config: stop before rechecking. if e := stop(); e != nil { return result, e } } if !healthy { id, e := managedScanCandidate(ctx, d, "frontend", managedFrontendHost) if e != nil { return result, e } if e := apply([]int64{id}); e != nil { return result, e } result.FrontendID = id } } // A frontend-only rollback boundary prevents cache failures undoing bootstrap. current, e := m.ActiveCandidates(d.db) if e != nil { return result, e } if len(current) != 1 || current[0] != result.FrontendID { if e := apply([]int64{result.FrontendID}); e != nil { return result, e } } host, e := d.discover(ctx) if ctx.Err() != nil { return result, ctx.Err() } if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") { result.GGCError = fmt.Errorf("exact GGC discovery failed") return result, nil } if e := stop(); e != nil { return result, e } args, report, e := revalidateGooglevideo(ctx, d.db, host, io.Discard, d.check) var cacheID int64 if e == nil && args != "" { e = saveGooglevideoRun(d.db, time.Now(), host, args, report, nil) if e != nil { return result, &managedRecoveryFatal{Err: e} } if e == nil { cacheID, e = newestManagedCandidateForScope(d.db, "googlevideo", host, args) if e != nil { return result, &managedRecoveryFatal{Err: e} } } } else if e == nil { cacheID, e = managedScanCandidate(ctx, d, "googlevideo", host) } if e == nil { e = apply([]int64{result.FrontendID, cacheID}) } if e != nil { var fatal *managedRecoveryFatal if errors.As(e, &fatal) { return result, e } if ctx.Err() != nil { return result, ctx.Err() } result.GGCError = fmt.Errorf("GGC validation or activation unresolved") return result, nil } result.GooglevideoID = cacheID return result, nil } func newestManagedCandidate(db *sql.DB, group string) (int64, error) { ids, err := qualifiedCandidateIDs(db, group) if err != nil { return 0, err } if len(ids) == 0 { return 0, fmt.Errorf("no qualified candidate") } return ids[0], nil } func newestManagedCandidateForScope(db *sql.DB, group, host, args string) (int64, error) { id, err := newestManagedCandidate(db, group) if err != nil { return 0, err } c, err := readStrategyCandidate(db, id) if err != nil { return 0, err } if c.Args != args || len(c.Endpoints) != 1 || c.Endpoints[0] != host { return 0, fmt.Errorf("latest saved candidate scope changed") } return id, nil } func managedRecheck(ctx context.Context, check cacheCheck, args, group, host string) ([]byte, error) { if err := ctx.Err(); err != nil { return nil, err } f, err := os.CreateTemp("", "blockcheckw-managed-*.txt") if err != nil { return nil, err } path := f.Name() if err := f.Close(); err != nil { return nil, err } return revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, check) } // Reuses proposal selection and transactional persistence with caller context. func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, host string) (int64, error) { if err := ctx.Err(); err != nil { return 0, err } started := time.Now() path, err := d.scan(ctx, "blockcheckw", host, io.Discard, io.Discard) var report []byte var args, output string if err == nil { var info os.FileInfo info, err = os.Stat(path + ".check.json") // ponytail: whole-second mtime permits same-second artifacts from injected // scans; production uses unique paths. Require that contract if precision grows. if err == nil && (!info.Mode().IsRegular() || info.ModTime().Before(started.Truncate(time.Second))) { err = fmt.Errorf("missing fresh scan shortlist") } } if err == nil { report, err = os.ReadFile(path + ".check.json") } if err == nil { args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, host) } if err == nil { report, err = revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, d.check) } if err == nil { err = ctx.Err() } if group == "frontend" { if err == nil { output, err = combinedBlockcheckOutputForHost(bytes.NewReader(report), args, host) } if e := saveFrontendRunForHost(d.db, started, args, output, report, err, host); e != nil { return 0, &managedRecoveryFatal{Err: e} } } else { var safe error if err != nil { safe = fmt.Errorf("cache scan or verification failed") } if e := saveGooglevideoRun(d.db, started, host, args, report, safe); e != nil { return 0, &managedRecoveryFatal{Err: e} } } if err != nil { return 0, err } id, err := newestManagedCandidateForScope(d.db, group, host, args) if err != nil { return 0, &managedRecoveryFatal{Err: err} } return id, nil } // Installs callbacks. runManagedMonitor performs startup // under its singleton. Target guards are repeated before every recovery/probe. func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDeps, reportGGC func(error)) error { if monitor == nil || monitor.db == nil || monitor.db != recovery.db || recovery.transaction == nil || reportGGC == nil || recovery.health == nil || recovery.run == nil || recovery.scan == nil || recovery.check == nil || recovery.discover == nil { return fmt.Errorf("matching monitor/recovery database and hooks required") } if recovery.transaction.Dir != monitor.target.directory || recovery.transaction.Name != "config" { return fmt.Errorf("recovery transaction must own the monitor target config") } guard := func() error { if _, err := managedRuntimeGuard(monitor.target); err != nil { return err } return managedCheckIdentity(recovery.transaction) } callback := func(ctx context.Context) error { if err := guard(); err != nil { return &managedRecoveryFatal{Err: err} } result, err := recoverManagedTarget(ctx, recovery) if result.GGCError != nil { reportGGC(result.GGCError) } // The supervisor reports ordinary recovery failures and retries only // after its durable cooldown. Safety failures (rollback/cleanup/drift) // are already typed managedRecoveryFatal by the transaction boundary. return err } monitor.bootstrap = func(ctx context.Context) error { if err := guard(); err != nil { return &managedRecoveryFatal{Err: err} } // Reconcile once at startup, never adopt a pending live deployment. transaction := *recovery.transaction transaction.Health = func() (string, error) { return recovery.health(ctx) } transaction.Reload = func() error { cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() return recovery.run(cleanup, managedZapretScript, "restart") } if err := transaction.Reconcile(recovery.db); err != nil { return &managedRecoveryFatal{Err: err} } ids, err := transaction.ActiveCandidates(recovery.db) if err != nil { return &managedRecoveryFatal{Err: err} } for _, id := range ids { candidate, err := readStrategyCandidate(recovery.db, id) if err != nil { return &managedRecoveryFatal{Err: err} } if candidate.Group == "frontend" && candidate.Qualified { return nil } } return callback(ctx) } monitor.recover = callback monitor.probe = func(ctx context.Context) error { if err := guard(); err != nil { return &managedRecoveryFatal{Err: err} } status, err := recovery.health(ctx) if err != nil { return err } if status != "OK" { return &managedHealthRejection{Status: status} } return nil } return nil }