Files
zapret-autotune/cmd/zapret-autotune/managed_recovery.go
T
ZeusinaandGPT-6 Bot e270810f33 refactor: execute adapter plans through shared core
Execute the private YouTube qualification plan through core-owned scheduling while retaining worker cleanup, proof sealing and legacy policy boundaries. Preserve unsafe collector failures alongside concurrent cancellation or deadlines.

Send bounded Player JSON through stdin instead of argv and update all shared runner callers. Generic controller cutover and remote runtime acceptance remain pending.

Co-authored-by: GPT-6 <gpt-6@openai.com>
2026-10-15 17:25:12 +03:00

817 lines
27 KiB
Go

package main
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"strings"
"syscall"
"time"
)
const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2"
const managedRecoveryJobTimeout = 90 * time.Minute
// Caller owns startup reconciliation and the monitor singleton. Discover and
// Health must bind to the guarded target SOCKS. No CLI parsing or remote work.
type managedRecoveryDeps struct {
db *sql.DB
transaction *managedTransaction
scan cacheScan
check cacheCheck
discover func(context.Context) (string, error)
health func(context.Context) (string, error)
run func(context.Context, string, ...string) error
jobTimeout time.Duration // Zero uses the production budget; tests may shorten it.
service bool
frontHealth func(context.Context) (string, error)
resources func(context.Context) (managedServiceInventory, error)
// Verify owns temporary raw-config activation and must join/restore before
// returning. It must not touch permanent config or the transaction journal.
verifyAssets func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error)
adaptive bool
mediaSamples func(context.Context) ([]mediaResource, error)
verifyGooglevideo func(context.Context, storedCandidate, []mediaResource) (GooglevideoQualificationProof, error)
recheckExperimental func(context.Context, storedCandidate, mediaResource, *managedTransaction) error
recheckGooglevideo func(context.Context, storedCandidate, []mediaResource, *managedTransaction) error
}
// Target-local adapter: reuses bounded Player requests without invoking a CLI.
// The injected command runner receives only fixed script/argv from orchestration.
func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, check cacheCheck, run func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
d := managedRecoveryDeps{db: db, scan: scan, check: check, run: run}
if _, err := requiredYoutubeAdapter(); err != nil {
return d, err
}
if target.lookup == nil || target.player == nil || db == nil || scan == nil || check == nil || run == nil {
return d, fmt.Errorf("complete target-local recovery adapter dependencies required")
}
if _, err := managedRuntimeGuard(target); err != nil {
return d, err
}
cfg, err := parseManagedHealthConfig(target.lookup)
if err != nil {
return d, err
}
d.transaction = &managedTransaction{Dir: target.directory, Name: "config"}
requestVideo := func(ctx context.Context, videoID string) ([]byte, error) {
if _, err := requiredYoutubeAdapter(); err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
if _, err := managedRuntimeGuard(target); err != nil {
return nil, &managedRecoveryFatal{Err: err}
}
bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
defer cancel()
body, err := requestPlayer(videoID, cfg.SOCKSProxy, func(_ context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error {
return target.player(bounded, args, stdin, out, diagnostics)
})
if bounded.Err() != nil {
return nil, managedRecoveryContextError(ctx, errors.Join(err, bounded.Err()))
}
return body, err
}
request := func(ctx context.Context) ([]byte, error) { return requestVideo(ctx, cfg.VideoID) }
mode, videoIDs, err := managedGooglevideoConfig(target.lookup, cfg.VideoID)
if err != nil {
return d, err
}
d.adaptive = mode == QualificationAdaptive
if d.adaptive {
d.mediaSamples = managedGooglevideoSamples(videoIDs, requestVideo)
}
d.health = func(ctx context.Context) (string, error) {
body, err := request(ctx)
if err != nil {
return "", err
}
var response struct {
Playability struct {
Status string `json:"status"`
} `json:"playabilityStatus"`
}
if err := json.Unmarshal(body, &response); err != nil {
return "", fmt.Errorf("malformed Player API health response")
}
if response.Playability.Status == "" {
return "", fmt.Errorf("missing Player API health status")
}
if response.Playability.Status != "OK" {
return response.Playability.Status, nil
}
if err := playerHealth(body); err != nil {
return "", err
}
return "OK", nil
}
d.discover = func(ctx context.Context) (string, error) {
body, err := request(ctx)
if err != nil {
return "", err
}
return playerHostname(body)
}
d.service, err = managedServiceEnabled(target.lookup)
if err != nil {
return d, err
}
if d.service {
d.frontHealth = d.health
// Construction is side-effect free; experiments may seek representative
// qualification without changing the legacy default exact-host mode.
d.mediaSamples = managedGooglevideoSamples(videoIDs, requestVideo)
d.resources = managedServiceResources(cfg, request, target.player)
d.health = managedServiceHealth(cfg.SOCKSProxy, d.frontHealth, d.resources, runResourceCurl)
d.verifyAssets = unwiredManagedAssetVerifier
}
return d, nil
}
type managedRecoveryFatal struct{ Err error }
// Only clean job expiration is restartable; safety failures retain their type.
type managedRecoveryTimeout struct{ Err error }
func (e *managedRecoveryTimeout) Error() string { return e.Err.Error() }
func (e *managedRecoveryTimeout) Unwrap() error { return e.Err }
var (
errNoQualifiedStrategy = errors.New("no fully passing check strategy for both tls12 and tls13")
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
errManagedRecoveryDeadline = errors.New("managed recovery total deadline exceeded")
)
// Validate every leaf, not just errors.Is: a joined cleanup failure must never
// authorize a restart. Callers must separately prove the job's own deadline.
func managedRecoveryTimeoutSafe(err error) bool {
if err == nil {
return true
}
switch e := err.(type) {
case *managedRecoveryFatal:
return false
case *exec.ExitError:
if e.ProcessState == nil {
return false
}
status, ok := e.Sys().(syscall.WaitStatus)
return ok && status.Signaled() && (status.Signal() == syscall.SIGTERM || status.Signal() == syscall.SIGKILL)
case interface{ Unwrap() []error }:
children := e.Unwrap()
if len(children) == 0 {
return false
}
for _, child := range children {
if !managedRecoveryTimeoutSafe(child) {
return false
}
}
return true
case interface{ Unwrap() error }:
child := e.Unwrap()
return child != nil && managedRecoveryTimeoutSafe(child)
default:
return err == context.Canceled || err == context.DeadlineExceeded || err == errManagedRecoveryDeadline
}
}
// Cancellation must prevent late success without erasing a concurrent safety
// or cleanup failure. Request-local timeouts alone leave the job context live.
func managedRecoveryContextError(ctx context.Context, err error) error {
if ctx.Err() == nil {
return err
}
joined := errors.Join(err, ctx.Err())
if !managedRecoveryTimeoutSafe(err) {
return &managedRecoveryFatal{Err: joined}
}
return joined
}
// Synchronous callbacks own cancellation cleanup/join. Classify expiration only
// after they return, so the monitor cannot release ownership before restoration.
func managedRecoveryJob(ctx context.Context, budget time.Duration, job func(context.Context) error) error {
if budget == 0 {
budget = managedRecoveryJobTimeout
}
jobCtx, cancel := context.WithTimeoutCause(ctx, budget, errManagedRecoveryDeadline)
defer cancel()
err := job(jobCtx)
if errors.Is(context.Cause(jobCtx), errManagedRecoveryDeadline) {
joined := errors.Join(err, errManagedRecoveryDeadline, context.DeadlineExceeded)
if !managedRecoveryTimeoutSafe(err) {
return &managedRecoveryFatal{Err: joined}
}
return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: joined}}
}
if err == nil {
return jobCtx.Err()
}
return err
}
func (e *managedRecoveryFatal) Error() string {
return "managed recovery safety failure: " + e.Err.Error()
}
func (e *managedRecoveryFatal) Unwrap() error { return e.Err }
type managedRecoveryResult struct {
FrontendID, GooglevideoID int64
GGCError error // Unresolved; never a playback/Player API health claim.
}
func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result managedRecoveryResult, err error) {
if d.db == nil || d.transaction == nil || d.scan == nil || d.check == nil || d.discover == nil || d.health == nil || d.run == nil {
return result, fmt.Errorf("complete managed recovery dependencies required")
}
if err := ctx.Err(); err != nil {
return result, err
}
m := *d.transaction
// Rollback must restart even after job cancellation; independently bounded.
var cleanupCtx context.Context
var cleanupCancel context.CancelFunc
cleanupContext := func() context.Context {
if cleanupCtx == nil {
cleanupCtx, cleanupCancel = context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
}
return cleanupCtx
}
defer func() {
if cleanupCancel != nil {
cleanupCancel()
}
}()
m.Reload = func() error {
if err := d.run(cleanupContext(), managedZapretScript, "restart"); err != nil {
return &managedRecoveryFatal{Err: err}
}
return nil
}
m.Health = func() (string, error) {
health := d.health
if d.service && d.frontHealth != nil {
health = d.frontHealth
}
// Commit frontend/GGC progress independently; the monitor's overall
// service probe still requires media and both images before READY.
status, err := health(ctx)
if ctx.Err() != nil {
return "", managedRecoveryContextError(ctx, err)
}
return status, err
}
// Nested asset/GGC recovery reads the same transaction before binding its
// apply-specific hooks. Production adapters start with path-only transactions.
d.transaction = &m
if d.service {
_, healthErr := d.health(ctx)
var fatal *managedRecoveryFatal
var failure *managedResourceFailure
j, e := m.load()
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
if j.Scope == GooglevideoScopeExperimental {
healthErr = managedResourceHealthError(ctx, healthErr)
}
if ctx.Err() != nil || errors.As(healthErr, &fatal) || errors.Is(healthErr, context.Canceled) || errors.Is(healthErr, context.DeadlineExceeded) {
return result, managedRecoveryContextError(ctx, healthErr)
}
// A previously authorized native suffix stays adaptive after promotion,
// even if the operator's legacy environment still says "cold". Stored
// validation permits reconstruction, never fresh proof authorization.
if j.Scope == GooglevideoScopeSuffix {
for _, id := range j.Candidates {
c, err := readStrategyCandidatePolicy(d.db, id, false)
if err != nil {
return result, &managedRecoveryFatal{Err: err}
}
if c.Group == "googlevideo" && c.Mode == QualificationAdaptive && c.Qualified {
d.adaptive = true
}
}
}
if errors.As(healthErr, &failure) && assetEndpoint(failure.Host) && j.Scope != GooglevideoScopeExperimental && !d.adaptive {
return result, repairManagedAssets(ctx, d, failure.Host)
}
}
ids, err := m.ActiveCandidates(d.db)
if err != nil {
return result, &managedRecoveryFatal{Err: err}
}
journal, err := m.load()
if err != nil {
return result, &managedRecoveryFatal{Err: err}
}
experimental := journal.Scope == GooglevideoScopeExperimental
for _, id := range ids {
c, e := readStrategyCandidate(d.db, id)
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
if c.Group == "frontend" {
result.FrontendID = id
}
}
stopped := false
defer func() {
if stopped {
// Never start bytes left pending by a failed rollback. Reconcile restores
// the last committed set or fails closed without activating unknown bytes.
if e := m.Reconcile(d.db); e != nil {
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
return
}
if e := d.run(cleanupContext(), managedZapretScript, "start"); e != nil {
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
}
}
}()
stop := func() error {
stopped = true // A partially failed stop also needs restoration.
return d.run(ctx, managedZapretScript, "stop")
}
apply := func(ids []int64) error {
if err := ctx.Err(); err != nil {
return err
}
if d.service || experimental {
current, e := m.ActiveCandidates(d.db)
if e != nil {
return &managedRecoveryFatal{Err: e}
}
ids, e = managedFullDesiredSet(d.db, current, ids)
if e != nil {
return &managedRecoveryFatal{Err: e}
}
}
cleanupReload := m.Reload
first := true
m.Reload = func() error {
if !first {
return cleanupReload()
}
first = false
activation, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if err := d.run(activation, managedZapretScript, "restart"); err != nil {
return &managedRecoveryFatal{Err: err}
}
return nil
}
defer func() { m.Reload = cleanupReload }()
defer func() {
// A rejected candidate's rollback is complete before the next attempt;
// that later attempt needs its own restoration reserve.
if ctx.Err() == nil && cleanupCancel != nil {
cleanupCancel()
cleanupCtx, cleanupCancel = nil, nil
}
}()
e := m.Apply(d.db, ids)
if e == nil {
stopped = false
}
var rollback *managedRollbackError
if errors.As(e, &rollback) {
return &managedRecoveryFatal{Err: e}
}
var rejection *managedHealthRejection
if e != nil && !errors.As(e, &rejection) && !errors.Is(e, context.Canceled) && !errors.Is(e, context.DeadlineExceeded) {
return &managedRecoveryFatal{Err: e}
}
return e
}
healthy := false
if result.FrontendID != 0 {
frontHealth := d.health
if d.service && d.frontHealth != nil {
frontHealth = d.frontHealth
}
status, e := frontHealth(ctx)
e = managedRecoveryContextError(ctx, e)
var fatal *managedRecoveryFatal
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
return result, e
}
if err := ctx.Err(); err != nil {
return result, e
}
// A bounded Player request timeout is unhealthy, not job cancellation.
healthy = e == nil && status == "OK"
}
if !healthy {
if e := stop(); e != nil {
return result, e
}
// Both bootstrap and live recovery check saved definitions before scanning.
saved, e := qualifiedCandidateIDs(d.db, "frontend")
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
for _, id := range saved {
c, e := readStrategyCandidate(d.db, id)
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
inScope := false
for _, endpoint := range c.Endpoints {
inScope = inScope || endpoint == managedFrontendHost || strings.HasSuffix(managedFrontendHost, "."+endpoint)
}
if !inScope {
return result, &managedRecoveryFatal{Err: fmt.Errorf("saved frontend scope excludes Player API host")}
}
_, e = managedRecheck(ctx, d.check, c.Args, "frontend", managedFrontendHost)
if errors.Is(e, errCandidateNonpass) {
continue
}
if e != nil {
return result, e
}
e = apply([]int64{id})
if e == nil {
result.FrontendID, healthy = id, true
break
}
var fatal *managedRecoveryFatal
var rejection *managedHealthRejection
if errors.As(e, &fatal) || !errors.As(e, &rejection) {
return result, e
}
// Health rollback restarted the prior config: stop before rechecking.
if e := stop(); e != nil {
return result, e
}
}
if !healthy {
id, e := managedScanCandidate(ctx, d, "frontend", managedFrontendHost)
if e != nil {
return result, e
}
if e := apply([]int64{id}); e != nil {
return result, e
}
result.FrontendID = id
}
}
// A frontend-only rollback boundary prevents cache failures undoing bootstrap.
current, e := m.ActiveCandidates(d.db)
if e != nil {
return result, e
}
if experimental {
result.GooglevideoID, result.GGCError = recoverManagedExperimental(ctx, d)
return result, result.GGCError
}
if !d.service && (len(current) != 1 || current[0] != result.FrontendID) {
if e := apply([]int64{result.FrontendID}); e != nil {
return result, e
}
}
if d.adaptive {
result.GooglevideoID, result.GGCError = recoverManagedGooglevideo(ctx, d)
var fatal *managedRecoveryFatal
if errors.As(result.GGCError, &fatal) || ctx.Err() != nil {
return result, result.GGCError
}
return result, nil
}
host, e := d.discover(ctx)
e = managedRecoveryContextError(ctx, e)
var discoveryFatal *managedRecoveryFatal
if errors.As(e, &discoveryFatal) {
return result, e
}
if ctx.Err() != nil {
return result, e
}
if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
result.GGCError = fmt.Errorf("exact GGC discovery failed")
return result, nil
}
if e := stop(); e != nil {
return result, e
}
args, report, e := revalidateGooglevideo(ctx, d.db, host, io.Discard, d.check)
var cacheID int64
if e == nil && args != "" {
e = saveGooglevideoRun(d.db, time.Now(), host, args, report, nil)
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
if e == nil {
cacheID, e = newestManagedCandidateForScope(d.db, "googlevideo", host, args)
if e != nil {
return result, &managedRecoveryFatal{Err: e}
}
}
} else if e == nil {
cacheID, e = managedScanCandidate(ctx, d, "googlevideo", host)
}
if e == nil {
e = apply([]int64{result.FrontendID, cacheID})
}
if e != nil {
e = managedRecoveryContextError(ctx, e)
var fatal *managedRecoveryFatal
if errors.As(e, &fatal) {
return result, e
}
if ctx.Err() != nil {
return result, e
}
if errors.Is(e, errNoQualifiedStrategy) {
return result, e
}
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
return result, nil
}
result.GooglevideoID = cacheID
return result, nil
}
func newestManagedCandidate(db *sql.DB, group string) (int64, error) {
ids, err := qualifiedCandidateIDs(db, group)
if err != nil {
return 0, err
}
if len(ids) == 0 {
return 0, fmt.Errorf("no qualified candidate")
}
return ids[0], nil
}
func newestManagedCandidateForScope(db *sql.DB, group, host, args string) (int64, error) {
id, err := newestManagedCandidate(db, group)
if err != nil {
return 0, err
}
c, err := readStrategyCandidate(db, id)
if err != nil {
return 0, err
}
if c.Args != args || len(c.Endpoints) != 1 || c.Endpoints[0] != host {
return 0, fmt.Errorf("latest saved candidate scope changed")
}
return id, nil
}
func managedRecheck(ctx context.Context, check cacheCheck, args, group, host string) ([]byte, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
f, err := os.CreateTemp("", "blockcheckw-managed-*.txt")
if err != nil {
return nil, err
}
path := f.Name()
if err := f.Close(); err != nil {
return nil, err
}
return revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, check)
}
// Reuses proposal selection and transactional persistence with caller context.
func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, host string) (int64, error) {
if err := ctx.Err(); err != nil {
return 0, err
}
started := time.Now()
var diagnostics bytes.Buffer
path, err := d.scan(ctx, "blockcheckw", host, io.Discard, &diagnostics)
// Only validated report evidence, never a runner error, authorizes absence.
if errors.Is(err, errNoQualifiedStrategy) || errors.Is(err, errCandidateNonpass) {
err = fmt.Errorf("candidate scanner failed: %v", err)
}
err = managedCheckerError(err, path+".check.json.stderr", diagnostics.Bytes())
var report []byte
var args, output string
if err == nil {
var info os.FileInfo
info, err = os.Stat(path + ".check.json")
// ponytail: whole-second mtime permits same-second artifacts from injected
// scans; production uses unique paths. Require that contract if precision grows.
if err == nil && (!info.Mode().IsRegular() || info.ModTime().Before(started.Truncate(time.Second))) {
err = fmt.Errorf("missing fresh scan shortlist")
}
}
if err == nil {
report, err = os.ReadFile(path + ".check.json")
}
if err == nil {
args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, host)
if err != nil && emptyManagedScanReport(report, host) {
err = errNoQualifiedStrategy
}
}
if err == nil {
report, err = revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, d.check)
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) && errors.Is(err, errCandidateNonpass) {
err = errNoQualifiedStrategy
}
}
if err == nil {
err = ctx.Err()
}
if group == "frontend" {
if err == nil {
output, err = combinedBlockcheckOutputForHost(bytes.NewReader(report), args, host)
}
if e := saveFrontendRunForHost(d.db, started, args, output, report, err, host); e != nil {
return 0, &managedRecoveryFatal{Err: e}
}
} else {
var safe error
if err != nil {
safe = fmt.Errorf("cache scan or verification failed")
}
if e := saveGooglevideoRun(d.db, started, host, args, report, safe); e != nil {
return 0, &managedRecoveryFatal{Err: e}
}
}
if err != nil {
return 0, err
}
id, err := newestManagedCandidateForScope(d.db, group, host, args)
if err != nil {
return 0, &managedRecoveryFatal{Err: err}
}
return id, nil
}
// A successful owned scan may omit rejected rows. Missing/null metadata is not
// absence evidence; qualification still requires the usual full TLS checks.
func emptyManagedScanReport(raw []byte, host string) bool {
var report struct {
Domain string `json:"domain"`
Total *int `json:"total"`
Working *int `json:"working"`
Strategies *[]json.RawMessage `json:"strategies"`
}
d := json.NewDecoder(bytes.NewReader(raw))
return d.Decode(&report) == nil && d.Decode(new(json.RawMessage)) == io.EOF &&
report.Domain == host && report.Total != nil && *report.Total > 0 &&
report.Working != nil && *report.Working == 0 && report.Strategies != nil && len(*report.Strategies) == 0
}
// Installs callbacks. runManagedMonitor performs startup
// under its singleton. Target guards are repeated before every recovery/probe.
func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDeps, reportGGC func(error)) error {
if monitor == nil || monitor.db == nil || monitor.db != recovery.db || recovery.transaction == nil || reportGGC == nil || recovery.health == nil || recovery.run == nil || recovery.scan == nil || recovery.check == nil || recovery.discover == nil {
return fmt.Errorf("matching monitor/recovery database and hooks required")
}
if recovery.transaction.Dir != monitor.target.directory || recovery.transaction.Name != "config" {
return fmt.Errorf("recovery transaction must own the monitor target config")
}
guard := func() error {
if _, err := managedRuntimeGuard(monitor.target); err != nil {
return err
}
return managedCheckIdentity(recovery.transaction)
}
callback := func(ctx context.Context) error {
if err := guard(); err != nil {
return &managedRecoveryFatal{Err: err}
}
result, err := recoverManagedTarget(ctx, recovery)
if result.GGCError != nil {
reportGGC(result.GGCError)
if recovery.adaptive && err == nil {
return result.GGCError
}
}
// The supervisor reports ordinary recovery failures and retries only
// after its durable cooldown. Safety failures (rollback/cleanup/drift)
// are already typed managedRecoveryFatal by the transaction boundary.
return err
}
bootstrap := func(ctx context.Context) error {
if err := guard(); err != nil {
return &managedRecoveryFatal{Err: err}
}
// Reconcile once at startup, never adopt a pending live deployment.
transaction := *recovery.transaction
transaction.Health = func() (string, error) { return recovery.health(ctx) }
transaction.Reload = func() error {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
return recovery.run(cleanup, managedZapretScript, "restart")
}
if err := transaction.Reconcile(recovery.db); err != nil {
return &managedRecoveryFatal{Err: err}
}
state, err := readMonitorState(monitor.db, monitor.serviceID)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
// Reconciliation and identity guards precede even a paused bootstrap.
if state.RecoveryRunning || monitor.now().Before(state.CooldownUntil) {
return nil
}
ids, err := transaction.ActiveCandidates(recovery.db)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
for _, id := range ids {
candidate, err := readStrategyCandidate(recovery.db, id)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
if candidate.Group == "frontend" && candidate.Qualified {
return nil
}
}
return callback(ctx)
}
monitor.bootstrap = func(ctx context.Context) error {
return managedRecoveryJob(ctx, recovery.jobTimeout, bootstrap)
}
monitor.recover = func(ctx context.Context) error {
return managedRecoveryJob(ctx, recovery.jobTimeout, callback)
}
monitor.probe = func(ctx context.Context) error {
if err := guard(); err != nil {
return &managedRecoveryFatal{Err: err}
}
transaction := *recovery.transaction
transaction.Health = func() (string, error) { return recovery.health(ctx) }
// ActiveCandidates is read-only; never permit a probe to reload config.
transaction.Reload = func() error { return fmt.Errorf("probe cannot reload config") }
ids, err := transaction.ActiveCandidates(recovery.db)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
// Keep Player API health separate from GGC scope (not playback health).
journal, err := transaction.load()
if err != nil {
return &managedRecoveryFatal{Err: err}
}
status, err := recovery.health(ctx)
if journal.Scope == GooglevideoScopeExperimental && recovery.service && recovery.recheckExperimental != nil {
err = managedResourceHealthError(ctx, err)
} else {
err = managedRecoveryContextError(ctx, err)
}
if err != nil {
return err
}
if status != "OK" {
return &managedHealthRejection{Status: status}
}
if journal.Scope == GooglevideoScopeExperimental {
if !recovery.service || recovery.recheckExperimental == nil {
return errManagedGGCUnresolved
}
// Operational service health is not family qualification.
reportGGC(errManagedExperimentalUnqualified)
return nil
}
// Two bounded Player requests reuse the existing health/discovery adapters.
host, err := recovery.discover(ctx)
err = managedRecoveryContextError(ctx, err)
if ctx.Err() != nil {
return err
}
var fatal *managedRecoveryFatal
if errors.As(err, &fatal) || errors.Is(err, context.Canceled) {
return err
}
if err != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
return errManagedGGCUnresolved
}
for _, id := range ids {
candidate, err := readStrategyCandidate(recovery.db, id)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
if candidate.Group == "googlevideo" && candidate.Mode == QualificationAdaptive && candidate.Qualified {
journal, e := transaction.load()
if e != nil {
return &managedRecoveryFatal{Err: e}
}
if journal.Scope == GooglevideoScopeSuffix {
return nil
}
}
if candidate.Group == "googlevideo" && len(candidate.Endpoints) == 1 && candidate.Endpoints[0] == host {
return nil
}
}
return errManagedGGCScopeChanged
}
return nil
}