Keep operator-owned native circular extrapolation distinct from cold/adaptive qualification and playback readiness. Preserve qualified frontend/assets, use guarded journal transactions and rollback, and apply private startup requests before the owned monitor begins. Retain legacy rendering and qualification gates. Co-authored-by: GPT-6 <gpt-6@openai.com>
778 lines
25 KiB
Go
778 lines
25 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2"
|
|
|
|
const managedRecoveryJobTimeout = 90 * time.Minute
|
|
|
|
// Caller owns startup reconciliation and the monitor singleton. Discover and
|
|
// Health must bind to the guarded target SOCKS. No CLI parsing or remote work.
|
|
type managedRecoveryDeps struct {
|
|
db *sql.DB
|
|
transaction *managedTransaction
|
|
scan cacheScan
|
|
check cacheCheck
|
|
discover func(context.Context) (string, error)
|
|
health func(context.Context) (string, error)
|
|
run func(context.Context, string, ...string) error
|
|
jobTimeout time.Duration // Zero uses the production budget; tests may shorten it.
|
|
service bool
|
|
frontHealth func(context.Context) (string, error)
|
|
resources func(context.Context) (managedServiceInventory, error)
|
|
// Verify owns temporary raw-config activation and must join/restore before
|
|
// returning. It must not touch permanent config or the transaction journal.
|
|
verifyAssets func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error)
|
|
adaptive bool
|
|
mediaSamples func(context.Context) ([]mediaResource, error)
|
|
verifyGooglevideo func(context.Context, storedCandidate, []mediaResource) (GooglevideoQualificationProof, error)
|
|
}
|
|
|
|
// Target-local adapter: reuses bounded Player requests without invoking a CLI.
|
|
// The injected command runner receives only fixed script/argv from orchestration.
|
|
func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, check cacheCheck, run func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
|
|
d := managedRecoveryDeps{db: db, scan: scan, check: check, run: run}
|
|
if target.lookup == nil || target.player == nil || db == nil || scan == nil || check == nil || run == nil {
|
|
return d, fmt.Errorf("complete target-local recovery adapter dependencies required")
|
|
}
|
|
if _, err := managedRuntimeGuard(target); err != nil {
|
|
return d, err
|
|
}
|
|
cfg, err := parseHealthConfig(target.lookup)
|
|
if err != nil {
|
|
return d, err
|
|
}
|
|
d.transaction = &managedTransaction{Dir: target.directory, Name: "config"}
|
|
requestVideo := func(ctx context.Context, videoID string) ([]byte, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := managedRuntimeGuard(target); err != nil {
|
|
return nil, &managedRecoveryFatal{Err: err}
|
|
}
|
|
bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
|
|
defer cancel()
|
|
body, err := requestPlayer(videoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error {
|
|
return target.player(bounded, args, out, diagnostics)
|
|
})
|
|
if bounded.Err() != nil {
|
|
return nil, managedRecoveryContextError(ctx, errors.Join(err, bounded.Err()))
|
|
}
|
|
return body, err
|
|
}
|
|
request := func(ctx context.Context) ([]byte, error) { return requestVideo(ctx, cfg.VideoID) }
|
|
mode, videoIDs, err := managedGooglevideoConfig(target.lookup, cfg.VideoID)
|
|
if err != nil {
|
|
return d, err
|
|
}
|
|
d.adaptive = mode == QualificationAdaptive
|
|
if d.adaptive {
|
|
d.mediaSamples = managedGooglevideoSamples(videoIDs, requestVideo)
|
|
}
|
|
d.health = func(ctx context.Context) (string, error) {
|
|
body, err := request(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var response struct {
|
|
Playability struct {
|
|
Status string `json:"status"`
|
|
} `json:"playabilityStatus"`
|
|
}
|
|
if err := json.Unmarshal(body, &response); err != nil {
|
|
return "", fmt.Errorf("malformed Player API health response")
|
|
}
|
|
if response.Playability.Status == "" {
|
|
return "", fmt.Errorf("missing Player API health status")
|
|
}
|
|
if response.Playability.Status != "OK" {
|
|
return response.Playability.Status, nil
|
|
}
|
|
if err := playerHealth(body); err != nil {
|
|
return "", err
|
|
}
|
|
return "OK", nil
|
|
}
|
|
d.discover = func(ctx context.Context) (string, error) {
|
|
body, err := request(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return playerHostname(body)
|
|
}
|
|
d.service, err = managedServiceEnabled(target.lookup)
|
|
if err != nil {
|
|
return d, err
|
|
}
|
|
if d.service {
|
|
d.frontHealth = d.health
|
|
d.resources = managedServiceResources(cfg, request, target.player)
|
|
d.health = managedServiceHealth(cfg.SOCKSProxy, d.frontHealth, d.resources, runResourceCurl)
|
|
d.verifyAssets = unwiredManagedAssetVerifier
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
type managedRecoveryFatal struct{ Err error }
|
|
|
|
// Only clean job expiration is restartable; safety failures retain their type.
|
|
type managedRecoveryTimeout struct{ Err error }
|
|
|
|
func (e *managedRecoveryTimeout) Error() string { return e.Err.Error() }
|
|
func (e *managedRecoveryTimeout) Unwrap() error { return e.Err }
|
|
|
|
var (
|
|
errNoQualifiedStrategy = errors.New("no fully passing check strategy for both tls12 and tls13")
|
|
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
|
|
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
|
|
errManagedRecoveryDeadline = errors.New("managed recovery total deadline exceeded")
|
|
)
|
|
|
|
// Validate every leaf, not just errors.Is: a joined cleanup failure must never
|
|
// authorize a restart. Callers must separately prove the job's own deadline.
|
|
func managedRecoveryTimeoutSafe(err error) bool {
|
|
if err == nil {
|
|
return true
|
|
}
|
|
switch e := err.(type) {
|
|
case *managedRecoveryFatal:
|
|
return false
|
|
case *exec.ExitError:
|
|
if e.ProcessState == nil {
|
|
return false
|
|
}
|
|
status, ok := e.Sys().(syscall.WaitStatus)
|
|
return ok && status.Signaled() && (status.Signal() == syscall.SIGTERM || status.Signal() == syscall.SIGKILL)
|
|
case interface{ Unwrap() []error }:
|
|
children := e.Unwrap()
|
|
if len(children) == 0 {
|
|
return false
|
|
}
|
|
for _, child := range children {
|
|
if !managedRecoveryTimeoutSafe(child) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
case interface{ Unwrap() error }:
|
|
child := e.Unwrap()
|
|
return child != nil && managedRecoveryTimeoutSafe(child)
|
|
default:
|
|
return err == context.Canceled || err == context.DeadlineExceeded || err == errManagedRecoveryDeadline
|
|
}
|
|
}
|
|
|
|
// Cancellation must prevent late success without erasing a concurrent safety
|
|
// or cleanup failure. Request-local timeouts alone leave the job context live.
|
|
func managedRecoveryContextError(ctx context.Context, err error) error {
|
|
if ctx.Err() == nil {
|
|
return err
|
|
}
|
|
joined := errors.Join(err, ctx.Err())
|
|
if !managedRecoveryTimeoutSafe(err) {
|
|
return &managedRecoveryFatal{Err: joined}
|
|
}
|
|
return joined
|
|
}
|
|
|
|
// Synchronous callbacks own cancellation cleanup/join. Classify expiration only
|
|
// after they return, so the monitor cannot release ownership before restoration.
|
|
func managedRecoveryJob(ctx context.Context, budget time.Duration, job func(context.Context) error) error {
|
|
if budget == 0 {
|
|
budget = managedRecoveryJobTimeout
|
|
}
|
|
jobCtx, cancel := context.WithTimeoutCause(ctx, budget, errManagedRecoveryDeadline)
|
|
defer cancel()
|
|
err := job(jobCtx)
|
|
if errors.Is(context.Cause(jobCtx), errManagedRecoveryDeadline) {
|
|
joined := errors.Join(err, errManagedRecoveryDeadline, context.DeadlineExceeded)
|
|
if !managedRecoveryTimeoutSafe(err) {
|
|
return &managedRecoveryFatal{Err: joined}
|
|
}
|
|
return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: joined}}
|
|
}
|
|
if err == nil {
|
|
return jobCtx.Err()
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (e *managedRecoveryFatal) Error() string {
|
|
return "managed recovery safety failure: " + e.Err.Error()
|
|
}
|
|
func (e *managedRecoveryFatal) Unwrap() error { return e.Err }
|
|
|
|
type managedRecoveryResult struct {
|
|
FrontendID, GooglevideoID int64
|
|
GGCError error // Unresolved; never a playback/Player API health claim.
|
|
}
|
|
|
|
func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result managedRecoveryResult, err error) {
|
|
if d.db == nil || d.transaction == nil || d.scan == nil || d.check == nil || d.discover == nil || d.health == nil || d.run == nil {
|
|
return result, fmt.Errorf("complete managed recovery dependencies required")
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return result, err
|
|
}
|
|
m := *d.transaction
|
|
// Rollback must restart even after job cancellation; independently bounded.
|
|
var cleanupCtx context.Context
|
|
var cleanupCancel context.CancelFunc
|
|
cleanupContext := func() context.Context {
|
|
if cleanupCtx == nil {
|
|
cleanupCtx, cleanupCancel = context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
|
}
|
|
return cleanupCtx
|
|
}
|
|
defer func() {
|
|
if cleanupCancel != nil {
|
|
cleanupCancel()
|
|
}
|
|
}()
|
|
m.Reload = func() error {
|
|
if err := d.run(cleanupContext(), managedZapretScript, "restart"); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
return nil
|
|
}
|
|
m.Health = func() (string, error) {
|
|
health := d.health
|
|
if d.service && d.frontHealth != nil {
|
|
health = d.frontHealth
|
|
}
|
|
// Commit frontend/GGC progress independently; the monitor's overall
|
|
// service probe still requires media and both images before READY.
|
|
status, err := health(ctx)
|
|
if ctx.Err() != nil {
|
|
return "", managedRecoveryContextError(ctx, err)
|
|
}
|
|
return status, err
|
|
}
|
|
// Nested asset/GGC recovery reads the same transaction before binding its
|
|
// apply-specific hooks. Production adapters start with path-only transactions.
|
|
d.transaction = &m
|
|
if d.service {
|
|
_, healthErr := d.health(ctx)
|
|
var fatal *managedRecoveryFatal
|
|
if ctx.Err() != nil || errors.As(healthErr, &fatal) || errors.Is(healthErr, context.Canceled) || errors.Is(healthErr, context.DeadlineExceeded) {
|
|
return result, managedRecoveryContextError(ctx, healthErr)
|
|
}
|
|
var failure *managedResourceFailure
|
|
if errors.As(healthErr, &failure) && assetEndpoint(failure.Host) {
|
|
return result, repairManagedAssets(ctx, d, failure.Host)
|
|
}
|
|
}
|
|
ids, err := m.ActiveCandidates(d.db)
|
|
if err != nil {
|
|
return result, &managedRecoveryFatal{Err: err}
|
|
}
|
|
journal, err := m.load()
|
|
if err != nil {
|
|
return result, &managedRecoveryFatal{Err: err}
|
|
}
|
|
experimental := journal.Scope == GooglevideoScopeExperimental
|
|
for _, id := range ids {
|
|
c, e := readStrategyCandidate(d.db, id)
|
|
if e != nil {
|
|
return result, &managedRecoveryFatal{Err: e}
|
|
}
|
|
if c.Group == "frontend" {
|
|
result.FrontendID = id
|
|
}
|
|
}
|
|
stopped := false
|
|
defer func() {
|
|
if stopped {
|
|
// Never start bytes left pending by a failed rollback. Reconcile restores
|
|
// the last committed set or fails closed without activating unknown bytes.
|
|
if e := m.Reconcile(d.db); e != nil {
|
|
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
|
return
|
|
}
|
|
if e := d.run(cleanupContext(), managedZapretScript, "start"); e != nil {
|
|
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
|
}
|
|
}
|
|
}()
|
|
stop := func() error {
|
|
stopped = true // A partially failed stop also needs restoration.
|
|
return d.run(ctx, managedZapretScript, "stop")
|
|
}
|
|
apply := func(ids []int64) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
if d.service || experimental {
|
|
current, e := m.ActiveCandidates(d.db)
|
|
if e != nil {
|
|
return &managedRecoveryFatal{Err: e}
|
|
}
|
|
ids, e = managedFullDesiredSet(d.db, current, ids)
|
|
if e != nil {
|
|
return &managedRecoveryFatal{Err: e}
|
|
}
|
|
}
|
|
cleanupReload := m.Reload
|
|
first := true
|
|
m.Reload = func() error {
|
|
if !first {
|
|
return cleanupReload()
|
|
}
|
|
first = false
|
|
activation, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
if err := d.run(activation, managedZapretScript, "restart"); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
return nil
|
|
}
|
|
defer func() { m.Reload = cleanupReload }()
|
|
defer func() {
|
|
// A rejected candidate's rollback is complete before the next attempt;
|
|
// that later attempt needs its own restoration reserve.
|
|
if ctx.Err() == nil && cleanupCancel != nil {
|
|
cleanupCancel()
|
|
cleanupCtx, cleanupCancel = nil, nil
|
|
}
|
|
}()
|
|
e := m.Apply(d.db, ids)
|
|
if e == nil {
|
|
stopped = false
|
|
}
|
|
var rollback *managedRollbackError
|
|
if errors.As(e, &rollback) {
|
|
return &managedRecoveryFatal{Err: e}
|
|
}
|
|
var rejection *managedHealthRejection
|
|
if e != nil && !errors.As(e, &rejection) && !errors.Is(e, context.Canceled) && !errors.Is(e, context.DeadlineExceeded) {
|
|
return &managedRecoveryFatal{Err: e}
|
|
}
|
|
return e
|
|
}
|
|
healthy := false
|
|
if result.FrontendID != 0 {
|
|
frontHealth := d.health
|
|
if d.service && d.frontHealth != nil {
|
|
frontHealth = d.frontHealth
|
|
}
|
|
status, e := frontHealth(ctx)
|
|
e = managedRecoveryContextError(ctx, e)
|
|
var fatal *managedRecoveryFatal
|
|
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
|
|
return result, e
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return result, e
|
|
}
|
|
// A bounded Player request timeout is unhealthy, not job cancellation.
|
|
healthy = e == nil && status == "OK"
|
|
}
|
|
if !healthy {
|
|
if e := stop(); e != nil {
|
|
return result, e
|
|
}
|
|
// Both bootstrap and live recovery check saved definitions before scanning.
|
|
saved, e := qualifiedCandidateIDs(d.db, "frontend")
|
|
if e != nil {
|
|
return result, &managedRecoveryFatal{Err: e}
|
|
}
|
|
for _, id := range saved {
|
|
c, e := readStrategyCandidate(d.db, id)
|
|
if e != nil {
|
|
return result, &managedRecoveryFatal{Err: e}
|
|
}
|
|
inScope := false
|
|
for _, endpoint := range c.Endpoints {
|
|
inScope = inScope || endpoint == managedFrontendHost || strings.HasSuffix(managedFrontendHost, "."+endpoint)
|
|
}
|
|
if !inScope {
|
|
return result, &managedRecoveryFatal{Err: fmt.Errorf("saved frontend scope excludes Player API host")}
|
|
}
|
|
_, e = managedRecheck(ctx, d.check, c.Args, "frontend", managedFrontendHost)
|
|
if errors.Is(e, errCandidateNonpass) {
|
|
continue
|
|
}
|
|
if e != nil {
|
|
return result, e
|
|
}
|
|
e = apply([]int64{id})
|
|
if e == nil {
|
|
result.FrontendID, healthy = id, true
|
|
break
|
|
}
|
|
var fatal *managedRecoveryFatal
|
|
var rejection *managedHealthRejection
|
|
if errors.As(e, &fatal) || !errors.As(e, &rejection) {
|
|
return result, e
|
|
}
|
|
// Health rollback restarted the prior config: stop before rechecking.
|
|
if e := stop(); e != nil {
|
|
return result, e
|
|
}
|
|
}
|
|
if !healthy {
|
|
id, e := managedScanCandidate(ctx, d, "frontend", managedFrontendHost)
|
|
if e != nil {
|
|
return result, e
|
|
}
|
|
if e := apply([]int64{id}); e != nil {
|
|
return result, e
|
|
}
|
|
result.FrontendID = id
|
|
}
|
|
}
|
|
// A frontend-only rollback boundary prevents cache failures undoing bootstrap.
|
|
current, e := m.ActiveCandidates(d.db)
|
|
if e != nil {
|
|
return result, e
|
|
}
|
|
if experimental {
|
|
// Operator extrapolation survives recovery in either configured mode.
|
|
// Frontend/asset repairs remain available, but cannot qualify this policy.
|
|
result.GGCError = errManagedGGCUnresolved
|
|
return result, result.GGCError
|
|
}
|
|
if !d.service && (len(current) != 1 || current[0] != result.FrontendID) {
|
|
if e := apply([]int64{result.FrontendID}); e != nil {
|
|
return result, e
|
|
}
|
|
}
|
|
if d.adaptive {
|
|
result.GooglevideoID, result.GGCError = recoverManagedGooglevideo(ctx, d)
|
|
var fatal *managedRecoveryFatal
|
|
if errors.As(result.GGCError, &fatal) || ctx.Err() != nil {
|
|
return result, result.GGCError
|
|
}
|
|
return result, nil
|
|
}
|
|
host, e := d.discover(ctx)
|
|
e = managedRecoveryContextError(ctx, e)
|
|
var discoveryFatal *managedRecoveryFatal
|
|
if errors.As(e, &discoveryFatal) {
|
|
return result, e
|
|
}
|
|
if ctx.Err() != nil {
|
|
return result, e
|
|
}
|
|
if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
|
|
result.GGCError = fmt.Errorf("exact GGC discovery failed")
|
|
return result, nil
|
|
}
|
|
if e := stop(); e != nil {
|
|
return result, e
|
|
}
|
|
args, report, e := revalidateGooglevideo(ctx, d.db, host, io.Discard, d.check)
|
|
var cacheID int64
|
|
if e == nil && args != "" {
|
|
e = saveGooglevideoRun(d.db, time.Now(), host, args, report, nil)
|
|
if e != nil {
|
|
return result, &managedRecoveryFatal{Err: e}
|
|
}
|
|
if e == nil {
|
|
cacheID, e = newestManagedCandidateForScope(d.db, "googlevideo", host, args)
|
|
if e != nil {
|
|
return result, &managedRecoveryFatal{Err: e}
|
|
}
|
|
}
|
|
} else if e == nil {
|
|
cacheID, e = managedScanCandidate(ctx, d, "googlevideo", host)
|
|
}
|
|
if e == nil {
|
|
e = apply([]int64{result.FrontendID, cacheID})
|
|
}
|
|
if e != nil {
|
|
e = managedRecoveryContextError(ctx, e)
|
|
var fatal *managedRecoveryFatal
|
|
if errors.As(e, &fatal) {
|
|
return result, e
|
|
}
|
|
if ctx.Err() != nil {
|
|
return result, e
|
|
}
|
|
if errors.Is(e, errNoQualifiedStrategy) {
|
|
return result, e
|
|
}
|
|
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
|
|
return result, nil
|
|
}
|
|
result.GooglevideoID = cacheID
|
|
return result, nil
|
|
}
|
|
|
|
func newestManagedCandidate(db *sql.DB, group string) (int64, error) {
|
|
ids, err := qualifiedCandidateIDs(db, group)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if len(ids) == 0 {
|
|
return 0, fmt.Errorf("no qualified candidate")
|
|
}
|
|
return ids[0], nil
|
|
}
|
|
|
|
func newestManagedCandidateForScope(db *sql.DB, group, host, args string) (int64, error) {
|
|
id, err := newestManagedCandidate(db, group)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
c, err := readStrategyCandidate(db, id)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if c.Args != args || len(c.Endpoints) != 1 || c.Endpoints[0] != host {
|
|
return 0, fmt.Errorf("latest saved candidate scope changed")
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
func managedRecheck(ctx context.Context, check cacheCheck, args, group, host string) ([]byte, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
f, err := os.CreateTemp("", "blockcheckw-managed-*.txt")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
path := f.Name()
|
|
if err := f.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
return revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, check)
|
|
}
|
|
|
|
// Reuses proposal selection and transactional persistence with caller context.
|
|
func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, host string) (int64, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
started := time.Now()
|
|
var diagnostics bytes.Buffer
|
|
path, err := d.scan(ctx, "blockcheckw", host, io.Discard, &diagnostics)
|
|
// Only validated report evidence, never a runner error, authorizes absence.
|
|
if errors.Is(err, errNoQualifiedStrategy) || errors.Is(err, errCandidateNonpass) {
|
|
err = fmt.Errorf("candidate scanner failed: %v", err)
|
|
}
|
|
err = managedCheckerError(err, path+".check.json.stderr", diagnostics.Bytes())
|
|
var report []byte
|
|
var args, output string
|
|
if err == nil {
|
|
var info os.FileInfo
|
|
info, err = os.Stat(path + ".check.json")
|
|
// ponytail: whole-second mtime permits same-second artifacts from injected
|
|
// scans; production uses unique paths. Require that contract if precision grows.
|
|
if err == nil && (!info.Mode().IsRegular() || info.ModTime().Before(started.Truncate(time.Second))) {
|
|
err = fmt.Errorf("missing fresh scan shortlist")
|
|
}
|
|
}
|
|
if err == nil {
|
|
report, err = os.ReadFile(path + ".check.json")
|
|
}
|
|
if err == nil {
|
|
args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, host)
|
|
if err != nil && emptyManagedScanReport(report, host) {
|
|
err = errNoQualifiedStrategy
|
|
}
|
|
}
|
|
if err == nil {
|
|
report, err = revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, d.check)
|
|
var fatal *managedRecoveryFatal
|
|
if !errors.As(err, &fatal) && errors.Is(err, errCandidateNonpass) {
|
|
err = errNoQualifiedStrategy
|
|
}
|
|
}
|
|
if err == nil {
|
|
err = ctx.Err()
|
|
}
|
|
if group == "frontend" {
|
|
if err == nil {
|
|
output, err = combinedBlockcheckOutputForHost(bytes.NewReader(report), args, host)
|
|
}
|
|
if e := saveFrontendRunForHost(d.db, started, args, output, report, err, host); e != nil {
|
|
return 0, &managedRecoveryFatal{Err: e}
|
|
}
|
|
} else {
|
|
var safe error
|
|
if err != nil {
|
|
safe = fmt.Errorf("cache scan or verification failed")
|
|
}
|
|
if e := saveGooglevideoRun(d.db, started, host, args, report, safe); e != nil {
|
|
return 0, &managedRecoveryFatal{Err: e}
|
|
}
|
|
}
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
id, err := newestManagedCandidateForScope(d.db, group, host, args)
|
|
if err != nil {
|
|
return 0, &managedRecoveryFatal{Err: err}
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
// A successful owned scan may omit rejected rows. Missing/null metadata is not
|
|
// absence evidence; qualification still requires the usual full TLS checks.
|
|
func emptyManagedScanReport(raw []byte, host string) bool {
|
|
var report struct {
|
|
Domain string `json:"domain"`
|
|
Total *int `json:"total"`
|
|
Working *int `json:"working"`
|
|
Strategies *[]json.RawMessage `json:"strategies"`
|
|
}
|
|
d := json.NewDecoder(bytes.NewReader(raw))
|
|
return d.Decode(&report) == nil && d.Decode(new(json.RawMessage)) == io.EOF &&
|
|
report.Domain == host && report.Total != nil && *report.Total > 0 &&
|
|
report.Working != nil && *report.Working == 0 && report.Strategies != nil && len(*report.Strategies) == 0
|
|
}
|
|
|
|
// Installs callbacks. runManagedMonitor performs startup
|
|
// under its singleton. Target guards are repeated before every recovery/probe.
|
|
func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDeps, reportGGC func(error)) error {
|
|
if monitor == nil || monitor.db == nil || monitor.db != recovery.db || recovery.transaction == nil || reportGGC == nil || recovery.health == nil || recovery.run == nil || recovery.scan == nil || recovery.check == nil || recovery.discover == nil {
|
|
return fmt.Errorf("matching monitor/recovery database and hooks required")
|
|
}
|
|
if recovery.transaction.Dir != monitor.target.directory || recovery.transaction.Name != "config" {
|
|
return fmt.Errorf("recovery transaction must own the monitor target config")
|
|
}
|
|
guard := func() error {
|
|
if _, err := managedRuntimeGuard(monitor.target); err != nil {
|
|
return err
|
|
}
|
|
return managedCheckIdentity(recovery.transaction)
|
|
}
|
|
callback := func(ctx context.Context) error {
|
|
if err := guard(); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
result, err := recoverManagedTarget(ctx, recovery)
|
|
if result.GGCError != nil {
|
|
reportGGC(result.GGCError)
|
|
if recovery.adaptive && err == nil {
|
|
return result.GGCError
|
|
}
|
|
}
|
|
// The supervisor reports ordinary recovery failures and retries only
|
|
// after its durable cooldown. Safety failures (rollback/cleanup/drift)
|
|
// are already typed managedRecoveryFatal by the transaction boundary.
|
|
return err
|
|
}
|
|
bootstrap := func(ctx context.Context) error {
|
|
if err := guard(); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
// Reconcile once at startup, never adopt a pending live deployment.
|
|
transaction := *recovery.transaction
|
|
transaction.Health = func() (string, error) { return recovery.health(ctx) }
|
|
transaction.Reload = func() error {
|
|
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
|
defer cancel()
|
|
return recovery.run(cleanup, managedZapretScript, "restart")
|
|
}
|
|
if err := transaction.Reconcile(recovery.db); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
state, err := readMonitorState(monitor.db, monitor.serviceID)
|
|
if err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
// Reconciliation and identity guards precede even a paused bootstrap.
|
|
if state.RecoveryRunning || monitor.now().Before(state.CooldownUntil) {
|
|
return nil
|
|
}
|
|
ids, err := transaction.ActiveCandidates(recovery.db)
|
|
if err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
for _, id := range ids {
|
|
candidate, err := readStrategyCandidate(recovery.db, id)
|
|
if err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
if candidate.Group == "frontend" && candidate.Qualified {
|
|
return nil
|
|
}
|
|
}
|
|
return callback(ctx)
|
|
}
|
|
monitor.bootstrap = func(ctx context.Context) error {
|
|
return managedRecoveryJob(ctx, recovery.jobTimeout, bootstrap)
|
|
}
|
|
monitor.recover = func(ctx context.Context) error {
|
|
return managedRecoveryJob(ctx, recovery.jobTimeout, callback)
|
|
}
|
|
monitor.probe = func(ctx context.Context) error {
|
|
if err := guard(); err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
status, err := recovery.health(ctx)
|
|
err = managedRecoveryContextError(ctx, err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if status != "OK" {
|
|
return &managedHealthRejection{Status: status}
|
|
}
|
|
transaction := *recovery.transaction
|
|
transaction.Health = func() (string, error) { return recovery.health(ctx) }
|
|
// ActiveCandidates is read-only; never permit a probe to reload config.
|
|
transaction.Reload = func() error { return fmt.Errorf("probe cannot reload config") }
|
|
ids, err := transaction.ActiveCandidates(recovery.db)
|
|
if err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
// Keep Player API health separate from GGC scope (not playback health).
|
|
journal, err := transaction.load()
|
|
if err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
if journal.Scope == GooglevideoScopeExperimental {
|
|
return errManagedGGCUnresolved
|
|
}
|
|
// Two bounded Player requests reuse the existing health/discovery adapters.
|
|
host, err := recovery.discover(ctx)
|
|
err = managedRecoveryContextError(ctx, err)
|
|
if ctx.Err() != nil {
|
|
return err
|
|
}
|
|
var fatal *managedRecoveryFatal
|
|
if errors.As(err, &fatal) || errors.Is(err, context.Canceled) {
|
|
return err
|
|
}
|
|
if err != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
|
|
return errManagedGGCUnresolved
|
|
}
|
|
for _, id := range ids {
|
|
candidate, err := readStrategyCandidate(recovery.db, id)
|
|
if err != nil {
|
|
return &managedRecoveryFatal{Err: err}
|
|
}
|
|
if recovery.adaptive && candidate.Group == "googlevideo" && candidate.Mode == QualificationAdaptive && candidate.Qualified {
|
|
journal, e := transaction.load()
|
|
if e != nil {
|
|
return &managedRecoveryFatal{Err: e}
|
|
}
|
|
if journal.Scope == GooglevideoScopeSuffix {
|
|
return nil
|
|
}
|
|
}
|
|
if candidate.Group == "googlevideo" && len(candidate.Endpoints) == 1 && candidate.Endpoints[0] == host {
|
|
return nil
|
|
}
|
|
}
|
|
return errManagedGGCScopeChanged
|
|
}
|
|
return nil
|
|
}
|