feat: recover experimental Googlevideo media autonomously
Preserve operator-owned profiles while rechecking fresh media and bounded native relearning in the deployed daemon. Keep mixed safety failures fatal and expose active coverage separately from qualification. Document source-only acceptance and retain existing recovery budgets and privacy boundaries. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
1a35cb3019
commit
a60f62de22
14 files changed
+878
-41
No files matched your search
@@ -38,6 +38,18 @@ func (e *managedResourceFailure) PublicError() string {
|
||||
}
|
||||
func (e *managedResourceFailure) Unwrap() error { return e.Err }
|
||||
|
||||
// Only a typed collector's complete known-negative error set may lose its
|
||||
// generic fatal wrapper at a live-health boundary. Missing resources remain
|
||||
// DEGRADED; joined guard/infrastructure/cancellation failures retain their type.
|
||||
func managedResourceHealthError(ctx context.Context, err error) error {
|
||||
var failure *managedResourceFailure
|
||||
var collector *resourceError
|
||||
if ctx.Err() == nil && errors.As(err, &failure) && (failure.ProbeKind == "media" || assetEndpoint(failure.Host) && failure.ProbeKind == managedImageProbeKind(failure.Host)) && errors.As(failure.Err, &collector) && managedResourceTransferNonpass(err, assetEndpoint(failure.Host)) {
|
||||
return &managedResourceFailure{ProbeKind: failure.ProbeKind, Host: failure.Host}
|
||||
}
|
||||
return managedRecoveryContextError(ctx, err)
|
||||
}
|
||||
|
||||
// Ephemeral inventory: never serialize URLs, page bodies or Player responses.
|
||||
type managedServiceInventory struct {
|
||||
images map[string]string
|
||||
@@ -85,7 +97,11 @@ func managedServiceResources(cfg healthConfig, player func(context.Context) ([]b
|
||||
return i, managedRecoveryContextError(ctx, err)
|
||||
}
|
||||
if err != nil || bounded.Err() != nil || out.exceeded {
|
||||
return i, &managedResourceFailure{"avatar", "yt3.ggpht.com", resourceFailure(bounded, errors.Join(err, bounded.Err(), errors.New("fresh root metadata request failed")))}
|
||||
cause := errors.Join(err, bounded.Err())
|
||||
if cause == nil || out.exceeded {
|
||||
cause = errors.Join(cause, errors.New("fresh root metadata request failed"))
|
||||
}
|
||||
return i, managedResourceHealthError(ctx, &managedResourceFailure{"avatar", "yt3.ggpht.com", resourceFailure(bounded, cause)})
|
||||
}
|
||||
if raw, e := avatarResourceFromPage(output.Bytes()); e == nil {
|
||||
i.images["yt3.ggpht.com"] = raw
|
||||
|
||||
@@ -272,6 +272,7 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB,
|
||||
return errManagedDrift
|
||||
}
|
||||
groups := map[string]bool{}
|
||||
operatorCoverage := map[string]bool{}
|
||||
assetHosts := map[string]bool{}
|
||||
known := j.State == "committed" && j.Version >= 2 && string(b) == managedBaseline && len(j.Candidates) == 0
|
||||
if j.State == "committed" && j.Version >= 2 && len(j.Candidates) != 0 {
|
||||
@@ -290,6 +291,7 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB,
|
||||
break
|
||||
}
|
||||
groups[c.Group] = true
|
||||
operatorCoverage[c.Group] = true
|
||||
if experimentalGooglevideo(j.Scope, c) {
|
||||
groups[c.Group] = false
|
||||
}
|
||||
@@ -304,6 +306,7 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB,
|
||||
}
|
||||
if !known {
|
||||
groups = map[string]bool{}
|
||||
operatorCoverage = map[string]bool{}
|
||||
// Unvalidated journal references are not active recovery candidates.
|
||||
j.Candidates = nil
|
||||
}
|
||||
@@ -321,14 +324,29 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB,
|
||||
if (groups["assets"] || (len(requestAssets) != 0 && requestAssets[0])) && (!known || len(assetHosts) != 2) {
|
||||
missing = append(missing, "assets")
|
||||
}
|
||||
// Coverage labels active profiles, not qualification or the full host inventory.
|
||||
covered := []string{}
|
||||
for _, group := range []string{"frontend", "assets", "googlevideo"} {
|
||||
if operatorCoverage[group] {
|
||||
covered = append(covered, group)
|
||||
}
|
||||
}
|
||||
scope := j.Scope
|
||||
if !known {
|
||||
scope = "unknown"
|
||||
} else if operatorCoverage["googlevideo"] && scope == "" {
|
||||
scope = GooglevideoScopeExact
|
||||
}
|
||||
if activeOnly {
|
||||
if !known {
|
||||
return fmt.Errorf("active candidate IDs unresolved; deployment preserved by hash")
|
||||
}
|
||||
return json.NewEncoder(out).Encode(struct {
|
||||
IDs []int64 `json:"active_candidate_ids"`
|
||||
Missing []string `json:"unresolved_groups"`
|
||||
}{append([]int64{}, j.Candidates...), missing})
|
||||
IDs []int64 `json:"active_candidate_ids"`
|
||||
Missing []string `json:"unresolved_groups"`
|
||||
Covered []string `json:"covered_groups"`
|
||||
Scope GooglevideoScope `json:"googlevideo_scope"`
|
||||
}{append([]int64{}, j.Candidates...), missing, covered, scope})
|
||||
}
|
||||
groupText := strings.Join(active, ",")
|
||||
if groupText == "" {
|
||||
@@ -341,9 +359,20 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB,
|
||||
if known {
|
||||
reportIDs = j.Candidates
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "managed-test generation=%d hash=%s state=%s groups=%s active_candidate_ids=%v unresolved_groups=%s\nLive health: NOT CHECKED (Player API is not GGC/video playback)\n", j.Generation, j.Hash, j.State, groupText, reportIDs, strings.Join(missing, ","))
|
||||
coverageText := strings.Join(covered, ",")
|
||||
if coverageText == "" {
|
||||
coverageText = "none"
|
||||
if !known {
|
||||
coverageText = "unknown"
|
||||
}
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "managed-test generation=%d hash=%s state=%s groups=%s active_candidate_ids=%v unresolved_groups=%s covered=%s\nLive health: NOT CHECKED (Player API is not GGC/video playback)\nCoverage: active config profiles only, not qualification or all-host/asset-inventory success.\n", j.Generation, j.Hash, j.State, groupText, reportIDs, strings.Join(missing, ","), coverageText)
|
||||
if err == nil && j.Version == 3 {
|
||||
_, err = fmt.Fprintf(out, "Googlevideo scope: %s (suffix is authorized extrapolation, not all-host qualification)\n", j.Scope)
|
||||
if known && scope == GooglevideoScopeExperimental {
|
||||
_, err = fmt.Fprintf(out, "Googlevideo scope: %s (experimental/unqualified; one authorized composite profile, not all hosts checked)\n", scope)
|
||||
} else {
|
||||
_, err = fmt.Fprintf(out, "Googlevideo scope: %s (suffix is authorized extrapolation, not all-host qualification)\n", scope)
|
||||
}
|
||||
}
|
||||
return err
|
||||
})
|
||||
|
||||
@@ -458,6 +458,144 @@ func TestManagedCLILegacyIDsUnresolved(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedStatusOperatorCoverage(t *testing.T) {
|
||||
for _, mode := range []string{"qualified", "experimental", "missing-candidate", "missing-db", "render-mismatch", "invalid-hash", "legacy", "pending"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
db := testStore(t)
|
||||
front := managedCandidateFixture(t, db, "frontend", "www.youtube.com", "--lua-desync=fake", true)
|
||||
ids := []int64{front}
|
||||
scope := GooglevideoScopeExact
|
||||
if mode == "qualified" {
|
||||
ids = append(ids, managedCandidateFixture(t, db, "googlevideo", "rr1.googlevideo.com", "--lua-desync=fake", true))
|
||||
} else {
|
||||
asset := unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true)
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ggc, err := insertStrategyCandidate(tx, "googlevideo", []string{"rr1---sn-alpha.googlevideo.com"}, managedGooglevideoArgs, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids = append(ids, asset, ggc)
|
||||
scope = GooglevideoScopeExperimental
|
||||
c, err := readStrategyCandidate(db, ggc)
|
||||
if err != nil || c.Qualified || c.Mode != QualificationCold {
|
||||
t.Fatalf("experimental candidate: %+v %v", c, err)
|
||||
}
|
||||
var evidence int
|
||||
if err := db.QueryRow("SELECT count(*) FROM candidate_evidence WHERE candidate_id=?", ggc).Scan(&evidence); err != nil || evidence != 0 {
|
||||
t.Fatalf("fabricated evidence: %d %v", evidence, err)
|
||||
}
|
||||
}
|
||||
m, _ := managedFixture(t)
|
||||
if err := m.Apply(db, ids, scope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j, err := m.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
openDB := func() (*sql.DB, error) { return openHistoryStore(databasePath()) }
|
||||
switch mode {
|
||||
case "missing-candidate":
|
||||
j.Candidates[len(ids)-1] = 999999
|
||||
case "missing-db":
|
||||
openDB = func() (*sql.DB, error) { return nil, errors.New("unavailable") }
|
||||
case "render-mismatch":
|
||||
b := []byte("# not the rendered candidates\n" + managedBaseline)
|
||||
if err := os.WriteFile(filepath.Join(m.Dir, m.Name), b, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j.Hash = managedHash(b)
|
||||
case "invalid-hash":
|
||||
j.Hash = managedHash([]byte("wrong hash"))
|
||||
case "legacy":
|
||||
j.Version, j.Scope = 2, ""
|
||||
case "pending":
|
||||
j.State, j.Next = "prepared", j.Hash
|
||||
j.PendingCandidates, j.PendingScope = append([]int64{}, ids...), scope
|
||||
}
|
||||
if err := m.journal(j); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := os.ReadFile(filepath.Join(m.Dir, m.Name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
journalBefore, err := os.ReadFile(filepath.Join(m.Dir, managedJournalName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Health = func() (string, error) { t.Fatal("status checked live health"); return "", nil }
|
||||
m.Reload = func() error { t.Fatal("status reloaded target"); return nil }
|
||||
var out bytes.Buffer
|
||||
err = managedReport(m, &out, openDB, false)
|
||||
if mode == "invalid-hash" {
|
||||
if !errors.Is(err, errManagedDrift) || out.Len() != 0 {
|
||||
t.Fatalf("drift trusted: %s %v", out.String(), err)
|
||||
}
|
||||
} else {
|
||||
if err != nil || !strings.Contains(out.String(), "Live health: NOT CHECKED") || strings.Contains(out.String(), "READY") {
|
||||
t.Fatalf("status: %s %v", out.String(), err)
|
||||
}
|
||||
want := []string{"groups=unknown ", "covered=unknown", "active_candidate_ids=[] ", "unresolved_groups=frontend,googlevideo"}
|
||||
if mode == "qualified" {
|
||||
want = []string{"groups=frontend,googlevideo ", "covered=frontend,googlevideo", "unresolved_groups= covered="}
|
||||
} else if mode == "experimental" {
|
||||
want = []string{"groups=frontend,assets ", "covered=frontend,assets,googlevideo", "unresolved_groups=googlevideo,assets ", "experimental/unqualified", "one authorized composite profile", "not all hosts checked"}
|
||||
}
|
||||
for _, text := range want {
|
||||
if !strings.Contains(out.String(), text) {
|
||||
t.Fatalf("missing %q: %s", text, out.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
out.Reset()
|
||||
err = managedReport(m, &out, openDB, true)
|
||||
if mode == "qualified" || mode == "experimental" {
|
||||
var state struct {
|
||||
IDs []int64 `json:"active_candidate_ids"`
|
||||
Missing []string `json:"unresolved_groups"`
|
||||
Covered []string `json:"covered_groups"`
|
||||
Scope GooglevideoScope `json:"googlevideo_scope"`
|
||||
}
|
||||
if err != nil || json.Unmarshal(out.Bytes(), &state) != nil || !reflect.DeepEqual(state.IDs, ids) || state.Scope != scope {
|
||||
t.Fatalf("JSON: %s %v", out.String(), err)
|
||||
}
|
||||
covered, missing := []string{"frontend", "googlevideo"}, []string(nil)
|
||||
if mode == "experimental" {
|
||||
covered, missing = []string{"frontend", "assets", "googlevideo"}, []string{"googlevideo", "assets"}
|
||||
}
|
||||
if !reflect.DeepEqual(state.Covered, covered) || !reflect.DeepEqual(state.Missing, missing) {
|
||||
t.Fatalf("coverage changed qualification: %+v", state)
|
||||
}
|
||||
encoded, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var roundtrip map[string]json.RawMessage
|
||||
if err := json.Unmarshal(encoded, &roundtrip); err != nil || len(roundtrip) != 4 {
|
||||
t.Fatalf("JSON roundtrip: %s %v", encoded, err)
|
||||
}
|
||||
} else if err == nil || out.Len() != 0 {
|
||||
t.Fatalf("invalid IDs trusted: %s %v", out.String(), err)
|
||||
}
|
||||
after, err := os.ReadFile(filepath.Join(m.Dir, m.Name))
|
||||
if err != nil || !bytes.Equal(before, after) {
|
||||
t.Fatal("status changed config", err)
|
||||
}
|
||||
journalAfter, err := os.ReadFile(filepath.Join(m.Dir, managedJournalName))
|
||||
if err != nil || !bytes.Equal(journalBefore, journalAfter) {
|
||||
t.Fatal("status changed journal", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMountpoint(t *testing.T) {
|
||||
for _, path := range []string{"/", filepath.Dir(managedTargetDirectory), managedTargetDirectory + "-other"} {
|
||||
if managedMountpoint("1 2 0:3 / "+path+" rw - tmpfs tmpfs rw\n", managedTargetDirectory) {
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os/exec"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
||||
for _, mode := range []string{"bad-range", "healthy", "unknown", "infra", "mixed", "guard", "cancel", "metadata-timeout", "metadata-http", "metadata-malformed", "metadata-mixed", "metadata-cancel", "metadata-infra", "image-timeout", "image-mixed"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, front, _ := recoveryFixture(t, true)
|
||||
target, _, _ := managedCLIFixture(t)
|
||||
initManagedCLI(t, target)
|
||||
d.transaction.Dir = target.directory
|
||||
asset := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
|
||||
id := managedCandidateFixture(t, d.db, "googlevideo", "rr1---sn-alpha.googlevideo.com", managedGooglevideoArgs, false)
|
||||
ids := []int64{front, asset, id}
|
||||
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeExperimental); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := d.transaction.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.service = true
|
||||
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
|
||||
d.resources = func(context.Context) (managedServiceInventory, error) {
|
||||
r := experimentalResourceFixture()
|
||||
if mode == "unknown" {
|
||||
r = mediaResource{}
|
||||
}
|
||||
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/test", "yt3.ggpht.com": "https://yt3.ggpht.com/test"}, media: r}, nil
|
||||
}
|
||||
knownTimeout := exec.Command("sh", "-c", "exit 28").Run()
|
||||
if strings.HasPrefix(mode, "metadata-") {
|
||||
d.resources = managedServiceResources(healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080"}, func(context.Context) ([]byte, error) {
|
||||
return []byte(fmt.Sprintf(`{"playabilityStatus":{"status":"OK"},"streamingData":{"formats":[{"url":%q,"mimeType":"video/mp4"}]}}`, experimentalResourceFixture().url)), nil
|
||||
}, func(_ context.Context, args []string, out, diagnostic io.Writer) error {
|
||||
if args[len(args)-1] != "https://www.youtube.com/" || diagnostic != io.Discard {
|
||||
t.Fatal("metadata origin/diagnostics changed")
|
||||
}
|
||||
switch mode {
|
||||
case "metadata-malformed":
|
||||
_, err := io.WriteString(out, "<html>not playable metadata</html>")
|
||||
return err
|
||||
case "metadata-http":
|
||||
return exec.Command("sh", "-c", "exit 22").Run()
|
||||
case "metadata-mixed":
|
||||
return errors.Join(knownTimeout, &managedRecoveryFatal{Err: errors.New("guard failed")})
|
||||
case "metadata-cancel":
|
||||
return errors.Join(knownTimeout, context.Canceled)
|
||||
case "metadata-infra":
|
||||
return errors.Join(knownTimeout, errors.New("collector unavailable"))
|
||||
default:
|
||||
return knownTimeout
|
||||
}
|
||||
})
|
||||
i, _ := d.resources(context.Background())
|
||||
if !freshExperimentalMedia(i.media) {
|
||||
t.Fatal("metadata failure discarded fresh media")
|
||||
}
|
||||
}
|
||||
calls := 0
|
||||
d.recheckExperimental = func(_ context.Context, c storedCandidate, r mediaResource) error {
|
||||
if c.ID != id || !freshExperimentalMedia(r) {
|
||||
t.Fatal("recovery changed candidate or lost fresh media")
|
||||
}
|
||||
calls++
|
||||
return nil
|
||||
}
|
||||
health := managedServiceHealth("socks5://127.0.0.1:11080", d.frontHealth, d.resources, func(_ context.Context, args []string, in io.Reader, out, diagnostic io.Writer) error {
|
||||
if strings.HasPrefix(mode, "metadata-") {
|
||||
t.Fatal("missing images must not become transfer evidence")
|
||||
}
|
||||
if mode == "image-timeout" {
|
||||
return knownTimeout
|
||||
}
|
||||
if mode == "image-mixed" {
|
||||
return errors.Join(knownTimeout, &managedRecoveryFatal{Err: errors.New("guard failed")})
|
||||
}
|
||||
config, _ := io.ReadAll(in)
|
||||
raw := strings.TrimSuffix(strings.TrimPrefix(string(config), "url = \""), "\"\n")
|
||||
var wire, diag bytes.Buffer
|
||||
wireMode := ""
|
||||
if strings.Contains(raw, ".googlevideo.com/") {
|
||||
wireMode = "media"
|
||||
if mode != "healthy" {
|
||||
wireMode = "bad-range"
|
||||
}
|
||||
}
|
||||
resourceWire(raw, wireMode, &wire, &diag)
|
||||
if strings.Contains(strings.Join(args, " "), "--tlsv1.3") {
|
||||
diag = *bytes.NewBufferString(strings.ReplaceAll(diag.String(), "TLSv1.2", "TLSv1.3"))
|
||||
}
|
||||
_, _ = out.Write(wire.Bytes())
|
||||
_, _ = diagnostic.Write(diag.Bytes())
|
||||
return nil
|
||||
})
|
||||
d.health = func(ctx context.Context) (string, error) {
|
||||
status, err := health(ctx)
|
||||
if mode != "healthy" && status != "DEGRADED" {
|
||||
t.Fatalf("resource failure became healthy: %s", status)
|
||||
}
|
||||
switch mode {
|
||||
case "infra":
|
||||
err = &managedResourceFailure{"media", experimentalResourceFixture().host, resourceFailure(ctx, errors.New("collector unavailable"))}
|
||||
case "mixed":
|
||||
err = errors.Join(err, &managedRecoveryFatal{Err: errors.New("guard failed")})
|
||||
case "guard":
|
||||
err = errors.Join(err, &managedRecoveryFatal{Err: errors.New("invalid media range or container")})
|
||||
case "cancel":
|
||||
err = errors.Join(err, context.Canceled)
|
||||
}
|
||||
return status, err
|
||||
}
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target = d.db, target
|
||||
monitor.threshold = 1
|
||||
if err := wireManagedRecovery(&monitor, d, func(error) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = monitor.probe(context.Background())
|
||||
var fatal *managedRecoveryFatal
|
||||
switch mode {
|
||||
case "healthy":
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "bad-range", "metadata-timeout", "metadata-http", "metadata-malformed", "image-timeout":
|
||||
if err == nil || errors.As(err, &fatal) {
|
||||
t.Fatalf("media damage killed monitor: %v", err)
|
||||
}
|
||||
case "unknown":
|
||||
if err == nil || errors.Is(err, errNoQualifiedStrategy) {
|
||||
t.Fatalf("unknown became green/absence: %v", err)
|
||||
}
|
||||
default:
|
||||
if !errors.As(err, &fatal) {
|
||||
t.Fatalf("safety failure normalized: %v", err)
|
||||
}
|
||||
}
|
||||
if mode != "healthy" && mode != "unknown" {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
monitor.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||
err = runManagedMonitor(ctx, monitor)
|
||||
if mode == "bad-range" || mode == "metadata-timeout" || mode == "metadata-http" || mode == "metadata-malformed" || mode == "image-timeout" {
|
||||
if !errors.Is(err, context.Canceled) || errors.As(err, &fatal) || calls != 1 {
|
||||
t.Fatalf("monitor recovery unreachable: %v calls=%d", err, calls)
|
||||
}
|
||||
} else if !errors.As(err, &fatal) || calls != 0 {
|
||||
t.Fatalf("fatal probe entered recovery: %v calls=%d", err, calls)
|
||||
}
|
||||
}
|
||||
after, err := d.transaction.load()
|
||||
if err != nil || !reflect.DeepEqual(before, after) {
|
||||
t.Fatalf("probe/recovery changed journal: %v", err)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, ids)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedResourceHealthClassificationKeepsSafetyAndEvidenceBoundaries(t *testing.T) {
|
||||
timeout := exec.Command("sh", "-c", "exit 28").Run()
|
||||
httpFailure := exec.Command("sh", "-c", "exit 22").Run()
|
||||
if managedGooglevideoTransferNonpass(resourceFailure(context.Background(), httpFailure)) {
|
||||
t.Fatal("metadata HTTP nonpass became media rejection evidence")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, tc := range []struct {
|
||||
ctx context.Context
|
||||
err error
|
||||
}{
|
||||
{context.Background(), &managedRecoveryFatal{Err: timeout}},
|
||||
{context.Background(), errors.Join(resourceFailure(context.Background(), timeout), &managedRecoveryFatal{Err: timeout})},
|
||||
{context.Background(), errors.Join(resourceFailure(context.Background(), timeout), errors.New("unknown collector failure"))},
|
||||
{ctx, resourceFailure(context.Background(), timeout)},
|
||||
} {
|
||||
err := managedResourceHealthError(tc.ctx, &managedResourceFailure{"avatar", "yt3.ggpht.com", tc.err})
|
||||
var failure *managedResourceFailure
|
||||
if !errors.As(err, &failure) || failure.Err == nil {
|
||||
t.Fatalf("safety failure normalized: %v", err)
|
||||
}
|
||||
if tc.ctx.Err() != nil && !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("parent cancellation lost")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExperimentalProductionRecheckAdapterInstalled(t *testing.T) {
|
||||
d, _, _ := recoveryFixture(t, true)
|
||||
target, env, _ := managedCLIFixture(t)
|
||||
env["ZAPRET_AUTOTUNE_YOUTUBE_SERVICE"] = "1"
|
||||
adapted, err := managedMonitorRecoveryAdapters(target, d.db, d.scan, d.check, d.run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !adapted.service || adapted.resources == nil || adapted.recheckExperimental == nil || adapted.run == nil || adapted.transaction == nil {
|
||||
t.Fatal("production recheck dependency missing")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var errManagedExperimentalUnqualified = errors.New("experimental googlevideo suffix: operational health is not family qualification")
|
||||
|
||||
// URLs are trusted Player inventory, never rewritten or persisted.
|
||||
func freshExperimentalMedia(r mediaResource) bool {
|
||||
u, err := privateResourceURL(r.url, r.host)
|
||||
if err != nil || !strings.HasSuffix(r.host, ".googlevideo.com") || u.Path != "/videoplayback" || (r.kind != "video/mp4" && r.kind != "audio/mp4") {
|
||||
return false
|
||||
}
|
||||
q := u.Query()
|
||||
expires, err := strconv.ParseInt(q.Get("expire"), 10, 64)
|
||||
return err == nil && expires > time.Now().Unix()+120 && (q.Get("sig") != "" || q.Get("signature") != "" || q.Get("lsig") != "")
|
||||
}
|
||||
|
||||
func experimentalMedia(ctx context.Context, d managedRecoveryDeps) (mediaResource, error) {
|
||||
i, err := d.resources(ctx)
|
||||
var fatal *managedRecoveryFatal
|
||||
var failure *managedResourceFailure
|
||||
// An unrelated metadata/image failure cannot conceal usable media, but a
|
||||
// mixed safety/infrastructure error must not be turned into transfer absence.
|
||||
unrelated := errors.As(err, &failure) && assetEndpoint(failure.Host) && (err == failure && failure.Err == nil || managedGooglevideoTransferNonpass(err))
|
||||
if ctx.Err() != nil || errors.As(err, &fatal) && !unrelated || errors.Is(err, context.Canceled) || err != nil && !unrelated {
|
||||
return mediaResource{}, managedRecoveryContextError(ctx, err)
|
||||
}
|
||||
if !freshExperimentalMedia(i.media) {
|
||||
return mediaResource{}, &managedResourceFailure{"media", "unknown", nil}
|
||||
}
|
||||
return i.media, nil
|
||||
}
|
||||
|
||||
func recoverManagedExperimental(ctx context.Context, d managedRecoveryDeps) (int64, error) {
|
||||
if !d.service || d.resources == nil || d.recheckExperimental == nil {
|
||||
return 0, errManagedGGCUnresolved
|
||||
}
|
||||
ids, err := d.transaction.ActiveCandidates(d.db)
|
||||
if err != nil {
|
||||
return 0, &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
j, err := d.transaction.load()
|
||||
if err != nil {
|
||||
return 0, &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
for _, id := range ids {
|
||||
c, err := readStrategyCandidate(d.db, id)
|
||||
if err != nil {
|
||||
return 0, &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
if experimentalGooglevideo(j.Scope, c) {
|
||||
r, err := experimentalMedia(ctx, d)
|
||||
if err == nil {
|
||||
err = d.recheckExperimental(ctx, c, r)
|
||||
}
|
||||
return id, managedRecoveryContextError(ctx, err)
|
||||
}
|
||||
}
|
||||
return 0, &managedRecoveryFatal{Err: errors.New("owned experimental candidate missing")}
|
||||
}
|
||||
|
||||
func newManagedExperimentalRecheck(target managedDeps, d managedRecoveryDeps) func(context.Context, storedCandidate, mediaResource) error {
|
||||
runtime := newManagedAssetRuntime(target)
|
||||
guard := func() error {
|
||||
if err := runtime.guard(); err != nil {
|
||||
return err
|
||||
}
|
||||
lua, err := os.ReadFile("/opt/zapret2/lua/zapret-auto.lua")
|
||||
if err != nil || fmt.Sprintf("%x", sha256.Sum256(lua)) != managedGooglevideoLuaHash {
|
||||
return errors.New("unverified native automation Lua source")
|
||||
}
|
||||
_, err = d.transaction.ActiveCandidates(d.db)
|
||||
return err
|
||||
}
|
||||
restart := func(ctx context.Context) (err error) {
|
||||
// Existing daemon protocol stops/joins before starting the same full set.
|
||||
// Never activate a pending or drifted deployment after a partial stop.
|
||||
defer func() {
|
||||
if err != nil {
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cancel()
|
||||
if e := d.transaction.Reconcile(d.db); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
} else {
|
||||
err = errors.Join(err, d.run(cleanup, managedZapretScript, "start"))
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err = d.run(ctx, managedZapretScript, "stop"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = d.transaction.Reconcile(d.db); err != nil {
|
||||
return err
|
||||
}
|
||||
return d.run(ctx, managedZapretScript, "start")
|
||||
}
|
||||
return func(ctx context.Context, c storedCandidate, resource mediaResource) error {
|
||||
if !experimentalGooglevideo(GooglevideoScopeExperimental, c) {
|
||||
return &managedRecoveryFatal{Err: errors.New("unaudited experimental candidate")}
|
||||
}
|
||||
ownedGuard := func() error {
|
||||
if err := guard(); err != nil {
|
||||
return err
|
||||
}
|
||||
j, err := d.transaction.load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if j.Scope == GooglevideoScopeExperimental {
|
||||
for _, id := range j.Candidates {
|
||||
if id == c.ID {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.New("experimental ownership changed")
|
||||
}
|
||||
return recheckDeployedExperimental(ctx, resource, ownedGuard, func(ctx context.Context) error {
|
||||
if err := ownedGuard(); err != nil {
|
||||
return err
|
||||
}
|
||||
return restart(ctx)
|
||||
}, func(ctx context.Context) (mediaResource, error) {
|
||||
return experimentalMedia(ctx, d)
|
||||
}, func(ctx context.Context, r mediaResource, protocol string) error {
|
||||
_, _, err := collectMediaTransfer(ctx, runtime.proxy, r, protocol, runtime.curl)
|
||||
return err
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// No temporary NFQ worker: successful learning stays in the deployed daemon RAM.
|
||||
func recheckDeployedExperimental(ctx context.Context, resource mediaResource, guard func() error, restart func(context.Context) error, fresh func(context.Context) (mediaResource, error), transfer func(context.Context, mediaResource, string) error) error {
|
||||
check := func(ctx context.Context, protocol string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := guard(); err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
if !freshExperimentalMedia(resource) {
|
||||
var err error
|
||||
resource, err = fresh(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !freshExperimentalMedia(resource) {
|
||||
return &managedResourceFailure{"media", "unknown", nil}
|
||||
}
|
||||
}
|
||||
return managedRecoveryContextError(ctx, transfer(ctx, resource, protocol))
|
||||
}
|
||||
sample := func() error {
|
||||
for _, protocol := range []string{"tls12", "tls13"} {
|
||||
for n := 0; n < 3; n++ {
|
||||
if err := check(ctx, protocol); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
err := sample()
|
||||
if err == nil || !managedGooglevideoTransferNonpass(err) {
|
||||
return err
|
||||
}
|
||||
if err := restart(ctx); err != nil {
|
||||
return &managedRecoveryFatal{Err: managedRecoveryContextError(ctx, err)}
|
||||
}
|
||||
warm, cancel := context.WithTimeout(ctx, 120*time.Second)
|
||||
defer cancel()
|
||||
learned := map[string]bool{}
|
||||
for n := 0; n < 10 && len(learned) < 2; n++ {
|
||||
protocol := []string{"tls12", "tls13"}[n%2]
|
||||
err = check(warm, protocol)
|
||||
if ctx.Err() != nil {
|
||||
return managedRecoveryContextError(ctx, err)
|
||||
}
|
||||
if warm.Err() != nil && managedRecoveryTimeoutSafe(err) {
|
||||
break
|
||||
}
|
||||
if err != nil && !managedGooglevideoTransferNonpass(err) {
|
||||
return err
|
||||
}
|
||||
if err == nil {
|
||||
learned[protocol] = true
|
||||
}
|
||||
}
|
||||
if len(learned) == 2 {
|
||||
err = sample()
|
||||
if err == nil || !managedGooglevideoTransferNonpass(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("experimental media relearning exhausted: %w", errNoQualifiedStrategy)
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func experimentalResourceFixture() mediaResource {
|
||||
return mediaResource{url: fmt.Sprintf("https://rr2---sn-new.googlevideo.com/videoplayback?expire=%d&sig=private", time.Now().Add(time.Hour).Unix()), host: "rr2---sn-new.googlevideo.com", kind: "video/mp4"}
|
||||
}
|
||||
|
||||
func TestExperimentalDeployedRecheck(t *testing.T) {
|
||||
for _, mode := range []string{"healthy", "learn", "exhausted", "mixed", "cancel", "guard"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
r := experimentalResourceFixture()
|
||||
calls, restarts := 0, 0
|
||||
protocols := []string{}
|
||||
bad := errors.New("invalid media range or container")
|
||||
infra := errors.New("collector unavailable")
|
||||
err := recheckDeployedExperimental(ctx, r, func() error {
|
||||
if mode == "guard" {
|
||||
return infra
|
||||
}
|
||||
return nil
|
||||
}, func(context.Context) error { restarts++; return nil }, func(context.Context) (mediaResource, error) {
|
||||
t.Fatal("unexpected refresh")
|
||||
return mediaResource{}, infra
|
||||
}, func(_ context.Context, got mediaResource, protocol string) error {
|
||||
if got != r {
|
||||
t.Fatal("signed resource rewritten")
|
||||
}
|
||||
calls++
|
||||
protocols = append(protocols, protocol)
|
||||
switch mode {
|
||||
case "learn":
|
||||
if calls == 1 {
|
||||
return resourceFailure(ctx, bad)
|
||||
}
|
||||
case "exhausted":
|
||||
return resourceFailure(ctx, bad)
|
||||
case "mixed":
|
||||
return errors.Join(resourceFailure(ctx, bad), infra)
|
||||
case "cancel":
|
||||
cancel()
|
||||
return context.Canceled
|
||||
}
|
||||
return nil
|
||||
})
|
||||
switch mode {
|
||||
case "healthy":
|
||||
if err != nil || restarts != 0 || !reflect.DeepEqual(protocols, []string{"tls12", "tls12", "tls12", "tls13", "tls13", "tls13"}) {
|
||||
t.Fatalf("%v %d %v", err, restarts, protocols)
|
||||
}
|
||||
case "learn":
|
||||
if err != nil || restarts != 1 || calls != 9 {
|
||||
t.Fatalf("%v restarts=%d calls=%d", err, restarts, calls)
|
||||
}
|
||||
case "exhausted":
|
||||
if !errors.Is(err, errNoQualifiedStrategy) || restarts != 1 || calls != 11 {
|
||||
t.Fatalf("%v restarts=%d calls=%d", err, restarts, calls)
|
||||
}
|
||||
default:
|
||||
if err == nil || errors.Is(err, errNoQualifiedStrategy) || restarts != 0 {
|
||||
t.Fatalf("unsafe classification: %v restarts=%d", err, restarts)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExperimentalServiceRecoveryPreservesOwnership(t *testing.T) {
|
||||
d, front, _ := recoveryFixture(t, true)
|
||||
target, _, _ := managedCLIFixture(t)
|
||||
initManagedCLI(t, target)
|
||||
d.transaction.Dir = target.directory
|
||||
asset := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
|
||||
id := managedCandidateFixture(t, d.db, "googlevideo", "rr1---sn-alpha.googlevideo.com", managedGooglevideoArgs, false)
|
||||
ids := []int64{front, asset, id}
|
||||
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeExperimental); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := d.transaction.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.service = true
|
||||
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
|
||||
d.health = func(context.Context) (string, error) { return "OK", nil }
|
||||
d.resources = func(context.Context) (managedServiceInventory, error) {
|
||||
return managedServiceInventory{media: experimentalResourceFixture()}, &managedResourceFailure{"avatar", "yt3.ggpht.com", nil}
|
||||
}
|
||||
calls := 0
|
||||
d.recheckExperimental = func(_ context.Context, c storedCandidate, r mediaResource) error {
|
||||
calls++
|
||||
if c.ID != id || !freshExperimentalMedia(r) {
|
||||
t.Fatal("wrong committed candidate/resource")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, adaptive := range []bool{false, true} {
|
||||
d.adaptive = adaptive
|
||||
result, err := recoverManagedTarget(context.Background(), d)
|
||||
if err != nil || result.GooglevideoID != id || result.GGCError != nil {
|
||||
t.Fatalf("%+v %v", result, err)
|
||||
}
|
||||
}
|
||||
after, err := d.transaction.load()
|
||||
if err != nil || !reflect.DeepEqual(before, after) || calls != 2 {
|
||||
t.Fatalf("journal changed: %v calls=%d", err, calls)
|
||||
}
|
||||
c, err := readStrategyCandidate(d.db, id)
|
||||
if err != nil || c.Qualified {
|
||||
t.Fatalf("qualification forged: %+v %v", c, err)
|
||||
}
|
||||
monitor := managedMonitorDeps{db: d.db, target: target}
|
||||
reports := 0
|
||||
if err := wireManagedRecovery(&monitor, d, func(err error) {
|
||||
if errors.Is(err, errManagedExperimentalUnqualified) {
|
||||
reports++
|
||||
}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := monitor.probe(context.Background()); err != nil || reports != 1 {
|
||||
t.Fatalf("operational probe: %v reports=%d", err, reports)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, ids)
|
||||
}
|
||||
|
||||
func TestExperimentalRefreshAndMixedFatalPrecedence(t *testing.T) {
|
||||
bad := errors.New("invalid media range or container")
|
||||
for _, err := range []error{
|
||||
&managedRecoveryFatal{Err: bad},
|
||||
fmt.Errorf("wrapped: %w", &managedRecoveryFatal{Err: bad}),
|
||||
errors.Join(resourceFailure(context.Background(), bad), &managedRecoveryFatal{Err: bad}),
|
||||
errors.Join(resourceFailure(context.Background(), bad), context.Canceled),
|
||||
} {
|
||||
if managedGooglevideoTransferNonpass(err) {
|
||||
t.Fatalf("fatal accepted: %v", err)
|
||||
}
|
||||
}
|
||||
refreshes, transfers := 0, 0
|
||||
err := recheckDeployedExperimental(context.Background(), mediaResource{}, func() error { return nil }, func(context.Context) error {
|
||||
t.Fatal("unknown media caused restart")
|
||||
return nil
|
||||
}, func(context.Context) (mediaResource, error) {
|
||||
refreshes++
|
||||
return experimentalResourceFixture(), nil
|
||||
}, func(context.Context, mediaResource, string) error { transfers++; return nil })
|
||||
if err != nil || refreshes != 1 || transfers != 6 {
|
||||
t.Fatalf("refresh: %v %d %d", err, refreshes, transfers)
|
||||
}
|
||||
err = recheckDeployedExperimental(context.Background(), mediaResource{}, func() error { return nil }, func(context.Context) error {
|
||||
t.Fatal("unknown media caused restart")
|
||||
return nil
|
||||
}, func(context.Context) (mediaResource, error) {
|
||||
return mediaResource{}, &managedResourceFailure{"media", "unknown", nil}
|
||||
}, func(context.Context, mediaResource, string) error { t.Fatal("unknown media transferred"); return nil })
|
||||
if err == nil || errors.Is(err, errNoQualifiedStrategy) {
|
||||
t.Fatalf("unknown treated as absence: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -78,16 +78,15 @@ func managedAdaptiveAssetsQualified(d managedRecoveryDeps, ids []int64) (bool, e
|
||||
}
|
||||
|
||||
func recoverManagedGooglevideo(ctx context.Context, d managedRecoveryDeps) (int64, error) {
|
||||
if !d.service || d.mediaSamples == nil || d.verifyGooglevideo == nil {
|
||||
return 0, &InsufficientAdaptiveSamples{}
|
||||
}
|
||||
m := *d.transaction
|
||||
journal, err := m.load()
|
||||
if err != nil {
|
||||
return 0, &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
if journal.Scope == GooglevideoScopeExperimental {
|
||||
// Explicit operator policy remains owned, never auto-qualified/replaced.
|
||||
return recoverManagedExperimental(ctx, d)
|
||||
}
|
||||
if !d.service || d.mediaSamples == nil || d.verifyGooglevideo == nil {
|
||||
return 0, &InsufficientAdaptiveSamples{}
|
||||
}
|
||||
current, err := m.ActiveCandidates(d.db)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -185,6 +186,43 @@ func (r managedAssetRuntime) verifyGooglevideo(ctx context.Context, c storedCand
|
||||
}
|
||||
|
||||
func managedGooglevideoTransferNonpass(err error) bool {
|
||||
return managedResourceTransferNonpass(err, false)
|
||||
}
|
||||
|
||||
func managedResourceTransferNonpass(err error, metadata bool) bool {
|
||||
// Collector normalization wraps known transfer failures as fatal for generic
|
||||
// consumers. Only that boundary may unwrap the wrapper; guard/cleanup failures
|
||||
// and mixed error sets must never authorize absence or a daemon restart.
|
||||
if e, ok := err.(*resourceError); ok {
|
||||
cause := e.cause
|
||||
if fatal, ok := cause.(*managedRecoveryFatal); ok {
|
||||
cause = fatal.Err
|
||||
}
|
||||
return managedResourceTransferNonpass(cause, metadata)
|
||||
}
|
||||
if _, ok := err.(*managedRecoveryFatal); ok {
|
||||
return false
|
||||
}
|
||||
if joined, ok := err.(interface{ Unwrap() []error }); ok {
|
||||
children := joined.Unwrap()
|
||||
if len(children) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, child := range children {
|
||||
if !managedResourceTransferNonpass(child, metadata) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
|
||||
return managedResourceTransferNonpass(wrapped.Unwrap(), metadata)
|
||||
}
|
||||
// Metadata curl uses --fail. A non-2xx response is missing inventory,
|
||||
// never media rejection or candidate-absence evidence.
|
||||
if exit, ok := err.(*exec.ExitError); metadata && ok && exit.ExitCode() == 22 {
|
||||
return true
|
||||
}
|
||||
if managedAssetTransferNonpass(err) {
|
||||
return true
|
||||
}
|
||||
@@ -194,9 +232,6 @@ func managedGooglevideoTransferNonpass(err error) bool {
|
||||
if err.Error() == "invalid media range or container" {
|
||||
return true
|
||||
}
|
||||
if e, ok := err.(interface{ Unwrap() error }); ok {
|
||||
return managedGooglevideoTransferNonpass(e.Unwrap())
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -191,6 +191,7 @@ func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand
|
||||
d, err := managedRecoveryDependencies(target, db, scan, check, run)
|
||||
if err == nil && d.service {
|
||||
d.verifyAssets = newManagedAssetVerifier(target)
|
||||
d.recheckExperimental = newManagedExperimentalRecheck(target, d)
|
||||
if d.adaptive {
|
||||
d.verifyGooglevideo = newManagedGooglevideoVerifier(target)
|
||||
}
|
||||
|
||||
@@ -488,7 +488,7 @@ func TestManagedMonitorProductionAbsenceDiagnostics(t *testing.T) {
|
||||
var fatal *managedRecoveryFatal
|
||||
wantFatal := strings.Contains(mode, "nfq")
|
||||
if errors.As(err, &fatal) != wantFatal || (!wantFatal && !errors.Is(err, context.Canceled)) {
|
||||
t.Fatalf("mode=%s err=%v", mode, err)
|
||||
t.Fatalf("mode=%s err=%v reported=%v", mode, err, reported)
|
||||
}
|
||||
if strings.Contains(errors.Join(err, reported).Error(), "secret-never-print") {
|
||||
t.Fatal("diagnostics leaked")
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestManagedSingleGroupCLI(t *testing.T) {
|
||||
t.Fatalf("apply: %d %s %s", code, out, diagnostic)
|
||||
}
|
||||
code, out, diagnostic := callManaged(t, d, "status")
|
||||
if code != 0 || !strings.Contains(out, "groups="+group+" ") || !strings.Contains(out, "unresolved_groups="+missing+"\n") || !strings.Contains(out, "Live health: NOT CHECKED") {
|
||||
if code != 0 || !strings.Contains(out, "groups="+group+" ") || !strings.Contains(out, "unresolved_groups="+missing+" covered="+group+"\n") || !strings.Contains(out, "Live health: NOT CHECKED") {
|
||||
t.Fatalf("partial status: %d %s %s", code, out, diagnostic)
|
||||
}
|
||||
code, out, diagnostic = callManaged(t, d, "active-ids")
|
||||
|
||||
@@ -35,10 +35,11 @@ type managedRecoveryDeps struct {
|
||||
resources func(context.Context) (managedServiceInventory, error)
|
||||
// Verify owns temporary raw-config activation and must join/restore before
|
||||
// returning. It must not touch permanent config or the transaction journal.
|
||||
verifyAssets func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error)
|
||||
adaptive bool
|
||||
mediaSamples func(context.Context) ([]mediaResource, error)
|
||||
verifyGooglevideo func(context.Context, storedCandidate, []mediaResource) (GooglevideoQualificationProof, error)
|
||||
verifyAssets func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error)
|
||||
adaptive bool
|
||||
mediaSamples func(context.Context) ([]mediaResource, error)
|
||||
verifyGooglevideo func(context.Context, storedCandidate, []mediaResource) (GooglevideoQualificationProof, error)
|
||||
recheckExperimental func(context.Context, storedCandidate, mediaResource) error
|
||||
}
|
||||
|
||||
// Target-local adapter: reuses bounded Player requests without invoking a CLI.
|
||||
@@ -267,11 +268,18 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
||||
if d.service {
|
||||
_, healthErr := d.health(ctx)
|
||||
var fatal *managedRecoveryFatal
|
||||
var failure *managedResourceFailure
|
||||
j, e := m.load()
|
||||
if e != nil {
|
||||
return result, &managedRecoveryFatal{Err: e}
|
||||
}
|
||||
if j.Scope == GooglevideoScopeExperimental {
|
||||
healthErr = managedResourceHealthError(ctx, healthErr)
|
||||
}
|
||||
if ctx.Err() != nil || errors.As(healthErr, &fatal) || errors.Is(healthErr, context.Canceled) || errors.Is(healthErr, context.DeadlineExceeded) {
|
||||
return result, managedRecoveryContextError(ctx, healthErr)
|
||||
}
|
||||
var failure *managedResourceFailure
|
||||
if errors.As(healthErr, &failure) && assetEndpoint(failure.Host) {
|
||||
if errors.As(healthErr, &failure) && assetEndpoint(failure.Host) && j.Scope != GooglevideoScopeExperimental {
|
||||
return result, repairManagedAssets(ctx, d, failure.Host)
|
||||
}
|
||||
}
|
||||
@@ -440,9 +448,7 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
||||
return result, e
|
||||
}
|
||||
if experimental {
|
||||
// Operator extrapolation survives recovery in either configured mode.
|
||||
// Frontend/asset repairs remain available, but cannot qualify this policy.
|
||||
result.GGCError = errManagedGGCUnresolved
|
||||
result.GooglevideoID, result.GGCError = recoverManagedExperimental(ctx, d)
|
||||
return result, result.GGCError
|
||||
}
|
||||
if !d.service && (len(current) != 1 || current[0] != result.FrontendID) {
|
||||
@@ -716,14 +722,6 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
||||
if err := guard(); err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
status, err := recovery.health(ctx)
|
||||
err = managedRecoveryContextError(ctx, err)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if status != "OK" {
|
||||
return &managedHealthRejection{Status: status}
|
||||
}
|
||||
transaction := *recovery.transaction
|
||||
transaction.Health = func() (string, error) { return recovery.health(ctx) }
|
||||
// ActiveCandidates is read-only; never permit a probe to reload config.
|
||||
@@ -737,8 +735,25 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
||||
if err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
status, err := recovery.health(ctx)
|
||||
if journal.Scope == GooglevideoScopeExperimental && recovery.service && recovery.recheckExperimental != nil {
|
||||
err = managedResourceHealthError(ctx, err)
|
||||
} else {
|
||||
err = managedRecoveryContextError(ctx, err)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if status != "OK" {
|
||||
return &managedHealthRejection{Status: status}
|
||||
}
|
||||
if journal.Scope == GooglevideoScopeExperimental {
|
||||
return errManagedGGCUnresolved
|
||||
if !recovery.service || recovery.recheckExperimental == nil {
|
||||
return errManagedGGCUnresolved
|
||||
}
|
||||
// Operational service health is not family qualification.
|
||||
reportGGC(errManagedExperimentalUnqualified)
|
||||
return nil
|
||||
}
|
||||
// Two bounded Player requests reuse the existing health/discovery adapters.
|
||||
host, err := recovery.discover(ctx)
|
||||
|
||||
Reference in new issue
Block a user