Keep verified curl peer TLS negatives unqualified without making them process-wide safety failures. Preserve local proxy/configuration, guard, cleanup and mixed cancellation vetoes; test owned verification, rollback and durable absence cooldown. Co-authored-by: GPT-6 <gpt-6@openai.com>
352 lines
15 KiB
Go
352 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestManagedServiceOptIn(t *testing.T) {
|
|
for _, value := range []string{"", "0", "1", "true", "secret"} {
|
|
enabled, err := managedServiceEnabled(func(key string) (string, bool) { return value, key == "ZAPRET_AUTOTUNE_YOUTUBE_SERVICE" && value != "" })
|
|
valid := value == "" || value == "0" || value == "1"
|
|
if (err == nil) != valid || (valid && enabled != (value == "1")) {
|
|
t.Fatalf("value %q: %v %v", value, enabled, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestManagedServiceMissingResourcesNeverGreen(t *testing.T) {
|
|
front := func(context.Context) (string, error) { return "OK", nil }
|
|
for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} {
|
|
i := managedServiceInventory{images: map[string]string{}}
|
|
if host == "yt3.ggpht.com" {
|
|
i.images["i.ytimg.com"] = "https://i.ytimg.com/test"
|
|
}
|
|
// The first missing image must fail before any resource process is spawned.
|
|
if host == "yt3.ggpht.com" {
|
|
continue
|
|
}
|
|
health := managedServiceHealth("socks5://127.0.0.1:11080", front, func(context.Context) (managedServiceInventory, error) { return i, nil }, func(context.Context, []string, io.Reader, io.Writer, io.Writer) error {
|
|
t.Fatal("unexpected transfer")
|
|
return nil
|
|
})
|
|
status, err := health(context.Background())
|
|
var failure *managedResourceFailure
|
|
if status != "DEGRADED" || !errors.As(err, &failure) || failure.Host != host {
|
|
t.Fatalf("%s %v", status, err)
|
|
}
|
|
}
|
|
if public := (&managedResourceFailure{"https://secret/?token=x", "secret/?token=x", errors.New("private")}).PublicError(); strings.Contains(public, "secret") || strings.Contains(public, "private") {
|
|
t.Fatal(public)
|
|
}
|
|
}
|
|
|
|
func TestManagedServiceFullDesiredSet(t *testing.T) {
|
|
d, front, _ := recoveryFixture(t, true)
|
|
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
|
|
thumb := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
|
|
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
|
|
newFront := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=multisplit", true)
|
|
ids, err := managedFullDesiredSet(d.db, []int64{front, ggc, thumb, avatar}, []int64{newFront})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(ids) != 4 || ids[0] != ggc || ids[1] != thumb || ids[2] != avatar || ids[3] != newFront {
|
|
t.Fatal(ids)
|
|
}
|
|
}
|
|
|
|
func TestManagedServiceSavedAssetRepairPreservesFullSet(t *testing.T) {
|
|
d, front, events := recoveryFixture(t, true)
|
|
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
|
|
old := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
|
|
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
|
|
saved := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=multisplit", true)
|
|
if err := d.transaction.Apply(d.db, []int64{front, ggc, old, avatar}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/fresh"}}, nil
|
|
}
|
|
joined := false
|
|
d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) {
|
|
if c.ID != saved || len(*events) == 0 || (*events)[len(*events)-1] != "stop" {
|
|
t.Fatal("not saved-first under downtime", c.ID, *events)
|
|
}
|
|
joined = true
|
|
return []assetMeasurement{unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args)}, nil
|
|
}
|
|
d.health = func(context.Context) (string, error) {
|
|
if !joined {
|
|
t.Fatal("apply before verifier joined")
|
|
}
|
|
return "OK", nil
|
|
}
|
|
if err := repairManagedAssets(context.Background(), d, "i.ytimg.com"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ids, err := d.transaction.ActiveCandidates(d.db)
|
|
if err != nil || len(ids) != 4 {
|
|
t.Fatal(ids, err)
|
|
}
|
|
for _, required := range []int64{front, ggc, avatar} {
|
|
found := false
|
|
for _, id := range ids {
|
|
found = found || id == required
|
|
}
|
|
if !found {
|
|
t.Fatal("lost sibling", required, ids)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestManagedServiceAssetRollbackAndCancellation(t *testing.T) {
|
|
for _, cancelJob := range []bool{false, true} {
|
|
t.Run(map[bool]string{false: "false-media-rollback", true: "cancel-joins"}[cancelJob], func(t *testing.T) {
|
|
d, front, events := recoveryFixture(t, true)
|
|
old := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
|
|
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
|
|
if err := d.transaction.Apply(d.db, []int64{front, old, avatar}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=multisplit", true)
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/fresh"}}, nil
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
joined := false
|
|
d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) {
|
|
joined = true
|
|
if cancelJob {
|
|
cancel()
|
|
return nil, context.Canceled
|
|
}
|
|
return []assetMeasurement{unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args)}, nil
|
|
}
|
|
d.health = func(context.Context) (string, error) {
|
|
return "DEGRADED", &managedResourceFailure{"media", "fresh.googlevideo.com", nil}
|
|
}
|
|
err := repairManagedAssets(ctx, d, "i.ytimg.com")
|
|
if err == nil || !joined {
|
|
t.Fatal("unverified success or unjoined verifier", err)
|
|
}
|
|
if cancelJob && (!errors.Is(err, context.Canceled) || (*events)[len(*events)-1] != "start") {
|
|
t.Fatal("missing bounded restoration", err, *events)
|
|
}
|
|
assertActiveCandidates(t, d.transaction, d.db, []int64{front, old, avatar})
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedServiceFreshPrivateInventory(t *testing.T) {
|
|
cfg := healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080"}
|
|
resources := managedServiceResources(cfg, func(context.Context) ([]byte, error) {
|
|
return []byte(`{"playabilityStatus":{"status":"OK"},"videoDetails":{"thumbnail":{"thumbnails":[{"url":"https://i.ytimg.com/fresh?private=thumbnail"}]}},"streamingData":{}}`), nil
|
|
}, func(_ context.Context, args []string, out, _ io.Writer) error {
|
|
if args[len(args)-1] != "https://www.youtube.com/" || strings.Contains(strings.Join(args, " "), "private=") {
|
|
t.Fatal(args)
|
|
}
|
|
_, err := io.WriteString(out, `var ytInitialData = {"avatar":{"url":"https://yt3.ggpht.com/fresh?private=avatar"}};`)
|
|
return err
|
|
})
|
|
i, err := resources(context.Background())
|
|
if err != nil || len(i.images) != 2 || i.media.url != "" {
|
|
t.Fatal("fresh inventory or SABR unknown contract", err)
|
|
}
|
|
}
|
|
|
|
func TestManagedServicePartialInventoryChecksAndRepairsAvailableImage(t *testing.T) {
|
|
for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} {
|
|
t.Run(host, func(t *testing.T) {
|
|
d, front, _ := recoveryFixture(t, true)
|
|
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
|
|
thumb := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
|
|
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
|
|
if err := d.transaction.Apply(d.db, []int64{front, ggc, thumb, avatar}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, _, _ := assetRuntimeFixture(t)
|
|
d.service = true
|
|
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
|
|
missingHost := "yt3.ggpht.com"
|
|
if host == missingHost {
|
|
missingHost = "i.ytimg.com"
|
|
}
|
|
missing := &managedResourceFailure{managedImageProbeKind(missingHost), missingHost, errors.New("private metadata unavailable")}
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
return managedServiceInventory{images: map[string]string{host: "https://" + host + "/image?token=private"}, proxy: r.proxy, runner: r.curl, resourceErr: &managedResourceFailure{"media", "unknown", nil}}, missing
|
|
}
|
|
repaired, checked := false, false
|
|
d.health = managedServiceHealth(r.proxy, d.frontHealth, d.resources, func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
|
|
checked = true
|
|
if !repaired {
|
|
resourceWire("https://"+host+"/image?token=private", "cn", out, diag)
|
|
return nil
|
|
}
|
|
return r.curl(ctx, args, in, out, diag)
|
|
})
|
|
status, err := d.health(context.Background())
|
|
var failure *managedResourceFailure
|
|
if status != "DEGRADED" || !checked || !errors.As(err, &failure) || failure.Host != host || failure.ProbeKind != managedImageProbeKind(host) || strings.Contains(err.Error(), "private") {
|
|
t.Fatal(status, checked, err)
|
|
}
|
|
d.verifyAssets = func(ctx context.Context, c storedCandidate, i managedServiceInventory) ([]assetMeasurement, error) {
|
|
m, err := r.verify(ctx, c, i)
|
|
repaired = err == nil
|
|
return m, err
|
|
}
|
|
if _, err := recoverManagedTarget(context.Background(), d); err != nil || !repaired {
|
|
t.Fatal("partial metadata blocked independent asset repair", err)
|
|
}
|
|
ids, err := d.transaction.ActiveCandidates(d.db)
|
|
if err != nil || len(ids) != 4 {
|
|
t.Fatal("lost full desired set", ids, err)
|
|
}
|
|
status, err = d.health(context.Background())
|
|
if status != "DEGRADED" || !errors.As(err, &failure) || failure.Host != missingHost {
|
|
t.Fatal("missing resources became green", status, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedServiceBootstrapCommitsFrontendBeforeAssetRepair(t *testing.T) {
|
|
d, _, _ := recoveryFixture(t, false)
|
|
d.service = true
|
|
r, _, _ := assetRuntimeFixture(t)
|
|
frontChecks := 0
|
|
d.frontHealth = func(context.Context) (string, error) {
|
|
frontChecks++
|
|
if frontChecks == 1 {
|
|
return "DEGRADED", nil
|
|
}
|
|
return "OK", nil
|
|
}
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}, proxy: r.proxy, runner: r.curl, resourceErr: &managedResourceFailure{"media", "unknown", nil}}, nil
|
|
}
|
|
repaired := false
|
|
d.health = managedServiceHealth(r.proxy, d.frontHealth, d.resources, func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
|
|
if !repaired {
|
|
resourceWire("https://i.ytimg.com/image", "cn", out, diag)
|
|
return nil
|
|
}
|
|
return r.curl(ctx, args, in, out, diag)
|
|
})
|
|
result, err := recoverManagedTarget(context.Background(), d)
|
|
if err != nil || result.FrontendID == 0 || result.GooglevideoID == 0 {
|
|
t.Fatal("broken thumbnail vetoed frontend commit", result, err)
|
|
}
|
|
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
|
|
d.verifyAssets = func(ctx context.Context, c storedCandidate, i managedServiceInventory) ([]assetMeasurement, error) {
|
|
m, err := r.verify(ctx, c, i)
|
|
repaired = err == nil
|
|
return m, err
|
|
}
|
|
if _, err := recoverManagedTarget(context.Background(), d); err != nil || !repaired {
|
|
t.Fatal("thumbnail repair blocked by missing media", err)
|
|
}
|
|
ids, err := d.transaction.ActiveCandidates(d.db)
|
|
if err != nil || len(ids) != 3 || ids[0] != result.FrontendID || ids[1] != result.GooglevideoID {
|
|
t.Fatal("asset commit lost frontend/GGC", ids, err)
|
|
}
|
|
status, err := d.health(context.Background())
|
|
var failure *managedResourceFailure
|
|
if status != "DEGRADED" || !errors.As(err, &failure) || failure.ProbeKind != "media" {
|
|
t.Fatal("missing media became service-ready", status, err)
|
|
}
|
|
}
|
|
|
|
func TestManagedServiceAssetLocalHealthFailureRollsBack(t *testing.T) {
|
|
d, front, _ := recoveryFixture(t, true)
|
|
old := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
|
|
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
|
|
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
|
|
prior := []int64{front, ggc, old, avatar}
|
|
if err := d.transaction.Apply(d.db, prior); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before, err := d.transaction.read("config")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, _, _ := assetRuntimeFixture(t)
|
|
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}, proxy: r.proxy, runner: func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
|
|
input, err := io.ReadAll(in)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if strings.Contains(string(input), "i.ytimg.com") {
|
|
resourceWire("https://i.ytimg.com/image", "truncated", out, diag)
|
|
return nil
|
|
}
|
|
return r.curl(ctx, args, strings.NewReader(string(input)), out, diag)
|
|
}}, nil
|
|
}
|
|
i, _ := d.resources(context.Background())
|
|
if status, err := managedAssetInventoryHealth(context.Background(), i, []string{"yt3.ggpht.com"}); status != "OK" || err != nil {
|
|
t.Fatal("old sibling must remain healthy", status, err)
|
|
}
|
|
d.verifyAssets = r.verify
|
|
if err := repairManagedAssets(context.Background(), d, "i.ytimg.com"); !errors.Is(err, errNoQualifiedStrategy) {
|
|
t.Fatal(err)
|
|
}
|
|
assertActiveCandidates(t, d.transaction, d.db, prior)
|
|
after, err := d.transaction.read("config")
|
|
if err != nil || string(before) != string(after) {
|
|
t.Fatal("rejected asset changed prior config", err)
|
|
}
|
|
}
|
|
|
|
func TestManagedAssetHealthNonpassPreservesMixedSafetyFailures(t *testing.T) {
|
|
badBody := resourceFailure(context.Background(), errors.New("invalid image body"))
|
|
for _, tc := range []struct {
|
|
err error
|
|
nonpass bool
|
|
}{
|
|
{badBody, true},
|
|
{&managedRecoveryFatal{Err: errors.New("invalid image body")}, false},
|
|
{errors.Join(badBody, &managedRecoveryFatal{Err: errors.New("guard failed")}), false},
|
|
{errors.Join(badBody, context.Canceled), false},
|
|
{&managedAssetCleanupFailure{Err: errors.New("cleanup failed")}, false},
|
|
} {
|
|
if managedAssetHealthNonpass(tc.err) != tc.nonpass {
|
|
t.Fatal(tc.err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestManagedAssetRepairMixedFatalNeverRetries(t *testing.T) {
|
|
for _, stage := range []string{"verify", "health"} {
|
|
t.Run(stage, func(t *testing.T) {
|
|
d, front, _ := recoveryFixture(t, true)
|
|
r, _, _ := assetRuntimeFixture(t)
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}, proxy: r.proxy, runner: r.curl}, nil
|
|
}
|
|
mixed := errors.Join(&managedRecoveryFatal{Err: errors.New("guard failed")}, errCandidateNonpass, &managedResourceFailure{"thumbnail", "i.ytimg.com", nil})
|
|
calls := 0
|
|
d.verifyAssets = func(ctx context.Context, c storedCandidate, i managedServiceInventory) ([]assetMeasurement, error) {
|
|
calls++
|
|
if stage == "verify" {
|
|
return nil, mixed
|
|
}
|
|
return r.verify(ctx, c, i)
|
|
}
|
|
d.frontHealth = func(context.Context) (string, error) { return "DEGRADED", mixed }
|
|
err := repairManagedAssets(context.Background(), d, "i.ytimg.com")
|
|
var fatal *managedRecoveryFatal
|
|
if !errors.As(err, &fatal) || calls != 1 {
|
|
t.Fatal("mixed safety failure retried as rejection", calls, err)
|
|
}
|
|
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
|
})
|
|
}
|
|
}
|