Files
zapret-autotune/cmd/zapret-autotune/managed_recovery_regression_test.go
T

157 lines
5.7 KiB
Go

package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"testing"
"time"
)
func TestManagedExpiredAdaptiveJournalRecovery(t *testing.T) {
for _, scenario := range []string{"committed", "prepared", "rollback"} {
t.Run(scenario, func(t *testing.T) {
d, front, _ := recoveryFixture(t, true)
r, c, resources, _ := googlevideoRuntimeFixture(t)
proof, err := r.verifyGooglevideo(context.Background(), c, resources)
if err != nil {
t.Fatal(err)
}
tx, err := d.db.Begin()
if err != nil {
t.Fatal(err)
}
id, err := insertStrategyCandidate(tx, c.Group, c.Endpoints, c.Args, nil, c.Mode)
if err == nil {
err = saveAdaptiveGooglevideoEvidence(tx, id, proof)
}
if err != nil {
_ = tx.Rollback()
t.Fatal(err)
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
ids := []int64{front, unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true), unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true), id}
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); err != nil {
t.Fatal(err)
}
before, err := d.transaction.read("config")
if err != nil {
t.Fatal(err)
}
if scenario == "prepared" {
crash := errors.New("process died after replacement")
d.transaction.fault = func(point string) error {
if point == "replaced" {
return crash
}
return nil
}
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); !errors.Is(err, crash) {
t.Fatal(err)
}
d.transaction.fault = nil
}
proof.Attestation.CheckedAt = time.Now().Add(-25 * time.Hour)
raw, err := json.Marshal(proof)
if err != nil {
t.Fatal(err)
}
if _, err := d.db.Exec("UPDATE googlevideo_scope_evidence SET proof=? WHERE candidate_id=?", string(raw), id); err != nil {
t.Fatal(err)
}
if err := d.transaction.Reconcile(d.db); err != nil {
t.Fatal("aged committed/pending refs prevented reconciliation", err)
}
assertActiveCandidates(t, d.transaction, d.db, ids)
stored, err := readStrategyCandidate(d.db, id)
if err != nil || stored.Qualified {
t.Fatal("aged proof authorized fresh qualification", stored, err)
}
if err := validateGooglevideoSuffixEvidence(d.db, stored); err == nil {
t.Fatal("aged proof authorized new suffix")
}
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); err == nil {
t.Fatal("aged proof authorized new apply")
}
if err := d.transaction.applyRenderedDB(before, d.db, ids...); err == nil {
t.Fatal("pre-rendered bytes bypassed freshness")
}
d.service, d.adaptive = true, true
d.frontHealth = d.health
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/sample.jpg", "yt3.ggpht.com": "https://yt3.ggpht.com/sample.jpg"}, proxy: r.proxy, runner: adaptiveImageRunner}, nil
}
samples, verifies := 0, 0
d.mediaSamples = func(context.Context) ([]mediaResource, error) {
samples++
return resources, nil
}
d.verifyGooglevideo = func(ctx context.Context, c storedCandidate, resources []mediaResource) (GooglevideoQualificationProof, error) {
verifies++
return r.verifyGooglevideo(ctx, c, resources)
}
if scenario == "rollback" {
d.health = func(context.Context) (string, error) {
return "DEGRADED", &managedResourceFailure{"media", resources[0].host, nil}
}
}
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || samples != 1 || verifies != 1 {
t.Fatal("aged active state blocked fresh owned verification", result, samples, verifies, err)
}
if scenario == "rollback" {
if result.GGCError == nil {
t.Fatal("unhealthy apply accepted")
}
assertActiveCandidates(t, d.transaction, d.db, ids)
after, err := d.transaction.read("config")
if err != nil || !reflect.DeepEqual(before, after) {
t.Fatal("aged rollback changed committed bytes", err)
}
} else {
if result.GGCError != nil || result.GooglevideoID <= id {
t.Fatal("fresh revalidation did not replace expired proof", result)
}
ids[len(ids)-1] = result.GooglevideoID
assertActiveCandidates(t, d.transaction, d.db, ids)
}
// Expiry alone is safe; malformed/invalid historical proof is not.
activeID := ids[len(ids)-1]
if _, err := d.db.Exec("UPDATE googlevideo_scope_evidence SET proof='{}' WHERE candidate_id=?", activeID); err != nil {
t.Fatal(err)
}
if _, err := d.transaction.ActiveCandidates(d.db); err == nil {
t.Fatal("invalid committed proof accepted")
}
if err := d.transaction.Reconcile(d.db); err == nil {
t.Fatal("invalid committed proof reconciled")
}
})
}
}
func TestManagedServiceDispatchSafetyPrecedesImageRepair(t *testing.T) {
for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} {
for _, failure := range []error{&managedRecoveryFatal{Err: errors.New("guard failed")}, context.Canceled, context.DeadlineExceeded} {
t.Run(host+"/"+failure.Error(), func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
d.service = true
mixed := errors.Join(&managedResourceFailure{managedImageProbeKind(host), host, nil}, failure)
d.health = func(context.Context) (string, error) { return "DEGRADED", mixed }
d.resources = func(context.Context) (managedServiceInventory, error) {
t.Fatal("mixed safety failure dispatched image repair")
return managedServiceInventory{}, nil
}
_, err := recoverManagedTarget(context.Background(), d)
if !errors.Is(err, failure) || len(*events) != 0 {
t.Fatal("joined safety failure swallowed/retried", err, *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
})
}
}
}