157 lines
5.7 KiB
Go
157 lines
5.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestManagedExpiredAdaptiveJournalRecovery(t *testing.T) {
|
|
for _, scenario := range []string{"committed", "prepared", "rollback"} {
|
|
t.Run(scenario, func(t *testing.T) {
|
|
d, front, _ := recoveryFixture(t, true)
|
|
r, c, resources, _ := googlevideoRuntimeFixture(t)
|
|
proof, err := r.verifyGooglevideo(context.Background(), c, resources)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tx, err := d.db.Begin()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id, err := insertStrategyCandidate(tx, c.Group, c.Endpoints, c.Args, nil, c.Mode)
|
|
if err == nil {
|
|
err = saveAdaptiveGooglevideoEvidence(tx, id, proof)
|
|
}
|
|
if err != nil {
|
|
_ = tx.Rollback()
|
|
t.Fatal(err)
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ids := []int64{front, unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true), unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true), id}
|
|
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before, err := d.transaction.read("config")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if scenario == "prepared" {
|
|
crash := errors.New("process died after replacement")
|
|
d.transaction.fault = func(point string) error {
|
|
if point == "replaced" {
|
|
return crash
|
|
}
|
|
return nil
|
|
}
|
|
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); !errors.Is(err, crash) {
|
|
t.Fatal(err)
|
|
}
|
|
d.transaction.fault = nil
|
|
}
|
|
proof.Attestation.CheckedAt = time.Now().Add(-25 * time.Hour)
|
|
raw, err := json.Marshal(proof)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := d.db.Exec("UPDATE googlevideo_scope_evidence SET proof=? WHERE candidate_id=?", string(raw), id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := d.transaction.Reconcile(d.db); err != nil {
|
|
t.Fatal("aged committed/pending refs prevented reconciliation", err)
|
|
}
|
|
assertActiveCandidates(t, d.transaction, d.db, ids)
|
|
stored, err := readStrategyCandidate(d.db, id)
|
|
if err != nil || stored.Qualified {
|
|
t.Fatal("aged proof authorized fresh qualification", stored, err)
|
|
}
|
|
if err := validateGooglevideoSuffixEvidence(d.db, stored); err == nil {
|
|
t.Fatal("aged proof authorized new suffix")
|
|
}
|
|
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); err == nil {
|
|
t.Fatal("aged proof authorized new apply")
|
|
}
|
|
if err := d.transaction.applyRenderedDB(before, d.db, ids...); err == nil {
|
|
t.Fatal("pre-rendered bytes bypassed freshness")
|
|
}
|
|
d.service, d.adaptive = true, true
|
|
d.frontHealth = d.health
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/sample.jpg", "yt3.ggpht.com": "https://yt3.ggpht.com/sample.jpg"}, proxy: r.proxy, runner: adaptiveImageRunner}, nil
|
|
}
|
|
samples, verifies := 0, 0
|
|
d.mediaSamples = func(context.Context) ([]mediaResource, error) {
|
|
samples++
|
|
return resources, nil
|
|
}
|
|
d.verifyGooglevideo = func(ctx context.Context, c storedCandidate, resources []mediaResource) (GooglevideoQualificationProof, error) {
|
|
verifies++
|
|
return r.verifyGooglevideo(ctx, c, resources)
|
|
}
|
|
if scenario == "rollback" {
|
|
d.health = func(context.Context) (string, error) {
|
|
return "DEGRADED", &managedResourceFailure{"media", resources[0].host, nil}
|
|
}
|
|
}
|
|
result, err := recoverManagedTarget(context.Background(), d)
|
|
if err != nil || samples != 1 || verifies != 1 {
|
|
t.Fatal("aged active state blocked fresh owned verification", result, samples, verifies, err)
|
|
}
|
|
if scenario == "rollback" {
|
|
if result.GGCError == nil {
|
|
t.Fatal("unhealthy apply accepted")
|
|
}
|
|
assertActiveCandidates(t, d.transaction, d.db, ids)
|
|
after, err := d.transaction.read("config")
|
|
if err != nil || !reflect.DeepEqual(before, after) {
|
|
t.Fatal("aged rollback changed committed bytes", err)
|
|
}
|
|
} else {
|
|
if result.GGCError != nil || result.GooglevideoID <= id {
|
|
t.Fatal("fresh revalidation did not replace expired proof", result)
|
|
}
|
|
ids[len(ids)-1] = result.GooglevideoID
|
|
assertActiveCandidates(t, d.transaction, d.db, ids)
|
|
}
|
|
// Expiry alone is safe; malformed/invalid historical proof is not.
|
|
activeID := ids[len(ids)-1]
|
|
if _, err := d.db.Exec("UPDATE googlevideo_scope_evidence SET proof='{}' WHERE candidate_id=?", activeID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := d.transaction.ActiveCandidates(d.db); err == nil {
|
|
t.Fatal("invalid committed proof accepted")
|
|
}
|
|
if err := d.transaction.Reconcile(d.db); err == nil {
|
|
t.Fatal("invalid committed proof reconciled")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedServiceDispatchSafetyPrecedesImageRepair(t *testing.T) {
|
|
for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} {
|
|
for _, failure := range []error{&managedRecoveryFatal{Err: errors.New("guard failed")}, context.Canceled, context.DeadlineExceeded} {
|
|
t.Run(host+"/"+failure.Error(), func(t *testing.T) {
|
|
d, front, events := recoveryFixture(t, true)
|
|
d.service = true
|
|
mixed := errors.Join(&managedResourceFailure{managedImageProbeKind(host), host, nil}, failure)
|
|
d.health = func(context.Context) (string, error) { return "DEGRADED", mixed }
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
t.Fatal("mixed safety failure dispatched image repair")
|
|
return managedServiceInventory{}, nil
|
|
}
|
|
_, err := recoverManagedTarget(context.Background(), d)
|
|
if !errors.Is(err, failure) || len(*events) != 0 {
|
|
t.Fatal("joined safety failure swallowed/retried", err, *events)
|
|
}
|
|
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
|
})
|
|
}
|
|
}
|
|
}
|