fix: keep rejected image payloads recoverable across verification
Type decoder payload rejections once and share the image nonpass classifier across live health and owned verification. Retain strict qualification, malformed metadata, source-kind separation and mixed safety vetoes; exercise seed fallback, joined workers, fresh evidence and durable absence cooldown. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
2804d97063
commit
e9a28c8cd3
9 files changed
+237
-19
No files matched your search
@@ -177,27 +177,36 @@ func validateAssetTransfers(m assetMeasurement) (float64, float64, error) {
|
||||
return latency[1], speed[1], nil
|
||||
}
|
||||
|
||||
// Only supplied body bytes can produce this typed rejection. Missing/corrupt
|
||||
// normalized evidence remains a local format error, never candidate absence.
|
||||
type imageBodyRejection string
|
||||
|
||||
func (e imageBodyRejection) Error() string { return string(e) }
|
||||
|
||||
// Bodies are transient. Stored evidence carries only verified normalized facts.
|
||||
func normalizeAssetBody(t *assetTransfer) error {
|
||||
if len(t.Body) != 0 {
|
||||
if len(t.Body) > 5<<20 {
|
||||
return fmt.Errorf("image body exceeds limit")
|
||||
return imageBodyRejection("image body exceeds limit")
|
||||
}
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(t.Body))
|
||||
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 || cfg.Width > (16<<20)/cfg.Height {
|
||||
return fmt.Errorf("invalid image dimensions")
|
||||
return imageBodyRejection("invalid image dimensions")
|
||||
}
|
||||
if format == "gif" {
|
||||
if _, err := gif.DecodeAll(bytes.NewReader(t.Body)); err != nil {
|
||||
return fmt.Errorf("invalid GIF body")
|
||||
return imageBodyRejection("invalid GIF body")
|
||||
}
|
||||
} else if _, _, err := image.Decode(bytes.NewReader(t.Body)); err != nil {
|
||||
return fmt.Errorf("invalid image body")
|
||||
return imageBodyRejection("invalid image body")
|
||||
}
|
||||
t.Format, t.Size, t.BodyHash = format, len(t.Body), fmt.Sprintf("%x", sha256.Sum256(t.Body))
|
||||
}
|
||||
want := map[string]string{"png": "image/png", "jpeg": "image/jpeg", "gif": "image/gif"}[t.Format]
|
||||
if want == "" || t.ContentType != want || t.Size <= 0 || t.Size > 5<<20 || len(t.BodyHash) != 64 || strings.Trim(t.BodyHash, "0123456789abcdef") != "" {
|
||||
if len(t.Body) != 0 {
|
||||
return imageBodyRejection("invalid image content type")
|
||||
}
|
||||
return fmt.Errorf("invalid normalized image evidence")
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func rejectedImageRunner(mode string) subprocessRunner {
|
||||
return func(_ context.Context, args []string, in io.Reader, out, diag io.Writer) error {
|
||||
config, err := io.ReadAll(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw := strings.TrimSuffix(strings.TrimPrefix(string(config), "url = \""), "\"\n")
|
||||
body, contentType := resourceFixture(), "image/png"
|
||||
switch mode {
|
||||
case "jpeg":
|
||||
var b bytes.Buffer
|
||||
if err := jpeg.Encode(&b, image.NewRGBA(image.Rect(0, 0, 8, 8)), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
body, contentType = b.Bytes(), "image/jpeg"
|
||||
if start := bytes.Index(body, []byte{0xff, 0xda}); start >= 0 {
|
||||
body = body[:start+12]
|
||||
}
|
||||
case "png":
|
||||
body = body[:len(body)/2]
|
||||
case "magic":
|
||||
body = []byte("not an image")
|
||||
case "gif":
|
||||
body, contentType = []byte("GIF89a\x01\x00\x01\x00\x00\x00\x00"), "image/gif"
|
||||
case "dimensions":
|
||||
body, contentType = []byte("GIF89a\xff\xff\xff\xff\x00\x00\x00"), "image/gif"
|
||||
case "type":
|
||||
contentType = "image/jpeg"
|
||||
case "unknown":
|
||||
return errors.New("invalid image body")
|
||||
}
|
||||
fmt.Fprintf(out, "HTTP/1.1 200 OK\r\nContent-Type: %s\r\nContent-Length: %d\r\n\r\n", contentType, len(body))
|
||||
if _, err := out.Write(body); err != nil {
|
||||
return err
|
||||
}
|
||||
metrics, _ := json.Marshal(map[string]interface{}{"url_effective": raw, "ssl_verify_result": 0, "num_redirects": 0, "http_code": 200, "time_total": 0.1, "speed_download": 10000, "size_download": len(body)})
|
||||
if mode == "footer" {
|
||||
metrics = []byte("{malformed")
|
||||
}
|
||||
fmt.Fprint(out, resourceTrailer, string(metrics))
|
||||
protocol := "TLSv1.2"
|
||||
if strings.Contains(strings.Join(args, " "), "--tlsv1.3") {
|
||||
protocol = "TLSv1.3"
|
||||
}
|
||||
fmt.Fprintln(diag, "SSL connection using "+protocol)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageBodyRejectionCollectorBoundaries(t *testing.T) {
|
||||
for _, mode := range []string{"jpeg", "png", "magic", "gif", "dimensions", "type", "unknown", "footer"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
m, err := collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", "https://i.ytimg.com/image", "i.ytimg.com", "tls12", "", rejectedImageRunner(mode))
|
||||
if err == nil || len(m.Transfers) != 0 {
|
||||
t.Fatal("invalid image qualified", m, err)
|
||||
}
|
||||
ordinary := mode != "unknown" && mode != "footer"
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(err, &fatal) == ordinary || managedAssetTransferNonpass(err) != ordinary || managedAssetHealthNonpass(err) != ordinary {
|
||||
t.Fatal("wrong rejection class", err)
|
||||
}
|
||||
if managedGooglevideoTransferNonpass(err) {
|
||||
t.Fatal("image rejection became media absence")
|
||||
}
|
||||
for _, veto := range []error{context.Canceled, errors.New("unknown infrastructure"), &managedRecoveryFatal{Err: imageBodyRejection("invalid image body")}} {
|
||||
mixed := resourceFailure(context.Background(), errors.Join(err, veto))
|
||||
if !errors.As(mixed, &fatal) || managedAssetTransferNonpass(mixed) || managedAssetHealthNonpass(mixed) {
|
||||
t.Fatal("mixed safety veto lost", mixed)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
_, _, metadataErr := playerMediaResource([]byte("{malformed"))
|
||||
if metadataErr == nil {
|
||||
t.Fatal("malformed Player JSON accepted")
|
||||
}
|
||||
for _, err := range []error{metadataErr, errors.New("invalid normalized image evidence")} {
|
||||
failure := resourceFailure(context.Background(), err)
|
||||
var fatal *managedRecoveryFatal
|
||||
if !errors.As(failure, &fatal) || managedAssetTransferNonpass(failure) || managedGooglevideoTransferNonpass(failure) {
|
||||
t.Fatal("local metadata damage became absence", failure)
|
||||
}
|
||||
}
|
||||
oversized := assetTransfer{Body: make([]byte, (5<<20)+1)}
|
||||
if err := normalizeAssetBody(&oversized); err == nil || !managedAssetTransferNonpass(resourceFailure(context.Background(), err)) {
|
||||
t.Fatal("image decoder budget rejection lost", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedTruncatedImageMonitorRecovery(t *testing.T) {
|
||||
for _, recoverable := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(recoverable), func(t *testing.T) {
|
||||
d, front, _ := recoveryFixture(t, true)
|
||||
target, _, _ := managedCLIFixture(t)
|
||||
initManagedCLI(t, target)
|
||||
d.transaction.Dir = target.directory
|
||||
r, events, _ := assetRuntimeFixture(t)
|
||||
good, start := r.curl, r.start
|
||||
attempts := 0
|
||||
r.start = func(args []string) (managedAssetWorker, error) { attempts++; return start(args) }
|
||||
r.curl = func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
|
||||
if recoverable && attempts >= 2 {
|
||||
return good(ctx, args, in, out, diag)
|
||||
}
|
||||
return rejectedImageRunner("jpeg")(ctx, args, in, out, diag)
|
||||
}
|
||||
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
|
||||
ids := []int64{front, ggc}
|
||||
if err := d.transaction.Apply(d.db, ids); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.service = true
|
||||
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
|
||||
d.resources = func(context.Context) (managedServiceInventory, error) {
|
||||
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}, proxy: r.proxy, runner: r.curl}, nil
|
||||
}
|
||||
d.verifyAssets = r.verify
|
||||
d.health = managedServiceHealth(r.proxy, d.frontHealth, d.resources, r.curl)
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target, monitor.threshold = d.db, target, 1
|
||||
if err := wireManagedRecovery(&monitor, d, func(error) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
monitor.reportRecoveryError = func(err error) {
|
||||
if recoverable || !errors.Is(err, errNoQualifiedStrategy) {
|
||||
t.Fatal("wrong recovery result", err)
|
||||
}
|
||||
}
|
||||
waits := 0
|
||||
monitor.wait = func(ctx context.Context, _ time.Duration) error {
|
||||
state, err := readMonitorState(d.db, monitor.serviceID)
|
||||
if err != nil || state.RecoveryRunning {
|
||||
t.Fatal("recovery ownership retained", state, err)
|
||||
}
|
||||
if !recoverable && state.CooldownUntil.Before(monitor.now().Add(2*time.Hour)) {
|
||||
t.Fatal("missing two-hour pause", state)
|
||||
}
|
||||
waits++
|
||||
if waits == 2 {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("invalid payload killed monitor", err, *events)
|
||||
}
|
||||
want := len(managedAssetSeeds)
|
||||
if recoverable {
|
||||
want = 2
|
||||
}
|
||||
joins := 0
|
||||
for _, event := range *events {
|
||||
if event == "join" {
|
||||
joins++
|
||||
}
|
||||
}
|
||||
if attempts != want || joins != want {
|
||||
t.Fatal("seed fallback/worker join failed", attempts, joins, *events)
|
||||
}
|
||||
active, err := d.transaction.ActiveCandidates(d.db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recoverable {
|
||||
if len(active) != 3 {
|
||||
t.Fatal("fresh image not applied", active)
|
||||
}
|
||||
fresh, err := readStrategyCandidate(d.db, active[2])
|
||||
if err != nil || !fresh.Qualified || fresh.Args != managedAssetSeeds[1] {
|
||||
t.Fatal("invalid fresh evidence", fresh, err)
|
||||
}
|
||||
ids = append(ids, fresh.ID)
|
||||
var rows int
|
||||
if err := d.db.QueryRow("SELECT count(*) FROM candidate_evidence WHERE candidate_id=? AND passes_ok=3 AND passes_total=3", fresh.ID).Scan(&rows); err != nil || rows != 2 {
|
||||
t.Fatal("missing both TLS 3/3 rows", rows, err)
|
||||
}
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, ids)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -350,7 +350,9 @@ func managedAssetTransferNonpass(err error) bool {
|
||||
case *managedRecoveryFatal:
|
||||
return false
|
||||
case *resourceError:
|
||||
return managedResourceTransferNonpass(e, false)
|
||||
return managedResourceTransferNonpass(e, false, true)
|
||||
case imageBodyRejection:
|
||||
return true
|
||||
case *resourcePeerTLSRejection:
|
||||
return true
|
||||
case *exec.ExitError:
|
||||
|
||||
@@ -53,7 +53,7 @@ func managedResourceHealthError(ctx context.Context, err error) error {
|
||||
}
|
||||
var failure *managedResourceFailure
|
||||
var collector *resourceError
|
||||
if ctx.Err() == nil && errors.As(err, &failure) && (failure.ProbeKind == "media" || assetEndpoint(failure.Host) && failure.ProbeKind == managedImageProbeKind(failure.Host)) && errors.As(failure.Err, &collector) && managedResourceTransferNonpass(err, assetEndpoint(failure.Host)) {
|
||||
if ctx.Err() == nil && errors.As(err, &failure) && (failure.ProbeKind == "media" || assetEndpoint(failure.Host) && failure.ProbeKind == managedImageProbeKind(failure.Host)) && errors.As(failure.Err, &collector) && managedResourceTransferNonpass(err, assetEndpoint(failure.Host), assetEndpoint(failure.Host)) {
|
||||
return &managedResourceFailure{ProbeKind: failure.ProbeKind, Host: failure.Host}
|
||||
}
|
||||
return managedRecoveryContextError(ctx, err)
|
||||
@@ -464,7 +464,7 @@ func managedAssetHealthNonpass(err error) bool {
|
||||
if err == nil || err == context.Canceled {
|
||||
return false
|
||||
}
|
||||
return managedAssetTransferNonpass(err) || err.Error() == "image body exceeds limit" || err.Error() == "invalid image dimensions" || err.Error() == "invalid image body"
|
||||
return managedAssetTransferNonpass(err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -186,10 +186,10 @@ func (r managedAssetRuntime) verifyGooglevideo(ctx context.Context, c storedCand
|
||||
}
|
||||
|
||||
func managedGooglevideoTransferNonpass(err error) bool {
|
||||
return managedResourceTransferNonpass(err, false)
|
||||
return managedResourceTransferNonpass(err, false, false)
|
||||
}
|
||||
|
||||
func managedResourceTransferNonpass(err error, metadata bool) bool {
|
||||
func managedResourceTransferNonpass(err error, metadata, image bool) bool {
|
||||
// Collector normalization wraps known transfer failures as fatal for generic
|
||||
// consumers. Only that boundary may unwrap the wrapper; guard/cleanup failures
|
||||
// and mixed error sets must never authorize absence or a daemon restart.
|
||||
@@ -198,7 +198,7 @@ func managedResourceTransferNonpass(err error, metadata bool) bool {
|
||||
if fatal, ok := cause.(*managedRecoveryFatal); ok {
|
||||
cause = fatal.Err
|
||||
}
|
||||
return managedResourceTransferNonpass(cause, metadata)
|
||||
return managedResourceTransferNonpass(cause, metadata, image)
|
||||
}
|
||||
if _, ok := err.(*managedRecoveryFatal); ok {
|
||||
return false
|
||||
@@ -209,14 +209,17 @@ func managedResourceTransferNonpass(err error, metadata bool) bool {
|
||||
return false
|
||||
}
|
||||
for _, child := range children {
|
||||
if !managedResourceTransferNonpass(child, metadata) {
|
||||
if !managedResourceTransferNonpass(child, metadata, image) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if wrapped, ok := err.(interface{ Unwrap() error }); ok {
|
||||
return managedResourceTransferNonpass(wrapped.Unwrap(), metadata)
|
||||
return managedResourceTransferNonpass(wrapped.Unwrap(), metadata, image)
|
||||
}
|
||||
if _, ok := err.(imageBodyRejection); ok {
|
||||
return image
|
||||
}
|
||||
// Metadata curl uses --fail. A non-2xx response is missing inventory,
|
||||
// never media rejection or candidate-absence evidence.
|
||||
|
||||
@@ -305,12 +305,13 @@ func TestManagedServiceAssetLocalHealthFailureRollsBack(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestManagedAssetHealthNonpassPreservesMixedSafetyFailures(t *testing.T) {
|
||||
badBody := resourceFailure(context.Background(), errors.New("invalid image body"))
|
||||
badBody := resourceFailure(context.Background(), imageBodyRejection("invalid image body"))
|
||||
for _, tc := range []struct {
|
||||
err error
|
||||
nonpass bool
|
||||
}{
|
||||
{badBody, true},
|
||||
{resourceFailure(context.Background(), errors.New("invalid image body")), false},
|
||||
{&managedRecoveryFatal{Err: errors.New("invalid image body")}, false},
|
||||
{errors.Join(badBody, &managedRecoveryFatal{Err: errors.New("guard failed")}), false},
|
||||
{errors.Join(badBody, context.Canceled), false},
|
||||
|
||||
@@ -40,12 +40,13 @@ type resourcePeerTLSRejection struct{ error }
|
||||
func (e *resourcePeerTLSRejection) Error() string { return "resource peer TLS rejected" }
|
||||
func (e *resourcePeerTLSRejection) Unwrap() error { return e.error }
|
||||
|
||||
func resourcePeerTLSOnly(err error) bool {
|
||||
if _, ok := err.(*resourcePeerTLSRejection); ok {
|
||||
func resourceRejectionOnly(err error) bool {
|
||||
switch err.(type) {
|
||||
case *resourcePeerTLSRejection, imageBodyRejection:
|
||||
return true
|
||||
}
|
||||
if e, ok := err.(*resourceError); ok {
|
||||
return resourcePeerTLSOnly(e.cause)
|
||||
return resourceRejectionOnly(e.cause)
|
||||
}
|
||||
if e, ok := err.(interface{ Unwrap() []error }); ok {
|
||||
children := e.Unwrap()
|
||||
@@ -53,7 +54,7 @@ func resourcePeerTLSOnly(err error) bool {
|
||||
return false
|
||||
}
|
||||
for _, child := range children {
|
||||
if !resourcePeerTLSOnly(child) {
|
||||
if !resourceRejectionOnly(child) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -75,7 +76,7 @@ func resourceFailure(ctx context.Context, err error) error {
|
||||
if ctx.Err() != nil {
|
||||
err = errors.Join(err, ctx.Err())
|
||||
}
|
||||
if !managedRecoveryTimeoutSafe(err) && !resourcePeerTLSOnly(err) {
|
||||
if !managedRecoveryTimeoutSafe(err) && !resourceRejectionOnly(err) {
|
||||
err = &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
return &resourceError{cause: err}
|
||||
|
||||
@@ -216,7 +216,7 @@ func TestResourceRejectionsAndCancellation(t *testing.T) {
|
||||
t.Fatal("accepted failure or exposed secret", err)
|
||||
}
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(err, &fatal) != (mode != "cancel") {
|
||||
if errors.As(err, &fatal) != (mode != "cancel" && mode != "truncated" && mode != "type") {
|
||||
t.Fatal("unsafe cancellation classification", err)
|
||||
}
|
||||
if (mode == "cancel" || mode == "mixed") && !errors.Is(err, context.Canceled) {
|
||||
|
||||
Reference in new issue
Block a user