fix: classify resource peer TLS rejections as recoverable damage
Keep verified curl peer TLS negatives unqualified without making them process-wide safety failures. Preserve local proxy/configuration, guard, cleanup and mixed cancellation vetoes; test owned verification, rollback and durable absence cooldown. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
50a78608ce
commit
2804d97063
7 files changed
+211
-6
No files matched your search
@@ -347,9 +347,15 @@ func managedAssetTransferNonpass(err error) bool {
|
||||
return false
|
||||
}
|
||||
switch e := err.(type) {
|
||||
case *managedRecoveryFatal:
|
||||
return false
|
||||
case *resourceError:
|
||||
return managedResourceTransferNonpass(e, false)
|
||||
case *resourcePeerTLSRejection:
|
||||
return true
|
||||
case *exec.ExitError:
|
||||
switch e.ExitCode() {
|
||||
case 6, 7, 28, 35, 51, 52, 55, 56, 60:
|
||||
case 6, 28, 35, 51, 52, 55, 56, 60:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -439,6 +439,14 @@ func managedAssetInventoryHealth(ctx context.Context, i managedServiceInventory,
|
||||
// Every joined leaf must be a known nonpass; mixed safety failures stay fatal.
|
||||
func managedAssetHealthNonpass(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case *managedRecoveryFatal:
|
||||
return false
|
||||
case *resourceError:
|
||||
cause := e.cause
|
||||
if fatal, ok := cause.(*managedRecoveryFatal); ok {
|
||||
cause = fatal.Err
|
||||
}
|
||||
return managedAssetHealthNonpass(cause)
|
||||
case interface{ Unwrap() []error }:
|
||||
children := e.Unwrap()
|
||||
if len(children) == 0 {
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
)
|
||||
|
||||
func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
||||
for _, mode := range []string{"bad-range", "healthy", "unknown", "infra", "mixed", "guard", "cancel", "metadata-timeout", "metadata-http", "metadata-malformed", "metadata-mixed", "metadata-cancel", "metadata-infra", "image-timeout", "image-mixed"} {
|
||||
for _, mode := range []string{"bad-range", "healthy", "unknown", "infra", "mixed", "guard", "cancel", "metadata-timeout", "metadata-http", "metadata-malformed", "metadata-mixed", "metadata-cancel", "metadata-infra", "image-timeout", "image-mixed", "media-peer35", "media-peer60", "image-peer35", "image-peer60", "image-local58", "image-proxy7"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, front, _ := recoveryFixture(t, true)
|
||||
target, _, _ := managedCLIFixture(t)
|
||||
@@ -89,6 +89,16 @@ func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
||||
}
|
||||
config, _ := io.ReadAll(in)
|
||||
raw := strings.TrimSuffix(strings.TrimPrefix(string(config), "url = \""), "\"\n")
|
||||
isMedia := strings.Contains(raw, ".googlevideo.com/")
|
||||
if strings.HasPrefix(mode, "media-peer") && isMedia || strings.HasPrefix(mode, "image-peer") && strings.Contains(raw, "yt3.ggpht.com/") {
|
||||
return exec.Command("sh", "-c", "exit "+mode[len(mode)-2:]).Run()
|
||||
}
|
||||
if mode == "image-local58" {
|
||||
return exec.Command("sh", "-c", "exit 58").Run()
|
||||
}
|
||||
if mode == "image-proxy7" {
|
||||
return exec.Command("sh", "-c", "exit 7").Run()
|
||||
}
|
||||
var wire, diag bytes.Buffer
|
||||
wireMode := ""
|
||||
if strings.Contains(raw, ".googlevideo.com/") {
|
||||
@@ -135,7 +145,7 @@ func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "bad-range", "metadata-timeout", "metadata-http", "metadata-malformed", "image-timeout":
|
||||
case "bad-range", "metadata-timeout", "metadata-http", "metadata-malformed", "image-timeout", "media-peer35", "media-peer60", "image-peer35", "image-peer60":
|
||||
if err == nil || errors.As(err, &fatal) {
|
||||
t.Fatalf("media damage killed monitor: %v", err)
|
||||
}
|
||||
@@ -153,7 +163,7 @@ func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
||||
defer cancel()
|
||||
monitor.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||
err = runManagedMonitor(ctx, monitor)
|
||||
if mode == "bad-range" || mode == "metadata-timeout" || mode == "metadata-http" || mode == "metadata-malformed" || mode == "image-timeout" {
|
||||
if mode == "bad-range" || mode == "metadata-timeout" || mode == "metadata-http" || mode == "metadata-malformed" || mode == "image-timeout" || strings.Contains(mode, "-peer") {
|
||||
if !errors.Is(err, context.Canceled) || errors.As(err, &fatal) || calls != 1 {
|
||||
t.Fatalf("monitor recovery unreachable: %v calls=%d", err, calls)
|
||||
}
|
||||
|
||||
@@ -305,12 +305,13 @@ func TestManagedServiceAssetLocalHealthFailureRollsBack(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestManagedAssetHealthNonpassPreservesMixedSafetyFailures(t *testing.T) {
|
||||
badBody := &managedRecoveryFatal{Err: errors.New("invalid image body")}
|
||||
badBody := resourceFailure(context.Background(), errors.New("invalid image body"))
|
||||
for _, tc := range []struct {
|
||||
err error
|
||||
nonpass bool
|
||||
}{
|
||||
{badBody, true},
|
||||
{&managedRecoveryFatal{Err: errors.New("invalid image body")}, false},
|
||||
{errors.Join(badBody, &managedRecoveryFatal{Err: errors.New("guard failed")}), false},
|
||||
{errors.Join(badBody, context.Canceled), false},
|
||||
{&managedAssetCleanupFailure{Err: errors.New("cleanup failed")}, false},
|
||||
|
||||
@@ -33,6 +33,35 @@ func runResourceCurl(ctx context.Context, args []string, stdin io.Reader, out, d
|
||||
|
||||
type resourceError struct{ cause error }
|
||||
|
||||
// Created only after proxy, URL and TLS inputs have passed collector validation.
|
||||
// Rejection is never qualification, nor evidence that local ownership failed.
|
||||
type resourcePeerTLSRejection struct{ error }
|
||||
|
||||
func (e *resourcePeerTLSRejection) Error() string { return "resource peer TLS rejected" }
|
||||
func (e *resourcePeerTLSRejection) Unwrap() error { return e.error }
|
||||
|
||||
func resourcePeerTLSOnly(err error) bool {
|
||||
if _, ok := err.(*resourcePeerTLSRejection); ok {
|
||||
return true
|
||||
}
|
||||
if e, ok := err.(*resourceError); ok {
|
||||
return resourcePeerTLSOnly(e.cause)
|
||||
}
|
||||
if e, ok := err.(interface{ Unwrap() []error }); ok {
|
||||
children := e.Unwrap()
|
||||
if len(children) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, child := range children {
|
||||
if !resourcePeerTLSOnly(child) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Safe metadata classifications contain neither page content nor resource URLs.
|
||||
type avatarMetadataError string
|
||||
|
||||
@@ -46,7 +75,7 @@ func resourceFailure(ctx context.Context, err error) error {
|
||||
if ctx.Err() != nil {
|
||||
err = errors.Join(err, ctx.Err())
|
||||
}
|
||||
if !managedRecoveryTimeoutSafe(err) {
|
||||
if !managedRecoveryTimeoutSafe(err) && !resourcePeerTLSOnly(err) {
|
||||
err = &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
return &resourceError{cause: err}
|
||||
@@ -227,6 +256,14 @@ func collectResourceTransfer(ctx context.Context, proxy, raw, host, protocol str
|
||||
err = errors.Join(err, errors.New("resource output exceeds limit"))
|
||||
}
|
||||
if err != nil || childCtx.Err() != nil {
|
||||
// Only actual curl peer negatives become typed resource damage. Mixed
|
||||
// cancellation, output overflow and explicit safety errors veto this path.
|
||||
if exit, ok := err.(*exec.ExitError); ok && childCtx.Err() == nil {
|
||||
switch exit.ExitCode() {
|
||||
case 35, 51, 60:
|
||||
err = &resourcePeerTLSRejection{error: err}
|
||||
}
|
||||
}
|
||||
return fail(errors.Join(err, childCtx.Err()))
|
||||
}
|
||||
i := bytes.LastIndex(output.Bytes(), []byte(resourceTrailer))
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestResourcePeerTLSCollectorBoundaries(t *testing.T) {
|
||||
for _, code := range []int{7, 35, 51, 58, 60, 65} {
|
||||
t.Run(fmt.Sprint(code), func(t *testing.T) {
|
||||
negative := exec.Command("sh", "-c", fmt.Sprintf("exit %d", code)).Run()
|
||||
for _, mixed := range []error{nil, context.Canceled, errors.New("unknown infrastructure"), &managedRecoveryFatal{Err: errors.New("guard failed")}} {
|
||||
runner := func(_ context.Context, args []string, _ io.Reader, _, _ io.Writer) error {
|
||||
if strings.Contains(strings.Join(args, " "), "--insecure") {
|
||||
t.Fatal("certificate bypass")
|
||||
}
|
||||
return errors.Join(negative, mixed)
|
||||
}
|
||||
// Use curl's actual exit type for the unmixed production path.
|
||||
if mixed == nil {
|
||||
runner = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { return negative }
|
||||
}
|
||||
m, err := collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", "https://yt3.ggpht.com/fresh?private=secret", "yt3.ggpht.com", "tls12", "", runner)
|
||||
if err == nil || len(m.Transfers) != 0 || strings.Contains(err.Error(), "secret") {
|
||||
t.Fatal("accepted/leaked rejection", err)
|
||||
}
|
||||
peer := code == 35 || code == 51 || code == 60
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(err, &fatal) != (!peer || mixed != nil) {
|
||||
t.Fatalf("code=%d mixed=%v fatal=%v", code, mixed, err)
|
||||
}
|
||||
if managedAssetTransferNonpass(err) != (peer && mixed == nil) || managedAssetHealthNonpass(err) != (peer && mixed == nil) {
|
||||
t.Fatal("unsafe nonpass classification", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedAvatarPeerTLSMonitorNoQualifiedCooldown(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
code int
|
||||
stage string
|
||||
}{{35, "verify"}, {60, "verify"}, {35, "apply"}, {60, "apply"}} {
|
||||
t.Run(fmt.Sprintf("%d/%s", tc.code, tc.stage), func(t *testing.T) {
|
||||
d, front, _ := recoveryFixture(t, true)
|
||||
r, events, _ := assetRuntimeFixture(t)
|
||||
good := r.curl
|
||||
negative := exec.Command("sh", "-c", fmt.Sprintf("exit %d", tc.code)).Run()
|
||||
reject := func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
|
||||
if strings.Contains(strings.Join(args, " "), "--insecure") {
|
||||
t.Fatal("certificate bypass")
|
||||
}
|
||||
return negative
|
||||
}
|
||||
if tc.stage == "verify" {
|
||||
r.curl = reject
|
||||
}
|
||||
thumbnail := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
|
||||
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
|
||||
ids := []int64{front, thumbnail, ggc}
|
||||
target, _, _ := managedCLIFixture(t)
|
||||
initManagedCLI(t, target)
|
||||
d.transaction.Dir = target.directory
|
||||
if err := d.transaction.Apply(d.db, ids); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := d.transaction.read("config")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.service = true
|
||||
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
|
||||
d.resources = func(context.Context) (managedServiceInventory, error) {
|
||||
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/fresh", "yt3.ggpht.com": "https://yt3.ggpht.com/fresh"}, proxy: r.proxy, runner: reject}, nil
|
||||
}
|
||||
d.verifyAssets = r.verify
|
||||
d.health = managedServiceHealth(r.proxy, d.frontHealth, d.resources, func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
|
||||
config, _ := io.ReadAll(in)
|
||||
if strings.Contains(string(config), "yt3.ggpht.com") {
|
||||
return negative
|
||||
}
|
||||
return good(ctx, args, strings.NewReader(string(config)), out, diag)
|
||||
})
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target, monitor.threshold = d.db, target, 1
|
||||
if err := wireManagedRecovery(&monitor, d, func(error) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
monitor.reportRecoveryError = func(err error) {
|
||||
if !errors.Is(err, errNoQualifiedStrategy) {
|
||||
t.Fatal("not genuine candidate exhaustion", err)
|
||||
}
|
||||
}
|
||||
waits := 0
|
||||
monitor.wait = func(ctx context.Context, _ time.Duration) error {
|
||||
state, err := readMonitorState(d.db, monitor.serviceID)
|
||||
if err != nil || state.RecoveryRunning || state.CooldownUntil.Before(monitor.now().Add(2*time.Hour)) {
|
||||
t.Fatal("absence pause not durable", state, err)
|
||||
}
|
||||
waits++
|
||||
if waits == 2 {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("peer rejection killed monitor", err)
|
||||
}
|
||||
joins := 0
|
||||
for _, event := range *events {
|
||||
if event == "join" {
|
||||
joins++
|
||||
}
|
||||
}
|
||||
if joins != len(managedAssetSeeds) {
|
||||
t.Fatal("candidate worker not joined", joins, *events)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, ids)
|
||||
after, err := d.transaction.read("config")
|
||||
if err != nil || string(before) != string(after) {
|
||||
t.Fatal("peer-negative apply changed committed config", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user