fix: classify resource peer TLS rejections as recoverable damage
Keep verified curl peer TLS negatives unqualified without making them process-wide safety failures. Preserve local proxy/configuration, guard, cleanup and mixed cancellation vetoes; test owned verification, rollback and durable absence cooldown. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
50a78608ce
commit
2804d97063
7 files changed
+211
-6
No files matched your search
@@ -4,6 +4,14 @@
|
|||||||
|
|
||||||
## Текущий статус
|
## Текущий статус
|
||||||
|
|
||||||
|
### Peer TLS rejection classification fix (2026-10-12)
|
||||||
|
|
||||||
|
- Read-only field inspection: RC source `50a78608ce40dfc26289b5a63d01eeb3fc569ace`, image `sha256:1a97f34cfbe3bd606d5fb13599767f8a51273b374fa441f5e67802c74d814701`, container `d361b5d581bf710af28f14d3998d6a9a699661b827248c12c9352182df94f98d` is now exited78 (finished `2026-10-12T18:44:23.941802233Z`); host port11082 has no TCP listener. The earlier `proof-v2/final.safe.json` observation at 18:44:10 reported it running, so it is not current uptime evidence. Private artifacts: `/home/agent/zapret2-autotune-rc-50a7860/proof-v2/`.
|
||||||
|
- That safe collector evidence contains a fresh allowed avatar URL, avatar TLS1.2 curl60 and TLS1.3 curl35, both zero samples/FAIL with generic fatal wrappers; thumbnail and sampled media passed. Terminal safe logs subsequently show avatar degradation, **thumbnail** degradation, recovery failure, and the generic safety/database/runtime terminal branch. They do not identify the exact terminal underlying error: do not claim avatar TLS alone is proven to have caused exit78 or that a real local guard failed.
|
||||||
|
- Source fix distinguishes validated resource-curl peer TLS rejection (35/51/60) from process-wide operational safety. It remains rejected/unqualified, with certificate verification, origin checks and SSLVerifyResult=0 qualification unchanged. Probe/recovery can remain DEGRADED; genuine exhausted asset verification enters the existing minimum two-hour pause with workers joined and committed profiles preserved. Local proxy connection7, client-certificate configuration58, unknown/mixed cancellation and explicit guard/cleanup failures remain infrastructure/safety failures. No certificate bypass, host pinning, DNS change, family promotion or READY claim.
|
||||||
|
- The corrected source has not been deployed or image-qualified. Existing remote containers, profiles and other working ports were not restarted or modified by this fix; a tested new image and renewed actual-resource checks are the next dispatch.
|
||||||
|
- Local affected-area verification: managed/resource/experimental/asset/Googlevideo/monitor tests passed normally and under `-race`; local CLI build and `git diff --check` passed. Fixture checks cover production avatar35/60 degradation, owned verification exhaustion and post-apply rejection/rollback, joined workers, durable two-hour cooldown, experimental same-profile media35/60 recovery, and retained local/mixed safety failures. These are not live image acceptance.
|
||||||
|
|
||||||
### Googlevideo suffix RC — browser-verified experiment (2026-10-12)
|
### Googlevideo suffix RC — browser-verified experiment (2026-10-12)
|
||||||
|
|
||||||
- Initial source RC `eade051c2bc335b71b1616d697326ca4c8019d70`, immutable image `sha256:fdb60593e29d7c337a413a47aaddfbcaaeabdeb877042c58f506d4abdc98a13b`, isolated host SOCKS `127.0.0.1:11082`. Historical initial journal: generation3, active candidates [1,2,3], scope `experimental_googlevideo_suffix`; `www.youtube.com` and `i.ytimg.com` profiles preserved. Older targets were unchanged. These are recorded field observations, not current candidate IDs or a new probe/deployment by this source update; its SHA is not the tested image source.
|
- Initial source RC `eade051c2bc335b71b1616d697326ca4c8019d70`, immutable image `sha256:fdb60593e29d7c337a413a47aaddfbcaaeabdeb877042c58f506d4abdc98a13b`, isolated host SOCKS `127.0.0.1:11082`. Historical initial journal: generation3, active candidates [1,2,3], scope `experimental_googlevideo_suffix`; `www.youtube.com` and `i.ytimg.com` profiles preserved. Older targets were unchanged. These are recorded field observations, not current candidate IDs or a new probe/deployment by this source update; its SHA is not the tested image source.
|
||||||
|
|||||||
@@ -347,9 +347,15 @@ func managedAssetTransferNonpass(err error) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
switch e := err.(type) {
|
switch e := err.(type) {
|
||||||
|
case *managedRecoveryFatal:
|
||||||
|
return false
|
||||||
|
case *resourceError:
|
||||||
|
return managedResourceTransferNonpass(e, false)
|
||||||
|
case *resourcePeerTLSRejection:
|
||||||
|
return true
|
||||||
case *exec.ExitError:
|
case *exec.ExitError:
|
||||||
switch e.ExitCode() {
|
switch e.ExitCode() {
|
||||||
case 6, 7, 28, 35, 51, 52, 55, 56, 60:
|
case 6, 28, 35, 51, 52, 55, 56, 60:
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
|
|||||||
@@ -439,6 +439,14 @@ func managedAssetInventoryHealth(ctx context.Context, i managedServiceInventory,
|
|||||||
// Every joined leaf must be a known nonpass; mixed safety failures stay fatal.
|
// Every joined leaf must be a known nonpass; mixed safety failures stay fatal.
|
||||||
func managedAssetHealthNonpass(err error) bool {
|
func managedAssetHealthNonpass(err error) bool {
|
||||||
switch e := err.(type) {
|
switch e := err.(type) {
|
||||||
|
case *managedRecoveryFatal:
|
||||||
|
return false
|
||||||
|
case *resourceError:
|
||||||
|
cause := e.cause
|
||||||
|
if fatal, ok := cause.(*managedRecoveryFatal); ok {
|
||||||
|
cause = fatal.Err
|
||||||
|
}
|
||||||
|
return managedAssetHealthNonpass(cause)
|
||||||
case interface{ Unwrap() []error }:
|
case interface{ Unwrap() []error }:
|
||||||
children := e.Unwrap()
|
children := e.Unwrap()
|
||||||
if len(children) == 0 {
|
if len(children) == 0 {
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
||||||
for _, mode := range []string{"bad-range", "healthy", "unknown", "infra", "mixed", "guard", "cancel", "metadata-timeout", "metadata-http", "metadata-malformed", "metadata-mixed", "metadata-cancel", "metadata-infra", "image-timeout", "image-mixed"} {
|
for _, mode := range []string{"bad-range", "healthy", "unknown", "infra", "mixed", "guard", "cancel", "metadata-timeout", "metadata-http", "metadata-malformed", "metadata-mixed", "metadata-cancel", "metadata-infra", "image-timeout", "image-mixed", "media-peer35", "media-peer60", "image-peer35", "image-peer60", "image-local58", "image-proxy7"} {
|
||||||
t.Run(mode, func(t *testing.T) {
|
t.Run(mode, func(t *testing.T) {
|
||||||
d, front, _ := recoveryFixture(t, true)
|
d, front, _ := recoveryFixture(t, true)
|
||||||
target, _, _ := managedCLIFixture(t)
|
target, _, _ := managedCLIFixture(t)
|
||||||
@@ -89,6 +89,16 @@ func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
|||||||
}
|
}
|
||||||
config, _ := io.ReadAll(in)
|
config, _ := io.ReadAll(in)
|
||||||
raw := strings.TrimSuffix(strings.TrimPrefix(string(config), "url = \""), "\"\n")
|
raw := strings.TrimSuffix(strings.TrimPrefix(string(config), "url = \""), "\"\n")
|
||||||
|
isMedia := strings.Contains(raw, ".googlevideo.com/")
|
||||||
|
if strings.HasPrefix(mode, "media-peer") && isMedia || strings.HasPrefix(mode, "image-peer") && strings.Contains(raw, "yt3.ggpht.com/") {
|
||||||
|
return exec.Command("sh", "-c", "exit "+mode[len(mode)-2:]).Run()
|
||||||
|
}
|
||||||
|
if mode == "image-local58" {
|
||||||
|
return exec.Command("sh", "-c", "exit 58").Run()
|
||||||
|
}
|
||||||
|
if mode == "image-proxy7" {
|
||||||
|
return exec.Command("sh", "-c", "exit 7").Run()
|
||||||
|
}
|
||||||
var wire, diag bytes.Buffer
|
var wire, diag bytes.Buffer
|
||||||
wireMode := ""
|
wireMode := ""
|
||||||
if strings.Contains(raw, ".googlevideo.com/") {
|
if strings.Contains(raw, ".googlevideo.com/") {
|
||||||
@@ -135,7 +145,7 @@ func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
case "bad-range", "metadata-timeout", "metadata-http", "metadata-malformed", "image-timeout":
|
case "bad-range", "metadata-timeout", "metadata-http", "metadata-malformed", "image-timeout", "media-peer35", "media-peer60", "image-peer35", "image-peer60":
|
||||||
if err == nil || errors.As(err, &fatal) {
|
if err == nil || errors.As(err, &fatal) {
|
||||||
t.Fatalf("media damage killed monitor: %v", err)
|
t.Fatalf("media damage killed monitor: %v", err)
|
||||||
}
|
}
|
||||||
@@ -153,7 +163,7 @@ func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
monitor.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
monitor.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||||
err = runManagedMonitor(ctx, monitor)
|
err = runManagedMonitor(ctx, monitor)
|
||||||
if mode == "bad-range" || mode == "metadata-timeout" || mode == "metadata-http" || mode == "metadata-malformed" || mode == "image-timeout" {
|
if mode == "bad-range" || mode == "metadata-timeout" || mode == "metadata-http" || mode == "metadata-malformed" || mode == "image-timeout" || strings.Contains(mode, "-peer") {
|
||||||
if !errors.Is(err, context.Canceled) || errors.As(err, &fatal) || calls != 1 {
|
if !errors.Is(err, context.Canceled) || errors.As(err, &fatal) || calls != 1 {
|
||||||
t.Fatalf("monitor recovery unreachable: %v calls=%d", err, calls)
|
t.Fatalf("monitor recovery unreachable: %v calls=%d", err, calls)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -305,12 +305,13 @@ func TestManagedServiceAssetLocalHealthFailureRollsBack(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestManagedAssetHealthNonpassPreservesMixedSafetyFailures(t *testing.T) {
|
func TestManagedAssetHealthNonpassPreservesMixedSafetyFailures(t *testing.T) {
|
||||||
badBody := &managedRecoveryFatal{Err: errors.New("invalid image body")}
|
badBody := resourceFailure(context.Background(), errors.New("invalid image body"))
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
err error
|
err error
|
||||||
nonpass bool
|
nonpass bool
|
||||||
}{
|
}{
|
||||||
{badBody, true},
|
{badBody, true},
|
||||||
|
{&managedRecoveryFatal{Err: errors.New("invalid image body")}, false},
|
||||||
{errors.Join(badBody, &managedRecoveryFatal{Err: errors.New("guard failed")}), false},
|
{errors.Join(badBody, &managedRecoveryFatal{Err: errors.New("guard failed")}), false},
|
||||||
{errors.Join(badBody, context.Canceled), false},
|
{errors.Join(badBody, context.Canceled), false},
|
||||||
{&managedAssetCleanupFailure{Err: errors.New("cleanup failed")}, false},
|
{&managedAssetCleanupFailure{Err: errors.New("cleanup failed")}, false},
|
||||||
|
|||||||
@@ -33,6 +33,35 @@ func runResourceCurl(ctx context.Context, args []string, stdin io.Reader, out, d
|
|||||||
|
|
||||||
type resourceError struct{ cause error }
|
type resourceError struct{ cause error }
|
||||||
|
|
||||||
|
// Created only after proxy, URL and TLS inputs have passed collector validation.
|
||||||
|
// Rejection is never qualification, nor evidence that local ownership failed.
|
||||||
|
type resourcePeerTLSRejection struct{ error }
|
||||||
|
|
||||||
|
func (e *resourcePeerTLSRejection) Error() string { return "resource peer TLS rejected" }
|
||||||
|
func (e *resourcePeerTLSRejection) Unwrap() error { return e.error }
|
||||||
|
|
||||||
|
func resourcePeerTLSOnly(err error) bool {
|
||||||
|
if _, ok := err.(*resourcePeerTLSRejection); ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if e, ok := err.(*resourceError); ok {
|
||||||
|
return resourcePeerTLSOnly(e.cause)
|
||||||
|
}
|
||||||
|
if e, ok := err.(interface{ Unwrap() []error }); ok {
|
||||||
|
children := e.Unwrap()
|
||||||
|
if len(children) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, child := range children {
|
||||||
|
if !resourcePeerTLSOnly(child) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// Safe metadata classifications contain neither page content nor resource URLs.
|
// Safe metadata classifications contain neither page content nor resource URLs.
|
||||||
type avatarMetadataError string
|
type avatarMetadataError string
|
||||||
|
|
||||||
@@ -46,7 +75,7 @@ func resourceFailure(ctx context.Context, err error) error {
|
|||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
err = errors.Join(err, ctx.Err())
|
err = errors.Join(err, ctx.Err())
|
||||||
}
|
}
|
||||||
if !managedRecoveryTimeoutSafe(err) {
|
if !managedRecoveryTimeoutSafe(err) && !resourcePeerTLSOnly(err) {
|
||||||
err = &managedRecoveryFatal{Err: err}
|
err = &managedRecoveryFatal{Err: err}
|
||||||
}
|
}
|
||||||
return &resourceError{cause: err}
|
return &resourceError{cause: err}
|
||||||
@@ -227,6 +256,14 @@ func collectResourceTransfer(ctx context.Context, proxy, raw, host, protocol str
|
|||||||
err = errors.Join(err, errors.New("resource output exceeds limit"))
|
err = errors.Join(err, errors.New("resource output exceeds limit"))
|
||||||
}
|
}
|
||||||
if err != nil || childCtx.Err() != nil {
|
if err != nil || childCtx.Err() != nil {
|
||||||
|
// Only actual curl peer negatives become typed resource damage. Mixed
|
||||||
|
// cancellation, output overflow and explicit safety errors veto this path.
|
||||||
|
if exit, ok := err.(*exec.ExitError); ok && childCtx.Err() == nil {
|
||||||
|
switch exit.ExitCode() {
|
||||||
|
case 35, 51, 60:
|
||||||
|
err = &resourcePeerTLSRejection{error: err}
|
||||||
|
}
|
||||||
|
}
|
||||||
return fail(errors.Join(err, childCtx.Err()))
|
return fail(errors.Join(err, childCtx.Err()))
|
||||||
}
|
}
|
||||||
i := bytes.LastIndex(output.Bytes(), []byte(resourceTrailer))
|
i := bytes.LastIndex(output.Bytes(), []byte(resourceTrailer))
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResourcePeerTLSCollectorBoundaries(t *testing.T) {
|
||||||
|
for _, code := range []int{7, 35, 51, 58, 60, 65} {
|
||||||
|
t.Run(fmt.Sprint(code), func(t *testing.T) {
|
||||||
|
negative := exec.Command("sh", "-c", fmt.Sprintf("exit %d", code)).Run()
|
||||||
|
for _, mixed := range []error{nil, context.Canceled, errors.New("unknown infrastructure"), &managedRecoveryFatal{Err: errors.New("guard failed")}} {
|
||||||
|
runner := func(_ context.Context, args []string, _ io.Reader, _, _ io.Writer) error {
|
||||||
|
if strings.Contains(strings.Join(args, " "), "--insecure") {
|
||||||
|
t.Fatal("certificate bypass")
|
||||||
|
}
|
||||||
|
return errors.Join(negative, mixed)
|
||||||
|
}
|
||||||
|
// Use curl's actual exit type for the unmixed production path.
|
||||||
|
if mixed == nil {
|
||||||
|
runner = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { return negative }
|
||||||
|
}
|
||||||
|
m, err := collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", "https://yt3.ggpht.com/fresh?private=secret", "yt3.ggpht.com", "tls12", "", runner)
|
||||||
|
if err == nil || len(m.Transfers) != 0 || strings.Contains(err.Error(), "secret") {
|
||||||
|
t.Fatal("accepted/leaked rejection", err)
|
||||||
|
}
|
||||||
|
peer := code == 35 || code == 51 || code == 60
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if errors.As(err, &fatal) != (!peer || mixed != nil) {
|
||||||
|
t.Fatalf("code=%d mixed=%v fatal=%v", code, mixed, err)
|
||||||
|
}
|
||||||
|
if managedAssetTransferNonpass(err) != (peer && mixed == nil) || managedAssetHealthNonpass(err) != (peer && mixed == nil) {
|
||||||
|
t.Fatal("unsafe nonpass classification", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedAvatarPeerTLSMonitorNoQualifiedCooldown(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
code int
|
||||||
|
stage string
|
||||||
|
}{{35, "verify"}, {60, "verify"}, {35, "apply"}, {60, "apply"}} {
|
||||||
|
t.Run(fmt.Sprintf("%d/%s", tc.code, tc.stage), func(t *testing.T) {
|
||||||
|
d, front, _ := recoveryFixture(t, true)
|
||||||
|
r, events, _ := assetRuntimeFixture(t)
|
||||||
|
good := r.curl
|
||||||
|
negative := exec.Command("sh", "-c", fmt.Sprintf("exit %d", tc.code)).Run()
|
||||||
|
reject := func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
|
||||||
|
if strings.Contains(strings.Join(args, " "), "--insecure") {
|
||||||
|
t.Fatal("certificate bypass")
|
||||||
|
}
|
||||||
|
return negative
|
||||||
|
}
|
||||||
|
if tc.stage == "verify" {
|
||||||
|
r.curl = reject
|
||||||
|
}
|
||||||
|
thumbnail := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
|
||||||
|
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
|
||||||
|
ids := []int64{front, thumbnail, ggc}
|
||||||
|
target, _, _ := managedCLIFixture(t)
|
||||||
|
initManagedCLI(t, target)
|
||||||
|
d.transaction.Dir = target.directory
|
||||||
|
if err := d.transaction.Apply(d.db, ids); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before, err := d.transaction.read("config")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d.service = true
|
||||||
|
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
|
||||||
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
||||||
|
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/fresh", "yt3.ggpht.com": "https://yt3.ggpht.com/fresh"}, proxy: r.proxy, runner: reject}, nil
|
||||||
|
}
|
||||||
|
d.verifyAssets = r.verify
|
||||||
|
d.health = managedServiceHealth(r.proxy, d.frontHealth, d.resources, func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
|
||||||
|
config, _ := io.ReadAll(in)
|
||||||
|
if strings.Contains(string(config), "yt3.ggpht.com") {
|
||||||
|
return negative
|
||||||
|
}
|
||||||
|
return good(ctx, args, strings.NewReader(string(config)), out, diag)
|
||||||
|
})
|
||||||
|
monitor := monitorRuntimeFixture(t)
|
||||||
|
monitor.db, monitor.target, monitor.threshold = d.db, target, 1
|
||||||
|
if err := wireManagedRecovery(&monitor, d, func(error) {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
monitor.reportRecoveryError = func(err error) {
|
||||||
|
if !errors.Is(err, errNoQualifiedStrategy) {
|
||||||
|
t.Fatal("not genuine candidate exhaustion", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
waits := 0
|
||||||
|
monitor.wait = func(ctx context.Context, _ time.Duration) error {
|
||||||
|
state, err := readMonitorState(d.db, monitor.serviceID)
|
||||||
|
if err != nil || state.RecoveryRunning || state.CooldownUntil.Before(monitor.now().Add(2*time.Hour)) {
|
||||||
|
t.Fatal("absence pause not durable", state, err)
|
||||||
|
}
|
||||||
|
waits++
|
||||||
|
if waits == 2 {
|
||||||
|
cancel()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal("peer rejection killed monitor", err)
|
||||||
|
}
|
||||||
|
joins := 0
|
||||||
|
for _, event := range *events {
|
||||||
|
if event == "join" {
|
||||||
|
joins++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if joins != len(managedAssetSeeds) {
|
||||||
|
t.Fatal("candidate worker not joined", joins, *events)
|
||||||
|
}
|
||||||
|
assertActiveCandidates(t, d.transaction, d.db, ids)
|
||||||
|
after, err := d.transaction.read("config")
|
||||||
|
if err != nil || string(before) != string(after) {
|
||||||
|
t.Fatal("peer-negative apply changed committed config", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user