fix: add privacy-safe qualification failure diagnostics

Report fixed operation and collector reason enums with measurement ordinals, keeping peer-negative classification and cleanup/ACK vetoes unchanged. Preserve original errors without emitting their values.

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-14 14:53:08 +03:00
1 parent 610916ce66
commit 63cdc7d490
7 files changed
+455 -35

No files matched your search

+12 -11
View File
@@ -234,12 +234,12 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st
} }
occupied, e := r.queue(0) occupied, e := r.queue(0)
if e != nil || occupied { if e != nil || occupied {
return &managedRecoveryFatal{Err: errors.Join(e, errors.New("asset queue unavailable"))} return &managedRecoveryFatal{Err: qualificationOperation(errors.Join(e, errors.New("asset queue unavailable")), stageWorkerQueue)}
} }
args := managedProbeWorkerArgs(endpoints, strategyArgs) args := managedProbeWorkerArgs(endpoints, strategyArgs)
worker, e := r.start(args) worker, e := r.start(args)
if e != nil { if e != nil {
return &managedRecoveryFatal{Err: e} return &managedRecoveryFatal{Err: qualificationOperation(e, stageWorkerStart)}
} }
joined, table := false, false joined, table := false, false
defer func() { defer func() {
@@ -249,28 +249,29 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st
// Remove queues before killing the consumer. // Remove queues before killing the consumer.
if table { if table {
_, cleanupErr = r.nft(cleanup, "delete table inet "+managedAssetTable+"\n") _, cleanupErr = r.nft(cleanup, "delete table inet "+managedAssetTable+"\n")
cleanupErr = qualificationOperation(cleanupErr, stageNFTDelete)
} }
if !joined { if !joined {
select { select {
case e := <-worker.done: case e := <-worker.done:
joined = true joined = true
cleanupErr = errors.Join(cleanupErr, e, errors.New("asset worker exited unexpectedly")) cleanupErr = errors.Join(cleanupErr, qualificationOperation(errors.Join(e, errors.New("asset worker exited unexpectedly")), stageStopJoin))
default: default:
} }
} }
if !joined { if !joined {
cleanupErr = errors.Join(cleanupErr, worker.stop()) cleanupErr = errors.Join(cleanupErr, qualificationOperation(worker.stop(), stageStopJoin))
select { select {
case e := <-worker.done: case e := <-worker.done:
joined = true joined = true
if e != nil { if e != nil {
var exit *exec.ExitError var exit *exec.ExitError
if !errors.As(e, &exit) || !exit.ProcessState.Sys().(syscall.WaitStatus).Signaled() || exit.ProcessState.Sys().(syscall.WaitStatus).Signal() != syscall.SIGKILL { if !errors.As(e, &exit) || !exit.ProcessState.Sys().(syscall.WaitStatus).Signaled() || exit.ProcessState.Sys().(syscall.WaitStatus).Signal() != syscall.SIGKILL {
cleanupErr = errors.Join(cleanupErr, e) cleanupErr = errors.Join(cleanupErr, qualificationOperation(e, stageStopJoin))
} }
} }
case <-cleanup.Done(): case <-cleanup.Done():
cleanupErr = errors.Join(cleanupErr, errors.New("asset worker join timeout")) cleanupErr = errors.Join(cleanupErr, qualificationOperation(errors.New("asset worker join timeout"), stageStopJoin))
} }
} }
if cleanupErr != nil { if cleanupErr != nil {
@@ -285,12 +286,12 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st
select { select {
case e := <-worker.done: case e := <-worker.done:
joined = true joined = true
return &managedRecoveryFatal{Err: errors.Join(e, errors.New("asset worker exited before READY"))} return &managedRecoveryFatal{Err: qualificationOperation(errors.Join(e, errors.New("asset worker exited before READY")), stageWorkerExit)}
default: default:
} }
ready, e := r.queue(worker.pid) ready, e := r.queue(worker.pid)
if e != nil { if e != nil {
return &managedRecoveryFatal{Err: e} return &managedRecoveryFatal{Err: qualificationOperation(e, stageWorkerQueue)}
} }
if ready { if ready {
break break
@@ -300,7 +301,7 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st
if ctx.Err() != nil { if ctx.Err() != nil {
return ctx.Err() return ctx.Err()
} }
return &managedRecoveryFatal{Err: errors.New("asset queue READY timeout")} return &managedRecoveryFatal{Err: qualificationOperation(errors.New("asset queue READY timeout"), stageWorkerQueue)}
case <-tick.C: case <-tick.C:
} }
} }
@@ -311,7 +312,7 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st
table = true table = true
// Recheck ownership after installing the queue and before the first capture. // Recheck ownership after installing the queue and before the first capture.
if ready, e := r.queue(worker.pid); e != nil || !ready { if ready, e := r.queue(worker.pid); e != nil || !ready {
return &managedRecoveryFatal{Err: errors.Join(e, errors.New("owned queue lost before capture"))} return &managedRecoveryFatal{Err: qualificationOperation(errors.Join(e, errors.New("owned queue lost before capture")), stageWorkerQueue)}
} }
probeCtx, stopProbe := context.WithCancel(ctx) probeCtx, stopProbe := context.WithCancel(ctx)
watchDone := make(chan struct{}) watchDone := make(chan struct{})
@@ -320,7 +321,7 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st
defer close(watchDone) defer close(watchDone)
select { select {
case e := <-worker.done: case e := <-worker.done:
exited <- errors.Join(e, errors.New("asset worker exited during probe")) exited <- qualificationOperation(errors.Join(e, errors.New("asset worker exited during probe")), stageWorkerExit)
stopProbe() stopProbe()
case <-probeCtx.Done(): case <-probeCtx.Done():
} }
@@ -0,0 +1,158 @@
package main
import (
"fmt"
"io"
)
// These enums are closed: no caller-controlled strings enter diagnostics.
type qualificationStage uint8
const (
stageUnknown qualificationStage = iota
stagePreDNSGuard
stagePostDNSGuard
stageDNS
stageTransfer
stageAttestation
stageWorkerStart
stageWorkerQueue
stageWorkerExit
stageNFTDelete
stageStopJoin
stageRestore
stageFinalStoreGuard
stageACKWrite
)
type qualificationReason uint8
const (
reasonUnknown qualificationReason = iota
reasonExecution
reasonOutputBound
reasonMetrics
reasonOrigin
reasonTLSEvidence
reasonFraming
reasonBody
reasonMIME
reasonWorkerIdentity
reasonRules
reasonFiles
reasonDefinition
reasonRuntime
reasonCollector
)
func (s qualificationStage) String() string {
names := [...]string{"unknown", "pre_dns_guard", "post_dns_guard", "dns", "transfer", "attestation", "worker_start", "worker_queue", "worker_exit", "nft_delete", "stop_join", "restore", "final_store_guard", "ack_write"}
if int(s) >= len(names) {
return names[0]
}
return names[s]
}
func (r qualificationReason) String() string {
names := [...]string{"unknown", "execution", "output_bound", "metrics", "origin", "tls_evidence", "framing", "body", "mime", "worker_identity", "rules", "files", "definition", "runtime", "collector"}
if int(r) >= len(names) {
return names[0]
}
return names[r]
}
type qualificationDiagnostic struct {
Err error
Stage qualificationStage
Reason qualificationReason
Candidate, Resource, Attempt int
TLS uint8
}
func (e *qualificationDiagnostic) Error() string { return "qualification operation failed" }
func (e *qualificationDiagnostic) Unwrap() error { return e.Err }
func qualificationOperation(err error, stage qualificationStage) error {
if err == nil {
return nil
}
return &qualificationDiagnostic{Err: err, Stage: stage}
}
func attestationMismatch(a, b qualificationAttestation) qualificationReason {
switch {
case a.PID != b.PID || a.StartTime != b.StartTime || a.WorkerArgsHash != b.WorkerArgsHash || a.Queue != b.Queue:
return reasonWorkerIdentity
case a.RulesHash != b.RulesHash:
return reasonRules
case a.LuaHash != b.LuaHash:
return reasonFiles
case a.Definition != b.Definition:
return reasonDefinition
case a.Runtime != b.Runtime:
return reasonRuntime
case a.Collector != b.Collector:
return reasonCollector
default:
return reasonUnknown
}
}
// Traverse only for reporting, never for classification. Bound even pathological
// error trees; output contains enums and ordinals, never Error() or type names.
func writeQualificationDiagnostics(w io.Writer, err error) {
budget := 32
var visit func(error, qualificationDiagnostic, bool)
visit = func(err error, inherited qualificationDiagnostic, cleanup bool) {
if err == nil || budget == 0 {
return
}
budget--
switch e := err.(type) {
case *qualificationDiagnostic:
if e.Candidate != 0 {
inherited.Candidate = e.Candidate
}
if e.Resource != 0 {
inherited.Resource = e.Resource
}
if e.Attempt != 0 {
inherited.Attempt = e.Attempt
}
if e.TLS != 0 {
inherited.TLS = e.TLS
}
if e.Stage != stageUnknown && (e.Stage != stageAttestation || inherited.Stage == stageUnknown) {
inherited.Stage = e.Stage
}
if e.Reason != reasonUnknown {
inherited.Reason = e.Reason
}
cleanup = cleanup || e.Stage == stageNFTDelete || e.Stage == stageStopJoin || e.Stage == stageRestore
visit(e.Err, inherited, cleanup)
return
case *resourceError:
inherited.Reason = e.reason
case interface{ Unwrap() []error }:
for _, child := range e.Unwrap() {
visit(child, inherited, cleanup)
}
return
}
if e, ok := err.(interface{ Unwrap() error }); ok {
visit(e.Unwrap(), inherited, cleanup)
return
}
role, tls := "primary", "none"
if cleanup {
role = "cleanup"
}
if inherited.TLS == 12 {
tls = "tls12"
} else if inherited.TLS == 13 {
tls = "tls13"
}
fmt.Fprintf(w, "qualification diagnostic: role=%s stage=%s reason=%s candidate=%d resource=%d attempt=%d tls=%s\n", role, inherited.Stage, inherited.Reason, inherited.Candidate, inherited.Resource, inherited.Attempt, tls)
}
visit(err, qualificationDiagnostic{}, false)
}
@@ -0,0 +1,211 @@
package main
import (
"bytes"
"context"
"errors"
"io"
"strings"
"testing"
)
func TestQualificationDiagnosticStagesAndPrivacy(t *testing.T) {
secret := errors.New("PRIVATE https://secret.example/path Authorization IP env")
for _, mode := range []string{"transfer", "next_guard", "post_guard", "attestation", "metrics", "mime", "cleanup"} {
t.Run(mode, func(t *testing.T) {
q, s, registry, _, calls := qualifierFixture(t)
original := q.checks.runner
q.checks.runner = func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
if *calls == 3 && (mode == "transfer" || mode == "cleanup") {
return secret
}
var wire bytes.Buffer
err := original(ctx, args, in, &wire, diag)
if *calls == 4 {
if mode == "metrics" {
wire.WriteString("PRIVATE")
}
if mode == "mime" {
wire = *bytes.NewBufferString(strings.Replace(wire.String(), "text/plain", "text/plain; PRIVATE", 1))
}
}
_, _ = out.Write(wire.Bytes())
return err
}
guards := 0
q.checks.guard = func() error {
guards++
if (mode == "next_guard" && guards == 9) || (mode == "post_guard" && guards == 8) {
return secret
}
return nil
}
attest := q.attest
q.attest = func(ctx context.Context, pid int, args []string, d strategyDefinition) (qualificationAttestation, error) {
a, err := attest(ctx, pid, args, d)
if mode == "attestation" && *calls == 4 {
a.RulesHash = strings.Repeat("b", 64)
}
return a, err
}
if mode == "cleanup" {
q.restore = func(context.Context) error { return secret }
}
_, err := q.complete(context.Background(), s, registry)
if err == nil {
t.Fatal("failure accepted")
}
var diagnostic bytes.Buffer
writeQualificationDiagnostics(&diagnostic, err)
got := diagnostic.String()
if strings.Contains(got, "PRIVATE") || strings.Contains(got, "example") || strings.Contains(got, "SECRET") {
t.Fatal(got)
}
want := map[string]string{"transfer": "stage=transfer reason=execution candidate=1 resource=1 attempt=1 tls=tls13", "cleanup": "stage=transfer reason=execution candidate=1 resource=1 attempt=1 tls=tls13", "next_guard": "stage=pre_dns_guard reason=unknown candidate=1 resource=1 attempt=2 tls=tls13", "post_guard": "stage=post_dns_guard reason=unknown candidate=1 resource=1 attempt=1 tls=tls13", "attestation": "stage=pre_dns_guard reason=rules", "metrics": "stage=transfer reason=metrics", "mime": "stage=transfer reason=mime"}[mode]
if !strings.Contains(got, want) {
t.Fatalf("want %s, got %s", want, got)
}
if mode == "cleanup" {
if !strings.Contains(got, "role=cleanup stage=restore") || qualificationCleanupVerified(err, func(context.Context) error { t.Fatal("cleanup retry"); return nil }) {
t.Fatal(got)
}
}
if mode == "transfer" && !errors.Is(err, secret) {
t.Fatal("lost original error")
}
})
}
}
func TestQualificationDiagnosticsDoNotChangeNegatives(t *testing.T) {
for _, mode := range []string{"status500", "mime_mismatch"} {
t.Run(mode, func(t *testing.T) {
q, s, registry, _, calls := qualifierFixture(t)
if mode == "mime_mismatch" {
source, err := q.source()
if err != nil {
t.Fatal(err)
}
for i := range source.config.Services {
for j := range source.config.Services[i].Resources {
source.config.Services[i].Resources[j].Expect.ContentType = "image/png"
}
}
q.source = func() (resourceMonitorSource, error) { return source, nil }
} else {
original := q.checks.runner
q.checks.runner = func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
var wire bytes.Buffer
err := original(ctx, args, in, &wire, diag)
_, _ = io.WriteString(out, strings.ReplaceAll(strings.ReplaceAll(wire.String(), "200 OK", "500 OK"), `"http_code":200`, `"http_code":500`))
return err
}
}
proof, err := q.complete(context.Background(), s, registry)
if err != nil || *calls != 12 {
t.Fatalf("%d %v", *calls, err)
}
for _, check := range proof.matrix.Runs[0].Checks {
if check.Outcome != "expectation_failed" || check.Attempts != 3 {
t.Fatal(check)
}
}
})
}
}
func TestQualificationDiagnosticVetoAndFinalStages(t *testing.T) {
secret := errors.New("PRIVATE sentinel")
for _, stage := range []qualificationStage{stageFinalStoreGuard, stageACKWrite, stageNFTDelete, stageStopJoin, stageWorkerQueue, stageWorkerExit} {
err := qualificationOperation(secret, stage)
var out bytes.Buffer
writeQualificationDiagnostics(&out, err)
if !strings.Contains(out.String(), "stage="+stage.String()) || strings.Contains(out.String(), "PRIVATE") || !errors.Is(err, secret) {
t.Fatal(out.String())
}
}
cancel := qualificationOperation(context.Canceled, stagePreDNSGuard)
if !managedCancellationOnly(cancel, context.Canceled) {
t.Fatal("pure cancellation changed")
}
for _, veto := range []error{secret, &managedRecoveryFatal{Err: context.Canceled}, &managedAssetCleanupFailure{Err: secret}} {
if managedCancellationOnly(errors.Join(cancel, veto), context.Canceled) {
t.Fatal("mixed veto lost")
}
}
if resourceRejectionOnly(qualificationOperation(&resourcePeerTLSRejection{error: secret}, stageTransfer)) {
t.Fatal("arbitrary rejection unwrapping added")
}
}
func TestQualificationFinalizeDiagnosticsAndACKGate(t *testing.T) {
secret := errors.New("PRIVATE sentinel")
for _, mode := range []string{"cleanup", "restore", "guard", "ack", "success"} {
t.Run(mode, func(t *testing.T) {
var primary error
if mode == "cleanup" {
primary = &qualificationCleanupFailure{Err: secret}
}
restores, guards, acks := 0, 0, 0
err := qualificationFinalize(primary, func(context.Context) error {
restores++
if mode == "restore" {
return secret
}
return nil
}, func() error {
guards++
if mode == "guard" {
return secret
}
return nil
}, func() error {
acks++
if mode == "ack" {
return secret
}
return nil
})
if mode == "success" {
if err != nil || acks != 1 {
t.Fatal(err, acks)
}
return
}
var out bytes.Buffer
writeQualificationDiagnostics(&out, err)
stage := stageRestore
if mode == "guard" {
stage = stageFinalStoreGuard
}
if mode == "ack" {
stage = stageACKWrite
}
if !strings.Contains(out.String(), "stage="+stage.String()) || strings.Contains(out.String(), "PRIVATE") {
t.Fatal(out.String())
}
if mode != "ack" && acks != 0 {
t.Fatal("ACK passed failed gate")
}
if mode == "cleanup" && (restores != 0 || guards != 0) {
t.Fatal("cleanup failure retried")
}
})
}
}
func TestQualificationAttestationErrorRetainsOwnershipVeto(t *testing.T) {
q, s, registry, _, _ := qualifierFixture(t)
attest, calls := q.attest, 0
q.attest = func(ctx context.Context, pid int, args []string, d strategyDefinition) (qualificationAttestation, error) {
calls++
if calls == 2 {
return qualificationAttestation{}, context.Canceled
}
return attest(ctx, pid, args, d)
}
_, err := q.complete(context.Background(), s, registry)
if !errors.Is(err, context.Canceled) || !errors.Is(err, errResourcePlan) || managedCancellationOnly(err, context.Canceled) {
t.Fatal("attestation ownership veto changed", err)
}
}
+5 -5
View File
@@ -374,7 +374,7 @@ type resourceCheckResult struct {
func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpec, protocol, server, proxy string, d resourceCheckDeps) (resourceCheckResult, error) { func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpec, protocol, server, proxy string, d resourceCheckDeps) (resourceCheckResult, error) {
result := resourceCheckResult{Service: service, Resource: r.ID, Group: r.Group, TLS: protocol, Outcome: "dns_failed"} result := resourceCheckResult{Service: service, Resource: r.ID, Group: r.Group, TLS: protocol, Outcome: "dns_failed"}
if err := d.guard(); err != nil { if err := d.guard(); err != nil {
return result, err return result, qualificationOperation(err, stagePreDNSGuard)
} }
probeCtx, cancel := context.WithTimeout(ctx, time.Duration(r.TimeoutMS)*time.Millisecond) probeCtx, cancel := context.WithTimeout(ctx, time.Duration(r.TimeoutMS)*time.Millisecond)
defer cancel() defer cancel()
@@ -383,11 +383,11 @@ func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpe
var dns *net.DNSError var dns *net.DNSError
knownTimeout := probeCtx.Err() != nil && managedRecoveryTimeoutSafe(err) knownTimeout := probeCtx.Err() != nil && managedRecoveryTimeoutSafe(err)
if !knownTimeout && (!errors.As(err, &dns) || !dns.IsNotFound) { if !knownTimeout && (!errors.As(err, &dns) || !dns.IsNotFound) {
return result, err return result, qualificationOperation(err, stageDNS)
} }
} }
if d.qualification && err == nil && (len(ips) == 0 || len(ips) > 64) { if d.qualification && err == nil && (len(ips) == 0 || len(ips) > 64) {
return result, errResourceConfig return result, qualificationOperation(errResourceConfig, stageDNS)
} }
if err == nil && len(ips) > 0 && len(ips) <= 64 { if err == nil && len(ips) > 0 && len(ips) <= 64 {
approved := true approved := true
@@ -399,7 +399,7 @@ func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpe
result.Outcome = "destination_denied" result.Outcome = "destination_denied"
if approved { if approved {
if err := d.guard(); err != nil { if err := d.guard(); err != nil {
return result, err return result, qualificationOperation(err, stagePostDNSGuard)
} }
pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443") pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443")
options := resourceTransferOptions{generic: true, method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n"} options := resourceTransferOptions{generic: true, method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n"}
@@ -407,7 +407,7 @@ func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpe
if d.qualification && err != nil { if d.qualification && err != nil {
knownTimeout := probeCtx.Err() != nil && managedRecoveryTimeoutSafe(err) knownTimeout := probeCtx.Err() != nil && managedRecoveryTimeoutSafe(err)
if !knownTimeout && !resourceRejectionOnly(err) && !managedAssetTransferNonpass(err) { if !knownTimeout && !resourceRejectionOnly(err) && !managedAssetTransferNonpass(err) {
return result, err return result, qualificationOperation(err, stageTransfer)
} }
} }
result.Outcome = "transfer_failed" result.Outcome = "transfer_failed"
+17 -2
View File
@@ -31,7 +31,10 @@ func runResourceCurl(ctx context.Context, args []string, stdin io.Reader, out, d
return cmd.Run() return cmd.Run()
} }
type resourceError struct{ cause error } type resourceError struct {
cause error
reason qualificationReason
}
// Created only after proxy, URL and TLS inputs have passed collector validation. // Created only after proxy, URL and TLS inputs have passed collector validation.
// Rejection is never qualification, nor evidence that local ownership failed. // Rejection is never qualification, nor evidence that local ownership failed.
@@ -240,8 +243,11 @@ type resourceTransferOptions struct {
} }
func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner, options resourceTransferOptions) (assetTransfer, http.Header, error) { func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner, options resourceTransferOptions) (assetTransfer, http.Header, error) {
reason := reasonOrigin
fail := func(err error) (assetTransfer, http.Header, error) { fail := func(err error) (assetTransfer, http.Header, error) {
return assetTransfer{}, nil, resourceFailure(ctx, err) failure := resourceFailure(ctx, err)
failure.(*resourceError).reason = reason
return assetTransfer{}, nil, failure
} }
proxy, proxyErr := managedProbeProxy(proxy) proxy, proxyErr := managedProbeProxy(proxy)
if proxyErr != nil || runner == nil { if proxyErr != nil || runner == nil {
@@ -288,10 +294,12 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
childCtx, cancel := context.WithTimeout(ctx, timeout) childCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel() defer cancel()
var output, diagnostic bytes.Buffer var output, diagnostic bytes.Buffer
reason = reasonExecution
out := &playerBoundedWriter{dst: &output, limit: limit + (96 << 10)} out := &playerBoundedWriter{dst: &output, limit: limit + (96 << 10)}
diag := &playerBoundedWriter{dst: &diagnostic, limit: 64 << 10} diag := &playerBoundedWriter{dst: &diagnostic, limit: 64 << 10}
err := runner(childCtx, args, strings.NewReader(config), out, diag) err := runner(childCtx, args, strings.NewReader(config), out, diag)
if out.exceeded || diag.exceeded { if out.exceeded || diag.exceeded {
reason = reasonOutputBound
err = errors.Join(err, errors.New("resource output exceeds limit")) err = errors.Join(err, errors.New("resource output exceeds limit"))
} }
if err != nil || childCtx.Err() != nil { if err != nil || childCtx.Err() != nil {
@@ -305,6 +313,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
} }
return fail(errors.Join(err, childCtx.Err())) return fail(errors.Join(err, childCtx.Err()))
} }
reason = reasonMetrics
i := bytes.LastIndex(output.Bytes(), []byte(resourceTrailer)) i := bytes.LastIndex(output.Bytes(), []byte(resourceTrailer))
if i < 0 { if i < 0 {
return fail(errors.New("missing transfer metrics")) return fail(errors.New("missing transfer metrics"))
@@ -321,9 +330,11 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
if json.Unmarshal(output.Bytes()[i+len(resourceTrailer):], &metrics) != nil { if json.Unmarshal(output.Bytes()[i+len(resourceTrailer):], &metrics) != nil {
return fail(errors.New("invalid transfer metrics")) return fail(errors.New("invalid transfer metrics"))
} }
reason = reasonOrigin
if _, err := privateResourceURL(metrics.URL, host); err != nil || metrics.URL != raw || metrics.Verify == nil || *metrics.Verify != 0 || metrics.Redirects == nil || *metrics.Redirects != 0 { if _, err := privateResourceURL(metrics.URL, host); err != nil || metrics.URL != raw || metrics.Verify == nil || *metrics.Verify != 0 || metrics.Redirects == nil || *metrics.Redirects != 0 {
return fail(errors.New("resource origin verification failed")) return fail(errors.New("resource origin verification failed"))
} }
reason = reasonTLSEvidence
actualTLS := "SSL connection using TLSv1.2" actualTLS := "SSL connection using TLSv1.2"
if version == 0x0304 { if version == 0x0304 {
actualTLS = "SSL connection using TLSv1.3" actualTLS = "SSL connection using TLSv1.3"
@@ -335,6 +346,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
if !strings.Contains(diagnostic.String(), actualTLS) || strings.Contains(diagnostic.String(), otherTLS) { if !strings.Contains(diagnostic.String(), actualTLS) || strings.Contains(diagnostic.String(), otherTLS) {
return fail(errors.New("actual TLS version mismatch")) return fail(errors.New("actual TLS version mismatch"))
} }
reason = reasonFraming
wire := output.Bytes()[:i] wire := output.Bytes()[:i]
headerEnd := bytes.Index(wire, []byte("\r\n\r\n")) headerEnd := bytes.Index(wire, []byte("\r\n\r\n"))
statusEnd := bytes.Index(wire, []byte("\r\n")) statusEnd := bytes.Index(wire, []byte("\r\n"))
@@ -371,6 +383,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
if options.generic && response.StatusCode >= 300 && response.StatusCode < 400 { if options.generic && response.StatusCode >= 300 && response.StatusCode < 400 {
return fail(&resourcePeerResponseRejection{error: errors.New("redirect rejected")}) return fail(&resourcePeerResponseRejection{error: errors.New("redirect rejected")})
} }
reason = reasonBody
body, err := io.ReadAll(io.LimitReader(response.Body, int64(limit+1))) body, err := io.ReadAll(io.LimitReader(response.Body, int64(limit+1)))
// With --raw curl counts the wire body, including chunk framing; the image // With --raw curl counts the wire body, including chunk framing; the image
// limit and durable byte identity apply to the fully decoded payload instead. // limit and durable byte identity apply to the fully decoded payload instead.
@@ -379,6 +392,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
if err != nil || remaining != io.EOF || (!options.generic && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status { if err != nil || remaining != io.EOF || (!options.generic && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status {
return fail(errors.New("incomplete or oversized resource body")) return fail(errors.New("incomplete or oversized resource body"))
} }
reason = reasonMetrics
for _, n := range []float64{metrics.Time, metrics.Speed} { for _, n := range []float64{metrics.Time, metrics.Speed} {
if n < 0 || (!options.generic && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) { if n < 0 || (!options.generic && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) {
return fail(errors.New("invalid resource metrics")) return fail(errors.New("invalid resource metrics"))
@@ -387,6 +401,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
if options.generic && metrics.Time > options.timeout.Seconds()+1 { if options.generic && metrics.Time > options.timeout.Seconds()+1 {
return fail(errors.New("resource timing exceeds budget")) return fail(errors.New("resource timing exceeds budget"))
} }
reason = reasonMIME
contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type")) contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type"))
if err != nil && (!options.generic || response.Header.Get("Content-Type") != "") { if err != nil && (!options.generic || response.Header.Get("Content-Type") != "") {
return fail(errors.New("invalid content type")) return fail(errors.New("invalid content type"))
+33 -17
View File
@@ -388,7 +388,7 @@ func (q resourceQualifier) complete(ctx context.Context, s *resourcePlanStore, r
} }
return nil return nil
} }
for _, definition := range registry.Strategies { for candidateIndex, definition := range registry.Strategies {
if _, e := canonicalStrategy(definition); e != nil { if _, e := canonicalStrategy(definition); e != nil {
return fail(e) return fail(e)
} }
@@ -423,10 +423,10 @@ func (q resourceQualifier) complete(ctx context.Context, s *resourcePlanStore, r
} }
a, e := q.attest(captureCtx, pid, args, definition) a, e := q.attest(captureCtx, pid, args, definition)
if e != nil { if e != nil {
return e return qualificationOperation(e, stageAttestation)
} }
if a.LuaHash != nativeBefore { if a.LuaHash != nativeBefore {
return errResourcePlan return &qualificationDiagnostic{Err: errResourcePlan, Stage: stageAttestation, Reason: reasonFiles}
} }
run.Attestation = a run.Attestation = a
checks := q.checks checks := q.checks
@@ -440,13 +440,18 @@ func (q resourceQualifier) complete(ctx context.Context, s *resourcePlanStore, r
return e return e
} }
current, e := q.attest(captureCtx, pid, args, definition) current, e := q.attest(captureCtx, pid, args, definition)
if e != nil || current != a { if e != nil {
return errResourcePlan return qualificationOperation(errors.Join(errResourcePlan, e), stageAttestation)
}
if current != a {
return &qualificationDiagnostic{Err: errResourcePlan, Stage: stageAttestation, Reason: attestationMismatch(a, current)}
} }
return nil return nil
} }
resourceIndex := 0
for _, service := range source.config.Services { for _, service := range source.config.Services {
for _, spec := range service.Resources { for _, spec := range service.Resources {
resourceIndex++
protocols := []string{spec.TLS} protocols := []string{spec.TLS}
if spec.TLS == "both" { if spec.TLS == "both" {
protocols = []string{"tls12", "tls13"} protocols = []string{"tls12", "tls13"}
@@ -457,14 +462,18 @@ func (q resourceQualifier) complete(ctx context.Context, s *resourcePlanStore, r
cell := qualificationCheck{Service: service.ID, Resource: spec.ID, Fingerprint: resourceRowFingerprint(s.files.key, source, spec), TLS: protocol, Outcome: "pass"} cell := qualificationCheck{Service: service.ID, Resource: spec.ID, Fingerprint: resourceRowFingerprint(s.files.key, source, spec), TLS: protocol, Outcome: "pass"}
for repeat := 0; repeat < 3; repeat++ { for repeat := 0; repeat < 3; repeat++ {
result, e := runResourceProtocol(captureCtx, service.ID, spec, protocol, source.server, source.proxy, checks) result, e := runResourceProtocol(captureCtx, service.ID, spec, protocol, source.server, source.proxy, checks)
if e == nil && captureCtx.Err() != nil {
e = captureCtx.Err()
}
if e == nil && result.Outcome == "cancelled" {
e = context.Canceled
}
if e != nil { if e != nil {
return e tls := uint8(12)
} if protocol == "tls13" {
if captureCtx.Err() != nil { tls = 13
return captureCtx.Err() }
} return &qualificationDiagnostic{Err: e, Candidate: candidateIndex + 1, Resource: resourceIndex, Attempt: repeat + 1, TLS: tls}
if result.Outcome == "cancelled" {
return context.Canceled
} }
if result.Outcome != "pass" { if result.Outcome != "pass" {
cell.Outcome = result.Outcome cell.Outcome = result.Outcome
@@ -479,8 +488,15 @@ func (q resourceQualifier) complete(ctx context.Context, s *resourcePlanStore, r
}) })
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
restoreErr := q.restore(cleanup) restoreErr := q.restore(cleanup)
if restoreErr != nil {
restoreErr = &qualificationDiagnostic{Err: restoreErr, Stage: stageRestore, Candidate: candidateIndex + 1}
}
cancel() cancel()
if err != nil || restoreErr != nil { if err != nil || restoreErr != nil {
err = qualificationOperation(err, stageUnknown)
if diagnostic, ok := err.(*qualificationDiagnostic); ok {
diagnostic.Candidate = candidateIndex + 1
}
if restoreErr != nil { if restoreErr != nil {
return fail(&qualificationCleanupFailure{Err: errors.Join(err, restoreErr)}) return fail(&qualificationCleanupFailure{Err: errors.Join(err, restoreErr)})
} }
@@ -588,11 +604,10 @@ func qualifyResourceCLI(config, strategies string, stdout, stderr io.Writer, d m
if code != 0 && code != 78 { if code != 0 && code != 78 {
return return
} }
if !qualificationCleanupVerified(sessionErr, q.restore) || s.guard() != nil { if e := qualificationFinalize(sessionErr, q.restore, s.guard, func() error {
code = 78 return writeQualificationCompletion(qualificationCompletionDir, s.files.key, os.Getpid(), code)
return }); e != nil {
} writeQualificationDiagnostics(stderr, e)
if writeQualificationCompletion(qualificationCompletionDir, s.files.key, os.Getpid(), code) != nil {
code = 78 code = 78
} }
}() }()
@@ -613,6 +628,7 @@ func qualifyResourceCLI(config, strategies string, stdout, stderr io.Writer, d m
return 64 return 64
} }
if e != nil || ctx.Err() != nil { if e != nil || ctx.Err() != nil {
writeQualificationDiagnostics(stderr, e)
fmt.Fprintln(stderr, "unknown: qualification incomplete or ownership/cleanup blocked; no completed matrix published") fmt.Fprintln(stderr, "unknown: qualification incomplete or ownership/cleanup blocked; no completed matrix published")
if signals.Err() != nil && managedCancellationOnly(e, signals.Err()) { if signals.Err() != nil && managedCancellationOnly(e, signals.Err()) {
return 0 return 0
@@ -84,6 +84,25 @@ func qualificationCleanupVerified(err error, restore func(context.Context) error
return restore(ctx) == nil return restore(ctx) == nil
} }
// Same cleanup gate and ordering as the controller's completion defer. Keeping
// operations separate makes ACK refusal diagnosable without exposing errors.
func qualificationFinalize(err error, restore func(context.Context) error, guard, ack func() error) error {
var restoreErr error
if !qualificationCleanupVerified(err, func(ctx context.Context) error {
restoreErr = restore(ctx)
return restoreErr
}) {
if restoreErr == nil {
restoreErr = errResourcePlan
}
return qualificationOperation(restoreErr, stageRestore)
}
if e := guard(); e != nil {
return qualificationOperation(e, stageFinalStoreGuard)
}
return qualificationOperation(ack(), stageACKWrite)
}
func qualificationCompletionCLI(args []string, d managedDeps) int { func qualificationCompletionCLI(args []string, d managedDeps) int {
if len(args) != 2 || os.Geteuid() != 0 || os.Getppid() != 1 { if len(args) != 2 || os.Geteuid() != 0 || os.Getppid() != 1 {
return 78 return 78