diff --git a/cmd/zapret-autotune/managed_asset_runtime.go b/cmd/zapret-autotune/managed_asset_runtime.go index d65245e..446293e 100644 --- a/cmd/zapret-autotune/managed_asset_runtime.go +++ b/cmd/zapret-autotune/managed_asset_runtime.go @@ -234,12 +234,12 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st } occupied, e := r.queue(0) if e != nil || occupied { - return &managedRecoveryFatal{Err: errors.Join(e, errors.New("asset queue unavailable"))} + return &managedRecoveryFatal{Err: qualificationOperation(errors.Join(e, errors.New("asset queue unavailable")), stageWorkerQueue)} } args := managedProbeWorkerArgs(endpoints, strategyArgs) worker, e := r.start(args) if e != nil { - return &managedRecoveryFatal{Err: e} + return &managedRecoveryFatal{Err: qualificationOperation(e, stageWorkerStart)} } joined, table := false, false defer func() { @@ -249,28 +249,29 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st // Remove queues before killing the consumer. if table { _, cleanupErr = r.nft(cleanup, "delete table inet "+managedAssetTable+"\n") + cleanupErr = qualificationOperation(cleanupErr, stageNFTDelete) } if !joined { select { case e := <-worker.done: joined = true - cleanupErr = errors.Join(cleanupErr, e, errors.New("asset worker exited unexpectedly")) + cleanupErr = errors.Join(cleanupErr, qualificationOperation(errors.Join(e, errors.New("asset worker exited unexpectedly")), stageStopJoin)) default: } } if !joined { - cleanupErr = errors.Join(cleanupErr, worker.stop()) + cleanupErr = errors.Join(cleanupErr, qualificationOperation(worker.stop(), stageStopJoin)) select { case e := <-worker.done: joined = true if e != nil { var exit *exec.ExitError if !errors.As(e, &exit) || !exit.ProcessState.Sys().(syscall.WaitStatus).Signaled() || exit.ProcessState.Sys().(syscall.WaitStatus).Signal() != syscall.SIGKILL { - cleanupErr = errors.Join(cleanupErr, e) + cleanupErr = errors.Join(cleanupErr, qualificationOperation(e, stageStopJoin)) } } case <-cleanup.Done(): - cleanupErr = errors.Join(cleanupErr, errors.New("asset worker join timeout")) + cleanupErr = errors.Join(cleanupErr, qualificationOperation(errors.New("asset worker join timeout"), stageStopJoin)) } } if cleanupErr != nil { @@ -285,12 +286,12 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st select { case e := <-worker.done: joined = true - return &managedRecoveryFatal{Err: errors.Join(e, errors.New("asset worker exited before READY"))} + return &managedRecoveryFatal{Err: qualificationOperation(errors.Join(e, errors.New("asset worker exited before READY")), stageWorkerExit)} default: } ready, e := r.queue(worker.pid) if e != nil { - return &managedRecoveryFatal{Err: e} + return &managedRecoveryFatal{Err: qualificationOperation(e, stageWorkerQueue)} } if ready { break @@ -300,7 +301,7 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st if ctx.Err() != nil { return ctx.Err() } - return &managedRecoveryFatal{Err: errors.New("asset queue READY timeout")} + return &managedRecoveryFatal{Err: qualificationOperation(errors.New("asset queue READY timeout"), stageWorkerQueue)} case <-tick.C: } } @@ -311,7 +312,7 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st table = true // Recheck ownership after installing the queue and before the first capture. if ready, e := r.queue(worker.pid); e != nil || !ready { - return &managedRecoveryFatal{Err: errors.Join(e, errors.New("owned queue lost before capture"))} + return &managedRecoveryFatal{Err: qualificationOperation(errors.Join(e, errors.New("owned queue lost before capture")), stageWorkerQueue)} } probeCtx, stopProbe := context.WithCancel(ctx) watchDone := make(chan struct{}) @@ -320,7 +321,7 @@ func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []st defer close(watchDone) select { case e := <-worker.done: - exited <- errors.Join(e, errors.New("asset worker exited during probe")) + exited <- qualificationOperation(errors.Join(e, errors.New("asset worker exited during probe")), stageWorkerExit) stopProbe() case <-probeCtx.Done(): } diff --git a/cmd/zapret-autotune/qualification_diagnostic.go b/cmd/zapret-autotune/qualification_diagnostic.go new file mode 100644 index 0000000..b8e130e --- /dev/null +++ b/cmd/zapret-autotune/qualification_diagnostic.go @@ -0,0 +1,158 @@ +package main + +import ( + "fmt" + "io" +) + +// These enums are closed: no caller-controlled strings enter diagnostics. +type qualificationStage uint8 + +const ( + stageUnknown qualificationStage = iota + stagePreDNSGuard + stagePostDNSGuard + stageDNS + stageTransfer + stageAttestation + stageWorkerStart + stageWorkerQueue + stageWorkerExit + stageNFTDelete + stageStopJoin + stageRestore + stageFinalStoreGuard + stageACKWrite +) + +type qualificationReason uint8 + +const ( + reasonUnknown qualificationReason = iota + reasonExecution + reasonOutputBound + reasonMetrics + reasonOrigin + reasonTLSEvidence + reasonFraming + reasonBody + reasonMIME + reasonWorkerIdentity + reasonRules + reasonFiles + reasonDefinition + reasonRuntime + reasonCollector +) + +func (s qualificationStage) String() string { + names := [...]string{"unknown", "pre_dns_guard", "post_dns_guard", "dns", "transfer", "attestation", "worker_start", "worker_queue", "worker_exit", "nft_delete", "stop_join", "restore", "final_store_guard", "ack_write"} + if int(s) >= len(names) { + return names[0] + } + return names[s] +} + +func (r qualificationReason) String() string { + names := [...]string{"unknown", "execution", "output_bound", "metrics", "origin", "tls_evidence", "framing", "body", "mime", "worker_identity", "rules", "files", "definition", "runtime", "collector"} + if int(r) >= len(names) { + return names[0] + } + return names[r] +} + +type qualificationDiagnostic struct { + Err error + Stage qualificationStage + Reason qualificationReason + Candidate, Resource, Attempt int + TLS uint8 +} + +func (e *qualificationDiagnostic) Error() string { return "qualification operation failed" } +func (e *qualificationDiagnostic) Unwrap() error { return e.Err } + +func qualificationOperation(err error, stage qualificationStage) error { + if err == nil { + return nil + } + return &qualificationDiagnostic{Err: err, Stage: stage} +} + +func attestationMismatch(a, b qualificationAttestation) qualificationReason { + switch { + case a.PID != b.PID || a.StartTime != b.StartTime || a.WorkerArgsHash != b.WorkerArgsHash || a.Queue != b.Queue: + return reasonWorkerIdentity + case a.RulesHash != b.RulesHash: + return reasonRules + case a.LuaHash != b.LuaHash: + return reasonFiles + case a.Definition != b.Definition: + return reasonDefinition + case a.Runtime != b.Runtime: + return reasonRuntime + case a.Collector != b.Collector: + return reasonCollector + default: + return reasonUnknown + } +} + +// Traverse only for reporting, never for classification. Bound even pathological +// error trees; output contains enums and ordinals, never Error() or type names. +func writeQualificationDiagnostics(w io.Writer, err error) { + budget := 32 + var visit func(error, qualificationDiagnostic, bool) + visit = func(err error, inherited qualificationDiagnostic, cleanup bool) { + if err == nil || budget == 0 { + return + } + budget-- + switch e := err.(type) { + case *qualificationDiagnostic: + if e.Candidate != 0 { + inherited.Candidate = e.Candidate + } + if e.Resource != 0 { + inherited.Resource = e.Resource + } + if e.Attempt != 0 { + inherited.Attempt = e.Attempt + } + if e.TLS != 0 { + inherited.TLS = e.TLS + } + if e.Stage != stageUnknown && (e.Stage != stageAttestation || inherited.Stage == stageUnknown) { + inherited.Stage = e.Stage + } + if e.Reason != reasonUnknown { + inherited.Reason = e.Reason + } + cleanup = cleanup || e.Stage == stageNFTDelete || e.Stage == stageStopJoin || e.Stage == stageRestore + visit(e.Err, inherited, cleanup) + return + case *resourceError: + inherited.Reason = e.reason + case interface{ Unwrap() []error }: + for _, child := range e.Unwrap() { + visit(child, inherited, cleanup) + } + return + } + if e, ok := err.(interface{ Unwrap() error }); ok { + visit(e.Unwrap(), inherited, cleanup) + return + } + role, tls := "primary", "none" + if cleanup { + role = "cleanup" + } + if inherited.TLS == 12 { + tls = "tls12" + } else if inherited.TLS == 13 { + tls = "tls13" + } + fmt.Fprintf(w, "qualification diagnostic: role=%s stage=%s reason=%s candidate=%d resource=%d attempt=%d tls=%s\n", role, inherited.Stage, inherited.Reason, inherited.Candidate, inherited.Resource, inherited.Attempt, tls) + } + visit(err, qualificationDiagnostic{}, false) +} diff --git a/cmd/zapret-autotune/qualification_diagnostic_test.go b/cmd/zapret-autotune/qualification_diagnostic_test.go new file mode 100644 index 0000000..4700d01 --- /dev/null +++ b/cmd/zapret-autotune/qualification_diagnostic_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" +) + +func TestQualificationDiagnosticStagesAndPrivacy(t *testing.T) { + secret := errors.New("PRIVATE https://secret.example/path Authorization IP env") + for _, mode := range []string{"transfer", "next_guard", "post_guard", "attestation", "metrics", "mime", "cleanup"} { + t.Run(mode, func(t *testing.T) { + q, s, registry, _, calls := qualifierFixture(t) + original := q.checks.runner + q.checks.runner = func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error { + if *calls == 3 && (mode == "transfer" || mode == "cleanup") { + return secret + } + var wire bytes.Buffer + err := original(ctx, args, in, &wire, diag) + if *calls == 4 { + if mode == "metrics" { + wire.WriteString("PRIVATE") + } + if mode == "mime" { + wire = *bytes.NewBufferString(strings.Replace(wire.String(), "text/plain", "text/plain; PRIVATE", 1)) + } + } + _, _ = out.Write(wire.Bytes()) + return err + } + guards := 0 + q.checks.guard = func() error { + guards++ + if (mode == "next_guard" && guards == 9) || (mode == "post_guard" && guards == 8) { + return secret + } + return nil + } + attest := q.attest + q.attest = func(ctx context.Context, pid int, args []string, d strategyDefinition) (qualificationAttestation, error) { + a, err := attest(ctx, pid, args, d) + if mode == "attestation" && *calls == 4 { + a.RulesHash = strings.Repeat("b", 64) + } + return a, err + } + if mode == "cleanup" { + q.restore = func(context.Context) error { return secret } + } + _, err := q.complete(context.Background(), s, registry) + if err == nil { + t.Fatal("failure accepted") + } + var diagnostic bytes.Buffer + writeQualificationDiagnostics(&diagnostic, err) + got := diagnostic.String() + if strings.Contains(got, "PRIVATE") || strings.Contains(got, "example") || strings.Contains(got, "SECRET") { + t.Fatal(got) + } + want := map[string]string{"transfer": "stage=transfer reason=execution candidate=1 resource=1 attempt=1 tls=tls13", "cleanup": "stage=transfer reason=execution candidate=1 resource=1 attempt=1 tls=tls13", "next_guard": "stage=pre_dns_guard reason=unknown candidate=1 resource=1 attempt=2 tls=tls13", "post_guard": "stage=post_dns_guard reason=unknown candidate=1 resource=1 attempt=1 tls=tls13", "attestation": "stage=pre_dns_guard reason=rules", "metrics": "stage=transfer reason=metrics", "mime": "stage=transfer reason=mime"}[mode] + if !strings.Contains(got, want) { + t.Fatalf("want %s, got %s", want, got) + } + if mode == "cleanup" { + if !strings.Contains(got, "role=cleanup stage=restore") || qualificationCleanupVerified(err, func(context.Context) error { t.Fatal("cleanup retry"); return nil }) { + t.Fatal(got) + } + } + if mode == "transfer" && !errors.Is(err, secret) { + t.Fatal("lost original error") + } + }) + } +} + +func TestQualificationDiagnosticsDoNotChangeNegatives(t *testing.T) { + for _, mode := range []string{"status500", "mime_mismatch"} { + t.Run(mode, func(t *testing.T) { + q, s, registry, _, calls := qualifierFixture(t) + if mode == "mime_mismatch" { + source, err := q.source() + if err != nil { + t.Fatal(err) + } + for i := range source.config.Services { + for j := range source.config.Services[i].Resources { + source.config.Services[i].Resources[j].Expect.ContentType = "image/png" + } + } + q.source = func() (resourceMonitorSource, error) { return source, nil } + } else { + original := q.checks.runner + q.checks.runner = func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error { + var wire bytes.Buffer + err := original(ctx, args, in, &wire, diag) + _, _ = io.WriteString(out, strings.ReplaceAll(strings.ReplaceAll(wire.String(), "200 OK", "500 OK"), `"http_code":200`, `"http_code":500`)) + return err + } + } + proof, err := q.complete(context.Background(), s, registry) + if err != nil || *calls != 12 { + t.Fatalf("%d %v", *calls, err) + } + for _, check := range proof.matrix.Runs[0].Checks { + if check.Outcome != "expectation_failed" || check.Attempts != 3 { + t.Fatal(check) + } + } + }) + } +} + +func TestQualificationDiagnosticVetoAndFinalStages(t *testing.T) { + secret := errors.New("PRIVATE sentinel") + for _, stage := range []qualificationStage{stageFinalStoreGuard, stageACKWrite, stageNFTDelete, stageStopJoin, stageWorkerQueue, stageWorkerExit} { + err := qualificationOperation(secret, stage) + var out bytes.Buffer + writeQualificationDiagnostics(&out, err) + if !strings.Contains(out.String(), "stage="+stage.String()) || strings.Contains(out.String(), "PRIVATE") || !errors.Is(err, secret) { + t.Fatal(out.String()) + } + } + cancel := qualificationOperation(context.Canceled, stagePreDNSGuard) + if !managedCancellationOnly(cancel, context.Canceled) { + t.Fatal("pure cancellation changed") + } + for _, veto := range []error{secret, &managedRecoveryFatal{Err: context.Canceled}, &managedAssetCleanupFailure{Err: secret}} { + if managedCancellationOnly(errors.Join(cancel, veto), context.Canceled) { + t.Fatal("mixed veto lost") + } + } + if resourceRejectionOnly(qualificationOperation(&resourcePeerTLSRejection{error: secret}, stageTransfer)) { + t.Fatal("arbitrary rejection unwrapping added") + } +} + +func TestQualificationFinalizeDiagnosticsAndACKGate(t *testing.T) { + secret := errors.New("PRIVATE sentinel") + for _, mode := range []string{"cleanup", "restore", "guard", "ack", "success"} { + t.Run(mode, func(t *testing.T) { + var primary error + if mode == "cleanup" { + primary = &qualificationCleanupFailure{Err: secret} + } + restores, guards, acks := 0, 0, 0 + err := qualificationFinalize(primary, func(context.Context) error { + restores++ + if mode == "restore" { + return secret + } + return nil + }, func() error { + guards++ + if mode == "guard" { + return secret + } + return nil + }, func() error { + acks++ + if mode == "ack" { + return secret + } + return nil + }) + if mode == "success" { + if err != nil || acks != 1 { + t.Fatal(err, acks) + } + return + } + var out bytes.Buffer + writeQualificationDiagnostics(&out, err) + stage := stageRestore + if mode == "guard" { + stage = stageFinalStoreGuard + } + if mode == "ack" { + stage = stageACKWrite + } + if !strings.Contains(out.String(), "stage="+stage.String()) || strings.Contains(out.String(), "PRIVATE") { + t.Fatal(out.String()) + } + if mode != "ack" && acks != 0 { + t.Fatal("ACK passed failed gate") + } + if mode == "cleanup" && (restores != 0 || guards != 0) { + t.Fatal("cleanup failure retried") + } + }) + } +} + +func TestQualificationAttestationErrorRetainsOwnershipVeto(t *testing.T) { + q, s, registry, _, _ := qualifierFixture(t) + attest, calls := q.attest, 0 + q.attest = func(ctx context.Context, pid int, args []string, d strategyDefinition) (qualificationAttestation, error) { + calls++ + if calls == 2 { + return qualificationAttestation{}, context.Canceled + } + return attest(ctx, pid, args, d) + } + _, err := q.complete(context.Background(), s, registry) + if !errors.Is(err, context.Canceled) || !errors.Is(err, errResourcePlan) || managedCancellationOnly(err, context.Canceled) { + t.Fatal("attestation ownership veto changed", err) + } +} diff --git a/cmd/zapret-autotune/resource_check.go b/cmd/zapret-autotune/resource_check.go index 9880725..c4af956 100644 --- a/cmd/zapret-autotune/resource_check.go +++ b/cmd/zapret-autotune/resource_check.go @@ -374,7 +374,7 @@ type resourceCheckResult struct { func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpec, protocol, server, proxy string, d resourceCheckDeps) (resourceCheckResult, error) { result := resourceCheckResult{Service: service, Resource: r.ID, Group: r.Group, TLS: protocol, Outcome: "dns_failed"} if err := d.guard(); err != nil { - return result, err + return result, qualificationOperation(err, stagePreDNSGuard) } probeCtx, cancel := context.WithTimeout(ctx, time.Duration(r.TimeoutMS)*time.Millisecond) defer cancel() @@ -383,11 +383,11 @@ func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpe var dns *net.DNSError knownTimeout := probeCtx.Err() != nil && managedRecoveryTimeoutSafe(err) if !knownTimeout && (!errors.As(err, &dns) || !dns.IsNotFound) { - return result, err + return result, qualificationOperation(err, stageDNS) } } if d.qualification && err == nil && (len(ips) == 0 || len(ips) > 64) { - return result, errResourceConfig + return result, qualificationOperation(errResourceConfig, stageDNS) } if err == nil && len(ips) > 0 && len(ips) <= 64 { approved := true @@ -399,7 +399,7 @@ func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpe result.Outcome = "destination_denied" if approved { if err := d.guard(); err != nil { - return result, err + return result, qualificationOperation(err, stagePostDNSGuard) } pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443") options := resourceTransferOptions{generic: true, method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n"} @@ -407,7 +407,7 @@ func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpe if d.qualification && err != nil { knownTimeout := probeCtx.Err() != nil && managedRecoveryTimeoutSafe(err) if !knownTimeout && !resourceRejectionOnly(err) && !managedAssetTransferNonpass(err) { - return result, err + return result, qualificationOperation(err, stageTransfer) } } result.Outcome = "transfer_failed" diff --git a/cmd/zapret-autotune/resource_collector.go b/cmd/zapret-autotune/resource_collector.go index 511b73f..d0c7a85 100644 --- a/cmd/zapret-autotune/resource_collector.go +++ b/cmd/zapret-autotune/resource_collector.go @@ -31,7 +31,10 @@ func runResourceCurl(ctx context.Context, args []string, stdin io.Reader, out, d return cmd.Run() } -type resourceError struct{ cause error } +type resourceError struct { + cause error + reason qualificationReason +} // Created only after proxy, URL and TLS inputs have passed collector validation. // Rejection is never qualification, nor evidence that local ownership failed. @@ -240,8 +243,11 @@ type resourceTransferOptions struct { } func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner, options resourceTransferOptions) (assetTransfer, http.Header, error) { + reason := reasonOrigin fail := func(err error) (assetTransfer, http.Header, error) { - return assetTransfer{}, nil, resourceFailure(ctx, err) + failure := resourceFailure(ctx, err) + failure.(*resourceError).reason = reason + return assetTransfer{}, nil, failure } proxy, proxyErr := managedProbeProxy(proxy) if proxyErr != nil || runner == nil { @@ -288,10 +294,12 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto childCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() var output, diagnostic bytes.Buffer + reason = reasonExecution out := &playerBoundedWriter{dst: &output, limit: limit + (96 << 10)} diag := &playerBoundedWriter{dst: &diagnostic, limit: 64 << 10} err := runner(childCtx, args, strings.NewReader(config), out, diag) if out.exceeded || diag.exceeded { + reason = reasonOutputBound err = errors.Join(err, errors.New("resource output exceeds limit")) } if err != nil || childCtx.Err() != nil { @@ -305,6 +313,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto } return fail(errors.Join(err, childCtx.Err())) } + reason = reasonMetrics i := bytes.LastIndex(output.Bytes(), []byte(resourceTrailer)) if i < 0 { return fail(errors.New("missing transfer metrics")) @@ -321,9 +330,11 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto if json.Unmarshal(output.Bytes()[i+len(resourceTrailer):], &metrics) != nil { return fail(errors.New("invalid transfer metrics")) } + reason = reasonOrigin if _, err := privateResourceURL(metrics.URL, host); err != nil || metrics.URL != raw || metrics.Verify == nil || *metrics.Verify != 0 || metrics.Redirects == nil || *metrics.Redirects != 0 { return fail(errors.New("resource origin verification failed")) } + reason = reasonTLSEvidence actualTLS := "SSL connection using TLSv1.2" if version == 0x0304 { actualTLS = "SSL connection using TLSv1.3" @@ -335,6 +346,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto if !strings.Contains(diagnostic.String(), actualTLS) || strings.Contains(diagnostic.String(), otherTLS) { return fail(errors.New("actual TLS version mismatch")) } + reason = reasonFraming wire := output.Bytes()[:i] headerEnd := bytes.Index(wire, []byte("\r\n\r\n")) statusEnd := bytes.Index(wire, []byte("\r\n")) @@ -371,6 +383,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto if options.generic && response.StatusCode >= 300 && response.StatusCode < 400 { return fail(&resourcePeerResponseRejection{error: errors.New("redirect rejected")}) } + reason = reasonBody body, err := io.ReadAll(io.LimitReader(response.Body, int64(limit+1))) // With --raw curl counts the wire body, including chunk framing; the image // limit and durable byte identity apply to the fully decoded payload instead. @@ -379,6 +392,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto if err != nil || remaining != io.EOF || (!options.generic && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status { return fail(errors.New("incomplete or oversized resource body")) } + reason = reasonMetrics for _, n := range []float64{metrics.Time, metrics.Speed} { if n < 0 || (!options.generic && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) { return fail(errors.New("invalid resource metrics")) @@ -387,6 +401,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto if options.generic && metrics.Time > options.timeout.Seconds()+1 { return fail(errors.New("resource timing exceeds budget")) } + reason = reasonMIME contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type")) if err != nil && (!options.generic || response.Header.Get("Content-Type") != "") { return fail(errors.New("invalid content type")) diff --git a/cmd/zapret-autotune/resource_qualify.go b/cmd/zapret-autotune/resource_qualify.go index 2439601..08f1d32 100644 --- a/cmd/zapret-autotune/resource_qualify.go +++ b/cmd/zapret-autotune/resource_qualify.go @@ -388,7 +388,7 @@ func (q resourceQualifier) complete(ctx context.Context, s *resourcePlanStore, r } return nil } - for _, definition := range registry.Strategies { + for candidateIndex, definition := range registry.Strategies { if _, e := canonicalStrategy(definition); e != nil { return fail(e) } @@ -423,10 +423,10 @@ func (q resourceQualifier) complete(ctx context.Context, s *resourcePlanStore, r } a, e := q.attest(captureCtx, pid, args, definition) if e != nil { - return e + return qualificationOperation(e, stageAttestation) } if a.LuaHash != nativeBefore { - return errResourcePlan + return &qualificationDiagnostic{Err: errResourcePlan, Stage: stageAttestation, Reason: reasonFiles} } run.Attestation = a checks := q.checks @@ -440,13 +440,18 @@ func (q resourceQualifier) complete(ctx context.Context, s *resourcePlanStore, r return e } current, e := q.attest(captureCtx, pid, args, definition) - if e != nil || current != a { - return errResourcePlan + if e != nil { + return qualificationOperation(errors.Join(errResourcePlan, e), stageAttestation) + } + if current != a { + return &qualificationDiagnostic{Err: errResourcePlan, Stage: stageAttestation, Reason: attestationMismatch(a, current)} } return nil } + resourceIndex := 0 for _, service := range source.config.Services { for _, spec := range service.Resources { + resourceIndex++ protocols := []string{spec.TLS} if spec.TLS == "both" { protocols = []string{"tls12", "tls13"} @@ -457,14 +462,18 @@ func (q resourceQualifier) complete(ctx context.Context, s *resourcePlanStore, r cell := qualificationCheck{Service: service.ID, Resource: spec.ID, Fingerprint: resourceRowFingerprint(s.files.key, source, spec), TLS: protocol, Outcome: "pass"} for repeat := 0; repeat < 3; repeat++ { result, e := runResourceProtocol(captureCtx, service.ID, spec, protocol, source.server, source.proxy, checks) + if e == nil && captureCtx.Err() != nil { + e = captureCtx.Err() + } + if e == nil && result.Outcome == "cancelled" { + e = context.Canceled + } if e != nil { - return e - } - if captureCtx.Err() != nil { - return captureCtx.Err() - } - if result.Outcome == "cancelled" { - return context.Canceled + tls := uint8(12) + if protocol == "tls13" { + tls = 13 + } + return &qualificationDiagnostic{Err: e, Candidate: candidateIndex + 1, Resource: resourceIndex, Attempt: repeat + 1, TLS: tls} } if result.Outcome != "pass" { cell.Outcome = result.Outcome @@ -479,8 +488,15 @@ func (q resourceQualifier) complete(ctx context.Context, s *resourcePlanStore, r }) cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) restoreErr := q.restore(cleanup) + if restoreErr != nil { + restoreErr = &qualificationDiagnostic{Err: restoreErr, Stage: stageRestore, Candidate: candidateIndex + 1} + } cancel() if err != nil || restoreErr != nil { + err = qualificationOperation(err, stageUnknown) + if diagnostic, ok := err.(*qualificationDiagnostic); ok { + diagnostic.Candidate = candidateIndex + 1 + } if restoreErr != nil { return fail(&qualificationCleanupFailure{Err: errors.Join(err, restoreErr)}) } @@ -588,11 +604,10 @@ func qualifyResourceCLI(config, strategies string, stdout, stderr io.Writer, d m if code != 0 && code != 78 { return } - if !qualificationCleanupVerified(sessionErr, q.restore) || s.guard() != nil { - code = 78 - return - } - if writeQualificationCompletion(qualificationCompletionDir, s.files.key, os.Getpid(), code) != nil { + if e := qualificationFinalize(sessionErr, q.restore, s.guard, func() error { + return writeQualificationCompletion(qualificationCompletionDir, s.files.key, os.Getpid(), code) + }); e != nil { + writeQualificationDiagnostics(stderr, e) code = 78 } }() @@ -613,6 +628,7 @@ func qualifyResourceCLI(config, strategies string, stdout, stderr io.Writer, d m return 64 } if e != nil || ctx.Err() != nil { + writeQualificationDiagnostics(stderr, e) fmt.Fprintln(stderr, "unknown: qualification incomplete or ownership/cleanup blocked; no completed matrix published") if signals.Err() != nil && managedCancellationOnly(e, signals.Err()) { return 0 diff --git a/cmd/zapret-autotune/resource_qualify_completion.go b/cmd/zapret-autotune/resource_qualify_completion.go index cd3554d..3ba45d0 100644 --- a/cmd/zapret-autotune/resource_qualify_completion.go +++ b/cmd/zapret-autotune/resource_qualify_completion.go @@ -84,6 +84,25 @@ func qualificationCleanupVerified(err error, restore func(context.Context) error return restore(ctx) == nil } +// Same cleanup gate and ordering as the controller's completion defer. Keeping +// operations separate makes ACK refusal diagnosable without exposing errors. +func qualificationFinalize(err error, restore func(context.Context) error, guard, ack func() error) error { + var restoreErr error + if !qualificationCleanupVerified(err, func(ctx context.Context) error { + restoreErr = restore(ctx) + return restoreErr + }) { + if restoreErr == nil { + restoreErr = errResourcePlan + } + return qualificationOperation(restoreErr, stageRestore) + } + if e := guard(); e != nil { + return qualificationOperation(e, stageFinalStoreGuard) + } + return qualificationOperation(ack(), stageACKWrite) +} + func qualificationCompletionCLI(args []string, d managedDeps) int { if len(args) != 2 || os.Geteuid() != 0 || os.Getppid() != 1 { return 78