fix: diagnose qualification worker scan failures

Report fixed procfs scan reasons, typed errno and numeric worker identities without exposing paths or errors. Preserve scan metadata through attestation and restore normalization while retaining fail-closed decisions and cleanup classification.

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-14 15:40:48 +03:00
1 parent 83921350fd
commit 5cdc0cd767
4 files changed
+264 -19

No files matched your search

@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"io"
"syscall"
)
// These enums are closed: no caller-controlled strings enter diagnostics.
@@ -61,6 +62,9 @@ const (
reasonNFTPresent
reasonBaseline
reasonPriorCleanupVeto
reasonWorkerScanRead
reasonWorkerScanReadlink
reasonWorkerScanForeign
)
func (s qualificationStage) String() string {
@@ -72,7 +76,7 @@ func (s qualificationStage) String() string {
}
func (r qualificationReason) String() string {
names := [...]string{"unknown", "execution", "output_bound", "metrics", "origin", "tls_evidence", "framing", "body", "mime", "worker_identity", "rules", "files", "definition", "runtime", "collector", "guard", "worker_scan", "engine", "argv", "parent", "starttime", "queue", "version", "rules_read", "mode", "plan_store", "context", "queue_present_or_unreadable", "nft_read", "nft_present", "baseline", "prior_cleanup_veto"}
names := [...]string{"unknown", "execution", "output_bound", "metrics", "origin", "tls_evidence", "framing", "body", "mime", "worker_identity", "rules", "files", "definition", "runtime", "collector", "guard", "worker_scan", "engine", "argv", "parent", "starttime", "queue", "version", "rules_read", "mode", "plan_store", "context", "queue_present_or_unreadable", "nft_read", "nft_present", "baseline", "prior_cleanup_veto", "worker_scan_read", "worker_scan_readlink", "worker_scan_foreign"}
if int(r) >= len(names) {
return names[0]
}
@@ -85,6 +89,60 @@ type qualificationDiagnostic struct {
Reason qualificationReason
Candidate, Resource, Attempt int
TLS uint8
Errno qualificationErrno
ScanPID, OwnedPID int
}
type qualificationErrno uint8
const (
errnoNone qualificationErrno = iota
errnoEACCES
errnoEPERM
errnoESRCH
errnoENOENT
errnoOther
)
func (e qualificationErrno) String() string {
names := [...]string{"none", "eacces", "eperm", "esrch", "enoent", "other"}
if int(e) >= len(names) {
return "other"
}
return names[e]
}
func qualificationScanErrno(err error) qualificationErrno {
for _, item := range []struct {
err error
code qualificationErrno
}{{syscall.EACCES, errnoEACCES}, {syscall.EPERM, errnoEPERM}, {syscall.ESRCH, errnoESRCH}, {syscall.ENOENT, errnoENOENT}} {
if errors.Is(err, item.err) {
return item.code
}
}
if err == nil {
return errnoNone
}
return errnoOther
}
func workerScanReason(reason qualificationReason) bool {
return reason == reasonWorkerScanRead || reason == reasonWorkerScanReadlink || reason == reasonWorkerScanForeign
}
// Normalize classification without retaining private predicate errors.
func qualificationPlanPredicate(err error, fallback qualificationReason) error {
d := &qualificationDiagnostic{Err: errResourcePlan, Reason: qualificationErrorReason(err, fallback)}
copyQualificationScanMetadata(d, err)
return d
}
func copyQualificationScanMetadata(d *qualificationDiagnostic, err error) {
var source *qualificationDiagnostic
if workerScanReason(d.Reason) && errors.As(err, &source) {
d.Errno, d.ScanPID, d.OwnedPID = source.Errno, source.ScanPID, source.OwnedPID
}
}
func (e *qualificationDiagnostic) Error() string { return "qualification operation failed" }
@@ -94,11 +152,15 @@ func qualificationOperation(err error, stage qualificationStage) error {
if err == nil {
return nil
}
return &qualificationDiagnostic{Err: err, Stage: stage, Reason: qualificationErrorReason(err, reasonUnknown)}
d := &qualificationDiagnostic{Err: err, Stage: stage, Reason: qualificationErrorReason(err, reasonUnknown)}
copyQualificationScanMetadata(d, err)
return d
}
func qualificationPredicate(err error, reason qualificationReason) error {
return &qualificationDiagnostic{Err: err, Reason: reason}
d := &qualificationDiagnostic{Err: err, Reason: reason}
copyQualificationScanMetadata(d, err)
return d
}
func qualificationErrorReason(err error, fallback qualificationReason) qualificationReason {
@@ -158,6 +220,9 @@ func writeQualificationDiagnostics(w io.Writer, err error) {
if e.Reason != reasonUnknown {
inherited.Reason = e.Reason
}
if workerScanReason(e.Reason) {
inherited.Errno, inherited.ScanPID, inherited.OwnedPID = e.Errno, e.ScanPID, e.OwnedPID
}
cleanup = cleanup || e.Stage == stageNFTDelete || e.Stage == stageStopJoin || e.Stage == stageRestore
visit(e.Err, inherited, cleanup)
return
@@ -182,7 +247,11 @@ func writeQualificationDiagnostics(w io.Writer, err error) {
} else if inherited.TLS == 13 {
tls = "tls13"
}
fmt.Fprintf(w, "qualification diagnostic: role=%s stage=%s reason=%s candidate=%d resource=%d attempt=%d tls=%s\n", role, inherited.Stage, inherited.Reason, inherited.Candidate, inherited.Resource, inherited.Attempt, tls)
fmt.Fprintf(w, "qualification diagnostic: role=%s stage=%s reason=%s candidate=%d resource=%d attempt=%d tls=%s", role, inherited.Stage, inherited.Reason, inherited.Candidate, inherited.Resource, inherited.Attempt, tls)
if workerScanReason(inherited.Reason) {
fmt.Fprintf(w, " errno=%s scan_pid=%d owned_pid=%d", inherited.Errno, inherited.ScanPID, inherited.OwnedPID)
}
fmt.Fprintln(w)
}
visit(err, qualificationDiagnostic{}, false)
}
@@ -107,7 +107,7 @@ func TestQualificationProductionPredicateMap(t *testing.T) {
return b.String()
}
var attest *ast.FuncLit
var restore, guard, noWorker, onlyWorker string
var restore, guard, noWorker, onlyWorker, scanWorkers string
ast.Inspect(f, func(n ast.Node) bool {
if fn, ok := n.(*ast.FuncDecl); ok {
switch fn.Name.Name {
@@ -115,6 +115,8 @@ func TestQualificationProductionPredicateMap(t *testing.T) {
noWorker = text(fn.Body)
case "qualificationOnlyWorker":
onlyWorker = text(fn.Body)
case "qualificationScanWorkers":
scanWorkers = text(fn.Body)
}
}
a, ok := n.(*ast.AssignStmt)
@@ -144,12 +146,13 @@ func TestQualificationProductionPredicateMap(t *testing.T) {
t.Fatal("production guard predicate changed", guard)
}
}
if !strings.Contains(noWorker, "qualificationOnlyWorker(0)") || !strings.Contains(noWorker, "qualificationPredicate(e, reasonWorkerScan)") ||
if !strings.Contains(noWorker, "qualificationOnlyWorker(0)") || !strings.Contains(noWorker, "qualificationPredicate(e, qualificationErrorReason(e, reasonWorkerScan))") ||
!strings.Contains(noWorker, "managedAssetQueueReady(0)") || !strings.Contains(noWorker, "e != nil || occupied") || !strings.Contains(noWorker, "qualificationPredicate(errResourcePlan, reasonQueuePresentOrUnreadable)") {
t.Fatal("production worker/queue predicate changed", noWorker)
}
if !strings.Contains(onlyWorker, "if errors.Is(e, os.ErrNotExist) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn errResourcePlan") {
t.Fatal("unexpected procfs readlink errors must remain fail-closed", onlyWorker)
if !strings.Contains(onlyWorker, "qualificationScanWorkers(owned, os.ReadDir, os.Readlink)") ||
!strings.Contains(scanWorkers, "if errors.Is(e, os.ErrNotExist) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn &qualificationDiagnostic{Err: errResourcePlan, Reason: reasonWorkerScanReadlink") {
t.Fatal("unexpected procfs readlink errors must remain fail-closed", onlyWorker, scanWorkers)
}
var got []string
for _, stmt := range attest.Body.List {
@@ -169,7 +172,7 @@ func TestQualificationProductionPredicateMap(t *testing.T) {
}
want := []string{
"e := q.guard(); e != nil => qualificationPredicate(errResourcePlan, qualificationErrorReason(e, reasonGuard))",
"qualificationOnlyWorker(pid) != nil => qualificationPredicate(errResourcePlan, reasonWorkerScan)",
"e := qualificationOnlyWorker(pid); e != nil => qualificationPlanPredicate(e, reasonWorkerScan)",
`engine == nil || engine.validate() != nil || engine.running(fmt.Sprintf("/proc/%d/exe", pid)) != nil => qualificationPredicate(errResourcePlan, reasonEngine)`,
`e != nil || string(cmd) != strings.Join(append([]string{qualificationEngine}, args...), "\x00")+"\x00" => qualificationPredicate(errResourcePlan, reasonArgv)`,
"e != nil => qualificationPredicate(e, reasonParent)",
@@ -0,0 +1,168 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
"testing"
)
func TestQualificationWorkerScanBranches(t *testing.T) {
secret := errors.New("PRIVATE path argv propertyID sentinel")
dir := t.TempDir()
for _, name := range []string{"0-nonnumeric", "41"} {
if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil {
t.Fatal(err)
}
}
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
err error
code qualificationErrno
}{
{"eacces", syscall.EACCES, errnoEACCES},
{"eperm", syscall.EPERM, errnoEPERM},
{"esrch", syscall.ESRCH, errnoESRCH},
{"enoent", syscall.ENOENT, errnoENOENT},
{"not_exist", os.ErrNotExist, errnoOther},
{"other", secret, errnoOther},
} {
for _, readDirFailure := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/readdir=%t", tc.name, readDirFailure), func(t *testing.T) {
wrapped := &os.PathError{Op: "PRIVATE", Path: secret.Error(), Err: errors.Join(tc.err, secret)}
calls := 0
err := qualificationScanWorkers(42, func(path string) ([]os.DirEntry, error) {
if path != "/proc" {
t.Fatal(path)
}
if readDirFailure {
return nil, wrapped
}
return entries, nil
}, func(path string) (string, error) {
calls++
if path != "/proc/41/exe" {
t.Fatal("nonnumeric entry scanned", path)
}
return "", wrapped
})
if !readDirFailure && errors.Is(wrapped, os.ErrNotExist) {
if err != nil || calls != 1 {
t.Fatal("wrapped ENOENT must skip", err, calls)
}
return
}
var d *qualificationDiagnostic
if !errors.As(err, &d) || d.Errno != tc.code || d.OwnedPID != 42 {
t.Fatal("missing typed metadata", err)
}
if readDirFailure {
if d.Reason != reasonWorkerScanRead || d.ScanPID != 0 || calls != 0 || !errors.Is(err, wrapped) {
t.Fatal("readdir classification changed", d, calls)
}
} else if d.Reason != reasonWorkerScanReadlink || d.ScanPID != 41 || !errors.Is(err, errResourcePlan) || errors.Is(err, wrapped) {
t.Fatal("readlink classification changed", d)
}
assertWorkerScanOutput(t, err, stageUnknown, d.Reason, tc.code, d.ScanPID, 42)
})
}
}
for _, tc := range []struct {
name, exe string
owned int
deny bool
}{
{"owned", "/PRIVATE/nfqws2", 41, false},
{"foreign", "/PRIVATE/nfqws2", 42, true},
// A foreign --version invocation has the same exe: never inspect argv
// or exempt it from the exclusive-worker predicate.
{"foreign_version", "/PRIVATE/nfqws2", 0, true},
{"unrelated", "/PRIVATE/other", 42, false},
} {
t.Run(tc.name, func(t *testing.T) {
err := qualificationScanWorkers(tc.owned, func(string) ([]os.DirEntry, error) { return entries, nil }, func(string) (string, error) { return tc.exe, nil })
if !tc.deny {
if err != nil {
t.Fatal(err)
}
return
}
if !errors.Is(err, errResourcePlan) {
t.Fatal("foreign worker accepted", err)
}
assertWorkerScanOutput(t, err, stageUnknown, reasonWorkerScanForeign, errnoNone, 41, tc.owned)
})
}
}
func assertWorkerScanOutput(t *testing.T, err error, stage qualificationStage, reason qualificationReason, errno qualificationErrno, scan, owned int) {
t.Helper()
var out bytes.Buffer
writeQualificationDiagnostics(&out, err)
want := fmt.Sprintf(" errno=%s scan_pid=%d owned_pid=%d", errno, scan, owned)
if !strings.Contains(out.String(), "stage="+stage.String()+" reason="+reason.String()) || !strings.Contains(out.String(), want) || strings.Contains(out.String(), "PRIVATE") {
t.Fatal(out.String())
}
}
func TestQualificationWorkerScanNormalization(t *testing.T) {
secret := errors.New("PRIVATE path argv propertyID sentinel")
for _, reason := range []qualificationReason{reasonWorkerScanRead, reasonWorkerScanReadlink, reasonWorkerScanForeign} {
for _, errno := range []qualificationErrno{errnoNone, errnoEACCES, errnoEPERM, errnoESRCH, errnoENOENT, errnoOther} {
scan := &qualificationDiagnostic{Err: secret, Reason: reason, Errno: errno, ScanPID: 41, OwnedPID: 42}
attestation := qualificationPlanPredicate(qualificationOperation(scan, stageAttestation), reasonWorkerScan)
if !errors.Is(attestation, errResourcePlan) || errors.Is(attestation, secret) {
t.Fatal("attestation classification changed", attestation)
}
assertWorkerScanOutput(t, qualificationOperation(attestation, stagePostDNSGuard), stagePostDNSGuard, reason, errno, 41, 42)
q := resourceQualifier{guard: func() error { return nil }}
q.runtime.nft = func(context.Context, string) (string, error) {
t.Fatal("restore passed scan veto")
return "", nil
}
restore := q.restoreBaseline(context.Background(), func() error {
return qualificationPredicate(scan, qualificationErrorReason(scan, reasonWorkerScan))
}, func() error { t.Fatal("unsafe baseline restore"); return nil })
if !errors.Is(restore, errResourcePlan) || errors.Is(restore, secret) {
t.Fatal("restore classification changed", restore)
}
assertWorkerScanOutput(t, qualificationOperation(restore, stageRestore), stageRestore, reason, errno, 41, 42)
}
}
var out bytes.Buffer
writeQualificationDiagnostics(&out, qualificationPredicate(secret, reasonEngine))
if strings.Contains(out.String(), "errno=") || strings.Contains(out.String(), "_pid=") || strings.Contains(out.String(), "PRIVATE") {
t.Fatal("scan metadata on non-scan diagnostic", out.String())
}
}
func TestQualificationWorkerScanPostDNSVeto(t *testing.T) {
q, s, registry, _, transfers := qualifierFixture(t)
attest, calls := q.attest, 0
q.attest = func(ctx context.Context, pid int, args []string, d strategyDefinition) (qualificationAttestation, error) {
calls++
if calls == 3 {
scan := &qualificationDiagnostic{Err: errors.New("PRIVATE path argv propertyID sentinel"), Reason: reasonWorkerScanReadlink, Errno: errnoEPERM, ScanPID: 41, OwnedPID: pid}
return qualificationAttestation{}, qualificationPlanPredicate(scan, reasonWorkerScan)
}
return attest(ctx, pid, args, d)
}
proof, err := q.complete(context.Background(), s, registry)
if !errors.Is(err, errResourcePlan) || proof.seal != "" || *transfers != 0 {
t.Fatal("ownership veto or execution order changed", err, *transfers)
}
var out bytes.Buffer
writeQualificationDiagnostics(&out, err)
if strings.Count(out.String(), "stage=post_dns_guard reason=worker_scan_readlink candidate=1 resource=1 attempt=1 tls=tls12 errno=eperm scan_pid=41 owned_pid=") != 2 || strings.Contains(out.String(), "PRIVATE") {
t.Fatal(out.String())
}
}
+15 -10
View File
@@ -151,7 +151,7 @@ func (s *resourcePlanStore) protectBaseline() error {
}
func qualificationNoWorker() error {
if e := qualificationOnlyWorker(0); e != nil {
return qualificationPredicate(e, reasonWorkerScan)
return qualificationPredicate(e, qualificationErrorReason(e, reasonWorkerScan))
}
occupied, e := managedAssetQueueReady(0)
if e != nil || occupied {
@@ -168,7 +168,7 @@ func (q resourceQualifier) restoreBaseline(ctx context.Context, noWorker, baseli
return qualificationPredicate(errResourcePlan, qualificationErrorReason(e, reasonGuard))
}
if e := noWorker(); e != nil {
return qualificationPredicate(errResourcePlan, qualificationErrorReason(e, reasonUnknown))
return qualificationPlanPredicate(e, reasonUnknown)
}
tables, e := q.runtime.nft(ctx, "list tables\n")
if e != nil {
@@ -184,23 +184,28 @@ func (q resourceQualifier) restoreBaseline(ctx context.Context, noWorker, baseli
}
func qualificationOnlyWorker(owned int) error {
entries, e := os.ReadDir("/proc")
return qualificationScanWorkers(owned, os.ReadDir, os.Readlink)
}
func qualificationScanWorkers(owned int, readDir func(string) ([]os.DirEntry, error), readlink func(string) (string, error)) error {
entries, e := readDir("/proc")
if e != nil {
return e
return &qualificationDiagnostic{Err: e, Reason: reasonWorkerScanRead, Errno: qualificationScanErrno(e), OwnedPID: owned}
}
for _, entry := range entries {
if _, e := strconv.Atoi(entry.Name()); e != nil {
pid, e := strconv.Atoi(entry.Name())
if e != nil {
continue
}
exe, e := os.Readlink(filepath.Join("/proc", entry.Name(), "exe"))
exe, e := readlink(filepath.Join("/proc", entry.Name(), "exe"))
if e != nil {
if errors.Is(e, os.ErrNotExist) {
continue
}
return errResourcePlan
return &qualificationDiagnostic{Err: errResourcePlan, Reason: reasonWorkerScanReadlink, Errno: qualificationScanErrno(e), ScanPID: pid, OwnedPID: owned}
}
if filepath.Base(exe) == "nfqws2" && entry.Name() != strconv.Itoa(owned) {
return errResourcePlan
return &qualificationDiagnostic{Err: errResourcePlan, Reason: reasonWorkerScanForeign, ScanPID: pid, OwnedPID: owned}
}
}
return nil
@@ -283,8 +288,8 @@ func newResourceQualifier(config, strategies string, d managedDeps, s *resourceP
if e := q.guard(); e != nil {
return qualificationAttestation{}, qualificationPredicate(errResourcePlan, qualificationErrorReason(e, reasonGuard))
}
if qualificationOnlyWorker(pid) != nil {
return qualificationAttestation{}, qualificationPredicate(errResourcePlan, reasonWorkerScan)
if e := qualificationOnlyWorker(pid); e != nil {
return qualificationAttestation{}, qualificationPlanPredicate(e, reasonWorkerScan)
}
if engine == nil || engine.validate() != nil || engine.running(fmt.Sprintf("/proc/%d/exe", pid)) != nil {
return qualificationAttestation{}, qualificationPredicate(errResourcePlan, reasonEngine)