fix: diagnose qualification worker scan failures
Report fixed procfs scan reasons, typed errno and numeric worker identities without exposing paths or errors. Preserve scan metadata through attestation and restore normalization while retaining fail-closed decisions and cleanup classification. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
83921350fd
commit
5cdc0cd767
4 files changed
+264
-19
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// These enums are closed: no caller-controlled strings enter diagnostics.
|
||||
@@ -61,6 +62,9 @@ const (
|
||||
reasonNFTPresent
|
||||
reasonBaseline
|
||||
reasonPriorCleanupVeto
|
||||
reasonWorkerScanRead
|
||||
reasonWorkerScanReadlink
|
||||
reasonWorkerScanForeign
|
||||
)
|
||||
|
||||
func (s qualificationStage) String() string {
|
||||
@@ -72,7 +76,7 @@ func (s qualificationStage) String() string {
|
||||
}
|
||||
|
||||
func (r qualificationReason) String() string {
|
||||
names := [...]string{"unknown", "execution", "output_bound", "metrics", "origin", "tls_evidence", "framing", "body", "mime", "worker_identity", "rules", "files", "definition", "runtime", "collector", "guard", "worker_scan", "engine", "argv", "parent", "starttime", "queue", "version", "rules_read", "mode", "plan_store", "context", "queue_present_or_unreadable", "nft_read", "nft_present", "baseline", "prior_cleanup_veto"}
|
||||
names := [...]string{"unknown", "execution", "output_bound", "metrics", "origin", "tls_evidence", "framing", "body", "mime", "worker_identity", "rules", "files", "definition", "runtime", "collector", "guard", "worker_scan", "engine", "argv", "parent", "starttime", "queue", "version", "rules_read", "mode", "plan_store", "context", "queue_present_or_unreadable", "nft_read", "nft_present", "baseline", "prior_cleanup_veto", "worker_scan_read", "worker_scan_readlink", "worker_scan_foreign"}
|
||||
if int(r) >= len(names) {
|
||||
return names[0]
|
||||
}
|
||||
@@ -85,6 +89,60 @@ type qualificationDiagnostic struct {
|
||||
Reason qualificationReason
|
||||
Candidate, Resource, Attempt int
|
||||
TLS uint8
|
||||
Errno qualificationErrno
|
||||
ScanPID, OwnedPID int
|
||||
}
|
||||
|
||||
type qualificationErrno uint8
|
||||
|
||||
const (
|
||||
errnoNone qualificationErrno = iota
|
||||
errnoEACCES
|
||||
errnoEPERM
|
||||
errnoESRCH
|
||||
errnoENOENT
|
||||
errnoOther
|
||||
)
|
||||
|
||||
func (e qualificationErrno) String() string {
|
||||
names := [...]string{"none", "eacces", "eperm", "esrch", "enoent", "other"}
|
||||
if int(e) >= len(names) {
|
||||
return "other"
|
||||
}
|
||||
return names[e]
|
||||
}
|
||||
|
||||
func qualificationScanErrno(err error) qualificationErrno {
|
||||
for _, item := range []struct {
|
||||
err error
|
||||
code qualificationErrno
|
||||
}{{syscall.EACCES, errnoEACCES}, {syscall.EPERM, errnoEPERM}, {syscall.ESRCH, errnoESRCH}, {syscall.ENOENT, errnoENOENT}} {
|
||||
if errors.Is(err, item.err) {
|
||||
return item.code
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
return errnoNone
|
||||
}
|
||||
return errnoOther
|
||||
}
|
||||
|
||||
func workerScanReason(reason qualificationReason) bool {
|
||||
return reason == reasonWorkerScanRead || reason == reasonWorkerScanReadlink || reason == reasonWorkerScanForeign
|
||||
}
|
||||
|
||||
// Normalize classification without retaining private predicate errors.
|
||||
func qualificationPlanPredicate(err error, fallback qualificationReason) error {
|
||||
d := &qualificationDiagnostic{Err: errResourcePlan, Reason: qualificationErrorReason(err, fallback)}
|
||||
copyQualificationScanMetadata(d, err)
|
||||
return d
|
||||
}
|
||||
|
||||
func copyQualificationScanMetadata(d *qualificationDiagnostic, err error) {
|
||||
var source *qualificationDiagnostic
|
||||
if workerScanReason(d.Reason) && errors.As(err, &source) {
|
||||
d.Errno, d.ScanPID, d.OwnedPID = source.Errno, source.ScanPID, source.OwnedPID
|
||||
}
|
||||
}
|
||||
|
||||
func (e *qualificationDiagnostic) Error() string { return "qualification operation failed" }
|
||||
@@ -94,11 +152,15 @@ func qualificationOperation(err error, stage qualificationStage) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return &qualificationDiagnostic{Err: err, Stage: stage, Reason: qualificationErrorReason(err, reasonUnknown)}
|
||||
d := &qualificationDiagnostic{Err: err, Stage: stage, Reason: qualificationErrorReason(err, reasonUnknown)}
|
||||
copyQualificationScanMetadata(d, err)
|
||||
return d
|
||||
}
|
||||
|
||||
func qualificationPredicate(err error, reason qualificationReason) error {
|
||||
return &qualificationDiagnostic{Err: err, Reason: reason}
|
||||
d := &qualificationDiagnostic{Err: err, Reason: reason}
|
||||
copyQualificationScanMetadata(d, err)
|
||||
return d
|
||||
}
|
||||
|
||||
func qualificationErrorReason(err error, fallback qualificationReason) qualificationReason {
|
||||
@@ -158,6 +220,9 @@ func writeQualificationDiagnostics(w io.Writer, err error) {
|
||||
if e.Reason != reasonUnknown {
|
||||
inherited.Reason = e.Reason
|
||||
}
|
||||
if workerScanReason(e.Reason) {
|
||||
inherited.Errno, inherited.ScanPID, inherited.OwnedPID = e.Errno, e.ScanPID, e.OwnedPID
|
||||
}
|
||||
cleanup = cleanup || e.Stage == stageNFTDelete || e.Stage == stageStopJoin || e.Stage == stageRestore
|
||||
visit(e.Err, inherited, cleanup)
|
||||
return
|
||||
@@ -182,7 +247,11 @@ func writeQualificationDiagnostics(w io.Writer, err error) {
|
||||
} else if inherited.TLS == 13 {
|
||||
tls = "tls13"
|
||||
}
|
||||
fmt.Fprintf(w, "qualification diagnostic: role=%s stage=%s reason=%s candidate=%d resource=%d attempt=%d tls=%s\n", role, inherited.Stage, inherited.Reason, inherited.Candidate, inherited.Resource, inherited.Attempt, tls)
|
||||
fmt.Fprintf(w, "qualification diagnostic: role=%s stage=%s reason=%s candidate=%d resource=%d attempt=%d tls=%s", role, inherited.Stage, inherited.Reason, inherited.Candidate, inherited.Resource, inherited.Attempt, tls)
|
||||
if workerScanReason(inherited.Reason) {
|
||||
fmt.Fprintf(w, " errno=%s scan_pid=%d owned_pid=%d", inherited.Errno, inherited.ScanPID, inherited.OwnedPID)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
}
|
||||
visit(err, qualificationDiagnostic{}, false)
|
||||
}
|
||||
@@ -107,7 +107,7 @@ func TestQualificationProductionPredicateMap(t *testing.T) {
|
||||
return b.String()
|
||||
}
|
||||
var attest *ast.FuncLit
|
||||
var restore, guard, noWorker, onlyWorker string
|
||||
var restore, guard, noWorker, onlyWorker, scanWorkers string
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
if fn, ok := n.(*ast.FuncDecl); ok {
|
||||
switch fn.Name.Name {
|
||||
@@ -115,6 +115,8 @@ func TestQualificationProductionPredicateMap(t *testing.T) {
|
||||
noWorker = text(fn.Body)
|
||||
case "qualificationOnlyWorker":
|
||||
onlyWorker = text(fn.Body)
|
||||
case "qualificationScanWorkers":
|
||||
scanWorkers = text(fn.Body)
|
||||
}
|
||||
}
|
||||
a, ok := n.(*ast.AssignStmt)
|
||||
@@ -144,12 +146,13 @@ func TestQualificationProductionPredicateMap(t *testing.T) {
|
||||
t.Fatal("production guard predicate changed", guard)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(noWorker, "qualificationOnlyWorker(0)") || !strings.Contains(noWorker, "qualificationPredicate(e, reasonWorkerScan)") ||
|
||||
if !strings.Contains(noWorker, "qualificationOnlyWorker(0)") || !strings.Contains(noWorker, "qualificationPredicate(e, qualificationErrorReason(e, reasonWorkerScan))") ||
|
||||
!strings.Contains(noWorker, "managedAssetQueueReady(0)") || !strings.Contains(noWorker, "e != nil || occupied") || !strings.Contains(noWorker, "qualificationPredicate(errResourcePlan, reasonQueuePresentOrUnreadable)") {
|
||||
t.Fatal("production worker/queue predicate changed", noWorker)
|
||||
}
|
||||
if !strings.Contains(onlyWorker, "if errors.Is(e, os.ErrNotExist) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn errResourcePlan") {
|
||||
t.Fatal("unexpected procfs readlink errors must remain fail-closed", onlyWorker)
|
||||
if !strings.Contains(onlyWorker, "qualificationScanWorkers(owned, os.ReadDir, os.Readlink)") ||
|
||||
!strings.Contains(scanWorkers, "if errors.Is(e, os.ErrNotExist) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn &qualificationDiagnostic{Err: errResourcePlan, Reason: reasonWorkerScanReadlink") {
|
||||
t.Fatal("unexpected procfs readlink errors must remain fail-closed", onlyWorker, scanWorkers)
|
||||
}
|
||||
var got []string
|
||||
for _, stmt := range attest.Body.List {
|
||||
@@ -169,7 +172,7 @@ func TestQualificationProductionPredicateMap(t *testing.T) {
|
||||
}
|
||||
want := []string{
|
||||
"e := q.guard(); e != nil => qualificationPredicate(errResourcePlan, qualificationErrorReason(e, reasonGuard))",
|
||||
"qualificationOnlyWorker(pid) != nil => qualificationPredicate(errResourcePlan, reasonWorkerScan)",
|
||||
"e := qualificationOnlyWorker(pid); e != nil => qualificationPlanPredicate(e, reasonWorkerScan)",
|
||||
`engine == nil || engine.validate() != nil || engine.running(fmt.Sprintf("/proc/%d/exe", pid)) != nil => qualificationPredicate(errResourcePlan, reasonEngine)`,
|
||||
`e != nil || string(cmd) != strings.Join(append([]string{qualificationEngine}, args...), "\x00")+"\x00" => qualificationPredicate(errResourcePlan, reasonArgv)`,
|
||||
"e != nil => qualificationPredicate(e, reasonParent)",
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQualificationWorkerScanBranches(t *testing.T) {
|
||||
secret := errors.New("PRIVATE path argv propertyID sentinel")
|
||||
dir := t.TempDir()
|
||||
for _, name := range []string{"0-nonnumeric", "41"} {
|
||||
if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
code qualificationErrno
|
||||
}{
|
||||
{"eacces", syscall.EACCES, errnoEACCES},
|
||||
{"eperm", syscall.EPERM, errnoEPERM},
|
||||
{"esrch", syscall.ESRCH, errnoESRCH},
|
||||
{"enoent", syscall.ENOENT, errnoENOENT},
|
||||
{"not_exist", os.ErrNotExist, errnoOther},
|
||||
{"other", secret, errnoOther},
|
||||
} {
|
||||
for _, readDirFailure := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/readdir=%t", tc.name, readDirFailure), func(t *testing.T) {
|
||||
wrapped := &os.PathError{Op: "PRIVATE", Path: secret.Error(), Err: errors.Join(tc.err, secret)}
|
||||
calls := 0
|
||||
err := qualificationScanWorkers(42, func(path string) ([]os.DirEntry, error) {
|
||||
if path != "/proc" {
|
||||
t.Fatal(path)
|
||||
}
|
||||
if readDirFailure {
|
||||
return nil, wrapped
|
||||
}
|
||||
return entries, nil
|
||||
}, func(path string) (string, error) {
|
||||
calls++
|
||||
if path != "/proc/41/exe" {
|
||||
t.Fatal("nonnumeric entry scanned", path)
|
||||
}
|
||||
return "", wrapped
|
||||
})
|
||||
if !readDirFailure && errors.Is(wrapped, os.ErrNotExist) {
|
||||
if err != nil || calls != 1 {
|
||||
t.Fatal("wrapped ENOENT must skip", err, calls)
|
||||
}
|
||||
return
|
||||
}
|
||||
var d *qualificationDiagnostic
|
||||
if !errors.As(err, &d) || d.Errno != tc.code || d.OwnedPID != 42 {
|
||||
t.Fatal("missing typed metadata", err)
|
||||
}
|
||||
if readDirFailure {
|
||||
if d.Reason != reasonWorkerScanRead || d.ScanPID != 0 || calls != 0 || !errors.Is(err, wrapped) {
|
||||
t.Fatal("readdir classification changed", d, calls)
|
||||
}
|
||||
} else if d.Reason != reasonWorkerScanReadlink || d.ScanPID != 41 || !errors.Is(err, errResourcePlan) || errors.Is(err, wrapped) {
|
||||
t.Fatal("readlink classification changed", d)
|
||||
}
|
||||
assertWorkerScanOutput(t, err, stageUnknown, d.Reason, tc.code, d.ScanPID, 42)
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name, exe string
|
||||
owned int
|
||||
deny bool
|
||||
}{
|
||||
{"owned", "/PRIVATE/nfqws2", 41, false},
|
||||
{"foreign", "/PRIVATE/nfqws2", 42, true},
|
||||
// A foreign --version invocation has the same exe: never inspect argv
|
||||
// or exempt it from the exclusive-worker predicate.
|
||||
{"foreign_version", "/PRIVATE/nfqws2", 0, true},
|
||||
{"unrelated", "/PRIVATE/other", 42, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := qualificationScanWorkers(tc.owned, func(string) ([]os.DirEntry, error) { return entries, nil }, func(string) (string, error) { return tc.exe, nil })
|
||||
if !tc.deny {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !errors.Is(err, errResourcePlan) {
|
||||
t.Fatal("foreign worker accepted", err)
|
||||
}
|
||||
assertWorkerScanOutput(t, err, stageUnknown, reasonWorkerScanForeign, errnoNone, 41, tc.owned)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertWorkerScanOutput(t *testing.T, err error, stage qualificationStage, reason qualificationReason, errno qualificationErrno, scan, owned int) {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
writeQualificationDiagnostics(&out, err)
|
||||
want := fmt.Sprintf(" errno=%s scan_pid=%d owned_pid=%d", errno, scan, owned)
|
||||
if !strings.Contains(out.String(), "stage="+stage.String()+" reason="+reason.String()) || !strings.Contains(out.String(), want) || strings.Contains(out.String(), "PRIVATE") {
|
||||
t.Fatal(out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestQualificationWorkerScanNormalization(t *testing.T) {
|
||||
secret := errors.New("PRIVATE path argv propertyID sentinel")
|
||||
for _, reason := range []qualificationReason{reasonWorkerScanRead, reasonWorkerScanReadlink, reasonWorkerScanForeign} {
|
||||
for _, errno := range []qualificationErrno{errnoNone, errnoEACCES, errnoEPERM, errnoESRCH, errnoENOENT, errnoOther} {
|
||||
scan := &qualificationDiagnostic{Err: secret, Reason: reason, Errno: errno, ScanPID: 41, OwnedPID: 42}
|
||||
attestation := qualificationPlanPredicate(qualificationOperation(scan, stageAttestation), reasonWorkerScan)
|
||||
if !errors.Is(attestation, errResourcePlan) || errors.Is(attestation, secret) {
|
||||
t.Fatal("attestation classification changed", attestation)
|
||||
}
|
||||
assertWorkerScanOutput(t, qualificationOperation(attestation, stagePostDNSGuard), stagePostDNSGuard, reason, errno, 41, 42)
|
||||
q := resourceQualifier{guard: func() error { return nil }}
|
||||
q.runtime.nft = func(context.Context, string) (string, error) {
|
||||
t.Fatal("restore passed scan veto")
|
||||
return "", nil
|
||||
}
|
||||
restore := q.restoreBaseline(context.Background(), func() error {
|
||||
return qualificationPredicate(scan, qualificationErrorReason(scan, reasonWorkerScan))
|
||||
}, func() error { t.Fatal("unsafe baseline restore"); return nil })
|
||||
if !errors.Is(restore, errResourcePlan) || errors.Is(restore, secret) {
|
||||
t.Fatal("restore classification changed", restore)
|
||||
}
|
||||
assertWorkerScanOutput(t, qualificationOperation(restore, stageRestore), stageRestore, reason, errno, 41, 42)
|
||||
}
|
||||
}
|
||||
var out bytes.Buffer
|
||||
writeQualificationDiagnostics(&out, qualificationPredicate(secret, reasonEngine))
|
||||
if strings.Contains(out.String(), "errno=") || strings.Contains(out.String(), "_pid=") || strings.Contains(out.String(), "PRIVATE") {
|
||||
t.Fatal("scan metadata on non-scan diagnostic", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestQualificationWorkerScanPostDNSVeto(t *testing.T) {
|
||||
q, s, registry, _, transfers := qualifierFixture(t)
|
||||
attest, calls := q.attest, 0
|
||||
q.attest = func(ctx context.Context, pid int, args []string, d strategyDefinition) (qualificationAttestation, error) {
|
||||
calls++
|
||||
if calls == 3 {
|
||||
scan := &qualificationDiagnostic{Err: errors.New("PRIVATE path argv propertyID sentinel"), Reason: reasonWorkerScanReadlink, Errno: errnoEPERM, ScanPID: 41, OwnedPID: pid}
|
||||
return qualificationAttestation{}, qualificationPlanPredicate(scan, reasonWorkerScan)
|
||||
}
|
||||
return attest(ctx, pid, args, d)
|
||||
}
|
||||
proof, err := q.complete(context.Background(), s, registry)
|
||||
if !errors.Is(err, errResourcePlan) || proof.seal != "" || *transfers != 0 {
|
||||
t.Fatal("ownership veto or execution order changed", err, *transfers)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
writeQualificationDiagnostics(&out, err)
|
||||
if strings.Count(out.String(), "stage=post_dns_guard reason=worker_scan_readlink candidate=1 resource=1 attempt=1 tls=tls12 errno=eperm scan_pid=41 owned_pid=") != 2 || strings.Contains(out.String(), "PRIVATE") {
|
||||
t.Fatal(out.String())
|
||||
}
|
||||
}
|
||||
@@ -151,7 +151,7 @@ func (s *resourcePlanStore) protectBaseline() error {
|
||||
}
|
||||
func qualificationNoWorker() error {
|
||||
if e := qualificationOnlyWorker(0); e != nil {
|
||||
return qualificationPredicate(e, reasonWorkerScan)
|
||||
return qualificationPredicate(e, qualificationErrorReason(e, reasonWorkerScan))
|
||||
}
|
||||
occupied, e := managedAssetQueueReady(0)
|
||||
if e != nil || occupied {
|
||||
@@ -168,7 +168,7 @@ func (q resourceQualifier) restoreBaseline(ctx context.Context, noWorker, baseli
|
||||
return qualificationPredicate(errResourcePlan, qualificationErrorReason(e, reasonGuard))
|
||||
}
|
||||
if e := noWorker(); e != nil {
|
||||
return qualificationPredicate(errResourcePlan, qualificationErrorReason(e, reasonUnknown))
|
||||
return qualificationPlanPredicate(e, reasonUnknown)
|
||||
}
|
||||
tables, e := q.runtime.nft(ctx, "list tables\n")
|
||||
if e != nil {
|
||||
@@ -184,23 +184,28 @@ func (q resourceQualifier) restoreBaseline(ctx context.Context, noWorker, baseli
|
||||
}
|
||||
|
||||
func qualificationOnlyWorker(owned int) error {
|
||||
entries, e := os.ReadDir("/proc")
|
||||
return qualificationScanWorkers(owned, os.ReadDir, os.Readlink)
|
||||
}
|
||||
|
||||
func qualificationScanWorkers(owned int, readDir func(string) ([]os.DirEntry, error), readlink func(string) (string, error)) error {
|
||||
entries, e := readDir("/proc")
|
||||
if e != nil {
|
||||
return e
|
||||
return &qualificationDiagnostic{Err: e, Reason: reasonWorkerScanRead, Errno: qualificationScanErrno(e), OwnedPID: owned}
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if _, e := strconv.Atoi(entry.Name()); e != nil {
|
||||
pid, e := strconv.Atoi(entry.Name())
|
||||
if e != nil {
|
||||
continue
|
||||
}
|
||||
exe, e := os.Readlink(filepath.Join("/proc", entry.Name(), "exe"))
|
||||
exe, e := readlink(filepath.Join("/proc", entry.Name(), "exe"))
|
||||
if e != nil {
|
||||
if errors.Is(e, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return errResourcePlan
|
||||
return &qualificationDiagnostic{Err: errResourcePlan, Reason: reasonWorkerScanReadlink, Errno: qualificationScanErrno(e), ScanPID: pid, OwnedPID: owned}
|
||||
}
|
||||
if filepath.Base(exe) == "nfqws2" && entry.Name() != strconv.Itoa(owned) {
|
||||
return errResourcePlan
|
||||
return &qualificationDiagnostic{Err: errResourcePlan, Reason: reasonWorkerScanForeign, ScanPID: pid, OwnedPID: owned}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -283,8 +288,8 @@ func newResourceQualifier(config, strategies string, d managedDeps, s *resourceP
|
||||
if e := q.guard(); e != nil {
|
||||
return qualificationAttestation{}, qualificationPredicate(errResourcePlan, qualificationErrorReason(e, reasonGuard))
|
||||
}
|
||||
if qualificationOnlyWorker(pid) != nil {
|
||||
return qualificationAttestation{}, qualificationPredicate(errResourcePlan, reasonWorkerScan)
|
||||
if e := qualificationOnlyWorker(pid); e != nil {
|
||||
return qualificationAttestation{}, qualificationPlanPredicate(e, reasonWorkerScan)
|
||||
}
|
||||
if engine == nil || engine.validate() != nil || engine.running(fmt.Sprintf("/proc/%d/exe", pid)) != nil {
|
||||
return qualificationAttestation{}, qualificationPredicate(errResourcePlan, reasonEngine)
|
||||
|
||||
Reference in new issue
Block a user