fix: validate live resource monitor ownership in status
Bind authenticated snapshots to controller PID/starttime and validate process identity and held inode anchors before reporting current check health. Preserve identical signed foreign replacements and fail closed when live ownership is unavailable. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
4efc066625
commit
5c85f3f0b9
5 files changed
+247
-2
No files matched your search
@@ -62,6 +62,13 @@ unavailable ownership. Resource failures continue to other resources; ownership
|
||||
failure stops all probes. These checks have unit/loopback-fixture coverage only,
|
||||
not new remote acceptance or generic automated recovery/controller support.
|
||||
|
||||
Generic `managed resource-status` validates the authenticated snapshot against
|
||||
the live controller's PID/starttime, executable, command, UID, namespaces and
|
||||
held directory/key/state/singleton inodes. Cached check results alone cannot
|
||||
report current health: missing ownership or an identical signed replacement
|
||||
returns `BLOCKED` with `stale: true` without modifying the foreign file.
|
||||
`CHECKS_ONLY`/`HEALTHY` still do not mean candidate or production readiness.
|
||||
|
||||
## Production preparation (isolated clean-run acceptance passed)
|
||||
|
||||
The standalone deployment candidate is a **new isolated target**, never an
|
||||
|
||||
@@ -316,6 +316,17 @@ func resourceMonitorCLIWithCheck(ctx context.Context, path string, stdout, stder
|
||||
}
|
||||
continue
|
||||
}
|
||||
start, err := readMonitorProcessStarttime(os.Getpid())
|
||||
state, loadErr := store.load()
|
||||
if err != nil || loadErr != nil {
|
||||
fmt.Fprintln(stderr, "BLOCKED: generic monitor ownership unavailable")
|
||||
return 78
|
||||
}
|
||||
state.Controller = &resourceMonitorOwner{PID: os.Getpid(), Start: start}
|
||||
if store.save(state) != nil {
|
||||
fmt.Fprintln(stderr, "BLOCKED: generic monitor ownership publication failed")
|
||||
return 78
|
||||
}
|
||||
}
|
||||
err := runResourceMonitor(ctx, resourceMonitorDeps{
|
||||
load: store.load, save: store.save, key: store.key,
|
||||
@@ -363,6 +374,10 @@ func resourceSingletonGuard(lock *managedMonitorSingleton, dir string) error {
|
||||
}
|
||||
|
||||
func resourceStatusCLI(stdout io.Writer, target managedDeps) int {
|
||||
return resourceStatusCLIAt(stdout, target, "/proc")
|
||||
}
|
||||
|
||||
func resourceStatusCLIAt(stdout io.Writer, target managedDeps, proc string) int {
|
||||
blocked := func() int { fmt.Fprintln(stdout, `{"health":"BLOCKED","stale":true}`); return 1 }
|
||||
store, err := openResourceMonitorStore(target.directory, false)
|
||||
if err != nil {
|
||||
@@ -370,10 +385,10 @@ func resourceStatusCLI(stdout io.Writer, target managedDeps) int {
|
||||
}
|
||||
defer store.close()
|
||||
state, err := store.load()
|
||||
if err != nil {
|
||||
path, _ := target.lookup("ZAPRET_AUTOTUNE_RESOURCE_CONFIG")
|
||||
if err != nil || resourceMonitorLiveOwner(store, state, path, proc) != nil {
|
||||
return blocked()
|
||||
}
|
||||
path, _ := target.lookup("ZAPRET_AUTOTUNE_RESOURCE_CONFIG")
|
||||
source, err := readResourceMonitorSource(path, target.lookup, store.key)
|
||||
check := productionResourceCheckDeps()
|
||||
if err != nil || check.guard() != nil || resourceManagedIdentity(target.directory) != nil {
|
||||
@@ -390,6 +405,11 @@ func resourceStatusCLI(stdout io.Writer, target managedDeps) int {
|
||||
if state.Blocked {
|
||||
health = "BLOCKED"
|
||||
}
|
||||
// Source/runtime validation may take time; don't emit a cache whose live
|
||||
// ownership disappeared or whose inode changed during those checks.
|
||||
if resourceMonitorLiveOwner(store, state, path, proc) != nil {
|
||||
return blocked()
|
||||
}
|
||||
if json.NewEncoder(stdout).Encode(struct {
|
||||
Health string `json:"health"`
|
||||
Stale bool `json:"stale"`
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// A signed cache is not a live ownership claim. Inspect only procfs metadata,
|
||||
// never the controller's private FD contents, and fail closed on uncertainty.
|
||||
func resourceMonitorLiveOwner(store *resourceMonitorStore, state resourceMonitorState, config, proc string) error {
|
||||
owner := state.Controller
|
||||
if owner == nil || owner.PID <= 1 || owner.Start <= 0 || config == "" {
|
||||
return errResourceState
|
||||
}
|
||||
base := filepath.Join(proc, strconv.Itoa(owner.PID))
|
||||
identity := func() bool {
|
||||
stat, err := os.ReadFile(filepath.Join(base, "stat"))
|
||||
start, parseErr := parseMonitorProcessStat(owner.PID, string(stat))
|
||||
if err != nil || parseErr != nil || start != owner.Start {
|
||||
return false
|
||||
}
|
||||
exe, err := os.Readlink(filepath.Join(proc, "self/exe"))
|
||||
self, selfErr := os.Stat(filepath.Join(proc, "self/exe"))
|
||||
other, otherErr := os.Stat(filepath.Join(base, "exe"))
|
||||
if err != nil || selfErr != nil || otherErr != nil || !os.SameFile(self, other) {
|
||||
return false
|
||||
}
|
||||
return resourceProcessOwned(proc, owner.PID, exe, []string{exe, "managed", "monitor", "--resource-config", config}) ||
|
||||
resourceProcessOwned(proc, owner.PID, exe, []string{exe, "managed", "monitor"})
|
||||
}
|
||||
if !identity() || store.snapshotGuard() != nil {
|
||||
return errResourceState
|
||||
}
|
||||
// Include the singleton and directory so another target's controller cannot
|
||||
// lend apparent liveness to a copied authenticated snapshot.
|
||||
anchors := []string{store.path, store.entryPath(resourceKeyName), store.entryPath(resourceStateName), store.entryPath(managedMonitorLockName)}
|
||||
fds, err := os.ReadDir(filepath.Join(base, "fd"))
|
||||
if err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
for _, anchor := range anchors {
|
||||
current, err := os.Lstat(anchor)
|
||||
if err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
matched := false
|
||||
for _, fd := range fds {
|
||||
held, err := os.Stat(filepath.Join(base, "fd", fd.Name()))
|
||||
if err == nil && os.SameFile(current, held) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return errResourceState
|
||||
}
|
||||
}
|
||||
if !identity() || store.snapshotGuard() != nil || managedPrivateFile(store.entryPath(managedMonitorLockName)) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestResourceStatusLiveOwnership(t *testing.T) {
|
||||
for _, change := range []string{"state", "key", "directory", "lock", "dead", "pid-reuse", "exe", "argv", "uid", "parent", "namespace", "missing-owner"} {
|
||||
t.Run(change, func(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "state")
|
||||
if err := os.Mkdir(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err := openResourceMonitorStore(dir, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.close()
|
||||
write := func(path, contents string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
link := func(path, target string) {
|
||||
t.Helper()
|
||||
if err := os.Symlink(target, path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
lockPath := filepath.Join(dir, managedMonitorLockName)
|
||||
write(lockPath, "")
|
||||
lock, err := os.Open(lockPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lock.Close()
|
||||
source, _ := resourceMonitorFixture(t)
|
||||
state := resourceMonitorState{Version: 1, Updated: time.Now().UnixMilli(), Controller: &resourceMonitorOwner{PID: 42, Start: 123}, Rows: reconcileResourceRows(store.key, source, nil)}
|
||||
for i := range state.Rows {
|
||||
state.Rows[i].Health, state.Rows[i].Outcome = "HEALTHY", "pass"
|
||||
state.Rows[i].LastCompleted, state.Rows[i].NextDue = state.Updated, state.Updated+60000
|
||||
}
|
||||
if err := store.save(state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err = store.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proc := t.TempDir()
|
||||
for _, sub := range []string{"self/ns", "42/ns", "42/fd"} {
|
||||
if err := os.MkdirAll(filepath.Join(proc, sub), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
link(filepath.Join(proc, "self/exe"), exe)
|
||||
link(filepath.Join(proc, "42/exe"), exe)
|
||||
for _, ns := range []string{"pid", "net"} {
|
||||
link(filepath.Join(proc, "self/ns", ns), ns+"[1]")
|
||||
link(filepath.Join(proc, "42/ns", ns), ns+"[1]")
|
||||
}
|
||||
fields := append([]string{"S"}, strings.Fields(strings.Repeat("0 ", 18))...)
|
||||
fields = append(fields, "123")
|
||||
stat := "42 (monitor) " + strings.Join(fields, " ")
|
||||
write(filepath.Join(proc, "42/stat"), stat)
|
||||
uid := strconv.Itoa(os.Geteuid())
|
||||
write(filepath.Join(proc, "42/status"), "PPid:\t1\nUid:\t"+strings.Join([]string{uid, uid, uid, uid}, "\t")+"\n")
|
||||
config := "/private/checks.json"
|
||||
write(filepath.Join(proc, "42/cmdline"), strings.Join([]string{exe, "managed", "monitor", "--resource-config", config}, "\x00")+"\x00")
|
||||
for i, f := range []*os.File{store.dir, store.keyFile, store.stateFile, lock} {
|
||||
link(filepath.Join(proc, "42/fd", strconv.Itoa(i)), "/proc/self/fd/"+strconv.FormatUint(uint64(f.Fd()), 10))
|
||||
}
|
||||
if err := resourceMonitorLiveOwner(store, state, config, proc); err != nil {
|
||||
t.Fatal("valid live owner refused", err)
|
||||
}
|
||||
if resourceSnapshotStale(state, time.Now().UnixMilli()) {
|
||||
t.Fatal("fixture already stale")
|
||||
}
|
||||
var preservedPath string
|
||||
var preserved []byte
|
||||
switch change {
|
||||
case "state", "key", "lock":
|
||||
name := map[string]string{"state": resourceStateName, "key": resourceKeyName, "lock": managedMonitorLockName}[change]
|
||||
preservedPath = filepath.Join(dir, name)
|
||||
preserved, err = os.ReadFile(preservedPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write(preservedPath+".replacement", string(preserved))
|
||||
if err := os.Rename(preservedPath+".replacement", preservedPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "directory":
|
||||
if err := os.Rename(dir, dir+".old"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Mkdir(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "dead":
|
||||
os.Remove(filepath.Join(proc, "42/stat"))
|
||||
case "pid-reuse":
|
||||
write(filepath.Join(proc, "42/stat"), strings.TrimSuffix(stat, "123")+"124")
|
||||
case "exe":
|
||||
os.Remove(filepath.Join(proc, "42/exe"))
|
||||
link(filepath.Join(proc, "42/exe"), "/bin/sh")
|
||||
case "argv":
|
||||
write(filepath.Join(proc, "42/cmdline"), exe+"\x00managed\x00resource-status\x00")
|
||||
case "uid":
|
||||
write(filepath.Join(proc, "42/status"), "PPid:\t1\nUid:\t9999\t9999\t9999\t9999\n")
|
||||
case "parent":
|
||||
write(filepath.Join(proc, "42/status"), "PPid:\t2\nUid:\t"+strings.Join([]string{uid, uid, uid, uid}, "\t")+"\n")
|
||||
case "namespace":
|
||||
os.Remove(filepath.Join(proc, "42/ns/net"))
|
||||
link(filepath.Join(proc, "42/ns/net"), "net[2]")
|
||||
case "missing-owner":
|
||||
state.Controller = nil
|
||||
if err := store.save(state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if resourceMonitorLiveOwner(store, state, config, proc) == nil {
|
||||
t.Fatal("invalid live ownership accepted")
|
||||
}
|
||||
target := managedDeps{directory: dir, lookup: func(name string) (string, bool) { return config, name == "ZAPRET_AUTOTUNE_RESOURCE_CONFIG" }}
|
||||
var out bytes.Buffer
|
||||
if resourceStatusCLIAt(&out, target, proc) != 1 || out.String() != "{\"health\":\"BLOCKED\",\"stale\":true}\n" {
|
||||
t.Fatal("cached status hid live failure", out.String())
|
||||
}
|
||||
if preservedPath != "" {
|
||||
got, err := os.ReadFile(preservedPath)
|
||||
if err != nil || !bytes.Equal(got, preserved) {
|
||||
t.Fatal("foreign entry modified", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -35,8 +35,14 @@ type resourceMonitorState struct {
|
||||
Updated int64 `json:"updated_ms"`
|
||||
Blocked bool `json:"blocked"`
|
||||
Rows []resourceMonitorRow `json:"rows"`
|
||||
Controller *resourceMonitorOwner `json:"controller,omitempty"`
|
||||
MAC string `json:"mac"`
|
||||
}
|
||||
|
||||
type resourceMonitorOwner struct {
|
||||
PID int `json:"pid"`
|
||||
Start int64 `json:"starttime"`
|
||||
}
|
||||
type resourceMonitorStore struct {
|
||||
dir *os.File
|
||||
path string
|
||||
|
||||
Reference in new issue
Block a user