fix: validate live resource monitor ownership in status
Bind authenticated snapshots to controller PID/starttime and validate process identity and held inode anchors before reporting current check health. Preserve identical signed foreign replacements and fail closed when live ownership is unavailable. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
4efc066625
commit
5c85f3f0b9
5 files changed
+252
-7
No files matched your search
@@ -62,6 +62,13 @@ unavailable ownership. Resource failures continue to other resources; ownership
|
|||||||
failure stops all probes. These checks have unit/loopback-fixture coverage only,
|
failure stops all probes. These checks have unit/loopback-fixture coverage only,
|
||||||
not new remote acceptance or generic automated recovery/controller support.
|
not new remote acceptance or generic automated recovery/controller support.
|
||||||
|
|
||||||
|
Generic `managed resource-status` validates the authenticated snapshot against
|
||||||
|
the live controller's PID/starttime, executable, command, UID, namespaces and
|
||||||
|
held directory/key/state/singleton inodes. Cached check results alone cannot
|
||||||
|
report current health: missing ownership or an identical signed replacement
|
||||||
|
returns `BLOCKED` with `stale: true` without modifying the foreign file.
|
||||||
|
`CHECKS_ONLY`/`HEALTHY` still do not mean candidate or production readiness.
|
||||||
|
|
||||||
## Production preparation (isolated clean-run acceptance passed)
|
## Production preparation (isolated clean-run acceptance passed)
|
||||||
|
|
||||||
The standalone deployment candidate is a **new isolated target**, never an
|
The standalone deployment candidate is a **new isolated target**, never an
|
||||||
|
|||||||
@@ -316,6 +316,17 @@ func resourceMonitorCLIWithCheck(ctx context.Context, path string, stdout, stder
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
start, err := readMonitorProcessStarttime(os.Getpid())
|
||||||
|
state, loadErr := store.load()
|
||||||
|
if err != nil || loadErr != nil {
|
||||||
|
fmt.Fprintln(stderr, "BLOCKED: generic monitor ownership unavailable")
|
||||||
|
return 78
|
||||||
|
}
|
||||||
|
state.Controller = &resourceMonitorOwner{PID: os.Getpid(), Start: start}
|
||||||
|
if store.save(state) != nil {
|
||||||
|
fmt.Fprintln(stderr, "BLOCKED: generic monitor ownership publication failed")
|
||||||
|
return 78
|
||||||
|
}
|
||||||
}
|
}
|
||||||
err := runResourceMonitor(ctx, resourceMonitorDeps{
|
err := runResourceMonitor(ctx, resourceMonitorDeps{
|
||||||
load: store.load, save: store.save, key: store.key,
|
load: store.load, save: store.save, key: store.key,
|
||||||
@@ -363,6 +374,10 @@ func resourceSingletonGuard(lock *managedMonitorSingleton, dir string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func resourceStatusCLI(stdout io.Writer, target managedDeps) int {
|
func resourceStatusCLI(stdout io.Writer, target managedDeps) int {
|
||||||
|
return resourceStatusCLIAt(stdout, target, "/proc")
|
||||||
|
}
|
||||||
|
|
||||||
|
func resourceStatusCLIAt(stdout io.Writer, target managedDeps, proc string) int {
|
||||||
blocked := func() int { fmt.Fprintln(stdout, `{"health":"BLOCKED","stale":true}`); return 1 }
|
blocked := func() int { fmt.Fprintln(stdout, `{"health":"BLOCKED","stale":true}`); return 1 }
|
||||||
store, err := openResourceMonitorStore(target.directory, false)
|
store, err := openResourceMonitorStore(target.directory, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -370,10 +385,10 @@ func resourceStatusCLI(stdout io.Writer, target managedDeps) int {
|
|||||||
}
|
}
|
||||||
defer store.close()
|
defer store.close()
|
||||||
state, err := store.load()
|
state, err := store.load()
|
||||||
if err != nil {
|
path, _ := target.lookup("ZAPRET_AUTOTUNE_RESOURCE_CONFIG")
|
||||||
|
if err != nil || resourceMonitorLiveOwner(store, state, path, proc) != nil {
|
||||||
return blocked()
|
return blocked()
|
||||||
}
|
}
|
||||||
path, _ := target.lookup("ZAPRET_AUTOTUNE_RESOURCE_CONFIG")
|
|
||||||
source, err := readResourceMonitorSource(path, target.lookup, store.key)
|
source, err := readResourceMonitorSource(path, target.lookup, store.key)
|
||||||
check := productionResourceCheckDeps()
|
check := productionResourceCheckDeps()
|
||||||
if err != nil || check.guard() != nil || resourceManagedIdentity(target.directory) != nil {
|
if err != nil || check.guard() != nil || resourceManagedIdentity(target.directory) != nil {
|
||||||
@@ -390,6 +405,11 @@ func resourceStatusCLI(stdout io.Writer, target managedDeps) int {
|
|||||||
if state.Blocked {
|
if state.Blocked {
|
||||||
health = "BLOCKED"
|
health = "BLOCKED"
|
||||||
}
|
}
|
||||||
|
// Source/runtime validation may take time; don't emit a cache whose live
|
||||||
|
// ownership disappeared or whose inode changed during those checks.
|
||||||
|
if resourceMonitorLiveOwner(store, state, path, proc) != nil {
|
||||||
|
return blocked()
|
||||||
|
}
|
||||||
if json.NewEncoder(stdout).Encode(struct {
|
if json.NewEncoder(stdout).Encode(struct {
|
||||||
Health string `json:"health"`
|
Health string `json:"health"`
|
||||||
Stale bool `json:"stale"`
|
Stale bool `json:"stale"`
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A signed cache is not a live ownership claim. Inspect only procfs metadata,
|
||||||
|
// never the controller's private FD contents, and fail closed on uncertainty.
|
||||||
|
func resourceMonitorLiveOwner(store *resourceMonitorStore, state resourceMonitorState, config, proc string) error {
|
||||||
|
owner := state.Controller
|
||||||
|
if owner == nil || owner.PID <= 1 || owner.Start <= 0 || config == "" {
|
||||||
|
return errResourceState
|
||||||
|
}
|
||||||
|
base := filepath.Join(proc, strconv.Itoa(owner.PID))
|
||||||
|
identity := func() bool {
|
||||||
|
stat, err := os.ReadFile(filepath.Join(base, "stat"))
|
||||||
|
start, parseErr := parseMonitorProcessStat(owner.PID, string(stat))
|
||||||
|
if err != nil || parseErr != nil || start != owner.Start {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
exe, err := os.Readlink(filepath.Join(proc, "self/exe"))
|
||||||
|
self, selfErr := os.Stat(filepath.Join(proc, "self/exe"))
|
||||||
|
other, otherErr := os.Stat(filepath.Join(base, "exe"))
|
||||||
|
if err != nil || selfErr != nil || otherErr != nil || !os.SameFile(self, other) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return resourceProcessOwned(proc, owner.PID, exe, []string{exe, "managed", "monitor", "--resource-config", config}) ||
|
||||||
|
resourceProcessOwned(proc, owner.PID, exe, []string{exe, "managed", "monitor"})
|
||||||
|
}
|
||||||
|
if !identity() || store.snapshotGuard() != nil {
|
||||||
|
return errResourceState
|
||||||
|
}
|
||||||
|
// Include the singleton and directory so another target's controller cannot
|
||||||
|
// lend apparent liveness to a copied authenticated snapshot.
|
||||||
|
anchors := []string{store.path, store.entryPath(resourceKeyName), store.entryPath(resourceStateName), store.entryPath(managedMonitorLockName)}
|
||||||
|
fds, err := os.ReadDir(filepath.Join(base, "fd"))
|
||||||
|
if err != nil {
|
||||||
|
return errResourceState
|
||||||
|
}
|
||||||
|
for _, anchor := range anchors {
|
||||||
|
current, err := os.Lstat(anchor)
|
||||||
|
if err != nil {
|
||||||
|
return errResourceState
|
||||||
|
}
|
||||||
|
matched := false
|
||||||
|
for _, fd := range fds {
|
||||||
|
held, err := os.Stat(filepath.Join(base, "fd", fd.Name()))
|
||||||
|
if err == nil && os.SameFile(current, held) {
|
||||||
|
matched = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !matched {
|
||||||
|
return errResourceState
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !identity() || store.snapshotGuard() != nil || managedPrivateFile(store.entryPath(managedMonitorLockName)) != nil {
|
||||||
|
return errResourceState
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResourceStatusLiveOwnership(t *testing.T) {
|
||||||
|
for _, change := range []string{"state", "key", "directory", "lock", "dead", "pid-reuse", "exe", "argv", "uid", "parent", "namespace", "missing-owner"} {
|
||||||
|
t.Run(change, func(t *testing.T) {
|
||||||
|
dir := filepath.Join(t.TempDir(), "state")
|
||||||
|
if err := os.Mkdir(dir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
store, err := openResourceMonitorStore(dir, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer store.close()
|
||||||
|
write := func(path, contents string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
link := func(path, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.Symlink(target, path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lockPath := filepath.Join(dir, managedMonitorLockName)
|
||||||
|
write(lockPath, "")
|
||||||
|
lock, err := os.Open(lockPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer lock.Close()
|
||||||
|
source, _ := resourceMonitorFixture(t)
|
||||||
|
state := resourceMonitorState{Version: 1, Updated: time.Now().UnixMilli(), Controller: &resourceMonitorOwner{PID: 42, Start: 123}, Rows: reconcileResourceRows(store.key, source, nil)}
|
||||||
|
for i := range state.Rows {
|
||||||
|
state.Rows[i].Health, state.Rows[i].Outcome = "HEALTHY", "pass"
|
||||||
|
state.Rows[i].LastCompleted, state.Rows[i].NextDue = state.Updated, state.Updated+60000
|
||||||
|
}
|
||||||
|
if err := store.save(state); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err = store.load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
proc := t.TempDir()
|
||||||
|
for _, sub := range []string{"self/ns", "42/ns", "42/fd"} {
|
||||||
|
if err := os.MkdirAll(filepath.Join(proc, sub), 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
link(filepath.Join(proc, "self/exe"), exe)
|
||||||
|
link(filepath.Join(proc, "42/exe"), exe)
|
||||||
|
for _, ns := range []string{"pid", "net"} {
|
||||||
|
link(filepath.Join(proc, "self/ns", ns), ns+"[1]")
|
||||||
|
link(filepath.Join(proc, "42/ns", ns), ns+"[1]")
|
||||||
|
}
|
||||||
|
fields := append([]string{"S"}, strings.Fields(strings.Repeat("0 ", 18))...)
|
||||||
|
fields = append(fields, "123")
|
||||||
|
stat := "42 (monitor) " + strings.Join(fields, " ")
|
||||||
|
write(filepath.Join(proc, "42/stat"), stat)
|
||||||
|
uid := strconv.Itoa(os.Geteuid())
|
||||||
|
write(filepath.Join(proc, "42/status"), "PPid:\t1\nUid:\t"+strings.Join([]string{uid, uid, uid, uid}, "\t")+"\n")
|
||||||
|
config := "/private/checks.json"
|
||||||
|
write(filepath.Join(proc, "42/cmdline"), strings.Join([]string{exe, "managed", "monitor", "--resource-config", config}, "\x00")+"\x00")
|
||||||
|
for i, f := range []*os.File{store.dir, store.keyFile, store.stateFile, lock} {
|
||||||
|
link(filepath.Join(proc, "42/fd", strconv.Itoa(i)), "/proc/self/fd/"+strconv.FormatUint(uint64(f.Fd()), 10))
|
||||||
|
}
|
||||||
|
if err := resourceMonitorLiveOwner(store, state, config, proc); err != nil {
|
||||||
|
t.Fatal("valid live owner refused", err)
|
||||||
|
}
|
||||||
|
if resourceSnapshotStale(state, time.Now().UnixMilli()) {
|
||||||
|
t.Fatal("fixture already stale")
|
||||||
|
}
|
||||||
|
var preservedPath string
|
||||||
|
var preserved []byte
|
||||||
|
switch change {
|
||||||
|
case "state", "key", "lock":
|
||||||
|
name := map[string]string{"state": resourceStateName, "key": resourceKeyName, "lock": managedMonitorLockName}[change]
|
||||||
|
preservedPath = filepath.Join(dir, name)
|
||||||
|
preserved, err = os.ReadFile(preservedPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
write(preservedPath+".replacement", string(preserved))
|
||||||
|
if err := os.Rename(preservedPath+".replacement", preservedPath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case "directory":
|
||||||
|
if err := os.Rename(dir, dir+".old"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Mkdir(dir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case "dead":
|
||||||
|
os.Remove(filepath.Join(proc, "42/stat"))
|
||||||
|
case "pid-reuse":
|
||||||
|
write(filepath.Join(proc, "42/stat"), strings.TrimSuffix(stat, "123")+"124")
|
||||||
|
case "exe":
|
||||||
|
os.Remove(filepath.Join(proc, "42/exe"))
|
||||||
|
link(filepath.Join(proc, "42/exe"), "/bin/sh")
|
||||||
|
case "argv":
|
||||||
|
write(filepath.Join(proc, "42/cmdline"), exe+"\x00managed\x00resource-status\x00")
|
||||||
|
case "uid":
|
||||||
|
write(filepath.Join(proc, "42/status"), "PPid:\t1\nUid:\t9999\t9999\t9999\t9999\n")
|
||||||
|
case "parent":
|
||||||
|
write(filepath.Join(proc, "42/status"), "PPid:\t2\nUid:\t"+strings.Join([]string{uid, uid, uid, uid}, "\t")+"\n")
|
||||||
|
case "namespace":
|
||||||
|
os.Remove(filepath.Join(proc, "42/ns/net"))
|
||||||
|
link(filepath.Join(proc, "42/ns/net"), "net[2]")
|
||||||
|
case "missing-owner":
|
||||||
|
state.Controller = nil
|
||||||
|
if err := store.save(state); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if resourceMonitorLiveOwner(store, state, config, proc) == nil {
|
||||||
|
t.Fatal("invalid live ownership accepted")
|
||||||
|
}
|
||||||
|
target := managedDeps{directory: dir, lookup: func(name string) (string, bool) { return config, name == "ZAPRET_AUTOTUNE_RESOURCE_CONFIG" }}
|
||||||
|
var out bytes.Buffer
|
||||||
|
if resourceStatusCLIAt(&out, target, proc) != 1 || out.String() != "{\"health\":\"BLOCKED\",\"stale\":true}\n" {
|
||||||
|
t.Fatal("cached status hid live failure", out.String())
|
||||||
|
}
|
||||||
|
if preservedPath != "" {
|
||||||
|
got, err := os.ReadFile(preservedPath)
|
||||||
|
if err != nil || !bytes.Equal(got, preserved) {
|
||||||
|
t.Fatal("foreign entry modified", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -31,11 +31,17 @@ type resourceMonitorRow struct {
|
|||||||
NextDue int64 `json:"next_due_ms"`
|
NextDue int64 `json:"next_due_ms"`
|
||||||
}
|
}
|
||||||
type resourceMonitorState struct {
|
type resourceMonitorState struct {
|
||||||
Version int `json:"version"`
|
Version int `json:"version"`
|
||||||
Updated int64 `json:"updated_ms"`
|
Updated int64 `json:"updated_ms"`
|
||||||
Blocked bool `json:"blocked"`
|
Blocked bool `json:"blocked"`
|
||||||
Rows []resourceMonitorRow `json:"rows"`
|
Rows []resourceMonitorRow `json:"rows"`
|
||||||
MAC string `json:"mac"`
|
Controller *resourceMonitorOwner `json:"controller,omitempty"`
|
||||||
|
MAC string `json:"mac"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type resourceMonitorOwner struct {
|
||||||
|
PID int `json:"pid"`
|
||||||
|
Start int64 `json:"starttime"`
|
||||||
}
|
}
|
||||||
type resourceMonitorStore struct {
|
type resourceMonitorStore struct {
|
||||||
dir *os.File
|
dir *os.File
|
||||||
|
|||||||
Reference in new issue
Block a user