feat: add guarded periodic resource monitoring
Add checks-only scheduling, read-only status, private authenticated persistence and guarded PID1 lifecycle. Keep strategy selection/recovery and remote fault acceptance out of scope. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
cfce8170d7
commit
4efc066625
16 files changed
+2307
-103
No files matched your search
@@ -0,0 +1,402 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
const resourceStateName = ".resource-monitor-state.json"
|
||||
const resourceKeyName = ".resource-monitor-key"
|
||||
|
||||
var errResourceState = errors.New("resource state blocked")
|
||||
|
||||
type resourceMonitorRow struct {
|
||||
Service string `json:"service"`
|
||||
Resource string `json:"resource"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Health string `json:"health"`
|
||||
Outcome string `json:"outcome"`
|
||||
Failures uint64 `json:"failures"`
|
||||
LastCompleted int64 `json:"last_completed_ms"`
|
||||
NextDue int64 `json:"next_due_ms"`
|
||||
}
|
||||
type resourceMonitorState struct {
|
||||
Version int `json:"version"`
|
||||
Updated int64 `json:"updated_ms"`
|
||||
Blocked bool `json:"blocked"`
|
||||
Rows []resourceMonitorRow `json:"rows"`
|
||||
MAC string `json:"mac"`
|
||||
}
|
||||
type resourceMonitorStore struct {
|
||||
dir *os.File
|
||||
path string
|
||||
key []byte
|
||||
previous []byte
|
||||
existed bool
|
||||
keyFile, stateFile *os.File
|
||||
// Deterministic filesystem interleavings in tests; nil in production.
|
||||
fault func(point, stage string)
|
||||
}
|
||||
|
||||
func resourceMAC(key, data []byte) string {
|
||||
m := hmac.New(sha256.New, key)
|
||||
m.Write(data)
|
||||
return hex.EncodeToString(m.Sum(nil))
|
||||
}
|
||||
func resourceStateMAC(key []byte, s resourceMonitorState) string {
|
||||
s.MAC = ""
|
||||
b, _ := json.Marshal(s)
|
||||
return resourceMAC(key, b)
|
||||
}
|
||||
func (s *resourceMonitorStore) guard() error {
|
||||
if managedDirectory(s.path) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
current, err := os.Stat(s.path)
|
||||
held, heldErr := s.dir.Stat()
|
||||
if err != nil || heldErr != nil || !os.SameFile(current, held) {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *resourceMonitorStore) entryPath(name string) string {
|
||||
return filepath.Join("/proc/self/fd", strconv.FormatUint(uint64(s.dir.Fd()), 10), name)
|
||||
}
|
||||
|
||||
func (s *resourceMonitorStore) close() error {
|
||||
var errs []error
|
||||
for _, f := range []*os.File{s.stateFile, s.keyFile, s.dir} {
|
||||
if f != nil {
|
||||
errs = append(errs, f.Close())
|
||||
}
|
||||
}
|
||||
s.stateFile, s.keyFile, s.dir = nil, nil, nil
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func resourcePrivateEntry(f *os.File) error {
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
st, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || st.Uid != uint32(os.Geteuid()) || st.Nlink != 1 || info.Size() > 65536 {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *resourceMonitorStore) entryMatches(name string, held *os.File) error {
|
||||
current, err := os.Lstat(s.entryPath(name))
|
||||
anchored, heldErr := held.Stat()
|
||||
if err != nil || heldErr != nil || !os.SameFile(current, anchored) || resourcePrivateEntry(held) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *resourceMonitorStore) openEntry(name string) (*os.File, []byte, error) {
|
||||
if s.guard() != nil {
|
||||
return nil, nil, errResourceState
|
||||
}
|
||||
fd, err := syscall.Openat(int(s.dir.Fd()), name, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK|syscall.O_CLOEXEC, 0)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
f := os.NewFile(uintptr(fd), name)
|
||||
if resourcePrivateEntry(f) != nil {
|
||||
f.Close()
|
||||
return nil, nil, errResourceState
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(f, 65537))
|
||||
if err != nil || len(b) > 65536 || s.entryMatches(name, f) != nil {
|
||||
f.Close()
|
||||
return nil, nil, errResourceState
|
||||
}
|
||||
return f, b, nil
|
||||
}
|
||||
|
||||
func (s *resourceMonitorStore) trustedRead(name string, held *os.File) ([]byte, error) {
|
||||
f, b, err := s.openEntry(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
if held == nil || s.entryMatches(name, held) != nil {
|
||||
return nil, errResourceState
|
||||
}
|
||||
readInfo, err := f.Stat()
|
||||
heldInfo, heldErr := held.Stat()
|
||||
if err != nil || heldErr != nil || !os.SameFile(readInfo, heldInfo) {
|
||||
return nil, errResourceState
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Both content and held inode identities must still match the trusted snapshot.
|
||||
// In particular, an identical signed replacement is not silently adopted.
|
||||
func (s *resourceMonitorStore) snapshotGuard() error {
|
||||
key, err := s.trustedRead(resourceKeyName, s.keyFile)
|
||||
if err != nil || !hmac.Equal(key, s.key) {
|
||||
return errResourceState
|
||||
}
|
||||
if s.existed {
|
||||
b, err := s.trustedRead(resourceStateName, s.stateFile)
|
||||
if err != nil || !bytes.Equal(b, s.previous) {
|
||||
return errResourceState
|
||||
}
|
||||
if _, err := s.decode(b); err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
} else {
|
||||
f, _, err := s.openEntry(resourceStateName)
|
||||
if f != nil {
|
||||
f.Close()
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return errResourceState
|
||||
}
|
||||
}
|
||||
return s.guard()
|
||||
}
|
||||
func openResourceMonitorStore(path string, create bool) (*resourceMonitorStore, error) {
|
||||
if managedDirectory(path) != nil {
|
||||
return nil, errResourceState
|
||||
}
|
||||
fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_DIRECTORY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0)
|
||||
if err != nil {
|
||||
return nil, errResourceState
|
||||
}
|
||||
s := &resourceMonitorStore{path: path, dir: os.NewFile(uintptr(fd), "resource state directory")}
|
||||
fail := func() (*resourceMonitorStore, error) { s.close(); return nil, errResourceState }
|
||||
stateFile, state, stateErr := s.openEntry(resourceStateName)
|
||||
s.stateFile = stateFile
|
||||
if stateErr != nil && !errors.Is(stateErr, os.ErrNotExist) {
|
||||
return fail()
|
||||
}
|
||||
keyFile, key, err := s.openEntry(resourceKeyName)
|
||||
s.keyFile = keyFile
|
||||
if errors.Is(err, os.ErrNotExist) && errors.Is(stateErr, os.ErrNotExist) && create {
|
||||
key = make([]byte, 32)
|
||||
if _, err = rand.Read(key); err != nil {
|
||||
return fail()
|
||||
}
|
||||
keyFD, err := syscall.Openat(fd, resourceKeyName, syscall.O_RDWR|syscall.O_CREAT|syscall.O_EXCL|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0600)
|
||||
if err != nil {
|
||||
return fail()
|
||||
}
|
||||
f := os.NewFile(uintptr(keyFD), "resource key")
|
||||
s.keyFile = f
|
||||
_, writeErr := f.Write(key)
|
||||
syncErr := f.Sync()
|
||||
if errors.Join(writeErr, syncErr, s.dir.Sync()) != nil {
|
||||
return fail()
|
||||
}
|
||||
} else if err != nil {
|
||||
return fail()
|
||||
}
|
||||
if len(key) != 32 {
|
||||
return fail()
|
||||
}
|
||||
s.key = append([]byte(nil), key...)
|
||||
s.previous, s.existed = state, stateErr == nil
|
||||
if s.existed {
|
||||
if _, err := s.decode(state); err != nil {
|
||||
return fail()
|
||||
}
|
||||
}
|
||||
if s.snapshotGuard() != nil {
|
||||
return fail()
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
func (s *resourceMonitorStore) decode(b []byte) (resourceMonitorState, error) {
|
||||
var v resourceMonitorState
|
||||
d := json.NewDecoder(bytes.NewReader(b))
|
||||
if uniqueResourceJSON(d, 0) != nil {
|
||||
return v, errResourceState
|
||||
}
|
||||
if _, err := d.Token(); err != io.EOF {
|
||||
return v, errResourceState
|
||||
}
|
||||
d = json.NewDecoder(bytes.NewReader(b))
|
||||
d.DisallowUnknownFields()
|
||||
if d.Decode(&v) != nil || v.Version != 1 || v.Updated < 0 || len(v.Rows) > 64 || !hmac.Equal([]byte(v.MAC), []byte(resourceStateMAC(s.key, v))) {
|
||||
return v, errResourceState
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
services := map[string]bool{}
|
||||
for _, r := range v.Rows {
|
||||
id := r.Service + "/" + r.Resource
|
||||
fp, err := hex.DecodeString(r.Fingerprint)
|
||||
if !resourceID.MatchString(r.Service) || !resourceID.MatchString(r.Resource) || seen[id] || err != nil || len(fp) != 32 || r.LastCompleted < 0 || r.NextDue < 0 {
|
||||
return v, errResourceState
|
||||
}
|
||||
seen[id] = true
|
||||
services[r.Service] = true
|
||||
if len(services) > 16 || r.LastCompleted > v.Updated || r.LastCompleted > 253402300799999 || (r.Health == "UNKNOWN" && r.NextDue != 0) || (r.Health != "UNKNOWN" && (r.NextDue < r.LastCompleted || r.NextDue-r.LastCompleted > 86400000)) {
|
||||
return v, errResourceState
|
||||
}
|
||||
if r.Health != "UNKNOWN" && r.Health != "HEALTHY" && r.Health != "DEGRADED" {
|
||||
return v, errResourceState
|
||||
}
|
||||
switch r.Outcome {
|
||||
case "unknown", "pass", "dns_failed", "destination_denied", "transfer_failed", "expectation_failed", "timeout", "cancelled":
|
||||
default:
|
||||
return v, errResourceState
|
||||
}
|
||||
if (r.Health == "UNKNOWN" && (r.Outcome != "unknown" || r.LastCompleted != 0 || r.Failures != 0)) || (r.Health == "HEALTHY" && (r.Outcome != "pass" || r.Failures != 0)) || (r.Health == "DEGRADED" && (r.Outcome == "unknown" || r.Outcome == "pass" || r.Failures == 0)) {
|
||||
return v, errResourceState
|
||||
}
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
func (s *resourceMonitorStore) load() (resourceMonitorState, error) {
|
||||
if s.guard() != nil {
|
||||
return resourceMonitorState{}, errResourceState
|
||||
}
|
||||
key, err := s.trustedRead(resourceKeyName, s.keyFile)
|
||||
if err != nil || !hmac.Equal(key, s.key) {
|
||||
return resourceMonitorState{}, errResourceState
|
||||
}
|
||||
b, err := s.trustedRead(resourceStateName, s.stateFile)
|
||||
if errors.Is(err, os.ErrNotExist) && !s.existed {
|
||||
return resourceMonitorState{Version: 1, Rows: []resourceMonitorRow{}}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return resourceMonitorState{}, errResourceState
|
||||
}
|
||||
v, err := s.decode(b)
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
if s.fault != nil {
|
||||
s.fault("loaded", "")
|
||||
}
|
||||
// Authenticated operator repair on the original held inode can be reloaded;
|
||||
// replaced files remain preserved/BLOCKED until deliberately repaired.
|
||||
key, err = s.trustedRead(resourceKeyName, s.keyFile)
|
||||
if err != nil || !hmac.Equal(key, s.key) || s.entryMatches(resourceStateName, s.stateFile) != nil {
|
||||
return v, errResourceState
|
||||
}
|
||||
current, err := s.trustedRead(resourceStateName, s.stateFile)
|
||||
if err != nil || !bytes.Equal(current, b) {
|
||||
return v, errResourceState
|
||||
}
|
||||
s.previous = b
|
||||
return v, nil
|
||||
}
|
||||
func (s *resourceMonitorStore) save(v resourceMonitorState) error {
|
||||
if s.snapshotGuard() != nil {
|
||||
return errResourceState
|
||||
}
|
||||
v.MAC = resourceStateMAC(s.key, v)
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
if _, err := s.decode(b); err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
// Keep the staged inode open from creation through publication; reopening a
|
||||
// path after rename must never adopt a foreign replacement as our own inode.
|
||||
f, err := os.CreateTemp(s.entryPath(""), ".resource-monitor-stage-")
|
||||
if err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
stage := f.Name()
|
||||
stageGuard := func() bool {
|
||||
if s.entryMatches(filepath.Base(stage), f) != nil {
|
||||
return false
|
||||
}
|
||||
contents, err := io.ReadAll(io.NewSectionReader(f, 0, 65537))
|
||||
// Exact bytes bind the entire validated update, including its HMAC;
|
||||
// truncation, append and in-place edits must fail before publication.
|
||||
return err == nil && bytes.Equal(contents, b) && s.entryMatches(filepath.Base(stage), f) == nil
|
||||
}
|
||||
adopted := false
|
||||
defer func() {
|
||||
// Delete only the known staged inode, never an operator replacement.
|
||||
info, err := os.Lstat(stage)
|
||||
held, heldErr := f.Stat()
|
||||
if err == nil && heldErr == nil && os.SameFile(info, held) && resourcePrivateEntry(f) == nil {
|
||||
contents, readErr := io.ReadAll(io.NewSectionReader(f, 0, 65537))
|
||||
if readErr == nil && bytes.Equal(contents, b) {
|
||||
os.Remove(stage)
|
||||
}
|
||||
}
|
||||
if !adopted {
|
||||
f.Close()
|
||||
}
|
||||
}()
|
||||
if _, err := f.Write(b); err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
if f.Sync() != nil {
|
||||
return errResourceState
|
||||
}
|
||||
if s.fault != nil {
|
||||
s.fault("staged", stage)
|
||||
}
|
||||
if s.snapshotGuard() != nil || !stageGuard() {
|
||||
return errResourceState
|
||||
}
|
||||
if s.fault != nil {
|
||||
s.fault("publish", stage)
|
||||
}
|
||||
// Detect benign/operator replacements immediately before publication. An
|
||||
// actively hostile same-UID actor racing between this final check and rename
|
||||
// is outside the guarantee: POSIX rename has no inode compare-and-swap.
|
||||
if s.existed {
|
||||
if s.snapshotGuard() != nil || !stageGuard() {
|
||||
return errResourceState
|
||||
}
|
||||
if os.Rename(stage, s.entryPath(resourceStateName)) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
} else {
|
||||
// Exclusive initial publication cannot clobber a newly appeared entry.
|
||||
key, err := s.trustedRead(resourceKeyName, s.keyFile)
|
||||
if err != nil || !hmac.Equal(key, s.key) || s.guard() != nil || !stageGuard() {
|
||||
return errResourceState
|
||||
}
|
||||
if os.Link(stage, s.entryPath(resourceStateName)) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
stageInfo, stageErr := os.Lstat(stage)
|
||||
heldInfo, heldErr := f.Stat()
|
||||
if stageErr != nil || heldErr != nil || !os.SameFile(stageInfo, heldInfo) {
|
||||
return errResourceState
|
||||
}
|
||||
if os.Remove(stage) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
}
|
||||
if s.entryMatches(resourceStateName, f) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
published, err := s.trustedRead(resourceStateName, f)
|
||||
if err != nil || !bytes.Equal(published, b) {
|
||||
return errResourceState
|
||||
}
|
||||
old := s.stateFile
|
||||
s.stateFile, s.previous, s.existed, adopted = f, b, true, true
|
||||
var closeErr error
|
||||
if old != nil {
|
||||
closeErr = old.Close()
|
||||
}
|
||||
if errors.Join(closeErr, s.dir.Sync()) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user