Add checks-only scheduling, read-only status, private authenticated persistence and guarded PID1 lifecycle. Keep strategy selection/recovery and remote fault acceptance out of scope. Co-authored-by: GPT-6 <gpt-6@openai.com>
327 lines
10 KiB
Go
327 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func resourceCompletedFixture(t *testing.T) (resourceMonitorSource, []byte, resourceMonitorState) {
|
|
t.Helper()
|
|
source, key := resourceMonitorFixture(t)
|
|
state := resourceMonitorState{Version: 1, Updated: 5000, Rows: reconcileResourceRows(key, source, nil)}
|
|
for i := range state.Rows {
|
|
state.Rows[i].Health, state.Rows[i].Outcome, state.Rows[i].LastCompleted = "HEALTHY", "pass", 5000
|
|
state.Rows[i].NextDue = 9000
|
|
}
|
|
return source, key, state
|
|
}
|
|
|
|
func TestResourceMonitorCancelledRowsUnchangedAndJoined(t *testing.T) {
|
|
for _, mixed := range []bool{false, true} {
|
|
t.Run(map[bool]string{false: "curl-cancel", true: "cancel-and-guard"}[mixed], func(t *testing.T) {
|
|
source, key, state := resourceCompletedFixture(t)
|
|
prior := append([]resourceMonitorRow(nil), state.Rows...)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
joined, writes := false, 0
|
|
check := resourceCheckDeps{resolve: func(context.Context, string, string) ([]net.IP, error) {
|
|
return []net.IP{net.ParseIP("93.184.216.34")}, nil
|
|
}, runner: func(ctx context.Context, _ []string, _ io.Reader, _ io.Writer, _ io.Writer) error {
|
|
cancel()
|
|
<-ctx.Done()
|
|
joined = true
|
|
return ctx.Err()
|
|
}}
|
|
d := resourceMonitorDeps{key: key, now: func() time.Time { return time.UnixMilli(10000) }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(s resourceMonitorState) error {
|
|
if !joined {
|
|
t.Error("persisted before joining curl")
|
|
}
|
|
writes++
|
|
state = s
|
|
return nil
|
|
}, source: func() (resourceMonitorSource, error) { return source, nil }, guard: func() error { return nil }, report: func(bool) {}, wait: func(context.Context, time.Duration) error { t.Error("wait after cancellation"); return nil }}
|
|
d.probe = func(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error) (string, error) {
|
|
outcome, err := probeResourceMonitor(ctx, service, spec, source, guard, check)
|
|
if mixed {
|
|
return outcome, errors.Join(ctx.Err(), errResourceConfig)
|
|
}
|
|
return outcome, err
|
|
}
|
|
err := runResourceMonitor(ctx, d)
|
|
if mixed {
|
|
if !errors.Is(err, errResourceConfig) || !errors.Is(err, context.Canceled) || !state.Blocked || writes != 1 {
|
|
t.Fatal("lost mixed shutdown guard error", state, writes, err)
|
|
}
|
|
} else if err != nil || writes != 0 {
|
|
t.Fatal("cancel published evidence", writes, err)
|
|
}
|
|
if !joined || !reflect.DeepEqual(prior, state.Rows) {
|
|
t.Fatal("cancel changed completed rows or failed to join", state.Rows)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResourceMonitorIdleNoPersistenceHeartbeat(t *testing.T) {
|
|
source, key, state := resourceCompletedFixture(t)
|
|
for i := range state.Rows {
|
|
state.Rows[i].NextDue = 305000
|
|
}
|
|
now := time.UnixMilli(10000)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
writes, waits := 0, 0
|
|
d := resourceMonitorDeps{key: key, now: func() time.Time { return now }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(s resourceMonitorState) error { writes++; state = s; return nil }, source: func() (resourceMonitorSource, error) { return source, nil }, guard: func() error { return nil }, report: func(bool) {}, probe: func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error) {
|
|
t.Error("probed resource before due")
|
|
return "pass", nil
|
|
}, wait: func(_ context.Context, d time.Duration) error {
|
|
now = now.Add(d)
|
|
waits++
|
|
if waits == 5 {
|
|
cancel()
|
|
}
|
|
return nil
|
|
}}
|
|
if err := runResourceMonitor(ctx, d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if writes != 0 || state.Updated != 5000 {
|
|
t.Fatal("idle disk heartbeat", writes, state.Updated)
|
|
}
|
|
if resourceSnapshotStale(state, 100000) || !resourceSnapshotStale(state, 370001) {
|
|
t.Fatal("staleness still depends on idle heartbeat")
|
|
}
|
|
}
|
|
|
|
func TestResourceMonitorBlockedTransitionsOnly(t *testing.T) {
|
|
source, key, state := resourceCompletedFixture(t)
|
|
for i := range state.Rows {
|
|
state.Rows[i].NextDue = 305000
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
tick, writes := 0, 0
|
|
d := resourceMonitorDeps{key: key, now: func() time.Time { return time.UnixMilli(10000 + int64(tick)*1000) }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(s resourceMonitorState) error { writes++; state = s; return nil }, source: func() (resourceMonitorSource, error) {
|
|
if tick < 3 {
|
|
return resourceMonitorSource{}, errResourceConfig
|
|
}
|
|
return source, nil
|
|
}, guard: func() error { return nil }, report: func(bool) {}, probe: func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error) {
|
|
t.Error("unexpected probe")
|
|
return "pass", nil
|
|
}, wait: func(context.Context, time.Duration) error {
|
|
tick++
|
|
if tick == 5 {
|
|
cancel()
|
|
}
|
|
return nil
|
|
}}
|
|
if err := runResourceMonitor(ctx, d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if writes != 2 || state.Blocked {
|
|
t.Fatal("blocked transition heartbeat or missing resume", writes, state)
|
|
}
|
|
}
|
|
|
|
func resourceBoundaryStore(t *testing.T, createState bool) (*resourceMonitorStore, resourceMonitorState) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
if err := os.Chmod(dir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err := openResourceMonitorStore(dir, true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { s.close() })
|
|
v, err := s.load()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if createState {
|
|
if err := s.save(v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return s, v
|
|
}
|
|
|
|
func replaceResourceEntry(t *testing.T, path string, data []byte) {
|
|
t.Helper()
|
|
if err := os.WriteFile(path+"-replacement", data, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Rename(path+"-replacement", path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestResourceMonitorHeldEntriesRejectReplacement(t *testing.T) {
|
|
for _, name := range []string{resourceStateName, resourceKeyName} {
|
|
for _, point := range []string{"before-load", "loaded", "staged", "publish"} {
|
|
t.Run(name+"/"+point, func(t *testing.T) {
|
|
s, v := resourceBoundaryStore(t, true)
|
|
path := filepath.Join(s.path, name)
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var replacement os.FileInfo
|
|
mutate := func() {
|
|
replaceResourceEntry(t, path, data)
|
|
replacement, err = os.Stat(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if point == "before-load" {
|
|
mutate()
|
|
} else {
|
|
s.fault = func(p, _ string) {
|
|
if p == point {
|
|
mutate()
|
|
}
|
|
}
|
|
}
|
|
if point == "before-load" || point == "loaded" {
|
|
if _, err := s.load(); err == nil {
|
|
t.Fatal("load adopted identical replaced inode")
|
|
}
|
|
}
|
|
if err := s.save(v); err == nil {
|
|
t.Fatal("save clobbered identical replacement")
|
|
}
|
|
current, err := os.Stat(path)
|
|
if err != nil || replacement == nil || !os.SameFile(current, replacement) {
|
|
t.Fatal("foreign inode changed", err)
|
|
}
|
|
got, err := os.ReadFile(path)
|
|
if err != nil || !bytes.Equal(got, data) {
|
|
t.Fatal("foreign bytes changed", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResourceMonitorMidStageCorruptionPreserved(t *testing.T) {
|
|
for _, name := range []string{resourceStateName, resourceKeyName, "stage", "stage-in-place"} {
|
|
t.Run(name, func(t *testing.T) {
|
|
s, v := resourceBoundaryStore(t, true)
|
|
bad := []byte("untrusted corrupt operator bytes")
|
|
path := ""
|
|
s.fault = func(point, stage string) {
|
|
if point != "staged" {
|
|
return
|
|
}
|
|
path = filepath.Join(s.path, name)
|
|
if name == "stage" || name == "stage-in-place" {
|
|
path = stage
|
|
}
|
|
if name == "stage-in-place" {
|
|
if err := os.WriteFile(path, bad, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
} else {
|
|
replaceResourceEntry(t, path, bad)
|
|
}
|
|
}
|
|
if err := s.save(v); err == nil {
|
|
t.Fatal("corruption overwritten")
|
|
}
|
|
got, err := os.ReadFile(path)
|
|
if err != nil || !bytes.Equal(got, bad) {
|
|
t.Fatal("unknown stage/file removed or clobbered", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResourceMonitorInitialPublicationExclusive(t *testing.T) {
|
|
s, v := resourceBoundaryStore(t, false)
|
|
path := filepath.Join(s.path, resourceStateName)
|
|
bad := []byte("new foreign entry created just before initial publication")
|
|
s.fault = func(point, _ string) {
|
|
if point == "publish" {
|
|
if err := os.WriteFile(path, bad, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
if err := s.save(v); err == nil {
|
|
t.Fatal("exclusive initial publish overwrote raced entry")
|
|
}
|
|
got, err := os.ReadFile(path)
|
|
if err != nil || !bytes.Equal(got, bad) {
|
|
t.Fatal("new entry clobbered", err)
|
|
}
|
|
if _, err := s.load(); err == nil {
|
|
t.Fatal("new entry silently adopted")
|
|
}
|
|
}
|
|
|
|
func TestResourceMonitorPublicationRetainsNewAnchor(t *testing.T) {
|
|
s, v := resourceBoundaryStore(t, true)
|
|
old := s.stateFile
|
|
v.Updated = 10000
|
|
if err := s.save(v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if old == s.stateFile {
|
|
t.Fatal("old inode retained instead of published inode")
|
|
}
|
|
if _, err := old.Stat(); err == nil {
|
|
t.Fatal("old anchor not closed after verified publication")
|
|
}
|
|
if s.entryMatches(resourceStateName, s.stateFile) != nil {
|
|
t.Fatal("published state not anchored")
|
|
}
|
|
if loaded, err := s.load(); err != nil || loaded.Updated != 10000 {
|
|
t.Fatal(loaded, err)
|
|
}
|
|
}
|
|
|
|
func TestResourceMonitorOriginalIdentityRepair(t *testing.T) {
|
|
s, v := resourceBoundaryStore(t, true)
|
|
path := filepath.Join(s.path, resourceStateName)
|
|
valid, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
original, err := os.Stat(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(`{"version":99}`), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.load(); err == nil {
|
|
t.Fatal("corrupt original trusted")
|
|
}
|
|
if err := s.save(v); err == nil {
|
|
t.Fatal("corrupt original overwritten")
|
|
}
|
|
if err := os.WriteFile(path, valid, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.load(); err != nil {
|
|
t.Fatal("valid original-identity repair blocked", err)
|
|
}
|
|
current, err := os.Stat(path)
|
|
if err != nil || !os.SameFile(current, original) {
|
|
t.Fatal("repair changed inode", err)
|
|
}
|
|
if err := s.save(v); err != nil {
|
|
t.Fatal("repaired original failed publish", err)
|
|
}
|
|
}
|