feat: add guarded periodic resource monitoring
Add checks-only scheduling, read-only status, private authenticated persistence and guarded PID1 lifecycle. Keep strategy selection/recovery and remote fault acceptance out of scope. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
cfce8170d7
commit
4efc066625
16 files changed
+2307
-103
No files matched your search
@@ -21,6 +21,10 @@ Commands:
|
||||
health youtube One-shot Player API health through configured loopback SOCKS
|
||||
managed init|reconcile|status|apply|monitor
|
||||
Guarded activation/monitoring in the dedicated managed-test target only
|
||||
managed monitor --resource-config FILE
|
||||
Periodic configured checks only (no recovery/apply)
|
||||
managed resource-status
|
||||
Read-only configured-check snapshot (explicit stale/blocked)
|
||||
check youtube Refuse checks until probe URLs/video IDs are configured and agreed
|
||||
check --config FILE
|
||||
Check configured HTTPS resources (checks only; no strategy selection)
|
||||
@@ -48,7 +52,18 @@ func runWithPlayer(args []string, stdout, stderr io.Writer, player playerRunner)
|
||||
return dnsForwarderCLI(args[1:], stdout, stderr)
|
||||
}
|
||||
if len(args) == 1 && args[0] == "target-config-check" {
|
||||
if _, err := parseManagedHealthConfig(os.LookupEnv); err != nil {
|
||||
path, _ := os.LookupEnv("ZAPRET_AUTOTUNE_RESOURCE_CONFIG")
|
||||
if path != "" {
|
||||
port, err := managedTargetSettings(os.LookupEnv)
|
||||
if err != nil || port != 1080 {
|
||||
fmt.Fprintln(stderr, "configerror: invalid generic target settings")
|
||||
return 64
|
||||
}
|
||||
if _, _, err := resourceStaticConfig(os.LookupEnv); err != nil {
|
||||
fmt.Fprintln(stderr, "configerror: invalid generic DNS/proxy configuration")
|
||||
return 64
|
||||
}
|
||||
} else if _, err := parseManagedHealthConfig(os.LookupEnv); err != nil {
|
||||
fmt.Fprintln(stderr, "configerror: invalid target health/service configuration")
|
||||
return 64
|
||||
}
|
||||
|
||||
@@ -379,25 +379,9 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB,
|
||||
}
|
||||
|
||||
func managedRuntimeGuard(d managedDeps) (int, error) {
|
||||
role, _ := d.lookup("ZAPRET_AUTOTUNE_TARGET_ROLE")
|
||||
if role != "managed-test" {
|
||||
return 2, fmt.Errorf("ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test")
|
||||
}
|
||||
for key, expected := range map[string]string{
|
||||
"ZAPRET_AUTOTUNE_TARGET_ID": managedTargetID,
|
||||
"ZAPRET_AUTOTUNE_TARGET_DIR": managedTargetDirectory,
|
||||
"ZAPRET_CONFIG": managedTargetConfig,
|
||||
} {
|
||||
if value, _ := d.lookup(key); value != expected {
|
||||
return 2, fmt.Errorf("%s does not match the fresh managed target contract", key)
|
||||
}
|
||||
}
|
||||
portText, _ := d.lookup("SOCKS_PORT")
|
||||
port, portErr := strconv.Atoi(portText)
|
||||
proxy, _ := d.lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
|
||||
proxy, proxyErr := managedProbeProxy(proxy)
|
||||
if portErr != nil || port < 1 || port > 65535 || strconv.Itoa(port) != portText || proxyErr != nil || proxy != "socks5h://127.0.0.1:"+portText {
|
||||
return 2, fmt.Errorf("managed SOCKS proxy must match the target SOCKS_PORT listener")
|
||||
port, err := managedTargetSettings(d.lookup)
|
||||
if err != nil {
|
||||
return 2, err
|
||||
}
|
||||
if d.verifyRuntime == nil || d.verifySOCKS == nil {
|
||||
return 1, fmt.Errorf("managed runtime validators unavailable")
|
||||
@@ -411,7 +395,34 @@ func managedRuntimeGuard(d managedDeps) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func managedTargetSettings(lookup func(string) (string, bool)) (int, error) {
|
||||
role, _ := lookup("ZAPRET_AUTOTUNE_TARGET_ROLE")
|
||||
if role != "managed-test" {
|
||||
return 2, fmt.Errorf("ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test")
|
||||
}
|
||||
for key, expected := range map[string]string{
|
||||
"ZAPRET_AUTOTUNE_TARGET_ID": managedTargetID,
|
||||
"ZAPRET_AUTOTUNE_TARGET_DIR": managedTargetDirectory,
|
||||
"ZAPRET_CONFIG": managedTargetConfig,
|
||||
} {
|
||||
if value, _ := lookup(key); value != expected {
|
||||
return 2, fmt.Errorf("%s does not match the fresh managed target contract", key)
|
||||
}
|
||||
}
|
||||
portText, _ := lookup("SOCKS_PORT")
|
||||
port, portErr := strconv.Atoi(portText)
|
||||
proxy, _ := lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
|
||||
proxy, proxyErr := managedProbeProxy(proxy)
|
||||
if portErr != nil || port < 1 || port > 65535 || strconv.Itoa(port) != portText || proxyErr != nil || proxy != "socks5h://127.0.0.1:"+portText {
|
||||
return 2, fmt.Errorf("managed SOCKS proxy must match the target SOCKS_PORT listener")
|
||||
}
|
||||
return port, nil
|
||||
}
|
||||
|
||||
func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
|
||||
if len(args) == 1 && args[0] == "resource-status" {
|
||||
return resourceStatusCLI(stdout, d)
|
||||
}
|
||||
if len(args) > 0 && args[0] == "monitor" {
|
||||
return managedMonitorCLI(args[1:], stdout, stderr, d)
|
||||
}
|
||||
|
||||
@@ -57,6 +57,17 @@ func managedMonitorCLI(args []string, stdout, stderr io.Writer, target managedDe
|
||||
}
|
||||
|
||||
func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) (code int) {
|
||||
resourcePath, _ := target.lookup("ZAPRET_AUTOTUNE_RESOURCE_CONFIG")
|
||||
if len(args) == 2 && args[0] == "--resource-config" && args[1] != "" {
|
||||
return resourceMonitorCLI(ctx, args[1], stdout, stderr, target)
|
||||
}
|
||||
if resourcePath != "" {
|
||||
if len(args) != 0 {
|
||||
fmt.Fprintln(stderr, "configerror: generic monitor accepts only --resource-config FILE")
|
||||
return 64
|
||||
}
|
||||
return resourceMonitorCLI(ctx, resourcePath, stdout, stderr, target)
|
||||
}
|
||||
cfg, err := managedMonitorConfig(args, target.lookup)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error: invalid managed monitor configuration")
|
||||
|
||||
@@ -702,7 +702,11 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
text = shortened
|
||||
}
|
||||
if mode == "stop-hang" {
|
||||
text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1)
|
||||
shortened := strings.Replace(text, "cleanup_command 30 "+zapret+" stop", "timeout -k 1 1 "+zapret+" stop", 1)
|
||||
if shortened == text {
|
||||
t.Fatal("fixture did not shorten stop watchdog")
|
||||
}
|
||||
text = shortened
|
||||
}
|
||||
entry := write("entrypoint", text)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -42,6 +43,8 @@ type resourceCheckSpec struct {
|
||||
Method string `json:"method"`
|
||||
TLS string `json:"tls"`
|
||||
TimeoutMS int `json:"timeout_ms"`
|
||||
IntervalMS *int `json:"interval_ms,omitempty"`
|
||||
CooldownMS *int `json:"cooldown_ms,omitempty"`
|
||||
Expect resourceCheckExpect `json:"expect"`
|
||||
HeadersEnv map[string]string `json:"headers_env,omitempty"`
|
||||
AllowPrivate bool `json:"allow_private,omitempty"`
|
||||
@@ -139,6 +142,18 @@ func parseResourceChecks(data []byte, lookup func(string) (string, bool)) (resou
|
||||
return fail()
|
||||
}
|
||||
ids[r.ID] = true
|
||||
for _, scheduling := range []struct {
|
||||
value **int
|
||||
fallback int
|
||||
}{{&r.IntervalMS, 300000}, {&r.CooldownMS, 30000}} {
|
||||
if *scheduling.value == nil {
|
||||
v := scheduling.fallback
|
||||
*scheduling.value = &v
|
||||
}
|
||||
if **scheduling.value < 100 || **scheduling.value > 86400000 {
|
||||
return fail()
|
||||
}
|
||||
}
|
||||
if (r.URL == "") == (r.URLEnv == "") {
|
||||
return fail()
|
||||
}
|
||||
@@ -202,7 +217,13 @@ func parseResourceChecks(data []byte, lookup func(string) (string, bool)) (resou
|
||||
return fail()
|
||||
}
|
||||
headers := map[string]bool{}
|
||||
for name, env := range r.HeadersEnv {
|
||||
names := make([]string, 0, len(r.HeadersEnv))
|
||||
for name := range r.HeadersEnv {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
env := r.HeadersEnv[name]
|
||||
lower := strings.ToLower(name)
|
||||
if !resourceHeader.MatchString(name) || headers[lower] || !resourceEnv.MatchString(env) {
|
||||
return fail()
|
||||
@@ -234,6 +255,10 @@ func parseResourceChecks(data []byte, lookup func(string) (string, bool)) (resou
|
||||
}
|
||||
|
||||
func readResourceChecks(path string, lookup func(string) (string, bool)) (resourceCheckConfig, error) {
|
||||
return readResourceChecksMode(path, lookup, false)
|
||||
}
|
||||
|
||||
func readResourceChecksMode(path string, lookup func(string) (string, bool), owned bool) (resourceCheckConfig, error) {
|
||||
fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK|syscall.O_CLOEXEC, 0)
|
||||
if err != nil {
|
||||
return resourceCheckConfig{}, errResourceConfig
|
||||
@@ -244,10 +269,22 @@ func readResourceChecks(path string, lookup func(string) (string, bool)) (resour
|
||||
if err != nil || !info.Mode().IsRegular() || info.Size() > 1<<20 {
|
||||
return resourceCheckConfig{}, errResourceConfig
|
||||
}
|
||||
if owned {
|
||||
st, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok || (st.Uid != 0 && st.Uid != uint32(os.Geteuid())) || st.Nlink != 1 || info.Mode().Perm()&0022 != 0 {
|
||||
return resourceCheckConfig{}, errResourceConfig
|
||||
}
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(f, (1<<20)+1))
|
||||
if err != nil {
|
||||
return resourceCheckConfig{}, errResourceConfig
|
||||
}
|
||||
if owned {
|
||||
current, err := os.Lstat(path)
|
||||
if err != nil || !os.SameFile(info, current) {
|
||||
return resourceCheckConfig{}, errResourceConfig
|
||||
}
|
||||
}
|
||||
return parseResourceChecks(data, lookup)
|
||||
}
|
||||
|
||||
@@ -329,6 +366,52 @@ type resourceCheckResult struct {
|
||||
Status int `json:"status"`
|
||||
}
|
||||
|
||||
// A synchronous runner: subprocess cancellation joins before this returns.
|
||||
// The guard is also called after DNS, immediately before the pinned transfer.
|
||||
func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpec, protocol, server, proxy string, d resourceCheckDeps) (resourceCheckResult, error) {
|
||||
result := resourceCheckResult{Service: service, Resource: r.ID, Group: r.Group, TLS: protocol, Outcome: "dns_failed"}
|
||||
if err := d.guard(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(ctx, time.Duration(r.TimeoutMS)*time.Millisecond)
|
||||
defer cancel()
|
||||
ips, err := d.resolve(probeCtx, server, r.host)
|
||||
if err == nil && len(ips) > 0 && len(ips) <= 64 {
|
||||
approved := true
|
||||
for _, ip := range ips {
|
||||
if !approvedResourceIP(r, ip) {
|
||||
approved = false
|
||||
}
|
||||
}
|
||||
result.Outcome = "destination_denied"
|
||||
if approved {
|
||||
if err := d.guard(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443")
|
||||
options := resourceTransferOptions{generic: true, method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n"}
|
||||
transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, false, d.runner, options)
|
||||
result.Outcome = "transfer_failed"
|
||||
if err == nil {
|
||||
result.Bytes, result.TimeMS, result.Status = len(transfer.Body), transfer.LatencyMS, transfer.Status
|
||||
result.Outcome = "expectation_failed"
|
||||
for _, status := range r.Expect.Statuses {
|
||||
if status == transfer.Status && len(transfer.Body) >= r.Expect.MinBytes && (r.Expect.ContentType == "" || transfer.ContentType == r.Expect.ContentType) {
|
||||
result.Outcome = "pass"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if probeCtx.Err() != nil {
|
||||
result.Outcome = "timeout"
|
||||
if ctx.Err() != nil {
|
||||
result.Outcome = "cancelled"
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func resourceCheckCLI(ctx context.Context, path string, stdout, stderr io.Writer, d resourceCheckDeps) int {
|
||||
if d.lookup == nil {
|
||||
fmt.Fprintln(stderr, "configerror: invalid resource check configuration")
|
||||
@@ -357,57 +440,11 @@ func resourceCheckCLI(ctx context.Context, path string, stdout, stderr io.Writer
|
||||
if ctx.Err() != nil {
|
||||
return 1
|
||||
}
|
||||
if d.guard() != nil {
|
||||
result, err := runResourceProtocol(ctx, s.ID, r, protocol, server, proxy, d)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "blocked: resource check ownership unavailable")
|
||||
return 2
|
||||
}
|
||||
result := resourceCheckResult{Service: s.ID, Resource: r.ID, Group: r.Group, TLS: protocol, Outcome: "dns_failed"}
|
||||
probeCtx, cancel := context.WithTimeout(ctx, time.Duration(r.TimeoutMS)*time.Millisecond)
|
||||
ips, err := d.resolve(probeCtx, server, r.host)
|
||||
if err == nil && len(ips) > 0 && len(ips) <= 64 {
|
||||
approved := true
|
||||
for _, ip := range ips {
|
||||
if !approvedResourceIP(r, ip) {
|
||||
approved = false
|
||||
}
|
||||
}
|
||||
result.Outcome = "destination_denied"
|
||||
if approved {
|
||||
if d.guard() != nil {
|
||||
cancel()
|
||||
fmt.Fprintln(stderr, "blocked: resource check ownership unavailable")
|
||||
return 2
|
||||
}
|
||||
// --connect-to changes the SOCKS CONNECT destination even for socks5h;
|
||||
// URL authority, HTTP Host, SNI and certificate verification stay original.
|
||||
pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443")
|
||||
options := resourceTransferOptions{generic: true, method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n"}
|
||||
transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, false, d.runner, options)
|
||||
result.Outcome = "transfer_failed"
|
||||
if err == nil {
|
||||
result.Bytes, result.TimeMS, result.Status = len(transfer.Body), transfer.LatencyMS, transfer.Status
|
||||
result.Outcome = "expectation_failed"
|
||||
statusOK := false
|
||||
for _, status := range r.Expect.Statuses {
|
||||
if status == transfer.Status {
|
||||
statusOK = true
|
||||
}
|
||||
}
|
||||
if statusOK {
|
||||
if len(transfer.Body) >= r.Expect.MinBytes && (r.Expect.ContentType == "" || transfer.ContentType == r.Expect.ContentType) {
|
||||
result.Outcome = "pass"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if probeCtx.Err() != nil {
|
||||
result.Outcome = "timeout"
|
||||
if ctx.Err() != nil {
|
||||
result.Outcome = "cancelled"
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
if result.Outcome != "pass" {
|
||||
code = 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,441 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// HEALTHY means only that the configured checks passed, not production READY.
|
||||
type resourceMonitorSource struct {
|
||||
config resourceCheckConfig
|
||||
server, proxy, fingerprint string
|
||||
}
|
||||
|
||||
func resourceStaticConfig(lookup func(string) (string, bool)) (string, string, error) {
|
||||
dns, err := parseManagedDNSConfig(lookup)
|
||||
raw, _ := lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
|
||||
proxy, proxyErr := managedProbeProxy(raw)
|
||||
if err != nil || proxyErr != nil || dns.Mode == "native" && dns.NativeServer == "" {
|
||||
return "", "", errResourceConfig
|
||||
}
|
||||
server := dns.serverAddress()
|
||||
if dns.Mode == "native" {
|
||||
server = net.JoinHostPort(server, "53")
|
||||
}
|
||||
return server, proxy, nil
|
||||
}
|
||||
func readResourceMonitorSource(path string, lookup func(string) (string, bool), key []byte) (resourceMonitorSource, error) {
|
||||
// Resolve each referenced environment value once in this immutable snapshot.
|
||||
values := map[string]struct {
|
||||
value string
|
||||
ok bool
|
||||
}{}
|
||||
snapshot := func(name string) (string, bool) {
|
||||
if v, ok := values[name]; ok {
|
||||
return v.value, v.ok
|
||||
}
|
||||
v, ok := lookup(name)
|
||||
if len(v) > 16384 {
|
||||
return "", false
|
||||
}
|
||||
values[name] = struct {
|
||||
value string
|
||||
ok bool
|
||||
}{v, ok}
|
||||
return v, ok
|
||||
}
|
||||
cfg, err := readResourceChecksMode(path, snapshot, true)
|
||||
server, proxy, staticErr := resourceStaticConfig(snapshot)
|
||||
if err != nil || staticErr != nil {
|
||||
return resourceMonitorSource{}, errResourceConfig
|
||||
}
|
||||
source := resourceMonitorSource{config: cfg, server: server, proxy: proxy}
|
||||
b, _ := json.Marshal(cfg)
|
||||
// Marshal omits the private curl headers; bind them explicitly as well.
|
||||
parts := []string{string(b), server, proxy}
|
||||
for _, s := range cfg.Services {
|
||||
for _, r := range s.Resources {
|
||||
parts = append(parts, r.privateConfig)
|
||||
}
|
||||
}
|
||||
b, _ = json.Marshal(parts)
|
||||
source.fingerprint = resourceMAC(key, b)
|
||||
return source, nil
|
||||
}
|
||||
func resourceRowFingerprint(key []byte, source resourceMonitorSource, r resourceCheckSpec) string {
|
||||
b, _ := json.Marshal(r)
|
||||
b, _ = json.Marshal([]string{string(b), r.privateConfig, source.server, source.proxy})
|
||||
return resourceMAC(key, b)
|
||||
}
|
||||
func reconcileResourceRows(key []byte, source resourceMonitorSource, prior []resourceMonitorRow) []resourceMonitorRow {
|
||||
old := map[string]resourceMonitorRow{}
|
||||
for _, r := range prior {
|
||||
old[r.Service+"/"+r.Resource] = r
|
||||
}
|
||||
rows := []resourceMonitorRow{}
|
||||
for _, s := range source.config.Services {
|
||||
for _, r := range s.Resources {
|
||||
fp := resourceRowFingerprint(key, source, r)
|
||||
row, ok := old[s.ID+"/"+r.ID]
|
||||
if !ok || row.Fingerprint != fp {
|
||||
row = resourceMonitorRow{Service: s.ID, Resource: r.ID, Fingerprint: fp, Health: "UNKNOWN", Outcome: "unknown"}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
type resourceMonitorDeps struct {
|
||||
load func() (resourceMonitorState, error)
|
||||
save func(resourceMonitorState) error
|
||||
source func() (resourceMonitorSource, error)
|
||||
guard func() error
|
||||
probe func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error)
|
||||
key []byte
|
||||
now func() time.Time
|
||||
wait func(context.Context, time.Duration) error
|
||||
report func(bool)
|
||||
}
|
||||
|
||||
// Sequential round robin with independent due times; reload at least every
|
||||
// second while idle/blocked. No recovery leases or strategy mutations.
|
||||
func runResourceMonitor(ctx context.Context, d resourceMonitorDeps) error {
|
||||
cursor := 0
|
||||
persistenceBlocked, failedMAC, attemptedMAC := false, "", ""
|
||||
var shutdownError error
|
||||
for ctx.Err() == nil {
|
||||
state, stateErr := d.load()
|
||||
if persistenceBlocked && stateErr == nil && state.MAC != failedMAC && state.MAC != attemptedMAC {
|
||||
persistenceBlocked = false
|
||||
}
|
||||
source, sourceErr := d.source()
|
||||
blocked := persistenceBlocked || stateErr != nil || sourceErr != nil || d.guard() != nil
|
||||
dirty := false
|
||||
if !blocked {
|
||||
rows := reconcileResourceRows(d.key, source, state.Rows)
|
||||
dirty = !reflect.DeepEqual(rows, state.Rows)
|
||||
state.Rows = rows
|
||||
if len(state.Rows) > 0 {
|
||||
cursor %= len(state.Rows)
|
||||
for n := 0; n < len(state.Rows); n++ {
|
||||
i := (cursor + n) % len(state.Rows)
|
||||
row := &state.Rows[i]
|
||||
if row.NextDue > d.now().UnixMilli() {
|
||||
continue
|
||||
}
|
||||
var spec resourceCheckSpec
|
||||
for _, s := range source.config.Services {
|
||||
if s.ID == row.Service {
|
||||
for _, r := range s.Resources {
|
||||
if r.ID == row.Resource {
|
||||
spec = r
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
guard := func() error {
|
||||
if d.guard() != nil {
|
||||
return errResourceConfig
|
||||
}
|
||||
fresh, err := d.source()
|
||||
if err != nil || fresh.fingerprint != source.fingerprint {
|
||||
return errResourceConfig
|
||||
}
|
||||
return nil
|
||||
}
|
||||
outcome, err := d.probe(ctx, row.Service, spec, source, guard)
|
||||
cursor = (i + 1) % len(state.Rows)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil || !managedCancellationOnly(err, ctx.Err()) {
|
||||
blocked = true
|
||||
if ctx.Err() != nil {
|
||||
shutdownError = err
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
// Cancellation joins the probe but is not completed evidence.
|
||||
// Keep the previous health, failure count and due time intact.
|
||||
if ctx.Err() != nil || outcome == "cancelled" {
|
||||
break
|
||||
}
|
||||
dirty = true
|
||||
row.Outcome, row.LastCompleted = outcome, d.now().UnixMilli()
|
||||
delay := *spec.CooldownMS
|
||||
if outcome == "pass" {
|
||||
row.Health, row.Failures, delay = "HEALTHY", 0, *spec.IntervalMS
|
||||
} else {
|
||||
row.Health = "DEGRADED"
|
||||
if row.Failures < ^uint64(0) {
|
||||
row.Failures++
|
||||
}
|
||||
}
|
||||
row.NextDue = row.LastCompleted + int64(delay)
|
||||
break // persist each completed (and joined) check before another probe
|
||||
}
|
||||
}
|
||||
}
|
||||
dirty = dirty || state.Blocked != blocked
|
||||
state.Blocked = blocked
|
||||
if dirty {
|
||||
state.Updated = d.now().UnixMilli()
|
||||
}
|
||||
if dirty && stateErr == nil && !persistenceBlocked {
|
||||
if d.save(state) != nil {
|
||||
blocked, persistenceBlocked, failedMAC, attemptedMAC = true, true, state.MAC, resourceStateMAC(d.key, state)
|
||||
}
|
||||
}
|
||||
d.report(blocked)
|
||||
if ctx.Err() != nil {
|
||||
if persistenceBlocked {
|
||||
return errors.Join(shutdownError, errResourceState)
|
||||
}
|
||||
return shutdownError
|
||||
}
|
||||
// Bound reloads, but don't delay an already due sibling by a cooldown.
|
||||
delay := time.Second
|
||||
if !blocked {
|
||||
for _, r := range state.Rows {
|
||||
milliseconds := r.NextDue - d.now().UnixMilli()
|
||||
if milliseconds > 1000 {
|
||||
milliseconds = 1000
|
||||
}
|
||||
remaining := time.Duration(milliseconds) * time.Millisecond
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
if remaining < delay {
|
||||
delay = remaining
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := d.wait(ctx, delay); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
if persistenceBlocked {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
if persistenceBlocked {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func probeResourceMonitor(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error, check resourceCheckDeps) (string, error) {
|
||||
check.guard = guard
|
||||
protocols := []string{spec.TLS}
|
||||
if spec.TLS == "both" {
|
||||
protocols = []string{"tls12", "tls13"}
|
||||
}
|
||||
outcome := "pass"
|
||||
for _, protocol := range protocols {
|
||||
if ctx.Err() != nil {
|
||||
return "cancelled", nil
|
||||
}
|
||||
result, err := runResourceProtocol(ctx, service, spec, protocol, source.server, source.proxy, check)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if result.Outcome != "pass" {
|
||||
outcome = result.Outcome
|
||||
}
|
||||
}
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
func resourceMonitorCLI(ctx context.Context, path string, stdout, stderr io.Writer, target managedDeps) (code int) {
|
||||
return resourceMonitorCLIWithCheck(ctx, path, stdout, stderr, target, productionResourceCheckDeps())
|
||||
}
|
||||
|
||||
func resourceMonitorCLIWithCheck(ctx context.Context, path string, stdout, stderr io.Writer, target managedDeps, check resourceCheckDeps) (code int) {
|
||||
port, settingsErr := managedTargetSettings(target.lookup)
|
||||
if settingsErr != nil || port != 1080 {
|
||||
fmt.Fprintln(stderr, "configerror: invalid generic target settings")
|
||||
return 64
|
||||
}
|
||||
if _, _, err := resourceStaticConfig(target.lookup); err != nil {
|
||||
fmt.Fprintln(stderr, "configerror: invalid generic DNS/proxy configuration")
|
||||
return 64
|
||||
}
|
||||
var lock *managedMonitorSingleton
|
||||
var store *resourceMonitorStore
|
||||
defer func() {
|
||||
if store != nil && store.close() != nil {
|
||||
code = 78
|
||||
}
|
||||
if lock != nil && lock.close() != nil {
|
||||
code = 78
|
||||
}
|
||||
}()
|
||||
lastBlocked := false
|
||||
report := func(blocked bool) {
|
||||
if blocked && !lastBlocked {
|
||||
fmt.Fprintln(stderr, "BLOCKED: generic resource monitor guard/state unavailable")
|
||||
}
|
||||
lastBlocked = blocked
|
||||
}
|
||||
fmt.Fprintln(stdout, "generic resource monitor: configured checks only; no recovery/apply; HEALTHY is not production READY")
|
||||
// Guard failures before store/lock acquisition remain cancellable and alive.
|
||||
for ctx.Err() == nil {
|
||||
if lock == nil {
|
||||
var err error
|
||||
lock, err = acquireManagedMonitorSingleton(target)
|
||||
if err != nil {
|
||||
if errors.Is(err, syscall.EWOULDBLOCK) {
|
||||
fmt.Fprintln(stderr, "BLOCKED: monitor singleton already held")
|
||||
return 78
|
||||
}
|
||||
report(true)
|
||||
if waitManagedMonitor(ctx, time.Second) != nil {
|
||||
return 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if store == nil {
|
||||
var err error
|
||||
store, err = openResourceMonitorStore(target.directory, true)
|
||||
if err != nil {
|
||||
report(true)
|
||||
if waitManagedMonitor(ctx, time.Second) != nil {
|
||||
return 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
err := runResourceMonitor(ctx, resourceMonitorDeps{
|
||||
load: store.load, save: store.save, key: store.key,
|
||||
source: func() (resourceMonitorSource, error) {
|
||||
return readResourceMonitorSource(path, target.lookup, store.key)
|
||||
},
|
||||
guard: func() error {
|
||||
if resourceSingletonGuard(lock, target.directory) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
// Probe guards validate the held snapshot, never reload/adopt a new
|
||||
// authenticated operator edit while an older check is in flight.
|
||||
if store.snapshotGuard() != nil {
|
||||
return errResourceState
|
||||
}
|
||||
if resourceManagedIdentity(target.directory) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
return check.guard()
|
||||
},
|
||||
probe: func(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error) (string, error) {
|
||||
return probeResourceMonitor(ctx, service, spec, source, guard, check)
|
||||
},
|
||||
now: time.Now, wait: waitManagedMonitor, report: report,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "BLOCKED: generic monitor safety/finalization failure")
|
||||
return 78
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func resourceSingletonGuard(lock *managedMonitorSingleton, dir string) error {
|
||||
path := filepath.Join(dir, managedMonitorLockName)
|
||||
if managedPrivateFile(path) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
current, err := os.Lstat(path)
|
||||
held, heldErr := lock.file.Stat()
|
||||
if err != nil || heldErr != nil || !os.SameFile(current, held) {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resourceStatusCLI(stdout io.Writer, target managedDeps) int {
|
||||
blocked := func() int { fmt.Fprintln(stdout, `{"health":"BLOCKED","stale":true}`); return 1 }
|
||||
store, err := openResourceMonitorStore(target.directory, false)
|
||||
if err != nil {
|
||||
return blocked()
|
||||
}
|
||||
defer store.close()
|
||||
state, err := store.load()
|
||||
if err != nil {
|
||||
return blocked()
|
||||
}
|
||||
path, _ := target.lookup("ZAPRET_AUTOTUNE_RESOURCE_CONFIG")
|
||||
source, err := readResourceMonitorSource(path, target.lookup, store.key)
|
||||
check := productionResourceCheckDeps()
|
||||
if err != nil || check.guard() != nil || resourceManagedIdentity(target.directory) != nil {
|
||||
return blocked()
|
||||
}
|
||||
rows := reconcileResourceRows(store.key, source, state.Rows)
|
||||
b1, _ := json.Marshal(rows)
|
||||
b2, _ := json.Marshal(state.Rows)
|
||||
stale := resourceSnapshotStale(state, time.Now().UnixMilli()) || string(b1) != string(b2)
|
||||
health := "CHECKS_ONLY"
|
||||
if stale {
|
||||
health = "STALE"
|
||||
}
|
||||
if state.Blocked {
|
||||
health = "BLOCKED"
|
||||
}
|
||||
if json.NewEncoder(stdout).Encode(struct {
|
||||
Health string `json:"health"`
|
||||
Stale bool `json:"stale"`
|
||||
State resourceMonitorState `json:"state"`
|
||||
}{health, stale, state}) != nil {
|
||||
return 1
|
||||
}
|
||||
if stale || state.Blocked {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Updated is a semantic-change timestamp, not a disk heartbeat. Evidence is
|
||||
// stale only once its next scheduled check plus the maximum run budget is past.
|
||||
func resourceSnapshotStale(state resourceMonitorState, now int64) bool {
|
||||
if state.Updated > now {
|
||||
return true
|
||||
}
|
||||
if len(state.Rows) == 0 {
|
||||
return now-state.Updated > 65000
|
||||
}
|
||||
for _, row := range state.Rows {
|
||||
if row.NextDue == 0 {
|
||||
if now-state.Updated > 65000 {
|
||||
return true
|
||||
}
|
||||
} else if now > row.NextDue && now-row.NextDue > 65000 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func resourceManagedIdentity(dir string) error {
|
||||
m := &managedTransaction{Dir: dir, Name: "config"}
|
||||
if managedCheckIdentity(m) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
j, err := m.load()
|
||||
if err != nil || j.State != "committed" {
|
||||
return errResourceState
|
||||
}
|
||||
b, err := m.read("config")
|
||||
if err != nil || managedHash(b) != j.Hash {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func resourceCompletedFixture(t *testing.T) (resourceMonitorSource, []byte, resourceMonitorState) {
|
||||
t.Helper()
|
||||
source, key := resourceMonitorFixture(t)
|
||||
state := resourceMonitorState{Version: 1, Updated: 5000, Rows: reconcileResourceRows(key, source, nil)}
|
||||
for i := range state.Rows {
|
||||
state.Rows[i].Health, state.Rows[i].Outcome, state.Rows[i].LastCompleted = "HEALTHY", "pass", 5000
|
||||
state.Rows[i].NextDue = 9000
|
||||
}
|
||||
return source, key, state
|
||||
}
|
||||
|
||||
func TestResourceMonitorCancelledRowsUnchangedAndJoined(t *testing.T) {
|
||||
for _, mixed := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "curl-cancel", true: "cancel-and-guard"}[mixed], func(t *testing.T) {
|
||||
source, key, state := resourceCompletedFixture(t)
|
||||
prior := append([]resourceMonitorRow(nil), state.Rows...)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
joined, writes := false, 0
|
||||
check := resourceCheckDeps{resolve: func(context.Context, string, string) ([]net.IP, error) {
|
||||
return []net.IP{net.ParseIP("93.184.216.34")}, nil
|
||||
}, runner: func(ctx context.Context, _ []string, _ io.Reader, _ io.Writer, _ io.Writer) error {
|
||||
cancel()
|
||||
<-ctx.Done()
|
||||
joined = true
|
||||
return ctx.Err()
|
||||
}}
|
||||
d := resourceMonitorDeps{key: key, now: func() time.Time { return time.UnixMilli(10000) }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(s resourceMonitorState) error {
|
||||
if !joined {
|
||||
t.Error("persisted before joining curl")
|
||||
}
|
||||
writes++
|
||||
state = s
|
||||
return nil
|
||||
}, source: func() (resourceMonitorSource, error) { return source, nil }, guard: func() error { return nil }, report: func(bool) {}, wait: func(context.Context, time.Duration) error { t.Error("wait after cancellation"); return nil }}
|
||||
d.probe = func(ctx context.Context, service string, spec resourceCheckSpec, source resourceMonitorSource, guard func() error) (string, error) {
|
||||
outcome, err := probeResourceMonitor(ctx, service, spec, source, guard, check)
|
||||
if mixed {
|
||||
return outcome, errors.Join(ctx.Err(), errResourceConfig)
|
||||
}
|
||||
return outcome, err
|
||||
}
|
||||
err := runResourceMonitor(ctx, d)
|
||||
if mixed {
|
||||
if !errors.Is(err, errResourceConfig) || !errors.Is(err, context.Canceled) || !state.Blocked || writes != 1 {
|
||||
t.Fatal("lost mixed shutdown guard error", state, writes, err)
|
||||
}
|
||||
} else if err != nil || writes != 0 {
|
||||
t.Fatal("cancel published evidence", writes, err)
|
||||
}
|
||||
if !joined || !reflect.DeepEqual(prior, state.Rows) {
|
||||
t.Fatal("cancel changed completed rows or failed to join", state.Rows)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorIdleNoPersistenceHeartbeat(t *testing.T) {
|
||||
source, key, state := resourceCompletedFixture(t)
|
||||
for i := range state.Rows {
|
||||
state.Rows[i].NextDue = 305000
|
||||
}
|
||||
now := time.UnixMilli(10000)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
writes, waits := 0, 0
|
||||
d := resourceMonitorDeps{key: key, now: func() time.Time { return now }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(s resourceMonitorState) error { writes++; state = s; return nil }, source: func() (resourceMonitorSource, error) { return source, nil }, guard: func() error { return nil }, report: func(bool) {}, probe: func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error) {
|
||||
t.Error("probed resource before due")
|
||||
return "pass", nil
|
||||
}, wait: func(_ context.Context, d time.Duration) error {
|
||||
now = now.Add(d)
|
||||
waits++
|
||||
if waits == 5 {
|
||||
cancel()
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
if err := runResourceMonitor(ctx, d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if writes != 0 || state.Updated != 5000 {
|
||||
t.Fatal("idle disk heartbeat", writes, state.Updated)
|
||||
}
|
||||
if resourceSnapshotStale(state, 100000) || !resourceSnapshotStale(state, 370001) {
|
||||
t.Fatal("staleness still depends on idle heartbeat")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorBlockedTransitionsOnly(t *testing.T) {
|
||||
source, key, state := resourceCompletedFixture(t)
|
||||
for i := range state.Rows {
|
||||
state.Rows[i].NextDue = 305000
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
tick, writes := 0, 0
|
||||
d := resourceMonitorDeps{key: key, now: func() time.Time { return time.UnixMilli(10000 + int64(tick)*1000) }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(s resourceMonitorState) error { writes++; state = s; return nil }, source: func() (resourceMonitorSource, error) {
|
||||
if tick < 3 {
|
||||
return resourceMonitorSource{}, errResourceConfig
|
||||
}
|
||||
return source, nil
|
||||
}, guard: func() error { return nil }, report: func(bool) {}, probe: func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error) {
|
||||
t.Error("unexpected probe")
|
||||
return "pass", nil
|
||||
}, wait: func(context.Context, time.Duration) error {
|
||||
tick++
|
||||
if tick == 5 {
|
||||
cancel()
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
if err := runResourceMonitor(ctx, d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if writes != 2 || state.Blocked {
|
||||
t.Fatal("blocked transition heartbeat or missing resume", writes, state)
|
||||
}
|
||||
}
|
||||
|
||||
func resourceBoundaryStore(t *testing.T, createState bool) (*resourceMonitorStore, resourceMonitorState) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.Chmod(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := openResourceMonitorStore(dir, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { s.close() })
|
||||
v, err := s.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if createState {
|
||||
if err := s.save(v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return s, v
|
||||
}
|
||||
|
||||
func replaceResourceEntry(t *testing.T, path string, data []byte) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path+"-replacement", data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(path+"-replacement", path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorHeldEntriesRejectReplacement(t *testing.T) {
|
||||
for _, name := range []string{resourceStateName, resourceKeyName} {
|
||||
for _, point := range []string{"before-load", "loaded", "staged", "publish"} {
|
||||
t.Run(name+"/"+point, func(t *testing.T) {
|
||||
s, v := resourceBoundaryStore(t, true)
|
||||
path := filepath.Join(s.path, name)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var replacement os.FileInfo
|
||||
mutate := func() {
|
||||
replaceResourceEntry(t, path, data)
|
||||
replacement, err = os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if point == "before-load" {
|
||||
mutate()
|
||||
} else {
|
||||
s.fault = func(p, _ string) {
|
||||
if p == point {
|
||||
mutate()
|
||||
}
|
||||
}
|
||||
}
|
||||
if point == "before-load" || point == "loaded" {
|
||||
if _, err := s.load(); err == nil {
|
||||
t.Fatal("load adopted identical replaced inode")
|
||||
}
|
||||
}
|
||||
if err := s.save(v); err == nil {
|
||||
t.Fatal("save clobbered identical replacement")
|
||||
}
|
||||
current, err := os.Stat(path)
|
||||
if err != nil || replacement == nil || !os.SameFile(current, replacement) {
|
||||
t.Fatal("foreign inode changed", err)
|
||||
}
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil || !bytes.Equal(got, data) {
|
||||
t.Fatal("foreign bytes changed", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorMidStageCorruptionPreserved(t *testing.T) {
|
||||
for _, name := range []string{resourceStateName, resourceKeyName, "stage", "stage-in-place"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s, v := resourceBoundaryStore(t, true)
|
||||
bad := []byte("untrusted corrupt operator bytes")
|
||||
path := ""
|
||||
s.fault = func(point, stage string) {
|
||||
if point != "staged" {
|
||||
return
|
||||
}
|
||||
path = filepath.Join(s.path, name)
|
||||
if name == "stage" || name == "stage-in-place" {
|
||||
path = stage
|
||||
}
|
||||
if name == "stage-in-place" {
|
||||
if err := os.WriteFile(path, bad, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
replaceResourceEntry(t, path, bad)
|
||||
}
|
||||
}
|
||||
if err := s.save(v); err == nil {
|
||||
t.Fatal("corruption overwritten")
|
||||
}
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil || !bytes.Equal(got, bad) {
|
||||
t.Fatal("unknown stage/file removed or clobbered", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorInitialPublicationExclusive(t *testing.T) {
|
||||
s, v := resourceBoundaryStore(t, false)
|
||||
path := filepath.Join(s.path, resourceStateName)
|
||||
bad := []byte("new foreign entry created just before initial publication")
|
||||
s.fault = func(point, _ string) {
|
||||
if point == "publish" {
|
||||
if err := os.WriteFile(path, bad, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := s.save(v); err == nil {
|
||||
t.Fatal("exclusive initial publish overwrote raced entry")
|
||||
}
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil || !bytes.Equal(got, bad) {
|
||||
t.Fatal("new entry clobbered", err)
|
||||
}
|
||||
if _, err := s.load(); err == nil {
|
||||
t.Fatal("new entry silently adopted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorPublicationRetainsNewAnchor(t *testing.T) {
|
||||
s, v := resourceBoundaryStore(t, true)
|
||||
old := s.stateFile
|
||||
v.Updated = 10000
|
||||
if err := s.save(v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if old == s.stateFile {
|
||||
t.Fatal("old inode retained instead of published inode")
|
||||
}
|
||||
if _, err := old.Stat(); err == nil {
|
||||
t.Fatal("old anchor not closed after verified publication")
|
||||
}
|
||||
if s.entryMatches(resourceStateName, s.stateFile) != nil {
|
||||
t.Fatal("published state not anchored")
|
||||
}
|
||||
if loaded, err := s.load(); err != nil || loaded.Updated != 10000 {
|
||||
t.Fatal(loaded, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorOriginalIdentityRepair(t *testing.T) {
|
||||
s, v := resourceBoundaryStore(t, true)
|
||||
path := filepath.Join(s.path, resourceStateName)
|
||||
valid, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(`{"version":99}`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.load(); err == nil {
|
||||
t.Fatal("corrupt original trusted")
|
||||
}
|
||||
if err := s.save(v); err == nil {
|
||||
t.Fatal("corrupt original overwritten")
|
||||
}
|
||||
if err := os.WriteFile(path, valid, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.load(); err != nil {
|
||||
t.Fatal("valid original-identity repair blocked", err)
|
||||
}
|
||||
current, err := os.Stat(path)
|
||||
if err != nil || !os.SameFile(current, original) {
|
||||
t.Fatal("repair changed inode", err)
|
||||
}
|
||||
if err := s.save(v); err != nil {
|
||||
t.Fatal("repaired original failed publish", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestResourceMonitorFinalPublicationStageGuard(t *testing.T) {
|
||||
for _, existing := range []bool{false, true} {
|
||||
for _, mutation := range []string{"byte", "truncate", "append", "growth", "path-replace", "key-replace", "key-in-place", "state-replace"} {
|
||||
name := "initial/"
|
||||
if existing {
|
||||
name = "existing/"
|
||||
}
|
||||
t.Run(name+mutation, func(t *testing.T) {
|
||||
s, v := resourceBoundaryStore(t, existing)
|
||||
path := filepath.Join(s.path, resourceStateName)
|
||||
oldBytes, oldErr := os.ReadFile(path)
|
||||
oldInfo, _ := os.Stat(path)
|
||||
if existing && oldErr != nil {
|
||||
t.Fatal(oldErr)
|
||||
}
|
||||
v.Updated = 10000
|
||||
var changedPath string
|
||||
var changedBytes []byte
|
||||
var changedInfo os.FileInfo
|
||||
s.fault = func(point, stage string) {
|
||||
if point != "publish" {
|
||||
return
|
||||
}
|
||||
data, err := os.ReadFile(stage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changedPath = stage
|
||||
switch mutation {
|
||||
case "byte":
|
||||
data[0] = '['
|
||||
case "truncate":
|
||||
data = data[:len(data)-1]
|
||||
case "append":
|
||||
data = append(data, ' ')
|
||||
case "growth":
|
||||
data = append(data, bytes.Repeat([]byte{'x'}, 65537)...)
|
||||
case "key-replace", "key-in-place":
|
||||
changedPath = filepath.Join(s.path, resourceKeyName)
|
||||
data, err = os.ReadFile(changedPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mutation == "key-in-place" {
|
||||
data[0] ^= 1
|
||||
}
|
||||
case "state-replace":
|
||||
changedPath = path
|
||||
if existing {
|
||||
data = append([]byte(nil), oldBytes...)
|
||||
}
|
||||
}
|
||||
changedBytes = append([]byte(nil), data...)
|
||||
if mutation == "path-replace" || mutation == "key-replace" || mutation == "state-replace" {
|
||||
replaceResourceEntry(t, changedPath, data)
|
||||
} else if err := os.WriteFile(changedPath, data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changedInfo, err = os.Stat(changedPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := s.save(v); !errors.Is(err, errResourceState) {
|
||||
t.Fatal("unsafe stage published", err)
|
||||
}
|
||||
current, err := os.ReadFile(changedPath)
|
||||
info, infoErr := os.Stat(changedPath)
|
||||
if err != nil || infoErr != nil || !bytes.Equal(current, changedBytes) || !os.SameFile(info, changedInfo) {
|
||||
t.Fatal("unknown artifact changed/deleted", err, infoErr)
|
||||
}
|
||||
if mutation != "state-replace" {
|
||||
if existing {
|
||||
current, err := os.ReadFile(path)
|
||||
info, infoErr := os.Stat(path)
|
||||
if err != nil || infoErr != nil || !bytes.Equal(current, oldBytes) || !os.SameFile(info, oldInfo) {
|
||||
t.Fatal("trusted old state lost before final guard", err, infoErr)
|
||||
}
|
||||
} else if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("initial corrupt state published", err)
|
||||
}
|
||||
}
|
||||
if existing && s.entryMatches(resourceStateName, s.stateFile) != nil && mutation != "state-replace" {
|
||||
t.Fatal("old trusted anchor lost")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorPublicationFailureBlocksFurtherProbes(t *testing.T) {
|
||||
source, _, _ := resourceCompletedFixture(t)
|
||||
s, _ := resourceBoundaryStore(t, true)
|
||||
state, err := s.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state.Updated = 5000
|
||||
state.Rows = reconcileResourceRows(s.key, source, nil)
|
||||
for i := range state.Rows {
|
||||
state.Rows[i].Health, state.Rows[i].Outcome = "HEALTHY", "pass"
|
||||
state.Rows[i].LastCompleted, state.Rows[i].NextDue = 5000, 9000
|
||||
}
|
||||
if err := s.save(state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(s.path, resourceStateName)
|
||||
old, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldInfo, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.fault = func(point, stage string) {
|
||||
if point == "publish" {
|
||||
if err := os.WriteFile(stage, []byte("unknown stage corruption"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
probes, waits, blocked := 0, 0, false
|
||||
d := resourceMonitorDeps{key: s.key, now: func() time.Time { return time.UnixMilli(10000) }, load: s.load, save: s.save, source: func() (resourceMonitorSource, error) { return source, nil }, guard: s.snapshotGuard, report: func(v bool) { blocked = v }, probe: func(context.Context, string, resourceCheckSpec, resourceMonitorSource, func() error) (string, error) {
|
||||
probes++
|
||||
return "pass", nil
|
||||
}, wait: func(context.Context, time.Duration) error {
|
||||
waits++
|
||||
if waits == 3 {
|
||||
cancel()
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
if err := runResourceMonitor(ctx, d); !errors.Is(err, errResourceState) {
|
||||
t.Fatal("lost unsafe publication failure", err)
|
||||
}
|
||||
if probes != 1 || !blocked {
|
||||
t.Fatal("continued probing after failed publication", probes, blocked)
|
||||
}
|
||||
current, err := os.ReadFile(path)
|
||||
currentInfo, infoErr := os.Stat(path)
|
||||
if err != nil || infoErr != nil || !bytes.Equal(current, old) || !os.SameFile(currentInfo, oldInfo) {
|
||||
t.Fatal("old completed snapshot lost", err, infoErr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
const resourceStateName = ".resource-monitor-state.json"
|
||||
const resourceKeyName = ".resource-monitor-key"
|
||||
|
||||
var errResourceState = errors.New("resource state blocked")
|
||||
|
||||
type resourceMonitorRow struct {
|
||||
Service string `json:"service"`
|
||||
Resource string `json:"resource"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Health string `json:"health"`
|
||||
Outcome string `json:"outcome"`
|
||||
Failures uint64 `json:"failures"`
|
||||
LastCompleted int64 `json:"last_completed_ms"`
|
||||
NextDue int64 `json:"next_due_ms"`
|
||||
}
|
||||
type resourceMonitorState struct {
|
||||
Version int `json:"version"`
|
||||
Updated int64 `json:"updated_ms"`
|
||||
Blocked bool `json:"blocked"`
|
||||
Rows []resourceMonitorRow `json:"rows"`
|
||||
MAC string `json:"mac"`
|
||||
}
|
||||
type resourceMonitorStore struct {
|
||||
dir *os.File
|
||||
path string
|
||||
key []byte
|
||||
previous []byte
|
||||
existed bool
|
||||
keyFile, stateFile *os.File
|
||||
// Deterministic filesystem interleavings in tests; nil in production.
|
||||
fault func(point, stage string)
|
||||
}
|
||||
|
||||
func resourceMAC(key, data []byte) string {
|
||||
m := hmac.New(sha256.New, key)
|
||||
m.Write(data)
|
||||
return hex.EncodeToString(m.Sum(nil))
|
||||
}
|
||||
func resourceStateMAC(key []byte, s resourceMonitorState) string {
|
||||
s.MAC = ""
|
||||
b, _ := json.Marshal(s)
|
||||
return resourceMAC(key, b)
|
||||
}
|
||||
func (s *resourceMonitorStore) guard() error {
|
||||
if managedDirectory(s.path) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
current, err := os.Stat(s.path)
|
||||
held, heldErr := s.dir.Stat()
|
||||
if err != nil || heldErr != nil || !os.SameFile(current, held) {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *resourceMonitorStore) entryPath(name string) string {
|
||||
return filepath.Join("/proc/self/fd", strconv.FormatUint(uint64(s.dir.Fd()), 10), name)
|
||||
}
|
||||
|
||||
func (s *resourceMonitorStore) close() error {
|
||||
var errs []error
|
||||
for _, f := range []*os.File{s.stateFile, s.keyFile, s.dir} {
|
||||
if f != nil {
|
||||
errs = append(errs, f.Close())
|
||||
}
|
||||
}
|
||||
s.stateFile, s.keyFile, s.dir = nil, nil, nil
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func resourcePrivateEntry(f *os.File) error {
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
st, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 || st.Uid != uint32(os.Geteuid()) || st.Nlink != 1 || info.Size() > 65536 {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *resourceMonitorStore) entryMatches(name string, held *os.File) error {
|
||||
current, err := os.Lstat(s.entryPath(name))
|
||||
anchored, heldErr := held.Stat()
|
||||
if err != nil || heldErr != nil || !os.SameFile(current, anchored) || resourcePrivateEntry(held) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *resourceMonitorStore) openEntry(name string) (*os.File, []byte, error) {
|
||||
if s.guard() != nil {
|
||||
return nil, nil, errResourceState
|
||||
}
|
||||
fd, err := syscall.Openat(int(s.dir.Fd()), name, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK|syscall.O_CLOEXEC, 0)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
f := os.NewFile(uintptr(fd), name)
|
||||
if resourcePrivateEntry(f) != nil {
|
||||
f.Close()
|
||||
return nil, nil, errResourceState
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(f, 65537))
|
||||
if err != nil || len(b) > 65536 || s.entryMatches(name, f) != nil {
|
||||
f.Close()
|
||||
return nil, nil, errResourceState
|
||||
}
|
||||
return f, b, nil
|
||||
}
|
||||
|
||||
func (s *resourceMonitorStore) trustedRead(name string, held *os.File) ([]byte, error) {
|
||||
f, b, err := s.openEntry(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
if held == nil || s.entryMatches(name, held) != nil {
|
||||
return nil, errResourceState
|
||||
}
|
||||
readInfo, err := f.Stat()
|
||||
heldInfo, heldErr := held.Stat()
|
||||
if err != nil || heldErr != nil || !os.SameFile(readInfo, heldInfo) {
|
||||
return nil, errResourceState
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Both content and held inode identities must still match the trusted snapshot.
|
||||
// In particular, an identical signed replacement is not silently adopted.
|
||||
func (s *resourceMonitorStore) snapshotGuard() error {
|
||||
key, err := s.trustedRead(resourceKeyName, s.keyFile)
|
||||
if err != nil || !hmac.Equal(key, s.key) {
|
||||
return errResourceState
|
||||
}
|
||||
if s.existed {
|
||||
b, err := s.trustedRead(resourceStateName, s.stateFile)
|
||||
if err != nil || !bytes.Equal(b, s.previous) {
|
||||
return errResourceState
|
||||
}
|
||||
if _, err := s.decode(b); err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
} else {
|
||||
f, _, err := s.openEntry(resourceStateName)
|
||||
if f != nil {
|
||||
f.Close()
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return errResourceState
|
||||
}
|
||||
}
|
||||
return s.guard()
|
||||
}
|
||||
func openResourceMonitorStore(path string, create bool) (*resourceMonitorStore, error) {
|
||||
if managedDirectory(path) != nil {
|
||||
return nil, errResourceState
|
||||
}
|
||||
fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_DIRECTORY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0)
|
||||
if err != nil {
|
||||
return nil, errResourceState
|
||||
}
|
||||
s := &resourceMonitorStore{path: path, dir: os.NewFile(uintptr(fd), "resource state directory")}
|
||||
fail := func() (*resourceMonitorStore, error) { s.close(); return nil, errResourceState }
|
||||
stateFile, state, stateErr := s.openEntry(resourceStateName)
|
||||
s.stateFile = stateFile
|
||||
if stateErr != nil && !errors.Is(stateErr, os.ErrNotExist) {
|
||||
return fail()
|
||||
}
|
||||
keyFile, key, err := s.openEntry(resourceKeyName)
|
||||
s.keyFile = keyFile
|
||||
if errors.Is(err, os.ErrNotExist) && errors.Is(stateErr, os.ErrNotExist) && create {
|
||||
key = make([]byte, 32)
|
||||
if _, err = rand.Read(key); err != nil {
|
||||
return fail()
|
||||
}
|
||||
keyFD, err := syscall.Openat(fd, resourceKeyName, syscall.O_RDWR|syscall.O_CREAT|syscall.O_EXCL|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0600)
|
||||
if err != nil {
|
||||
return fail()
|
||||
}
|
||||
f := os.NewFile(uintptr(keyFD), "resource key")
|
||||
s.keyFile = f
|
||||
_, writeErr := f.Write(key)
|
||||
syncErr := f.Sync()
|
||||
if errors.Join(writeErr, syncErr, s.dir.Sync()) != nil {
|
||||
return fail()
|
||||
}
|
||||
} else if err != nil {
|
||||
return fail()
|
||||
}
|
||||
if len(key) != 32 {
|
||||
return fail()
|
||||
}
|
||||
s.key = append([]byte(nil), key...)
|
||||
s.previous, s.existed = state, stateErr == nil
|
||||
if s.existed {
|
||||
if _, err := s.decode(state); err != nil {
|
||||
return fail()
|
||||
}
|
||||
}
|
||||
if s.snapshotGuard() != nil {
|
||||
return fail()
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
func (s *resourceMonitorStore) decode(b []byte) (resourceMonitorState, error) {
|
||||
var v resourceMonitorState
|
||||
d := json.NewDecoder(bytes.NewReader(b))
|
||||
if uniqueResourceJSON(d, 0) != nil {
|
||||
return v, errResourceState
|
||||
}
|
||||
if _, err := d.Token(); err != io.EOF {
|
||||
return v, errResourceState
|
||||
}
|
||||
d = json.NewDecoder(bytes.NewReader(b))
|
||||
d.DisallowUnknownFields()
|
||||
if d.Decode(&v) != nil || v.Version != 1 || v.Updated < 0 || len(v.Rows) > 64 || !hmac.Equal([]byte(v.MAC), []byte(resourceStateMAC(s.key, v))) {
|
||||
return v, errResourceState
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
services := map[string]bool{}
|
||||
for _, r := range v.Rows {
|
||||
id := r.Service + "/" + r.Resource
|
||||
fp, err := hex.DecodeString(r.Fingerprint)
|
||||
if !resourceID.MatchString(r.Service) || !resourceID.MatchString(r.Resource) || seen[id] || err != nil || len(fp) != 32 || r.LastCompleted < 0 || r.NextDue < 0 {
|
||||
return v, errResourceState
|
||||
}
|
||||
seen[id] = true
|
||||
services[r.Service] = true
|
||||
if len(services) > 16 || r.LastCompleted > v.Updated || r.LastCompleted > 253402300799999 || (r.Health == "UNKNOWN" && r.NextDue != 0) || (r.Health != "UNKNOWN" && (r.NextDue < r.LastCompleted || r.NextDue-r.LastCompleted > 86400000)) {
|
||||
return v, errResourceState
|
||||
}
|
||||
if r.Health != "UNKNOWN" && r.Health != "HEALTHY" && r.Health != "DEGRADED" {
|
||||
return v, errResourceState
|
||||
}
|
||||
switch r.Outcome {
|
||||
case "unknown", "pass", "dns_failed", "destination_denied", "transfer_failed", "expectation_failed", "timeout", "cancelled":
|
||||
default:
|
||||
return v, errResourceState
|
||||
}
|
||||
if (r.Health == "UNKNOWN" && (r.Outcome != "unknown" || r.LastCompleted != 0 || r.Failures != 0)) || (r.Health == "HEALTHY" && (r.Outcome != "pass" || r.Failures != 0)) || (r.Health == "DEGRADED" && (r.Outcome == "unknown" || r.Outcome == "pass" || r.Failures == 0)) {
|
||||
return v, errResourceState
|
||||
}
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
func (s *resourceMonitorStore) load() (resourceMonitorState, error) {
|
||||
if s.guard() != nil {
|
||||
return resourceMonitorState{}, errResourceState
|
||||
}
|
||||
key, err := s.trustedRead(resourceKeyName, s.keyFile)
|
||||
if err != nil || !hmac.Equal(key, s.key) {
|
||||
return resourceMonitorState{}, errResourceState
|
||||
}
|
||||
b, err := s.trustedRead(resourceStateName, s.stateFile)
|
||||
if errors.Is(err, os.ErrNotExist) && !s.existed {
|
||||
return resourceMonitorState{Version: 1, Rows: []resourceMonitorRow{}}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return resourceMonitorState{}, errResourceState
|
||||
}
|
||||
v, err := s.decode(b)
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
if s.fault != nil {
|
||||
s.fault("loaded", "")
|
||||
}
|
||||
// Authenticated operator repair on the original held inode can be reloaded;
|
||||
// replaced files remain preserved/BLOCKED until deliberately repaired.
|
||||
key, err = s.trustedRead(resourceKeyName, s.keyFile)
|
||||
if err != nil || !hmac.Equal(key, s.key) || s.entryMatches(resourceStateName, s.stateFile) != nil {
|
||||
return v, errResourceState
|
||||
}
|
||||
current, err := s.trustedRead(resourceStateName, s.stateFile)
|
||||
if err != nil || !bytes.Equal(current, b) {
|
||||
return v, errResourceState
|
||||
}
|
||||
s.previous = b
|
||||
return v, nil
|
||||
}
|
||||
func (s *resourceMonitorStore) save(v resourceMonitorState) error {
|
||||
if s.snapshotGuard() != nil {
|
||||
return errResourceState
|
||||
}
|
||||
v.MAC = resourceStateMAC(s.key, v)
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
if _, err := s.decode(b); err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
// Keep the staged inode open from creation through publication; reopening a
|
||||
// path after rename must never adopt a foreign replacement as our own inode.
|
||||
f, err := os.CreateTemp(s.entryPath(""), ".resource-monitor-stage-")
|
||||
if err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
stage := f.Name()
|
||||
stageGuard := func() bool {
|
||||
if s.entryMatches(filepath.Base(stage), f) != nil {
|
||||
return false
|
||||
}
|
||||
contents, err := io.ReadAll(io.NewSectionReader(f, 0, 65537))
|
||||
// Exact bytes bind the entire validated update, including its HMAC;
|
||||
// truncation, append and in-place edits must fail before publication.
|
||||
return err == nil && bytes.Equal(contents, b) && s.entryMatches(filepath.Base(stage), f) == nil
|
||||
}
|
||||
adopted := false
|
||||
defer func() {
|
||||
// Delete only the known staged inode, never an operator replacement.
|
||||
info, err := os.Lstat(stage)
|
||||
held, heldErr := f.Stat()
|
||||
if err == nil && heldErr == nil && os.SameFile(info, held) && resourcePrivateEntry(f) == nil {
|
||||
contents, readErr := io.ReadAll(io.NewSectionReader(f, 0, 65537))
|
||||
if readErr == nil && bytes.Equal(contents, b) {
|
||||
os.Remove(stage)
|
||||
}
|
||||
}
|
||||
if !adopted {
|
||||
f.Close()
|
||||
}
|
||||
}()
|
||||
if _, err := f.Write(b); err != nil {
|
||||
return errResourceState
|
||||
}
|
||||
if f.Sync() != nil {
|
||||
return errResourceState
|
||||
}
|
||||
if s.fault != nil {
|
||||
s.fault("staged", stage)
|
||||
}
|
||||
if s.snapshotGuard() != nil || !stageGuard() {
|
||||
return errResourceState
|
||||
}
|
||||
if s.fault != nil {
|
||||
s.fault("publish", stage)
|
||||
}
|
||||
// Detect benign/operator replacements immediately before publication. An
|
||||
// actively hostile same-UID actor racing between this final check and rename
|
||||
// is outside the guarantee: POSIX rename has no inode compare-and-swap.
|
||||
if s.existed {
|
||||
if s.snapshotGuard() != nil || !stageGuard() {
|
||||
return errResourceState
|
||||
}
|
||||
if os.Rename(stage, s.entryPath(resourceStateName)) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
} else {
|
||||
// Exclusive initial publication cannot clobber a newly appeared entry.
|
||||
key, err := s.trustedRead(resourceKeyName, s.keyFile)
|
||||
if err != nil || !hmac.Equal(key, s.key) || s.guard() != nil || !stageGuard() {
|
||||
return errResourceState
|
||||
}
|
||||
if os.Link(stage, s.entryPath(resourceStateName)) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
stageInfo, stageErr := os.Lstat(stage)
|
||||
heldInfo, heldErr := f.Stat()
|
||||
if stageErr != nil || heldErr != nil || !os.SameFile(stageInfo, heldInfo) {
|
||||
return errResourceState
|
||||
}
|
||||
if os.Remove(stage) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
}
|
||||
if s.entryMatches(resourceStateName, f) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
published, err := s.trustedRead(resourceStateName, f)
|
||||
if err != nil || !bytes.Equal(published, b) {
|
||||
return errResourceState
|
||||
}
|
||||
old := s.stateFile
|
||||
s.stateFile, s.previous, s.existed, adopted = f, b, true, true
|
||||
var closeErr error
|
||||
if old != nil {
|
||||
closeErr = old.Close()
|
||||
}
|
||||
if errors.Join(closeErr, s.dir.Sync()) != nil {
|
||||
return errResourceState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func resourceMonitorFixture(t *testing.T) (resourceMonitorSource, []byte) {
|
||||
t.Helper()
|
||||
key := bytes.Repeat([]byte{7}, 32)
|
||||
path := resourceCheckFile(t, resourceCheckExample)
|
||||
s, err := readResourceMonitorSource(path, resourceCheckLookup, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s, key
|
||||
}
|
||||
|
||||
func TestResourceMonitorSchedule(t *testing.T) {
|
||||
source, key := resourceMonitorFixture(t)
|
||||
a, b, cooldown := 1000, 4000, 200
|
||||
source.config.Services[0].Resources[0].IntervalMS = &a
|
||||
source.config.Services[0].Resources[0].CooldownMS = &cooldown
|
||||
source.config.Services[1].Resources[0].IntervalMS = &b
|
||||
state := resourceMonitorState{Version: 1}
|
||||
now := time.UnixMilli(10000)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var visits []string
|
||||
var elapsed []int64
|
||||
d := resourceMonitorDeps{key: key, now: func() time.Time { return now }, source: func() (resourceMonitorSource, error) { return source, nil }, guard: func() error { return nil }, load: func() (resourceMonitorState, error) { return state, nil }, save: func(v resourceMonitorState) error { state = v; return nil }, report: func(bool) {}, wait: func(_ context.Context, d time.Duration) error { now = now.Add(d); return nil }}
|
||||
d.probe = func(_ context.Context, service string, _ resourceCheckSpec, _ resourceMonitorSource, guard func() error) (string, error) {
|
||||
if err := guard(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
visits = append(visits, service)
|
||||
elapsed = append(elapsed, now.UnixMilli()-10000)
|
||||
if len(visits) == 8 {
|
||||
cancel()
|
||||
}
|
||||
if len(visits) == 1 {
|
||||
return "timeout", nil
|
||||
}
|
||||
return "pass", nil
|
||||
}
|
||||
if err := runResourceMonitor(ctx, d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(visits, []string{"first", "second", "first", "first", "first", "first", "second", "first"}) || !reflect.DeepEqual(elapsed, []int64{0, 0, 200, 1200, 2200, 3200, 4000, 4200}) {
|
||||
t.Fatalf("unfair/due schedule: %v %v", visits, elapsed)
|
||||
}
|
||||
if state.Rows[0].Health != "HEALTHY" || state.Rows[0].Failures != 0 || state.Rows[1].NextDue != 18000 {
|
||||
t.Fatal(state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorBlockedRepairAndPersistenceFailure(t *testing.T) {
|
||||
for _, kind := range []string{"source", "ownership", "storage", "save"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
source, key := resourceMonitorFixture(t)
|
||||
state := resourceMonitorState{Version: 1, MAC: "initial"}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
tick, probes := 0, 0
|
||||
d := resourceMonitorDeps{key: key, now: func() time.Time { return time.UnixMilli(10000 + int64(tick)*1000) }, report: func(bool) {}, wait: func(_ context.Context, _ time.Duration) error {
|
||||
tick++
|
||||
if tick == 4 {
|
||||
cancel()
|
||||
}
|
||||
return nil
|
||||
}, source: func() (resourceMonitorSource, error) {
|
||||
if kind == "source" && tick < 2 {
|
||||
return resourceMonitorSource{}, errResourceConfig
|
||||
}
|
||||
return source, nil
|
||||
}, guard: func() error {
|
||||
if kind == "ownership" && tick < 2 {
|
||||
return errResourceConfig
|
||||
}
|
||||
return nil
|
||||
}, load: func() (resourceMonitorState, error) {
|
||||
if kind == "storage" && tick < 2 {
|
||||
return state, errResourceState
|
||||
}
|
||||
return state, nil
|
||||
}, save: func(v resourceMonitorState) error {
|
||||
if kind == "save" {
|
||||
return errResourceState
|
||||
}
|
||||
state = v
|
||||
return nil
|
||||
}}
|
||||
d.probe = func(_ context.Context, _ string, _ resourceCheckSpec, _ resourceMonitorSource, g func() error) (string, error) {
|
||||
if err := g(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
probes++
|
||||
return "pass", nil
|
||||
}
|
||||
if err := runResourceMonitor(ctx, d); (err != nil) != (kind == "save") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kind == "save" {
|
||||
if probes != 1 {
|
||||
t.Fatalf("probed after persistence failure: %d", probes)
|
||||
}
|
||||
} else if probes != 2 {
|
||||
t.Fatalf("did not resume safely: %d", probes)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorConfigReconcilePrivacy(t *testing.T) {
|
||||
source, key := resourceMonitorFixture(t)
|
||||
rows := reconcileResourceRows(key, source, nil)
|
||||
rows[0].Health, rows[0].Outcome, rows[0].LastCompleted = "HEALTHY", "pass", 10000
|
||||
rows[1].Health, rows[1].Outcome, rows[1].LastCompleted = "HEALTHY", "pass", 10000
|
||||
old := rows[1].Fingerprint
|
||||
source.config.Services[0].Resources[0].URL = "https://example.com/changed-secret"
|
||||
changed := reconcileResourceRows(key, source, rows)
|
||||
if changed[0].Health != "UNKNOWN" || changed[1].Health != "HEALTHY" || changed[1].Fingerprint != old {
|
||||
t.Fatal(changed)
|
||||
}
|
||||
source.config.Services = source.config.Services[1:]
|
||||
if got := reconcileResourceRows(key, source, changed); len(got) != 1 || got[0].Service != "second" {
|
||||
t.Fatal(got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorStore(t *testing.T) {
|
||||
source, key := resourceMonitorFixture(t)
|
||||
dir := t.TempDir()
|
||||
if err := os.Chmod(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := openResourceMonitorStore(dir, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key = s.key
|
||||
state, err := s.load()
|
||||
if err != nil || len(state.Rows) != 0 {
|
||||
t.Fatal(state, err)
|
||||
}
|
||||
state.Updated = 10000
|
||||
state.Rows = reconcileResourceRows(key, source, nil)
|
||||
if err := s.save(state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.close()
|
||||
s, err = openResourceMonitorStore(dir, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.close()
|
||||
loaded, err := s.load()
|
||||
if err != nil || !reflect.DeepEqual(loaded.Rows, state.Rows) {
|
||||
t.Fatal(loaded, err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(dir, resourceStateName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, secret := range []string{"example.com", "private-token", "secret=value", "example.org", "Authorization"} {
|
||||
if bytes.Contains(data, []byte(secret)) {
|
||||
t.Fatal("secret serialized")
|
||||
}
|
||||
}
|
||||
for _, name := range []string{resourceStateName, resourceKeyName} {
|
||||
info, err := os.Stat(filepath.Join(dir, name))
|
||||
if err != nil || info.Mode().Perm() != 0600 {
|
||||
t.Fatal(info, err)
|
||||
}
|
||||
}
|
||||
// Corruption is preserved; restoring a valid signed version resumes loading.
|
||||
bad := []byte(`{"version":99}`)
|
||||
if err := os.WriteFile(filepath.Join(dir, resourceStateName), bad, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.load(); err == nil {
|
||||
t.Fatal("corrupt state trusted")
|
||||
}
|
||||
if err := s.save(state); err == nil {
|
||||
t.Fatal("corrupt state overwritten")
|
||||
}
|
||||
got, _ := os.ReadFile(filepath.Join(dir, resourceStateName))
|
||||
if !bytes.Equal(got, bad) {
|
||||
t.Fatal("evidence changed")
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, resourceStateName), data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.load(); err != nil {
|
||||
t.Fatal("valid repair refused", err)
|
||||
}
|
||||
if err := os.Remove(filepath.Join(dir, resourceKeyName)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := openResourceMonitorStore(dir, true); err == nil {
|
||||
t.Fatal("missing key silently regenerated")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorUnsafeStorePreserved(t *testing.T) {
|
||||
for _, kind := range []string{"symlink", "hardlink", "permissions", "wrong-owner", "directory-replaced"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "state")
|
||||
if err := os.Mkdir(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := openResourceMonitorStore(dir, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.close()
|
||||
path := filepath.Join(dir, resourceKeyName)
|
||||
original, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch kind {
|
||||
case "symlink":
|
||||
if err := os.Rename(path, path+"-original"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(path+"-original", path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "hardlink":
|
||||
if err := os.Link(path, path+"-alias"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "permissions":
|
||||
if err := os.Chmod(path, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "wrong-owner":
|
||||
if os.Geteuid() != 0 {
|
||||
t.Skip("requires chown capability")
|
||||
}
|
||||
if err := os.Chown(path, 12345, -1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "directory-replaced":
|
||||
if err := os.Rename(dir, dir+"-old"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Mkdir(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path = filepath.Join(dir+"-old", resourceKeyName)
|
||||
}
|
||||
if _, err := s.load(); err == nil {
|
||||
t.Fatal("unsafe storage accepted")
|
||||
}
|
||||
if err := s.save(resourceMonitorState{Version: 1}); err == nil {
|
||||
t.Fatal("unsafe storage overwritten")
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil || !bytes.Equal(data, original) {
|
||||
t.Fatal("key changed", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorReloadBeforeDNSAndTransfer(t *testing.T) {
|
||||
for _, phase := range []string{"before-dns", "after-dns", "replace-after-dns", "env-after-dns", "owner-after-dns"} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
key := bytes.Repeat([]byte{1}, 32)
|
||||
path := resourceCheckFile(t, resourceCheckExample)
|
||||
envChanged, ownerLost := false, false
|
||||
lookup := func(k string) (string, bool) {
|
||||
if envChanged && k == "CHECK_TOKEN" {
|
||||
return "changed secret", true
|
||||
}
|
||||
return resourceCheckLookup(k)
|
||||
}
|
||||
source, err := readResourceMonitorSource(path, lookup, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transfers, dnsCalls := 0, 0
|
||||
guard := func() error {
|
||||
if ownerLost {
|
||||
return errResourceConfig
|
||||
}
|
||||
fresh, err := readResourceMonitorSource(path, lookup, key)
|
||||
if err != nil || fresh.fingerprint != source.fingerprint {
|
||||
return errResourceConfig
|
||||
}
|
||||
return nil
|
||||
}
|
||||
mutate := func() {
|
||||
switch phase {
|
||||
case "env-after-dns":
|
||||
envChanged = true
|
||||
case "owner-after-dns":
|
||||
ownerLost = true
|
||||
case "replace-after-dns":
|
||||
data := strings.Replace(resourceCheckExample, "secret=value", "secret=rotated", 1)
|
||||
if err := os.WriteFile(path+"-next", []byte(data), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(path+"-next", path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
default:
|
||||
if err := os.WriteFile(path, []byte(`{"version":1}`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if phase == "before-dns" {
|
||||
mutate()
|
||||
}
|
||||
check := resourceCheckDeps{resolve: func(context.Context, string, string) ([]net.IP, error) {
|
||||
dnsCalls++
|
||||
mutate()
|
||||
return []net.IP{net.ParseIP("93.184.216.34")}, nil
|
||||
}, runner: func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { transfers++; return nil }}
|
||||
_, err = probeResourceMonitor(context.Background(), "first", source.config.Services[0].Resources[0], source, guard, check)
|
||||
if err == nil || transfers != 0 || (phase == "before-dns" && dnsCalls != 0) {
|
||||
t.Fatal("unsafe probe", transfers, dnsCalls, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorBothTLSAndCancellationJoin(t *testing.T) {
|
||||
source, _ := resourceMonitorFixture(t)
|
||||
spec := source.config.Services[1].Resources[0]
|
||||
check := resourceCheckDeps{resolve: func(context.Context, string, string) ([]net.IP, error) {
|
||||
return []net.IP{net.ParseIP("93.184.216.34")}, nil
|
||||
}}
|
||||
calls := 0
|
||||
check.runner = func(_ context.Context, args []string, _ io.Reader, out, diag io.Writer) error {
|
||||
calls++
|
||||
protocol := "tls12"
|
||||
mode := ""
|
||||
if strings.Contains(strings.Join(args, " "), "--tlsv1.3") {
|
||||
protocol = "tls13"
|
||||
mode = "verify"
|
||||
}
|
||||
configuredWire(spec.URL, spec.Method, protocol, mode, out, diag)
|
||||
return nil
|
||||
}
|
||||
outcome, err := probeResourceMonitor(context.Background(), "second", spec, source, func() error { return nil }, check)
|
||||
if err != nil || outcome == "pass" || calls != 2 {
|
||||
t.Fatal(outcome, calls, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
joined := false
|
||||
check.runner = func(ctx context.Context, _ []string, _ io.Reader, _ io.Writer, _ io.Writer) error {
|
||||
cancel()
|
||||
<-ctx.Done()
|
||||
joined = true
|
||||
return ctx.Err()
|
||||
}
|
||||
outcome, err = probeResourceMonitor(ctx, "second", spec, source, func() error { return nil }, check)
|
||||
if err != nil || outcome != "cancelled" || !joined {
|
||||
t.Fatal("did not join cancellation", outcome, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorSchemaScheduling(t *testing.T) {
|
||||
for _, field := range []string{"interval_ms", "cooldown_ms"} {
|
||||
for _, value := range []string{"0", "-1", "99", "86400001", "null", "\"1000\""} {
|
||||
data := strings.Replace(resourceCheckExample, `"timeout_ms":1000`, `"timeout_ms":1000,"`+field+`":`+value, 1)
|
||||
if _, err := parseResourceChecks([]byte(data), resourceCheckLookup); err == nil {
|
||||
t.Fatal(field, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorSingletonAndGenericDispatch(t *testing.T) {
|
||||
d, env, _ := managedCLIFixture(t)
|
||||
lock, err := acquireManagedMonitorSingleton(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lock.close()
|
||||
if _, err := acquireManagedMonitorSingleton(d); !errors.Is(err, syscall.EWOULDBLOCK) {
|
||||
t.Fatal("singleton conflict ignored", err)
|
||||
}
|
||||
// Generic dispatch precedes invalid YouTube/adaptive settings. A cancelled
|
||||
// generic startup needs no runtime/network, database or valid source JSON.
|
||||
env["SOCKS_PORT"] = "1080"
|
||||
env["ZAPRET_AUTOTUNE_SOCKS_PROXY"] = "socks5h://127.0.0.1:1080"
|
||||
env["ZAPRET_AUTOTUNE_RESOURCE_CONFIG"] = "/missing-invalid-config"
|
||||
env["ZAPRET_AUTOTUNE_VIDEO_ID"] = "bad"
|
||||
env["ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE"] = "invalid"
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
var out, diag bytes.Buffer
|
||||
code := managedMonitorCLIContext(ctx, nil, &out, &diag, d, func(context.Context, managedMonitorDeps) error { t.Fatal("YouTube supervisor invoked"); return nil })
|
||||
if code != 0 || !strings.Contains(out.String(), "generic resource monitor") {
|
||||
t.Fatal(code, out.String(), diag.String())
|
||||
}
|
||||
delete(env, "ZAPRET_AUTOTUNE_RESOURCE_CONFIG")
|
||||
if code := managedMonitorCLIContext(ctx, nil, &out, &diag, d, nil); code != 64 {
|
||||
t.Fatal("legacy invalid config no longer rejected", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorInvalidInitialConfigAliveAndTERM(t *testing.T) {
|
||||
d, env, _ := managedCLIFixture(t)
|
||||
env["SOCKS_PORT"] = "1080"
|
||||
env["ZAPRET_AUTOTUNE_SOCKS_PROXY"] = "socks5h://127.0.0.1:1080"
|
||||
d.verifySOCKS = func(int) error { return nil }
|
||||
initManagedCLI(t, d)
|
||||
path := resourceCheckFile(t, `{"invalid":true}`)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
started := make(chan struct{}, 1)
|
||||
check := resourceCheckDeps{guard: func() error {
|
||||
select {
|
||||
case started <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
}, resolve: func(context.Context, string, string) ([]net.IP, error) {
|
||||
t.Error("DNS with invalid source")
|
||||
return nil, nil
|
||||
}, runner: func(context.Context, []string, io.Reader, io.Writer, io.Writer) error {
|
||||
t.Error("transfer with invalid source")
|
||||
return nil
|
||||
}}
|
||||
var out, diag bytes.Buffer
|
||||
done := make(chan int, 1)
|
||||
go func() { done <- resourceMonitorCLIWithCheck(ctx, path, &out, &diag, d, check) }()
|
||||
// Source error short-circuits the runtime guard, so wait for authenticated
|
||||
// BLOCKED persistence rather than relying on a diagnostic log race.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
s, err := openResourceMonitorStore(d.directory, false)
|
||||
if err == nil {
|
||||
state, err := s.load()
|
||||
s.close()
|
||||
if err == nil && state.Blocked {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
select {
|
||||
case code := <-done:
|
||||
t.Fatalf("invalid source terminated controller: %d", code)
|
||||
default:
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case code := <-done:
|
||||
if code != 0 {
|
||||
t.Fatal(code, diag.String())
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("TERM did not join controller")
|
||||
}
|
||||
if !strings.Contains(diag.String(), "BLOCKED") {
|
||||
t.Fatal("no blocked status", diag.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorConcurrentEnvironmentReload(t *testing.T) {
|
||||
path := resourceCheckFile(t, resourceCheckExample)
|
||||
t.Setenv("CHECK_URL", "https://example.org/private")
|
||||
t.Setenv("CHECK_TOKEN", "initial-secret")
|
||||
t.Setenv("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5h://127.0.0.1:1080")
|
||||
key := bytes.Repeat([]byte{3}, 32)
|
||||
baseline, err := readResourceMonitorSource(path, os.LookupEnv, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for i := 0; i < 100; i++ {
|
||||
os.Setenv("CHECK_TOKEN", "rotated-secret")
|
||||
os.Setenv("CHECK_TOKEN", "initial-secret")
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 100; i++ {
|
||||
s, err := readResourceMonitorSource(path, os.LookupEnv, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.fingerprint == "" {
|
||||
t.Fatal("empty fingerprint")
|
||||
}
|
||||
}
|
||||
<-done
|
||||
if err := os.Setenv("CHECK_TOKEN", "rotated-secret"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fresh, err := readResourceMonitorSource(path, os.LookupEnv, key)
|
||||
if err != nil || fresh.fingerprint == baseline.fingerprint {
|
||||
t.Fatal("secret rotation not bound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMonitorTargetConfigBoundary(t *testing.T) {
|
||||
_, env, _ := managedCLIFixture(t)
|
||||
for k, v := range env {
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
t.Setenv("SOCKS_PORT", "1080")
|
||||
t.Setenv("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5h://127.0.0.1:1080")
|
||||
t.Setenv("ZAPRET_AUTOTUNE_RESOURCE_CONFIG", resourceCheckFile(t, `{"invalid":true}`))
|
||||
t.Setenv("ZAPRET_AUTOTUNE_VIDEO_ID", "invalid")
|
||||
t.Setenv("ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE", "invalid")
|
||||
if code := run([]string{"target-config-check"}, io.Discard, io.Discard); code != 0 {
|
||||
t.Fatal("bad resource JSON/YouTube blocked safe generic startup", code)
|
||||
}
|
||||
t.Setenv("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5h://foreign.example:1080")
|
||||
if code := run([]string{"target-config-check"}, io.Discard, io.Discard); code != 64 {
|
||||
t.Fatal("bad proxy allowed", code)
|
||||
}
|
||||
t.Setenv("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5h://127.0.0.1:1080")
|
||||
t.Setenv("ZAPRET_AUTOTUNE_DNS_MODE", "invalid")
|
||||
if code := run([]string{"target-config-check"}, io.Discard, io.Discard); code != 64 {
|
||||
t.Fatal("bad DNS allowed", code)
|
||||
}
|
||||
t.Setenv("ZAPRET_AUTOTUNE_DNS_MODE", "doh")
|
||||
t.Setenv("ZAPRET_AUTOTUNE_TARGET_ROLE", "foreign")
|
||||
if code := run([]string{"target-config-check"}, io.Discard, io.Discard); code != 64 {
|
||||
t.Fatal("bad target role allowed", code)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user