fix(runtime): keep supervision and absence retries in container
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
226d93e2aa
commit
3f9fd2f36f
21 files changed
+1056
-137
No files matched your search
@@ -160,11 +160,11 @@ func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand
|
||||
return managedRecoveryDeps{}, fmt.Errorf("complete managed process adapters required")
|
||||
}
|
||||
// The blockcheck process-group runner cancels and waits descendants.
|
||||
scan := func(ctx context.Context, _ string, host string, _, _ io.Writer) (string, error) {
|
||||
return scanCommand(ctx, "/usr/bin/blockcheckw", host, io.Discard, io.Discard)
|
||||
scan := func(ctx context.Context, _ string, host string, _ io.Writer, diagnostics io.Writer) (string, error) {
|
||||
return scanCommand(ctx, "/usr/bin/blockcheckw", host, io.Discard, diagnostics)
|
||||
}
|
||||
check := func(ctx context.Context, _ string, path, args, host string, _, _ io.Writer) error {
|
||||
return checkCommand(ctx, "/usr/bin/blockcheckw", path, args, host, io.Discard, io.Discard)
|
||||
check := func(ctx context.Context, _ string, path, args, host string, _ io.Writer, diagnostics io.Writer) error {
|
||||
return checkCommand(ctx, "/usr/bin/blockcheckw", path, args, host, io.Discard, diagnostics)
|
||||
}
|
||||
run := func(ctx context.Context, path string, args ...string) error {
|
||||
if path != managedZapretScript || len(args) != 1 || (args[0] != "start" && args[0] != "stop" && args[0] != "restart") {
|
||||
|
||||
@@ -3,11 +3,13 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
@@ -123,7 +125,7 @@ func TestManagedEntrypointPrestartReconciliation(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write("awk", "#!/bin/sh\nexit 0\n")
|
||||
child := "#!/bin/sh\ntrap 'exit 0' TERM INT\nwhile :; do sleep 0.1 & wait $!; done\n"
|
||||
child := "#!/bin/sh\nsleeper=\ntrap '[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; exit 0' TERM INT\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n"
|
||||
write("ss-server", child)
|
||||
write("ss-local", child)
|
||||
log := filepath.Join(root, "log")
|
||||
@@ -137,10 +139,16 @@ func TestManagedEntrypointPrestartReconciliation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry := write("entrypoint", strings.NewReplacer("/run/zapret-autotune-ss", ss, "/usr/local/bin/zapret-autotune", autotune, managedZapretScript, zapret, `"$ZAPRET_CONFIG"`, `"$FIXTURE_CONFIG"`).Replace(string(data)))
|
||||
helper, err := filepath.Abs("../../managed-target-supervision.sh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry := write("entrypoint", strings.NewReplacer("/opt/managed-target-supervision.sh", helper, "/run/zapret-autotune-ss", ss, "/usr/local/bin/zapret-autotune", autotune, managedZapretScript, zapret, `"$ZAPRET_CONFIG"`, `"$FIXTURE_CONFIG"`).Replace(string(data)))
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "/bin/sh", entry)
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
|
||||
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "SS_PASSWORD="+strings.Repeat("a", 32), "MANAGED_PRESTART_HELPER=1", "FIXTURE_BINARY="+binary, "FIXTURE_TARGET="+d.directory, "FIXTURE_SCRIPT="+zapret)
|
||||
for key, value := range env {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
@@ -354,6 +362,125 @@ func TestManagedMonitorProductionAdapterBinding(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorProductionAbsenceDiagnostics(t *testing.T) {
|
||||
for _, mode := range []string{"empty-nfq", "partial-nfq", "combined-nfq", "combined-capture-nfq", "success-scan-nfq", "success-combined-nfq", "artifact-nfq", "empty-timeout", "artifact-timeout", "malformed"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
recovery, _, _ := recoveryFixture(t, false)
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
||||
if err := os.Chmod(recovery.transaction.Dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const infra = "FAIL 0/1 nfqws2 exited immediately (code 1) secret-never-print"
|
||||
adapted, err := managedMonitorRecoveryAdapters(monitor.target, recovery.db,
|
||||
func(_ context.Context, _, host string, out, diagnostics io.Writer) (string, error) {
|
||||
if out != io.Discard || diagnostics == io.Discard {
|
||||
t.Fatal("streams not privately isolated")
|
||||
}
|
||||
// Model exec's independent stdout/stderr writers without sharing a buffer.
|
||||
done := make(chan struct{})
|
||||
go func() { io.WriteString(out, "secret-never-print"); close(done) }()
|
||||
message := "FAIL 0/1 timeout"
|
||||
if strings.Contains(mode, "nfq") || mode == "artifact-timeout" {
|
||||
message = infra
|
||||
}
|
||||
if strings.Contains(mode, "combined") {
|
||||
message = "FAIL 0/1 timeout"
|
||||
}
|
||||
io.WriteString(diagnostics, message)
|
||||
<-done
|
||||
data := []byte(`{"domain":"` + host + `","total":2,"working":0,"strategies":[]}`)
|
||||
if mode == "partial-nfq" {
|
||||
var report struct {
|
||||
Strategies []json.RawMessage `json:"strategies"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(combinedReportFixture("--lua-desync=fake")), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report.Strategies = report.Strategies[:1]
|
||||
partial, marshalErr := json.Marshal(report)
|
||||
if marshalErr != nil {
|
||||
t.Fatal(marshalErr)
|
||||
}
|
||||
data = partial
|
||||
}
|
||||
if strings.Contains(mode, "combined") || mode == "success-scan-nfq" {
|
||||
data = []byte(combinedReportFixture("--lua-desync=fake"))
|
||||
}
|
||||
if mode == "malformed" {
|
||||
data = []byte(`{"strategies":[]}`)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "scan")
|
||||
if mode == "artifact-nfq" || mode == "artifact-timeout" {
|
||||
artifact := infra
|
||||
if mode == "artifact-timeout" {
|
||||
artifact = "FAIL 0/1 timeout"
|
||||
}
|
||||
if err := os.WriteFile(path+".check.json.stderr", []byte(artifact), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return path, os.WriteFile(path+".check.json", data, 0600)
|
||||
},
|
||||
func(_ context.Context, _, path, _, _ string, out, diagnostics io.Writer) error {
|
||||
if out != io.Discard || diagnostics == io.Discard {
|
||||
t.Fatal("check streams not privately isolated")
|
||||
}
|
||||
io.WriteString(diagnostics, infra)
|
||||
if mode != "combined-capture-nfq" {
|
||||
if err := os.WriteFile(path+".combined.json.stderr", []byte(infra), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
data := nonpassingManagedReport(t)
|
||||
if mode == "success-combined-nfq" {
|
||||
data = []byte(combinedReportFixture("--lua-desync=fake"))
|
||||
}
|
||||
return os.WriteFile(path+".combined.json", data, 0600)
|
||||
}, recovery.run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := wireManagedRecovery(&monitor, adapted, func(error) { t.Fatal("unexpected GGC report") }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
monitor.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||
var reported error
|
||||
monitor.reportRecoveryError = func(err error) { reported = err }
|
||||
err = runManagedMonitor(ctx, monitor)
|
||||
var fatal *managedRecoveryFatal
|
||||
wantFatal := strings.Contains(mode, "nfq")
|
||||
if errors.As(err, &fatal) != wantFatal || (!wantFatal && !errors.Is(err, context.Canceled)) {
|
||||
t.Fatalf("mode=%s err=%v", mode, err)
|
||||
}
|
||||
if strings.Contains(errors.Join(err, reported).Error(), "secret-never-print") {
|
||||
t.Fatal("diagnostics leaked")
|
||||
}
|
||||
state, stateErr := readMonitorState(monitor.db, monitor.serviceID)
|
||||
if stateErr != nil {
|
||||
t.Fatal(stateErr)
|
||||
}
|
||||
wantAbsence := mode == "empty-timeout" || mode == "artifact-timeout"
|
||||
if errors.Is(reported, errNoQualifiedStrategy) != wantAbsence || errors.Is(err, errNoQualifiedStrategy) {
|
||||
t.Fatalf("err=%v reported=%v", err, reported)
|
||||
}
|
||||
if wantAbsence {
|
||||
if !state.CooldownUntil.Equal(monitorTestTime.Add(2 * time.Hour)) {
|
||||
t.Fatalf("pause=%v", state.CooldownUntil)
|
||||
}
|
||||
} else if !state.CooldownUntil.IsZero() {
|
||||
t.Fatalf("unexpected pause=%v", state.CooldownUntil)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) {
|
||||
recovery, _, _ := recoveryFixture(t, false)
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
@@ -468,7 +595,7 @@ func TestManagedMonitorBootstrapExistingFrontend(t *testing.T) {
|
||||
// Execute only rewritten target-local contracts and fake programs; no network,
|
||||
// Docker, real credentials, zapret or host firewall. Exercise the actual traps.
|
||||
func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
modes := []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"}
|
||||
modes := []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-early", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"}
|
||||
for _, stage := range []string{"init", "reconcile", "start"} {
|
||||
for _, failure := range []string{"hang", "75", "124", "137", "7"} {
|
||||
modes = append(modes, "startup-"+stage+"-"+failure)
|
||||
@@ -499,21 +626,37 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
write("timeout", "#!/bin/sh\nif [ \"$3\" = 45 ] || { [ \"$3\" = 30 ] && [ \"$5\" = start ]; }; then\nshift 3\nexec \"$REAL_TIMEOUT\" -k 1 1 \"$@\"\nfi\nexec \"$REAL_TIMEOUT\" \"$@\"\n")
|
||||
t.Setenv("REAL_TIMEOUT", timeout)
|
||||
}
|
||||
startupFailure := "case \"$MODE\" in startup-$stage-hang) trap '' TERM INT; while :; do sleep 0.1; done;; startup-$stage-*) exit \"${MODE##*-}\";; esac\n"
|
||||
// Accelerate only restart cooldowns; watchdog deadlines remain real.
|
||||
realSleep, err := exec.LookPath("sleep")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write("sleep", "#!/bin/sh\ncase \"$1\" in 15|30|60) exec \"$REAL_SLEEP\" 0.02;; *) exec \"$REAL_SLEEP\" \"$@\";; esac\n")
|
||||
t.Setenv("REAL_SLEEP", realSleep)
|
||||
startupFailure := "case \"$MODE\" in startup-$stage-hang) trap '' TERM INT; exec sleep 300;; startup-$stage-*) exit \"${MODE##*-}\";; esac\n"
|
||||
write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n")
|
||||
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & wait $!; done\n"
|
||||
joinSleeper := "[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; "
|
||||
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\nsleeper=\ntrap '" + joinSleeper + "echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n"
|
||||
write("ss-server", child)
|
||||
write("ss-local", child)
|
||||
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nstage=$1\n"+startupFailure+"if [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; while :; do sleep 0.1; done; fi\n")
|
||||
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then\necho managed-$2 >> \"$LOG\"\nstage=$2\n"+startupFailure+"exit 0\nfi\ntrap 'echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; fi\necho managed-monitor >> \"$LOG\"\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & wait $!; done\n")
|
||||
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nstage=$1\n"+startupFailure+"if [ \"$MODE:$1\" = ss-early:start ] && [ \"$(grep -c zapret-start \"$LOG\")\" -gt 1 ]; then\nuntil [ \"$(grep -c 'stopped-.*/ss-local' \"$LOG\")\" -ge \"$(grep -c zapret-start \"$LOG\")\" ]; do sleep 0.02; done\nsleep 0.02\nfi\nif [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; exec sleep 300; fi\n")
|
||||
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then\necho managed-$2 >> \"$LOG\"\nstage=$2\n"+startupFailure+"exit 0\nfi\nsleeper=\ntrap '"+joinSleeper+"echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\necho managed-monitor >> \"$LOG\"\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; exec sleep 300; fi\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n")
|
||||
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := strings.NewReplacer("/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
|
||||
helper, err := filepath.Abs("../../managed-target-supervision.sh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := strings.NewReplacer("/opt/managed-target-supervision.sh", helper, "/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
|
||||
if mode == "term-hang" {
|
||||
// Shorten only the fixture's watchdog, not the production contract.
|
||||
text = strings.Replace(text, `bounded_wait "$monitor_pid" 60`, `bounded_wait "$monitor_pid" 1`, 1)
|
||||
shortened := strings.Replace(text, `bounded_wait "$monitor_pid" 45`, `bounded_wait "$monitor_pid" 1`, 1)
|
||||
if shortened == text {
|
||||
t.Fatal("fixture did not shorten monitor watchdog")
|
||||
}
|
||||
text = shortened
|
||||
}
|
||||
if mode == "stop-hang" {
|
||||
text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1)
|
||||
@@ -524,6 +667,21 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
cmd := exec.CommandContext(ctx, "/bin/sh", entry)
|
||||
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "ZAPRET_CONFIG="+filepath.Join(root, "config"), "SS_PASSWORD="+strings.Repeat("a", 32))
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
var shutdownSnapshot string
|
||||
cmd.Cancel = func() error {
|
||||
// Snapshot only this fixture's process group before deadline cleanup;
|
||||
// no environment or private credential/config contents are collected.
|
||||
processes, _ := exec.Command("ps", "-eo", "pid,ppid,pgid,stat,wchan:24,args").Output()
|
||||
for _, line := range strings.Split(string(processes), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 3 && fields[2] == strconv.Itoa(cmd.Process.Pid) {
|
||||
shutdownSnapshot += line + "\n"
|
||||
}
|
||||
}
|
||||
fixtureLog, _ := os.ReadFile(log)
|
||||
shutdownSnapshot += "fixture log:\n" + string(fixtureLog)
|
||||
return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &output, &output
|
||||
if err := cmd.Start(); err != nil {
|
||||
@@ -553,9 +711,11 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
}
|
||||
err = <-done
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("entrypoint did not join children", ctx.Err())
|
||||
t.Fatal("entrypoint did not join children", ctx.Err(), shutdownSnapshot, output.String())
|
||||
}
|
||||
want := 75
|
||||
// Repeated technical failures are supervised inside PID 1; after three
|
||||
// starts the budget fails closed, rather than exporting exit 75.
|
||||
want := 78
|
||||
if strings.HasPrefix(mode, "startup-") {
|
||||
want = 78
|
||||
}
|
||||
@@ -579,6 +739,14 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text = string(b)
|
||||
retrying := mode == "technical" || mode == "panic" || mode == "killed" || mode == "segv" || mode == "zero" || mode == "ss" || mode == "ss-early"
|
||||
starts := strings.Count(text, "zapret-start\n")
|
||||
if retrying && starts != 3 || !retrying && !strings.HasPrefix(mode, "startup-") && starts != 1 {
|
||||
t.Fatal("unexpected internal restart count", starts, text)
|
||||
}
|
||||
if mode == "ss-early" && strings.Count(text, "managed-monitor\n") != 1 {
|
||||
t.Fatal("forked monitor after replacement SS child already exited", text)
|
||||
}
|
||||
if strings.HasPrefix(mode, "startup-") {
|
||||
if strings.Contains(text, "managed-monitor") || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-server")) || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-local")) {
|
||||
t.Fatal("startup failure escaped cleanup", text)
|
||||
|
||||
@@ -107,6 +107,7 @@ func (e *managedRecoveryTimeout) Error() string { return e.Err.Error() }
|
||||
func (e *managedRecoveryTimeout) Unwrap() error { return e.Err }
|
||||
|
||||
var (
|
||||
errNoQualifiedStrategy = errors.New("no fully passing check strategy for both tls12 and tls13")
|
||||
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
|
||||
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
|
||||
errManagedRecoveryDeadline = errors.New("managed recovery total deadline exceeded")
|
||||
@@ -420,6 +421,9 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
||||
if ctx.Err() != nil {
|
||||
return result, e
|
||||
}
|
||||
if errors.Is(e, errNoQualifiedStrategy) {
|
||||
return result, e
|
||||
}
|
||||
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
|
||||
return result, nil
|
||||
}
|
||||
@@ -474,7 +478,13 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos
|
||||
return 0, err
|
||||
}
|
||||
started := time.Now()
|
||||
path, err := d.scan(ctx, "blockcheckw", host, io.Discard, io.Discard)
|
||||
var diagnostics bytes.Buffer
|
||||
path, err := d.scan(ctx, "blockcheckw", host, io.Discard, &diagnostics)
|
||||
// Only validated report evidence, never a runner error, authorizes absence.
|
||||
if errors.Is(err, errNoQualifiedStrategy) || errors.Is(err, errCandidateNonpass) {
|
||||
err = fmt.Errorf("candidate scanner failed: %v", err)
|
||||
}
|
||||
err = managedCheckerError(err, path+".check.json.stderr", diagnostics.Bytes())
|
||||
var report []byte
|
||||
var args, output string
|
||||
if err == nil {
|
||||
@@ -491,9 +501,16 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos
|
||||
}
|
||||
if err == nil {
|
||||
args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, host)
|
||||
if err != nil && emptyManagedScanReport(report, host) {
|
||||
err = errNoQualifiedStrategy
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
report, err = revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, d.check)
|
||||
var fatal *managedRecoveryFatal
|
||||
if !errors.As(err, &fatal) && errors.Is(err, errCandidateNonpass) {
|
||||
err = errNoQualifiedStrategy
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = ctx.Err()
|
||||
@@ -524,6 +541,21 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// A successful owned scan may omit rejected rows. Missing/null metadata is not
|
||||
// absence evidence; qualification still requires the usual full TLS checks.
|
||||
func emptyManagedScanReport(raw []byte, host string) bool {
|
||||
var report struct {
|
||||
Domain string `json:"domain"`
|
||||
Total *int `json:"total"`
|
||||
Working *int `json:"working"`
|
||||
Strategies *[]json.RawMessage `json:"strategies"`
|
||||
}
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
return d.Decode(&report) == nil && d.Decode(new(json.RawMessage)) == io.EOF &&
|
||||
report.Domain == host && report.Total != nil && *report.Total > 0 &&
|
||||
report.Working != nil && *report.Working == 0 && report.Strategies != nil && len(*report.Strategies) == 0
|
||||
}
|
||||
|
||||
// Installs callbacks. runManagedMonitor performs startup
|
||||
// under its singleton. Target guards are repeated before every recovery/probe.
|
||||
func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDeps, reportGGC func(error)) error {
|
||||
@@ -567,6 +599,14 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
||||
if err := transaction.Reconcile(recovery.db); err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
state, err := readMonitorState(monitor.db, monitor.serviceID)
|
||||
if err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
// Reconciliation and identity guards precede even a paused bootstrap.
|
||||
if state.RecoveryRunning || monitor.now().Before(state.CooldownUntil) {
|
||||
return nil
|
||||
}
|
||||
ids, err := transaction.ActiveCandidates(recovery.db)
|
||||
if err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
|
||||
@@ -86,6 +86,202 @@ func TestManagedRecoveryBootstrap(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedSavedGGCInfrastructureCannotAuthorizeFallback(t *testing.T) {
|
||||
for _, artifact := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint("artifact=", artifact), func(t *testing.T) {
|
||||
d, front, _ := recoveryFixture(t, true)
|
||||
managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target.directory = d.db, d.transaction.Dir
|
||||
monitor.threshold = 1
|
||||
if err := os.Chmod(d.transaction.Dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||
if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.scan = func(context.Context, string, string, io.Writer, io.Writer) (string, error) {
|
||||
t.Fatal("infrastructure failure authorized fallback scan")
|
||||
return "", nil
|
||||
}
|
||||
checks := 0
|
||||
d.check = func(_ context.Context, _, path, _, host string, out, diagnostics io.Writer) error {
|
||||
checks++
|
||||
if host != "fresh.googlevideo.com" || out != io.Discard {
|
||||
t.Fatal("unexpected saved check")
|
||||
}
|
||||
const infra = "FAIL 0/1 timeout: nfqws2 exited immediately (code 1) private-secret"
|
||||
if artifact {
|
||||
if err := os.WriteFile(path+".combined.json.stderr", []byte(infra), 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
io.WriteString(diagnostics, infra)
|
||||
}
|
||||
return os.WriteFile(path+".combined.json", nonpassingManagedReport(t), 0600)
|
||||
}
|
||||
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("infrastructure became unresolved GGC") }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
monitor.probe = func(context.Context) error { return errMonitorTestUnhealthy }
|
||||
monitor.reportRecoveryError = func(err error) {
|
||||
var fatal *managedRecoveryFatal
|
||||
if !errors.As(err, &fatal) || errors.Is(err, errNoQualifiedStrategy) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
err := runManagedMonitor(context.Background(), monitor)
|
||||
var fatal *managedRecoveryFatal
|
||||
if !errors.As(err, &fatal) || errors.Is(err, errCandidateNonpass) || strings.Contains(err.Error(), "private-secret") || checks != 1 {
|
||||
t.Fatalf("checks=%d err=%v", checks, err)
|
||||
}
|
||||
state, stateErr := readMonitorState(d.db, monitor.serviceID)
|
||||
if stateErr != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(monitor.cooldown)) {
|
||||
t.Fatalf("%+v %v", state, stateErr)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func nonpassingManagedReport(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var report struct {
|
||||
Strategies []map[string]any `json:"strategies"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(combinedReportFixture("--lua-desync=fake")), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, row := range report.Strategies {
|
||||
row["passes_ok"], row["success_rate"] = 0, 0
|
||||
}
|
||||
raw, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func TestManagedAbsenceClassification(t *testing.T) {
|
||||
for _, mode := range []string{"shortlist", "combined", "malformed", "empty", "empty-valid", "empty-nfq", "empty-wrong-target", "runner"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, _, _ := recoveryFixture(t, false)
|
||||
raw := nonpassingManagedReport(t)
|
||||
d.scan = func(_ context.Context, _, host string, _, diagnostics io.Writer) (string, error) {
|
||||
if mode == "runner" {
|
||||
return "", errCandidateNonpass
|
||||
}
|
||||
data := raw
|
||||
switch mode {
|
||||
case "combined":
|
||||
data = []byte(combinedReportFixture("--lua-desync=fake"))
|
||||
case "malformed":
|
||||
data = []byte(`{"strategies":[{"protocol":"tls12"}]}`)
|
||||
case "empty":
|
||||
data = []byte(`{"strategies":[]}`)
|
||||
case "empty-valid", "empty-nfq", "empty-wrong-target":
|
||||
if mode == "empty-nfq" {
|
||||
fmt.Fprintln(diagnostics, "nfqws2 exited with status 1")
|
||||
}
|
||||
if mode == "empty-wrong-target" {
|
||||
host = "untrusted.example"
|
||||
}
|
||||
data = []byte(fmt.Sprintf(`{"domain":%q,"total":10,"working":0,"strategies":[]}`, host))
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "scan")
|
||||
return path, os.WriteFile(path+".check.json", data, 0600)
|
||||
}
|
||||
d.check = func(_ context.Context, _, path, _, _ string, _, _ io.Writer) error {
|
||||
return os.WriteFile(path+".combined.json", raw, 0600)
|
||||
}
|
||||
_, err := recoverManagedTarget(context.Background(), d)
|
||||
want := mode == "shortlist" || mode == "combined" || mode == "empty-valid"
|
||||
if err == nil || errors.Is(err, errNoQualifiedStrategy) != want {
|
||||
t.Fatalf("classification: %v", err)
|
||||
}
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(err, &fatal) && mode != "empty-nfq" {
|
||||
t.Fatalf("unexpected fatal: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedBootstrapAbsencePauseRestartAndExpiry(t *testing.T) {
|
||||
d, _, _ := recoveryFixture(t, false)
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target.directory = d.db, d.transaction.Dir
|
||||
if err := os.Chmod(d.transaction.Dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||
if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := monitorTestTime
|
||||
monitor.now = func() time.Time { return now }
|
||||
scans := 0
|
||||
d.scan = func(_ context.Context, _, _ string, _, _ io.Writer) (string, error) {
|
||||
scans++
|
||||
path := filepath.Join(t.TempDir(), "scan")
|
||||
return path, os.WriteFile(path+".check.json", nonpassingManagedReport(t), 0600)
|
||||
}
|
||||
d.health = func(context.Context) (string, error) { return "UNHEALTHY", nil }
|
||||
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("unexpected GGC error") }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
monitor.reportRecoveryError = func(err error) {
|
||||
if !errors.Is(err, errNoQualifiedStrategy) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for phase, elapsed := range []time.Duration{0, time.Hour, 2 * time.Hour} {
|
||||
now = monitorTestTime.Add(elapsed)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
probes := 0
|
||||
monitor.wait = func(ctx context.Context, _ time.Duration) error {
|
||||
probes++
|
||||
if probes == 4 {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel()
|
||||
wantScans := 1
|
||||
if phase == 2 {
|
||||
wantScans = 2
|
||||
}
|
||||
state, err := readMonitorState(monitor.db, monitor.serviceID)
|
||||
if err != nil || scans != wantScans || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(time.Duration(wantScans)*2*time.Hour)) {
|
||||
t.Fatalf("phase=%d scans=%d state=%+v err=%v", phase, scans, state, err)
|
||||
}
|
||||
// Reopen the same durable database before the next monitor instance.
|
||||
var dbPath string
|
||||
var seq int
|
||||
var name string
|
||||
if err := monitor.db.QueryRow(`PRAGMA database_list`).Scan(&seq, &name, &dbPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := monitor.db.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := openStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { reopened.Close() })
|
||||
monitor.db, d.db = reopened, reopened
|
||||
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("unexpected GGC error") }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryJobDeadline(t *testing.T) {
|
||||
for _, mode := range []string{"bootstrap-scan", "bootstrap-saved", "periodic-saved", "periodic-apply", "periodic-ggc", "parent-cancel", "parent-deadline"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
|
||||
@@ -198,8 +198,8 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
|
||||
if err := validateMonitorTime(serviceID, expires); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
|
||||
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), serviceID, token)
|
||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = MAX(COALESCE(cooldown_until, ?), ?)
|
||||
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), expires.UnixNano(), serviceID, token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -210,6 +210,23 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
|
||||
return err
|
||||
}
|
||||
|
||||
// Extend bootstrap backoff without changing probes or any recovery ownership.
|
||||
func pauseMonitorRecovery(db *sql.DB, serviceID string, now time.Time, cooldown time.Duration) error {
|
||||
if err := validateMonitorTime(serviceID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if cooldown <= 0 {
|
||||
return fmt.Errorf("recovery cooldown must be positive")
|
||||
}
|
||||
expires := now.Add(cooldown)
|
||||
if err := validateMonitorTime(serviceID, expires); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := db.Exec(`INSERT INTO monitor_state (service_id, cooldown_until) VALUES (?, ?)
|
||||
ON CONFLICT(service_id) DO UPDATE SET cooldown_until = MAX(COALESCE(monitor_state.cooldown_until, excluded.cooldown_until), excluded.cooldown_until)`, serviceID, expires.UnixNano())
|
||||
return err
|
||||
}
|
||||
|
||||
// Startup policy only: call for each configured service before monitoring.
|
||||
// A legacy/indeterminate owner is an error, never permission to unlock.
|
||||
// The conditional write also protects against a finish/reacquire during the read.
|
||||
@@ -251,9 +268,9 @@ func reconcileMonitorRecovery(db *sql.DB, serviceID string, cooldown time.Durati
|
||||
}
|
||||
}
|
||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL,
|
||||
recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
|
||||
recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = MAX(COALESCE(cooldown_until, ?), ?)
|
||||
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?
|
||||
AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), serviceID, token, pid.Int64, start.Int64)
|
||||
AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), expires.UnixNano(), serviceID, token, pid.Int64, start.Int64)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
@@ -14,6 +14,16 @@ import (
|
||||
|
||||
const managedMonitorLockName = ".autotune-monitor-lock"
|
||||
|
||||
const noQualifiedStrategyCooldown = 2 * time.Hour
|
||||
|
||||
func managedRecoveryCooldown(err error, configured time.Duration) time.Duration {
|
||||
var fatal *managedRecoveryFatal
|
||||
if !errors.As(err, &fatal) && errors.Is(err, errNoQualifiedStrategy) && configured < noQualifiedStrategyCooldown {
|
||||
return noQualifiedStrategyCooldown
|
||||
}
|
||||
return configured
|
||||
}
|
||||
|
||||
// Callbacks must honor ctx and join
|
||||
// their children before returning; the supervisor never detaches callbacks.
|
||||
type managedMonitorDeps struct {
|
||||
@@ -130,6 +140,11 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if errors.Is(err, errNoQualifiedStrategy) {
|
||||
if err := pauseMonitorRecovery(d.db, d.serviceID, d.now(), managedRecoveryCooldown(err, d.cooldown)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Bootstrap scans can validly find no qualified strategy. Keep the
|
||||
// supervisor alive so durable probes/cooldown govern the next attempt.
|
||||
d.reportRecoveryError(err)
|
||||
@@ -157,8 +172,9 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
||||
}
|
||||
if token != "" {
|
||||
recoveryErr := d.recover(ctx)
|
||||
cooldown := managedRecoveryCooldown(recoveryErr, d.cooldown)
|
||||
// Finalize even on cancellation, using the non-context DB operation.
|
||||
if err := finishMonitorRecovery(d.db, d.serviceID, token, d.now(), d.cooldown); err != nil {
|
||||
if err := finishMonitorRecovery(d.db, d.serviceID, token, d.now(), cooldown); err != nil {
|
||||
return err
|
||||
}
|
||||
if recoveryErr != nil {
|
||||
|
||||
@@ -11,6 +11,95 @@ import (
|
||||
|
||||
var errMonitorTestUnhealthy = errors.New("unhealthy")
|
||||
|
||||
func TestManagedMonitorBootstrapConfiguredLongerPause(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
d.cooldown = 4 * time.Hour
|
||||
d.bootstrap = func(context.Context) error { return errNoQualifiedStrategy }
|
||||
d.reportRecoveryError = func(err error) {
|
||||
if !errors.Is(err, errNoQualifiedStrategy) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := readMonitorState(d.db, d.serviceID)
|
||||
if err != nil || !state.CooldownUntil.Equal(monitorTestTime.Add(4*time.Hour)) {
|
||||
t.Fatalf("%+v %v", state, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorPeriodicAbsenceCooldown(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
d.threshold = 1
|
||||
now := monitorTestTime
|
||||
d.now = func() time.Time { return now }
|
||||
d.probe = func(context.Context) error { return errMonitorTestUnhealthy }
|
||||
recoveries := 0
|
||||
d.recover = func(context.Context) error { recoveries++; return errNoQualifiedStrategy }
|
||||
d.reportRecoveryError = func(err error) {
|
||||
if !errors.Is(err, errNoQualifiedStrategy) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
waits := 0
|
||||
d.wait = func(ctx context.Context, _ time.Duration) error {
|
||||
waits++
|
||||
state, err := readMonitorState(d.db, d.serviceID)
|
||||
want := monitorTestTime.Add(2 * time.Hour)
|
||||
if waits == 3 {
|
||||
want = monitorTestTime.Add(4 * time.Hour)
|
||||
}
|
||||
if err != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(want) {
|
||||
t.Fatalf("%+v %v", state, err)
|
||||
}
|
||||
if waits == 3 {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
now = now.Add(time.Hour)
|
||||
return nil
|
||||
}
|
||||
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recoveries != 2 {
|
||||
t.Fatalf("recoveries=%d", recoveries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorPausePreservesLongerCooldownAndLease(t *testing.T) {
|
||||
d := monitorRuntimeFixture(t)
|
||||
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime)
|
||||
if err != nil || token == "" {
|
||||
t.Fatalf("token=%s err=%v", token, err)
|
||||
}
|
||||
for _, duration := range []time.Duration{4 * time.Hour, 2 * time.Hour} {
|
||||
if err := pauseMonitorRecovery(d.db, d.serviceID, monitorTestTime, duration); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
state, err := readMonitorState(d.db, d.serviceID)
|
||||
if err != nil || !state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(4*time.Hour)) {
|
||||
t.Fatalf("%+v %v", state, err)
|
||||
}
|
||||
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, 30*time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err = readMonitorState(d.db, d.serviceID)
|
||||
if err != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(4*time.Hour)) {
|
||||
t.Fatalf("%+v %v", state, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise the production process reader under the real singleton. PID reuse
|
||||
// proves the old owner is gone and clears only that lease; a live owner stays
|
||||
// busy, while an owner without v5 identity makes startup fail closed.
|
||||
@@ -307,7 +396,7 @@ func TestManagedMonitorStartupLeaseAfterReopenStartsCrashCooldown(t *testing.T)
|
||||
if err != nil || oldToken == "" {
|
||||
t.Fatalf("token=%q err=%v", oldToken, err)
|
||||
}
|
||||
expires := monitorTestTime.Add(d.cooldown)
|
||||
expires := monitorTestTime.Add(time.Hour)
|
||||
if _, err := db.Exec(`UPDATE monitor_state SET cooldown_until = ? WHERE service_id = ?`, monitorTestTime.Add(time.Hour).UnixNano(), d.serviceID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -61,10 +61,11 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
|
||||
return nil, fmt.Errorf("candidate diagnostic artifact already exists or cannot be inspected: %v", err)
|
||||
}
|
||||
var raw bytes.Buffer
|
||||
err := check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw)
|
||||
err := check(ctx, "blockcheckw", path, c.Args, host, io.Discard, &raw)
|
||||
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
err = managedCheckerError(err, path+".combined.json.stderr", raw.Bytes())
|
||||
if err != nil {
|
||||
// Only our report parser may issue the nonpass authorization.
|
||||
if errors.Is(err, errCandidateNonpass) {
|
||||
@@ -91,6 +92,27 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
|
||||
return report, err
|
||||
}
|
||||
|
||||
// Check infrastructure before interpreting any positive or negative evidence.
|
||||
// Prefer the owned invocation's artifact over earlier scan/seed diagnostics;
|
||||
// captured stderr is private, and stdout never shares its buffer.
|
||||
func managedCheckerError(err error, artifact string, captured []byte) error {
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(err, &fatal) {
|
||||
return err
|
||||
}
|
||||
diagnostics, readErr := os.ReadFile(artifact)
|
||||
if readErr != nil {
|
||||
if !errors.Is(readErr, os.ErrNotExist) {
|
||||
return &managedRecoveryFatal{Err: fmt.Errorf("cannot inspect owned checker diagnostics")}
|
||||
}
|
||||
diagnostics = captured
|
||||
}
|
||||
if strings.Contains(strings.ToLower(string(diagnostics)), "nfqws2 exited") {
|
||||
return &managedRecoveryFatal{Err: fmt.Errorf("checker nfqws2 worker exited unexpectedly")}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var earlyTimeoutResult = regexp.MustCompile(`^ FAIL ([0-2])/([1-3]) timeout$`)
|
||||
var earlySuccessResult = regexp.MustCompile(`^ OK median [0-9]+ms, [0-9]+\.[0-9] KB/s$`)
|
||||
var earlyCheckPreamble = regexp.MustCompile("^(! --workers has no effect on `check` \\(it verifies strategies sequentially by design\\)\n)?" +
|
||||
|
||||
@@ -211,7 +211,7 @@ func selectBlockcheckEvidence(r io.Reader, expected, group, host string, recover
|
||||
for _, protocol := range []string{"tls12", "tls13"} {
|
||||
c, ok := best[protocol]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("no fully passing check strategy for both tls12 and tls13")
|
||||
return "", errNoQualifiedStrategy
|
||||
}
|
||||
if protocol == "tls13" && c.Args == best["tls12"].Args {
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user