Execute the private YouTube qualification plan through core-owned scheduling while retaining worker cleanup, proof sealing and legacy policy boundaries. Preserve unsafe collector failures alongside concurrent cancellation or deadlines. Send bounded Player JSON through stdin instead of argv and update all shared runner callers. Generic controller cutover and remote runtime acceptance remain pending. Co-authored-by: GPT-6 <gpt-6@openai.com>
841 lines
35 KiB
Go
841 lines
35 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// Subprocess adapter exercises the real CLI/transaction/DB behind the shell
|
|
// entrypoint. Only runtime/socket validation and fixed fixture paths are fake.
|
|
func TestManagedPrestartHelper(t *testing.T) {
|
|
if os.Getenv("MANAGED_PRESTART_HELPER") != "1" {
|
|
return
|
|
}
|
|
args := os.Args
|
|
for len(args) > 0 && args[0] != "--" {
|
|
args = args[1:]
|
|
}
|
|
if len(args) < 2 {
|
|
os.Exit(1)
|
|
}
|
|
args = args[1:] // -- COMMAND ...
|
|
if len(args) == 1 && args[0] == "target-config-check" {
|
|
// Use the production validator, not a fixture-only success response.
|
|
os.Exit(run(args, os.Stdout, os.Stderr))
|
|
}
|
|
if len(args) == 2 && args[0] == "dns-forwarder" && (args[1] == "mode" || args[1] == "address") {
|
|
os.Exit(run(args, os.Stdout, os.Stderr))
|
|
}
|
|
if len(args) < 2 || args[0] != "managed" {
|
|
os.Exit(1)
|
|
}
|
|
args = args[1:] // managed COMMAND
|
|
if args[0] == "monitor" {
|
|
os.Exit(78) // Stop after initial activation; no scans or network.
|
|
}
|
|
d := managedDependencies(nil)
|
|
d.directory, d.script = os.Getenv("FIXTURE_TARGET"), os.Getenv("FIXTURE_SCRIPT")
|
|
d.verifyRuntime = func(string) error { return nil }
|
|
d.verifySOCKS = func(int) error { return nil }
|
|
os.Exit(managedCLI(args, os.Stdout, os.Stderr, d))
|
|
}
|
|
|
|
func TestManagedEntrypointPrestartReconciliation(t *testing.T) {
|
|
for _, mode := range []string{"fresh", "baseline", "committed", "prepared", "drift", "unknown", "legacy", "corrupt-backup", "unqualified", "reload-failure", "invalid-dns", "invalid-service"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
d, env, _ := managedCLIFixture(t)
|
|
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
|
|
m := &managedTransaction{Dir: d.directory, Name: "config", Reload: func() error { return nil }, Health: func() (string, error) { return "OK", nil }}
|
|
known := []byte(managedBaseline)
|
|
if mode != "fresh" {
|
|
initManagedCLI(t, d)
|
|
}
|
|
if mode == "committed" || mode == "prepared" || mode == "corrupt-backup" || mode == "unqualified" || mode == "reload-failure" {
|
|
db, err := openStore(filepath.Join(d.directory, "candidates.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id := managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
|
|
if mode != "committed" {
|
|
m.fault = func(point string) error {
|
|
if point == "replaced" {
|
|
return errors.New("simulated process death")
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
err = m.Apply(db, []int64{id})
|
|
if (mode == "committed") != (err == nil) {
|
|
t.Fatal(err)
|
|
}
|
|
if mode == "committed" {
|
|
known, err = renderManagedConfig(db, []int64{id})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
} else {
|
|
j, err := m.load()
|
|
b, readErr := m.read("config")
|
|
if err != nil || readErr != nil || j.State != "prepared" || managedHash(b) != j.Next || j.Next == j.Hash {
|
|
t.Fatal("not a true interrupted replacement fixture", j, err, readErr)
|
|
}
|
|
}
|
|
if mode == "unqualified" {
|
|
if _, err := db.Exec("DELETE FROM candidate_evidence WHERE candidate_id = ?", id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := db.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if mode == "drift" || mode == "unknown" || mode == "corrupt-backup" {
|
|
name := "config"
|
|
if mode == "corrupt-backup" {
|
|
name = managedBackupName
|
|
}
|
|
if err := m.replace(name, []byte("UNRECOGNIZED=1\n")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if mode == "unknown" || mode == "legacy" {
|
|
j, err := m.load()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if mode == "legacy" {
|
|
j.Version = 1
|
|
} else {
|
|
j.Hash = managedHash([]byte("UNRECOGNIZED=1\n"))
|
|
}
|
|
if err := m.journal(j); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
root := t.TempDir()
|
|
write := func(name, text string) string {
|
|
path := filepath.Join(root, name)
|
|
if err := os.WriteFile(path, []byte(text), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return path
|
|
}
|
|
ss := filepath.Join(root, "ss")
|
|
if err := os.Mkdir(ss, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
write("awk", "#!/bin/sh\nexit 0\n")
|
|
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD-}\" ] || exit 1\n[ \"$#\" -eq 2 ] || { [ \"$#\" -eq 4 ] && [ \"$3\" = -d ] && [ \"$4\" = \"$SS_DNS_SERVER\" ]; } || exit 1\n: > \"$FIXTURE_CHILD_STARTED\"\nsleeper=\ntrap '[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; exit 0' TERM INT\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n"
|
|
write("ss-server", child)
|
|
write("ss-local", child)
|
|
log := filepath.Join(root, "log")
|
|
zapret := write("zapret", "#!/bin/sh\nprintf '%s ' \"$1\" >> \"$LOG\"\nsha256sum \"$FIXTURE_CONFIG\" >> \"$LOG\"\n[ \"$MODE:$1\" != reload-failure:restart ]\n")
|
|
binary, err := os.Executable()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
autotune := write("autotune", "#!/bin/sh\nexec \"$FIXTURE_BINARY\" -test.run='^TestManagedPrestartHelper$' -- \"$@\"\n")
|
|
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
helper, err := filepath.Abs("../../managed-target-supervision.sh")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entry := write("entrypoint", strings.NewReplacer("/opt/managed-target-supervision.sh", helper, "/run/zapret-autotune-ss", ss, "/usr/local/bin/zapret-autotune", autotune, managedZapretScript, zapret, `"$ZAPRET_CONFIG"`, `"$FIXTURE_CONFIG"`).Replace(string(data)))
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
cmd := exec.CommandContext(ctx, "/bin/sh", entry)
|
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
|
cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
|
|
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "SS_PASSWORD="+strings.Repeat("a", 32), "MANAGED_PRESTART_HELPER=1", "FIXTURE_BINARY="+binary, "FIXTURE_TARGET="+d.directory, "FIXTURE_SCRIPT="+zapret)
|
|
for key, value := range env {
|
|
cmd.Env = append(cmd.Env, key+"="+value)
|
|
}
|
|
// The CLI guard sees fixed production bindings; only fake daemon reads
|
|
// use the fixture config, via its own script rather than CLI path flags.
|
|
cmd.Env = append(cmd.Env, "FIXTURE_CONFIG="+filepath.Join(d.directory, "config"))
|
|
childStarted := filepath.Join(root, "child-started")
|
|
// This legacy daemon fixture deliberately exercises explicit native DNS.
|
|
cmd.Env = append(cmd.Env, "FIXTURE_CHILD_STARTED="+childStarted, "ZAPRET_AUTOTUNE_DNS_MODE=native", "SS_DNS_SERVER=8.8.8.8")
|
|
wantExit := 78
|
|
if mode == "invalid-dns" {
|
|
cmd.Env = append(cmd.Env, "SS_DNS_SERVER=8.8.8.8\n--invalid")
|
|
wantExit = 64
|
|
}
|
|
if mode == "invalid-service" {
|
|
cmd.Env = append(cmd.Env, "ZAPRET_AUTOTUNE_YOUTUBE_SERVICE=invalid")
|
|
wantExit = 64
|
|
}
|
|
output, err := cmd.CombinedOutput()
|
|
var exit *exec.ExitError
|
|
if ctx.Err() != nil || !errors.As(err, &exit) || exit.ExitCode() != wantExit {
|
|
t.Fatal(err, ctx.Err(), string(output))
|
|
}
|
|
if bytes.Contains(output, []byte(strings.Repeat("a", 32))) {
|
|
t.Fatal("startup exposed password")
|
|
}
|
|
if wantExit == 64 {
|
|
if _, err := os.Stat(childStarted); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatal("invalid configuration launched a service", err)
|
|
}
|
|
files, err := os.ReadDir(ss)
|
|
if err != nil || len(files) != 0 || !strings.Contains(string(output), "configerror:") {
|
|
t.Fatal("invalid configuration did not fail before config creation", err, string(output))
|
|
}
|
|
}
|
|
b, _ := os.ReadFile(log)
|
|
allowed := mode == "fresh" || mode == "baseline" || mode == "committed" || mode == "prepared"
|
|
if !allowed && mode != "reload-failure" && len(b) != 0 {
|
|
t.Fatal("unsafe startup invoked daemon script", string(b))
|
|
}
|
|
if allowed && !strings.Contains("\n"+string(b), "\nstart "+managedHash(known)) {
|
|
t.Fatal("no committed initial activation", string(b), string(output))
|
|
}
|
|
for _, line := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
|
if line != "" && !strings.Contains(line, managedHash(known)) {
|
|
t.Fatal("daemon saw uncommitted bytes", line)
|
|
}
|
|
}
|
|
if mode == "reload-failure" && (strings.HasPrefix(string(b), "start ") || strings.Contains(string(b), "\nstart ")) {
|
|
t.Fatal("failed rollback continued initial activation", string(b))
|
|
}
|
|
if err := m.locked(func() error { return nil }); err != nil {
|
|
t.Fatal("reconciliation leaked transaction lock", err)
|
|
}
|
|
if allowed {
|
|
for i := 0; i < 2; i++ {
|
|
if code, _, diagnostic := callManaged(t, d, "reconcile"); code != 0 {
|
|
t.Fatal("non-idempotent reconciliation/lock leak", code, diagnostic)
|
|
}
|
|
}
|
|
j, err := m.load()
|
|
if err != nil || j.State != "committed" || j.Hash != managedHash(known) {
|
|
t.Fatal(j, err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedReconcileDatabaseGuards(t *testing.T) {
|
|
for _, mode := range []string{"path", "symlink", "public", "flags"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
d, env, calls := managedCLIFixture(t)
|
|
initManagedCLI(t, d)
|
|
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
|
|
path := filepath.Join(d.directory, "candidates.db")
|
|
args := []string{"reconcile"}
|
|
switch mode {
|
|
case "path":
|
|
env["ZAPRET_AUTOTUNE_DB"] = "/other/db"
|
|
case "symlink":
|
|
if err := os.Symlink(filepath.Join(t.TempDir(), "missing"), path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case "public":
|
|
if err := os.WriteFile(path, []byte("unchanged"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case "flags":
|
|
args = append(args, "--db", path)
|
|
}
|
|
if code, _, _ := callManaged(t, d, args...); code == 0 || code == 75 || *calls != 0 {
|
|
t.Fatal("unsafe guard accepted", code)
|
|
}
|
|
if mode == "path" || mode == "flags" {
|
|
if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatal("guard opened database", err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedMonitorArguments(t *testing.T) {
|
|
lookup := func(string) (string, bool) { return "", false }
|
|
cfg, err := managedMonitorConfig(nil, lookup)
|
|
if err != nil || cfg.HealthInterval != 5*time.Minute || cfg.FailureThreshold != 3 || cfg.RecoveryCooldown != 30*time.Minute {
|
|
t.Fatal(cfg, err)
|
|
}
|
|
cfg, err = managedMonitorConfig([]string{"--interval", "2s", "--failure-threshold", "2", "--cooldown", "1m"}, lookup)
|
|
if err != nil || cfg.HealthInterval != 2*time.Second || cfg.FailureThreshold != 2 || cfg.RecoveryCooldown != time.Minute {
|
|
t.Fatal(cfg, err)
|
|
}
|
|
for _, args := range [][]string{{"--interval", "0"}, {"--interval", "25h"}, {"--cooldown", "169h"}, {"--cooldown", "-1s"}, {"--failure-threshold", "1001"}, {"--failure-threshold", "0"}, {"--command", "secret"}, {"--db", "/etc/db"}, {"extra"}} {
|
|
if _, err := managedMonitorConfig(args, lookup); err == nil {
|
|
t.Fatal("accepted", args)
|
|
}
|
|
}
|
|
if _, err := managedMonitorConfig(nil, func(key string) (string, bool) { return "999h", key == "ZAPRET_AUTOTUNE_HEALTH_INTERVAL" }); err == nil {
|
|
t.Fatal("accepted oversized environment policy")
|
|
}
|
|
}
|
|
|
|
func TestManagedMonitorCancellationSafety(t *testing.T) {
|
|
if !managedCancellationOnly(errors.Join(context.Canceled), context.Canceled) || managedCancellationOnly(errors.Join(context.Canceled, errors.New("DB failure")), context.Canceled) || managedCancellationOnly(&managedRecoveryFatal{Err: context.Canceled}, context.Canceled) {
|
|
t.Fatal("cancellation hid a fatal error")
|
|
}
|
|
}
|
|
|
|
func TestManagedMonitorExitClassification(t *testing.T) {
|
|
for _, mode := range []string{"timeout", "timeout-cleanup", "timeout-inner-cleanup", "fatal-timeout", "fatal-timeout-cleanup", "timeout-fatal", "job-timeout", "job-timeout-cleanup", "unknown", "zero", "cancel", "cancel-cleanup"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
d, env, _ := managedCLIFixture(t)
|
|
initManagedCLI(t, d)
|
|
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
want := 78
|
|
var out bytes.Buffer
|
|
code := managedMonitorCLIContext(ctx, nil, &out, &out, d, func(context.Context, managedMonitorDeps) error {
|
|
switch mode {
|
|
case "timeout":
|
|
want = 75
|
|
return &managedRecoveryTimeout{Err: context.DeadlineExceeded}
|
|
case "timeout-cleanup":
|
|
return errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("diagnostic close failed"))
|
|
case "timeout-inner-cleanup":
|
|
return &managedRecoveryTimeout{Err: errors.Join(context.DeadlineExceeded, errors.New("diagnostic close failed"))}
|
|
case "fatal-timeout":
|
|
want = 75 // Fatal is also the monitor's stop signal for clean timeout.
|
|
return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: context.DeadlineExceeded}}
|
|
case "fatal-timeout-cleanup":
|
|
return &managedRecoveryFatal{Err: errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("cleanup"))}
|
|
case "timeout-fatal":
|
|
return &managedRecoveryTimeout{Err: &managedRecoveryFatal{Err: errors.New("cleanup")}}
|
|
case "job-timeout", "job-timeout-cleanup":
|
|
if mode == "job-timeout" {
|
|
want = 75
|
|
}
|
|
return managedRecoveryJob(ctx, time.Millisecond, func(jobCtx context.Context) error {
|
|
<-jobCtx.Done()
|
|
if mode == "job-timeout-cleanup" {
|
|
return &managedRecoveryFatal{Err: errors.Join(jobCtx.Err(), errors.New("cleanup"))}
|
|
}
|
|
return jobCtx.Err()
|
|
})
|
|
case "zero":
|
|
want = 75
|
|
return nil
|
|
case "cancel":
|
|
want = 0
|
|
cancel()
|
|
return ctx.Err()
|
|
case "cancel-cleanup":
|
|
cancel()
|
|
return errors.Join(ctx.Err(), errors.New("cleanup"))
|
|
default:
|
|
return errors.New("unknown")
|
|
}
|
|
})
|
|
if code != want {
|
|
t.Fatal(code, want, out.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedMonitorProductionAdapterBinding(t *testing.T) {
|
|
recovery, _, _ := recoveryFixture(t, false)
|
|
target, _, _ := managedCLIFixture(t)
|
|
target.directory = recovery.transaction.Dir
|
|
target.player = func(ctx context.Context, args []string, _ io.Reader, out, _ io.Writer) error {
|
|
if ctx.Err() != nil {
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
if !strings.Contains(strings.Join(args, "|"), "--noproxy||--proxy|socks5h://127.0.0.1:11080|") {
|
|
t.Fatal(args)
|
|
}
|
|
_, err := io.WriteString(out, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://fresh.googlevideo.com/video?secret=never-print"}}`)
|
|
return err
|
|
}
|
|
scans, checks, commands := 0, 0, 0
|
|
adapted, err := managedMonitorRecoveryAdapters(target, recovery.db,
|
|
func(ctx context.Context, binary, host string, out, diagnostic io.Writer) (string, error) {
|
|
if binary != "/usr/bin/blockcheckw" {
|
|
t.Fatal(binary)
|
|
}
|
|
scans++
|
|
return recovery.scan(ctx, "blockcheckw", host, out, diagnostic)
|
|
},
|
|
func(ctx context.Context, binary, path, args, host string, out, diagnostic io.Writer) error {
|
|
if binary != "/usr/bin/blockcheckw" {
|
|
t.Fatal(binary)
|
|
}
|
|
checks++
|
|
return recovery.check(ctx, "blockcheckw", path, args, host, out, diagnostic)
|
|
},
|
|
func(ctx context.Context, path string, args ...string) error {
|
|
if _, ok := ctx.Deadline(); !ok {
|
|
t.Fatal("unbounded daemon operation")
|
|
}
|
|
commands++
|
|
return recovery.run(ctx, path, args...)
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result, err := recoverManagedTarget(context.Background(), adapted)
|
|
if err != nil || result.GGCError != nil || result.FrontendID == 0 || result.GooglevideoID == 0 || scans != 2 || checks != 2 || commands == 0 {
|
|
t.Fatal(result, err, scans, checks, commands)
|
|
}
|
|
if err := adapted.run(context.Background(), "/bin/sh", "-c", "password"); err == nil {
|
|
t.Fatal("accepted shell command")
|
|
}
|
|
}
|
|
|
|
func TestManagedMonitorProductionAbsenceDiagnostics(t *testing.T) {
|
|
for _, mode := range []string{"empty-nfq", "partial-nfq", "combined-nfq", "combined-capture-nfq", "success-scan-nfq", "success-combined-nfq", "artifact-nfq", "empty-timeout", "artifact-timeout", "malformed"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
recovery, _, _ := recoveryFixture(t, false)
|
|
monitor := monitorRuntimeFixture(t)
|
|
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
|
if err := os.Chmod(recovery.transaction.Dir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
|
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
const infra = "FAIL 0/1 nfqws2 exited immediately (code 1) secret-never-print"
|
|
adapted, err := managedMonitorRecoveryAdapters(monitor.target, recovery.db,
|
|
func(_ context.Context, _, host string, out, diagnostics io.Writer) (string, error) {
|
|
if out != io.Discard || diagnostics == io.Discard {
|
|
t.Fatal("streams not privately isolated")
|
|
}
|
|
// Model exec's independent stdout/stderr writers without sharing a buffer.
|
|
done := make(chan struct{})
|
|
go func() { io.WriteString(out, "secret-never-print"); close(done) }()
|
|
message := "FAIL 0/1 timeout"
|
|
if strings.Contains(mode, "nfq") || mode == "artifact-timeout" {
|
|
message = infra
|
|
}
|
|
if strings.Contains(mode, "combined") {
|
|
message = "FAIL 0/1 timeout"
|
|
}
|
|
io.WriteString(diagnostics, message)
|
|
<-done
|
|
data := []byte(`{"domain":"` + host + `","total":2,"working":0,"strategies":[]}`)
|
|
if mode == "partial-nfq" {
|
|
var report struct {
|
|
Strategies []json.RawMessage `json:"strategies"`
|
|
}
|
|
if err := json.Unmarshal([]byte(combinedReportFixture("--lua-desync=fake")), &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
report.Strategies = report.Strategies[:1]
|
|
partial, marshalErr := json.Marshal(report)
|
|
if marshalErr != nil {
|
|
t.Fatal(marshalErr)
|
|
}
|
|
data = partial
|
|
}
|
|
if strings.Contains(mode, "combined") || mode == "success-scan-nfq" {
|
|
data = []byte(combinedReportFixture("--lua-desync=fake"))
|
|
}
|
|
if mode == "malformed" {
|
|
data = []byte(`{"strategies":[]}`)
|
|
}
|
|
path := filepath.Join(t.TempDir(), "scan")
|
|
if mode == "artifact-nfq" || mode == "artifact-timeout" {
|
|
artifact := infra
|
|
if mode == "artifact-timeout" {
|
|
artifact = "FAIL 0/1 timeout"
|
|
}
|
|
if err := os.WriteFile(path+".check.json.stderr", []byte(artifact), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return path, os.WriteFile(path+".check.json", data, 0600)
|
|
},
|
|
func(_ context.Context, _, path, _, _ string, out, diagnostics io.Writer) error {
|
|
if out != io.Discard || diagnostics == io.Discard {
|
|
t.Fatal("check streams not privately isolated")
|
|
}
|
|
io.WriteString(diagnostics, infra)
|
|
if mode != "combined-capture-nfq" {
|
|
if err := os.WriteFile(path+".combined.json.stderr", []byte(infra), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
data := nonpassingManagedReport(t)
|
|
if mode == "success-combined-nfq" {
|
|
data = []byte(combinedReportFixture("--lua-desync=fake"))
|
|
}
|
|
return os.WriteFile(path+".combined.json", data, 0600)
|
|
}, recovery.run)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := wireManagedRecovery(&monitor, adapted, func(error) { t.Fatal("unexpected GGC report") }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
monitor.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
|
var reported error
|
|
monitor.reportRecoveryError = func(err error) { reported = err }
|
|
err = runManagedMonitor(ctx, monitor)
|
|
var fatal *managedRecoveryFatal
|
|
wantFatal := strings.Contains(mode, "nfq")
|
|
if errors.As(err, &fatal) != wantFatal || (!wantFatal && !errors.Is(err, context.Canceled)) {
|
|
t.Fatalf("mode=%s err=%v reported=%v", mode, err, reported)
|
|
}
|
|
if strings.Contains(errors.Join(err, reported).Error(), "secret-never-print") {
|
|
t.Fatal("diagnostics leaked")
|
|
}
|
|
state, stateErr := readMonitorState(monitor.db, monitor.serviceID)
|
|
if stateErr != nil {
|
|
t.Fatal(stateErr)
|
|
}
|
|
wantAbsence := mode == "empty-timeout" || mode == "artifact-timeout"
|
|
if errors.Is(reported, errNoQualifiedStrategy) != wantAbsence || errors.Is(err, errNoQualifiedStrategy) {
|
|
t.Fatalf("err=%v reported=%v", err, reported)
|
|
}
|
|
if wantAbsence {
|
|
if !state.CooldownUntil.Equal(monitorTestTime.Add(2 * time.Hour)) {
|
|
t.Fatalf("pause=%v", state.CooldownUntil)
|
|
}
|
|
} else if !state.CooldownUntil.IsZero() {
|
|
t.Fatalf("unexpected pause=%v", state.CooldownUntil)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) {
|
|
recovery, _, _ := recoveryFixture(t, false)
|
|
monitor := monitorRuntimeFixture(t)
|
|
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
|
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
|
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recovery.discover = func(context.Context) (string, error) { return "", errors.New("secret") }
|
|
reports := 0
|
|
if err := wireManagedRecovery(&monitor, recovery, func(error) { reports++ }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := monitor.bootstrap(context.Background()); err != nil || reports != 1 {
|
|
t.Fatal(err, reports)
|
|
}
|
|
if err := monitor.probe(context.Background()); !errors.Is(err, errManagedGGCUnresolved) {
|
|
t.Fatal("unresolved GGC did not fail scope probe", err)
|
|
}
|
|
}
|
|
|
|
func TestManagedMonitorCLIGuardsBeforeLock(t *testing.T) {
|
|
for _, mode := range []string{"role", "identity", "db", "alias", "config"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
d, env, _ := managedCLIFixture(t)
|
|
initManagedCLI(t, d)
|
|
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
|
|
switch mode {
|
|
case "role":
|
|
env["ZAPRET_AUTOTUNE_TARGET_ROLE"] = "old-target"
|
|
case "identity":
|
|
os.Remove(filepath.Join(d.directory, managedMarkerName))
|
|
case "db":
|
|
env["ZAPRET_AUTOTUNE_DB"] = "/other/db"
|
|
case "alias":
|
|
if err := os.Symlink(filepath.Join(t.TempDir(), "db"), filepath.Join(d.directory, "candidates.db")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case "config":
|
|
env["ZAPRET_AUTOTUNE_VIDEO_ID"] = "password-do-not-print"
|
|
}
|
|
var out bytes.Buffer
|
|
code := managedMonitorCLIContext(context.Background(), nil, &out, &out, d, func(context.Context, managedMonitorDeps) error { t.Fatal("unsafe supervisor start"); return nil })
|
|
want := 78
|
|
if mode == "config" {
|
|
want = 64
|
|
}
|
|
if code != want || strings.Contains(out.String(), "password-do-not-print") {
|
|
t.Fatal(code, out.String())
|
|
}
|
|
if _, err := os.Lstat(filepath.Join(d.directory, managedMonitorLockName)); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatal("guard acquired lock", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedMonitorCLIBindingAndCancellation(t *testing.T) {
|
|
d, env, _ := managedCLIFixture(t)
|
|
initManagedCLI(t, d)
|
|
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
requests := 0
|
|
d.player = func(ctx context.Context, args []string, _ io.Reader, out, _ io.Writer) error {
|
|
requests++
|
|
if !strings.Contains(strings.Join(args, "|"), "--noproxy||--proxy|socks5h://127.0.0.1:11080|") {
|
|
t.Fatal(args)
|
|
}
|
|
if requests == 1 {
|
|
_, err := io.WriteString(out, `{"playabilityStatus":{"status":"OK"}}`)
|
|
return err
|
|
}
|
|
cancel()
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
}
|
|
var out bytes.Buffer
|
|
code := managedMonitorCLIContext(ctx, nil, &out, &out, d, func(ctx context.Context, m managedMonitorDeps) error {
|
|
if m.bootstrap == nil || m.recover == nil || m.interval != 5*time.Minute || m.threshold != 3 || m.cooldown != 30*time.Minute {
|
|
t.Fatal("missing wiring/defaults")
|
|
}
|
|
if err := m.probe(ctx); !errors.Is(err, context.Canceled) {
|
|
t.Fatal(err)
|
|
}
|
|
return ctx.Err()
|
|
})
|
|
if code != 0 || requests != 2 {
|
|
t.Fatal(code, requests, out.String())
|
|
}
|
|
}
|
|
|
|
func TestManagedMonitorBootstrapExistingFrontend(t *testing.T) {
|
|
recovery, _, events := recoveryFixture(t, true)
|
|
monitor := monitorRuntimeFixture(t)
|
|
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
|
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
|
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := wireManagedRecovery(&monitor, recovery, func(error) { t.Fatal("unexpected GGC recovery") }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := monitor.bootstrap(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(*events) != 0 {
|
|
t.Fatal("startup rescanned qualified active frontend", *events)
|
|
}
|
|
}
|
|
|
|
// Execute only rewritten target-local contracts and fake programs; no network,
|
|
// Docker, real credentials, zapret or host firewall. Exercise the actual traps.
|
|
func TestManagedEntrypointSupervision(t *testing.T) {
|
|
modes := []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-early", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"}
|
|
for _, stage := range []string{"init", "reconcile", "start"} {
|
|
for _, failure := range []string{"hang", "75", "124", "137", "7"} {
|
|
modes = append(modes, "startup-"+stage+"-"+failure)
|
|
}
|
|
}
|
|
for _, mode := range modes {
|
|
t.Run(mode, func(t *testing.T) {
|
|
root := t.TempDir()
|
|
config := filepath.Join(root, "ss")
|
|
if err := os.Mkdir(config, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
write := func(name, text string) string {
|
|
path := filepath.Join(root, name)
|
|
if err := os.WriteFile(path, []byte(text), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return path
|
|
}
|
|
log := filepath.Join(root, "log")
|
|
// Shorten only startup watchdogs in this fixture. Execute a real
|
|
// timeout, including its forced-kill path, without a 45-second test.
|
|
if strings.HasPrefix(mode, "startup-") {
|
|
timeout, err := exec.LookPath("timeout")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
write("timeout", "#!/bin/sh\nif [ \"$3\" = 45 ] || { [ \"$3\" = 30 ] && [ \"$5\" = start ]; }; then\nshift 3\nexec \"$REAL_TIMEOUT\" -k 1 1 \"$@\"\nfi\nexec \"$REAL_TIMEOUT\" \"$@\"\n")
|
|
t.Setenv("REAL_TIMEOUT", timeout)
|
|
}
|
|
// Accelerate only restart cooldowns; watchdog deadlines remain real.
|
|
realSleep, err := exec.LookPath("sleep")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
write("sleep", "#!/bin/sh\ncase \"$1\" in 15|30|60) exec \"$REAL_SLEEP\" 0.02;; *) exec \"$REAL_SLEEP\" \"$@\";; esac\n")
|
|
t.Setenv("REAL_SLEEP", realSleep)
|
|
startupFailure := "case \"$MODE\" in startup-$stage-hang) trap '' TERM INT; exec sleep 300;; startup-$stage-*) exit \"${MODE##*-}\";; esac\n"
|
|
write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n")
|
|
joinSleeper := "[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; "
|
|
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\nsleeper=\ntrap '" + joinSleeper + "echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n"
|
|
write("ss-server", child)
|
|
write("ss-local", child)
|
|
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nstage=$1\n"+startupFailure+"if [ \"$MODE:$1\" = ss-early:start ] && [ \"$(grep -c zapret-start \"$LOG\")\" -gt 1 ]; then\nuntil [ \"$(grep -c 'stopped-.*/ss-local' \"$LOG\")\" -ge \"$(grep -c zapret-start \"$LOG\")\" ]; do sleep 0.02; done\nsleep 0.02\nfi\nif [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; exec sleep 300; fi\n")
|
|
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then\necho managed-$2 >> \"$LOG\"\nstage=$2\n"+startupFailure+"exit 0\nfi\nsleeper=\ntrap '"+joinSleeper+"echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\necho managed-monitor >> \"$LOG\"\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; exec sleep 300; fi\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n")
|
|
// This legacy lifecycle fixture has no DNS service. Implement only
|
|
// the explicit native getters; dedicated Python fixtures own DoH ACKs.
|
|
monitorData, err := os.ReadFile(monitor)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
getters := "if [ \"$1\" = dns-forwarder ]; then\ncase \"$2\" in mode) echo native;; address) echo;; *) exit 64;; esac\nexit 0\nfi\n"
|
|
write("autotune", strings.Replace(string(monitorData), "#!/bin/sh\n", "#!/bin/sh\n"+getters, 1))
|
|
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
helper, err := filepath.Abs("../../managed-target-supervision.sh")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
text := strings.NewReplacer("/opt/managed-target-supervision.sh", helper, "/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
|
|
if mode == "term-hang" {
|
|
// Shorten only the fixture's watchdog, not the production contract.
|
|
shortened := strings.Replace(text, `bounded_wait "$monitor_pid" 45`, `bounded_wait "$monitor_pid" 1`, 1)
|
|
if shortened == text {
|
|
t.Fatal("fixture did not shorten monitor watchdog")
|
|
}
|
|
text = shortened
|
|
}
|
|
if mode == "stop-hang" {
|
|
shortened := strings.Replace(text, "cleanup_command 30 "+zapret+" stop", "timeout -k 1 1 "+zapret+" stop", 1)
|
|
if shortened == text {
|
|
t.Fatal("fixture did not shorten stop watchdog")
|
|
}
|
|
text = shortened
|
|
}
|
|
entry := write("entrypoint", text)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
cmd := exec.CommandContext(ctx, "/bin/sh", entry)
|
|
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "ZAPRET_CONFIG="+filepath.Join(root, "config"), "SS_PASSWORD="+strings.Repeat("a", 32), "ZAPRET_AUTOTUNE_DNS_MODE=native")
|
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
|
var shutdownSnapshot string
|
|
stopMonitorSession := func() {
|
|
ready, _ := os.ReadFile(filepath.Join(config, "monitor-ready"))
|
|
sid, err := strconv.Atoi(strings.TrimSpace(string(ready)))
|
|
if err != nil || sid <= 1 || sid == os.Getpid() || sid == cmd.Process.Pid {
|
|
return
|
|
}
|
|
// Deadline cleanup must cover the fixture's verified private SID,
|
|
// including separate check PGIDs, not just the entrypoint PGID.
|
|
_ = exec.Command("/bin/sh", "-c", `sid=$1; helper=$2; shift 2; . "$helper"; signal_session "$sid" KILL`, "cleanup", strconv.Itoa(sid), helper).Run()
|
|
}
|
|
cmd.Cancel = func() error {
|
|
// Snapshot only this fixture's process group before deadline cleanup;
|
|
// no environment or private credential/config contents are collected.
|
|
ready, _ := os.ReadFile(filepath.Join(config, "monitor-ready"))
|
|
ownedSID := strings.TrimSpace(string(ready))
|
|
processes, _ := exec.Command("ps", "-eo", "pid,ppid,pgid,sid,stat,wchan:24,args").Output()
|
|
for _, line := range strings.Split(string(processes), "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) >= 4 && (fields[2] == strconv.Itoa(cmd.Process.Pid) || fields[3] == ownedSID) {
|
|
shutdownSnapshot += line + "\n"
|
|
}
|
|
}
|
|
fixtureLog, _ := os.ReadFile(log)
|
|
shutdownSnapshot += "fixture log:\n" + string(fixtureLog)
|
|
stopMonitorSession()
|
|
return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
|
}
|
|
var output bytes.Buffer
|
|
cmd.Stdout, cmd.Stderr = &output, &output
|
|
if err := cmd.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
done := make(chan error, 1)
|
|
exited := make(chan struct{})
|
|
go func() { done <- cmd.Wait(); close(exited) }()
|
|
t.Cleanup(func() {
|
|
select {
|
|
case <-exited:
|
|
return
|
|
default:
|
|
stopMonitorSession()
|
|
syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
|
<-exited
|
|
}
|
|
})
|
|
if strings.HasPrefix(mode, "term") {
|
|
for {
|
|
b, _ := os.ReadFile(log)
|
|
if strings.Contains(string(b), "managed-monitor") {
|
|
break
|
|
}
|
|
select {
|
|
case err := <-done:
|
|
t.Fatal("premature exit", err)
|
|
case <-ctx.Done():
|
|
t.Fatal(ctx.Err())
|
|
case <-time.After(20 * time.Millisecond):
|
|
}
|
|
}
|
|
if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
err = <-done
|
|
if ctx.Err() != nil {
|
|
t.Fatal("entrypoint did not join children", ctx.Err(), shutdownSnapshot, output.String())
|
|
}
|
|
// Repeated technical failures are supervised inside PID 1; after three
|
|
// starts the budget fails closed, rather than exporting exit 75.
|
|
want := 78
|
|
if strings.HasPrefix(mode, "startup-") {
|
|
want = 78
|
|
}
|
|
switch mode {
|
|
case "term":
|
|
want = 0
|
|
case "fatal", "unknown", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang":
|
|
want = 78
|
|
}
|
|
if want == 0 && err != nil {
|
|
t.Fatal(err, output.String())
|
|
}
|
|
if want != 0 {
|
|
var exit *exec.ExitError
|
|
if !errors.As(err, &exit) || exit.ExitCode() != want {
|
|
t.Fatal(err, "want", want, output.String())
|
|
}
|
|
}
|
|
b, err := os.ReadFile(log)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
text = string(b)
|
|
retrying := mode == "technical" || mode == "panic" || mode == "killed" || mode == "segv" || mode == "zero" || mode == "ss" || mode == "ss-early"
|
|
starts := strings.Count(text, "zapret-start\n")
|
|
if retrying && starts != 3 || !retrying && !strings.HasPrefix(mode, "startup-") && starts != 1 {
|
|
t.Fatal("unexpected internal restart count", starts, text)
|
|
}
|
|
if mode == "ss-early" && strings.Count(text, "managed-monitor\n") != 1 {
|
|
t.Fatal("forked monitor after replacement SS child already exited", text)
|
|
}
|
|
if strings.HasPrefix(mode, "startup-") {
|
|
if strings.Contains(text, "managed-monitor") || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-server")) || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-local")) {
|
|
t.Fatal("startup failure escaped cleanup", text)
|
|
}
|
|
if strings.HasPrefix(mode, "startup-start-") {
|
|
if !strings.Contains(text, "managed-init\nmanaged-reconcile\nzapret-start\n") || !strings.Contains(text, "zapret-stop") {
|
|
t.Fatal("startup ordering/daemon cleanup", text)
|
|
}
|
|
} else if strings.Contains(text, "zapret-") {
|
|
t.Fatal("daemon invoked after failed prestart guard", text)
|
|
}
|
|
return
|
|
}
|
|
if !strings.Contains(text, "managed-init\nmanaged-reconcile\nzapret-start\nmanaged-monitor\n") || !strings.Contains(text, "zapret-stop") || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-server")) || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-local")) {
|
|
t.Fatal(text)
|
|
}
|
|
if mode == "term" && (!strings.Contains(text, "monitor-stopped") || strings.Index(text, "monitor-stopped") > strings.Index(text, "zapret-stop")) {
|
|
t.Fatal("stop before monitor joined", text)
|
|
}
|
|
})
|
|
}
|
|
}
|