Files
zapret-autotune/cmd/zapret-autotune/managed_monitor_cli_test.go
T
2026-10-10 22:51:58 +03:00

364 lines
15 KiB
Go

package main
import (
"bytes"
"context"
"errors"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
)
func TestManagedMonitorArguments(t *testing.T) {
lookup := func(string) (string, bool) { return "", false }
cfg, err := managedMonitorConfig(nil, lookup)
if err != nil || cfg.HealthInterval != 5*time.Minute || cfg.FailureThreshold != 3 || cfg.RecoveryCooldown != 30*time.Minute {
t.Fatal(cfg, err)
}
cfg, err = managedMonitorConfig([]string{"--interval", "2s", "--failure-threshold", "2", "--cooldown", "1m"}, lookup)
if err != nil || cfg.HealthInterval != 2*time.Second || cfg.FailureThreshold != 2 || cfg.RecoveryCooldown != time.Minute {
t.Fatal(cfg, err)
}
for _, args := range [][]string{{"--interval", "0"}, {"--interval", "25h"}, {"--cooldown", "169h"}, {"--cooldown", "-1s"}, {"--failure-threshold", "1001"}, {"--failure-threshold", "0"}, {"--command", "secret"}, {"--db", "/etc/db"}, {"extra"}} {
if _, err := managedMonitorConfig(args, lookup); err == nil {
t.Fatal("accepted", args)
}
}
if _, err := managedMonitorConfig(nil, func(key string) (string, bool) { return "999h", key == "ZAPRET_AUTOTUNE_HEALTH_INTERVAL" }); err == nil {
t.Fatal("accepted oversized environment policy")
}
}
func TestManagedMonitorCancellationSafety(t *testing.T) {
if !managedCancellationOnly(errors.Join(context.Canceled), context.Canceled) || managedCancellationOnly(errors.Join(context.Canceled, errors.New("DB failure")), context.Canceled) || managedCancellationOnly(&managedRecoveryFatal{Err: context.Canceled}, context.Canceled) {
t.Fatal("cancellation hid a fatal error")
}
}
func TestManagedMonitorExitClassification(t *testing.T) {
for _, mode := range []string{"timeout", "timeout-cleanup", "timeout-inner-cleanup", "fatal-timeout", "fatal-timeout-cleanup", "timeout-fatal", "job-timeout", "job-timeout-cleanup", "unknown", "zero", "cancel", "cancel-cleanup"} {
t.Run(mode, func(t *testing.T) {
d, env, _ := managedCLIFixture(t)
initManagedCLI(t, d)
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
want := 78
var out bytes.Buffer
code := managedMonitorCLIContext(ctx, nil, &out, &out, d, func(context.Context, managedMonitorDeps) error {
switch mode {
case "timeout":
want = 75
return &managedRecoveryTimeout{Err: context.DeadlineExceeded}
case "timeout-cleanup":
return errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("diagnostic close failed"))
case "timeout-inner-cleanup":
return &managedRecoveryTimeout{Err: errors.Join(context.DeadlineExceeded, errors.New("diagnostic close failed"))}
case "fatal-timeout":
want = 75 // Fatal is also the monitor's stop signal for clean timeout.
return &managedRecoveryFatal{Err: &managedRecoveryTimeout{Err: context.DeadlineExceeded}}
case "fatal-timeout-cleanup":
return &managedRecoveryFatal{Err: errors.Join(&managedRecoveryTimeout{Err: context.DeadlineExceeded}, errors.New("cleanup"))}
case "timeout-fatal":
return &managedRecoveryTimeout{Err: &managedRecoveryFatal{Err: errors.New("cleanup")}}
case "job-timeout", "job-timeout-cleanup":
if mode == "job-timeout" {
want = 75
}
return managedRecoveryJob(ctx, time.Millisecond, func(jobCtx context.Context) error {
<-jobCtx.Done()
if mode == "job-timeout-cleanup" {
return &managedRecoveryFatal{Err: errors.Join(jobCtx.Err(), errors.New("cleanup"))}
}
return jobCtx.Err()
})
case "zero":
want = 75
return nil
case "cancel":
want = 0
cancel()
return ctx.Err()
case "cancel-cleanup":
cancel()
return errors.Join(ctx.Err(), errors.New("cleanup"))
default:
return errors.New("unknown")
}
})
if code != want {
t.Fatal(code, want, out.String())
}
})
}
}
func TestManagedMonitorProductionAdapterBinding(t *testing.T) {
recovery, _, _ := recoveryFixture(t, false)
target, _, _ := managedCLIFixture(t)
target.directory = recovery.transaction.Dir
target.player = func(ctx context.Context, args []string, out, _ io.Writer) error {
if ctx.Err() != nil {
t.Fatal(ctx.Err())
}
if !strings.Contains(strings.Join(args, "|"), "--noproxy||--proxy|socks5://127.0.0.1:11080|") {
t.Fatal(args)
}
_, err := io.WriteString(out, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://fresh.googlevideo.com/video?secret=never-print"}}`)
return err
}
scans, checks, commands := 0, 0, 0
adapted, err := managedMonitorRecoveryAdapters(target, recovery.db,
func(ctx context.Context, binary, host string, out, diagnostic io.Writer) (string, error) {
if binary != "/usr/bin/blockcheckw" {
t.Fatal(binary)
}
scans++
return recovery.scan(ctx, "blockcheckw", host, out, diagnostic)
},
func(ctx context.Context, binary, path, args, host string, out, diagnostic io.Writer) error {
if binary != "/usr/bin/blockcheckw" {
t.Fatal(binary)
}
checks++
return recovery.check(ctx, "blockcheckw", path, args, host, out, diagnostic)
},
func(ctx context.Context, path string, args ...string) error {
if _, ok := ctx.Deadline(); !ok {
t.Fatal("unbounded daemon operation")
}
commands++
return recovery.run(ctx, path, args...)
})
if err != nil {
t.Fatal(err)
}
result, err := recoverManagedTarget(context.Background(), adapted)
if err != nil || result.GGCError != nil || result.FrontendID == 0 || result.GooglevideoID == 0 || scans != 2 || checks != 2 || commands == 0 {
t.Fatal(result, err, scans, checks, commands)
}
if err := adapted.run(context.Background(), "/bin/sh", "-c", "password"); err == nil {
t.Fatal("accepted shell command")
}
}
func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) {
recovery, _, _ := recoveryFixture(t, false)
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
recovery.discover = func(context.Context) (string, error) { return "", errors.New("secret") }
reports := 0
if err := wireManagedRecovery(&monitor, recovery, func(error) { reports++ }); err != nil {
t.Fatal(err)
}
if err := monitor.bootstrap(context.Background()); err != nil || reports != 1 {
t.Fatal(err, reports)
}
if err := monitor.probe(context.Background()); !errors.Is(err, errManagedGGCUnresolved) {
t.Fatal("unresolved GGC did not fail scope probe", err)
}
}
func TestManagedMonitorCLIGuardsBeforeLock(t *testing.T) {
for _, mode := range []string{"role", "identity", "db", "alias", "config"} {
t.Run(mode, func(t *testing.T) {
d, env, _ := managedCLIFixture(t)
initManagedCLI(t, d)
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
switch mode {
case "role":
env["ZAPRET_AUTOTUNE_TARGET_ROLE"] = "old-target"
case "identity":
os.Remove(filepath.Join(d.directory, managedMarkerName))
case "db":
env["ZAPRET_AUTOTUNE_DB"] = "/other/db"
case "alias":
if err := os.Symlink(filepath.Join(t.TempDir(), "db"), filepath.Join(d.directory, "candidates.db")); err != nil {
t.Fatal(err)
}
case "config":
env["ZAPRET_AUTOTUNE_VIDEO_ID"] = "password-do-not-print"
}
var out bytes.Buffer
code := managedMonitorCLIContext(context.Background(), nil, &out, &out, d, func(context.Context, managedMonitorDeps) error { t.Fatal("unsafe supervisor start"); return nil })
want := 78
if mode == "config" {
want = 64
}
if code != want || strings.Contains(out.String(), "password-do-not-print") {
t.Fatal(code, out.String())
}
if _, err := os.Lstat(filepath.Join(d.directory, managedMonitorLockName)); !errors.Is(err, os.ErrNotExist) {
t.Fatal("guard acquired lock", err)
}
})
}
}
func TestManagedMonitorCLIBindingAndCancellation(t *testing.T) {
d, env, _ := managedCLIFixture(t)
initManagedCLI(t, d)
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
requests := 0
d.player = func(ctx context.Context, args []string, out, _ io.Writer) error {
requests++
if !strings.Contains(strings.Join(args, "|"), "--noproxy||--proxy|socks5://127.0.0.1:11080|") {
t.Fatal(args)
}
if requests == 1 {
_, err := io.WriteString(out, `{"playabilityStatus":{"status":"OK"}}`)
return err
}
cancel()
<-ctx.Done()
return ctx.Err()
}
var out bytes.Buffer
code := managedMonitorCLIContext(ctx, nil, &out, &out, d, func(ctx context.Context, m managedMonitorDeps) error {
if m.bootstrap == nil || m.recover == nil || m.interval != 5*time.Minute || m.threshold != 3 || m.cooldown != 30*time.Minute {
t.Fatal("missing wiring/defaults")
}
if err := m.probe(ctx); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
return ctx.Err()
})
if code != 0 || requests != 2 {
t.Fatal(code, requests, out.String())
}
}
func TestManagedMonitorBootstrapExistingFrontend(t *testing.T) {
recovery, _, events := recoveryFixture(t, true)
monitor := monitorRuntimeFixture(t)
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
t.Fatal(err)
}
if err := wireManagedRecovery(&monitor, recovery, func(error) { t.Fatal("unexpected GGC recovery") }); err != nil {
t.Fatal(err)
}
if err := monitor.bootstrap(context.Background()); err != nil {
t.Fatal(err)
}
if len(*events) != 0 {
t.Fatal("startup rescanned qualified active frontend", *events)
}
}
// Execute only rewritten target-local contracts and fake programs; no network,
// Docker, real credentials, zapret or host firewall. Exercise the actual traps.
func TestManagedEntrypointSupervision(t *testing.T) {
for _, mode := range []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"} {
t.Run(mode, func(t *testing.T) {
root := t.TempDir()
config := filepath.Join(root, "ss")
if err := os.Mkdir(config, 0700); err != nil {
t.Fatal(err)
}
write := func(name, text string) string {
path := filepath.Join(root, name)
if err := os.WriteFile(path, []byte(text), 0700); err != nil {
t.Fatal(err)
}
return path
}
log := filepath.Join(root, "log")
write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n")
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & wait $!; done\n"
write("ss-server", child)
write("ss-local", child)
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nif [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; while :; do sleep 0.1; done; fi\n")
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then echo managed-$2 >> \"$LOG\"; exit 0; fi\ntrap 'echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; fi\necho managed-monitor >> \"$LOG\"\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & wait $!; done\n")
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
if err != nil {
t.Fatal(err)
}
text := strings.NewReplacer("/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
if mode == "term-hang" {
// Shorten only the fixture's watchdog, not the production contract.
text = strings.Replace(text, `bounded_wait "$monitor_pid" 60`, `bounded_wait "$monitor_pid" 1`, 1)
}
if mode == "stop-hang" {
text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1)
}
entry := write("entrypoint", text)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "/bin/sh", entry)
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "ZAPRET_CONFIG="+filepath.Join(root, "config"), "SS_PASSWORD="+strings.Repeat("a", 32))
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
var output bytes.Buffer
cmd.Stdout, cmd.Stderr = &output, &output
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
done := make(chan error, 1)
exited := make(chan struct{})
go func() { done <- cmd.Wait(); close(exited) }()
t.Cleanup(func() { syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); <-exited })
if strings.HasPrefix(mode, "term") {
for {
b, _ := os.ReadFile(log)
if strings.Contains(string(b), "managed-monitor") {
break
}
select {
case err := <-done:
t.Fatal("premature exit", err)
case <-ctx.Done():
t.Fatal(ctx.Err())
case <-time.After(20 * time.Millisecond):
}
}
if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err)
}
}
err = <-done
if ctx.Err() != nil {
t.Fatal("entrypoint did not join children", ctx.Err())
}
want := 75
switch mode {
case "term":
want = 0
case "fatal", "unknown", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang":
want = 78
}
if want == 0 && err != nil {
t.Fatal(err, output.String())
}
if want != 0 {
var exit *exec.ExitError
if !errors.As(err, &exit) || exit.ExitCode() != want {
t.Fatal(err, "want", want, output.String())
}
}
b, err := os.ReadFile(log)
if err != nil {
t.Fatal(err)
}
text = string(b)
if !strings.Contains(text, "managed-init\nzapret-start\nmanaged-monitor\n") || !strings.Contains(text, "zapret-stop") || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-server")) || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-local")) {
t.Fatal(text)
}
if mode == "term" && (!strings.Contains(text, "monitor-stopped") || strings.Index(text, "monitor-stopped") > strings.Index(text, "zapret-stop")) {
t.Fatal("stop before monitor joined", text)
}
})
}
}