From 043df9ff77dd9d95a205030e8900f21e1f2978f4 Mon Sep 17 00:00:00 2001 From: Zeusina Date: Fri, 9 Oct 2026 20:54:45 +0300 Subject: [PATCH] feat: bootstrap zapret2 autotune Co-authored-by: GPT-6 --- .dockerignore | 6 + .gitignore | 6 + Dockerfile | 13 + README.md | 12 + autotune-entrypoint.sh | 11 + cmd/zapret-autotune/blockcheck.go | 157 ++++++++ cmd/zapret-autotune/blockcheck_test.go | 342 +++++++++++++++++ cmd/zapret-autotune/main.go | 76 ++++ cmd/zapret-autotune/main_test.go | 79 ++++ cmd/zapret-autotune/render.go | 184 +++++++++ cmd/zapret-autotune/render_test.go | 182 +++++++++ .../testdata/blockcheck-check.json | 6 + .../testdata/blockcheck-scan.json | 14 + .../testdata/blockcheck-scan.txt | 4 + cmd/zapret-autotune/tune.go | 182 +++++++++ cmd/zapret-autotune/tune_test.go | 358 ++++++++++++++++++ go.mod | 3 + 17 files changed, 1635 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 autotune-entrypoint.sh create mode 100644 cmd/zapret-autotune/blockcheck.go create mode 100644 cmd/zapret-autotune/blockcheck_test.go create mode 100644 cmd/zapret-autotune/main.go create mode 100644 cmd/zapret-autotune/main_test.go create mode 100644 cmd/zapret-autotune/render.go create mode 100644 cmd/zapret-autotune/render_test.go create mode 100644 cmd/zapret-autotune/testdata/blockcheck-check.json create mode 100644 cmd/zapret-autotune/testdata/blockcheck-scan.json create mode 100644 cmd/zapret-autotune/testdata/blockcheck-scan.txt create mode 100644 cmd/zapret-autotune/tune.go create mode 100644 cmd/zapret-autotune/tune_test.go create mode 100644 go.mod diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..76eb745 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +* +!go.mod +!cmd/ +!cmd/** +!Dockerfile +!autotune-entrypoint.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3d125fd --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.ssh/ +.config/ssh-mcp/ +Architecture.md +Autotune.md +Requirements.md +opencode.jsonc diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1384318 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.25.3 AS build +WORKDIR /src +ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOPROXY=off +COPY go.mod ./ +COPY cmd/ ./cmd/ +RUN go build -trimpath -o /out/zapret-autotune ./cmd/zapret-autotune + +FROM vernette/ss-zapret2@sha256:ba158180f27cd71a291ee521c291490374fd3fcf2f81eb1434ddfb19d35de45b +COPY --from=build /out/zapret-autotune /usr/local/bin/zapret-autotune +COPY autotune-entrypoint.sh /opt/autotune-entrypoint.sh +USER root +ENTRYPOINT ["/bin/sh", "/opt/autotune-entrypoint.sh"] +CMD ["status"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..bd485e2 --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +# zapret2-autotune + +Go CLI for scanning and checking YouTube TLS bypass strategies with `blockcheckw`. + +Build the Docker image with: + +```sh +docker build -t zapret-autotune . +``` + +The `tune youtube` command scans and checks candidate strategies; it prints a +proposed `NFQWS2_OPT` and does not apply configuration or restart services. diff --git a/autotune-entrypoint.sh b/autotune-entrypoint.sh new file mode 100644 index 0000000..ecf7052 --- /dev/null +++ b/autotune-entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +# Never invoke the service entrypoint or overwrite a mounted custom runtime. +if [ ! -e /opt/zapret2/lua ] && [ ! -L /opt/zapret2/lua ]; then + cp -a /opt/zapret2/lua.dist /opt/zapret2/lua +fi +if [ ! -d /opt/zapret2/lua ]; then + echo 'error: /opt/zapret2/lua must be a Lua runtime directory' >&2 + exit 1 +fi +exec /usr/local/bin/zapret-autotune "$@" diff --git a/cmd/zapret-autotune/blockcheck.go b/cmd/zapret-autotune/blockcheck.go new file mode 100644 index 0000000..bb6b992 --- /dev/null +++ b/cmd/zapret-autotune/blockcheck.go @@ -0,0 +1,157 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" +) + +// runBlockcheck leaves the output file (including partial results on failure) to +// the caller to read and remove. stdout/stderr receive the unmodified streams. +// The caller controls cancellation through ctx; no overall deadline is imposed here. +func runBlockcheck(ctx context.Context, binary, domain string, stdout, stderr io.Writer) (string, error) { + if !validDomain(domain) { + return "", fmt.Errorf("invalid DNS hostname: %q", domain) + } + if err := ctx.Err(); err != nil { + return "", err + } + output, err := os.CreateTemp("", "blockcheckw-*.txt") + if err != nil { + return "", err + } + path := output.Name() + if err := output.Close(); err != nil { + os.Remove(path) + return "", err + } + workdir, err := os.MkdirTemp("", "blockcheckw-scan-*") + if err != nil { + return path, err + } + // Resolve relative executable paths before changing the child's directory. + if strings.ContainsRune(binary, os.PathSeparator) && !filepath.IsAbs(binary) { + binary, err = filepath.Abs(binary) + if err != nil { + return path, fmt.Errorf("scan artifacts in %s (output %s): %w", workdir, path, err) + } + } + domain = strings.ToLower(domain) + err = runBlockcheckCommand(ctx, binary, workdir, []string{"--workers", "8", "scan", "--domain", domain, + "--protocols", "tls12,tls13", "--auto", "--no-conflict-cleanup", "--output", path}, stdout, stderr) + if err != nil { + return path, fmt.Errorf("scan artifacts in %s (output %s): %w", workdir, path, err) + } + vanilla, err := blockcheckVanillaReport(workdir) + if err != nil { + return path, fmt.Errorf("scan artifacts in %s (output %s): %w", workdir, path, err) + } + // Leave shortlist semantics to blockcheckw. Only check's JSON contains + // repeated real-data measurements; scan stdout is not evidence. + err = runBlockcheckCommand(ctx, binary, workdir, []string{"--workers", "32", "check", "--from-file", vanilla, + "--domain", domain, "--take", "10", "--passes", "3", "--auto", "--no-conflict-cleanup", + "--output", path + ".check.json"}, stdout, stderr) + if err != nil { + return path, fmt.Errorf("check (vanilla %s, scan output %s, artifacts in %s): %w", vanilla, path, workdir, err) + } + if err := os.RemoveAll(workdir); err != nil { + return path, fmt.Errorf("removing scan artifacts in %s: %w", workdir, err) + } + return path, nil +} + +func blockcheckVanillaReport(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", err + } + var report string + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), "_report_vanilla.txt") { + continue + } + info, err := entry.Info() + if err != nil { + return "", err + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("vanilla report %q is not a regular file", entry.Name()) + } + if report != "" { + return "", fmt.Errorf("multiple vanilla reports") + } + report = filepath.Join(dir, entry.Name()) + } + if report == "" { + return "", fmt.Errorf("no *_report_vanilla.txt generated") + } + return report, nil +} + +func combinedVanillaRows(args string) (string, error) { + c := candidate{Args: args} + if strings.TrimSpace(args) == "" { + return "", fmt.Errorf("empty combined args") + } + if err := c.validateArgs(); err != nil { + return "", err + } + return "curl_test_https_tls12 ipv4 youtube.com : nfqws2 " + args + "\n" + + "curl_test_https_tls13 ipv4 youtube.com : nfqws2 " + args + "\n", nil +} + +// Retain the exact combined input and report, including on failure. +func runCombinedBlockcheck(ctx context.Context, binary, path, args string, stdout, stderr io.Writer) error { + rows, err := combinedVanillaRows(args) + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + input := path + ".combined.txt" + if err := os.WriteFile(input, []byte(rows), 0600); err != nil { + return err + } + if err := runBlockcheckCommand(ctx, binary, "", []string{"--workers", "32", "check", "--from-file", input, + "--domain", "youtube.com", "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", + "--output", path + ".combined.json"}, stdout, stderr); err != nil { + return fmt.Errorf("combined check (input %s, output %s): %w", input, path+".combined.json", err) + } + return nil +} + +func runBlockcheckCommand(ctx context.Context, binary, dir string, args []string, stdout, stderr io.Writer) error { + cmd := exec.CommandContext(ctx, binary, args...) + cmd.Dir = dir + cmd.Stdout, cmd.Stderr = stdout, stderr + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM); err != nil { + if errors.Is(err, syscall.ESRCH) { + return os.ErrProcessDone + } + return err + } + // Wait also waits for Cancel: finish group cleanup even if the leader + // exits on TERM while a descendant ignores it and holds the pipes open. + time.Sleep(5 * time.Second) + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); !errors.Is(err, syscall.ESRCH) { + return err + } + return nil + } + cmd.WaitDelay = 5 * time.Second + err := cmd.Run() + if ctx.Err() != nil { + err = errors.Join(ctx.Err(), err) + } + return err +} diff --git a/cmd/zapret-autotune/blockcheck_test.go b/cmd/zapret-autotune/blockcheck_test.go new file mode 100644 index 0000000..604c802 --- /dev/null +++ b/cmd/zapret-autotune/blockcheck_test.go @@ -0,0 +1,342 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path/filepath" + "reflect" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// The test executable doubles as blockcheckw; no shell or network is involved. +func init() { + mode := os.Getenv("ZAPRET_TEST_BLOCKCHECK") + if mode == "" { + return + } + if len(os.Args) == 2 && os.Args[1] == "--child" { + if mode == "kill" { + signal.Ignore(syscall.SIGTERM) + } else { + signal.Reset(syscall.SIGTERM) + } + fmt.Fprintln(os.Stdout, os.Getpid()) + for { + time.Sleep(time.Hour) + } + } + if mode == "term" || mode == "kill" { + if mode == "term" { + signal.Ignore(syscall.SIGTERM) + } + child := exec.Command(os.Args[0], "--child") + child.Stdout, child.Stderr = os.Stdout, os.Stderr + if err := child.Run(); err != nil && mode == "kill" { + os.Exit(2) + } + os.Exit(0) + } + if os.Args[3] == "scan" { + name := "2026-10-09_14-42_report_vanilla.txt" + var err error + switch mode { + case "missing": + err = os.WriteFile("unrelated.txt", []byte("curl_test_"), 0600) + case "directory": + err = os.Mkdir(name, 0700) + case "symlink": + err = os.Symlink(os.Args[len(os.Args)-1], name) + default: + err = os.WriteFile(name, []byte("curl_test_https_tls12 example.com : --fake\n"), 0600) + if err == nil && mode == "multiple" { + err = os.WriteFile("2026-10-09_14-43_report_vanilla.txt", []byte("curl_test_"), 0600) + } + } + if err != nil { + os.Exit(3) + } + } else if os.Args[3] == "check" { + data, err := os.ReadFile(os.Args[5]) + if err != nil || !strings.Contains(string(data), "curl_test_") { + fmt.Fprintln(os.Stderr, "no strategies found") + os.Exit(4) + } + } + if strings.HasPrefix(mode, "tune") { + path := os.Args[len(os.Args)-1] + contents := "# blockcheckw scan results\n* SUMMARY\n" + if os.Args[3] == "check" { + contents = os.Getenv("ZAPRET_TEST_REPORT") + if strings.HasSuffix(path, ".combined.json") { + data, err := os.ReadFile(os.Args[5]) + if err != nil { + os.Exit(3) + } + line := strings.SplitN(string(data), "\n", 2)[0] + _, args, ok := strings.Cut(line, " : nfqws2 ") + if !ok { + os.Exit(4) + } + contents = combinedReportFixture(args) + if mode == "tune-combined-fail" { + contents = strings.Replace(contents, `"passes_ok":3`, `"passes_ok":2`, 1) + } + } + } + if err := os.WriteFile(path, []byte(contents), 0600); err != nil { + os.Exit(3) + } + data, _ := json.Marshal(os.Args[1:]) + file, err := os.OpenFile(os.Getenv("ZAPRET_TEST_ARGV"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + os.Exit(3) + } + if _, err := file.Write(append(data, '\n')); err != nil { + os.Exit(3) + } + file.Close() + fmt.Fprintln(os.Stdout, "untrusted scanner output") + fmt.Fprintln(os.Stdout, os.Getenv("ZAPRET_TEST_REPORT")) + if mode == "tune-error" || mode == "tune-check-error" && os.Args[3] == "check" || mode == "tune-combined-error" && strings.HasSuffix(path, ".combined.json") { + os.Exit(7) + } + os.Exit(0) + } + json.NewEncoder(os.Stdout).Encode(os.Args[1:]) + fmt.Fprint(os.Stderr, "raw stderr\n") + if err := os.WriteFile(os.Args[len(os.Args)-1], []byte("result\n"), 0600); err != nil { + os.Exit(3) + } + if mode == "error" || mode == "check-error" && os.Args[3] == "check" { + os.Exit(7) + } + os.Exit(0) +} + +func fakeBlockcheck(t *testing.T, mode string) string { + t.Helper() + t.Setenv("ZAPRET_TEST_BLOCKCHECK", mode) + source, err := os.Executable() + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(source) + if err != nil { + t.Fatal(err) + } + binary := filepath.Join(t.TempDir(), "fake blockcheckw") + if err := os.WriteFile(binary, data, 0700); err != nil { + t.Fatal(err) + } + return binary +} + +func TestRunBlockcheckArgvAndStreams(t *testing.T) { + tmp := t.TempDir() + t.Setenv("TMPDIR", tmp) + // A report from another run must not be considered, even with the same name. + if err := os.WriteFile(filepath.Join(tmp, "2026-10-09_14-42_report_vanilla.txt"), []byte("stale"), 0600); err != nil { + t.Fatal(err) + } + binary := fakeBlockcheck(t, "success") + var stdout, stderr bytes.Buffer + path, err := runBlockcheck(context.Background(), binary, "Example.COM", &stdout, &stderr) + if err != nil { + t.Fatal(err) + } + defer os.Remove(path) + defer os.Remove(path + ".check.json") + var args []string + decoder := json.NewDecoder(&stdout) + if err := decoder.Decode(&args); err != nil { + t.Fatal(err) + } + want := []string{"--workers", "8", "scan", "--domain", "example.com", "--protocols", "tls12,tls13", "--auto", "--no-conflict-cleanup", "--output", path} + if !reflect.DeepEqual(args, want) || stderr.String() != "raw stderr\nraw stderr\n" { + t.Fatalf("args=%q stderr=%q", args, stderr.String()) + } + if err := decoder.Decode(&args); err != nil { + t.Fatal(err) + } + vanilla := args[4] + if vanilla == path || filepath.Base(vanilla) != "2026-10-09_14-42_report_vanilla.txt" || filepath.Dir(vanilla) == filepath.Dir(path) { + t.Fatalf("check did not use isolated vanilla report: %q", vanilla) + } + if _, err := os.Stat(filepath.Dir(vanilla)); !os.IsNotExist(err) { + t.Fatalf("successful scan directory not removed: %v", err) + } + want = []string{"--workers", "32", "check", "--from-file", vanilla, "--domain", "example.com", "--take", "10", "--passes", "3", "--auto", "--no-conflict-cleanup", "--output", path + ".check.json"} + if !reflect.DeepEqual(args, want) { + t.Fatalf("check args=%q", args) + } + data, err := os.ReadFile(path) + if err != nil || string(data) != "result\n" { + t.Fatalf("output=%q err=%v", data, err) + } + var second bytes.Buffer + other, err := runBlockcheck(context.Background(), binary, "example.com", &second, io.Discard) + defer os.Remove(other) + defer os.Remove(other + ".check.json") + if err != nil || other == path { + t.Fatalf("second output=%q err=%v", other, err) + } + decoder = json.NewDecoder(&second) + if err := decoder.Decode(&args); err != nil { + t.Fatal(err) + } + if err := decoder.Decode(&args); err != nil || filepath.Dir(args[4]) == filepath.Dir(vanilla) { + t.Fatalf("scan working directory reused: args=%q err=%v", args, err) + } +} + +func TestRunBlockcheckValidation(t *testing.T) { + for _, domain := range []string{"", "--auto", "a;touch x", "a/b", "a..b", "-a.com", strings.Repeat("a", 64) + ".com"} { + path, err := runBlockcheck(context.Background(), "nonexistent", domain, io.Discard, io.Discard) + if err == nil || path != "" || !strings.Contains(err.Error(), "invalid DNS hostname") { + t.Fatalf("domain=%q path=%q err=%v", domain, path, err) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + path, err := runBlockcheck(ctx, "nonexistent", "example.com", nil, nil) + if path != "" || !errors.Is(err, context.Canceled) { + t.Fatalf("path=%q err=%v", path, err) + } +} + +func TestRunBlockcheckReportFailuresRetainArtifacts(t *testing.T) { + for _, tc := range []struct{ mode, message string }{ + {"missing", "no *_report_vanilla.txt generated"}, + {"multiple", "multiple vanilla reports"}, + {"directory", "not a regular file"}, + {"symlink", "not a regular file"}, + {"error", "scan artifacts"}, + {"check-error", "check (vanilla"}, + } { + t.Run(tc.mode, func(t *testing.T) { + tmp := t.TempDir() + t.Setenv("TMPDIR", tmp) + binary := fakeBlockcheck(t, tc.mode) + var stdout bytes.Buffer + path, runErr := runBlockcheck(context.Background(), binary, "example.com", &stdout, io.Discard) + if runErr == nil || !strings.Contains(runErr.Error(), tc.message) || !strings.Contains(runErr.Error(), path) { + t.Fatalf("path=%q err=%v", path, runErr) + } + if data, err := os.ReadFile(path); err != nil || string(data) != "result\n" { + t.Fatalf("scan output not retained: %q %v", data, err) + } + dirs, err := filepath.Glob(filepath.Join(tmp, "blockcheckw-scan-*")) + if err != nil || len(dirs) != 1 { + t.Fatalf("scan directory not retained: %q %v", dirs, err) + } + if !strings.Contains(runErr.Error(), dirs[0]) { + t.Fatalf("error omitted scan directory: %v", runErr) + } + decoder := json.NewDecoder(&stdout) + var args []string + if err := decoder.Decode(&args); err != nil { + t.Fatal(err) + } + if tc.mode == "check-error" { + if err := decoder.Decode(&args); err != nil || filepath.Dir(args[4]) != dirs[0] || args[4] == path { + t.Fatalf("incorrect check input: %q %v", args, err) + } + if _, err := os.Stat(args[4]); err != nil { + t.Fatalf("vanilla report not retained: %v", err) + } + if _, err := os.Stat(path + ".check.json"); err != nil { + t.Fatalf("partial check output not retained: %v", err) + } + } else if err := decoder.Decode(&args); err != io.EOF { + t.Fatalf("check ran after failed scan/discovery: %q %v", args, err) + } + }) + } +} + +func TestRunBlockcheckProcessError(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + binary := fakeBlockcheck(t, "error") + var stdout, stderr bytes.Buffer + path, err := runBlockcheck(context.Background(), binary, "example.com", &stdout, &stderr) + defer os.Remove(path) + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 7 || stdout.Len() == 0 || stderr.String() != "raw stderr\n" { + t.Fatalf("err=%v stdout=%q stderr=%q", err, stdout.String(), stderr.String()) + } + if data, err := os.ReadFile(path); err != nil || string(data) != "result\n" { + t.Fatalf("partial output=%q err=%v", data, err) + } +} + +type processReady struct{ pid chan int } + +func (w processReady) Write(p []byte) (int, error) { + pid, err := strconv.Atoi(strings.TrimSpace(string(p))) + if err != nil { + return 0, err + } + w.pid <- pid + return len(p), nil +} + +func TestRunBlockcheckCancellation(t *testing.T) { + for _, mode := range []string{"term", "kill"} { + t.Run(mode, func(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + binary := fakeBlockcheck(t, mode) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + ready := processReady{make(chan int, 1)} + done := make(chan error, 1) + go func() { + path, err := runBlockcheck(ctx, binary, "example.com", ready, io.Discard) + os.Remove(path) + done <- err + }() + var pid int + select { + case pid = <-ready.pid: + case err := <-done: + t.Fatalf("exited before child ready: %v", err) + case <-ctx.Done(): + t.Fatal("child never became ready") + } + start := time.Now() + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("err=%v", err) + } + case <-time.After(12 * time.Second): + t.Fatal("cancellation did not finish") + } + if elapsed := time.Since(start); elapsed < 5*time.Second || elapsed > 11*time.Second { + t.Fatalf("unexpected cancellation grace: %v", elapsed) + } + // A killed orphan may briefly be a zombie until the host's init reaps + // it; it must not still be running or holding any pipe open. + data, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)) + if err == nil { + _, state, _ := strings.Cut(string(data), ") ") + if !strings.HasPrefix(state, "Z ") { + t.Fatalf("child still alive: %s", data) + } + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + }) + } +} diff --git a/cmd/zapret-autotune/main.go b/cmd/zapret-autotune/main.go new file mode 100644 index 0000000..f96dfe5 --- /dev/null +++ b/cmd/zapret-autotune/main.go @@ -0,0 +1,76 @@ +package main + +import ( + "fmt" + "io" + "os" +) + +const usage = `Usage: zapret-autotune + +Commands: + status Show skeleton capabilities (not live service health) + discover youtube Refuse discovery until provider/config is agreed + check youtube Refuse checks until probe URLs/video IDs are configured and agreed + tune youtube Test youtube.com frontend only; print options without applying + render Select normalized offline results and print NFQWS2_OPT +` + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) +} + +func run(args []string, stdout, stderr io.Writer) int { + if len(args) == 2 && args[0] == "tune" && args[1] == "youtube" { + return tuneYoutube(stdout, stderr, runBlockcheck) + } + if len(args) == 2 && args[0] == "render" { + file, err := os.Open(args[1]) + if err != nil { + fmt.Fprintln(stderr, "error: cannot open results:", err) + return 1 + } + defer file.Close() + output, err := renderResults(file) + if err != nil { + fmt.Fprintln(stderr, "error: cannot render results:", err) + return 1 + } + if _, err := fmt.Fprintln(stdout, output); err != nil { + fmt.Fprintln(stderr, "error: cannot write results:", err) + return 1 + } + return 0 + } + if len(args) == 1 { + switch args[0] { + case "status": + _, err := fmt.Fprintln(stdout, `Status: no network requests, subprocesses, or production changes made by this command. +Tune: youtube.com frontend only via blockcheckw; apply: BLOCKED. Isolation, route/probe equivalence, version compatibility, and non-disruptive activation are unconfirmed. +Discovery: unavailable; YouTube provider/config is not implemented. +Production ss-zapret2 and Xray health: NOT CHECKED.`) + if err != nil { + fmt.Fprintln(stderr, "error: cannot write status:", err) + return 1 + } + return 0 + case "help", "--help", "-h": + if _, err := fmt.Fprint(stdout, usage); err != nil { + fmt.Fprintln(stderr, "error: cannot write usage:", err) + return 1 + } + return 0 + } + } + if len(args) == 2 && args[0] == "discover" && args[1] == "youtube" { + fmt.Fprintln(stderr, "error: discover youtube is unavailable: YouTube provider/config is not implemented; probe URLs/video IDs and API credentials/cookie requirements have not been configured or agreed. No network requests were made; no endpoints were generated.") + return 1 + } + if len(args) == 2 && args[0] == "check" && args[1] == "youtube" { + fmt.Fprintln(stderr, "error: check youtube is unavailable: YouTube probe URLs/video IDs have not been configured or agreed. No network requests were made; production health was not checked.") + return 1 + } + fmt.Fprintln(stderr, "error: missing, unknown, or unsupported command/arguments") + fmt.Fprint(stderr, usage) + return 2 +} diff --git a/cmd/zapret-autotune/main_test.go b/cmd/zapret-autotune/main_test.go new file mode 100644 index 0000000..64dcf15 --- /dev/null +++ b/cmd/zapret-autotune/main_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "bytes" + "errors" + "strings" + "testing" +) + +func TestRun(t *testing.T) { + tests := []struct { + name string + args []string + code int + out string + err string + }{ + {"status", []string{"status"}, 0, "apply: BLOCKED", ""}, + {"discovery refused", []string{"discover", "youtube"}, 1, "", "probe URLs/video IDs and API credentials/cookie requirements"}, + {"check refused", []string{"check", "youtube"}, 1, "", "YouTube probe URLs/video IDs have not been configured or agreed"}, + {"unsupported check provider", []string{"check", "other"}, 2, "", usage}, + {"missing check provider", []string{"check"}, 2, "", usage}, + {"extra check argument", []string{"check", "youtube", "extra"}, 2, "", usage}, + {"check flag rejected", []string{"check", "youtube", "--help"}, 2, "", usage}, + {"missing command", nil, 2, "", usage}, + {"unknown command", []string{"unknown"}, 2, "", usage}, + {"unsupported provider", []string{"discover", "other"}, 2, "", usage}, + {"missing provider", []string{"discover"}, 2, "", usage}, + {"extra status argument", []string{"status", "extra"}, 2, "", usage}, + {"extra discovery argument", []string{"discover", "youtube", "extra"}, 2, "", usage}, + {"unsupported tune provider", []string{"tune", "other"}, 2, "", usage}, + {"apply unavailable", []string{"apply"}, 2, "", usage}, + {"disruption flag rejected", []string{"tune", "--allow-disruption"}, 2, "", usage}, + {"help", []string{"--help"}, 0, usage, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + if got := run(tt.args, &stdout, &stderr); got != tt.code { + t.Fatalf("exit code = %d, want %d", got, tt.code) + } + for _, stream := range []struct{ got, want string }{ + {stdout.String(), tt.out}, {stderr.String(), tt.err}, + } { + if (stream.want == "" && stream.got != "") || !strings.Contains(stream.got, stream.want) { + t.Errorf("output = %q, want %q", stream.got, stream.want) + } + } + if tt.name == "status" && !strings.Contains(stdout.String(), "NOT CHECKED") { + t.Error("status must not claim production health") + } + if tt.name == "discovery refused" && !strings.Contains(stderr.String(), "no endpoints were generated") { + t.Error("discovery must explicitly refuse endpoint generation") + } + if tt.name == "check refused" { + for _, want := range []string{"No network requests were made", "production health was not checked"} { + if !strings.Contains(stderr.String(), want) { + t.Errorf("check refusal must include %q", want) + } + } + } + }) + } +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, errors.New("write failed") +} + +func TestOutputFailure(t *testing.T) { + for _, command := range []string{"status", "--help"} { + var stderr bytes.Buffer + if run([]string{command}, failingWriter{}, &stderr) != 1 { + t.Errorf("%s: output failure must return nonzero", command) + } + } +} diff --git a/cmd/zapret-autotune/render.go b/cmd/zapret-autotune/render.go new file mode 100644 index 0000000..fe7d88a --- /dev/null +++ b/cmd/zapret-autotune/render.go @@ -0,0 +1,184 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "math" + "regexp" + "sort" + "strings" +) + +type candidate struct { + Group string `json:"group"` + Domains []string `json:"domains"` + Args string `json:"args"` + SuccessRate float64 `json:"success_rate"` + Coverage float64 `json:"coverage"` + PassesOK int `json:"passes_ok"` + PassesTotal int `json:"passes_total"` + MedianLatency float64 `json:"median_latency_ms"` + MedianSpeed float64 `json:"median_speed_kbps"` +} + +// renderResults consumes normalized measurements only; it never runs probes or applies options. +func renderResults(r io.Reader) (string, error) { + decoder := json.NewDecoder(r) + var records []json.RawMessage + if err := decoder.Decode(&records); err != nil { + return "", err + } + if err := decoder.Decode(new(json.RawMessage)); err != io.EOF { + return "", fmt.Errorf("expected exactly one JSON array") + } + best := make(map[string]candidate) + for i, record := range records { + var fields map[string]json.RawMessage + if err := json.Unmarshal(record, &fields); err != nil { + return "", err + } + for _, key := range []string{"group", "domains", "args", "success_rate", "coverage", "passes_ok", "passes_total", "median_latency_ms", "median_speed_kbps"} { + if value, ok := fields[key]; !ok || bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + return "", fmt.Errorf("candidate %d: missing or null %s", i, key) + } + } + var c candidate + d := json.NewDecoder(bytes.NewReader(record)) + d.DisallowUnknownFields() + if err := d.Decode(&c); err != nil { + return "", fmt.Errorf("candidate %d: %w", i, err) + } + if err := c.validate(); err != nil { + return "", fmt.Errorf("candidate %d: %w", i, err) + } + if !fullyPassing(c) { + continue + } + if old, ok := best[c.Group]; !ok || better(c, old) { + best[c.Group] = c + } + } + var profiles []string + for _, group := range []string{"frontend", "googlevideo"} { + c, ok := best[group] + if !ok { + return "", fmt.Errorf("no fully passing candidate for %s", group) + } + profiles = append(profiles, "--filter-tcp=443 --filter-l7=tls --hostlist-domains="+strings.Join(c.Domains, ",")+" "+c.Args) + } + return "NFQWS2_OPT='" + strings.Join(profiles, " --new ") + "'", nil +} + +func (c *candidate) validate() error { + if err := c.validateMeasurements(); err != nil { + return err + } + return c.validateArgs() +} + +func (c *candidate) validateMeasurements() error { + if c.Group != "frontend" && c.Group != "googlevideo" { + return fmt.Errorf("unknown group") + } + if len(c.Domains) == 0 { + return fmt.Errorf("domains must not be empty") + } + for i, domain := range c.Domains { + if !validDomain(domain) { + return fmt.Errorf("invalid DNS hostname") + } + c.Domains[i] = strings.ToLower(domain) + } + sort.Strings(c.Domains) + for _, metric := range []float64{c.SuccessRate, c.Coverage, c.MedianLatency, c.MedianSpeed} { + if math.IsNaN(metric) || math.IsInf(metric, 0) || metric < 0 { + return fmt.Errorf("invalid metric") + } + } + if c.SuccessRate > 1 || c.Coverage > 1 || c.PassesOK < 0 || c.PassesTotal < 0 || c.PassesOK > c.PassesTotal { + return fmt.Errorf("invalid success/coverage/pass counts") + } + if len(strings.Fields(c.Args)) == 0 { + return fmt.Errorf("args must not be empty") + } + return nil +} + +func (c *candidate) validateArgs() error { + // Only shell-inert, unquoted option values can be rendered verbatim. + for _, ch := range c.Args { + if !(ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch >= '0' && ch <= '9' || strings.ContainsRune(" _-=:,./+@%", ch)) { + return fmt.Errorf("unsafe args character") + } + } + for _, arg := range strings.Fields(c.Args) { + option, value, ok := strings.Cut(arg, "=") + valid := false + if ok { + switch option { + case "--lua-desync": + valid = validStage(value) + case "--payload": + valid = payloadValue.MatchString(value) + case "--in-range", "--out-range": + valid = rangeValue.MatchString(value) + case "--lua-init": + // ponytail: file-based init only; inline Lua needs a separate, + // non-shell output contract and is deliberately rejected. + valid = initValue.MatchString(value) && !strings.Contains(value, "..") + } + } + if !valid { + return fmt.Errorf("unknown or invalid args option/value: %q", arg) + } + } + return nil +} + +var ( + stageValue = regexp.MustCompile(`^[A-Za-z_][A-Za-z_0-9]*(:[A-Za-z_][A-Za-z_0-9]*(=[A-Za-z0-9_.,/+@%-]+)?)*$`) + payloadValue = regexp.MustCompile(`^(all|unknown|empty|known|tls_client_hello|tls_server_hello)(,(all|unknown|empty|known|tls_client_hello|tls_server_hello))*$`) + rangeValue = regexp.MustCompile(`^([nbsd]?[0-9]+|-[nbsd]?[0-9]+|[nbsd]?[0-9]+-[nbsd]?[0-9]+)$`) + initValue = regexp.MustCompile(`^@/opt/zapret2/lua/[A-Za-z0-9_./-]+\.lua(\.gz)?$`) +) + +func validStage(stage string) bool { return stageValue.MatchString(stage) } + +func validDomain(domain string) bool { + if len(domain) == 0 || len(domain) > 253 { + return false + } + for _, label := range strings.Split(domain, ".") { + if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for _, ch := range label { + if !(ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch >= '0' && ch <= '9' || ch == '-') { + return false + } + } + } + return true +} + +func better(a, b candidate) bool { + if a.MedianSpeed != b.MedianSpeed { + return a.MedianSpeed > b.MedianSpeed + } + if a.MedianLatency != b.MedianLatency { + return a.MedianLatency < b.MedianLatency + } + if len(strings.Fields(a.Args)) != len(strings.Fields(b.Args)) { + return len(strings.Fields(a.Args)) < len(strings.Fields(b.Args)) + } + if a.Args != b.Args { + return a.Args < b.Args + } + return strings.Join(a.Domains, ",") < strings.Join(b.Domains, ",") +} + +func fullyPassing(c candidate) bool { + return c.SuccessRate == 1 && c.Coverage == 1 && c.PassesOK == c.PassesTotal && c.PassesTotal > 0 +} diff --git a/cmd/zapret-autotune/render_test.go b/cmd/zapret-autotune/render_test.go new file mode 100644 index 0000000..da690e4 --- /dev/null +++ b/cmd/zapret-autotune/render_test.go @@ -0,0 +1,182 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func passing(group string) candidate { + return candidate{group, []string{group + ".example.com"}, "--lua-desync=fake:blob=tls_clienthello", 1, 1, 3, 3, 10, 100} +} + +func encodeCandidates(t *testing.T, candidates ...candidate) string { + t.Helper() + data, err := json.Marshal(candidates) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func TestRenderSelection(t *testing.T) { + base := passing("frontend") + tests := []struct { + name string + change func(*candidate) + wins bool + }{ + {"higher speed", func(c *candidate) { c.MedianSpeed++; c.MedianLatency++ }, true}, + {"lower speed", func(c *candidate) { c.MedianSpeed-- }, false}, + {"lower latency", func(c *candidate) { c.MedianLatency-- }, true}, + {"higher latency", func(c *candidate) { c.MedianLatency++ }, false}, + {"more stages", func(c *candidate) { c.Args = "--lua-desync=fake --lua-desync=multisplit" }, false}, + {"lexical args", func(c *candidate) { c.Args = "--lua-desync=aaa" }, true}, + {"domain tie", func(c *candidate) { c.Domains = []string{"aaa.example.com"} }, true}, + {"partial success", func(c *candidate) { c.SuccessRate = .99; c.MedianSpeed++ }, false}, + {"partial coverage", func(c *candidate) { c.Coverage = .99; c.MedianSpeed++ }, false}, + {"incomplete passes", func(c *candidate) { c.PassesOK--; c.MedianSpeed++ }, false}, + {"no passes", func(c *candidate) { c.PassesTotal = 0; c.PassesOK = 0; c.MedianSpeed++ }, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + other := passing("frontend") + other.Domains = []string{"other.example.com"} + tt.change(&other) + want := base + if tt.wins { + want = other + } + video := passing("googlevideo") + expected, err := renderResults(strings.NewReader(encodeCandidates(t, want, video))) + if err != nil { + t.Fatal(err) + } + for _, candidates := range [][]candidate{{base, other, video}, {video, other, base}} { + got, err := renderResults(strings.NewReader(encodeCandidates(t, candidates...))) + if err != nil || got != expected { + t.Fatalf("got %q, %v; want %q", got, err, expected) + } + } + }) + } +} + +func TestRenderValidation(t *testing.T) { + tests := []struct { + name string + change func(*candidate) + }{ + {"unknown group", func(c *candidate) { c.Group = "other" }}, + {"no domains", func(c *candidate) { c.Domains = nil }}, + {"negative speed", func(c *candidate) { c.MedianSpeed = -1 }}, + {"negative latency", func(c *candidate) { c.MedianLatency = -1 }}, + {"success over one", func(c *candidate) { c.SuccessRate = 2 }}, + {"coverage negative", func(c *candidate) { c.Coverage = -1 }}, + {"negative passes", func(c *candidate) { c.PassesOK = -1 }}, + {"impossible passes", func(c *candidate) { c.PassesOK = 4 }}, + } + for _, domain := range []string{"https://example.com", "*.example.com", "a;b.com", "a$b.com", "a b.com", "a_b.com", "-a.com", "a-.com", "a..com", "a.com.", strings.Repeat("a", 64) + ".com", strings.Repeat("a.", 127) + "com"} { + domain := domain + tests = append(tests, struct { + name string + change func(*candidate) + }{"domain " + domain, func(c *candidate) { c.Domains = []string{domain} }}) + } + for _, args := range []string{"", "--new", "--filter-tcp=80", "--lua-desync=", "--lua-desync=123", "--lua-desync=fake extra", "--lua-desync=fake\n", "--lua-desync=fake\r", "--lua-desync=fake\t", "--lua-desync='fake'", "--lua-desync=\"fake\"", "--lua-desync=`fake`", "--lua-desync=$fake", "--lua-desync=fake;", "--lua-desync=fake\\", "--lua-desync=fake|cat", "--lua-desync=fake&", "--lua-desync=fake>file"} { + args := args + tests = append(tests, struct { + name string + change func(*candidate) + }{"args " + args, func(c *candidate) { c.Args = args }}) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bad := passing("frontend") + tt.change(&bad) + got, err := renderResults(strings.NewReader(encodeCandidates(t, bad, passing("frontend"), passing("googlevideo")))) + if err == nil || got != "" { + t.Fatalf("untrusted candidate accepted: %q, %v", got, err) + } + }) + } +} + +func TestRenderJSON(t *testing.T) { + valid := encodeCandidates(t, passing("frontend"), passing("googlevideo")) + for _, input := range []string{"", "[", "{}", "null", "[]", "[null]", valid + " {}", valid + " garbage", strings.Replace(valid, `"group":"frontend",`, "", 1), strings.Replace(valid, `"coverage":1`, `"coverage":null`, 1), strings.Replace(valid, `"coverage":1`, `"coverage":1,"unknown":true`, 1), strings.Replace(valid, `"coverage":1`, `"coverage":"1"`, 1), strings.Replace(valid, `"coverage":1`, `"coverage":1e999`, 1)} { + if got, err := renderResults(strings.NewReader(input)); err == nil || got != "" { + t.Errorf("accepted invalid JSON %q: %q, %v", input, got, err) + } + } + for _, group := range []string{"frontend", "googlevideo"} { + if got, err := renderResults(strings.NewReader(encodeCandidates(t, passing(group)))); err == nil || got != "" { + t.Errorf("accepted only %s: %q, %v", group, got, err) + } + } +} + +func TestStrategyOptionGrammar(t *testing.T) { + for _, args := range []string{ + "--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:tcp_md5:repeats=1", + "--in-range=-s1 --lua-desync=oob:urp=midsld", + "--out-range=s1-s10 --lua-init=@/opt/zapret2/lua/zapret-lib.lua --lua-desync=multisplit:pos=1,midsld", + } { + c := passing("frontend") + c.Args = args + if err := c.validate(); err != nil || c.Args != args { + t.Fatalf("valid args changed/rejected: %q, %v", c.Args, err) + } + } + for _, args := range []string{ + "nfqws2 --lua-desync=oob", "--payload=nope", "--payload=", "--in-range=xyz", "--out-range=s1--s2", + "--lua-desync=oob:", "--lua-desync=oob::urp=midsld", "--lua-desync=oob:urp=", "--lua-desync=oob:1bad=x", + "--lua-init=os.execute=bad", "--lua-init=@/tmp/x.lua", "--lua-init=@/opt/zapret2/lua/../x.lua", + "--lua-init=@/opt/zapret2/lua/x.lua;id", "--payload='tls_client_hello'", "--in-range=$(id)", + } { + c := passing("frontend") + c.Args = args + if err := c.validate(); err == nil { + t.Fatalf("invalid args accepted: %q", args) + } + } +} + +func TestRenderCLI(t *testing.T) { + front := passing("frontend") + front.Domains = []string{"Z.Example.com", "a.example.com"} + front.Args = " --lua-desync=fake:blob=tls_clienthello --lua-desync=multisplit:pos=1 " + path := filepath.Join(t.TempDir(), "results.json") + if err := os.WriteFile(path, []byte(encodeCandidates(t, front, passing("googlevideo"))+"\n "), 0600); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + if code := run([]string{"render", path}, &stdout, &stderr); code != 0 || stderr.Len() != 0 { + t.Fatalf("code %d, stderr %q", code, stderr.String()) + } + want := "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=a.example.com,z.example.com " + front.Args + " --new --filter-tcp=443 --filter-l7=tls --hostlist-domains=googlevideo.example.com --lua-desync=fake:blob=tls_clienthello'\n" + if stdout.String() != want { + t.Fatalf("got %q, want %q", stdout.String(), want) + } + if run([]string{"render", path}, failingWriter{}, &stderr) != 1 { + t.Error("write failure must fail") + } + stdout.Reset() + if run([]string{"render", path + ".missing"}, &stdout, &stderr) != 1 || stdout.Len() != 0 { + t.Error("missing file must fail without output") + } + if err := os.WriteFile(path, []byte("[]"), 0600); err != nil { + t.Fatal(err) + } + if run([]string{"render", path}, &stdout, &stderr) != 1 || stdout.Len() != 0 { + t.Error("no candidates must fail without output") + } + for _, args := range [][]string{{"render"}, {"render", path, "extra"}} { + if run(args, &stdout, &stderr) != 2 { + t.Error("wrong arity must return usage error") + } + } +} diff --git a/cmd/zapret-autotune/testdata/blockcheck-check.json b/cmd/zapret-autotune/testdata/blockcheck-check.json new file mode 100644 index 0000000..9f3ed0b --- /dev/null +++ b/cmd/zapret-autotune/testdata/blockcheck-check.json @@ -0,0 +1,6 @@ +{ + "strategies":[ + {"protocol":"HTTPS/TLS1.2","args":"--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:tcp_md5:repeats=1","coverage":1,"success_rate":1,"median_latency_ms":10,"median_speed_kbps":100,"passes_ok":3,"passes_total":3}, + {"protocol":"HTTPS/TLS1.3","args":"--in-range=-s1 --lua-desync=oob:urp=midsld","coverage":1,"success_rate":1,"median_latency_ms":10,"median_speed_kbps":100,"passes_ok":3,"passes_total":3} + ] +} diff --git a/cmd/zapret-autotune/testdata/blockcheck-scan.json b/cmd/zapret-autotune/testdata/blockcheck-scan.json new file mode 100644 index 0000000..167d383 --- /dev/null +++ b/cmd/zapret-autotune/testdata/blockcheck-scan.json @@ -0,0 +1,14 @@ +{ + "domain":"youtube.com", + "timestamp":"2026-10-09T00:00:00Z", + "block_type":"dpi", + "dns_spoofed":false, + "total":2, + "working":2, + "blocked":0, + "protocols":["tls12","tls13"], + "strategies":[ + {"protocol":"tls12","args":"--lua-desync=fake:blob=tls_clienthello","success_rate":1,"coverage":1,"passes_ok":3,"total":3,"median_latency_ms":10,"median_speed_kbps":100}, + {"protocol":"tls13","args":"--lua-desync=fake:blob=tls_clienthello","success_rate":1,"coverage":1,"passes_ok":3,"total":3,"median_latency_ms":10,"median_speed_kbps":100} + ] +} diff --git a/cmd/zapret-autotune/testdata/blockcheck-scan.txt b/cmd/zapret-autotune/testdata/blockcheck-scan.txt new file mode 100644 index 0000000..bf343d0 --- /dev/null +++ b/cmd/zapret-autotune/testdata/blockcheck-scan.txt @@ -0,0 +1,4 @@ +# blockcheckw vanilla scan report (v0.9.0 format excerpt) +* SUMMARY +youtube.com HTTPS/TLS1.2 : nfqws2 --payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:tcp_md5:repeats=1 +youtube.com HTTPS/TLS1.3 : nfqws2 --in-range=-s1 --lua-desync=oob:urp=midsld diff --git a/cmd/zapret-autotune/tune.go b/cmd/zapret-autotune/tune.go new file mode 100644 index 0000000..e0b9492 --- /dev/null +++ b/cmd/zapret-autotune/tune.go @@ -0,0 +1,182 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" +) + +func tuneYoutube(stdout, stderr io.Writer, scan func(context.Context, string, string, io.Writer, io.Writer) (string, error)) int { + return tuneYoutubeWithCheck(stdout, stderr, scan, runCombinedBlockcheck) +} + +func tuneYoutubeWithCheck(stdout, stderr io.Writer, scan func(context.Context, string, string, io.Writer, io.Writer) (string, error), check func(context.Context, string, string, string, io.Writer, io.Writer) error) int { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + fmt.Fprintln(stderr, "Testing frontend youtube.com only; GGC and other frontend domains have not been tested. No options will be applied.") + fmt.Fprintln(stderr, "Full TLS1.2 and TLS1.3 scan with 8 workers, then real-data check with 32 workers, --take 10, 3 passes; no overall time limit; selection requires fully passing results for both tls12 and tls13.") + fmt.Fprintln(stderr, "The best individually verified TLS strategies are combined into one profile, then checked together for 3/3 passes on both TLS versions before output. No options will be applied.") + var raw bytes.Buffer + path, err := scan(ctx, "blockcheckw", "youtube.com", io.MultiWriter(&raw, stderr), stderr) + if path != "" { + fmt.Fprintln(stderr, "Raw blockcheck report retained at:", path) + fmt.Fprintln(stderr, "Blockcheck check JSON path:", path+".check.json") + if saveErr := os.WriteFile(path+".stdout", raw.Bytes(), 0600); saveErr != nil { + err = fmt.Errorf("retaining stdout: %w", saveErr) + } + } + if err == nil { + var report []byte + report, err = os.ReadFile(path + ".check.json") + if err == nil { + var args, output string + args, err = selectBlockcheckReport(bytes.NewReader(report)) + if err == nil { + fmt.Fprintln(stderr, "Combined blockcheck JSON path:", path+".combined.json") + err = check(ctx, "blockcheckw", path, args, io.MultiWriter(&raw, stderr), stderr) + if saveErr := os.WriteFile(path+".stdout", raw.Bytes(), 0600); saveErr != nil { + err = fmt.Errorf("retaining stdout: %w", saveErr) + } + } + if err == nil { + report, err = os.ReadFile(path + ".combined.json") + } + if err == nil { + output, err = combinedBlockcheckOutput(bytes.NewReader(report), args) + } + if err == nil { + err = ctx.Err() + } + if err == nil { + _, err = fmt.Fprintln(stdout, output) + } + } + } + if err != nil { + fmt.Fprintln(stderr, "error: tune youtube:", err) + return 1 + } + return 0 +} + +func normalizeProtocol(protocol string) string { + switch protocol { + case "HTTPS/TLS1.2": + return "tls12" + case "HTTPS/TLS1.3": + return "tls13" + default: + return "" + } +} + +func selectBlockcheckReport(r io.Reader) (string, error) { + return selectBlockcheckArgs(r, "") +} + +func combinedBlockcheckOutput(r io.Reader, args string) (string, error) { + if args == "" { + return "", fmt.Errorf("empty combined args") + } + if _, err := selectBlockcheckArgs(r, args); err != nil { + return "", err + } + return "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=youtube.com " + args + "'", nil +} + +func selectBlockcheckArgs(r io.Reader, expected string) (string, error) { + d := json.NewDecoder(r) + var report struct { + Strategies []json.RawMessage `json:"strategies"` + } + if err := d.Decode(&report); err != nil { + return "", err + } + if err := d.Decode(new(json.RawMessage)); err != io.EOF { + return "", fmt.Errorf("expected exactly one JSON check report object") + } + if len(report.Strategies) == 0 { + return "", fmt.Errorf("check report requires a nonempty strategies array") + } + best := make(map[string]candidate) + seen := make(map[string]bool) + for i, record := range report.Strategies { + var fields map[string]json.RawMessage + if err := json.Unmarshal(record, &fields); err != nil { + return "", err + } + for _, key := range []string{"protocol", "args", "success_rate", "coverage", "passes_ok", "passes_total", "median_latency_ms", "median_speed_kbps"} { + if v, ok := fields[key]; !ok || bytes.Equal(bytes.TrimSpace(v), []byte("null")) { + return "", fmt.Errorf("result %d: missing or null %s", i, key) + } + } + var row struct { + candidate + Protocol string `json:"protocol"` + } + // v0.9.0 includes additional report metadata. Required measurements + // are checked above; unrelated metadata is not interpreted as evidence. + if err := json.Unmarshal(record, &row); err != nil { + return "", fmt.Errorf("result %d: %w", i, err) + } + protocol := normalizeProtocol(row.Protocol) + if protocol == "" { + return "", fmt.Errorf("result %d: unsupported check protocol %q", i, row.Protocol) + } + if _, exists := fields["group"]; !exists { + row.Group = "frontend" + } + if _, exists := fields["domains"]; !exists { + row.Domains = []string{"youtube.com"} + } + if err := row.validateMeasurements(); err != nil { + return "", fmt.Errorf("result %d: %w", i, err) + } + if row.Group != "frontend" || len(row.Domains) != 1 || row.Domains[0] != "youtube.com" { + return "", fmt.Errorf("result %d: unexpected target", i) + } + key := protocol + "\x00" + row.Args + if seen[key] { + return "", fmt.Errorf("result %d: duplicate args/protocol", i) + } + seen[key] = true + if expected != "" && (row.Args != expected || !fullyPassing(row.candidate) || row.PassesTotal != 3) { + return "", fmt.Errorf("result %d: combined args must pass exactly 3/3 for both protocols", i) + } + // Valid check evidence may describe args our inert output cannot represent. + // Exclude that candidate only; malformed measurements still fail above. + if err := row.validateArgs(); err != nil { + continue + } + if !fullyPassing(row.candidate) || row.PassesTotal < 3 { + continue + } + if old, ok := best[protocol]; !ok || better(row.candidate, old) { + best[protocol] = row.candidate + } + } + var args []string + payloads := make(map[string]bool) + for _, protocol := range []string{"tls12", "tls13"} { + c, ok := best[protocol] + if !ok { + return "", fmt.Errorf("no fully passing check strategy for both tls12 and tls13") + } + for _, token := range strings.Fields(c.Args) { + if strings.HasPrefix(token, "--payload=") { + if payloads[token] { + continue + } + payloads[token] = true + } + args = append(args, token) + } + } + return strings.Join(args, " "), nil +} diff --git a/cmd/zapret-autotune/tune_test.go b/cmd/zapret-autotune/tune_test.go new file mode 100644 index 0000000..9c0bf38 --- /dev/null +++ b/cmd/zapret-autotune/tune_test.go @@ -0,0 +1,358 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func reportFixture(t *testing.T) string { + t.Helper() + data, err := os.ReadFile("testdata/blockcheck-check.json") + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func TestBlockcheckAdapter(t *testing.T) { + valid := reportFixture(t) + got, err := selectBlockcheckReport(strings.NewReader(valid)) + want := "--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:tcp_md5:repeats=1 --in-range=-s1 --lua-desync=oob:urp=midsld" + if err != nil || got != want { + t.Fatalf("got %q, %v", got, err) + } + scan, err := os.ReadFile("testdata/blockcheck-scan.json") + if err != nil { + t.Fatal(err) + } + vanilla, err := os.ReadFile("testdata/blockcheck-scan.txt") + if err != nil { + t.Fatal(err) + } + for _, input := range []string{ + string(scan), string(vanilla), "", "null", "[]", `{}`, `{"strategies":[null]}`, valid + " {}", + strings.Replace(valid, `"HTTPS/TLS1.3"`, `"tls13"`, 1), + strings.Replace(valid, `"HTTPS/TLS1.3"`, `"QUIC"`, 1), + strings.Replace(valid, `"passes_total":3`, `"total":3`, 1), + strings.Replace(valid, `"coverage":1`, `"coverage":null`, 1), + strings.Replace(valid, `"coverage":1`, `"coverage":0.9`, 1), + strings.Replace(valid, `"success_rate":1`, `"success_rate":0.9`, 1), + strings.Replace(valid, `"passes_ok":3`, `"passes_ok":2`, 1), + strings.ReplaceAll(strings.ReplaceAll(valid, `"passes_ok":3`, `"passes_ok":1`), `"passes_total":3`, `"passes_total":1`), + strings.Replace(valid, `"median_speed_kbps":100`, `"median_speed_kbps":-1`, 1), + strings.Replace(valid, "--in-range=-s1", "nfqws2 --in-range=-s1", 1), + } { + if got, err := selectBlockcheckReport(strings.NewReader(input)); err == nil || got != "" { + t.Errorf("accepted unverified/invalid report: %s => %q, %v", input, got, err) + } + } + for _, protocol := range []string{"HTTPS/TLS1.2", "HTTPS/TLS1.3"} { + if normalizeProtocol(protocol) == "" { + t.Fatal("actual protocol not normalized") + } + } +} + +func TestBlockcheckIndependentSelection(t *testing.T) { + var report struct { + Strategies []map[string]interface{} `json:"strategies"` + } + if err := json.Unmarshal([]byte(reportFixture(t)), &report); err != nil { + t.Fatal(err) + } + row := make(map[string]interface{}) + for key, value := range report.Strategies[0] { + row[key] = value + } + row["args"], row["median_speed_kbps"] = "--lua-desync=multisplit", 200 + report.Strategies = append(report.Strategies, row) + data, _ := json.Marshal(report) + got, err := selectBlockcheckReport(bytes.NewReader(data)) + if err != nil || got != "--lua-desync=multisplit --in-range=-s1 --lua-desync=oob:urp=midsld" { + t.Fatalf("independent winner lost: %q, %v", got, err) + } + row["passes_ok"] = 2 + data, _ = json.Marshal(report) + got, err = selectBlockcheckReport(bytes.NewReader(data)) + if err != nil || strings.Contains(got, "multisplit") { + t.Fatalf("partial winner accepted: %q, %v", got, err) + } + report.Strategies = append(report.Strategies, report.Strategies[0]) + data, _ = json.Marshal(report) + if _, err := selectBlockcheckReport(bytes.NewReader(data)); err == nil { + t.Fatal("duplicate accepted") + } +} + +func TestBlockcheckUnsupportedCandidates(t *testing.T) { + var report struct { + Strategies []map[string]interface{} `json:"strategies"` + } + if err := json.Unmarshal([]byte(reportFixture(t)), &report); err != nil { + t.Fatal(err) + } + want, err := selectBlockcheckReport(strings.NewReader(reportFixture(t))) + if err != nil { + t.Fatal(err) + } + // Confirmed v0.9.0 inline Lua: successful evidence, but not shell-inert args. + unsupported := "--lua-init=fake_default_tls=tls_mod(fake_default_tls,'rnd') --lua-desync=multisplit" + for _, original := range report.Strategies { + row := make(map[string]interface{}) + for key, value := range original { + row[key] = value + } + row["args"], row["median_speed_kbps"] = unsupported, 1000 + report.Strategies = append(report.Strategies, row) + } + check := func(wantError bool) { + t.Helper() + data, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + got, err := selectBlockcheckReport(bytes.NewReader(data)) + if wantError { + if err == nil || got != "" { + t.Fatalf("invalid report emitted options: %q, %v", got, err) + } + } else if err != nil || got != want { + t.Fatalf("safe winners lost: %q, %v", got, err) + } + } + check(false) + // Both unsuccessful and successful unsupported rows are candidate-local. + row := report.Strategies[2] + row["passes_ok"], row["success_rate"] = 2, 0.5 + check(false) + row["passes_ok"], row["success_rate"] = 3, 1 + row["args"] = "--unsupported-option=1" + check(false) + row["args"] = unsupported + // Unsupported grammar must not hide malformed measurements or targeting. + row["passes_ok"] = 4 + check(true) + row["passes_ok"] = 3 + row["median_speed_kbps"] = -1 + check(true) + row["median_speed_kbps"] = 1000 + delete(row, "coverage") + check(true) + row["coverage"] = 1 + row["domains"] = []string{"example.com"} + check(true) + delete(row, "domains") + // Losing the only supported TLS1.2 row must fail even with supported TLS1.3. + report.Strategies = report.Strategies[1:] + check(true) + // No supported candidates for either TLS protocol also fails closed. + report.Strategies = report.Strategies[1:] + check(true) +} + +func TestTuneCLI(t *testing.T) { + for _, mode := range []string{"tune", "tune-invalid", "tune-error", "tune-check-error", "tune-combined-error", "tune-combined-fail"} { + t.Run(mode, func(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + binary := fakeBlockcheck(t, mode) + installed := filepath.Join(filepath.Dir(binary), "blockcheckw") + if err := os.Rename(binary, installed); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", filepath.Dir(installed)) + report := reportFixture(t) + if mode == "tune-invalid" { + report = `{"strategies":[]}` + } + t.Setenv("ZAPRET_TEST_REPORT", report) + argv := filepath.Join(t.TempDir(), "argv.jsonl") + t.Setenv("ZAPRET_TEST_ARGV", argv) + var stdout, stderr bytes.Buffer + code := run([]string{"tune", "youtube"}, &stdout, &stderr) + if mode == "tune" { + combined, err := selectBlockcheckReport(strings.NewReader(report)) + want := "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=youtube.com " + combined + "'\n" + if err != nil || code != 0 || stdout.String() != want || strings.Contains(stdout.String(), "--new") { + t.Fatalf("code=%d out=%q err=%q", code, stdout.String(), stderr.String()) + } + } else if code == 0 || stdout.Len() != 0 { + t.Fatalf("failure emitted options: code=%d out=%q", code, stdout.String()) + } + data, err := os.ReadFile(argv) + if err != nil { + t.Fatal(err) + } + decoder := json.NewDecoder(bytes.NewReader(data)) + var args []string + if err := decoder.Decode(&args); err != nil { + t.Fatal(err) + } + path := args[len(args)-1] + if data, err := os.ReadFile(path); err != nil || string(data) != "# blockcheckw scan results\n* SUMMARY\n" { + t.Fatalf("scan output not preserved: %q, %v", data, err) + } + if mode != "tune-error" { + if err := decoder.Decode(&args); err != nil || args[2] != "check" || args[4] == path || filepath.Base(args[4]) != "2026-10-09_14-42_report_vanilla.txt" { + t.Fatalf("scan not followed by check: %q, %v", args, err) + } + if data, err := os.ReadFile(path + ".check.json"); err != nil || string(data) != report { + t.Fatalf("check report not preserved: %q, %v", data, err) + } + } + if mode == "tune" || strings.HasPrefix(mode, "tune-combined-") { + if err := decoder.Decode(&args); err != nil { + t.Fatal(err) + } + want := []string{"--workers", "32", "check", "--from-file", path + ".combined.txt", "--domain", "youtube.com", "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", "--output", path + ".combined.json"} + if fmt.Sprint(args) != fmt.Sprint(want) { + t.Fatalf("combined check argv: %q", args) + } + } + if err := decoder.Decode(&args); err != io.EOF { + t.Fatalf("unexpected extra command: %q, %v", args, err) + } + if data, err := os.ReadFile(path + ".stdout"); err != nil || !strings.Contains(string(data), "untrusted scanner output") { + t.Fatalf("stdout not retained: %q, %v", data, err) + } + }) + } +} + +func TestTuneUsesCheckFileAndPreservesFailures(t *testing.T) { + for _, mode := range []string{"success", "scan", "parse", "write", "missing-check", "combined-error", "combined-fail", "combined-missing", "combined-wrong-args"} { + t.Run(mode, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "scan.txt") + report := reportFixture(t) + if mode == "parse" { + report = "[" + } + scan := func(ctx context.Context, _, _ string, out, _ io.Writer) (string, error) { + if _, ok := ctx.Deadline(); ok { + t.Fatal("unexpected deadline") + } + if err := os.WriteFile(path, []byte("vanilla"), 0600); err != nil { + t.Fatal(err) + } + // Even a valid check-shaped stdout object must never be selected. + fmt.Fprint(out, reportFixture(t)) + if mode != "missing-check" { + if err := os.WriteFile(path+".check.json", []byte(report), 0600); err != nil { + t.Fatal(err) + } + } + if mode == "scan" { + return path, fmt.Errorf("scan failed") + } + return path, nil + } + var stdout, stderr bytes.Buffer + var out io.Writer = &stdout + if mode == "write" { + out = failingWriter{} + } + checked := false + check := func(ctx context.Context, binary, gotPath, args string, _, _ io.Writer) error { + checked = true + if ctx.Err() != nil || binary != "blockcheckw" || gotPath != path { + t.Fatal("incorrect combined check invocation") + } + want, err := selectBlockcheckReport(strings.NewReader(reportFixture(t))) + if err != nil || args != want { + t.Fatalf("combined args=%q, want=%q, err=%v", args, want, err) + } + if mode == "combined-error" { + return fmt.Errorf("combined check failed") + } + if mode == "combined-missing" { + return nil + } + combined := combinedReportFixture(args) + if mode == "combined-fail" { + combined = strings.Replace(combined, `"passes_ok":3`, `"passes_ok":2`, 1) + } + if mode == "combined-wrong-args" { + combined = combinedReportFixture("--lua-desync=multisplit") + } + return os.WriteFile(path+".combined.json", []byte(combined), 0600) + } + code := tuneYoutubeWithCheck(out, &stderr, scan, check) + if checked != (mode == "success" || mode == "write" || strings.HasPrefix(mode, "combined-")) { + t.Fatalf("unexpected combined check: %v", checked) + } + if mode == "success" { + if code != 0 { + t.Fatal(stderr.String()) + } + } else if code != 1 || stdout.Len() != 0 { + t.Fatalf("failed run emitted options: %d %q", code, stdout.String()) + } + if data, err := os.ReadFile(path); err != nil || string(data) != "vanilla" { + t.Fatal("vanilla report changed or removed") + } + if data, err := os.ReadFile(path + ".stdout"); err != nil || string(data) != reportFixture(t) { + t.Fatal("raw stdout changed or removed") + } + }) + } +} + +func combinedReportFixture(args string) string { + encoded, _ := json.Marshal(args) + row := `{"protocol":%q,"args":%s,"success_rate":1,"coverage":1,"passes_ok":3,"passes_total":3,"median_latency_ms":10,"median_speed_kbps":100}` + return `{"strategies":[` + fmt.Sprintf(row, "HTTPS/TLS1.2", encoded) + "," + fmt.Sprintf(row, "HTTPS/TLS1.3", encoded) + "]}" +} + +func TestCombinedVanillaRowsAndSelection(t *testing.T) { + args12 := "--payload=tls_client_hello --lua-desync=hostfakesplit:ip_autottl=-2,3-20:repeats=1" + args13 := "--payload=tls_client_hello --lua-desync=multidisorder:pos=2,midsld:seqovl=1" + combined := args12 + " --lua-desync=multidisorder:pos=2,midsld:seqovl=1" + report := combinedReportFixture(args12) + // Only the TLS1.3 row changes; selection must preserve TLS1.2-then-TLS1.3 order. + i := strings.LastIndex(report, args12) + report = report[:i] + args13 + report[i+len(args12):] + got, err := selectBlockcheckReport(strings.NewReader(report)) + if err != nil || got != combined { + t.Fatalf("combined=%q, err=%v", got, err) + } + rows, err := combinedVanillaRows(got) + want := "curl_test_https_tls12 ipv4 youtube.com : nfqws2 " + combined + "\n" + + "curl_test_https_tls13 ipv4 youtube.com : nfqws2 " + combined + "\n" + if err != nil || rows != want { + t.Fatalf("rows=%q, err=%v", rows, err) + } + for _, unsafe := range []string{"", "--lua-desync=multisplit\n--new", "--payload=tls_client_hello;exit", "--new"} { + if rows, err := combinedVanillaRows(unsafe); err == nil || rows != "" { + t.Fatalf("unsafe vanilla accepted: %q", rows) + } + } +} + +func TestCombinedBlockcheckOutput(t *testing.T) { + args := "--payload=tls_client_hello --lua-desync=hostfakesplit --lua-desync=multidisorder" + valid := combinedReportFixture(args) + want := "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=youtube.com " + args + "'" + if got, err := combinedBlockcheckOutput(strings.NewReader(valid), args); err != nil || got != want { + t.Fatalf("single profile=%q, err=%v", got, err) + } + for _, report := range []string{ + valid + " {}", `{"strategies":[]}`, + strings.Replace(valid, `"passes_ok":3`, `"passes_ok":2`, 1), + strings.ReplaceAll(strings.ReplaceAll(valid, `"passes_ok":3`, `"passes_ok":4`), `"passes_total":3`, `"passes_total":4`), + strings.Replace(valid, `"coverage":1`, `"coverage":0.9`, 1), + strings.Replace(valid, `"success_rate":1`, `"success_rate":0`, 1), + strings.Replace(valid, "HTTPS/TLS1.3", "HTTPS/TLS1.2", 1), + strings.Replace(valid, args, "--lua-desync=multisplit", 1), + strings.Replace(valid, `"passes_total":3`, `"passes_total":null`, 1), + } { + if got, err := combinedBlockcheckOutput(strings.NewReader(report), args); err == nil || got != "" { + t.Fatalf("failed combined check emitted options: %q, %v", got, err) + } + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d68e531 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module zapret-autotune + +go 1.22