Files
zapret-autotune/cmd/zapret-autotune/blockcheck.go
T
2026-10-10 22:51:58 +03:00

348 lines
12 KiB
Go

package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
)
// Managed seeds are only starting hypotheses from an existing known-good
// profile; blockcheckw must re-rank and re-verify them for the exact hostname.
func managedSeedArgs(host string) string {
host = strings.ToLower(host)
switch {
case host == "www.youtube.com":
return "--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld"
case host == "googlevideo.com" || strings.HasSuffix(host, ".googlevideo.com"):
return "--payload=tls_client_hello --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN"
default:
return ""
}
}
// runBlockcheck leaves the output file (including partial results on failure) to
// the caller to read and remove. stdout/stderr receive the unmodified streams.
// The caller controls cancellation through ctx; no overall deadline is imposed here.
func runBlockcheck(ctx context.Context, binary, domain string, stdout, stderr io.Writer) (string, error) {
return runBlockcheckWithConflictCleanup(ctx, binary, domain, stdout, stderr, false)
}
// Managed recovery tests known seeds before the default corpus. Each invocation
// cleans its own process group; foreign processes remain protected.
func runManagedBlockcheck(ctx context.Context, binary, domain string, stdout, stderr io.Writer) (string, error) {
domain = strings.ToLower(domain)
if args := managedSeedArgs(domain); args != "" {
// A transient endpoint timeout can invalidate one protocol's strict 3/3
// check even when the seeded candidate works; retry this tiny check before
// spending an hour on the full built-in corpus.
for attempt := 0; attempt < 5; attempt++ {
path, err := runManagedSeedCheck(ctx, binary, domain, args, stdout, stderr)
if err != nil {
return path, err
}
report, readErr := os.ReadFile(path + ".check.json")
group := "frontend"
if domain == "googlevideo.com" || strings.HasSuffix(domain, ".googlevideo.com") {
group = "googlevideo"
}
if readErr == nil {
if _, selectErr := selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, strings.ToLower(domain)); selectErr == nil {
return path, nil
}
}
_ = os.Remove(path)
_ = os.Remove(path + ".check.json")
}
}
return runBlockcheckWithConflictCleanup(ctx, binary, domain, stdout, stderr, false)
}
func runBlockcheckWithConflictCleanup(ctx context.Context, binary, domain string, stdout, stderr io.Writer, allowConflictCleanup bool) (string, error) {
if !validDomain(domain) {
return "", fmt.Errorf("invalid DNS hostname: %q", domain)
}
if err := ctx.Err(); err != nil {
return "", err
}
output, err := os.CreateTemp("", "blockcheckw-*.txt")
if err != nil {
return "", err
}
path := output.Name()
if err := output.Close(); err != nil {
os.Remove(path)
return "", err
}
workdir, err := os.MkdirTemp("", "blockcheckw-scan-*")
if err != nil {
return path, err
}
// Resolve relative executable paths before changing the child's directory.
if strings.ContainsRune(binary, os.PathSeparator) && !filepath.IsAbs(binary) {
binary, err = filepath.Abs(binary)
if err != nil {
return path, fmt.Errorf("scan artifacts in %s (output %s): %w", workdir, path, err)
}
}
domain = strings.ToLower(domain)
scanArgs := []string{"--workers", "8", "scan", "--domain", domain, "--protocols", "tls12,tls13", "--auto"}
if !allowConflictCleanup {
scanArgs = append(scanArgs, "--no-conflict-cleanup")
}
scanArgs = append(scanArgs, "--output", path)
err = runOwnedBlockcheckCommand(ctx, binary, workdir, scanArgs, stdout, stderr)
if err != nil {
return path, fmt.Errorf("scan artifacts in %s (output %s): %w", workdir, path, err)
}
vanilla, err := blockcheckVanillaReport(workdir)
if err != nil {
return path, fmt.Errorf("scan artifacts in %s (output %s): %w", workdir, path, err)
}
// Leave shortlist semantics to blockcheckw. Only check's JSON contains
// repeated real-data measurements; scan stdout is not evidence.
checkArgs := []string{"--workers", "32", "check", "--from-file", vanilla, "--domain", domain, "--take", "10", "--passes", "3", "--auto"}
if !allowConflictCleanup {
checkArgs = append(checkArgs, "--no-conflict-cleanup")
}
checkArgs = append(checkArgs, "--output", path+".check.json")
err = runOwnedBlockcheckCommand(ctx, binary, workdir, checkArgs, stdout, stderr)
if err != nil {
return path, fmt.Errorf("check (vanilla %s, scan output %s, artifacts in %s): %w", vanilla, path, workdir, err)
}
if err := os.RemoveAll(workdir); err != nil {
return path, fmt.Errorf("removing scan artifacts in %s: %w", workdir, err)
}
return path, nil
}
func runManagedSeedCheck(ctx context.Context, binary, host, args string, stdout, stderr io.Writer) (string, error) {
rows, err := combinedVanillaRowsForHost(args, host)
if err != nil {
return "", err
}
f, err := os.CreateTemp("", "blockcheckw-managed-seed-*.txt")
if err != nil {
return "", err
}
path := f.Name()
if err := f.Close(); err != nil {
return path, err
}
input := path + ".input"
if err := os.WriteFile(input, []byte(rows), 0600); err != nil {
return path, err
}
checkArgs := []string{"--workers", "32", "check", "--from-file", input, "--domain", strings.ToLower(host), "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", "--output", path + ".check.json"}
if err := runOwnedBlockcheckCommand(ctx, binary, "", checkArgs, stdout, stderr); err != nil {
return path, err
}
return path, nil
}
func blockcheckVanillaReport(dir string) (string, error) {
entries, err := os.ReadDir(dir)
if err != nil {
return "", err
}
var report string
for _, entry := range entries {
if !strings.HasSuffix(entry.Name(), "_report_vanilla.txt") {
continue
}
info, err := entry.Info()
if err != nil {
return "", err
}
if !info.Mode().IsRegular() {
return "", fmt.Errorf("vanilla report %q is not a regular file", entry.Name())
}
if report != "" {
return "", fmt.Errorf("multiple vanilla reports")
}
report = filepath.Join(dir, entry.Name())
}
if report == "" {
return "", fmt.Errorf("no *_report_vanilla.txt generated")
}
return report, nil
}
func combinedVanillaRows(args string) (string, error) {
return combinedVanillaRowsForHost(args, "youtube.com")
}
func combinedVanillaRowsForHost(args, host string) (string, error) {
if !validDomain(host) {
return "", fmt.Errorf("invalid combined hostname")
}
c := candidate{Args: args}
if strings.TrimSpace(args) == "" {
return "", fmt.Errorf("empty combined args")
}
if err := c.validateArgs(); err != nil {
return "", err
}
return "curl_test_https_tls12 ipv4 " + host + " : nfqws2 " + args + "\n" +
"curl_test_https_tls13 ipv4 " + host + " : nfqws2 " + args + "\n", nil
}
// Retain the exact combined input and report, including on failure.
func runCombinedBlockcheck(ctx context.Context, binary, path, args string, stdout, stderr io.Writer) error {
return runCombinedBlockcheckForHost(ctx, binary, path, args, "youtube.com", stdout, stderr)
}
func runCombinedBlockcheckForHost(ctx context.Context, binary, path, args, host string, stdout, stderr io.Writer) error {
return runCombinedBlockcheckForHostWithConflictCleanup(ctx, binary, path, args, host, stdout, stderr, false)
}
func runManagedCombinedBlockcheckForHost(ctx context.Context, binary, path, args, host string, stdout, stderr io.Writer) error {
return runCombinedBlockcheckForHostWithConflictCleanup(ctx, binary, path, args, host, stdout, stderr, false)
}
func runCombinedBlockcheckForHostWithConflictCleanup(ctx context.Context, binary, path, args, host string, stdout, stderr io.Writer, allowConflictCleanup bool) error {
rows, err := combinedVanillaRowsForHost(args, host)
if err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
input := path + ".combined.txt"
if err := os.WriteFile(input, []byte(rows), 0600); err != nil {
return err
}
checkArgs := []string{"--workers", "32", "check", "--from-file", input, "--domain", host, "--take", "1", "--passes", "3", "--auto"}
if !allowConflictCleanup {
checkArgs = append(checkArgs, "--no-conflict-cleanup")
}
checkArgs = append(checkArgs, "--output", path+".combined.json")
if err := runOwnedBlockcheckCommand(ctx, binary, "", checkArgs, stdout, stderr); err != nil {
return fmt.Errorf("combined check (input %s, output %s): %w", input, path+".combined.json", err)
}
return nil
}
func runBlockcheckCommand(ctx context.Context, binary, dir string, args []string, stdout, stderr io.Writer) error {
return runProcessGroupCommand(ctx, binary, dir, args, stdout, stderr, false)
}
func runOwnedBlockcheckCommand(ctx context.Context, binary, dir string, args []string, stdout, stderr io.Writer) error {
for i, arg := range args {
if arg == "--output" && i+1 < len(args) {
f, err := os.OpenFile(args[i+1]+".stderr", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return err
}
if stderr == nil {
stderr = io.Discard
}
err = runProcessGroupCommand(ctx, binary, dir, args, stdout, io.MultiWriter(stderr, f), true)
return errors.Join(err, f.Close())
}
}
return runProcessGroupCommand(ctx, binary, dir, args, stdout, stderr, true)
}
func runProcessGroupCommand(ctx context.Context, binary, dir string, args []string, stdout, stderr io.Writer, cleanup bool) error {
cmd := exec.CommandContext(ctx, binary, args...)
cmd.Dir = dir
cmd.Stdout, cmd.Stderr = stdout, stderr
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
cmd.Cancel = func() error {
if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM); err != nil {
if errors.Is(err, syscall.ESRCH) {
return os.ErrProcessDone
}
return err
}
// Wait also waits for Cancel: finish group cleanup even if the leader
// exits on TERM while a descendant ignores it and holds the pipes open.
time.Sleep(5 * time.Second)
if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); !errors.Is(err, syscall.ESRCH) {
return err
}
return nil
}
cmd.WaitDelay = 5 * time.Second
if err := cmd.Start(); err != nil {
return err
}
err := cmd.Wait()
// Workers can outlive a successful leader and retain NFQUEUE bindings.
// Setpgid scopes cleanup to this invocation, never all nfqws2 processes.
if cleanup {
if cleanupErr := cleanupBlockcheckGroup(cmd.Process.Pid); cleanupErr != nil {
err = errors.Join(err, &managedRecoveryFatal{Err: cleanupErr})
}
}
if ctx.Err() != nil {
err = errors.Join(ctx.Err(), err)
}
return err
}
func cleanupBlockcheckGroup(pgid int) error {
for _, signal := range []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL} {
if err := syscall.Kill(-pgid, signal); err != nil && !errors.Is(err, syscall.ESRCH) {
return fmt.Errorf("blockcheck worker cleanup: %w", err)
}
deadline := time.Now().Add(time.Second)
for {
alive, err := blockcheckGroupAlive(pgid)
if err != nil || !alive {
return err
}
if time.Now().After(deadline) {
break
}
time.Sleep(10 * time.Millisecond)
}
}
return fmt.Errorf("blockcheck workers remain in process group %d", pgid)
}
func blockcheckGroupAlive(pgid int) (bool, error) {
entries, err := os.ReadDir("/proc")
if err != nil {
return false, err
}
for _, entry := range entries {
if _, err := strconv.Atoi(entry.Name()); err != nil {
continue
}
data, err := os.ReadFile(filepath.Join("/proc", entry.Name(), "stat"))
if errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ESRCH) {
continue
}
if err != nil {
return false, err
}
// comm may contain spaces/parentheses; pgrp is field 5 after comm.
end := strings.LastIndexByte(string(data), ')')
if end < 0 {
return false, fmt.Errorf("invalid process stat")
}
fields := strings.Fields(string(data[end+1:]))
if len(fields) < 3 {
return false, fmt.Errorf("short process stat")
}
group, err := strconv.Atoi(fields[2])
if err != nil {
return false, err
}
if group == pgid && fields[0] != "Z" && fields[0] != "X" {
return true, nil
}
}
return false, nil
}