From d508132b4f30e06ddfcba2e0ee07be149d2f6f4f Mon Sep 17 00:00:00 2001 From: Zeusina Date: Sat, 10 Oct 2026 14:50:23 +0300 Subject: [PATCH] fix(blockcheck): reap owned workers after each invocation Co-authored-by: GPT-6 --- cmd/zapret-autotune/blockcheck.go | 111 ++++++++++++++++++++++--- cmd/zapret-autotune/blockcheck_test.go | 45 ++++++++-- deploy/managed-target/check_static.py | 2 +- deploy/managed-target/compose.yaml | 4 +- 4 files changed, 141 insertions(+), 21 deletions(-) diff --git a/cmd/zapret-autotune/blockcheck.go b/cmd/zapret-autotune/blockcheck.go index 05457bc..111c31f 100644 --- a/cmd/zapret-autotune/blockcheck.go +++ b/cmd/zapret-autotune/blockcheck.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -35,9 +36,8 @@ func runBlockcheck(ctx context.Context, binary, domain string, stdout, stderr io return runBlockcheckWithConflictCleanup(ctx, binary, domain, stdout, stderr, false) } -// Managed recovery has already stopped zapret inside its dedicated network -// namespace, so blockcheck may clean up its own scan workers before shortlist -// checks. Other callers preserve foreign nfqws2 processes by default. +// Managed recovery tests known seeds before the default corpus. Each invocation +// cleans its own process group; foreign processes remain protected. func runManagedBlockcheck(ctx context.Context, binary, domain string, stdout, stderr io.Writer) (string, error) { domain = strings.ToLower(domain) if args := managedSeedArgs(domain); args != "" { @@ -47,7 +47,7 @@ func runManagedBlockcheck(ctx context.Context, binary, domain string, stdout, st for attempt := 0; attempt < 5; attempt++ { path, err := runManagedSeedCheck(ctx, binary, domain, args, stdout, stderr) if err != nil { - break + return path, err } report, readErr := os.ReadFile(path + ".check.json") group := "frontend" @@ -63,7 +63,7 @@ func runManagedBlockcheck(ctx context.Context, binary, domain string, stdout, st _ = os.Remove(path + ".check.json") } } - return runBlockcheckWithConflictCleanup(ctx, binary, domain, stdout, stderr, true) + return runBlockcheckWithConflictCleanup(ctx, binary, domain, stdout, stderr, false) } func runBlockcheckWithConflictCleanup(ctx context.Context, binary, domain string, stdout, stderr io.Writer, allowConflictCleanup bool) (string, error) { @@ -99,7 +99,7 @@ func runBlockcheckWithConflictCleanup(ctx context.Context, binary, domain string scanArgs = append(scanArgs, "--no-conflict-cleanup") } scanArgs = append(scanArgs, "--output", path) - err = runBlockcheckCommand(ctx, binary, workdir, scanArgs, stdout, stderr) + err = runOwnedBlockcheckCommand(ctx, binary, workdir, scanArgs, stdout, stderr) if err != nil { return path, fmt.Errorf("scan artifacts in %s (output %s): %w", workdir, path, err) } @@ -114,7 +114,7 @@ func runBlockcheckWithConflictCleanup(ctx context.Context, binary, domain string checkArgs = append(checkArgs, "--no-conflict-cleanup") } checkArgs = append(checkArgs, "--output", path+".check.json") - err = runBlockcheckCommand(ctx, binary, workdir, checkArgs, stdout, stderr) + err = runOwnedBlockcheckCommand(ctx, binary, workdir, checkArgs, stdout, stderr) if err != nil { return path, fmt.Errorf("check (vanilla %s, scan output %s, artifacts in %s): %w", vanilla, path, workdir, err) } @@ -141,8 +141,8 @@ func runManagedSeedCheck(ctx context.Context, binary, host, args string, stdout, if err := os.WriteFile(input, []byte(rows), 0600); err != nil { return path, err } - checkArgs := []string{"--workers", "32", "check", "--from-file", input, "--domain", strings.ToLower(host), "--take", "1", "--passes", "3", "--auto", "--output", path + ".check.json"} - if err := runBlockcheckCommand(ctx, binary, "", checkArgs, stdout, stderr); err != nil { + checkArgs := []string{"--workers", "32", "check", "--from-file", input, "--domain", strings.ToLower(host), "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", "--output", path + ".check.json"} + if err := runOwnedBlockcheckCommand(ctx, binary, "", checkArgs, stdout, stderr); err != nil { return path, err } return path, nil @@ -205,7 +205,7 @@ func runCombinedBlockcheckForHost(ctx context.Context, binary, path, args, host } func runManagedCombinedBlockcheckForHost(ctx context.Context, binary, path, args, host string, stdout, stderr io.Writer) error { - return runCombinedBlockcheckForHostWithConflictCleanup(ctx, binary, path, args, host, stdout, stderr, true) + return runCombinedBlockcheckForHostWithConflictCleanup(ctx, binary, path, args, host, stdout, stderr, false) } func runCombinedBlockcheckForHostWithConflictCleanup(ctx context.Context, binary, path, args, host string, stdout, stderr io.Writer, allowConflictCleanup bool) error { @@ -225,13 +225,34 @@ func runCombinedBlockcheckForHostWithConflictCleanup(ctx context.Context, binary checkArgs = append(checkArgs, "--no-conflict-cleanup") } checkArgs = append(checkArgs, "--output", path+".combined.json") - if err := runBlockcheckCommand(ctx, binary, "", checkArgs, stdout, stderr); err != nil { + if err := runOwnedBlockcheckCommand(ctx, binary, "", checkArgs, stdout, stderr); err != nil { return fmt.Errorf("combined check (input %s, output %s): %w", input, path+".combined.json", err) } return nil } func runBlockcheckCommand(ctx context.Context, binary, dir string, args []string, stdout, stderr io.Writer) error { + return runProcessGroupCommand(ctx, binary, dir, args, stdout, stderr, false) +} + +func runOwnedBlockcheckCommand(ctx context.Context, binary, dir string, args []string, stdout, stderr io.Writer) error { + for i, arg := range args { + if arg == "--output" && i+1 < len(args) { + f, err := os.OpenFile(args[i+1]+".stderr", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + if stderr == nil { + stderr = io.Discard + } + err = runProcessGroupCommand(ctx, binary, dir, args, stdout, io.MultiWriter(stderr, f), true) + return errors.Join(err, f.Close()) + } + } + return runProcessGroupCommand(ctx, binary, dir, args, stdout, stderr, true) +} + +func runProcessGroupCommand(ctx context.Context, binary, dir string, args []string, stdout, stderr io.Writer, cleanup bool) error { cmd := exec.CommandContext(ctx, binary, args...) cmd.Dir = dir cmd.Stdout, cmd.Stderr = stdout, stderr @@ -252,9 +273,75 @@ func runBlockcheckCommand(ctx context.Context, binary, dir string, args []string return nil } cmd.WaitDelay = 5 * time.Second - err := cmd.Run() + if err := cmd.Start(); err != nil { + return err + } + err := cmd.Wait() + // Workers can outlive a successful leader and retain NFQUEUE bindings. + // Setpgid scopes cleanup to this invocation, never all nfqws2 processes. + if cleanup { + if cleanupErr := cleanupBlockcheckGroup(cmd.Process.Pid); cleanupErr != nil { + err = errors.Join(err, &managedRecoveryFatal{Err: cleanupErr}) + } + } if ctx.Err() != nil { err = errors.Join(ctx.Err(), err) } return err } + +func cleanupBlockcheckGroup(pgid int) error { + for _, signal := range []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL} { + if err := syscall.Kill(-pgid, signal); err != nil && !errors.Is(err, syscall.ESRCH) { + return fmt.Errorf("blockcheck worker cleanup: %w", err) + } + deadline := time.Now().Add(time.Second) + for { + alive, err := blockcheckGroupAlive(pgid) + if err != nil || !alive { + return err + } + if time.Now().After(deadline) { + break + } + time.Sleep(10 * time.Millisecond) + } + } + return fmt.Errorf("blockcheck workers remain in process group %d", pgid) +} + +func blockcheckGroupAlive(pgid int) (bool, error) { + entries, err := os.ReadDir("/proc") + if err != nil { + return false, err + } + for _, entry := range entries { + if _, err := strconv.Atoi(entry.Name()); err != nil { + continue + } + data, err := os.ReadFile(filepath.Join("/proc", entry.Name(), "stat")) + if errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ESRCH) { + continue + } + if err != nil { + return false, err + } + // comm may contain spaces/parentheses; pgrp is field 5 after comm. + end := strings.LastIndexByte(string(data), ')') + if end < 0 { + return false, fmt.Errorf("invalid process stat") + } + fields := strings.Fields(string(data[end+1:])) + if len(fields) < 3 { + return false, fmt.Errorf("short process stat") + } + group, err := strconv.Atoi(fields[2]) + if err != nil { + return false, err + } + if group == pgid && fields[0] != "Z" && fields[0] != "X" { + return true, nil + } + } + return false, nil +} diff --git a/cmd/zapret-autotune/blockcheck_test.go b/cmd/zapret-autotune/blockcheck_test.go index 1f4ed63..04f8ace 100644 --- a/cmd/zapret-autotune/blockcheck_test.go +++ b/cmd/zapret-autotune/blockcheck_test.go @@ -26,6 +26,20 @@ func init() { if mode == "" { return } + if mode == "orphan" { + if len(os.Args) == 2 && os.Args[1] == "--child" { + signal.Ignore(syscall.SIGTERM) + for { + time.Sleep(time.Hour) + } + } + child := exec.Command(os.Args[0], "--child") + if err := child.Start(); err != nil { + os.Exit(3) + } + fmt.Fprintln(os.Stdout, child.Process.Pid) + os.Exit(0) + } if len(os.Args) == 2 && os.Args[1] == "--child" { if mode == "kill" { signal.Ignore(syscall.SIGTERM) @@ -201,7 +215,7 @@ func TestRunBlockcheckArgvAndStreams(t *testing.T) { } } -func TestRunManagedBlockcheckAllowsWorkerCleanup(t *testing.T) { +func TestRunManagedBlockcheckProtectsForeignWorkers(t *testing.T) { t.Setenv("TMPDIR", t.TempDir()) binary := fakeBlockcheck(t, "success") var stdout bytes.Buffer @@ -220,10 +234,8 @@ func TestRunManagedBlockcheckAllowsWorkerCleanup(t *testing.T) { t.Fatal(err) } for _, args := range [][]string{scanArgs, checkArgs} { - for _, arg := range args { - if arg == "--no-conflict-cleanup" { - t.Fatalf("managed blockcheck retained no-cleanup flag: %q", args) - } + if !slices.Contains(args, "--no-conflict-cleanup") { + t.Fatalf("managed blockcheck permits foreign cleanup: %q", args) } } } @@ -257,7 +269,7 @@ func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) { if err := decoder.Decode(&checkArgs); err != nil { t.Fatal(err) } - if !slices.Contains(checkArgs, "--from-file") || !slices.Contains(checkArgs, "--passes") || slices.Contains(checkArgs, "--no-conflict-cleanup") { + if !slices.Contains(checkArgs, "--from-file") || !slices.Contains(checkArgs, "--passes") || !slices.Contains(checkArgs, "--no-conflict-cleanup") { t.Fatalf("managed seeded check=%q", checkArgs) } if seed := managedSeedArgs("rr3.googlevideo.com"); !strings.Contains(seed, "fakeddisorder") { @@ -347,6 +359,27 @@ func TestRunBlockcheckProcessError(t *testing.T) { type processReady struct{ pid chan int } +func TestBlockcheckCleansWorkersAfterSuccessfulLeader(t *testing.T) { + binary := fakeBlockcheck(t, "orphan") + var stdout bytes.Buffer + if err := runOwnedBlockcheckCommand(context.Background(), binary, "", nil, &stdout, io.Discard); err != nil { + t.Fatal(err) + } + pid, err := strconv.Atoi(strings.TrimSpace(stdout.String())) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)) + if err == nil { + fields := strings.Fields(string(data[strings.LastIndexByte(string(data), ')')+1:])) + if fields[0] != "Z" && fields[0] != "X" { + t.Fatalf("worker survived cleanup: %s", data) + } + } else if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, syscall.ESRCH) { + t.Fatal(err) + } +} + func (w processReady) Write(p []byte) (int, error) { pid, err := strconv.Atoi(strings.TrimSpace(string(p))) if err != nil { diff --git a/deploy/managed-target/check_static.py b/deploy/managed-target/check_static.py index 4db9b6f..5867b5a 100644 --- a/deploy/managed-target/check_static.py +++ b/deploy/managed-target/check_static.py @@ -12,7 +12,7 @@ readme = (directory / "README.md").read_text() # parser if this definition grows beyond its single target service. assert " privileged: false\n" in compose assert " cap_drop: [ALL]\n" in compose -assert " cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID]\n" in compose +assert " cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID, KILL]\n" in compose assert compose.count(" - type: bind\n") == 1 assert " source: ./target-state\n" in compose assert " target: /var/lib/zapret-autotune/target\n" in compose diff --git a/deploy/managed-target/compose.yaml b/deploy/managed-target/compose.yaml index 0dbe085..0134f01 100644 --- a/deploy/managed-target/compose.yaml +++ b/deploy/managed-target/compose.yaml @@ -7,8 +7,8 @@ services: dockerfile: Dockerfile.target privileged: false cap_drop: [ALL] - # nft/NFQUEUE, nfqws2 raw sends, and --user=tpws UID/GID switching. - cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID] + # nft/NFQUEUE, raw sends, UID/GID switching, cleanup of dropped-UID workers. + cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID, KILL] environment: SS_PASSWORD: ${SS_PASSWORD:?Set SS_PASSWORD in the deployment environment} SOCKS_PORT: "1080"