fix(blockcheck): reap owned workers after each invocation
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
59df7d58f5
commit
d508132b4f
4 files changed
+141
-21
No files matched your search
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -35,9 +36,8 @@ func runBlockcheck(ctx context.Context, binary, domain string, stdout, stderr io
|
||||
return runBlockcheckWithConflictCleanup(ctx, binary, domain, stdout, stderr, false)
|
||||
}
|
||||
|
||||
// Managed recovery has already stopped zapret inside its dedicated network
|
||||
// namespace, so blockcheck may clean up its own scan workers before shortlist
|
||||
// checks. Other callers preserve foreign nfqws2 processes by default.
|
||||
// Managed recovery tests known seeds before the default corpus. Each invocation
|
||||
// cleans its own process group; foreign processes remain protected.
|
||||
func runManagedBlockcheck(ctx context.Context, binary, domain string, stdout, stderr io.Writer) (string, error) {
|
||||
domain = strings.ToLower(domain)
|
||||
if args := managedSeedArgs(domain); args != "" {
|
||||
@@ -47,7 +47,7 @@ func runManagedBlockcheck(ctx context.Context, binary, domain string, stdout, st
|
||||
for attempt := 0; attempt < 5; attempt++ {
|
||||
path, err := runManagedSeedCheck(ctx, binary, domain, args, stdout, stderr)
|
||||
if err != nil {
|
||||
break
|
||||
return path, err
|
||||
}
|
||||
report, readErr := os.ReadFile(path + ".check.json")
|
||||
group := "frontend"
|
||||
@@ -63,7 +63,7 @@ func runManagedBlockcheck(ctx context.Context, binary, domain string, stdout, st
|
||||
_ = os.Remove(path + ".check.json")
|
||||
}
|
||||
}
|
||||
return runBlockcheckWithConflictCleanup(ctx, binary, domain, stdout, stderr, true)
|
||||
return runBlockcheckWithConflictCleanup(ctx, binary, domain, stdout, stderr, false)
|
||||
}
|
||||
|
||||
func runBlockcheckWithConflictCleanup(ctx context.Context, binary, domain string, stdout, stderr io.Writer, allowConflictCleanup bool) (string, error) {
|
||||
@@ -99,7 +99,7 @@ func runBlockcheckWithConflictCleanup(ctx context.Context, binary, domain string
|
||||
scanArgs = append(scanArgs, "--no-conflict-cleanup")
|
||||
}
|
||||
scanArgs = append(scanArgs, "--output", path)
|
||||
err = runBlockcheckCommand(ctx, binary, workdir, scanArgs, stdout, stderr)
|
||||
err = runOwnedBlockcheckCommand(ctx, binary, workdir, scanArgs, stdout, stderr)
|
||||
if err != nil {
|
||||
return path, fmt.Errorf("scan artifacts in %s (output %s): %w", workdir, path, err)
|
||||
}
|
||||
@@ -114,7 +114,7 @@ func runBlockcheckWithConflictCleanup(ctx context.Context, binary, domain string
|
||||
checkArgs = append(checkArgs, "--no-conflict-cleanup")
|
||||
}
|
||||
checkArgs = append(checkArgs, "--output", path+".check.json")
|
||||
err = runBlockcheckCommand(ctx, binary, workdir, checkArgs, stdout, stderr)
|
||||
err = runOwnedBlockcheckCommand(ctx, binary, workdir, checkArgs, stdout, stderr)
|
||||
if err != nil {
|
||||
return path, fmt.Errorf("check (vanilla %s, scan output %s, artifacts in %s): %w", vanilla, path, workdir, err)
|
||||
}
|
||||
@@ -141,8 +141,8 @@ func runManagedSeedCheck(ctx context.Context, binary, host, args string, stdout,
|
||||
if err := os.WriteFile(input, []byte(rows), 0600); err != nil {
|
||||
return path, err
|
||||
}
|
||||
checkArgs := []string{"--workers", "32", "check", "--from-file", input, "--domain", strings.ToLower(host), "--take", "1", "--passes", "3", "--auto", "--output", path + ".check.json"}
|
||||
if err := runBlockcheckCommand(ctx, binary, "", checkArgs, stdout, stderr); err != nil {
|
||||
checkArgs := []string{"--workers", "32", "check", "--from-file", input, "--domain", strings.ToLower(host), "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", "--output", path + ".check.json"}
|
||||
if err := runOwnedBlockcheckCommand(ctx, binary, "", checkArgs, stdout, stderr); err != nil {
|
||||
return path, err
|
||||
}
|
||||
return path, nil
|
||||
@@ -205,7 +205,7 @@ func runCombinedBlockcheckForHost(ctx context.Context, binary, path, args, host
|
||||
}
|
||||
|
||||
func runManagedCombinedBlockcheckForHost(ctx context.Context, binary, path, args, host string, stdout, stderr io.Writer) error {
|
||||
return runCombinedBlockcheckForHostWithConflictCleanup(ctx, binary, path, args, host, stdout, stderr, true)
|
||||
return runCombinedBlockcheckForHostWithConflictCleanup(ctx, binary, path, args, host, stdout, stderr, false)
|
||||
}
|
||||
|
||||
func runCombinedBlockcheckForHostWithConflictCleanup(ctx context.Context, binary, path, args, host string, stdout, stderr io.Writer, allowConflictCleanup bool) error {
|
||||
@@ -225,13 +225,34 @@ func runCombinedBlockcheckForHostWithConflictCleanup(ctx context.Context, binary
|
||||
checkArgs = append(checkArgs, "--no-conflict-cleanup")
|
||||
}
|
||||
checkArgs = append(checkArgs, "--output", path+".combined.json")
|
||||
if err := runBlockcheckCommand(ctx, binary, "", checkArgs, stdout, stderr); err != nil {
|
||||
if err := runOwnedBlockcheckCommand(ctx, binary, "", checkArgs, stdout, stderr); err != nil {
|
||||
return fmt.Errorf("combined check (input %s, output %s): %w", input, path+".combined.json", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runBlockcheckCommand(ctx context.Context, binary, dir string, args []string, stdout, stderr io.Writer) error {
|
||||
return runProcessGroupCommand(ctx, binary, dir, args, stdout, stderr, false)
|
||||
}
|
||||
|
||||
func runOwnedBlockcheckCommand(ctx context.Context, binary, dir string, args []string, stdout, stderr io.Writer) error {
|
||||
for i, arg := range args {
|
||||
if arg == "--output" && i+1 < len(args) {
|
||||
f, err := os.OpenFile(args[i+1]+".stderr", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stderr == nil {
|
||||
stderr = io.Discard
|
||||
}
|
||||
err = runProcessGroupCommand(ctx, binary, dir, args, stdout, io.MultiWriter(stderr, f), true)
|
||||
return errors.Join(err, f.Close())
|
||||
}
|
||||
}
|
||||
return runProcessGroupCommand(ctx, binary, dir, args, stdout, stderr, true)
|
||||
}
|
||||
|
||||
func runProcessGroupCommand(ctx context.Context, binary, dir string, args []string, stdout, stderr io.Writer, cleanup bool) error {
|
||||
cmd := exec.CommandContext(ctx, binary, args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Stdout, cmd.Stderr = stdout, stderr
|
||||
@@ -252,9 +273,75 @@ func runBlockcheckCommand(ctx context.Context, binary, dir string, args []string
|
||||
return nil
|
||||
}
|
||||
cmd.WaitDelay = 5 * time.Second
|
||||
err := cmd.Run()
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
err := cmd.Wait()
|
||||
// Workers can outlive a successful leader and retain NFQUEUE bindings.
|
||||
// Setpgid scopes cleanup to this invocation, never all nfqws2 processes.
|
||||
if cleanup {
|
||||
if cleanupErr := cleanupBlockcheckGroup(cmd.Process.Pid); cleanupErr != nil {
|
||||
err = errors.Join(err, &managedRecoveryFatal{Err: cleanupErr})
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
err = errors.Join(ctx.Err(), err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func cleanupBlockcheckGroup(pgid int) error {
|
||||
for _, signal := range []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL} {
|
||||
if err := syscall.Kill(-pgid, signal); err != nil && !errors.Is(err, syscall.ESRCH) {
|
||||
return fmt.Errorf("blockcheck worker cleanup: %w", err)
|
||||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
alive, err := blockcheckGroupAlive(pgid)
|
||||
if err != nil || !alive {
|
||||
return err
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("blockcheck workers remain in process group %d", pgid)
|
||||
}
|
||||
|
||||
func blockcheckGroupAlive(pgid int) (bool, error) {
|
||||
entries, err := os.ReadDir("/proc")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if _, err := strconv.Atoi(entry.Name()); err != nil {
|
||||
continue
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join("/proc", entry.Name(), "stat"))
|
||||
if errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ESRCH) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// comm may contain spaces/parentheses; pgrp is field 5 after comm.
|
||||
end := strings.LastIndexByte(string(data), ')')
|
||||
if end < 0 {
|
||||
return false, fmt.Errorf("invalid process stat")
|
||||
}
|
||||
fields := strings.Fields(string(data[end+1:]))
|
||||
if len(fields) < 3 {
|
||||
return false, fmt.Errorf("short process stat")
|
||||
}
|
||||
group, err := strconv.Atoi(fields[2])
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if group == pgid && fields[0] != "Z" && fields[0] != "X" {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
@@ -26,6 +26,20 @@ func init() {
|
||||
if mode == "" {
|
||||
return
|
||||
}
|
||||
if mode == "orphan" {
|
||||
if len(os.Args) == 2 && os.Args[1] == "--child" {
|
||||
signal.Ignore(syscall.SIGTERM)
|
||||
for {
|
||||
time.Sleep(time.Hour)
|
||||
}
|
||||
}
|
||||
child := exec.Command(os.Args[0], "--child")
|
||||
if err := child.Start(); err != nil {
|
||||
os.Exit(3)
|
||||
}
|
||||
fmt.Fprintln(os.Stdout, child.Process.Pid)
|
||||
os.Exit(0)
|
||||
}
|
||||
if len(os.Args) == 2 && os.Args[1] == "--child" {
|
||||
if mode == "kill" {
|
||||
signal.Ignore(syscall.SIGTERM)
|
||||
@@ -201,7 +215,7 @@ func TestRunBlockcheckArgvAndStreams(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunManagedBlockcheckAllowsWorkerCleanup(t *testing.T) {
|
||||
func TestRunManagedBlockcheckProtectsForeignWorkers(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
binary := fakeBlockcheck(t, "success")
|
||||
var stdout bytes.Buffer
|
||||
@@ -220,10 +234,8 @@ func TestRunManagedBlockcheckAllowsWorkerCleanup(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, args := range [][]string{scanArgs, checkArgs} {
|
||||
for _, arg := range args {
|
||||
if arg == "--no-conflict-cleanup" {
|
||||
t.Fatalf("managed blockcheck retained no-cleanup flag: %q", args)
|
||||
}
|
||||
if !slices.Contains(args, "--no-conflict-cleanup") {
|
||||
t.Fatalf("managed blockcheck permits foreign cleanup: %q", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -257,7 +269,7 @@ func TestRunManagedBlockcheckSeedsKnownProfileThroughBlockcheck(t *testing.T) {
|
||||
if err := decoder.Decode(&checkArgs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(checkArgs, "--from-file") || !slices.Contains(checkArgs, "--passes") || slices.Contains(checkArgs, "--no-conflict-cleanup") {
|
||||
if !slices.Contains(checkArgs, "--from-file") || !slices.Contains(checkArgs, "--passes") || !slices.Contains(checkArgs, "--no-conflict-cleanup") {
|
||||
t.Fatalf("managed seeded check=%q", checkArgs)
|
||||
}
|
||||
if seed := managedSeedArgs("rr3.googlevideo.com"); !strings.Contains(seed, "fakeddisorder") {
|
||||
@@ -347,6 +359,27 @@ func TestRunBlockcheckProcessError(t *testing.T) {
|
||||
|
||||
type processReady struct{ pid chan int }
|
||||
|
||||
func TestBlockcheckCleansWorkersAfterSuccessfulLeader(t *testing.T) {
|
||||
binary := fakeBlockcheck(t, "orphan")
|
||||
var stdout bytes.Buffer
|
||||
if err := runOwnedBlockcheckCommand(context.Background(), binary, "", nil, &stdout, io.Discard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pid, err := strconv.Atoi(strings.TrimSpace(stdout.String()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
|
||||
if err == nil {
|
||||
fields := strings.Fields(string(data[strings.LastIndexByte(string(data), ')')+1:]))
|
||||
if fields[0] != "Z" && fields[0] != "X" {
|
||||
t.Fatalf("worker survived cleanup: %s", data)
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, syscall.ESRCH) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (w processReady) Write(p []byte) (int, error) {
|
||||
pid, err := strconv.Atoi(strings.TrimSpace(string(p)))
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user