diff --git a/.dockerignore b/.dockerignore index 76eb745..aa87397 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ * !go.mod +!go.sum !cmd/ !cmd/** !Dockerfile diff --git a/Dockerfile b/Dockerfile index 1384318..75cba62 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ FROM golang:1.25.3 AS build WORKDIR /src -ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOPROXY=off -COPY go.mod ./ +ENV CGO_ENABLED=0 GOTOOLCHAIN=local +COPY go.mod go.sum ./ +RUN go mod download COPY cmd/ ./cmd/ RUN go build -trimpath -o /out/zapret-autotune ./cmd/zapret-autotune diff --git a/README.md b/README.md index bd485e2..ff57911 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,24 @@ docker build -t zapret-autotune . The `tune youtube` command scans and checks candidate strategies; it prints a proposed `NFQWS2_OPT` and does not apply configuration or restart services. + +Tune runs are recorded in pure-Go SQLite (no CGO). A verified combined candidate +and its TLS1.2/TLS1.3 3/3 pass results are committed together before options are +printed; failed scans/checks are recorded as failed runs without a candidate. +Database initialization or persistence errors return nonzero and emit no options. +`status` shows the latest recorded run, or explicitly says there is no recorded +run. Candidates are **PROPOSED, not active**; there is no apply command and +production health is not checked. A later failed run is shown instead of an older +proposal. An output-write failure can leave a verified proposal recorded, but +still returns nonzero. + +Set `ZAPRET_AUTOTUNE_DB` to override the default database path +`/var/lib/zapret-autotune/autotune.db`. The parent directory is created if needed; +it must be writable. In Docker, mount that directory to retain history: + +```sh +docker run --rm -v autotune-data:/var/lib/zapret-autotune zapret-autotune status +``` + +Schema version 1 is initialized transactionally on `tune` and `status`; subsequent +opens are idempotent and newer, unsupported schema versions are rejected. diff --git a/cmd/zapret-autotune/main.go b/cmd/zapret-autotune/main.go index f96dfe5..40a289a 100644 --- a/cmd/zapret-autotune/main.go +++ b/cmd/zapret-autotune/main.go @@ -9,7 +9,7 @@ import ( const usage = `Usage: zapret-autotune Commands: - status Show skeleton capabilities (not live service health) + status Show latest recorded tune run/proposal (not live service health) discover youtube Refuse discovery until provider/config is agreed check youtube Refuse checks until probe URLs/video IDs are configured and agreed tune youtube Test youtube.com frontend only; print options without applying @@ -45,15 +45,7 @@ func run(args []string, stdout, stderr io.Writer) int { if len(args) == 1 { switch args[0] { case "status": - _, err := fmt.Fprintln(stdout, `Status: no network requests, subprocesses, or production changes made by this command. -Tune: youtube.com frontend only via blockcheckw; apply: BLOCKED. Isolation, route/probe equivalence, version compatibility, and non-disruptive activation are unconfirmed. -Discovery: unavailable; YouTube provider/config is not implemented. -Production ss-zapret2 and Xray health: NOT CHECKED.`) - if err != nil { - fmt.Fprintln(stderr, "error: cannot write status:", err) - return 1 - } - return 0 + return status(stdout, stderr) case "help", "--help", "-h": if _, err := fmt.Fprint(stdout, usage); err != nil { fmt.Fprintln(stderr, "error: cannot write usage:", err) diff --git a/cmd/zapret-autotune/main_test.go b/cmd/zapret-autotune/main_test.go index 64dcf15..837ff43 100644 --- a/cmd/zapret-autotune/main_test.go +++ b/cmd/zapret-autotune/main_test.go @@ -3,11 +3,13 @@ package main import ( "bytes" "errors" + "path/filepath" "strings" "testing" ) func TestRun(t *testing.T) { + t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(t.TempDir(), "autotune.db")) tests := []struct { name string args []string @@ -70,6 +72,7 @@ func (failingWriter) Write([]byte) (int, error) { } func TestOutputFailure(t *testing.T) { + t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(t.TempDir(), "autotune.db")) for _, command := range []string{"status", "--help"} { var stderr bytes.Buffer if run([]string{command}, failingWriter{}, &stderr) != 1 { diff --git a/cmd/zapret-autotune/store.go b/cmd/zapret-autotune/store.go new file mode 100644 index 0000000..a77b8cf --- /dev/null +++ b/cmd/zapret-autotune/store.go @@ -0,0 +1,215 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +func databasePath() string { + if path := os.Getenv("ZAPRET_AUTOTUNE_DB"); path != "" { + return path + } + return "/var/lib/zapret-autotune/autotune.db" +} + +func openStore(path string) (*sql.DB, error) { + // Configuration is a filesystem path, never a caller-supplied SQLite DSN. + if path == "" || path == ":memory:" || strings.HasPrefix(path, ":memory:?") || strings.HasPrefix(strings.ToLower(path), "file:") { + return nil, fmt.Errorf("database location must be a filesystem path, not a SQLite DSN") + } + path, err := filepath.Abs(path) + if err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, err + } + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return nil, err + } + if err := file.Chmod(0600); err != nil { + file.Close() + return nil, err + } + if err := file.Close(); err != nil { + return nil, err + } + // Escape path characters so they cannot become URI options or fragments. + dsn := (&url.URL{Scheme: "file", Path: filepath.ToSlash(path)}).String() + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + if _, err = db.Exec("PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;"); err == nil { + err = migrate(db) + } + if err != nil { + db.Close() + return nil, err + } + return db, nil +} + +func migrate(db *sql.DB) error { + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + // Serialize migrations before reading the version; a deferred transaction + // cannot wait via busy_timeout when upgrading a contended read lock. + if _, err = conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil { + return err + } + defer conn.ExecContext(ctx, "ROLLBACK") + var version int + if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil { + return err + } + if version > 1 { + return fmt.Errorf("unsupported database schema version %d", version) + } + if version == 0 { + _, err = conn.ExecContext(ctx, ` +CREATE TABLE tune_runs ( + id INTEGER PRIMARY KEY, + started_at TEXT NOT NULL, + finished_at TEXT NOT NULL, + target TEXT NOT NULL CHECK (target = 'youtube.com'), + state TEXT NOT NULL CHECK (state IN ('PROPOSED', 'FAILED')), + args TEXT, + output TEXT, + error TEXT, + CHECK ((state = 'PROPOSED' AND args IS NOT NULL AND length(args) > 0 AND output IS NOT NULL AND length(output) > 0 AND error IS NULL) + OR (state = 'FAILED' AND args IS NULL AND output IS NULL AND error IS NOT NULL AND length(error) > 0)) +); +CREATE TABLE protocol_results ( + run_id INTEGER NOT NULL REFERENCES tune_runs(id) ON DELETE CASCADE, + protocol TEXT NOT NULL CHECK (protocol IN ('tls12', 'tls13')), + passes_ok INTEGER NOT NULL CHECK (passes_ok = 3), + passes_total INTEGER NOT NULL CHECK (passes_total = 3), + PRIMARY KEY (run_id, protocol) +); +CREATE TRIGGER proposed_results_only BEFORE INSERT ON protocol_results + WHEN (SELECT state FROM tune_runs WHERE id = NEW.run_id) != 'PROPOSED' + BEGIN SELECT RAISE(ABORT, 'protocol results require a proposed run'); END; +PRAGMA user_version = 1;`) + if err != nil { + return err + } + } + _, err = conn.ExecContext(ctx, "COMMIT") + return err +} + +type storedRun struct { + ID int64 + StartedAt, FinishedAt, State string + Args, Output, Error sql.NullString +} + +// Validation is shared with tune's output path; only combined 3/3 evidence is stored. +func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, runErr error) error { + var evidence struct { + Strategies []struct { + Protocol string `json:"protocol"` + PassesOK int `json:"passes_ok"` + PassesTotal int `json:"passes_total"` + } `json:"strategies"` + } + state := "FAILED" + var savedArgs, savedOutput, savedError interface{} + if runErr == nil { + verifiedOutput, err := combinedBlockcheckOutput(bytes.NewReader(report), args) + if err != nil { + return err + } + if output != verifiedOutput { + return fmt.Errorf("output does not match verified candidate") + } + if err := json.Unmarshal(report, &evidence); err != nil { + return err + } + state, savedArgs, savedOutput = "PROPOSED", args, output + } else { + savedError = runErr.Error() + } + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + result, err := tx.Exec(`INSERT INTO tune_runs (started_at, finished_at, target, state, args, output, error) VALUES (?, ?, 'youtube.com', ?, ?, ?, ?)`, + started.UTC().Format(time.RFC3339Nano), time.Now().UTC().Format(time.RFC3339Nano), state, savedArgs, savedOutput, savedError) + if err != nil { + return err + } + id, err := result.LastInsertId() + if err != nil { + return err + } + for _, row := range evidence.Strategies { + if _, err := tx.Exec(`INSERT INTO protocol_results (run_id, protocol, passes_ok, passes_total) VALUES (?, ?, ?, ?)`, id, normalizeProtocol(row.Protocol), row.PassesOK, row.PassesTotal); err != nil { + return err + } + } + return tx.Commit() +} + +func latestRun(db *sql.DB) (storedRun, error) { + var r storedRun + err := db.QueryRow(`SELECT id, started_at, finished_at, state, args, output, error FROM tune_runs ORDER BY id DESC LIMIT 1`).Scan( + &r.ID, &r.StartedAt, &r.FinishedAt, &r.State, &r.Args, &r.Output, &r.Error) + if err == nil && r.State == "PROPOSED" { + var results int + err = db.QueryRow(`SELECT count(*) FROM protocol_results WHERE run_id = ? AND protocol IN ('tls12', 'tls13') AND passes_ok = 3 AND passes_total = 3`, r.ID).Scan(&results) + if err == nil && results != 2 { + err = fmt.Errorf("proposed run %d has incomplete protocol evidence", r.ID) + } + } + return r, err +} + +func status(stdout, stderr io.Writer) int { + db, err := openStore(databasePath()) + if err != nil { + fmt.Fprintln(stderr, "error: status database:", err) + return 1 + } + defer db.Close() + r, err := latestRun(db) + var text string + if err == sql.ErrNoRows { + text = "Status: no recorded tune run.\n" + } else if err != nil { + fmt.Fprintln(stderr, "error: status database:", err) + return 1 + } else { + text = fmt.Sprintf("Latest tune run: %d; youtube.com frontend only; %s; finished %s\n", r.ID, r.State, r.FinishedAt) + if r.State == "PROPOSED" { + text += "Candidate: PROPOSED, not active; verified TLS1.2 3/3 and TLS1.3 3/3.\n" + r.Output.String + "\n" + } else { + text += "No proposed candidate from this run. Error: " + r.Error.String + "\n" + } + } + text += "apply: BLOCKED (not implemented). No configuration applied.\nProduction ss-zapret2 and Xray health: NOT CHECKED.\n" + if _, err := fmt.Fprint(stdout, text); err != nil { + fmt.Fprintln(stderr, "error: cannot write status:", err) + return 1 + } + return 0 +} diff --git a/cmd/zapret-autotune/store_test.go b/cmd/zapret-autotune/store_test.go new file mode 100644 index 0000000..6821345 --- /dev/null +++ b/cmd/zapret-autotune/store_test.go @@ -0,0 +1,384 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "errors" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func testStore(t *testing.T) *sql.DB { + t.Helper() + t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(t.TempDir(), "data", "autotune.db")) + db, err := openStore(databasePath()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func saveProposal(t *testing.T, db *sql.DB, args string) string { + t.Helper() + report := combinedReportFixture(args) + output, err := combinedBlockcheckOutput(strings.NewReader(report), args) + if err != nil { + t.Fatal(err) + } + if err := saveRun(db, time.Now(), args, output, []byte(report), nil); err != nil { + t.Fatal(err) + } + return output +} + +func assertRowCounts(t *testing.T, db *sql.DB, runs, results int) { + t.Helper() + for table, want := range map[string]int{"tune_runs": runs, "protocol_results": results} { + var got int + if err := db.QueryRow("SELECT count(*) FROM " + table).Scan(&got); err != nil || got != want { + t.Fatalf("%s: count=%d want=%d err=%v", table, got, want, err) + } + } +} + +func assertProposedRun(t *testing.T, args, output string) { + t.Helper() + db, err := openStore(databasePath()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + r, err := latestRun(db) + if err != nil || r.State != "PROPOSED" || r.Args.String != args || r.Output.String != output || r.Error.Valid { + t.Fatalf("latest=%+v err=%v", r, err) + } + assertRowCounts(t, db, 1, 2) + for _, protocol := range []string{"tls12", "tls13"} { + var ok, total int + if err := db.QueryRow("SELECT passes_ok, passes_total FROM protocol_results WHERE run_id = ? AND protocol = ?", r.ID, protocol).Scan(&ok, &total); err != nil || ok != 3 || total != 3 { + t.Fatalf("%s: %d/%d err=%v", protocol, ok, total, err) + } + } +} + +func TestStoreFilePermissions(t *testing.T) { + for _, name := range []string{"new", "existing"} { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "data", "autotune.db") + wantRuns := 0 + if name == "existing" { + db, err := openStore(path) + if err != nil { + t.Fatal(err) + } + if err := saveRun(db, time.Now(), "", "", nil, errors.New("scan failed")); err != nil { + db.Close() + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0666); err != nil { + t.Fatal(err) + } + wantRuns = 1 + } + db, err := openStore(path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0600 { + t.Fatalf("database mode = %04o, want 0600", got) + } + assertRowCounts(t, db, wantRuns, 0) + }) + } +} + +func TestDatabasePathEnvironment(t *testing.T) { + t.Setenv("ZAPRET_AUTOTUNE_DB", "") + if got := databasePath(); got != "/var/lib/zapret-autotune/autotune.db" { + t.Fatalf("default database path = %q", got) + } + t.Setenv("ZAPRET_AUTOTUNE_DB", "data/custom?.db") + if got := databasePath(); got != "data/custom?.db" { + t.Fatalf("configured database path = %q", got) + } +} + +func TestStoreRejectsDSNs(t *testing.T) { + for _, path := range []string{"", ":memory:", ":memory:?cache=shared", "file::memory:?cache=shared", "file:test.db?mode=memory", "file:///tmp/test.db", "FILE:test.db"} { + t.Run(path, func(t *testing.T) { + db, err := openStore(path) + if db != nil { + db.Close() + } + if err == nil { + t.Fatalf("accepted SQLite DSN %q", path) + } + }) + } +} + +func TestStoreFilesystemPaths(t *testing.T) { + for _, name := range []string{"plain.db", "question?.db", "options.db?mode=memory&cache=shared", "pragma.db?_pragma=query_only(1)", "hash#.db", "percent%3f.db", "space + ΓΌ.db"} { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), name) + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + relative, err := filepath.Rel(cwd, path) + if err != nil { + t.Fatal(err) + } + t.Setenv("ZAPRET_AUTOTUNE_DB", relative) + db, err := openStore(databasePath()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var sequence int + var schema, filename string + if err := db.QueryRow("PRAGMA database_list").Scan(&sequence, &schema, &filename); err != nil { + t.Fatal(err) + } + if filename != path { + t.Fatalf("SQLite opened %q, want %q", filename, path) + } + if err := saveRun(db, time.Now(), "", "", nil, errors.New("persisted failure")); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Size() == 0 || info.Mode().Perm() != 0600 { + t.Fatalf("database size=%d mode=%04o", info.Size(), info.Mode().Perm()) + } + entries, err := os.ReadDir(filepath.Dir(path)) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name() != name { + t.Fatalf("unexpected database files: %v", entries) + } + db, err = openStore(path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + assertRowCounts(t, db, 1, 0) + }) + } +} + +func TestConcurrentFirstOpens(t *testing.T) { + path := filepath.Join(t.TempDir(), "autotune.db") + const callers = 8 + start := make(chan struct{}) + type openResult struct { + db *sql.DB + err error + } + results := make(chan openResult, callers) + var ready sync.WaitGroup + ready.Add(callers) + for i := 0; i < callers; i++ { + go func() { + ready.Done() + <-start + db, err := openStore(path) + results <- openResult{db, err} + }() + } + ready.Wait() + close(start) + // Collect every result before assertions so failures still close all stores. + stores := make([]*sql.DB, 0, callers) + for i := 0; i < callers; i++ { + result := <-results + if result.err != nil { + t.Errorf("concurrent open: %v", result.err) + continue + } + stores = append(stores, result.db) + defer result.db.Close() + } + for _, db := range stores { + var version, objects int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 1 { + t.Fatalf("version=%d err=%v", version, err) + } + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE + (type = 'table' AND name IN ('tune_runs', 'protocol_results')) OR + (type = 'trigger' AND name = 'proposed_results_only')`).Scan(&objects); err != nil || objects != 3 { + t.Fatalf("schema objects=%d err=%v", objects, err) + } + assertRowCounts(t, db, 0, 0) + var integrity string + if err := db.QueryRow("PRAGMA integrity_check").Scan(&integrity); err != nil || integrity != "ok" { + t.Fatalf("integrity=%q err=%v", integrity, err) + } + } +} + +func TestMigrationAndLatest(t *testing.T) { + db := testStore(t) + if err := migrate(db); err != nil { + t.Fatal(err) + } + if _, err := latestRun(db); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("empty latest: %v", err) + } + args := "--lua-desync=multisplit" + output := saveProposal(t, db, args) + if err := migrate(db); err != nil { + t.Fatal(err) + } + assertProposedRun(t, args, output) + if err := saveRun(db, time.Now(), "", "", nil, errors.New("scan failed")); err != nil { + t.Fatal(err) + } + r, err := latestRun(db) + if err != nil || r.ID != 2 || r.State != "FAILED" || r.Error.String != "scan failed" || r.Args.Valid || r.Output.Valid { + t.Fatalf("latest=%+v err=%v", r, err) + } + assertRowCounts(t, db, 2, 2) + var stdout, stderr bytes.Buffer + if status(&stdout, &stderr) != 0 || !strings.Contains(stdout.String(), "FAILED") || strings.Contains(stdout.String(), output) { + t.Fatalf("failed status: %q %q", stdout.String(), stderr.String()) + } +} + +func TestStatusNoDataAndProposed(t *testing.T) { + db := testStore(t) + var stdout, stderr bytes.Buffer + if run([]string{"status"}, &stdout, &stderr) != 0 || !strings.Contains(stdout.String(), "no recorded tune run") || strings.Contains(stdout.String(), "NFQWS2_OPT") { + t.Fatalf("empty status: %q %q", stdout.String(), stderr.String()) + } + output := saveProposal(t, db, "--lua-desync=multisplit") + stdout.Reset() + if run([]string{"status"}, &stdout, &stderr) != 0 { + t.Fatal(stderr.String()) + } + for _, want := range []string{"PROPOSED, not active", "TLS1.2 3/3", "TLS1.3 3/3", output, "apply: BLOCKED", "NOT CHECKED"} { + if !strings.Contains(stdout.String(), want) { + t.Errorf("status %q missing %q", stdout.String(), want) + } + } +} + +func TestPersistenceRollback(t *testing.T) { + db := testStore(t) + output := saveProposal(t, db, "--lua-desync=multisplit") + if _, err := db.Exec(`CREATE TRIGGER reject_tls13 BEFORE INSERT ON protocol_results WHEN NEW.protocol = 'tls13' BEGIN SELECT RAISE(ABORT, 'injected persistence failure'); END`); err != nil { + t.Fatal(err) + } + args := "--lua-desync=multisplit" + if err := saveRun(db, time.Now(), args, output, []byte(combinedReportFixture(args)), nil); err == nil { + t.Fatal("expected persistence failure") + } + assertRowCounts(t, db, 1, 2) + r, err := latestRun(db) + if err != nil || r.ID != 1 { + t.Fatalf("rollback changed latest: %+v %v", r, err) + } +} + +func TestMigrationRollback(t *testing.T) { + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "broken.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + if _, err := db.Exec("CREATE TABLE protocol_results (id INTEGER)"); err != nil { + t.Fatal(err) + } + if err := migrate(db); err == nil { + t.Fatal("expected conflicting schema failure") + } + var version, tables int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 0 { + t.Fatalf("version=%d err=%v", version, err) + } + if err := db.QueryRow("SELECT count(*) FROM sqlite_master WHERE name = 'tune_runs'").Scan(&tables); err != nil || tables != 0 { + t.Fatalf("partial schema count=%d err=%v", tables, err) + } + if _, err := db.Exec("DROP TABLE protocol_results"); err != nil { + t.Fatal(err) + } + if err := migrate(db); err != nil { + t.Fatalf("migration after rollback: %v", err) + } +} + +func TestDatabaseFailureFailsClosed(t *testing.T) { + parent := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(parent, []byte("x"), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(parent, "autotune.db")) + var stdout, stderr bytes.Buffer + scan := func(context.Context, string, string, io.Writer, io.Writer) (string, error) { + t.Fatal("scan must not start with an unavailable database") + return "", nil + } + if tuneYoutube(&stdout, &stderr, scan) != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "database") { + t.Fatalf("tune: out=%q err=%q", stdout.String(), stderr.String()) + } + stderr.Reset() + if run([]string{"status"}, &stdout, &stderr) != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "database") { + t.Fatalf("status: out=%q err=%q", stdout.String(), stderr.String()) + } +} + +func TestSchemaConstraintsAndIncompleteEvidence(t *testing.T) { + db := testStore(t) + saveProposal(t, db, "--lua-desync=multisplit") + for _, statement := range []string{ + `INSERT INTO protocol_results VALUES (999, 'tls12', 3, 3)`, + `UPDATE protocol_results SET protocol = 'quic' WHERE protocol = 'tls12'`, + `UPDATE protocol_results SET passes_ok = 2 WHERE protocol = 'tls12'`, + `UPDATE protocol_results SET passes_total = 4 WHERE protocol = 'tls12'`, + `UPDATE tune_runs SET state = 'ACTIVE'`, + `UPDATE tune_runs SET args = NULL`, + } { + if _, err := db.Exec(statement); err == nil { + t.Fatalf("constraint accepted %s", statement) + } + } + if _, err := db.Exec(`DELETE FROM protocol_results WHERE protocol = 'tls13'`); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + if status(&stdout, &stderr) != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "incomplete protocol evidence") { + t.Fatalf("incomplete status: %q %q", stdout.String(), stderr.String()) + } +} + +func TestUnsupportedSchemaVersion(t *testing.T) { + db := testStore(t) + if _, err := db.Exec("PRAGMA user_version = 2"); err != nil { + t.Fatal(err) + } + if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") { + t.Fatalf("future schema accepted: %v", err) + } +} diff --git a/cmd/zapret-autotune/tune.go b/cmd/zapret-autotune/tune.go index e0b9492..33c285a 100644 --- a/cmd/zapret-autotune/tune.go +++ b/cmd/zapret-autotune/tune.go @@ -10,6 +10,7 @@ import ( "os/signal" "strings" "syscall" + "time" ) func tuneYoutube(stdout, stderr io.Writer, scan func(context.Context, string, string, io.Writer, io.Writer) (string, error)) int { @@ -17,12 +18,21 @@ func tuneYoutube(stdout, stderr io.Writer, scan func(context.Context, string, st } func tuneYoutubeWithCheck(stdout, stderr io.Writer, scan func(context.Context, string, string, io.Writer, io.Writer) (string, error), check func(context.Context, string, string, string, io.Writer, io.Writer) error) int { + started := time.Now() + db, err := openStore(databasePath()) + if err != nil { + fmt.Fprintln(stderr, "error: tune youtube database:", err) + return 1 + } + defer db.Close() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() fmt.Fprintln(stderr, "Testing frontend youtube.com only; GGC and other frontend domains have not been tested. No options will be applied.") fmt.Fprintln(stderr, "Full TLS1.2 and TLS1.3 scan with 8 workers, then real-data check with 32 workers, --take 10, 3 passes; no overall time limit; selection requires fully passing results for both tls12 and tls13.") fmt.Fprintln(stderr, "The best individually verified TLS strategies are combined into one profile, then checked together for 3/3 passes on both TLS versions before output. No options will be applied.") var raw bytes.Buffer + var report []byte + var args, output string path, err := scan(ctx, "blockcheckw", "youtube.com", io.MultiWriter(&raw, stderr), stderr) if path != "" { fmt.Fprintln(stderr, "Raw blockcheck report retained at:", path) @@ -32,10 +42,8 @@ func tuneYoutubeWithCheck(stdout, stderr io.Writer, scan func(context.Context, s } } if err == nil { - var report []byte report, err = os.ReadFile(path + ".check.json") if err == nil { - var args, output string args, err = selectBlockcheckReport(bytes.NewReader(report)) if err == nil { fmt.Fprintln(stderr, "Combined blockcheck JSON path:", path+".combined.json") @@ -53,11 +61,18 @@ func tuneYoutubeWithCheck(stdout, stderr io.Writer, scan func(context.Context, s if err == nil { err = ctx.Err() } - if err == nil { - _, err = fmt.Fprintln(stdout, output) - } } } + if saveErr := saveRun(db, started, args, output, report, err); saveErr != nil { + fmt.Fprintln(stderr, "error: tune youtube database:", saveErr) + if err != nil { + fmt.Fprintln(stderr, "error: tune youtube:", err) + } + return 1 + } + if err == nil { + _, err = fmt.Fprintln(stdout, output) + } if err != nil { fmt.Fprintln(stderr, "error: tune youtube:", err) return 1 diff --git a/cmd/zapret-autotune/tune_test.go b/cmd/zapret-autotune/tune_test.go index 9c0bf38..12a0857 100644 --- a/cmd/zapret-autotune/tune_test.go +++ b/cmd/zapret-autotune/tune_test.go @@ -161,6 +161,7 @@ func TestTuneCLI(t *testing.T) { for _, mode := range []string{"tune", "tune-invalid", "tune-error", "tune-check-error", "tune-combined-error", "tune-combined-fail"} { t.Run(mode, func(t *testing.T) { t.Setenv("TMPDIR", t.TempDir()) + t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(t.TempDir(), "autotune.db")) binary := fakeBlockcheck(t, mode) installed := filepath.Join(filepath.Dir(binary), "blockcheckw") if err := os.Rename(binary, installed); err != nil { @@ -182,6 +183,7 @@ func TestTuneCLI(t *testing.T) { if err != nil || code != 0 || stdout.String() != want || strings.Contains(stdout.String(), "--new") { t.Fatalf("code=%d out=%q err=%q", code, stdout.String(), stderr.String()) } + assertProposedRun(t, combined, strings.TrimSpace(want)) } else if code == 0 || stdout.Len() != 0 { t.Fatalf("failure emitted options: code=%d out=%q", code, stdout.String()) } @@ -228,6 +230,7 @@ func TestTuneCLI(t *testing.T) { func TestTuneUsesCheckFileAndPreservesFailures(t *testing.T) { for _, mode := range []string{"success", "scan", "parse", "write", "missing-check", "combined-error", "combined-fail", "combined-missing", "combined-wrong-args"} { t.Run(mode, func(t *testing.T) { + t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(t.TempDir(), "autotune.db")) path := filepath.Join(t.TempDir(), "scan.txt") report := reportFixture(t) if mode == "parse" { @@ -290,6 +293,11 @@ func TestTuneUsesCheckFileAndPreservesFailures(t *testing.T) { if code != 0 { t.Fatal(stderr.String()) } + args, err := selectBlockcheckReport(strings.NewReader(report)) + if err != nil { + t.Fatal(err) + } + assertProposedRun(t, args, strings.TrimSpace(stdout.String())) } else if code != 1 || stdout.Len() != 0 { t.Fatalf("failed run emitted options: %d %q", code, stdout.String()) } @@ -303,6 +311,60 @@ func TestTuneUsesCheckFileAndPreservesFailures(t *testing.T) { } } +func TestTunePersistenceFailureAfterVerification(t *testing.T) { + t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(t.TempDir(), "autotune.db")) + path := filepath.Join(t.TempDir(), "scan.txt") + report := reportFixture(t) + wantArgs, err := selectBlockcheckReport(strings.NewReader(report)) + if err != nil { + t.Fatal(err) + } + scan := func(context.Context, string, string, io.Writer, io.Writer) (string, error) { + // tune has initialized its DB before scanning; fail only verified persistence. + db, err := openStore(databasePath()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TRIGGER reject_tls13 BEFORE INSERT ON protocol_results WHEN NEW.protocol = 'tls13' BEGIN SELECT RAISE(ABORT, 'injected persistence failure'); END`); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("vanilla"), 0600); err != nil { + t.Fatal(err) + } + return path, os.WriteFile(path+".check.json", []byte(report), 0600) + } + checked := false + check := func(ctx context.Context, binary, gotPath, args string, _, _ io.Writer) error { + if ctx.Err() != nil || binary != "blockcheckw" || gotPath != path || args != wantArgs { + t.Fatal("incorrect combined check invocation") + } + if err := os.WriteFile(path+".combined.json", []byte(combinedReportFixture(args)), 0600); err != nil { + return err + } + checked = true + return nil + } + var stdout, stderr bytes.Buffer + code := tuneYoutubeWithCheck(&stdout, &stderr, scan, check) + if !checked { + t.Fatal("candidate verification was not completed") + } + if code == 0 || stdout.Len() != 0 { + t.Fatalf("persistence failure emitted candidate: code=%d out=%q err=%q", code, stdout.String(), stderr.String()) + } + // This trigger fires only when saveRun inserts successful protocol evidence. + if !strings.Contains(stderr.String(), "error: tune youtube database:") || !strings.Contains(stderr.String(), "injected persistence failure") { + t.Fatal(stderr.String()) + } + db, err := openStore(databasePath()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + assertRowCounts(t, db, 0, 0) +} + func combinedReportFixture(args string) string { encoded, _ := json.Marshal(args) row := `{"protocol":%q,"args":%s,"success_rate":1,"coverage":1,"passes_ok":3,"passes_total":3,"median_latency_ms":10,"median_speed_kbps":100}` diff --git a/go.mod b/go.mod index d68e531..14170a0 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,17 @@ module zapret-autotune go 1.22 + +require modernc.org/sqlite v1.34.5 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.22.0 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5424fe4 --- /dev/null +++ b/go.sum @@ -0,0 +1,43 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=