From 04fd9b370df279b9600f23e52e72ae331e6ce5c2 Mon Sep 17 00:00:00 2001 From: Zeusina Date: Fri, 9 Oct 2026 21:56:49 +0300 Subject: [PATCH] feat(storage): fingerprint proposed strategies Co-authored-by: GPT-6 --- README.md | 6 +- cmd/zapret-autotune/fingerprint_test.go | 164 ++++++++++++++++++++++++ cmd/zapret-autotune/store.go | 114 ++++++++++++++-- cmd/zapret-autotune/store_test.go | 133 ++++++++++++++++++- 4 files changed, 405 insertions(+), 12 deletions(-) create mode 100644 cmd/zapret-autotune/fingerprint_test.go diff --git a/README.md b/README.md index ff57911..644156d 100644 --- a/README.md +++ b/README.md @@ -29,5 +29,9 @@ it must be writable. In Docker, mount that directory to retain history: docker run --rm -v autotune-data:/var/lib/zapret-autotune zapret-autotune status ``` -Schema version 1 is initialized transactionally on `tune` and `status`; subsequent +Schema version 2 is initialized transactionally on `tune` and `status`; subsequent opens are idempotent and newer, unsupported schema versions are rejected. +Proposed runs store a SHA-256 fingerprint of validated arguments with separator +whitespace normalized and argument order preserved; repeated proposals retain +separate history rows. Failed runs have no fingerprint. Version 1 history is +upgraded atomically, rejecting invalid proposed arguments without changing it. diff --git a/cmd/zapret-autotune/fingerprint_test.go b/cmd/zapret-autotune/fingerprint_test.go new file mode 100644 index 0000000..682964b --- /dev/null +++ b/cmd/zapret-autotune/fingerprint_test.go @@ -0,0 +1,164 @@ +package main + +import ( + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestStrategyFingerprint(t *testing.T) { + args := "--payload=tls_client_hello --lua-desync=fake --lua-desync=multisplit" + want := fmt.Sprintf("%x", sha256.Sum256([]byte(args))) + for _, input := range []string{args, " " + args + " ", strings.ReplaceAll(args, " ", " \t\r\n\v\f ")} { + got, err := strategyFingerprint(input) + if err != nil || got != want { + t.Fatalf("fingerprint(%q)=%q err=%v want=%q", input, got, err, want) + } + } + for _, input := range []string{ + "--lua-desync=fake --payload=tls_client_hello --lua-desync=multisplit", + args + " --lua-desync=multisplit", + "--payload=tls_client_hello --lua-desync=fake --lua-desync=multidisorder", + } { + got, err := strategyFingerprint(input) + if err != nil || got == want { + t.Fatalf("different arguments have fingerprint %q err=%v", got, err) + } + } + // Meaningful value spaces are unsupported, not silently removed. + for _, input := range []string{"", " \t\n", "--lua-desync=fa ke", `--lua-desync="fa ke"`, "--lua-desync=fake;id", "--unknown=fake"} { + if _, err := strategyFingerprint(input); err == nil { + t.Fatalf("accepted invalid args %q", input) + } + } +} + +func TestFingerprintPersistence(t *testing.T) { + db := testStore(t) + args := "--lua-desync=fake --lua-desync=multisplit" + want, err := strategyFingerprint(args) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + saveProposal(t, db, args) + r, err := latestRun(db) + if err != nil || !r.Fingerprint.Valid || r.Fingerprint.String != want { + t.Fatalf("proposed=%+v err=%v", r, err) + } + } + if err := saveRun(db, time.Now(), args, "unused", nil, errors.New("scan failed")); err != nil { + t.Fatal(err) + } + r, err := latestRun(db) + if err != nil || r.Fingerprint.Valid { + t.Fatalf("failed=%+v err=%v", r, err) + } + assertRowCounts(t, db, 3, 4) +} + +func populatedV1Store(t *testing.T, args string) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "v1.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + db.SetMaxOpenConns(1) + _, err = db.Exec(` +PRAGMA foreign_keys = ON; +CREATE TABLE tune_runs ( + id INTEGER PRIMARY KEY, + started_at TEXT NOT NULL, + finished_at TEXT NOT NULL, + target TEXT NOT NULL CHECK (target = 'youtube.com'), + state TEXT NOT NULL CHECK (state IN ('PROPOSED', 'FAILED')), + args TEXT, output TEXT, error TEXT, + CHECK ((state = 'PROPOSED' AND args IS NOT NULL AND length(args) > 0 AND output IS NOT NULL AND length(output) > 0 AND error IS NULL) + OR (state = 'FAILED' AND args IS NULL AND output IS NULL AND error IS NOT NULL AND length(error) > 0)) +); +CREATE TABLE protocol_results ( + run_id INTEGER NOT NULL REFERENCES tune_runs(id) ON DELETE CASCADE, + protocol TEXT NOT NULL CHECK (protocol IN ('tls12', 'tls13')), + passes_ok INTEGER NOT NULL CHECK (passes_ok = 3), + passes_total INTEGER NOT NULL CHECK (passes_total = 3), + PRIMARY KEY (run_id, protocol) +); +CREATE TRIGGER proposed_results_only BEFORE INSERT ON protocol_results + WHEN (SELECT state FROM tune_runs WHERE id = NEW.run_id) != 'PROPOSED' + BEGIN SELECT RAISE(ABORT, 'protocol results require a proposed run'); END; +PRAGMA user_version = 1; +INSERT INTO tune_runs VALUES (7, 'start', 'finish', 'youtube.com', 'PROPOSED', '--lua-desync=fake', 'original output', NULL); +INSERT INTO tune_runs VALUES (8, 'failed start', 'failed finish', 'youtube.com', 'FAILED', NULL, NULL, 'original failure'); +INSERT INTO protocol_results VALUES (7, 'tls12', 3, 3), (7, 'tls13', 3, 3); +`) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("INSERT INTO tune_runs VALUES (9, 'start', 'finish', 'youtube.com', 'PROPOSED', ?, 'second output', NULL)", args); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("INSERT INTO protocol_results VALUES (9, 'tls12', 3, 3), (9, 'tls13', 3, 3)"); err != nil { + t.Fatal(err) + } + return db +} + +func TestPopulatedV1FingerprintMigration(t *testing.T) { + args := " --lua-desync=fake --lua-desync=multisplit " + db := populatedV1Store(t, args) + if err := migrate(db); err != nil { + t.Fatal(err) + } + assertRowCounts(t, db, 3, 4) + var version int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 2 { + t.Fatalf("version=%d err=%v", version, err) + } + for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} { + var savedArgs, fingerprint, started, finished, output string + err := db.QueryRow("SELECT args, fingerprint, started_at, finished_at, output FROM tune_runs WHERE id = ?", id).Scan(&savedArgs, &fingerprint, &started, &finished, &output) + want, _ := strategyFingerprint(input) + wantOutput := "original output" + if id == 9 { + wantOutput = "second output" + } + if err != nil || savedArgs != input || fingerprint != want || started != "start" || finished != "finish" || output != wantOutput { + t.Fatalf("run %d: args=%q fingerprint=%q timestamps=%q/%q output=%q err=%v", id, savedArgs, fingerprint, started, finished, output, err) + } + } + var fingerprint sql.NullString + var failure string + if err := db.QueryRow("SELECT fingerprint, error FROM tune_runs WHERE id = 8").Scan(&fingerprint, &failure); err != nil || fingerprint.Valid || failure != "original failure" { + t.Fatalf("failed fingerprint=%+v failure=%q err=%v", fingerprint, failure, err) + } + if err := migrate(db); err != nil { + t.Fatalf("idempotent migration: %v", err) + } +} + +func TestInvalidV1FingerprintMigrationRollback(t *testing.T) { + db := populatedV1Store(t, "--lua-desync=fa ke") + if err := migrate(db); err == nil || !strings.Contains(err.Error(), "run 9 fingerprint") { + t.Fatalf("invalid old args accepted: %v", err) + } + assertRowCounts(t, db, 3, 4) + var version, columns int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 1 { + t.Fatalf("version=%d err=%v", version, err) + } + if err := db.QueryRow("SELECT count(*) FROM pragma_table_info('tune_runs') WHERE name = 'fingerprint'").Scan(&columns); err != nil || columns != 0 { + t.Fatalf("partial migration columns=%d err=%v", columns, err) + } + if _, err := db.Exec("UPDATE tune_runs SET args = '--lua-desync=fake' WHERE id = 9"); err != nil { + t.Fatal(err) + } + if err := migrate(db); err != nil { + t.Fatalf("migration after repair: %v", err) + } +} diff --git a/cmd/zapret-autotune/store.go b/cmd/zapret-autotune/store.go index c7eff99..8e648dd 100644 --- a/cmd/zapret-autotune/store.go +++ b/cmd/zapret-autotune/store.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "crypto/sha256" "database/sql" "encoding/json" "fmt" @@ -113,7 +114,7 @@ func migrate(db *sql.DB) error { if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil { return err } - if version > 1 { + if version > 2 { return fmt.Errorf("unsupported database schema version %d", version) } if version == 0 { @@ -145,14 +146,105 @@ PRAGMA user_version = 1;`) return err } } + if version < 2 { + // Check the required source columns before changing either schema or data. + for _, schema := range []struct { + table string + columns string + count int + }{ + {"tune_runs", `(name = 'id' AND upper(type) = 'INTEGER') OR (name IN ('started_at', 'finished_at', 'target', 'state', 'args', 'output', 'error') AND upper(type) = 'TEXT')`, 8}, + {"protocol_results", `(name IN ('run_id', 'passes_ok', 'passes_total') AND upper(type) = 'INTEGER') OR (name = 'protocol' AND upper(type) = 'TEXT')`, 4}, + } { + var columns int + err := conn.QueryRowContext(ctx, "SELECT count(*) FROM pragma_table_info(?) WHERE ("+schema.columns+") AND EXISTS (SELECT 1 FROM sqlite_master WHERE name = ? AND type = 'table')", schema.table, schema.table).Scan(&columns) + if err != nil { + return err + } + if columns != schema.count { + return fmt.Errorf("invalid v1 schema: %s requires %d expected columns, found %d", schema.table, schema.count, columns) + } + } + if _, err := conn.ExecContext(ctx, "ALTER TABLE tune_runs ADD COLUMN fingerprint TEXT"); err != nil { + return err + } + rows, err := conn.QueryContext(ctx, "SELECT id, args FROM tune_runs WHERE state = 'PROPOSED'") + if err != nil { + return err + } + var fingerprints []struct { + id int64 + value string + } + for rows.Next() { + var id int64 + var args string + if err := rows.Scan(&id, &args); err != nil { + rows.Close() + return err + } + value, err := strategyFingerprint(args) + if err != nil { + rows.Close() + return fmt.Errorf("run %d fingerprint: %w", id, err) + } + fingerprints = append(fingerprints, struct { + id int64 + value string + }{id, value}) + } + err = rows.Err() + closeErr := rows.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + for _, fingerprint := range fingerprints { + if _, err := conn.ExecContext(ctx, "UPDATE tune_runs SET fingerprint = ? WHERE id = ?", fingerprint.value, fingerprint.id); err != nil { + return err + } + } + // Install only after backfill, so existing proposals can acquire their hash. + for _, event := range []string{"INSERT", "UPDATE"} { + if _, err := conn.ExecContext(ctx, `CREATE TRIGGER fingerprint_before_`+strings.ToLower(event)+` BEFORE `+event+` ON tune_runs + WHEN (NEW.state = 'PROPOSED' AND (NEW.fingerprint IS NULL + OR typeof(NEW.fingerprint) != 'text' OR length(NEW.fingerprint) != 64 + OR length(CAST(NEW.fingerprint AS BLOB)) != 64 + OR NEW.fingerprint GLOB '*[^0-9a-f]*')) + OR (NEW.state = 'FAILED' AND NEW.fingerprint IS NOT NULL) + BEGIN SELECT RAISE(ABORT, 'invalid state-dependent fingerprint'); END;`); err != nil { + return err + } + } + if _, err := conn.ExecContext(ctx, "PRAGMA user_version = 2"); err != nil { + return err + } + } _, err = conn.ExecContext(ctx, "COMMIT") return err } type storedRun struct { - ID int64 - StartedAt, FinishedAt, State string - Args, Output, Error sql.NullString + ID int64 + StartedAt, FinishedAt, State string + Args, Output, Error, Fingerprint sql.NullString +} + +func strategyFingerprint(args string) (string, error) { + // The validated grammar permits only unquoted options, with no spaces in + // values. Collapse separators only; never sort or deduplicate arguments. + args = strings.NewReplacer("\t", " ", "\r", " ", "\n", " ", "\v", " ", "\f", " ").Replace(args) + c := candidate{Args: args} + if err := c.validateArgs(); err != nil { + return "", err + } + args = strings.Join(strings.Fields(args), " ") + if args == "" { + return "", fmt.Errorf("args must not be empty") + } + return fmt.Sprintf("%x", sha256.Sum256([]byte(args))), nil } // Validation is shared with tune's output path; only combined 3/3 evidence is stored. @@ -165,7 +257,7 @@ func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, } `json:"strategies"` } state := "FAILED" - var savedArgs, savedOutput, savedError interface{} + var savedArgs, savedOutput, savedError, fingerprint interface{} if runErr == nil { verifiedOutput, err := combinedBlockcheckOutput(bytes.NewReader(report), args) if err != nil { @@ -177,6 +269,10 @@ func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, if err := json.Unmarshal(report, &evidence); err != nil { return err } + fingerprint, err = strategyFingerprint(args) + if err != nil { + return err + } state, savedArgs, savedOutput = "PROPOSED", args, output } else { savedError = runErr.Error() @@ -186,8 +282,8 @@ func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, return err } defer tx.Rollback() - result, err := tx.Exec(`INSERT INTO tune_runs (started_at, finished_at, target, state, args, output, error) VALUES (?, ?, 'youtube.com', ?, ?, ?, ?)`, - started.UTC().Format(time.RFC3339Nano), time.Now().UTC().Format(time.RFC3339Nano), state, savedArgs, savedOutput, savedError) + result, err := tx.Exec(`INSERT INTO tune_runs (started_at, finished_at, target, state, args, output, error, fingerprint) VALUES (?, ?, 'youtube.com', ?, ?, ?, ?, ?)`, + started.UTC().Format(time.RFC3339Nano), time.Now().UTC().Format(time.RFC3339Nano), state, savedArgs, savedOutput, savedError, fingerprint) if err != nil { return err } @@ -205,8 +301,8 @@ func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, func latestRun(db *sql.DB) (storedRun, error) { var r storedRun - err := db.QueryRow(`SELECT id, started_at, finished_at, state, args, output, error FROM tune_runs ORDER BY id DESC LIMIT 1`).Scan( - &r.ID, &r.StartedAt, &r.FinishedAt, &r.State, &r.Args, &r.Output, &r.Error) + err := db.QueryRow(`SELECT id, started_at, finished_at, state, args, output, error, fingerprint FROM tune_runs ORDER BY id DESC LIMIT 1`).Scan( + &r.ID, &r.StartedAt, &r.FinishedAt, &r.State, &r.Args, &r.Output, &r.Error, &r.Fingerprint) if err == nil && r.State == "PROPOSED" { var results int err = db.QueryRow(`SELECT count(*) FROM protocol_results WHERE run_id = ? AND protocol IN ('tls12', 'tls13') AND passes_ok = 3 AND passes_total = 3`, r.ID).Scan(&results) diff --git a/cmd/zapret-autotune/store_test.go b/cmd/zapret-autotune/store_test.go index 6821345..7d2794f 100644 --- a/cmd/zapret-autotune/store_test.go +++ b/cmd/zapret-autotune/store_test.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "errors" + "fmt" "io" "os" "path/filepath" @@ -222,7 +223,7 @@ func TestConcurrentFirstOpens(t *testing.T) { } for _, db := range stores { var version, objects int - if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 1 { + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 2 { t.Fatalf("version=%d err=%v", version, err) } if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE @@ -375,10 +376,138 @@ func TestSchemaConstraintsAndIncompleteEvidence(t *testing.T) { func TestUnsupportedSchemaVersion(t *testing.T) { db := testStore(t) - if _, err := db.Exec("PRAGMA user_version = 2"); err != nil { + if _, err := db.Exec("PRAGMA user_version = 3"); err != nil { t.Fatal(err) } if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") { t.Fatalf("future schema accepted: %v", err) } } + +func TestFingerprintSchemaConstraints(t *testing.T) { + for _, source := range []string{"v0", "v1"} { + t.Run(source, func(t *testing.T) { + var db *sql.DB + if source == "v1" { + db = populatedV1Store(t, "--lua-desync=fake") + if err := migrate(db); err != nil { + t.Fatal(err) + } + } else { + db = testStore(t) + } + valid := strings.Repeat("a", 64) + for _, state := range []string{"PROPOSED", "FAILED"} { + var args, output, failure interface{} + if state == "PROPOSED" { + args, output = "--lua-desync=fake", "output" + } else { + failure = "failure" + } + var initial interface{} + if state == "PROPOSED" { + initial = valid + } + result, err := db.Exec(`INSERT INTO tune_runs (started_at, finished_at, target, state, args, output, error, fingerprint) VALUES ('start', 'finish', 'youtube.com', ?, ?, ?, ?, ?)`, state, args, output, failure, initial) + if err != nil { + t.Fatal(err) + } + id, err := result.LastInsertId() + if err != nil { + t.Fatal(err) + } + invalid := []interface{}{"", strings.Repeat("a", 63), strings.Repeat("a", 65), strings.Repeat("A", 64), strings.Repeat("g", 64), valid + "\x00x"} + if state == "PROPOSED" { + invalid = append(invalid, nil) + } else { + invalid = append(invalid, valid) + } + for _, fingerprint := range invalid { + if _, err := db.Exec(`INSERT INTO tune_runs (started_at, finished_at, target, state, args, output, error, fingerprint) VALUES ('start', 'finish', 'youtube.com', ?, ?, ?, ?, ?)`, state, args, output, failure, fingerprint); err == nil { + t.Fatalf("INSERT accepted %s fingerprint %q", state, fingerprint) + } + if _, err := db.Exec("UPDATE tune_runs SET fingerprint = ? WHERE id = ?", fingerprint, id); err == nil { + t.Fatalf("UPDATE accepted %s fingerprint %q", state, fingerprint) + } + } + // Change all other state-dependent fields too: only the hash is wrong. + statement := "UPDATE tune_runs SET state = 'FAILED', args = NULL, output = NULL, error = 'failure' WHERE id = ?" + if state == "FAILED" { + statement = "UPDATE tune_runs SET state = 'PROPOSED', args = '--lua-desync=fake', output = 'output', error = NULL WHERE id = ?" + } + if _, err := db.Exec(statement, id); err == nil { + t.Fatalf("accepted state change from %s with mismatched fingerprint", state) + } + } + }) + } +} + +func TestMalformedV1SchemaRollback(t *testing.T) { + for name, damage := range map[string]string{ + "missing protocol table": "DROP TABLE protocol_results", + "altered protocol column": "ALTER TABLE protocol_results RENAME COLUMN passes_total TO unexpected", + "altered run column": "ALTER TABLE tune_runs RENAME COLUMN error TO unexpected", + "altered protocol type": `ALTER TABLE protocol_results RENAME TO old_results; + CREATE TABLE protocol_results (run_id INTEGER, protocol TEXT, passes_ok TEXT, passes_total INTEGER); + INSERT INTO protocol_results SELECT * FROM old_results; DROP TABLE old_results;`, + } { + t.Run(name, func(t *testing.T) { + db := populatedV1Store(t, "--lua-desync=fake") + if _, err := db.Exec(damage); err != nil { + t.Fatal(err) + } + snapshot := func() (string, string) { + t.Helper() + var schema, history string + if err := db.QueryRow(`SELECT group_concat(sql, ';') FROM (SELECT sql FROM sqlite_master ORDER BY name)`).Scan(&schema); err != nil { + t.Fatal(err) + } + for _, table := range []string{"tune_runs", "protocol_results"} { + if table == "protocol_results" && name == "missing protocol table" { + continue + } + rows, err := db.Query("SELECT * FROM " + table + " ORDER BY 1, 2") + if err != nil { + t.Fatal(err) + } + columns, err := rows.Columns() + if err != nil { + rows.Close() + t.Fatal(err) + } + values := make([]interface{}, len(columns)) + pointers := make([]interface{}, len(columns)) + for i := range values { + pointers[i] = &values[i] + } + for rows.Next() { + if err := rows.Scan(pointers...); err != nil { + rows.Close() + t.Fatal(err) + } + history += fmt.Sprintf("%s:%#v;", table, values) + } + err = rows.Err() + rows.Close() + if err != nil { + t.Fatal(err) + } + } + return schema, history + } + beforeSchema, beforeHistory := snapshot() + if err := migrate(db); err == nil || !strings.Contains(err.Error(), "invalid v1 schema") { + t.Fatalf("malformed schema accepted: %v", err) + } + var version int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 1 { + t.Fatalf("version=%d err=%v", version, err) + } + afterSchema, afterHistory := snapshot() + if afterSchema != beforeSchema || afterHistory != beforeHistory { + t.Fatal("failed migration changed schema or history") + } + }) + } +}