Files
zapret-autotune/cmd/zapret-autotune/store.go
T
2026-10-10 08:04:48 +03:00

433 lines
14 KiB
Go

package main
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/json"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strings"
"time"
_ "modernc.org/sqlite"
)
func databasePath() string {
if path := os.Getenv("ZAPRET_AUTOTUNE_DB"); path != "" {
return path
}
return "/var/lib/zapret-autotune/autotune.db"
}
func storeURI(path string) (*url.URL, error) {
// Configuration is a filesystem path, never a caller-supplied SQLite DSN.
if path == "" || path == ":memory:" || strings.HasPrefix(path, ":memory:?") || strings.HasPrefix(strings.ToLower(path), "file:") {
return nil, fmt.Errorf("database location must be a filesystem path, not a SQLite DSN")
}
path, err := filepath.Abs(path)
if err != nil {
return nil, err
}
// Escape path characters so they cannot become URI options or fragments.
return &url.URL{Scheme: "file", Path: filepath.ToSlash(path)}, nil
}
// A nil database means the validated path does not exist. Never initialize it.
func openHistoryStore(path string) (*sql.DB, error) {
uri, err := storeURI(path)
if err != nil {
return nil, err
}
if _, err := os.Stat(filepath.FromSlash(uri.Path)); err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
uri.RawQuery = "mode=ro"
db, err := sql.Open("sqlite", uri.String())
if err != nil {
return nil, err
}
db.SetMaxOpenConns(1)
if err := db.Ping(); err != nil {
db.Close()
return nil, err
}
return db, nil
}
func openStore(path string) (*sql.DB, error) {
uri, err := storeURI(path)
if err != nil {
return nil, err
}
path = filepath.FromSlash(uri.Path)
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return nil, err
}
file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600)
if err != nil {
return nil, err
}
if err := file.Chmod(0600); err != nil {
file.Close()
return nil, err
}
if err := file.Close(); err != nil {
return nil, err
}
db, err := sql.Open("sqlite", uri.String())
if err != nil {
return nil, err
}
db.SetMaxOpenConns(1)
if _, err = db.Exec("PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;"); err == nil {
err = migrate(db)
}
if err != nil {
db.Close()
return nil, err
}
return db, nil
}
func migrate(db *sql.DB) error {
ctx := context.Background()
conn, err := db.Conn(ctx)
if err != nil {
return err
}
defer conn.Close()
// Serialize migrations before reading the version; a deferred transaction
// cannot wait via busy_timeout when upgrading a contended read lock.
if _, err = conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil {
return err
}
defer conn.ExecContext(ctx, "ROLLBACK")
var version int
if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil {
return err
}
if version > 4 {
return fmt.Errorf("unsupported database schema version %d", version)
}
if version == 0 {
_, err = conn.ExecContext(ctx, `
CREATE TABLE tune_runs (
id INTEGER PRIMARY KEY,
started_at TEXT NOT NULL,
finished_at TEXT NOT NULL,
target TEXT NOT NULL CHECK (target = 'youtube.com'),
state TEXT NOT NULL CHECK (state IN ('PROPOSED', 'FAILED')),
args TEXT,
output TEXT,
error TEXT,
CHECK ((state = 'PROPOSED' AND args IS NOT NULL AND length(args) > 0 AND output IS NOT NULL AND length(output) > 0 AND error IS NULL)
OR (state = 'FAILED' AND args IS NULL AND output IS NULL AND error IS NOT NULL AND length(error) > 0))
);
CREATE TABLE protocol_results (
run_id INTEGER NOT NULL REFERENCES tune_runs(id) ON DELETE CASCADE,
protocol TEXT NOT NULL CHECK (protocol IN ('tls12', 'tls13')),
passes_ok INTEGER NOT NULL CHECK (passes_ok = 3),
passes_total INTEGER NOT NULL CHECK (passes_total = 3),
PRIMARY KEY (run_id, protocol)
);
CREATE TRIGGER proposed_results_only BEFORE INSERT ON protocol_results
WHEN (SELECT state FROM tune_runs WHERE id = NEW.run_id) != 'PROPOSED'
BEGIN SELECT RAISE(ABORT, 'protocol results require a proposed run'); END;
PRAGMA user_version = 1;`)
if err != nil {
return err
}
}
if version < 2 {
// Check the required source columns before changing either schema or data.
for _, schema := range []struct {
table string
columns string
count int
}{
{"tune_runs", `(name = 'id' AND upper(type) = 'INTEGER') OR (name IN ('started_at', 'finished_at', 'target', 'state', 'args', 'output', 'error') AND upper(type) = 'TEXT')`, 8},
{"protocol_results", `(name IN ('run_id', 'passes_ok', 'passes_total') AND upper(type) = 'INTEGER') OR (name = 'protocol' AND upper(type) = 'TEXT')`, 4},
} {
var columns int
err := conn.QueryRowContext(ctx, "SELECT count(*) FROM pragma_table_info(?) WHERE ("+schema.columns+") AND EXISTS (SELECT 1 FROM sqlite_master WHERE name = ? AND type = 'table')", schema.table, schema.table).Scan(&columns)
if err != nil {
return err
}
if columns != schema.count {
return fmt.Errorf("invalid v1 schema: %s requires %d expected columns, found %d", schema.table, schema.count, columns)
}
}
if _, err := conn.ExecContext(ctx, "ALTER TABLE tune_runs ADD COLUMN fingerprint TEXT"); err != nil {
return err
}
rows, err := conn.QueryContext(ctx, "SELECT id, args FROM tune_runs WHERE state = 'PROPOSED'")
if err != nil {
return err
}
var fingerprints []struct {
id int64
value string
}
for rows.Next() {
var id int64
var args string
if err := rows.Scan(&id, &args); err != nil {
rows.Close()
return err
}
value, err := strategyFingerprint(args)
if err != nil {
rows.Close()
return fmt.Errorf("run %d fingerprint: %w", id, err)
}
fingerprints = append(fingerprints, struct {
id int64
value string
}{id, value})
}
err = rows.Err()
closeErr := rows.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
for _, fingerprint := range fingerprints {
if _, err := conn.ExecContext(ctx, "UPDATE tune_runs SET fingerprint = ? WHERE id = ?", fingerprint.value, fingerprint.id); err != nil {
return err
}
}
// Install only after backfill, so existing proposals can acquire their hash.
for _, event := range []string{"INSERT", "UPDATE"} {
if _, err := conn.ExecContext(ctx, `CREATE TRIGGER fingerprint_before_`+strings.ToLower(event)+` BEFORE `+event+` ON tune_runs
WHEN (NEW.state = 'PROPOSED' AND (NEW.fingerprint IS NULL
OR typeof(NEW.fingerprint) != 'text' OR length(NEW.fingerprint) != 64
OR length(CAST(NEW.fingerprint AS BLOB)) != 64
OR NEW.fingerprint GLOB '*[^0-9a-f]*'))
OR (NEW.state = 'FAILED' AND NEW.fingerprint IS NOT NULL)
BEGIN SELECT RAISE(ABORT, 'invalid state-dependent fingerprint'); END;`); err != nil {
return err
}
}
if _, err := conn.ExecContext(ctx, "PRAGMA user_version = 2"); err != nil {
return err
}
}
if version < 3 {
if _, err := conn.ExecContext(ctx, candidateSchema); err != nil {
return err
}
}
// Additive v3 cache-run journal: the legacy frontend target constraint stays intact.
if _, err := conn.ExecContext(ctx, googlevideoRunSchema); err != nil {
return err
}
if version < 4 {
if _, err := conn.ExecContext(ctx, monitorSchema); err != nil {
return err
}
}
_, err = conn.ExecContext(ctx, "COMMIT")
return err
}
type storedRun struct {
ID int64
StartedAt, FinishedAt, State string
Args, Output, Error, Fingerprint sql.NullString
}
func strategyFingerprint(args string) (string, error) {
// The validated grammar permits only unquoted options, with no spaces in
// values. Collapse separators only; never sort or deduplicate arguments.
args = strings.NewReplacer("\t", " ", "\r", " ", "\n", " ", "\v", " ", "\f", " ").Replace(args)
c := candidate{Args: args}
if err := c.validateArgs(); err != nil {
return "", err
}
args = strings.Join(strings.Fields(args), " ")
if args == "" {
return "", fmt.Errorf("args must not be empty")
}
return fmt.Sprintf("%x", sha256.Sum256([]byte(args))), nil
}
// Validation is shared with tune's output path; only combined 3/3 evidence is stored.
func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, runErr error) error {
var evidence struct {
Strategies []json.RawMessage `json:"strategies"`
}
state := "FAILED"
var savedArgs, savedOutput, savedError, fingerprint interface{}
if runErr == nil {
verifiedOutput, err := combinedBlockcheckOutput(bytes.NewReader(report), args)
if err != nil {
return err
}
if output != verifiedOutput {
return fmt.Errorf("output does not match verified candidate")
}
if err := json.Unmarshal(report, &evidence); err != nil {
return err
}
fingerprint, err = strategyFingerprint(args)
if err != nil {
return err
}
state, savedArgs, savedOutput = "PROPOSED", args, output
} else {
savedError = runErr.Error()
}
tx, err := db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
result, err := tx.Exec(`INSERT INTO tune_runs (started_at, finished_at, target, state, args, output, error, fingerprint) VALUES (?, ?, 'youtube.com', ?, ?, ?, ?, ?)`,
started.UTC().Format(time.RFC3339Nano), time.Now().UTC().Format(time.RFC3339Nano), state, savedArgs, savedOutput, savedError, fingerprint)
if err != nil {
return err
}
id, err := result.LastInsertId()
if err != nil {
return err
}
var candidateID int64
checked := time.Now().UTC()
if runErr == nil {
candidateID, err = insertStrategyCandidate(tx, "frontend", []string{"youtube.com"}, args, id)
if err != nil {
return err
}
}
for _, raw := range evidence.Strategies {
var row struct {
candidate
Protocol string `json:"protocol"`
}
if err := json.Unmarshal(raw, &row); err != nil {
return err
}
if _, err := tx.Exec(`INSERT INTO protocol_results (run_id, protocol, passes_ok, passes_total) VALUES (?, ?, ?, ?)`, id, normalizeProtocol(row.Protocol), row.PassesOK, row.PassesTotal); err != nil {
return err
}
if err := saveCandidateEvidence(tx, candidateID, "youtube.com", normalizeProtocol(row.Protocol), raw, checked); err != nil {
return err
}
}
return tx.Commit()
}
func latestRun(db *sql.DB) (storedRun, error) {
var r storedRun
err := db.QueryRow(`SELECT id, started_at, finished_at, state, args, output, error, fingerprint FROM tune_runs ORDER BY id DESC LIMIT 1`).Scan(
&r.ID, &r.StartedAt, &r.FinishedAt, &r.State, &r.Args, &r.Output, &r.Error, &r.Fingerprint)
if err == nil && r.State == "PROPOSED" {
var results int
err = db.QueryRow(`SELECT count(*) FROM protocol_results WHERE run_id = ? AND protocol IN ('tls12', 'tls13') AND passes_ok = 3 AND passes_total = 3`, r.ID).Scan(&results)
if err == nil && results != 2 {
err = fmt.Errorf("proposed run %d has incomplete protocol evidence", r.ID)
}
}
return r, err
}
func proposedStrategies(db *sql.DB) (string, error) {
rows, err := db.Query(`SELECT id, finished_at, args,
(SELECT count(*) FROM protocol_results WHERE run_id = tune_runs.id
AND protocol IN ('tls12', 'tls13') AND passes_ok = 3 AND passes_total = 3)
FROM tune_runs WHERE state = 'PROPOSED' AND target = 'youtube.com' ORDER BY id DESC`)
if err != nil {
return "", err
}
defer rows.Close()
var text strings.Builder
for rows.Next() {
var id int64
var finished, args string
var results int
if err := rows.Scan(&id, &finished, &args, &results); err != nil {
return "", err
}
if results != 2 {
return "", fmt.Errorf("proposed run %d has incomplete protocol evidence", id)
}
fmt.Fprintf(&text, "Run %d; finished %s; youtube.com frontend only; PROPOSED, NOT ACTIVE\nArgs: %s\nRecorded combined check evidence: TLS1.2 3/3; TLS1.3 3/3.\n", id, finished, args)
}
if err := rows.Err(); err != nil {
return "", err
}
if text.Len() == 0 {
text.WriteString("No recorded strategies for youtube.com.\n")
}
text.WriteString("No configuration applied. Throughput, GGC and production health: NOT CHECKED by this command.\n")
return text.String(), nil
}
func strategiesYoutube(stdout, stderr io.Writer) int {
db, err := openStore(databasePath())
if err != nil {
fmt.Fprintln(stderr, "error: strategies youtube database:", err)
return 1
}
defer db.Close()
text, err := proposedStrategies(db)
if err != nil {
fmt.Fprintln(stderr, "error: strategies youtube database:", err)
return 1
}
cacheText, err := googlevideoStrategies(db)
if err != nil {
fmt.Fprintln(stderr, "error: strategies youtube database:", err)
return 1
}
if cacheText != "" {
text = strings.Replace(text, "Throughput, GGC and production health: NOT CHECKED by this command.", "Frontend-only scope above; GGC evidence below is exact-host TLS/data-transfer only, not playback or production health.", 1)
text += cacheText
}
if _, err := fmt.Fprint(stdout, text); err != nil {
fmt.Fprintln(stderr, "error: cannot write strategies:", err)
return 1
}
return 0
}
func status(stdout, stderr io.Writer) int {
db, err := openStore(databasePath())
if err != nil {
fmt.Fprintln(stderr, "error: status database:", err)
return 1
}
defer db.Close()
r, err := latestRun(db)
var text string
if err == sql.ErrNoRows {
text = "Status: no recorded tune run.\n"
} else if err != nil {
fmt.Fprintln(stderr, "error: status database:", err)
return 1
} else {
text = fmt.Sprintf("Latest tune run: %d; youtube.com frontend only; %s; finished %s\n", r.ID, r.State, r.FinishedAt)
if r.State == "PROPOSED" {
text += "Candidate: PROPOSED, not active; verified TLS1.2 3/3 and TLS1.3 3/3.\n" + r.Output.String + "\n"
} else {
text += "No proposed candidate from this run. Error: " + r.Error.String + "\n"
}
}
text += "apply: BLOCKED (not implemented). No configuration applied.\nProduction ss-zapret2 and Xray health: NOT CHECKED.\n"
if _, err := fmt.Fprint(stdout, text); err != nil {
fmt.Fprintln(stderr, "error: cannot write status:", err)
return 1
}
return 0
}