433 lines
14 KiB
Go
433 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
func databasePath() string {
|
|
if path := os.Getenv("ZAPRET_AUTOTUNE_DB"); path != "" {
|
|
return path
|
|
}
|
|
return "/var/lib/zapret-autotune/autotune.db"
|
|
}
|
|
|
|
func storeURI(path string) (*url.URL, error) {
|
|
// Configuration is a filesystem path, never a caller-supplied SQLite DSN.
|
|
if path == "" || path == ":memory:" || strings.HasPrefix(path, ":memory:?") || strings.HasPrefix(strings.ToLower(path), "file:") {
|
|
return nil, fmt.Errorf("database location must be a filesystem path, not a SQLite DSN")
|
|
}
|
|
path, err := filepath.Abs(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Escape path characters so they cannot become URI options or fragments.
|
|
return &url.URL{Scheme: "file", Path: filepath.ToSlash(path)}, nil
|
|
}
|
|
|
|
// A nil database means the validated path does not exist. Never initialize it.
|
|
func openHistoryStore(path string) (*sql.DB, error) {
|
|
uri, err := storeURI(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := os.Stat(filepath.FromSlash(uri.Path)); err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
uri.RawQuery = "mode=ro"
|
|
db, err := sql.Open("sqlite", uri.String())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
db.SetMaxOpenConns(1)
|
|
if err := db.Ping(); err != nil {
|
|
db.Close()
|
|
return nil, err
|
|
}
|
|
return db, nil
|
|
}
|
|
|
|
func openStore(path string) (*sql.DB, error) {
|
|
uri, err := storeURI(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
path = filepath.FromSlash(uri.Path)
|
|
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
|
return nil, err
|
|
}
|
|
file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := file.Chmod(0600); err != nil {
|
|
file.Close()
|
|
return nil, err
|
|
}
|
|
if err := file.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
db, err := sql.Open("sqlite", uri.String())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
db.SetMaxOpenConns(1)
|
|
if _, err = db.Exec("PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;"); err == nil {
|
|
err = migrate(db)
|
|
}
|
|
if err != nil {
|
|
db.Close()
|
|
return nil, err
|
|
}
|
|
return db, nil
|
|
}
|
|
|
|
func migrate(db *sql.DB) error {
|
|
ctx := context.Background()
|
|
conn, err := db.Conn(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer conn.Close()
|
|
// Serialize migrations before reading the version; a deferred transaction
|
|
// cannot wait via busy_timeout when upgrading a contended read lock.
|
|
if _, err = conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil {
|
|
return err
|
|
}
|
|
defer conn.ExecContext(ctx, "ROLLBACK")
|
|
var version int
|
|
if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil {
|
|
return err
|
|
}
|
|
if version > 4 {
|
|
return fmt.Errorf("unsupported database schema version %d", version)
|
|
}
|
|
if version == 0 {
|
|
_, err = conn.ExecContext(ctx, `
|
|
CREATE TABLE tune_runs (
|
|
id INTEGER PRIMARY KEY,
|
|
started_at TEXT NOT NULL,
|
|
finished_at TEXT NOT NULL,
|
|
target TEXT NOT NULL CHECK (target = 'youtube.com'),
|
|
state TEXT NOT NULL CHECK (state IN ('PROPOSED', 'FAILED')),
|
|
args TEXT,
|
|
output TEXT,
|
|
error TEXT,
|
|
CHECK ((state = 'PROPOSED' AND args IS NOT NULL AND length(args) > 0 AND output IS NOT NULL AND length(output) > 0 AND error IS NULL)
|
|
OR (state = 'FAILED' AND args IS NULL AND output IS NULL AND error IS NOT NULL AND length(error) > 0))
|
|
);
|
|
CREATE TABLE protocol_results (
|
|
run_id INTEGER NOT NULL REFERENCES tune_runs(id) ON DELETE CASCADE,
|
|
protocol TEXT NOT NULL CHECK (protocol IN ('tls12', 'tls13')),
|
|
passes_ok INTEGER NOT NULL CHECK (passes_ok = 3),
|
|
passes_total INTEGER NOT NULL CHECK (passes_total = 3),
|
|
PRIMARY KEY (run_id, protocol)
|
|
);
|
|
CREATE TRIGGER proposed_results_only BEFORE INSERT ON protocol_results
|
|
WHEN (SELECT state FROM tune_runs WHERE id = NEW.run_id) != 'PROPOSED'
|
|
BEGIN SELECT RAISE(ABORT, 'protocol results require a proposed run'); END;
|
|
PRAGMA user_version = 1;`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if version < 2 {
|
|
// Check the required source columns before changing either schema or data.
|
|
for _, schema := range []struct {
|
|
table string
|
|
columns string
|
|
count int
|
|
}{
|
|
{"tune_runs", `(name = 'id' AND upper(type) = 'INTEGER') OR (name IN ('started_at', 'finished_at', 'target', 'state', 'args', 'output', 'error') AND upper(type) = 'TEXT')`, 8},
|
|
{"protocol_results", `(name IN ('run_id', 'passes_ok', 'passes_total') AND upper(type) = 'INTEGER') OR (name = 'protocol' AND upper(type) = 'TEXT')`, 4},
|
|
} {
|
|
var columns int
|
|
err := conn.QueryRowContext(ctx, "SELECT count(*) FROM pragma_table_info(?) WHERE ("+schema.columns+") AND EXISTS (SELECT 1 FROM sqlite_master WHERE name = ? AND type = 'table')", schema.table, schema.table).Scan(&columns)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if columns != schema.count {
|
|
return fmt.Errorf("invalid v1 schema: %s requires %d expected columns, found %d", schema.table, schema.count, columns)
|
|
}
|
|
}
|
|
if _, err := conn.ExecContext(ctx, "ALTER TABLE tune_runs ADD COLUMN fingerprint TEXT"); err != nil {
|
|
return err
|
|
}
|
|
rows, err := conn.QueryContext(ctx, "SELECT id, args FROM tune_runs WHERE state = 'PROPOSED'")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var fingerprints []struct {
|
|
id int64
|
|
value string
|
|
}
|
|
for rows.Next() {
|
|
var id int64
|
|
var args string
|
|
if err := rows.Scan(&id, &args); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
value, err := strategyFingerprint(args)
|
|
if err != nil {
|
|
rows.Close()
|
|
return fmt.Errorf("run %d fingerprint: %w", id, err)
|
|
}
|
|
fingerprints = append(fingerprints, struct {
|
|
id int64
|
|
value string
|
|
}{id, value})
|
|
}
|
|
err = rows.Err()
|
|
closeErr := rows.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if closeErr != nil {
|
|
return closeErr
|
|
}
|
|
for _, fingerprint := range fingerprints {
|
|
if _, err := conn.ExecContext(ctx, "UPDATE tune_runs SET fingerprint = ? WHERE id = ?", fingerprint.value, fingerprint.id); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// Install only after backfill, so existing proposals can acquire their hash.
|
|
for _, event := range []string{"INSERT", "UPDATE"} {
|
|
if _, err := conn.ExecContext(ctx, `CREATE TRIGGER fingerprint_before_`+strings.ToLower(event)+` BEFORE `+event+` ON tune_runs
|
|
WHEN (NEW.state = 'PROPOSED' AND (NEW.fingerprint IS NULL
|
|
OR typeof(NEW.fingerprint) != 'text' OR length(NEW.fingerprint) != 64
|
|
OR length(CAST(NEW.fingerprint AS BLOB)) != 64
|
|
OR NEW.fingerprint GLOB '*[^0-9a-f]*'))
|
|
OR (NEW.state = 'FAILED' AND NEW.fingerprint IS NOT NULL)
|
|
BEGIN SELECT RAISE(ABORT, 'invalid state-dependent fingerprint'); END;`); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := conn.ExecContext(ctx, "PRAGMA user_version = 2"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if version < 3 {
|
|
if _, err := conn.ExecContext(ctx, candidateSchema); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// Additive v3 cache-run journal: the legacy frontend target constraint stays intact.
|
|
if _, err := conn.ExecContext(ctx, googlevideoRunSchema); err != nil {
|
|
return err
|
|
}
|
|
if version < 4 {
|
|
if _, err := conn.ExecContext(ctx, monitorSchema); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
_, err = conn.ExecContext(ctx, "COMMIT")
|
|
return err
|
|
}
|
|
|
|
type storedRun struct {
|
|
ID int64
|
|
StartedAt, FinishedAt, State string
|
|
Args, Output, Error, Fingerprint sql.NullString
|
|
}
|
|
|
|
func strategyFingerprint(args string) (string, error) {
|
|
// The validated grammar permits only unquoted options, with no spaces in
|
|
// values. Collapse separators only; never sort or deduplicate arguments.
|
|
args = strings.NewReplacer("\t", " ", "\r", " ", "\n", " ", "\v", " ", "\f", " ").Replace(args)
|
|
c := candidate{Args: args}
|
|
if err := c.validateArgs(); err != nil {
|
|
return "", err
|
|
}
|
|
args = strings.Join(strings.Fields(args), " ")
|
|
if args == "" {
|
|
return "", fmt.Errorf("args must not be empty")
|
|
}
|
|
return fmt.Sprintf("%x", sha256.Sum256([]byte(args))), nil
|
|
}
|
|
|
|
// Validation is shared with tune's output path; only combined 3/3 evidence is stored.
|
|
func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, runErr error) error {
|
|
var evidence struct {
|
|
Strategies []json.RawMessage `json:"strategies"`
|
|
}
|
|
state := "FAILED"
|
|
var savedArgs, savedOutput, savedError, fingerprint interface{}
|
|
if runErr == nil {
|
|
verifiedOutput, err := combinedBlockcheckOutput(bytes.NewReader(report), args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if output != verifiedOutput {
|
|
return fmt.Errorf("output does not match verified candidate")
|
|
}
|
|
if err := json.Unmarshal(report, &evidence); err != nil {
|
|
return err
|
|
}
|
|
fingerprint, err = strategyFingerprint(args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
state, savedArgs, savedOutput = "PROPOSED", args, output
|
|
} else {
|
|
savedError = runErr.Error()
|
|
}
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
result, err := tx.Exec(`INSERT INTO tune_runs (started_at, finished_at, target, state, args, output, error, fingerprint) VALUES (?, ?, 'youtube.com', ?, ?, ?, ?, ?)`,
|
|
started.UTC().Format(time.RFC3339Nano), time.Now().UTC().Format(time.RFC3339Nano), state, savedArgs, savedOutput, savedError, fingerprint)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
id, err := result.LastInsertId()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var candidateID int64
|
|
checked := time.Now().UTC()
|
|
if runErr == nil {
|
|
candidateID, err = insertStrategyCandidate(tx, "frontend", []string{"youtube.com"}, args, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, raw := range evidence.Strategies {
|
|
var row struct {
|
|
candidate
|
|
Protocol string `json:"protocol"`
|
|
}
|
|
if err := json.Unmarshal(raw, &row); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(`INSERT INTO protocol_results (run_id, protocol, passes_ok, passes_total) VALUES (?, ?, ?, ?)`, id, normalizeProtocol(row.Protocol), row.PassesOK, row.PassesTotal); err != nil {
|
|
return err
|
|
}
|
|
if err := saveCandidateEvidence(tx, candidateID, "youtube.com", normalizeProtocol(row.Protocol), raw, checked); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
func latestRun(db *sql.DB) (storedRun, error) {
|
|
var r storedRun
|
|
err := db.QueryRow(`SELECT id, started_at, finished_at, state, args, output, error, fingerprint FROM tune_runs ORDER BY id DESC LIMIT 1`).Scan(
|
|
&r.ID, &r.StartedAt, &r.FinishedAt, &r.State, &r.Args, &r.Output, &r.Error, &r.Fingerprint)
|
|
if err == nil && r.State == "PROPOSED" {
|
|
var results int
|
|
err = db.QueryRow(`SELECT count(*) FROM protocol_results WHERE run_id = ? AND protocol IN ('tls12', 'tls13') AND passes_ok = 3 AND passes_total = 3`, r.ID).Scan(&results)
|
|
if err == nil && results != 2 {
|
|
err = fmt.Errorf("proposed run %d has incomplete protocol evidence", r.ID)
|
|
}
|
|
}
|
|
return r, err
|
|
}
|
|
|
|
func proposedStrategies(db *sql.DB) (string, error) {
|
|
rows, err := db.Query(`SELECT id, finished_at, args,
|
|
(SELECT count(*) FROM protocol_results WHERE run_id = tune_runs.id
|
|
AND protocol IN ('tls12', 'tls13') AND passes_ok = 3 AND passes_total = 3)
|
|
FROM tune_runs WHERE state = 'PROPOSED' AND target = 'youtube.com' ORDER BY id DESC`)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer rows.Close()
|
|
var text strings.Builder
|
|
for rows.Next() {
|
|
var id int64
|
|
var finished, args string
|
|
var results int
|
|
if err := rows.Scan(&id, &finished, &args, &results); err != nil {
|
|
return "", err
|
|
}
|
|
if results != 2 {
|
|
return "", fmt.Errorf("proposed run %d has incomplete protocol evidence", id)
|
|
}
|
|
fmt.Fprintf(&text, "Run %d; finished %s; youtube.com frontend only; PROPOSED, NOT ACTIVE\nArgs: %s\nRecorded combined check evidence: TLS1.2 3/3; TLS1.3 3/3.\n", id, finished, args)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return "", err
|
|
}
|
|
if text.Len() == 0 {
|
|
text.WriteString("No recorded strategies for youtube.com.\n")
|
|
}
|
|
text.WriteString("No configuration applied. Throughput, GGC and production health: NOT CHECKED by this command.\n")
|
|
return text.String(), nil
|
|
}
|
|
|
|
func strategiesYoutube(stdout, stderr io.Writer) int {
|
|
db, err := openStore(databasePath())
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "error: strategies youtube database:", err)
|
|
return 1
|
|
}
|
|
defer db.Close()
|
|
text, err := proposedStrategies(db)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "error: strategies youtube database:", err)
|
|
return 1
|
|
}
|
|
cacheText, err := googlevideoStrategies(db)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "error: strategies youtube database:", err)
|
|
return 1
|
|
}
|
|
if cacheText != "" {
|
|
text = strings.Replace(text, "Throughput, GGC and production health: NOT CHECKED by this command.", "Frontend-only scope above; GGC evidence below is exact-host TLS/data-transfer only, not playback or production health.", 1)
|
|
text += cacheText
|
|
}
|
|
if _, err := fmt.Fprint(stdout, text); err != nil {
|
|
fmt.Fprintln(stderr, "error: cannot write strategies:", err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func status(stdout, stderr io.Writer) int {
|
|
db, err := openStore(databasePath())
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "error: status database:", err)
|
|
return 1
|
|
}
|
|
defer db.Close()
|
|
r, err := latestRun(db)
|
|
var text string
|
|
if err == sql.ErrNoRows {
|
|
text = "Status: no recorded tune run.\n"
|
|
} else if err != nil {
|
|
fmt.Fprintln(stderr, "error: status database:", err)
|
|
return 1
|
|
} else {
|
|
text = fmt.Sprintf("Latest tune run: %d; youtube.com frontend only; %s; finished %s\n", r.ID, r.State, r.FinishedAt)
|
|
if r.State == "PROPOSED" {
|
|
text += "Candidate: PROPOSED, not active; verified TLS1.2 3/3 and TLS1.3 3/3.\n" + r.Output.String + "\n"
|
|
} else {
|
|
text += "No proposed candidate from this run. Error: " + r.Error.String + "\n"
|
|
}
|
|
}
|
|
text += "apply: BLOCKED (not implemented). No configuration applied.\nProduction ss-zapret2 and Xray health: NOT CHECKED.\n"
|
|
if _, err := fmt.Fprint(stdout, text); err != nil {
|
|
fmt.Fprintln(stderr, "error: cannot write status:", err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|