feat(tune): add proposal-only Googlevideo scans

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-09 23:04:53 +03:00
1 parent 7fad9b3b5c
commit 6220eb0b31
10 files changed
+638 -27

No files matched your search

+15 -4
View File
@@ -96,6 +96,13 @@ func blockcheckVanillaReport(dir string) (string, error) {
}
func combinedVanillaRows(args string) (string, error) {
return combinedVanillaRowsForHost(args, "youtube.com")
}
func combinedVanillaRowsForHost(args, host string) (string, error) {
if !validDomain(host) {
return "", fmt.Errorf("invalid combined hostname")
}
c := candidate{Args: args}
if strings.TrimSpace(args) == "" {
return "", fmt.Errorf("empty combined args")
@@ -103,13 +110,17 @@ func combinedVanillaRows(args string) (string, error) {
if err := c.validateArgs(); err != nil {
return "", err
}
return "curl_test_https_tls12 ipv4 youtube.com : nfqws2 " + args + "\n" +
"curl_test_https_tls13 ipv4 youtube.com : nfqws2 " + args + "\n", nil
return "curl_test_https_tls12 ipv4 " + host + " : nfqws2 " + args + "\n" +
"curl_test_https_tls13 ipv4 " + host + " : nfqws2 " + args + "\n", nil
}
// Retain the exact combined input and report, including on failure.
func runCombinedBlockcheck(ctx context.Context, binary, path, args string, stdout, stderr io.Writer) error {
rows, err := combinedVanillaRows(args)
return runCombinedBlockcheckForHost(ctx, binary, path, args, "youtube.com", stdout, stderr)
}
func runCombinedBlockcheckForHost(ctx context.Context, binary, path, args, host string, stdout, stderr io.Writer) error {
rows, err := combinedVanillaRowsForHost(args, host)
if err != nil {
return err
}
@@ -121,7 +132,7 @@ func runCombinedBlockcheck(ctx context.Context, binary, path, args string, stdou
return err
}
if err := runBlockcheckCommand(ctx, binary, "", []string{"--workers", "32", "check", "--from-file", input,
"--domain", "youtube.com", "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup",
"--domain", host, "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup",
"--output", path + ".combined.json"}, stdout, stderr); err != nil {
return fmt.Errorf("combined check (input %s, output %s): %w", input, path+".combined.json", err)
}
+14 -6
View File
@@ -103,17 +103,25 @@ func playerHostname(body []byte) (string, error) {
return host, nil
}
func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunner) int {
func discoveryFlags(args []string) (string, string, error) {
flags := flag.NewFlagSet("discover youtube", flag.ContinueOnError)
flags.SetOutput(io.Discard) // flag errors can contain untrusted arguments.
videoID := flags.String("video-id", "", "")
proxy := flags.String("proxy", "", "")
if flags.Parse(args) != nil || flags.NArg() != 0 || !videoIDPattern.MatchString(*videoID) || !validDiscoveryProxy(*proxy) {
fmt.Fprintln(stderr, "error: discover youtube requires --video-id (11 characters) and --proxy socks5://HOST:PORT without credentials")
return "", "", errors.New("requires --video-id (11 characters) and --proxy socks5://HOST:PORT without credentials")
}
return *videoID, *proxy, nil
}
func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunner) int {
videoID, proxy, err := discoveryFlags(args)
if err != nil {
fmt.Fprintln(stderr, "error: discover youtube", err)
return 2
}
payload, _ := json.Marshal(map[string]interface{}{
"videoId": *videoID,
"videoId": videoID,
"context": map[string]interface{}{"client": map[string]interface{}{
"clientName": "ANDROID", "clientVersion": "21.02.35", "androidSdkVersion": 30,
"userAgent": playerUserAgent, "osName": "Android", "osVersion": "11",
@@ -125,8 +133,8 @@ func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunne
output := &playerBoundedWriter{dst: &body, limit: playerResponseLimit}
diagnostics := &playerBoundedWriter{dst: io.Discard, limit: 64 << 10}
// Disable curlrc and NO_PROXY bypasses: the explicit proxy must be used.
err := runner(ctx, []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30",
"--noproxy", "", "--proxy", *proxy, "--user-agent", playerUserAgent, "--json", string(payload), playerURL}, output, diagnostics)
err = runner(ctx, []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30",
"--noproxy", "", "--proxy", proxy, "--user-agent", playerUserAgent, "--json", string(payload), playerURL}, output, diagnostics)
if err != nil || output.exceeded || diagnostics.exceeded {
fmt.Fprintln(stderr, "error: player request failed or exceeded output limit")
return 1
@@ -139,7 +147,7 @@ func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunne
result := struct {
VideoID string `json:"videoId"`
Hostnames []string `json:"hostnames"`
}{*videoID, []string{host}}
}{videoID, []string{host}}
if json.NewEncoder(stdout).Encode(result) != nil {
fmt.Fprintln(stderr, "error: cannot write discovery result")
return 1
+230
View File
@@ -0,0 +1,230 @@
package main
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"fmt"
"io"
"os"
"os/signal"
"strings"
"syscall"
"time"
)
// Separate journal preserves the legacy frontend-only tune_runs constraint.
// Both journals refer to the same v3 immutable candidate/evidence model.
const googlevideoRunSchema = `CREATE TABLE IF NOT EXISTS googlevideo_tune_runs (
id INTEGER PRIMARY KEY,
started_at TEXT NOT NULL,
finished_at TEXT NOT NULL,
endpoint TEXT,
state TEXT NOT NULL CHECK (state IN ('PROPOSED', 'FAILED')),
candidate_id INTEGER UNIQUE REFERENCES strategy_candidates(id),
error TEXT,
CHECK ((state = 'PROPOSED' AND endpoint IS NOT NULL AND candidate_id IS NOT NULL AND error IS NULL)
OR (state = 'FAILED' AND candidate_id IS NULL AND error IS NOT NULL AND length(error) > 0))
);`
type cacheScan func(context.Context, string, string, io.Writer, io.Writer) (string, error)
type cacheCheck func(context.Context, string, string, string, string, io.Writer, io.Writer) error
func tuneGooglevideo(flags []string, stdout, stderr io.Writer, discover playerRunner, scan cacheScan, check cacheCheck) int {
if _, _, err := discoveryFlags(flags); err != nil {
fmt.Fprintln(stderr, "error: tune googlevideo", err)
return 2
}
started := time.Now()
db, err := openStore(databasePath())
if err != nil {
fmt.Fprintln(stderr, "error: tune googlevideo database:", err)
return 1
}
defer db.Close()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
var discovered, diagnostics bytes.Buffer
var host, args, output string
var report []byte
// Reuse the validated Android player request; never relay its body or runner errors.
if discoverYoutube(flags, &discovered, &diagnostics, discover) != 0 {
err = fmt.Errorf("cache discovery failed")
} else {
var result struct {
Hostnames []string `json:"hostnames"`
}
if json.Unmarshal(discovered.Bytes(), &result) != nil || len(result.Hostnames) != 1 {
err = fmt.Errorf("cache discovery requires exactly one hostname")
} else {
host = result.Hostnames[0]
}
}
if err == nil {
fmt.Fprintln(stderr, "Testing exact googlevideo endpoint:", host, "(proposal only; no configuration applied).")
fmt.Fprintln(stderr, "TLS1.2+TLS1.3 scan; real-data shortlist --take 10 --passes 3; exact combined check 3/3 on both protocols. No Selectel ranking or playback claim.")
var raw bytes.Buffer
var path string
path, err = scan(ctx, "blockcheckw", host, &raw, &raw)
// Scanner diagnostics stay in restricted artifacts, never on the output path.
if path != "" {
fmt.Fprintln(stderr, "Blockcheck artifacts:", path)
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
err = e
}
}
if err == nil {
report, err = os.ReadFile(path + ".check.json")
}
if err == nil {
args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", "googlevideo", host)
}
if err == nil {
err = check(ctx, "blockcheckw", path, args, host, &raw, &raw)
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
err = e
}
}
if err == nil {
report, err = os.ReadFile(path + ".combined.json")
}
if err == nil {
output, err = googlevideoOutput(report, args, host)
}
if err == nil {
err = ctx.Err()
}
}
// Do not store or print arbitrary runner errors (they might contain signed URLs).
var safeError error
if err != nil {
safeError = fmt.Errorf("discovery, scan or combined TLS verification failed; no candidate")
}
if saveErr := saveGooglevideoRun(db, started, host, args, report, safeError); saveErr != nil {
// Best effort failed-run recording after the successful transaction rolled back.
_ = saveGooglevideoRun(db, started, host, "", nil, fmt.Errorf("candidate persistence failed"))
fmt.Fprintln(stderr, "error: tune googlevideo database: candidate persistence failed; no options emitted")
return 1
}
if safeError != nil {
fmt.Fprintln(stderr, "error: tune googlevideo:", safeError)
return 1
}
if _, err := fmt.Fprintln(stdout, output); err != nil {
fmt.Fprintln(stderr, "error: cannot write googlevideo proposal")
return 1
}
return 0
}
func googlevideoOutput(report []byte, args, host string) (string, error) {
if !discoveryHostname(host) || !strings.HasSuffix(host, ".googlevideo.com") || host != strings.ToLower(host) || args == "" {
return "", fmt.Errorf("invalid exact cache scope or empty args")
}
if _, err := selectBlockcheckArgsForTarget(bytes.NewReader(report), args, "googlevideo", host); err != nil {
return "", err
}
// Bare hostlist entries also match subdomains; evidence covers only this host.
return "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=^" + host + " " + args + "'", nil
}
func saveGooglevideoRun(db *sql.DB, started time.Time, host, args string, report []byte, runErr error) error {
var evidence struct {
Strategies []json.RawMessage `json:"strategies"`
}
if runErr == nil {
if _, err := googlevideoOutput(report, args, host); err != nil {
return err
}
if err := json.Unmarshal(report, &evidence); err != nil {
return err
}
}
tx, err := db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
checked := time.Now().UTC()
var endpoint, candidateID, savedError interface{}
if host != "" {
endpoint = host
}
state := "FAILED"
if runErr != nil {
savedError = runErr.Error()
} else {
id, err := insertStrategyCandidate(tx, "googlevideo", []string{host}, args, nil)
if err != nil {
return err
}
candidateID, state = id, "PROPOSED"
for _, raw := range evidence.Strategies {
var row struct {
Protocol string `json:"protocol"`
}
if err := json.Unmarshal(raw, &row); err != nil {
return err
}
if err := saveCandidateEvidence(tx, id, host, normalizeProtocol(row.Protocol), raw, checked); err != nil {
return err
}
}
}
if _, err := tx.Exec(`INSERT INTO googlevideo_tune_runs (started_at, finished_at, endpoint, state, candidate_id, error) VALUES (?, ?, ?, ?, ?, ?)`, started.UTC().Format(time.RFC3339Nano), checked.Format(time.RFC3339Nano), endpoint, state, candidateID, savedError); err != nil {
return err
}
return tx.Commit()
}
func hasGooglevideoRuns(db *sql.DB) (bool, error) {
var exists int
if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = 'googlevideo_tune_runs'`).Scan(&exists); err != nil {
return false, err
}
if exists == 0 {
return false, nil
}
if err := db.QueryRow(`SELECT count(*) FROM googlevideo_tune_runs`).Scan(&exists); err != nil {
return false, err
}
return exists > 0, nil
}
func googlevideoStrategies(db *sql.DB) (string, error) {
rows, err := db.Query(`SELECT id, finished_at, candidate_id FROM googlevideo_tune_runs WHERE state = 'PROPOSED' ORDER BY id DESC`)
if err != nil {
return "", err
}
type proposal struct {
id, candidateID int64
finished string
}
var proposals []proposal
for rows.Next() {
var p proposal
if err := rows.Scan(&p.id, &p.finished, &p.candidateID); err != nil {
rows.Close()
return "", err
}
proposals = append(proposals, p)
}
err = rows.Err()
rows.Close() // single-connection store: finish the query before reading candidates.
if err != nil {
return "", err
}
var text strings.Builder
for _, p := range proposals {
c, err := readStrategyCandidate(db, p.candidateID)
if err != nil {
return "", err
}
if !c.Qualified || c.Group != "googlevideo" || len(c.Endpoints) != 1 {
return "", fmt.Errorf("googlevideo candidate has incomplete endpoint/TLS evidence")
}
fmt.Fprintf(&text, "Googlevideo run %d; finished %s; candidate %d; group googlevideo; exact endpoint %s; PROPOSED, NOT ACTIVE\nArgs: %s\nRecorded combined check evidence: TLS1.2 3/3; TLS1.3 3/3. No wildcard or playback evidence.\n", p.id, p.finished, c.ID, c.Endpoints[0], c.Args)
}
return text.String(), nil
}
+285
View File
@@ -0,0 +1,285 @@
package main
import (
"bytes"
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
)
const cacheTestHost = "rr16---sn-n8v7znsz.googlevideo.com"
var cacheTestFlags = []string{"--video-id", "dQw4w9WgXcQ", "--proxy", "socks5://localhost:1080"}
func cachePlayer(t *testing.T, order *[]string) playerRunner {
t.Helper()
return func(_ context.Context, args []string, out, diagnostics io.Writer) error {
*order = append(*order, "discover")
invocation := strings.Join(args, " ")
for _, want := range []string{"--proxy socks5://localhost:1080", "21.02.35", "dQw4w9WgXcQ", playerURL} {
if !strings.Contains(invocation, want) {
t.Fatalf("discovery missing %q: %s", want, invocation)
}
}
fmt.Fprint(diagnostics, "https://secret.invalid/signed?token=SECRET")
_, err := fmt.Fprintf(out, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://%s/videoplayback?token=SECRET"}}`, cacheTestHost)
return err
}
}
func TestTuneGooglevideoPipeline(t *testing.T) {
dir := t.TempDir()
t.Setenv("TMPDIR", dir)
t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(dir, "autotune.db"))
scanReport := filepath.Join(dir, "shortlist.json")
combinedReport := filepath.Join(dir, "combined.json")
argv := filepath.Join(dir, "argv")
if err := os.WriteFile(scanReport, []byte(reportFixture(t)), 0600); err != nil {
t.Fatal(err)
}
binary := filepath.Join(dir, "blockcheckw")
// Local fake executable exercises the real scan/check command construction.
script := `#!/bin/sh
printf '%s\n' "$*" >> "$CACHE_ARGV"
kind="$3"
input=""
output=""
while [ "$#" -gt 0 ]; do
case "$1" in
--from-file) input="$2"; shift ;;
--output) output="$2"; shift ;;
esac
shift
done
if [ "$kind" = scan ]; then
printf 'scan\n' > "$output"
printf 'vanilla\n' > fake_report_vanilla.txt
elif [ "$input" = "${output%.json}.txt" ]; then
cat "$CACHE_COMBINED" > "$output"
else
cat "$CACHE_SHORTLIST" > "$output"
fi
`
if err := os.WriteFile(binary, []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("CACHE_ARGV", argv)
t.Setenv("CACHE_SHORTLIST", scanReport)
t.Setenv("CACHE_COMBINED", combinedReport)
var order []string
var path, combinedArgs string
scan := func(ctx context.Context, bin, host string, out, diagnostics io.Writer) (string, error) {
order = append(order, "scan-shortlist")
if bin != "blockcheckw" || host != cacheTestHost {
t.Fatal("scan scope changed")
}
var err error
path, err = runBlockcheck(ctx, binary, host, out, diagnostics)
return path, err
}
check := func(ctx context.Context, bin, gotPath, args, host string, out, diagnostics io.Writer) error {
order = append(order, "combined")
if bin != "blockcheckw" || gotPath != path || host != cacheTestHost {
t.Fatal("combined scope changed")
}
combinedArgs = args
if err := os.WriteFile(combinedReport, []byte(combinedReportFixture(args)), 0600); err != nil {
return err
}
return runCombinedBlockcheckForHost(ctx, binary, gotPath, args, host, out, diagnostics)
}
var stdout, stderr bytes.Buffer
if code := tuneGooglevideo(cacheTestFlags, &stdout, &stderr, cachePlayer(t, &order), scan, check); code != 0 {
t.Fatalf("code=%d err=%s", code, stderr.String())
}
if strings.Join(order, ",") != "discover,scan-shortlist,combined" {
t.Fatal(order)
}
want := "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=^" + cacheTestHost + " " + combinedArgs + "'\n"
if stdout.String() != want || strings.Contains(stdout.String()+stderr.String(), "SECRET") {
t.Fatalf("unsafe output: %q %q", stdout.String(), stderr.String())
}
commands, err := os.ReadFile(argv)
if err != nil {
t.Fatal(err)
}
lines := strings.Split(strings.TrimSpace(string(commands)), "\n")
if len(lines) != 3 {
t.Fatalf("commands: %s", commands)
}
for i, line := range lines {
if !strings.Contains(line, "--domain "+cacheTestHost+" ") || strings.Contains(line, "youtube.com") {
t.Fatal(line)
}
want := "--take 10 --passes 3"
if i == 0 {
want = "--protocols tls12,tls13"
}
if i == 2 {
want = "--take 1 --passes 3"
}
if !strings.Contains(line, want) {
t.Fatal(line)
}
}
rows, err := os.ReadFile(path + ".combined.txt")
wantRows, rowErr := combinedVanillaRowsForHost(combinedArgs, cacheTestHost)
if err != nil || rowErr != nil || string(rows) != wantRows {
t.Fatalf("combined input: %q %v", rows, err)
}
db, err := openStore(databasePath())
if err != nil {
t.Fatal(err)
}
defer db.Close()
var id int64
if err := db.QueryRow(`SELECT candidate_id FROM googlevideo_tune_runs WHERE state = 'PROPOSED' AND endpoint = ?`, cacheTestHost).Scan(&id); err != nil {
t.Fatal(err)
}
c, err := readStrategyCandidate(db, id)
if err != nil || !c.Qualified || c.Group != "googlevideo" || len(c.Endpoints) != 1 || c.Endpoints[0] != cacheTestHost || c.Args != combinedArgs {
t.Fatalf("candidate: %+v %v", c, err)
}
var evidence, legacy, version int
if err := db.QueryRow(`SELECT count(*) FROM candidate_evidence WHERE candidate_id = ? AND endpoint = ? AND passes_ok = 3 AND passes_total = 3 AND checked_at != '' AND json_valid(raw_measurement)`, id, cacheTestHost).Scan(&evidence); err != nil {
t.Fatal(err)
}
if err := db.QueryRow(`SELECT count(*) FROM tune_runs`).Scan(&legacy); err != nil {
t.Fatal(err)
}
if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
t.Fatal(err)
}
if evidence != 2 || legacy != 0 || version != 3 {
t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version)
}
for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} {
stdout.Reset()
stderr.Reset()
if code := run(command, &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), cacheTestHost) || !strings.Contains(stdout.String(), "googlevideo") || !strings.Contains(stdout.String(), "NOT ACTIVE") {
t.Fatalf("%v: %d %s %s", command, code, stdout.String(), stderr.String())
}
}
}
func TestGooglevideoOutputExactScope(t *testing.T) {
args := "--lua-desync=fake"
report := []byte(combinedReportFixture(args))
got, err := googlevideoOutput(report, args, cacheTestHost)
want := "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=^" + cacheTestHost + " " + args + "'"
if err != nil || got != want {
t.Fatalf("exact host output: %q, %v", got, err)
}
for _, host := range []string{
"^" + cacheTestHost,
cacheTestHost + ",other.googlevideo.com",
"rr,other.googlevideo.com",
"googlevideo.com",
"evilgooglevideo.com",
cacheTestHost + ".evil.test",
"rr..googlevideo.com",
"-rr.googlevideo.com",
strings.ToUpper(cacheTestHost),
} {
if output, err := googlevideoOutput(report, args, host); err == nil || output != "" {
t.Fatalf("unsafe host %q accepted: %q, %v", host, output, err)
}
}
}
func TestTuneGooglevideoFailures(t *testing.T) {
for _, mode := range []string{"missing-flags", "invalid-proxy", "discovery", "incomplete", "failed", "wrong-scope", "wrong-args", "database-open", "database-evidence"} {
t.Run(mode, func(t *testing.T) {
dir := t.TempDir()
t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(dir, "autotune.db"))
flags := cacheTestFlags
if mode == "missing-flags" {
flags = nil
}
if mode == "invalid-proxy" {
flags = []string{"--video-id", "dQw4w9WgXcQ"}
}
if mode == "database-open" {
t.Setenv("ZAPRET_AUTOTUNE_DB", dir)
}
var order []string
player := cachePlayer(t, &order)
if mode == "discovery" {
player = func(context.Context, []string, io.Writer, io.Writer) error {
order = append(order, "discover")
return fmt.Errorf("https://signed.invalid/?SECRET")
}
}
path := filepath.Join(dir, "scan.txt")
scan := func(_ context.Context, _, host string, _, _ io.Writer) (string, error) {
order = append(order, "scan")
if host != cacheTestHost {
t.Fatal(host)
}
if mode == "database-evidence" {
db, err := openStore(databasePath())
if err != nil {
t.Fatal(err)
}
_, err = db.Exec(`CREATE TRIGGER reject_cache_tls13 BEFORE INSERT ON candidate_evidence WHEN NEW.protocol = 'tls13' BEGIN SELECT RAISE(ABORT, 'injected failure'); END`)
db.Close()
if err != nil {
t.Fatal(err)
}
}
return path, os.WriteFile(path+".check.json", []byte(reportFixture(t)), 0600)
}
check := func(_ context.Context, _, _, args, host string, _, _ io.Writer) error {
order = append(order, "combined")
if host != cacheTestHost {
t.Fatal(host)
}
report := combinedReportFixture(args)
switch mode {
case "incomplete":
report = `{"strategies":[` + strings.Split(strings.TrimPrefix(report, `{"strategies":[`), `},{`)[0] + `}]}`
case "failed":
report = strings.Replace(report, `"passes_ok":3`, `"passes_ok":2`, 1)
case "wrong-args":
report = combinedReportFixture("--lua-desync=fake")
case "wrong-scope":
report = strings.ReplaceAll(report, `"protocol":`, `"group":"googlevideo","domains":["other.googlevideo.com"],"protocol":`)
}
return os.WriteFile(path+".combined.json", []byte(report), 0600)
}
var stdout, stderr bytes.Buffer
code := tuneGooglevideo(flags, &stdout, &stderr, player, scan, check)
if code == 0 || stdout.Len() != 0 || strings.Contains(stderr.String(), "SECRET") {
t.Fatalf("failure leaked output: %d %s %s", code, stdout.String(), stderr.String())
}
if mode == "missing-flags" || mode == "invalid-proxy" || mode == "database-open" {
if len(order) != 0 {
t.Fatal(order)
}
return
}
if mode == "discovery" && strings.Join(order, ",") != "discover" {
t.Fatal(order)
}
db, err := openStore(databasePath())
if err != nil {
t.Fatal(err)
}
defer db.Close()
for _, table := range []string{"strategy_candidates", "candidate_evidence"} {
var n int
if err := db.QueryRow("SELECT count(*) FROM " + table).Scan(&n); err != nil || n != 0 {
t.Fatalf("%s: %d %v", table, n, err)
}
}
var failed int
if err := db.QueryRow(`SELECT count(*) FROM googlevideo_tune_runs WHERE state = 'FAILED' AND candidate_id IS NULL`).Scan(&failed); err != nil || failed != 1 {
t.Fatalf("failed journal: %d %v", failed, err)
}
})
}
}
+22 -4
View File
@@ -46,11 +46,29 @@ func history(args []string, stdout, stderr io.Writer) int {
}
func tuneHistory(db *sql.DB, limit int) (string, error) {
rows, err := db.Query(`SELECT id, finished_at, target, state, args, error,
query := `SELECT id, finished_at, target || ' frontend only' AS target, state, args, error,
(SELECT count(*) FROM protocol_results WHERE run_id = tune_runs.id),
(SELECT count(DISTINCT protocol) FROM protocol_results WHERE run_id = tune_runs.id
AND protocol IN ('tls12', 'tls13') AND passes_ok = 3 AND passes_total = 3)
FROM tune_runs ORDER BY id DESC LIMIT ?`, limit)
FROM tune_runs`
hasCache, err := hasGooglevideoRuns(db)
if err != nil {
return "", err
}
if hasCache {
query = `SELECT id, finished_at, target, state, args, error, total, verified FROM (` +
`SELECT id, started_at, finished_at, target || ' frontend only' AS target, state, args, error,
(SELECT count(*) FROM protocol_results WHERE run_id = tune_runs.id) AS total,
(SELECT count(DISTINCT protocol) FROM protocol_results WHERE run_id = tune_runs.id AND protocol IN ('tls12','tls13') AND passes_ok = 3 AND passes_total = 3) AS verified FROM tune_runs
UNION ALL SELECT r.id, r.started_at, r.finished_at, coalesce(r.endpoint, '(discovery unavailable)') || ' googlevideo exact endpoint', r.state, c.args, r.error,
(SELECT count(*) FROM candidate_evidence WHERE candidate_id = c.id),
(SELECT count(*) FROM candidate_evidence WHERE candidate_id = c.id AND endpoint = r.endpoint AND protocol IN ('tls12','tls13') AND passes_ok = 3 AND passes_total = 3 AND success_rate = 1 AND coverage = 1)
FROM googlevideo_tune_runs r LEFT JOIN strategy_candidates c ON c.id = r.candidate_id
) ORDER BY started_at DESC, id DESC LIMIT ?`
} else {
query += ` ORDER BY id DESC LIMIT ?`
}
rows, err := db.Query(query, limit)
if err != nil {
return "", err
}
@@ -72,12 +90,12 @@ func tuneHistory(db *sql.DB, limit int) (string, error) {
if !args.Valid || args.String == "" || runError.Valid {
return "", fmt.Errorf("proposed run %d has malformed outcome", id)
}
fmt.Fprintf(&text, "Run %d; finished %s; %s frontend only; PROPOSED, NOT ACTIVE; Args: %q; recorded TLS1.2 3/3; TLS1.3 3/3\n", id, finished, target, args.String)
fmt.Fprintf(&text, "Run %d; finished %s; %s; PROPOSED, NOT ACTIVE; Args: %q; recorded TLS1.2 3/3; TLS1.3 3/3\n", id, finished, target, args.String)
case "FAILED":
if !runError.Valid || runError.String == "" || args.Valid {
return "", fmt.Errorf("failed run %d has malformed outcome", id)
}
fmt.Fprintf(&text, "Run %d; finished %s; %s frontend only; FAILED; Error: %q\n", id, finished, target, runError.String)
fmt.Fprintf(&text, "Run %d; finished %s; %s; FAILED; Error: %q\n", id, finished, target, runError.String)
default:
return "", fmt.Errorf("run %d has unknown state %q", id, state)
}
+6 -1
View File
@@ -11,11 +11,13 @@ const usage = `Usage: zapret-autotune <command>
Commands:
status Show latest recorded tune run/proposal (not live service health)
history [--limit N] List recorded tune attempts (default 20; proposals not active)
strategies youtube List recorded proposals (not active), newest first
strategies youtube List recorded proposals by group (not active)
discover youtube --video-id ID --proxy socks5://HOST:PORT
Discover cache hostname only (no media-transfer check)
check youtube Refuse checks until probe URLs/video IDs are configured and agreed
tune youtube Test youtube.com frontend only; print options without applying
tune googlevideo --video-id ID --proxy socks5://HOST:PORT
Discover and verify one exact cache host; proposal only
render <json-file> Select normalized offline results and print NFQWS2_OPT
`
@@ -24,6 +26,9 @@ func main() {
}
func run(args []string, stdout, stderr io.Writer) int {
if len(args) >= 2 && args[0] == "tune" && args[1] == "googlevideo" {
return tuneGooglevideo(args[2:], stdout, stderr, runPlayerCurl, runBlockcheck, runCombinedBlockcheckForHost)
}
if len(args) > 0 && args[0] == "history" {
return history(args[1:], stdout, stderr)
}
+13
View File
@@ -227,6 +227,10 @@ PRAGMA user_version = 1;`)
return err
}
}
// Additive v3 cache-run journal: the legacy frontend target constraint stays intact.
if _, err := conn.ExecContext(ctx, googlevideoRunSchema); err != nil {
return err
}
_, err = conn.ExecContext(ctx, "COMMIT")
return err
}
@@ -376,6 +380,15 @@ func strategiesYoutube(stdout, stderr io.Writer) int {
fmt.Fprintln(stderr, "error: strategies youtube database:", err)
return 1
}
cacheText, err := googlevideoStrategies(db)
if err != nil {
fmt.Fprintln(stderr, "error: strategies youtube database:", err)
return 1
}
if cacheText != "" {
text = strings.Replace(text, "Throughput, GGC and production health: NOT CHECKED by this command.", "Frontend-only scope above; GGC evidence below is exact-host TLS/data-transfer only, not playback or production health.", 1)
text += cacheText
}
if _, err := fmt.Fprint(stdout, text); err != nil {
fmt.Fprintln(stderr, "error: cannot write strategies:", err)
return 1
+7 -3
View File
@@ -106,6 +106,10 @@ func combinedBlockcheckOutput(r io.Reader, args string) (string, error) {
}
func selectBlockcheckArgs(r io.Reader, expected string) (string, error) {
return selectBlockcheckArgsForTarget(r, expected, "frontend", "youtube.com")
}
func selectBlockcheckArgsForTarget(r io.Reader, expected, group, host string) (string, error) {
d := json.NewDecoder(r)
var report struct {
Strategies []json.RawMessage `json:"strategies"`
@@ -145,15 +149,15 @@ func selectBlockcheckArgs(r io.Reader, expected string) (string, error) {
return "", fmt.Errorf("result %d: unsupported check protocol %q", i, row.Protocol)
}
if _, exists := fields["group"]; !exists {
row.Group = "frontend"
row.Group = group
}
if _, exists := fields["domains"]; !exists {
row.Domains = []string{"youtube.com"}
row.Domains = []string{host}
}
if err := row.validateMeasurements(); err != nil {
return "", fmt.Errorf("result %d: %w", i, err)
}
if row.Group != "frontend" || len(row.Domains) != 1 || row.Domains[0] != "youtube.com" {
if row.Group != group || len(row.Domains) != 1 || row.Domains[0] != host {
return "", fmt.Errorf("result %d: unexpected target", i)
}
key := protocol + "\x00" + row.Args