From 6220eb0b31a964b6373182248f2059c7d55444a4 Mon Sep 17 00:00:00 2001 From: Zeusina Date: Fri, 9 Oct 2026 23:04:53 +0300 Subject: [PATCH] feat(tune): add proposal-only Googlevideo scans Co-authored-by: GPT-6 --- PROGRESS.md | 16 +- README.md | 39 +++- cmd/zapret-autotune/blockcheck.go | 19 +- cmd/zapret-autotune/discover.go | 20 +- cmd/zapret-autotune/googlevideo.go | 230 +++++++++++++++++++ cmd/zapret-autotune/googlevideo_test.go | 285 ++++++++++++++++++++++++ cmd/zapret-autotune/history.go | 26 ++- cmd/zapret-autotune/main.go | 7 +- cmd/zapret-autotune/store.go | 13 ++ cmd/zapret-autotune/tune.go | 10 +- 10 files changed, 638 insertions(+), 27 deletions(-) create mode 100644 cmd/zapret-autotune/googlevideo.go create mode 100644 cmd/zapret-autotune/googlevideo_test.go diff --git a/PROGRESS.md b/PROGRESS.md index 1e44ed5..2cb8224 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -4,7 +4,9 @@ ## Готово -- CLI на Go: `tune youtube`, `discover youtube`, `status`, `history`, `strategies youtube`, `render`. `check youtube` пока намеренно отказывает; `apply` и `rollback` не реализованы. +- CLI на Go: `tune youtube`, `tune googlevideo --video-id ID --proxy socks5://HOST:PORT`, `discover youtube`, `status`, `history`, `strategies youtube`, `render`. `check youtube` пока намеренно отказывает; `apply` и `rollback` не реализованы. +- Proposal-only GGC pipeline: Android 21.02.35 discovery через явный proxy → scan TLS1.2/1.3 только одного найденного hostname → data-transfer shortlist (`--take 10 --passes 3`) → проверка точной комбинации 3/3 на обоих TLS → один инертный профиль с exact-host фильтром. Ранжирование по blockcheckw, не Selectel. SQLite v3 атомарно сохраняет googlevideo candidate и оба endpoint×TLS evidence; отдельный additive journal сохраняет failed runs. `history` и `strategies youtube` показывают точный endpoint/group; `status` пока относится только к frontend journal. +- Добавлены injectable unit-тесты GGC success/order/scope/filter/persistence, обязательных флагов, discovery failure, неполной/неуспешной combined проверки и fail-closed DB. После исправления exact-host фильтра `go test ./...`, `go build` и `git diff --check` прошли локально. Полный новый `tune googlevideo` ещё не прогонялся удалённо. - `tune youtube` сканирует frontend `youtube.com`, проверяет shortlist через `blockcheckw` и выдаёт один TLS-профиль только после совместной проверки TLS 1.2 и 1.3 по 3/3. Это предложение конфигурации, оно ничего не применяет. - SQLite без CGO: миграции до схемы v3, атомарное сохранение успешных/неуспешных tune-запусков, SHA-256 fingerprint нормализованных аргументов, `status`, read-only `history` и список предложенных стратегий. v3 добавляет неизменяемые определения кандидатов и evidence по endpoint/protocol без active-состояния; старые предложения не переносятся в кандидаты. База и каталоги создаются с ограниченными правами. - `discover youtube --video-id ID --proxy socks5://HOST:PORT` делает запрос к YouTube Android Player API, валидирует `serverAbrStreamingUrl` и выдаёт только hostname из `googlevideo.com`; подписанный URL не выводится. @@ -20,19 +22,19 @@ ## Выполняется сейчас - На удалённом тестовом сервере работает полный `blockcheckw scan` для найденного `rr16---sn-n8v7znsz.googlevideo.com` (контейнер `zapret-autotune-googlevideo-scan`). Baseline для TLS 1.2/1.3 заблокирован; scan TLS 1.2 начал перебор 8 648 стратегий. TLS 1.3 и последующий `check --take 10 --passes 3` ещё не завершены. -- Сам scan запускается вручную; его интеграции в `tune youtube` пока нет. +- Этот scan был запущен вручную и не затрагивается новым кодом. Новый отдельный `tune googlevideo` proposal-only и не меняет frontend-only `tune youtube`; удалённый end-to-end запуск команды ещё не выполнен. ## Не готово -- Автоматический GGC pipeline: discovery → scan/check на найденных cache hostnames → выбор и совместная проверка GGC-профиля. Сейчас реализована только команда discovery; текущий scan выполняется вручную. - Проверка реального YouTube video-transfer/SABR. Прямой GET `serverAbrStreamingUrl` возвращает `sabr.malformed_config`; hostname discovery и `blockcheckw` data-transfer checks не равны проверке воспроизведения видео. -- Ранжирование GGC-кандидатов по `blockcheckw` TLS/data-transfer метрикам и сохранение GGC-результатов в SQLite. +- Перепроверка сохранённых кандидатов как alternatives-first recovery. SQLite хранит evidence, но нет recovery-команды, которая выбирает и повторно проверяет эти сценарии перед новым поиском. - Управляемое применение конфигурации, проверка после применения, rollback, активный профиль и health-check production-сервиса. CLI не перезапускает и не перенастраивает `zapret2-ss`. - Полный сервис из требований: YAML-конфигурация сервисов, endpoint-история/покрытие, daemon/периодический мониторинг, API/метрики и multi-service режим. ## Открытые ограничения и следующие шаги 1. Дождаться завершения GGC scan; выполнить `blockcheckw check` для shortlist до 10 стратегий на TLS-протокол с 3 проходами; выбирать только по проверенным TLS/data-transfer результатам. -2. На основании результатов добавить отдельный GGC-путь в `tune youtube` и проверять точную комбинацию профиля на TLS 1.2/1.3. Не помещать frontend- и GGC-правила в одинаковые фильтры, где одно затенит другое. -3. До реализации `apply` согласовать и проверить safety gates из `Architecture.md`: эквивалентность сетевого пути, совместимость версий, точку подключения конфигурации, безопасную активацию/откат и атомарность частичного результата. -4. Отдельно определить способ реальной video/SABR-проверки и лимиты трафика; не считать Selectel speedtest заменой YouTube playback. +2. После завершения ручного scan выполнить `check --take 10 --passes 3`; для полного end-to-end теста `tune googlevideo` учесть, что команда запускает новый scan, а не переиспользует текущий. +3. Добавить alternatives-first повторную проверку сохранённых кандидатов и затем health-check/monitoring; пока это не меняет конфигурацию. +4. До реализации `apply` согласовать и проверить safety gates из `Architecture.md`: эквивалентность сетевого пути, совместимость версий, точку подключения конфигурации, безопасную активацию/откат и атомарность частичного результата. +5. Отдельно определить способ реальной video/SABR-проверки и лимиты трафика; не считать Selectel speedtest заменой YouTube playback. diff --git a/README.md b/README.md index 5540ba8..a2bb5c9 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,41 @@ Go CLI for scanning and checking YouTube TLS bypass strategies with `blockcheckw`. +## Exact cache-host proposal + +```sh +zapret-autotune tune googlevideo --video-id dQw4w9WgXcQ --proxy socks5://localhost:1080 +``` + +Both flags are required. The validated Android 21.02.35 player request discovers +exactly one current cache hostname through the explicit proxy. `blockcheckw` +then scans only that hostname for TLS1.2/TLS1.3, checks the shortlist with +`--take 10 --passes 3`, ranks fully passing strategies by measured data-transfer +speed/latency, and checks the exact combined arguments 3/3 on both protocols. +The proxy is for discovery; blockcheckw uses its own network path, not that proxy. +No Selectel speedtest is used. This is not a YouTube playback/SABR test. + +Only after successful transactional persistence does this command print one +inert `NFQWS2_OPT` profile filtered by the **exact discovered hostname**, never +the `googlevideo.com` wildcard. Cache hosts rotate; one host is not wildcard +evidence. GGC proposals use `--hostlist-domains=^`: the `^` +prefix disables Zapret2's default subdomain matching, so untested subdomains +are not included. Only validated lowercase `*.googlevideo.com` hostnames are +accepted; commas and a leading `^` in the discovered hostname are rejected. +Schema-v3 candidates use group `googlevideo`, one exact endpoint, and +both endpoint×TLS measurements with raw JSON and timestamps. An additive v3 +cache-run journal records failed attempts without candidates and successful +attempts atomically with their candidate/evidence; legacy frontend runs keep +their existing target constraint. Database failures emit no options. +`history` and `strategies youtube` include the cache group and exact endpoint; +`status` still describes the legacy frontend journal only. + +Signed URLs are not printed. Scanner diagnostics are retained in restricted +artifact files rather than relayed to the terminal. No configuration is applied, +no profile is marked active, and no service is restarted. The staged pipeline is +scan/propose now → eventual install → monitoring/recovery; the latter stages are +**not implemented**. No-argument `tune youtube` remains frontend-only and unchanged. + ## Cache-host discovery ```sh @@ -60,8 +95,8 @@ Each newly successful tune saves its exact verified combined frontend profile fo in the same transaction as the existing run. Qualification requires every declared endpoint to pass exactly 3/3 for both TLS1.2 and TLS1.3, with full success/coverage. Existing v2 history is unchanged and is not backfilled into independent candidates. -Candidates have no active/installed state; apply, monitor and GGC integration remain -unimplemented. +Candidates have no active/installed state; apply and monitor remain unimplemented. +The explicit `tune googlevideo` path adds exact-cache proposals only. Proposed runs store a SHA-256 fingerprint of validated arguments with separator whitespace normalized and argument order preserved; repeated proposals retain separate history rows. Failed runs have no fingerprint. Version 1 history is diff --git a/cmd/zapret-autotune/blockcheck.go b/cmd/zapret-autotune/blockcheck.go index bb6b992..02b7ae4 100644 --- a/cmd/zapret-autotune/blockcheck.go +++ b/cmd/zapret-autotune/blockcheck.go @@ -96,6 +96,13 @@ func blockcheckVanillaReport(dir string) (string, error) { } func combinedVanillaRows(args string) (string, error) { + return combinedVanillaRowsForHost(args, "youtube.com") +} + +func combinedVanillaRowsForHost(args, host string) (string, error) { + if !validDomain(host) { + return "", fmt.Errorf("invalid combined hostname") + } c := candidate{Args: args} if strings.TrimSpace(args) == "" { return "", fmt.Errorf("empty combined args") @@ -103,13 +110,17 @@ func combinedVanillaRows(args string) (string, error) { if err := c.validateArgs(); err != nil { return "", err } - return "curl_test_https_tls12 ipv4 youtube.com : nfqws2 " + args + "\n" + - "curl_test_https_tls13 ipv4 youtube.com : nfqws2 " + args + "\n", nil + return "curl_test_https_tls12 ipv4 " + host + " : nfqws2 " + args + "\n" + + "curl_test_https_tls13 ipv4 " + host + " : nfqws2 " + args + "\n", nil } // Retain the exact combined input and report, including on failure. func runCombinedBlockcheck(ctx context.Context, binary, path, args string, stdout, stderr io.Writer) error { - rows, err := combinedVanillaRows(args) + return runCombinedBlockcheckForHost(ctx, binary, path, args, "youtube.com", stdout, stderr) +} + +func runCombinedBlockcheckForHost(ctx context.Context, binary, path, args, host string, stdout, stderr io.Writer) error { + rows, err := combinedVanillaRowsForHost(args, host) if err != nil { return err } @@ -121,7 +132,7 @@ func runCombinedBlockcheck(ctx context.Context, binary, path, args string, stdou return err } if err := runBlockcheckCommand(ctx, binary, "", []string{"--workers", "32", "check", "--from-file", input, - "--domain", "youtube.com", "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", + "--domain", host, "--take", "1", "--passes", "3", "--auto", "--no-conflict-cleanup", "--output", path + ".combined.json"}, stdout, stderr); err != nil { return fmt.Errorf("combined check (input %s, output %s): %w", input, path+".combined.json", err) } diff --git a/cmd/zapret-autotune/discover.go b/cmd/zapret-autotune/discover.go index 474415d..bc81734 100644 --- a/cmd/zapret-autotune/discover.go +++ b/cmd/zapret-autotune/discover.go @@ -103,17 +103,25 @@ func playerHostname(body []byte) (string, error) { return host, nil } -func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunner) int { +func discoveryFlags(args []string) (string, string, error) { flags := flag.NewFlagSet("discover youtube", flag.ContinueOnError) flags.SetOutput(io.Discard) // flag errors can contain untrusted arguments. videoID := flags.String("video-id", "", "") proxy := flags.String("proxy", "", "") if flags.Parse(args) != nil || flags.NArg() != 0 || !videoIDPattern.MatchString(*videoID) || !validDiscoveryProxy(*proxy) { - fmt.Fprintln(stderr, "error: discover youtube requires --video-id (11 characters) and --proxy socks5://HOST:PORT without credentials") + return "", "", errors.New("requires --video-id (11 characters) and --proxy socks5://HOST:PORT without credentials") + } + return *videoID, *proxy, nil +} + +func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunner) int { + videoID, proxy, err := discoveryFlags(args) + if err != nil { + fmt.Fprintln(stderr, "error: discover youtube", err) return 2 } payload, _ := json.Marshal(map[string]interface{}{ - "videoId": *videoID, + "videoId": videoID, "context": map[string]interface{}{"client": map[string]interface{}{ "clientName": "ANDROID", "clientVersion": "21.02.35", "androidSdkVersion": 30, "userAgent": playerUserAgent, "osName": "Android", "osVersion": "11", @@ -125,8 +133,8 @@ func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunne output := &playerBoundedWriter{dst: &body, limit: playerResponseLimit} diagnostics := &playerBoundedWriter{dst: io.Discard, limit: 64 << 10} // Disable curlrc and NO_PROXY bypasses: the explicit proxy must be used. - err := runner(ctx, []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", - "--noproxy", "", "--proxy", *proxy, "--user-agent", playerUserAgent, "--json", string(payload), playerURL}, output, diagnostics) + err = runner(ctx, []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", + "--noproxy", "", "--proxy", proxy, "--user-agent", playerUserAgent, "--json", string(payload), playerURL}, output, diagnostics) if err != nil || output.exceeded || diagnostics.exceeded { fmt.Fprintln(stderr, "error: player request failed or exceeded output limit") return 1 @@ -139,7 +147,7 @@ func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunne result := struct { VideoID string `json:"videoId"` Hostnames []string `json:"hostnames"` - }{*videoID, []string{host}} + }{videoID, []string{host}} if json.NewEncoder(stdout).Encode(result) != nil { fmt.Fprintln(stderr, "error: cannot write discovery result") return 1 diff --git a/cmd/zapret-autotune/googlevideo.go b/cmd/zapret-autotune/googlevideo.go new file mode 100644 index 0000000..73d0ba8 --- /dev/null +++ b/cmd/zapret-autotune/googlevideo.go @@ -0,0 +1,230 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" + "time" +) + +// Separate journal preserves the legacy frontend-only tune_runs constraint. +// Both journals refer to the same v3 immutable candidate/evidence model. +const googlevideoRunSchema = `CREATE TABLE IF NOT EXISTS googlevideo_tune_runs ( + id INTEGER PRIMARY KEY, + started_at TEXT NOT NULL, + finished_at TEXT NOT NULL, + endpoint TEXT, + state TEXT NOT NULL CHECK (state IN ('PROPOSED', 'FAILED')), + candidate_id INTEGER UNIQUE REFERENCES strategy_candidates(id), + error TEXT, + CHECK ((state = 'PROPOSED' AND endpoint IS NOT NULL AND candidate_id IS NOT NULL AND error IS NULL) + OR (state = 'FAILED' AND candidate_id IS NULL AND error IS NOT NULL AND length(error) > 0)) +);` + +type cacheScan func(context.Context, string, string, io.Writer, io.Writer) (string, error) +type cacheCheck func(context.Context, string, string, string, string, io.Writer, io.Writer) error + +func tuneGooglevideo(flags []string, stdout, stderr io.Writer, discover playerRunner, scan cacheScan, check cacheCheck) int { + if _, _, err := discoveryFlags(flags); err != nil { + fmt.Fprintln(stderr, "error: tune googlevideo", err) + return 2 + } + started := time.Now() + db, err := openStore(databasePath()) + if err != nil { + fmt.Fprintln(stderr, "error: tune googlevideo database:", err) + return 1 + } + defer db.Close() + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + var discovered, diagnostics bytes.Buffer + var host, args, output string + var report []byte + // Reuse the validated Android player request; never relay its body or runner errors. + if discoverYoutube(flags, &discovered, &diagnostics, discover) != 0 { + err = fmt.Errorf("cache discovery failed") + } else { + var result struct { + Hostnames []string `json:"hostnames"` + } + if json.Unmarshal(discovered.Bytes(), &result) != nil || len(result.Hostnames) != 1 { + err = fmt.Errorf("cache discovery requires exactly one hostname") + } else { + host = result.Hostnames[0] + } + } + if err == nil { + fmt.Fprintln(stderr, "Testing exact googlevideo endpoint:", host, "(proposal only; no configuration applied).") + fmt.Fprintln(stderr, "TLS1.2+TLS1.3 scan; real-data shortlist --take 10 --passes 3; exact combined check 3/3 on both protocols. No Selectel ranking or playback claim.") + var raw bytes.Buffer + var path string + path, err = scan(ctx, "blockcheckw", host, &raw, &raw) + // Scanner diagnostics stay in restricted artifacts, never on the output path. + if path != "" { + fmt.Fprintln(stderr, "Blockcheck artifacts:", path) + if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil { + err = e + } + } + if err == nil { + report, err = os.ReadFile(path + ".check.json") + } + if err == nil { + args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", "googlevideo", host) + } + if err == nil { + err = check(ctx, "blockcheckw", path, args, host, &raw, &raw) + if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil { + err = e + } + } + if err == nil { + report, err = os.ReadFile(path + ".combined.json") + } + if err == nil { + output, err = googlevideoOutput(report, args, host) + } + if err == nil { + err = ctx.Err() + } + } + // Do not store or print arbitrary runner errors (they might contain signed URLs). + var safeError error + if err != nil { + safeError = fmt.Errorf("discovery, scan or combined TLS verification failed; no candidate") + } + if saveErr := saveGooglevideoRun(db, started, host, args, report, safeError); saveErr != nil { + // Best effort failed-run recording after the successful transaction rolled back. + _ = saveGooglevideoRun(db, started, host, "", nil, fmt.Errorf("candidate persistence failed")) + fmt.Fprintln(stderr, "error: tune googlevideo database: candidate persistence failed; no options emitted") + return 1 + } + if safeError != nil { + fmt.Fprintln(stderr, "error: tune googlevideo:", safeError) + return 1 + } + if _, err := fmt.Fprintln(stdout, output); err != nil { + fmt.Fprintln(stderr, "error: cannot write googlevideo proposal") + return 1 + } + return 0 +} + +func googlevideoOutput(report []byte, args, host string) (string, error) { + if !discoveryHostname(host) || !strings.HasSuffix(host, ".googlevideo.com") || host != strings.ToLower(host) || args == "" { + return "", fmt.Errorf("invalid exact cache scope or empty args") + } + if _, err := selectBlockcheckArgsForTarget(bytes.NewReader(report), args, "googlevideo", host); err != nil { + return "", err + } + // Bare hostlist entries also match subdomains; evidence covers only this host. + return "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=^" + host + " " + args + "'", nil +} + +func saveGooglevideoRun(db *sql.DB, started time.Time, host, args string, report []byte, runErr error) error { + var evidence struct { + Strategies []json.RawMessage `json:"strategies"` + } + if runErr == nil { + if _, err := googlevideoOutput(report, args, host); err != nil { + return err + } + if err := json.Unmarshal(report, &evidence); err != nil { + return err + } + } + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + checked := time.Now().UTC() + var endpoint, candidateID, savedError interface{} + if host != "" { + endpoint = host + } + state := "FAILED" + if runErr != nil { + savedError = runErr.Error() + } else { + id, err := insertStrategyCandidate(tx, "googlevideo", []string{host}, args, nil) + if err != nil { + return err + } + candidateID, state = id, "PROPOSED" + for _, raw := range evidence.Strategies { + var row struct { + Protocol string `json:"protocol"` + } + if err := json.Unmarshal(raw, &row); err != nil { + return err + } + if err := saveCandidateEvidence(tx, id, host, normalizeProtocol(row.Protocol), raw, checked); err != nil { + return err + } + } + } + if _, err := tx.Exec(`INSERT INTO googlevideo_tune_runs (started_at, finished_at, endpoint, state, candidate_id, error) VALUES (?, ?, ?, ?, ?, ?)`, started.UTC().Format(time.RFC3339Nano), checked.Format(time.RFC3339Nano), endpoint, state, candidateID, savedError); err != nil { + return err + } + return tx.Commit() +} + +func hasGooglevideoRuns(db *sql.DB) (bool, error) { + var exists int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = 'googlevideo_tune_runs'`).Scan(&exists); err != nil { + return false, err + } + if exists == 0 { + return false, nil + } + if err := db.QueryRow(`SELECT count(*) FROM googlevideo_tune_runs`).Scan(&exists); err != nil { + return false, err + } + return exists > 0, nil +} + +func googlevideoStrategies(db *sql.DB) (string, error) { + rows, err := db.Query(`SELECT id, finished_at, candidate_id FROM googlevideo_tune_runs WHERE state = 'PROPOSED' ORDER BY id DESC`) + if err != nil { + return "", err + } + type proposal struct { + id, candidateID int64 + finished string + } + var proposals []proposal + for rows.Next() { + var p proposal + if err := rows.Scan(&p.id, &p.finished, &p.candidateID); err != nil { + rows.Close() + return "", err + } + proposals = append(proposals, p) + } + err = rows.Err() + rows.Close() // single-connection store: finish the query before reading candidates. + if err != nil { + return "", err + } + var text strings.Builder + for _, p := range proposals { + c, err := readStrategyCandidate(db, p.candidateID) + if err != nil { + return "", err + } + if !c.Qualified || c.Group != "googlevideo" || len(c.Endpoints) != 1 { + return "", fmt.Errorf("googlevideo candidate has incomplete endpoint/TLS evidence") + } + fmt.Fprintf(&text, "Googlevideo run %d; finished %s; candidate %d; group googlevideo; exact endpoint %s; PROPOSED, NOT ACTIVE\nArgs: %s\nRecorded combined check evidence: TLS1.2 3/3; TLS1.3 3/3. No wildcard or playback evidence.\n", p.id, p.finished, c.ID, c.Endpoints[0], c.Args) + } + return text.String(), nil +} diff --git a/cmd/zapret-autotune/googlevideo_test.go b/cmd/zapret-autotune/googlevideo_test.go new file mode 100644 index 0000000..7dba0c4 --- /dev/null +++ b/cmd/zapret-autotune/googlevideo_test.go @@ -0,0 +1,285 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +const cacheTestHost = "rr16---sn-n8v7znsz.googlevideo.com" + +var cacheTestFlags = []string{"--video-id", "dQw4w9WgXcQ", "--proxy", "socks5://localhost:1080"} + +func cachePlayer(t *testing.T, order *[]string) playerRunner { + t.Helper() + return func(_ context.Context, args []string, out, diagnostics io.Writer) error { + *order = append(*order, "discover") + invocation := strings.Join(args, " ") + for _, want := range []string{"--proxy socks5://localhost:1080", "21.02.35", "dQw4w9WgXcQ", playerURL} { + if !strings.Contains(invocation, want) { + t.Fatalf("discovery missing %q: %s", want, invocation) + } + } + fmt.Fprint(diagnostics, "https://secret.invalid/signed?token=SECRET") + _, err := fmt.Fprintf(out, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://%s/videoplayback?token=SECRET"}}`, cacheTestHost) + return err + } +} + +func TestTuneGooglevideoPipeline(t *testing.T) { + dir := t.TempDir() + t.Setenv("TMPDIR", dir) + t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(dir, "autotune.db")) + scanReport := filepath.Join(dir, "shortlist.json") + combinedReport := filepath.Join(dir, "combined.json") + argv := filepath.Join(dir, "argv") + if err := os.WriteFile(scanReport, []byte(reportFixture(t)), 0600); err != nil { + t.Fatal(err) + } + binary := filepath.Join(dir, "blockcheckw") + // Local fake executable exercises the real scan/check command construction. + script := `#!/bin/sh +printf '%s\n' "$*" >> "$CACHE_ARGV" +kind="$3" +input="" +output="" +while [ "$#" -gt 0 ]; do + case "$1" in + --from-file) input="$2"; shift ;; + --output) output="$2"; shift ;; + esac + shift +done +if [ "$kind" = scan ]; then + printf 'scan\n' > "$output" + printf 'vanilla\n' > fake_report_vanilla.txt +elif [ "$input" = "${output%.json}.txt" ]; then + cat "$CACHE_COMBINED" > "$output" +else + cat "$CACHE_SHORTLIST" > "$output" +fi +` + if err := os.WriteFile(binary, []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("CACHE_ARGV", argv) + t.Setenv("CACHE_SHORTLIST", scanReport) + t.Setenv("CACHE_COMBINED", combinedReport) + var order []string + var path, combinedArgs string + scan := func(ctx context.Context, bin, host string, out, diagnostics io.Writer) (string, error) { + order = append(order, "scan-shortlist") + if bin != "blockcheckw" || host != cacheTestHost { + t.Fatal("scan scope changed") + } + var err error + path, err = runBlockcheck(ctx, binary, host, out, diagnostics) + return path, err + } + check := func(ctx context.Context, bin, gotPath, args, host string, out, diagnostics io.Writer) error { + order = append(order, "combined") + if bin != "blockcheckw" || gotPath != path || host != cacheTestHost { + t.Fatal("combined scope changed") + } + combinedArgs = args + if err := os.WriteFile(combinedReport, []byte(combinedReportFixture(args)), 0600); err != nil { + return err + } + return runCombinedBlockcheckForHost(ctx, binary, gotPath, args, host, out, diagnostics) + } + var stdout, stderr bytes.Buffer + if code := tuneGooglevideo(cacheTestFlags, &stdout, &stderr, cachePlayer(t, &order), scan, check); code != 0 { + t.Fatalf("code=%d err=%s", code, stderr.String()) + } + if strings.Join(order, ",") != "discover,scan-shortlist,combined" { + t.Fatal(order) + } + want := "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=^" + cacheTestHost + " " + combinedArgs + "'\n" + if stdout.String() != want || strings.Contains(stdout.String()+stderr.String(), "SECRET") { + t.Fatalf("unsafe output: %q %q", stdout.String(), stderr.String()) + } + commands, err := os.ReadFile(argv) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(commands)), "\n") + if len(lines) != 3 { + t.Fatalf("commands: %s", commands) + } + for i, line := range lines { + if !strings.Contains(line, "--domain "+cacheTestHost+" ") || strings.Contains(line, "youtube.com") { + t.Fatal(line) + } + want := "--take 10 --passes 3" + if i == 0 { + want = "--protocols tls12,tls13" + } + if i == 2 { + want = "--take 1 --passes 3" + } + if !strings.Contains(line, want) { + t.Fatal(line) + } + } + rows, err := os.ReadFile(path + ".combined.txt") + wantRows, rowErr := combinedVanillaRowsForHost(combinedArgs, cacheTestHost) + if err != nil || rowErr != nil || string(rows) != wantRows { + t.Fatalf("combined input: %q %v", rows, err) + } + db, err := openStore(databasePath()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int64 + if err := db.QueryRow(`SELECT candidate_id FROM googlevideo_tune_runs WHERE state = 'PROPOSED' AND endpoint = ?`, cacheTestHost).Scan(&id); err != nil { + t.Fatal(err) + } + c, err := readStrategyCandidate(db, id) + if err != nil || !c.Qualified || c.Group != "googlevideo" || len(c.Endpoints) != 1 || c.Endpoints[0] != cacheTestHost || c.Args != combinedArgs { + t.Fatalf("candidate: %+v %v", c, err) + } + var evidence, legacy, version int + if err := db.QueryRow(`SELECT count(*) FROM candidate_evidence WHERE candidate_id = ? AND endpoint = ? AND passes_ok = 3 AND passes_total = 3 AND checked_at != '' AND json_valid(raw_measurement)`, id, cacheTestHost).Scan(&evidence); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`SELECT count(*) FROM tune_runs`).Scan(&legacy); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil { + t.Fatal(err) + } + if evidence != 2 || legacy != 0 || version != 3 { + t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version) + } + for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} { + stdout.Reset() + stderr.Reset() + if code := run(command, &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), cacheTestHost) || !strings.Contains(stdout.String(), "googlevideo") || !strings.Contains(stdout.String(), "NOT ACTIVE") { + t.Fatalf("%v: %d %s %s", command, code, stdout.String(), stderr.String()) + } + } +} + +func TestGooglevideoOutputExactScope(t *testing.T) { + args := "--lua-desync=fake" + report := []byte(combinedReportFixture(args)) + got, err := googlevideoOutput(report, args, cacheTestHost) + want := "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=^" + cacheTestHost + " " + args + "'" + if err != nil || got != want { + t.Fatalf("exact host output: %q, %v", got, err) + } + for _, host := range []string{ + "^" + cacheTestHost, + cacheTestHost + ",other.googlevideo.com", + "rr,other.googlevideo.com", + "googlevideo.com", + "evilgooglevideo.com", + cacheTestHost + ".evil.test", + "rr..googlevideo.com", + "-rr.googlevideo.com", + strings.ToUpper(cacheTestHost), + } { + if output, err := googlevideoOutput(report, args, host); err == nil || output != "" { + t.Fatalf("unsafe host %q accepted: %q, %v", host, output, err) + } + } +} + +func TestTuneGooglevideoFailures(t *testing.T) { + for _, mode := range []string{"missing-flags", "invalid-proxy", "discovery", "incomplete", "failed", "wrong-scope", "wrong-args", "database-open", "database-evidence"} { + t.Run(mode, func(t *testing.T) { + dir := t.TempDir() + t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(dir, "autotune.db")) + flags := cacheTestFlags + if mode == "missing-flags" { + flags = nil + } + if mode == "invalid-proxy" { + flags = []string{"--video-id", "dQw4w9WgXcQ"} + } + if mode == "database-open" { + t.Setenv("ZAPRET_AUTOTUNE_DB", dir) + } + var order []string + player := cachePlayer(t, &order) + if mode == "discovery" { + player = func(context.Context, []string, io.Writer, io.Writer) error { + order = append(order, "discover") + return fmt.Errorf("https://signed.invalid/?SECRET") + } + } + path := filepath.Join(dir, "scan.txt") + scan := func(_ context.Context, _, host string, _, _ io.Writer) (string, error) { + order = append(order, "scan") + if host != cacheTestHost { + t.Fatal(host) + } + if mode == "database-evidence" { + db, err := openStore(databasePath()) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(`CREATE TRIGGER reject_cache_tls13 BEFORE INSERT ON candidate_evidence WHEN NEW.protocol = 'tls13' BEGIN SELECT RAISE(ABORT, 'injected failure'); END`) + db.Close() + if err != nil { + t.Fatal(err) + } + } + return path, os.WriteFile(path+".check.json", []byte(reportFixture(t)), 0600) + } + check := func(_ context.Context, _, _, args, host string, _, _ io.Writer) error { + order = append(order, "combined") + if host != cacheTestHost { + t.Fatal(host) + } + report := combinedReportFixture(args) + switch mode { + case "incomplete": + report = `{"strategies":[` + strings.Split(strings.TrimPrefix(report, `{"strategies":[`), `},{`)[0] + `}]}` + case "failed": + report = strings.Replace(report, `"passes_ok":3`, `"passes_ok":2`, 1) + case "wrong-args": + report = combinedReportFixture("--lua-desync=fake") + case "wrong-scope": + report = strings.ReplaceAll(report, `"protocol":`, `"group":"googlevideo","domains":["other.googlevideo.com"],"protocol":`) + } + return os.WriteFile(path+".combined.json", []byte(report), 0600) + } + var stdout, stderr bytes.Buffer + code := tuneGooglevideo(flags, &stdout, &stderr, player, scan, check) + if code == 0 || stdout.Len() != 0 || strings.Contains(stderr.String(), "SECRET") { + t.Fatalf("failure leaked output: %d %s %s", code, stdout.String(), stderr.String()) + } + if mode == "missing-flags" || mode == "invalid-proxy" || mode == "database-open" { + if len(order) != 0 { + t.Fatal(order) + } + return + } + if mode == "discovery" && strings.Join(order, ",") != "discover" { + t.Fatal(order) + } + db, err := openStore(databasePath()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, table := range []string{"strategy_candidates", "candidate_evidence"} { + var n int + if err := db.QueryRow("SELECT count(*) FROM " + table).Scan(&n); err != nil || n != 0 { + t.Fatalf("%s: %d %v", table, n, err) + } + } + var failed int + if err := db.QueryRow(`SELECT count(*) FROM googlevideo_tune_runs WHERE state = 'FAILED' AND candidate_id IS NULL`).Scan(&failed); err != nil || failed != 1 { + t.Fatalf("failed journal: %d %v", failed, err) + } + }) + } +} diff --git a/cmd/zapret-autotune/history.go b/cmd/zapret-autotune/history.go index ad282f7..2ccdd15 100644 --- a/cmd/zapret-autotune/history.go +++ b/cmd/zapret-autotune/history.go @@ -46,11 +46,29 @@ func history(args []string, stdout, stderr io.Writer) int { } func tuneHistory(db *sql.DB, limit int) (string, error) { - rows, err := db.Query(`SELECT id, finished_at, target, state, args, error, + query := `SELECT id, finished_at, target || ' frontend only' AS target, state, args, error, (SELECT count(*) FROM protocol_results WHERE run_id = tune_runs.id), (SELECT count(DISTINCT protocol) FROM protocol_results WHERE run_id = tune_runs.id AND protocol IN ('tls12', 'tls13') AND passes_ok = 3 AND passes_total = 3) - FROM tune_runs ORDER BY id DESC LIMIT ?`, limit) + FROM tune_runs` + hasCache, err := hasGooglevideoRuns(db) + if err != nil { + return "", err + } + if hasCache { + query = `SELECT id, finished_at, target, state, args, error, total, verified FROM (` + + `SELECT id, started_at, finished_at, target || ' frontend only' AS target, state, args, error, + (SELECT count(*) FROM protocol_results WHERE run_id = tune_runs.id) AS total, + (SELECT count(DISTINCT protocol) FROM protocol_results WHERE run_id = tune_runs.id AND protocol IN ('tls12','tls13') AND passes_ok = 3 AND passes_total = 3) AS verified FROM tune_runs + UNION ALL SELECT r.id, r.started_at, r.finished_at, coalesce(r.endpoint, '(discovery unavailable)') || ' googlevideo exact endpoint', r.state, c.args, r.error, + (SELECT count(*) FROM candidate_evidence WHERE candidate_id = c.id), + (SELECT count(*) FROM candidate_evidence WHERE candidate_id = c.id AND endpoint = r.endpoint AND protocol IN ('tls12','tls13') AND passes_ok = 3 AND passes_total = 3 AND success_rate = 1 AND coverage = 1) + FROM googlevideo_tune_runs r LEFT JOIN strategy_candidates c ON c.id = r.candidate_id + ) ORDER BY started_at DESC, id DESC LIMIT ?` + } else { + query += ` ORDER BY id DESC LIMIT ?` + } + rows, err := db.Query(query, limit) if err != nil { return "", err } @@ -72,12 +90,12 @@ func tuneHistory(db *sql.DB, limit int) (string, error) { if !args.Valid || args.String == "" || runError.Valid { return "", fmt.Errorf("proposed run %d has malformed outcome", id) } - fmt.Fprintf(&text, "Run %d; finished %s; %s frontend only; PROPOSED, NOT ACTIVE; Args: %q; recorded TLS1.2 3/3; TLS1.3 3/3\n", id, finished, target, args.String) + fmt.Fprintf(&text, "Run %d; finished %s; %s; PROPOSED, NOT ACTIVE; Args: %q; recorded TLS1.2 3/3; TLS1.3 3/3\n", id, finished, target, args.String) case "FAILED": if !runError.Valid || runError.String == "" || args.Valid { return "", fmt.Errorf("failed run %d has malformed outcome", id) } - fmt.Fprintf(&text, "Run %d; finished %s; %s frontend only; FAILED; Error: %q\n", id, finished, target, runError.String) + fmt.Fprintf(&text, "Run %d; finished %s; %s; FAILED; Error: %q\n", id, finished, target, runError.String) default: return "", fmt.Errorf("run %d has unknown state %q", id, state) } diff --git a/cmd/zapret-autotune/main.go b/cmd/zapret-autotune/main.go index 16aaa38..aca492f 100644 --- a/cmd/zapret-autotune/main.go +++ b/cmd/zapret-autotune/main.go @@ -11,11 +11,13 @@ const usage = `Usage: zapret-autotune Commands: status Show latest recorded tune run/proposal (not live service health) history [--limit N] List recorded tune attempts (default 20; proposals not active) - strategies youtube List recorded proposals (not active), newest first + strategies youtube List recorded proposals by group (not active) discover youtube --video-id ID --proxy socks5://HOST:PORT Discover cache hostname only (no media-transfer check) check youtube Refuse checks until probe URLs/video IDs are configured and agreed tune youtube Test youtube.com frontend only; print options without applying + tune googlevideo --video-id ID --proxy socks5://HOST:PORT + Discover and verify one exact cache host; proposal only render Select normalized offline results and print NFQWS2_OPT ` @@ -24,6 +26,9 @@ func main() { } func run(args []string, stdout, stderr io.Writer) int { + if len(args) >= 2 && args[0] == "tune" && args[1] == "googlevideo" { + return tuneGooglevideo(args[2:], stdout, stderr, runPlayerCurl, runBlockcheck, runCombinedBlockcheckForHost) + } if len(args) > 0 && args[0] == "history" { return history(args[1:], stdout, stderr) } diff --git a/cmd/zapret-autotune/store.go b/cmd/zapret-autotune/store.go index 70283f7..cdb95c1 100644 --- a/cmd/zapret-autotune/store.go +++ b/cmd/zapret-autotune/store.go @@ -227,6 +227,10 @@ PRAGMA user_version = 1;`) return err } } + // Additive v3 cache-run journal: the legacy frontend target constraint stays intact. + if _, err := conn.ExecContext(ctx, googlevideoRunSchema); err != nil { + return err + } _, err = conn.ExecContext(ctx, "COMMIT") return err } @@ -376,6 +380,15 @@ func strategiesYoutube(stdout, stderr io.Writer) int { fmt.Fprintln(stderr, "error: strategies youtube database:", err) return 1 } + cacheText, err := googlevideoStrategies(db) + if err != nil { + fmt.Fprintln(stderr, "error: strategies youtube database:", err) + return 1 + } + if cacheText != "" { + text = strings.Replace(text, "Throughput, GGC and production health: NOT CHECKED by this command.", "Frontend-only scope above; GGC evidence below is exact-host TLS/data-transfer only, not playback or production health.", 1) + text += cacheText + } if _, err := fmt.Fprint(stdout, text); err != nil { fmt.Fprintln(stderr, "error: cannot write strategies:", err) return 1 diff --git a/cmd/zapret-autotune/tune.go b/cmd/zapret-autotune/tune.go index 33c285a..77cc069 100644 --- a/cmd/zapret-autotune/tune.go +++ b/cmd/zapret-autotune/tune.go @@ -106,6 +106,10 @@ func combinedBlockcheckOutput(r io.Reader, args string) (string, error) { } func selectBlockcheckArgs(r io.Reader, expected string) (string, error) { + return selectBlockcheckArgsForTarget(r, expected, "frontend", "youtube.com") +} + +func selectBlockcheckArgsForTarget(r io.Reader, expected, group, host string) (string, error) { d := json.NewDecoder(r) var report struct { Strategies []json.RawMessage `json:"strategies"` @@ -145,15 +149,15 @@ func selectBlockcheckArgs(r io.Reader, expected string) (string, error) { return "", fmt.Errorf("result %d: unsupported check protocol %q", i, row.Protocol) } if _, exists := fields["group"]; !exists { - row.Group = "frontend" + row.Group = group } if _, exists := fields["domains"]; !exists { - row.Domains = []string{"youtube.com"} + row.Domains = []string{host} } if err := row.validateMeasurements(); err != nil { return "", fmt.Errorf("result %d: %w", i, err) } - if row.Group != "frontend" || len(row.Domains) != 1 || row.Domains[0] != "youtube.com" { + if row.Group != group || len(row.Domains) != 1 || row.Domains[0] != host { return "", fmt.Errorf("result %d: unexpected target", i) } key := protocol + "\x00" + row.Args