From aaa8f751c6523738de474d9de08405317f3fc0e6 Mon Sep 17 00:00:00 2001 From: Zeusina Date: Fri, 9 Oct 2026 23:10:25 +0300 Subject: [PATCH] feat(recovery): retry saved Googlevideo scenarios first Co-authored-by: GPT-6 --- PROGRESS.md | 8 +- README.md | 22 +++- cmd/zapret-autotune/googlevideo.go | 20 ++- cmd/zapret-autotune/main.go | 5 + cmd/zapret-autotune/recover.go | 80 ++++++++++++ cmd/zapret-autotune/recover_test.go | 193 ++++++++++++++++++++++++++++ cmd/zapret-autotune/tune.go | 24 +++- 7 files changed, 345 insertions(+), 7 deletions(-) create mode 100644 cmd/zapret-autotune/recover.go create mode 100644 cmd/zapret-autotune/recover_test.go diff --git a/PROGRESS.md b/PROGRESS.md index 2cb8224..eb76225 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -4,7 +4,10 @@ ## Готово -- CLI на Go: `tune youtube`, `tune googlevideo --video-id ID --proxy socks5://HOST:PORT`, `discover youtube`, `status`, `history`, `strategies youtube`, `render`. `check youtube` пока намеренно отказывает; `apply` и `rollback` не реализованы. +- Добавлена proposal-only команда `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`: discovery одного свежего exact host → qualified schema-v3 сценарии newest-first → combined TLS1.2/1.3 3/3 точных сохранённых args. Первый успешный сценарий сохраняется новым immutable candidate/run и выдаётся как NOT ACTIVE без scan. Только отсутствие eligible candidates или валидные non-pass результаты разрешают обычный scan на уже найденном host; infrastructure/malformed-report ошибки fail closed. Apply/monitor и удалённый scan не затронуты. +- Unit-тесты recovery порядка, fresh-host scope, short-circuit, fallback и fail-closed ошибок прошли; локальные `go test ./...`, `go build` и `git diff --check` успешны. + +- CLI на Go: `tune youtube`, `tune googlevideo --video-id ID --proxy socks5://HOST:PORT`, `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`, `discover youtube`, `status`, `history`, `strategies youtube`, `render`. `check youtube` пока намеренно отказывает; `apply`, `rollback` и мониторинг не реализованы. - Proposal-only GGC pipeline: Android 21.02.35 discovery через явный proxy → scan TLS1.2/1.3 только одного найденного hostname → data-transfer shortlist (`--take 10 --passes 3`) → проверка точной комбинации 3/3 на обоих TLS → один инертный профиль с exact-host фильтром. Ранжирование по blockcheckw, не Selectel. SQLite v3 атомарно сохраняет googlevideo candidate и оба endpoint×TLS evidence; отдельный additive journal сохраняет failed runs. `history` и `strategies youtube` показывают точный endpoint/group; `status` пока относится только к frontend journal. - Добавлены injectable unit-тесты GGC success/order/scope/filter/persistence, обязательных флагов, discovery failure, неполной/неуспешной combined проверки и fail-closed DB. После исправления exact-host фильтра `go test ./...`, `go build` и `git diff --check` прошли локально. Полный новый `tune googlevideo` ещё не прогонялся удалённо. - `tune youtube` сканирует frontend `youtube.com`, проверяет shortlist через `blockcheckw` и выдаёт один TLS-профиль только после совместной проверки TLS 1.2 и 1.3 по 3/3. Это предложение конфигурации, оно ничего не применяет. @@ -27,7 +30,6 @@ ## Не готово - Проверка реального YouTube video-transfer/SABR. Прямой GET `serverAbrStreamingUrl` возвращает `sabr.malformed_config`; hostname discovery и `blockcheckw` data-transfer checks не равны проверке воспроизведения видео. -- Перепроверка сохранённых кандидатов как alternatives-first recovery. SQLite хранит evidence, но нет recovery-команды, которая выбирает и повторно проверяет эти сценарии перед новым поиском. - Управляемое применение конфигурации, проверка после применения, rollback, активный профиль и health-check production-сервиса. CLI не перезапускает и не перенастраивает `zapret2-ss`. - Полный сервис из требований: YAML-конфигурация сервисов, endpoint-история/покрытие, daemon/периодический мониторинг, API/метрики и multi-service режим. @@ -35,6 +37,6 @@ 1. Дождаться завершения GGC scan; выполнить `blockcheckw check` для shortlist до 10 стратегий на TLS-протокол с 3 проходами; выбирать только по проверенным TLS/data-transfer результатам. 2. После завершения ручного scan выполнить `check --take 10 --passes 3`; для полного end-to-end теста `tune googlevideo` учесть, что команда запускает новый scan, а не переиспользует текущий. -3. Добавить alternatives-first повторную проверку сохранённых кандидатов и затем health-check/monitoring; пока это не меняет конфигурацию. +3. Проверить новую proposal-only recovery-команду; затем добавить health-check/monitoring с согласованными safety gates. Recovery пока не меняет конфигурацию. 4. До реализации `apply` согласовать и проверить safety gates из `Architecture.md`: эквивалентность сетевого пути, совместимость версий, точку подключения конфигурации, безопасную активацию/откат и атомарность частичного результата. 5. Отдельно определить способ реальной video/SABR-проверки и лимиты трафика; не считать Selectel speedtest заменой YouTube playback. diff --git a/README.md b/README.md index a2bb5c9..320087e 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,26 @@ their existing target constraint. Database failures emit no options. Signed URLs are not printed. Scanner diagnostics are retained in restricted artifact files rather than relayed to the terminal. No configuration is applied, no profile is marked active, and no service is restarted. The staged pipeline is -scan/propose now → eventual install → monitoring/recovery; the latter stages are -**not implemented**. No-argument `tune youtube` remains frontend-only and unchanged. +scan/propose now → eventual install → monitoring; install and monitoring are +**not implemented**. Manual proposal-only recovery is available below. +No-argument `tune youtube` remains frontend-only and unchanged. + +## Alternatives-first recovery (proposal only) + +```sh +zapret-autotune recover googlevideo --video-id dQw4w9WgXcQ --proxy socks5://localhost:1080 +``` + +Recovery discovers the current exact cache host once, then rechecks qualified +schema-v3 `googlevideo` candidates newest-first using their exact saved arguments. +Qualification covers every saved endpoint × TLS protocol; revalidation checks +the fresh host for combined TLS1.2+TLS1.3 3/3. The first passing scenario is saved +as a new immutable candidate/run scoped to that host and emitted as an exact-host +proposal, **not active**. No scan runs after a passing saved scenario. +Only no eligible candidates or complete, valid non-passing evidence permit the +normal tune scan on the already-discovered host. Database, discovery, cancellation, +runner and malformed/incomplete-report errors fail closed without a fallback scan. +This command does not apply configuration, restart services or start monitoring. ## Cache-host discovery diff --git a/cmd/zapret-autotune/googlevideo.go b/cmd/zapret-autotune/googlevideo.go index 73d0ba8..f09d1cb 100644 --- a/cmd/zapret-autotune/googlevideo.go +++ b/cmd/zapret-autotune/googlevideo.go @@ -32,6 +32,14 @@ type cacheScan func(context.Context, string, string, io.Writer, io.Writer) (stri type cacheCheck func(context.Context, string, string, string, string, io.Writer, io.Writer) error func tuneGooglevideo(flags []string, stdout, stderr io.Writer, discover playerRunner, scan cacheScan, check cacheCheck) int { + return googlevideoProposal(flags, stdout, stderr, discover, scan, check, false) +} + +func recoverGooglevideo(flags []string, stdout, stderr io.Writer, discover playerRunner, scan cacheScan, check cacheCheck) int { + return googlevideoProposal(flags, stdout, stderr, discover, scan, check, true) +} + +func googlevideoProposal(flags []string, stdout, stderr io.Writer, discover playerRunner, scan cacheScan, check cacheCheck, recoverSaved bool) int { if _, _, err := discoveryFlags(flags); err != nil { fmt.Fprintln(stderr, "error: tune googlevideo", err) return 2 @@ -61,7 +69,14 @@ func tuneGooglevideo(flags []string, stdout, stderr io.Writer, discover playerRu host = result.Hostnames[0] } } - if err == nil { + if err == nil && recoverSaved { + fmt.Fprintln(stderr, "Revalidating qualified saved googlevideo scenarios newest-first; PROPOSED, NOT ACTIVE; no configuration applied.") + args, report, err = revalidateGooglevideo(ctx, db, host, stderr, check) + if err == nil && args != "" { + output, err = googlevideoOutput(report, args, host) + } + } + if err == nil && args == "" { fmt.Fprintln(stderr, "Testing exact googlevideo endpoint:", host, "(proposal only; no configuration applied).") fmt.Fprintln(stderr, "TLS1.2+TLS1.3 scan; real-data shortlist --take 10 --passes 3; exact combined check 3/3 on both protocols. No Selectel ranking or playback claim.") var raw bytes.Buffer @@ -96,6 +111,9 @@ func tuneGooglevideo(flags []string, stdout, stderr io.Writer, discover playerRu err = ctx.Err() } } + if err == nil { + err = ctx.Err() + } // Do not store or print arbitrary runner errors (they might contain signed URLs). var safeError error if err != nil { diff --git a/cmd/zapret-autotune/main.go b/cmd/zapret-autotune/main.go index aca492f..6ed27b3 100644 --- a/cmd/zapret-autotune/main.go +++ b/cmd/zapret-autotune/main.go @@ -18,6 +18,8 @@ Commands: tune youtube Test youtube.com frontend only; print options without applying tune googlevideo --video-id ID --proxy socks5://HOST:PORT Discover and verify one exact cache host; proposal only + recover googlevideo --video-id ID --proxy socks5://HOST:PORT + Recheck saved scenarios before scanning; proposal, not active render Select normalized offline results and print NFQWS2_OPT ` @@ -26,6 +28,9 @@ func main() { } func run(args []string, stdout, stderr io.Writer) int { + if len(args) >= 2 && args[0] == "recover" && args[1] == "googlevideo" { + return recoverGooglevideo(args[2:], stdout, stderr, runPlayerCurl, runBlockcheck, runCombinedBlockcheckForHost) + } if len(args) >= 2 && args[0] == "tune" && args[1] == "googlevideo" { return tuneGooglevideo(args[2:], stdout, stderr, runPlayerCurl, runBlockcheck, runCombinedBlockcheckForHost) } diff --git a/cmd/zapret-autotune/recover.go b/cmd/zapret-autotune/recover.go new file mode 100644 index 0000000..909c2dd --- /dev/null +++ b/cmd/zapret-autotune/recover.go @@ -0,0 +1,80 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" +) + +// Only complete, well-formed non-passing measurements permit scan fallback. +var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols") + +func revalidateGooglevideo(ctx context.Context, db *sql.DB, host string, stderr io.Writer, check cacheCheck) (string, []byte, error) { + rows, err := db.Query(`SELECT id FROM strategy_candidates WHERE strategy_group = 'googlevideo' ORDER BY id DESC`) + if err != nil { + return "", nil, err + } + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + rows.Close() + return "", nil, err + } + ids = append(ids, id) + } + err = rows.Err() + closeErr := rows.Close() // Release the store's single connection before candidate reads. + if err == nil { + err = closeErr + } + if err != nil { + return "", nil, err + } + for _, id := range ids { + if err := ctx.Err(); err != nil { + return "", nil, err + } + c, err := readStrategyCandidate(db, id) + if err != nil { + return "", nil, err + } + if !c.Qualified { + continue + } + artifact, err := os.CreateTemp("", "blockcheckw-recover-*.txt") + if err != nil { + return "", nil, err + } + path := artifact.Name() + if err := artifact.Close(); err != nil { + return "", nil, err + } + fmt.Fprintln(stderr, "Rechecking saved candidate:", id, "exact endpoint:", host, "Blockcheck artifacts:", path) + var raw bytes.Buffer + err = check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw) + if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil { + err = e + } + if err != nil { + return "", nil, err + } + if err := ctx.Err(); err != nil { + return "", nil, err + } + report, err := os.ReadFile(path + ".combined.json") + if err != nil { + return "", nil, err + } + _, err = selectBlockcheckEvidence(bytes.NewReader(report), c.Args, "googlevideo", host, true) + if errors.Is(err, errCandidateNonpass) { + continue + } + return c.Args, report, err + } + return "", nil, ctx.Err() +} diff --git a/cmd/zapret-autotune/recover_test.go b/cmd/zapret-autotune/recover_test.go new file mode 100644 index 0000000..b5ecbbc --- /dev/null +++ b/cmd/zapret-autotune/recover_test.go @@ -0,0 +1,193 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestRecoverGooglevideo(t *testing.T) { + for _, mode := range []string{"newest-pass", "older-pass", "no-candidates", "unqualified", "all-failed", "runner-error", "cancelled", "malformed", "missing-report", "incomplete", "wrong-host", "wrong-args", "incomplete-passes", "discovery-error", "database-open", "database-read", "database-save"} { + t.Run(mode, func(t *testing.T) { + dir := t.TempDir() + t.Setenv("TMPDIR", dir) + t.Setenv("ZAPRET_AUTOTUNE_DB", filepath.Join(dir, "autotune.db")) + config := filepath.Join(dir, "active.conf") + if err := os.WriteFile(config, []byte("ACTIVE UNCHANGED"), 0600); err != nil { + t.Fatal(err) + } + db, err := openStore(databasePath()) + if err != nil { + t.Fatal(err) + } + oldArgs, newArgs := "--lua-desync=fake", "--lua-desync=multisplit" + seeded := 0 + if mode != "no-candidates" && mode != "unqualified" { + for _, args := range []string{oldArgs, newArgs} { + if err := saveGooglevideoRun(db, time.Now(), "old.googlevideo.com", args, []byte(combinedReportFixture(args)), nil); err != nil { + t.Fatal(err) + } + seeded++ + } + } + if mode != "no-candidates" { + // Newest incomplete endpoint × TLS evidence must never be eligible. + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + if _, err := insertStrategyCandidate(tx, "googlevideo", []string{"old.googlevideo.com", "other.googlevideo.com"}, "--lua-desync=disorder", nil); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + } + if mode == "database-read" { + if _, err := db.Exec(`DROP TABLE candidate_evidence`); err != nil { + t.Fatal(err) + } + } + if mode == "database-save" { + if _, err := db.Exec(`CREATE TRIGGER reject_recovery BEFORE INSERT ON candidate_evidence BEGIN SELECT RAISE(ABORT, 'injected failure'); END`); err != nil { + t.Fatal(err) + } + } + db.Close() + if mode == "database-open" { + t.Setenv("ZAPRET_AUTOTUNE_DB", dir) + } + var order, checked []string + player := cachePlayer(t, &order) + if mode == "discovery-error" { + player = func(context.Context, []string, io.Writer, io.Writer) error { + order = append(order, "discover") + return fmt.Errorf("SECRET discovery failure") + } + } + scans := 0 + scan := func(_ context.Context, _, host string, _, _ io.Writer) (string, error) { + scans++ + order = append(order, "scan") + if host != cacheTestHost { + t.Fatal(host) + } + path := filepath.Join(dir, "scan") + return path, os.WriteFile(path+".check.json", []byte(reportFixture(t)), 0600) + } + check := func(_ context.Context, binary, path, args, host string, _, _ io.Writer) error { + if binary != "blockcheckw" || host != cacheTestHost { + t.Fatalf("wrong fresh scope: %s %s", binary, host) + } + order = append(order, "check") + if scans == 0 { + checked = append(checked, args) + } + report := combinedReportFixture(args) + if scans == 0 { + switch mode { + case "runner-error": + return fmt.Errorf("SECRET runner failure") + case "cancelled": + return context.Canceled + case "missing-report": + return nil + case "malformed": + report = `{"strategies":null}` + case "incomplete": + report = strings.Replace(report, "HTTPS/TLS1.3", "HTTPS/TLS1.2", 1) + case "wrong-host": + report = strings.ReplaceAll(report, `"protocol":`, `"domains":["old.googlevideo.com"],"protocol":`) + case "wrong-args": + report = combinedReportFixture(oldArgs) + case "incomplete-passes": + report = strings.ReplaceAll(report, `"passes_total":3`, `"passes_total":4`) + case "older-pass", "all-failed": + if mode == "all-failed" || args == newArgs { + report = strings.Replace(report, `"passes_ok":3`, `"passes_ok":2`, 1) + } + } + } + return os.WriteFile(path+".combined.json", []byte(report), 0600) + } + var stdout, stderr bytes.Buffer + code := recoverGooglevideo(cacheTestFlags, &stdout, &stderr, player, scan, check) + fallback := mode == "no-candidates" || mode == "unqualified" || mode == "all-failed" + success := fallback || mode == "newest-pass" || mode == "older-pass" + if (code == 0) != success || strings.Contains(stdout.String()+stderr.String(), "SECRET") { + t.Fatalf("code %d output %s diagnostics %s", code, stdout.String(), stderr.String()) + } + wantScans := 0 + if fallback { + wantScans = 1 + } + if scans != wantScans { + t.Fatalf("scans %d want %d", scans, wantScans) + } + if mode != "database-open" && (len(order) == 0 || order[0] != "discover" || strings.Count(strings.Join(order, ","), "discover") != 1) { + t.Fatal(order) + } + if len(checked) > 0 && checked[0] != newArgs { + t.Fatal(checked) + } + if mode == "older-pass" || mode == "all-failed" { + if strings.Join(checked, ",") != newArgs+","+oldArgs { + t.Fatal(checked) + } + } else if mode == "newest-pass" && len(checked) != 1 { + t.Fatal(checked) + } + if !success && stdout.Len() != 0 { + t.Fatal("failed recovery emitted proposal") + } + if success && (!strings.Contains(stdout.String(), "--hostlist-domains=^"+cacheTestHost+" ") || !strings.Contains(stderr.String(), "NOT ACTIVE")) { + t.Fatal(stdout.String(), stderr.String()) + } + active, err := os.ReadFile(config) + if err != nil || string(active) != "ACTIVE UNCHANGED" { + t.Fatalf("active config changed: %s %v", active, err) + } + if mode == "database-open" || mode == "database-read" { + return + } + db, err = openStore(databasePath()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var legacy, proposals int + if err := db.QueryRow(`SELECT count(*) FROM tune_runs`).Scan(&legacy); err != nil || legacy != 0 { + t.Fatalf("frontend changed: %d %v", legacy, err) + } + if err := db.QueryRow(`SELECT count(*) FROM googlevideo_tune_runs WHERE state = 'PROPOSED'`).Scan(&proposals); err != nil { + t.Fatal(err) + } + want := seeded + if success { + want++ + } + if proposals != want { + t.Fatalf("proposals %d want %d", proposals, want) + } + if success { + var id int64 + if err := db.QueryRow(`SELECT candidate_id FROM googlevideo_tune_runs ORDER BY id DESC LIMIT 1`).Scan(&id); err != nil { + t.Fatal(err) + } + c, err := readStrategyCandidate(db, id) + if err != nil || !c.Qualified || len(c.Endpoints) != 1 || c.Endpoints[0] != cacheTestHost { + t.Fatalf("fresh candidate: %+v %v", c, err) + } + if mode == "newest-pass" && c.Args != newArgs || mode == "older-pass" && c.Args != oldArgs { + t.Fatal(c.Args) + } + } + }) + } +} diff --git a/cmd/zapret-autotune/tune.go b/cmd/zapret-autotune/tune.go index 77cc069..0b567db 100644 --- a/cmd/zapret-autotune/tune.go +++ b/cmd/zapret-autotune/tune.go @@ -110,6 +110,10 @@ func selectBlockcheckArgs(r io.Reader, expected string) (string, error) { } func selectBlockcheckArgsForTarget(r io.Reader, expected, group, host string) (string, error) { + return selectBlockcheckEvidence(r, expected, group, host, false) +} + +func selectBlockcheckEvidence(r io.Reader, expected, group, host string, recovery bool) (string, error) { d := json.NewDecoder(r) var report struct { Strategies []json.RawMessage `json:"strategies"` @@ -125,6 +129,7 @@ func selectBlockcheckArgsForTarget(r io.Reader, expected, group, host string) (s } best := make(map[string]candidate) seen := make(map[string]bool) + nonpass := false for i, record := range report.Strategies { var fields map[string]json.RawMessage if err := json.Unmarshal(record, &fields); err != nil { @@ -165,7 +170,15 @@ func selectBlockcheckArgsForTarget(r io.Reader, expected, group, host string) (s return "", fmt.Errorf("result %d: duplicate args/protocol", i) } seen[key] = true - if expected != "" && (row.Args != expected || !fullyPassing(row.candidate) || row.PassesTotal != 3) { + if recovery { + if row.Args != expected || row.PassesTotal != 3 { + return "", fmt.Errorf("unexpected saved args or incomplete check") + } + if err := row.validateArgs(); err != nil { + return "", err + } + nonpass = nonpass || !fullyPassing(row.candidate) + } else if expected != "" && (row.Args != expected || !fullyPassing(row.candidate) || row.PassesTotal != 3) { return "", fmt.Errorf("result %d: combined args must pass exactly 3/3 for both protocols", i) } // Valid check evidence may describe args our inert output cannot represent. @@ -180,6 +193,15 @@ func selectBlockcheckArgsForTarget(r io.Reader, expected, group, host string) (s best[protocol] = row.candidate } } + if recovery { + if len(seen) != 2 || !seen["tls12\x00"+expected] || !seen["tls13\x00"+expected] { + return "", fmt.Errorf("incomplete combined TLS evidence") + } + if nonpass { + return "", errCandidateNonpass + } + return expected, nil + } var args []string payloads := make(map[string]bool) for _, protocol := range []string{"tls12", "tls13"} {