feat(discovery): find YouTube cache hosts

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-09 22:11:13 +03:00
1 parent 04fd9b370d
commit db92cff2b5
5 files changed
+357 -9

No files matched your search

+21
View File
@@ -2,6 +2,27 @@
Go CLI for scanning and checking YouTube TLS bypass strategies with `blockcheckw`. Go CLI for scanning and checking YouTube TLS bypass strategies with `blockcheckw`.
## Cache-host discovery
```sh
zapret-autotune discover youtube --video-id dQw4w9WgXcQ --proxy socks5://localhost:1080
```
Both flags are required. Use a proxy address reachable from the CLI; in Docker,
`localhost` is the container itself, not the Docker host. Proxy credentials,
cookies, and API credentials are not supported. This command requires `curl`
with `--json` support and POSTs to the YouTube player API through SOCKS5 using
Android client 21.02.35, SDK 30, Android 11. Requests time out after 30 seconds;
responses are limited to 1 MiB and must have playability status `OK` and a valid
HTTPS `serverAbrStreamingUrl` under `.googlevideo.com`.
Output is JSON containing only `videoId` and `hostnames`; signed streaming URLs
are never printed. Cache hosts may rotate. This is **cache-host discovery only**,
not verification of real video transfer: a discovered SABR URL can return
`sabr.malformed_config` on GET. Discovery does not fetch media, persist results,
apply production configuration, or restart services. Invalid input and failed
requests return nonzero with sanitized errors.
Build the Docker image with: Build the Docker image with:
```sh ```sh
+148
View File
@@ -0,0 +1,148 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net"
"net/url"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
)
const playerURL = "https://www.youtube.com/youtubei/v1/player"
const playerUserAgent = "com.google.youtube/21.02.35(Linux;U;Android11)gzip"
const playerResponseLimit = 1 << 20
var videoIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{11}$`)
var discoveryDNSLabel = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`)
type playerRunner func(context.Context, []string, io.Writer, io.Writer) error
func runPlayerCurl(ctx context.Context, args []string, stdout, stderr io.Writer) error {
cmd := exec.CommandContext(ctx, "curl", args...)
cmd.Stdout, cmd.Stderr = stdout, stderr
return cmd.Run()
}
// A failed write stops collection rather than retaining a truncated response.
type playerBoundedWriter struct {
dst io.Writer
n int
limit int
exceeded bool
}
func (w *playerBoundedWriter) Write(p []byte) (int, error) {
if len(p) > w.limit-w.n {
w.exceeded = true
return 0, errors.New("player output limit exceeded")
}
w.n += len(p)
return w.dst.Write(p)
}
func discoveryHostname(host string) bool {
if len(host) > 253 || host == "" {
return false
}
for _, label := range strings.Split(strings.ToLower(host), ".") {
if !discoveryDNSLabel.MatchString(label) {
return false
}
}
return true
}
func validDiscoveryProxy(raw string) bool {
u, err := url.Parse(raw)
if err != nil || u.Scheme != "socks5" || u.User != nil || u.Opaque != "" ||
u.Path != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" ||
strings.ContainsAny(raw, " \t\r\n#") {
return false
}
host, port, err := net.SplitHostPort(u.Host)
if err != nil || (net.ParseIP(host) == nil && !discoveryHostname(host)) {
return false
}
n, err := strconv.Atoi(port)
return err == nil && n > 0 && n <= 65535
}
func playerHostname(body []byte) (string, error) {
var response struct {
PlayabilityStatus struct {
Status string `json:"status"`
} `json:"playabilityStatus"`
StreamingData struct {
ServerAbrStreamingURL string `json:"serverAbrStreamingUrl"`
} `json:"streamingData"`
}
if len(body) > playerResponseLimit || json.Unmarshal(body, &response) != nil {
return "", errors.New("invalid player response")
}
if response.PlayabilityStatus.Status != "OK" {
return "", errors.New("player response is not playable")
}
u, err := url.Parse(response.StreamingData.ServerAbrStreamingURL)
if err != nil || u.Scheme != "https" || u.User != nil || u.Opaque != "" {
return "", errors.New("missing or invalid streaming URL")
}
host := strings.ToLower(u.Hostname())
if !discoveryHostname(host) || !strings.HasSuffix(host, ".googlevideo.com") ||
(u.Port() != "" && u.Port() != "443") || strings.HasSuffix(u.Host, ":") {
return "", errors.New("invalid cache hostname")
}
return host, nil
}
func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunner) int {
flags := flag.NewFlagSet("discover youtube", flag.ContinueOnError)
flags.SetOutput(io.Discard) // flag errors can contain untrusted arguments.
videoID := flags.String("video-id", "", "")
proxy := flags.String("proxy", "", "")
if flags.Parse(args) != nil || flags.NArg() != 0 || !videoIDPattern.MatchString(*videoID) || !validDiscoveryProxy(*proxy) {
fmt.Fprintln(stderr, "error: discover youtube requires --video-id (11 characters) and --proxy socks5://HOST:PORT without credentials")
return 2
}
payload, _ := json.Marshal(map[string]interface{}{
"videoId": *videoID,
"context": map[string]interface{}{"client": map[string]interface{}{
"clientName": "ANDROID", "clientVersion": "21.02.35", "androidSdkVersion": 30,
"userAgent": playerUserAgent, "osName": "Android", "osVersion": "11",
}},
})
ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
defer cancel()
var body bytes.Buffer
output := &playerBoundedWriter{dst: &body, limit: playerResponseLimit}
diagnostics := &playerBoundedWriter{dst: io.Discard, limit: 64 << 10}
// Disable curlrc and NO_PROXY bypasses: the explicit proxy must be used.
err := runner(ctx, []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30",
"--noproxy", "", "--proxy", *proxy, "--user-agent", playerUserAgent, "--json", string(payload), playerURL}, output, diagnostics)
if err != nil || output.exceeded || diagnostics.exceeded {
fmt.Fprintln(stderr, "error: player request failed or exceeded output limit")
return 1
}
host, err := playerHostname(body.Bytes())
if err != nil {
fmt.Fprintln(stderr, "error:", err)
return 1
}
result := struct {
VideoID string `json:"videoId"`
Hostnames []string `json:"hostnames"`
}{*videoID, []string{host}}
if json.NewEncoder(stdout).Encode(result) != nil {
fmt.Fprintln(stderr, "error: cannot write discovery result")
return 1
}
return 0
}
+182
View File
@@ -0,0 +1,182 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"reflect"
"strings"
"testing"
)
const discoveryFixture = `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://rr16---sn-n8v7znsz.googlevideo.com/videoplayback?signature=SECRET"}}`
func discoveryArgs() []string {
return []string{"--video-id", "dQw4w9WgXcQ", "--proxy", "socks5://localhost:1080"}
}
func TestDiscoverYoutubeSuccess(t *testing.T) {
var stdout, stderr bytes.Buffer
called := false
runner := func(ctx context.Context, args []string, out, diagnostics io.Writer) error {
called = true
if _, ok := ctx.Deadline(); !ok {
t.Error("request must have a deadline")
}
prefix := []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", "--noproxy", "", "--proxy", "socks5://localhost:1080", "--user-agent", playerUserAgent, "--json"}
if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(args)-1] != playerURL {
t.Fatalf("unexpected curl arguments: %q", args)
}
var payload struct {
VideoID string `json:"videoId"`
Context struct {
Client struct {
ClientName string `json:"clientName"`
ClientVersion string `json:"clientVersion"`
AndroidSDKVersion int `json:"androidSdkVersion"`
UserAgent string `json:"userAgent"`
OSName string `json:"osName"`
OSVersion string `json:"osVersion"`
} `json:"client"`
} `json:"context"`
}
if err := json.Unmarshal([]byte(args[len(prefix)]), &payload); err != nil {
t.Fatal(err)
}
client := payload.Context.Client
if payload.VideoID != "dQw4w9WgXcQ" || client.ClientName != "ANDROID" || client.ClientVersion != "21.02.35" ||
client.AndroidSDKVersion != 30 || client.UserAgent != playerUserAgent || client.OSName != "Android" || client.OSVersion != "11" {
t.Fatal("unexpected player payload")
}
io.WriteString(diagnostics, "SECRET signed URL must not be printed")
_, err := io.WriteString(out, discoveryFixture)
return err
}
if code := discoverYoutube(discoveryArgs(), &stdout, &stderr, runner); code != 0 {
t.Fatalf("code %d: %s", code, &stderr)
}
if !called || stderr.Len() != 0 || stdout.String() != "{\"videoId\":\"dQw4w9WgXcQ\",\"hostnames\":[\"rr16---sn-n8v7znsz.googlevideo.com\"]}\n" {
t.Fatalf("unexpected output: stdout=%q stderr=%q", &stdout, &stderr)
}
}
func TestDiscoverYoutubeInvalidArgs(t *testing.T) {
tests := [][]string{
nil, {"--video-id", "dQw4w9WgXcQ"}, {"--proxy", "socks5://localhost:1080"},
{"--video-id", "short", "--proxy", "socks5://localhost:1080"},
{"--video-id", "dQw4w9WgXc!", "--proxy", "socks5://localhost:1080"},
{"--SECRET", "SECRET"}, append(discoveryArgs(), "SECRET"),
}
for _, proxy := range []string{
"http://localhost:1080", "socks5h://localhost:1080", "socks5://user:SECRET@localhost:1080",
"socks5://:1080", "socks5://localhost", "socks5://localhost:0", "socks5://localhost:65536",
"socks5://localhost:bad", "socks5://localhost:1080/path", "socks5://localhost:1080?SECRET",
"socks5://localhost:1080#SECRET", "socks5://bad_host:1080", "socks5://localhost:1080?",
} {
tests = append(tests, []string{"--video-id", "dQw4w9WgXcQ", "--proxy", proxy})
}
for _, args := range tests {
var stdout, stderr bytes.Buffer
runner := func(context.Context, []string, io.Writer, io.Writer) error {
t.Fatal("invalid arguments must not invoke curl")
return nil
}
if code := discoverYoutube(args, &stdout, &stderr, runner); code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Errorf("args %q: code=%d stdout=%q stderr=%q", args, code, &stdout, &stderr)
}
if strings.Contains(stderr.String(), "SECRET") || strings.Contains(stderr.String(), "localhost") {
t.Error("argument leaked")
}
}
for _, proxy := range []string{"socks5://localhost:1080", "socks5://127.0.0.1:1080", "socks5://[::1]:1080", "socks5://proxy.example:1080"} {
if !validDiscoveryProxy(proxy) {
t.Errorf("valid proxy rejected: %s", proxy)
}
}
}
func TestPlayerHostnameFailsClosed(t *testing.T) {
fixtures := []string{
`SECRET`, `{} trailing SECRET`, `null`, `{}`, `{"playabilityStatus":{"status":"LOGIN_REQUIRED","reason":"SECRET"}}`,
`{"playabilityStatus":{"status":"OK"}}`, `{"playabilityStatus":{"status":"OK"},"streamingData":null}`,
}
for _, rawURL := range []string{
"", "http://rr.googlevideo.com/?SECRET", "//rr.googlevideo.com/?SECRET", "https://googlevideo.com/?SECRET",
"https://evilgooglevideo.com/?SECRET", "https://rr.googlevideo.com.evil.test/?SECRET",
"https://user:SECRET@rr.googlevideo.com/", "https://rr..googlevideo.com/?SECRET",
"https://-rr.googlevideo.com/?SECRET", "https://rr.googlevideo.com./?SECRET",
"https://rr.googlevideo.com:80/?SECRET", "https://rr.googlevideo.com:/?SECRET", "https://%SECRET",
} {
encoded, _ := json.Marshal(rawURL)
fixtures = append(fixtures, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":`+string(encoded)+`}}`)
}
for _, fixture := range fixtures {
var stdout, stderr bytes.Buffer
runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error {
io.WriteString(diagnostics, "SECRET")
_, err := io.WriteString(out, fixture)
return err
}
if code := discoverYoutube(discoveryArgs(), &stdout, &stderr, runner); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Errorf("fixture %q: code=%d stdout=%q stderr=%q", fixture, code, &stdout, &stderr)
}
if strings.Contains(stderr.String(), "SECRET") || strings.Contains(stderr.String(), "https://") {
t.Error("response leaked")
}
}
upper := strings.Replace(discoveryFixture, "rr16---sn-n8v7znsz.googlevideo.com", "RR16---SN-N8V7ZNSZ.GOOGLEVIDEO.COM:443", 1)
if host, err := playerHostname([]byte(upper)); err != nil || host != "rr16---sn-n8v7znsz.googlevideo.com" {
t.Fatalf("hostname normalization: %q %v", host, err)
}
}
func TestDiscoverYoutubeRequestFailures(t *testing.T) {
for _, mode := range []string{"runner", "stdout limit", "stderr limit", "write result"} {
t.Run(mode, func(t *testing.T) {
var stdout, stderr bytes.Buffer
runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error {
switch mode {
case "runner":
io.WriteString(out, discoveryFixture)
io.WriteString(diagnostics, "https://signed.example/?SECRET")
return errors.New("SECRET socks5://user:password@localhost:1080")
case "stdout limit":
// Ignore the write error to ensure the command still fails closed.
out.Write(bytes.Repeat([]byte("S"), playerResponseLimit+1))
case "stderr limit":
diagnostics.Write(bytes.Repeat([]byte("S"), (64<<10)+1))
}
_, err := io.WriteString(out, discoveryFixture)
return err
}
var dest io.Writer = &stdout
if mode == "write result" {
dest = failingWriter{}
}
if code := discoverYoutube(discoveryArgs(), dest, &stderr, runner); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, &stdout, &stderr)
}
for _, forbidden := range []string{"SECRET", "https://", "password", "localhost"} {
if strings.Contains(stderr.String(), forbidden) {
t.Errorf("leaked %s", forbidden)
}
}
})
}
}
func TestPlayerOutputBound(t *testing.T) {
var dst bytes.Buffer
w := &playerBoundedWriter{dst: &dst, limit: 4}
if n, err := w.Write([]byte("1234")); n != 4 || err != nil {
t.Fatal("exact limit must be accepted")
}
if n, err := w.Write([]byte("5")); n != 0 || err == nil || !w.exceeded || dst.String() != "1234" {
t.Fatal("overflow must not be retained")
}
if _, err := playerHostname(bytes.Repeat([]byte(" "), playerResponseLimit+1)); err == nil {
t.Fatal("oversized parser input accepted")
}
}
+4 -4
View File
@@ -12,7 +12,8 @@ Commands:
status Show latest recorded tune run/proposal (not live service health) status Show latest recorded tune run/proposal (not live service health)
history [--limit N] List recorded tune attempts (default 20; proposals not active) history [--limit N] List recorded tune attempts (default 20; proposals not active)
strategies youtube List recorded proposals (not active), newest first strategies youtube List recorded proposals (not active), newest first
discover youtube Refuse discovery until provider/config is agreed discover youtube --video-id ID --proxy socks5://HOST:PORT
Discover cache hostname only (no media-transfer check)
check youtube Refuse checks until probe URLs/video IDs are configured and agreed check youtube Refuse checks until probe URLs/video IDs are configured and agreed
tune youtube Test youtube.com frontend only; print options without applying tune youtube Test youtube.com frontend only; print options without applying
render <json-file> Select normalized offline results and print NFQWS2_OPT render <json-file> Select normalized offline results and print NFQWS2_OPT
@@ -62,9 +63,8 @@ func run(args []string, stdout, stderr io.Writer) int {
return 0 return 0
} }
} }
if len(args) == 2 && args[0] == "discover" && args[1] == "youtube" { if len(args) >= 2 && args[0] == "discover" && args[1] == "youtube" {
fmt.Fprintln(stderr, "error: discover youtube is unavailable: YouTube provider/config is not implemented; probe URLs/video IDs and API credentials/cookie requirements have not been configured or agreed. No network requests were made; no endpoints were generated.") return discoverYoutube(args[2:], stdout, stderr, runPlayerCurl)
return 1
} }
if len(args) == 2 && args[0] == "check" && args[1] == "youtube" { if len(args) == 2 && args[0] == "check" && args[1] == "youtube" {
fmt.Fprintln(stderr, "error: check youtube is unavailable: YouTube probe URLs/video IDs have not been configured or agreed. No network requests were made; production health was not checked.") fmt.Fprintln(stderr, "error: check youtube is unavailable: YouTube probe URLs/video IDs have not been configured or agreed. No network requests were made; production health was not checked.")
+2 -5
View File
@@ -18,7 +18,7 @@ func TestRun(t *testing.T) {
err string err string
}{ }{
{"status", []string{"status"}, 0, "apply: BLOCKED", ""}, {"status", []string{"status"}, 0, "apply: BLOCKED", ""},
{"discovery refused", []string{"discover", "youtube"}, 1, "", "probe URLs/video IDs and API credentials/cookie requirements"}, {"discovery missing flags", []string{"discover", "youtube"}, 2, "", "requires --video-id"},
{"check refused", []string{"check", "youtube"}, 1, "", "YouTube probe URLs/video IDs have not been configured or agreed"}, {"check refused", []string{"check", "youtube"}, 1, "", "YouTube probe URLs/video IDs have not been configured or agreed"},
{"unsupported check provider", []string{"check", "other"}, 2, "", usage}, {"unsupported check provider", []string{"check", "other"}, 2, "", usage},
{"missing check provider", []string{"check"}, 2, "", usage}, {"missing check provider", []string{"check"}, 2, "", usage},
@@ -29,7 +29,7 @@ func TestRun(t *testing.T) {
{"unsupported provider", []string{"discover", "other"}, 2, "", usage}, {"unsupported provider", []string{"discover", "other"}, 2, "", usage},
{"missing provider", []string{"discover"}, 2, "", usage}, {"missing provider", []string{"discover"}, 2, "", usage},
{"extra status argument", []string{"status", "extra"}, 2, "", usage}, {"extra status argument", []string{"status", "extra"}, 2, "", usage},
{"extra discovery argument", []string{"discover", "youtube", "extra"}, 2, "", usage}, {"extra discovery argument", []string{"discover", "youtube", "extra"}, 2, "", "requires --video-id"},
{"unsupported tune provider", []string{"tune", "other"}, 2, "", usage}, {"unsupported tune provider", []string{"tune", "other"}, 2, "", usage},
{"apply unavailable", []string{"apply"}, 2, "", usage}, {"apply unavailable", []string{"apply"}, 2, "", usage},
{"disruption flag rejected", []string{"tune", "--allow-disruption"}, 2, "", usage}, {"disruption flag rejected", []string{"tune", "--allow-disruption"}, 2, "", usage},
@@ -51,9 +51,6 @@ func TestRun(t *testing.T) {
if tt.name == "status" && !strings.Contains(stdout.String(), "NOT CHECKED") { if tt.name == "status" && !strings.Contains(stdout.String(), "NOT CHECKED") {
t.Error("status must not claim production health") t.Error("status must not claim production health")
} }
if tt.name == "discovery refused" && !strings.Contains(stderr.String(), "no endpoints were generated") {
t.Error("discovery must explicitly refuse endpoint generation")
}
if tt.name == "check refused" { if tt.name == "check refused" {
for _, want := range []string{"No network requests were made", "production health was not checked"} { for _, want := range []string{"No network requests were made", "production health was not checked"} {
if !strings.Contains(stderr.String(), want) { if !strings.Contains(stderr.String(), want) {