feat(discovery): find YouTube cache hosts
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
04fd9b370d
commit
db92cff2b5
5 files changed
+357
-9
No files matched your search
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const playerURL = "https://www.youtube.com/youtubei/v1/player"
|
||||
const playerUserAgent = "com.google.youtube/21.02.35(Linux;U;Android11)gzip"
|
||||
const playerResponseLimit = 1 << 20
|
||||
|
||||
var videoIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{11}$`)
|
||||
var discoveryDNSLabel = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`)
|
||||
|
||||
type playerRunner func(context.Context, []string, io.Writer, io.Writer) error
|
||||
|
||||
func runPlayerCurl(ctx context.Context, args []string, stdout, stderr io.Writer) error {
|
||||
cmd := exec.CommandContext(ctx, "curl", args...)
|
||||
cmd.Stdout, cmd.Stderr = stdout, stderr
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
// A failed write stops collection rather than retaining a truncated response.
|
||||
type playerBoundedWriter struct {
|
||||
dst io.Writer
|
||||
n int
|
||||
limit int
|
||||
exceeded bool
|
||||
}
|
||||
|
||||
func (w *playerBoundedWriter) Write(p []byte) (int, error) {
|
||||
if len(p) > w.limit-w.n {
|
||||
w.exceeded = true
|
||||
return 0, errors.New("player output limit exceeded")
|
||||
}
|
||||
w.n += len(p)
|
||||
return w.dst.Write(p)
|
||||
}
|
||||
|
||||
func discoveryHostname(host string) bool {
|
||||
if len(host) > 253 || host == "" {
|
||||
return false
|
||||
}
|
||||
for _, label := range strings.Split(strings.ToLower(host), ".") {
|
||||
if !discoveryDNSLabel.MatchString(label) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validDiscoveryProxy(raw string) bool {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Scheme != "socks5" || u.User != nil || u.Opaque != "" ||
|
||||
u.Path != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" ||
|
||||
strings.ContainsAny(raw, " \t\r\n#") {
|
||||
return false
|
||||
}
|
||||
host, port, err := net.SplitHostPort(u.Host)
|
||||
if err != nil || (net.ParseIP(host) == nil && !discoveryHostname(host)) {
|
||||
return false
|
||||
}
|
||||
n, err := strconv.Atoi(port)
|
||||
return err == nil && n > 0 && n <= 65535
|
||||
}
|
||||
|
||||
func playerHostname(body []byte) (string, error) {
|
||||
var response struct {
|
||||
PlayabilityStatus struct {
|
||||
Status string `json:"status"`
|
||||
} `json:"playabilityStatus"`
|
||||
StreamingData struct {
|
||||
ServerAbrStreamingURL string `json:"serverAbrStreamingUrl"`
|
||||
} `json:"streamingData"`
|
||||
}
|
||||
if len(body) > playerResponseLimit || json.Unmarshal(body, &response) != nil {
|
||||
return "", errors.New("invalid player response")
|
||||
}
|
||||
if response.PlayabilityStatus.Status != "OK" {
|
||||
return "", errors.New("player response is not playable")
|
||||
}
|
||||
u, err := url.Parse(response.StreamingData.ServerAbrStreamingURL)
|
||||
if err != nil || u.Scheme != "https" || u.User != nil || u.Opaque != "" {
|
||||
return "", errors.New("missing or invalid streaming URL")
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
if !discoveryHostname(host) || !strings.HasSuffix(host, ".googlevideo.com") ||
|
||||
(u.Port() != "" && u.Port() != "443") || strings.HasSuffix(u.Host, ":") {
|
||||
return "", errors.New("invalid cache hostname")
|
||||
}
|
||||
return host, nil
|
||||
}
|
||||
|
||||
func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunner) int {
|
||||
flags := flag.NewFlagSet("discover youtube", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard) // flag errors can contain untrusted arguments.
|
||||
videoID := flags.String("video-id", "", "")
|
||||
proxy := flags.String("proxy", "", "")
|
||||
if flags.Parse(args) != nil || flags.NArg() != 0 || !videoIDPattern.MatchString(*videoID) || !validDiscoveryProxy(*proxy) {
|
||||
fmt.Fprintln(stderr, "error: discover youtube requires --video-id (11 characters) and --proxy socks5://HOST:PORT without credentials")
|
||||
return 2
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]interface{}{
|
||||
"videoId": *videoID,
|
||||
"context": map[string]interface{}{"client": map[string]interface{}{
|
||||
"clientName": "ANDROID", "clientVersion": "21.02.35", "androidSdkVersion": 30,
|
||||
"userAgent": playerUserAgent, "osName": "Android", "osVersion": "11",
|
||||
}},
|
||||
})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
|
||||
defer cancel()
|
||||
var body bytes.Buffer
|
||||
output := &playerBoundedWriter{dst: &body, limit: playerResponseLimit}
|
||||
diagnostics := &playerBoundedWriter{dst: io.Discard, limit: 64 << 10}
|
||||
// Disable curlrc and NO_PROXY bypasses: the explicit proxy must be used.
|
||||
err := runner(ctx, []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30",
|
||||
"--noproxy", "", "--proxy", *proxy, "--user-agent", playerUserAgent, "--json", string(payload), playerURL}, output, diagnostics)
|
||||
if err != nil || output.exceeded || diagnostics.exceeded {
|
||||
fmt.Fprintln(stderr, "error: player request failed or exceeded output limit")
|
||||
return 1
|
||||
}
|
||||
host, err := playerHostname(body.Bytes())
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error:", err)
|
||||
return 1
|
||||
}
|
||||
result := struct {
|
||||
VideoID string `json:"videoId"`
|
||||
Hostnames []string `json:"hostnames"`
|
||||
}{*videoID, []string{host}}
|
||||
if json.NewEncoder(stdout).Encode(result) != nil {
|
||||
fmt.Fprintln(stderr, "error: cannot write discovery result")
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const discoveryFixture = `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://rr16---sn-n8v7znsz.googlevideo.com/videoplayback?signature=SECRET"}}`
|
||||
|
||||
func discoveryArgs() []string {
|
||||
return []string{"--video-id", "dQw4w9WgXcQ", "--proxy", "socks5://localhost:1080"}
|
||||
}
|
||||
|
||||
func TestDiscoverYoutubeSuccess(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
called := false
|
||||
runner := func(ctx context.Context, args []string, out, diagnostics io.Writer) error {
|
||||
called = true
|
||||
if _, ok := ctx.Deadline(); !ok {
|
||||
t.Error("request must have a deadline")
|
||||
}
|
||||
prefix := []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", "--noproxy", "", "--proxy", "socks5://localhost:1080", "--user-agent", playerUserAgent, "--json"}
|
||||
if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(args)-1] != playerURL {
|
||||
t.Fatalf("unexpected curl arguments: %q", args)
|
||||
}
|
||||
var payload struct {
|
||||
VideoID string `json:"videoId"`
|
||||
Context struct {
|
||||
Client struct {
|
||||
ClientName string `json:"clientName"`
|
||||
ClientVersion string `json:"clientVersion"`
|
||||
AndroidSDKVersion int `json:"androidSdkVersion"`
|
||||
UserAgent string `json:"userAgent"`
|
||||
OSName string `json:"osName"`
|
||||
OSVersion string `json:"osVersion"`
|
||||
} `json:"client"`
|
||||
} `json:"context"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(args[len(prefix)]), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := payload.Context.Client
|
||||
if payload.VideoID != "dQw4w9WgXcQ" || client.ClientName != "ANDROID" || client.ClientVersion != "21.02.35" ||
|
||||
client.AndroidSDKVersion != 30 || client.UserAgent != playerUserAgent || client.OSName != "Android" || client.OSVersion != "11" {
|
||||
t.Fatal("unexpected player payload")
|
||||
}
|
||||
io.WriteString(diagnostics, "SECRET signed URL must not be printed")
|
||||
_, err := io.WriteString(out, discoveryFixture)
|
||||
return err
|
||||
}
|
||||
if code := discoverYoutube(discoveryArgs(), &stdout, &stderr, runner); code != 0 {
|
||||
t.Fatalf("code %d: %s", code, &stderr)
|
||||
}
|
||||
if !called || stderr.Len() != 0 || stdout.String() != "{\"videoId\":\"dQw4w9WgXcQ\",\"hostnames\":[\"rr16---sn-n8v7znsz.googlevideo.com\"]}\n" {
|
||||
t.Fatalf("unexpected output: stdout=%q stderr=%q", &stdout, &stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverYoutubeInvalidArgs(t *testing.T) {
|
||||
tests := [][]string{
|
||||
nil, {"--video-id", "dQw4w9WgXcQ"}, {"--proxy", "socks5://localhost:1080"},
|
||||
{"--video-id", "short", "--proxy", "socks5://localhost:1080"},
|
||||
{"--video-id", "dQw4w9WgXc!", "--proxy", "socks5://localhost:1080"},
|
||||
{"--SECRET", "SECRET"}, append(discoveryArgs(), "SECRET"),
|
||||
}
|
||||
for _, proxy := range []string{
|
||||
"http://localhost:1080", "socks5h://localhost:1080", "socks5://user:SECRET@localhost:1080",
|
||||
"socks5://:1080", "socks5://localhost", "socks5://localhost:0", "socks5://localhost:65536",
|
||||
"socks5://localhost:bad", "socks5://localhost:1080/path", "socks5://localhost:1080?SECRET",
|
||||
"socks5://localhost:1080#SECRET", "socks5://bad_host:1080", "socks5://localhost:1080?",
|
||||
} {
|
||||
tests = append(tests, []string{"--video-id", "dQw4w9WgXcQ", "--proxy", proxy})
|
||||
}
|
||||
for _, args := range tests {
|
||||
var stdout, stderr bytes.Buffer
|
||||
runner := func(context.Context, []string, io.Writer, io.Writer) error {
|
||||
t.Fatal("invalid arguments must not invoke curl")
|
||||
return nil
|
||||
}
|
||||
if code := discoverYoutube(args, &stdout, &stderr, runner); code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Errorf("args %q: code=%d stdout=%q stderr=%q", args, code, &stdout, &stderr)
|
||||
}
|
||||
if strings.Contains(stderr.String(), "SECRET") || strings.Contains(stderr.String(), "localhost") {
|
||||
t.Error("argument leaked")
|
||||
}
|
||||
}
|
||||
for _, proxy := range []string{"socks5://localhost:1080", "socks5://127.0.0.1:1080", "socks5://[::1]:1080", "socks5://proxy.example:1080"} {
|
||||
if !validDiscoveryProxy(proxy) {
|
||||
t.Errorf("valid proxy rejected: %s", proxy)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlayerHostnameFailsClosed(t *testing.T) {
|
||||
fixtures := []string{
|
||||
`SECRET`, `{} trailing SECRET`, `null`, `{}`, `{"playabilityStatus":{"status":"LOGIN_REQUIRED","reason":"SECRET"}}`,
|
||||
`{"playabilityStatus":{"status":"OK"}}`, `{"playabilityStatus":{"status":"OK"},"streamingData":null}`,
|
||||
}
|
||||
for _, rawURL := range []string{
|
||||
"", "http://rr.googlevideo.com/?SECRET", "//rr.googlevideo.com/?SECRET", "https://googlevideo.com/?SECRET",
|
||||
"https://evilgooglevideo.com/?SECRET", "https://rr.googlevideo.com.evil.test/?SECRET",
|
||||
"https://user:SECRET@rr.googlevideo.com/", "https://rr..googlevideo.com/?SECRET",
|
||||
"https://-rr.googlevideo.com/?SECRET", "https://rr.googlevideo.com./?SECRET",
|
||||
"https://rr.googlevideo.com:80/?SECRET", "https://rr.googlevideo.com:/?SECRET", "https://%SECRET",
|
||||
} {
|
||||
encoded, _ := json.Marshal(rawURL)
|
||||
fixtures = append(fixtures, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":`+string(encoded)+`}}`)
|
||||
}
|
||||
for _, fixture := range fixtures {
|
||||
var stdout, stderr bytes.Buffer
|
||||
runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error {
|
||||
io.WriteString(diagnostics, "SECRET")
|
||||
_, err := io.WriteString(out, fixture)
|
||||
return err
|
||||
}
|
||||
if code := discoverYoutube(discoveryArgs(), &stdout, &stderr, runner); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Errorf("fixture %q: code=%d stdout=%q stderr=%q", fixture, code, &stdout, &stderr)
|
||||
}
|
||||
if strings.Contains(stderr.String(), "SECRET") || strings.Contains(stderr.String(), "https://") {
|
||||
t.Error("response leaked")
|
||||
}
|
||||
}
|
||||
upper := strings.Replace(discoveryFixture, "rr16---sn-n8v7znsz.googlevideo.com", "RR16---SN-N8V7ZNSZ.GOOGLEVIDEO.COM:443", 1)
|
||||
if host, err := playerHostname([]byte(upper)); err != nil || host != "rr16---sn-n8v7znsz.googlevideo.com" {
|
||||
t.Fatalf("hostname normalization: %q %v", host, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverYoutubeRequestFailures(t *testing.T) {
|
||||
for _, mode := range []string{"runner", "stdout limit", "stderr limit", "write result"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error {
|
||||
switch mode {
|
||||
case "runner":
|
||||
io.WriteString(out, discoveryFixture)
|
||||
io.WriteString(diagnostics, "https://signed.example/?SECRET")
|
||||
return errors.New("SECRET socks5://user:password@localhost:1080")
|
||||
case "stdout limit":
|
||||
// Ignore the write error to ensure the command still fails closed.
|
||||
out.Write(bytes.Repeat([]byte("S"), playerResponseLimit+1))
|
||||
case "stderr limit":
|
||||
diagnostics.Write(bytes.Repeat([]byte("S"), (64<<10)+1))
|
||||
}
|
||||
_, err := io.WriteString(out, discoveryFixture)
|
||||
return err
|
||||
}
|
||||
var dest io.Writer = &stdout
|
||||
if mode == "write result" {
|
||||
dest = failingWriter{}
|
||||
}
|
||||
if code := discoverYoutube(discoveryArgs(), dest, &stderr, runner); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, &stdout, &stderr)
|
||||
}
|
||||
for _, forbidden := range []string{"SECRET", "https://", "password", "localhost"} {
|
||||
if strings.Contains(stderr.String(), forbidden) {
|
||||
t.Errorf("leaked %s", forbidden)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlayerOutputBound(t *testing.T) {
|
||||
var dst bytes.Buffer
|
||||
w := &playerBoundedWriter{dst: &dst, limit: 4}
|
||||
if n, err := w.Write([]byte("1234")); n != 4 || err != nil {
|
||||
t.Fatal("exact limit must be accepted")
|
||||
}
|
||||
if n, err := w.Write([]byte("5")); n != 0 || err == nil || !w.exceeded || dst.String() != "1234" {
|
||||
t.Fatal("overflow must not be retained")
|
||||
}
|
||||
if _, err := playerHostname(bytes.Repeat([]byte(" "), playerResponseLimit+1)); err == nil {
|
||||
t.Fatal("oversized parser input accepted")
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,8 @@ Commands:
|
||||
status Show latest recorded tune run/proposal (not live service health)
|
||||
history [--limit N] List recorded tune attempts (default 20; proposals not active)
|
||||
strategies youtube List recorded proposals (not active), newest first
|
||||
discover youtube Refuse discovery until provider/config is agreed
|
||||
discover youtube --video-id ID --proxy socks5://HOST:PORT
|
||||
Discover cache hostname only (no media-transfer check)
|
||||
check youtube Refuse checks until probe URLs/video IDs are configured and agreed
|
||||
tune youtube Test youtube.com frontend only; print options without applying
|
||||
render <json-file> Select normalized offline results and print NFQWS2_OPT
|
||||
@@ -62,9 +63,8 @@ func run(args []string, stdout, stderr io.Writer) int {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
if len(args) == 2 && args[0] == "discover" && args[1] == "youtube" {
|
||||
fmt.Fprintln(stderr, "error: discover youtube is unavailable: YouTube provider/config is not implemented; probe URLs/video IDs and API credentials/cookie requirements have not been configured or agreed. No network requests were made; no endpoints were generated.")
|
||||
return 1
|
||||
if len(args) >= 2 && args[0] == "discover" && args[1] == "youtube" {
|
||||
return discoverYoutube(args[2:], stdout, stderr, runPlayerCurl)
|
||||
}
|
||||
if len(args) == 2 && args[0] == "check" && args[1] == "youtube" {
|
||||
fmt.Fprintln(stderr, "error: check youtube is unavailable: YouTube probe URLs/video IDs have not been configured or agreed. No network requests were made; production health was not checked.")
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestRun(t *testing.T) {
|
||||
err string
|
||||
}{
|
||||
{"status", []string{"status"}, 0, "apply: BLOCKED", ""},
|
||||
{"discovery refused", []string{"discover", "youtube"}, 1, "", "probe URLs/video IDs and API credentials/cookie requirements"},
|
||||
{"discovery missing flags", []string{"discover", "youtube"}, 2, "", "requires --video-id"},
|
||||
{"check refused", []string{"check", "youtube"}, 1, "", "YouTube probe URLs/video IDs have not been configured or agreed"},
|
||||
{"unsupported check provider", []string{"check", "other"}, 2, "", usage},
|
||||
{"missing check provider", []string{"check"}, 2, "", usage},
|
||||
@@ -29,7 +29,7 @@ func TestRun(t *testing.T) {
|
||||
{"unsupported provider", []string{"discover", "other"}, 2, "", usage},
|
||||
{"missing provider", []string{"discover"}, 2, "", usage},
|
||||
{"extra status argument", []string{"status", "extra"}, 2, "", usage},
|
||||
{"extra discovery argument", []string{"discover", "youtube", "extra"}, 2, "", usage},
|
||||
{"extra discovery argument", []string{"discover", "youtube", "extra"}, 2, "", "requires --video-id"},
|
||||
{"unsupported tune provider", []string{"tune", "other"}, 2, "", usage},
|
||||
{"apply unavailable", []string{"apply"}, 2, "", usage},
|
||||
{"disruption flag rejected", []string{"tune", "--allow-disruption"}, 2, "", usage},
|
||||
@@ -51,9 +51,6 @@ func TestRun(t *testing.T) {
|
||||
if tt.name == "status" && !strings.Contains(stdout.String(), "NOT CHECKED") {
|
||||
t.Error("status must not claim production health")
|
||||
}
|
||||
if tt.name == "discovery refused" && !strings.Contains(stderr.String(), "no endpoints were generated") {
|
||||
t.Error("discovery must explicitly refuse endpoint generation")
|
||||
}
|
||||
if tt.name == "check refused" {
|
||||
for _, want := range []string{"No network requests were made", "production health was not checked"} {
|
||||
if !strings.Contains(stderr.String(), want) {
|
||||
|
||||
Reference in new issue
Block a user