diff --git a/README.md b/README.md index 644156d..af498a5 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,27 @@ Go CLI for scanning and checking YouTube TLS bypass strategies with `blockcheckw`. +## Cache-host discovery + +```sh +zapret-autotune discover youtube --video-id dQw4w9WgXcQ --proxy socks5://localhost:1080 +``` + +Both flags are required. Use a proxy address reachable from the CLI; in Docker, +`localhost` is the container itself, not the Docker host. Proxy credentials, +cookies, and API credentials are not supported. This command requires `curl` +with `--json` support and POSTs to the YouTube player API through SOCKS5 using +Android client 21.02.35, SDK 30, Android 11. Requests time out after 30 seconds; +responses are limited to 1 MiB and must have playability status `OK` and a valid +HTTPS `serverAbrStreamingUrl` under `.googlevideo.com`. + +Output is JSON containing only `videoId` and `hostnames`; signed streaming URLs +are never printed. Cache hosts may rotate. This is **cache-host discovery only**, +not verification of real video transfer: a discovered SABR URL can return +`sabr.malformed_config` on GET. Discovery does not fetch media, persist results, +apply production configuration, or restart services. Invalid input and failed +requests return nonzero with sanitized errors. + Build the Docker image with: ```sh diff --git a/cmd/zapret-autotune/discover.go b/cmd/zapret-autotune/discover.go new file mode 100644 index 0000000..474415d --- /dev/null +++ b/cmd/zapret-autotune/discover.go @@ -0,0 +1,148 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net" + "net/url" + "os/exec" + "regexp" + "strconv" + "strings" + "time" +) + +const playerURL = "https://www.youtube.com/youtubei/v1/player" +const playerUserAgent = "com.google.youtube/21.02.35(Linux;U;Android11)gzip" +const playerResponseLimit = 1 << 20 + +var videoIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{11}$`) +var discoveryDNSLabel = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`) + +type playerRunner func(context.Context, []string, io.Writer, io.Writer) error + +func runPlayerCurl(ctx context.Context, args []string, stdout, stderr io.Writer) error { + cmd := exec.CommandContext(ctx, "curl", args...) + cmd.Stdout, cmd.Stderr = stdout, stderr + return cmd.Run() +} + +// A failed write stops collection rather than retaining a truncated response. +type playerBoundedWriter struct { + dst io.Writer + n int + limit int + exceeded bool +} + +func (w *playerBoundedWriter) Write(p []byte) (int, error) { + if len(p) > w.limit-w.n { + w.exceeded = true + return 0, errors.New("player output limit exceeded") + } + w.n += len(p) + return w.dst.Write(p) +} + +func discoveryHostname(host string) bool { + if len(host) > 253 || host == "" { + return false + } + for _, label := range strings.Split(strings.ToLower(host), ".") { + if !discoveryDNSLabel.MatchString(label) { + return false + } + } + return true +} + +func validDiscoveryProxy(raw string) bool { + u, err := url.Parse(raw) + if err != nil || u.Scheme != "socks5" || u.User != nil || u.Opaque != "" || + u.Path != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || + strings.ContainsAny(raw, " \t\r\n#") { + return false + } + host, port, err := net.SplitHostPort(u.Host) + if err != nil || (net.ParseIP(host) == nil && !discoveryHostname(host)) { + return false + } + n, err := strconv.Atoi(port) + return err == nil && n > 0 && n <= 65535 +} + +func playerHostname(body []byte) (string, error) { + var response struct { + PlayabilityStatus struct { + Status string `json:"status"` + } `json:"playabilityStatus"` + StreamingData struct { + ServerAbrStreamingURL string `json:"serverAbrStreamingUrl"` + } `json:"streamingData"` + } + if len(body) > playerResponseLimit || json.Unmarshal(body, &response) != nil { + return "", errors.New("invalid player response") + } + if response.PlayabilityStatus.Status != "OK" { + return "", errors.New("player response is not playable") + } + u, err := url.Parse(response.StreamingData.ServerAbrStreamingURL) + if err != nil || u.Scheme != "https" || u.User != nil || u.Opaque != "" { + return "", errors.New("missing or invalid streaming URL") + } + host := strings.ToLower(u.Hostname()) + if !discoveryHostname(host) || !strings.HasSuffix(host, ".googlevideo.com") || + (u.Port() != "" && u.Port() != "443") || strings.HasSuffix(u.Host, ":") { + return "", errors.New("invalid cache hostname") + } + return host, nil +} + +func discoverYoutube(args []string, stdout, stderr io.Writer, runner playerRunner) int { + flags := flag.NewFlagSet("discover youtube", flag.ContinueOnError) + flags.SetOutput(io.Discard) // flag errors can contain untrusted arguments. + videoID := flags.String("video-id", "", "") + proxy := flags.String("proxy", "", "") + if flags.Parse(args) != nil || flags.NArg() != 0 || !videoIDPattern.MatchString(*videoID) || !validDiscoveryProxy(*proxy) { + fmt.Fprintln(stderr, "error: discover youtube requires --video-id (11 characters) and --proxy socks5://HOST:PORT without credentials") + return 2 + } + payload, _ := json.Marshal(map[string]interface{}{ + "videoId": *videoID, + "context": map[string]interface{}{"client": map[string]interface{}{ + "clientName": "ANDROID", "clientVersion": "21.02.35", "androidSdkVersion": 30, + "userAgent": playerUserAgent, "osName": "Android", "osVersion": "11", + }}, + }) + ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second) + defer cancel() + var body bytes.Buffer + output := &playerBoundedWriter{dst: &body, limit: playerResponseLimit} + diagnostics := &playerBoundedWriter{dst: io.Discard, limit: 64 << 10} + // Disable curlrc and NO_PROXY bypasses: the explicit proxy must be used. + err := runner(ctx, []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", + "--noproxy", "", "--proxy", *proxy, "--user-agent", playerUserAgent, "--json", string(payload), playerURL}, output, diagnostics) + if err != nil || output.exceeded || diagnostics.exceeded { + fmt.Fprintln(stderr, "error: player request failed or exceeded output limit") + return 1 + } + host, err := playerHostname(body.Bytes()) + if err != nil { + fmt.Fprintln(stderr, "error:", err) + return 1 + } + result := struct { + VideoID string `json:"videoId"` + Hostnames []string `json:"hostnames"` + }{*videoID, []string{host}} + if json.NewEncoder(stdout).Encode(result) != nil { + fmt.Fprintln(stderr, "error: cannot write discovery result") + return 1 + } + return 0 +} diff --git a/cmd/zapret-autotune/discover_test.go b/cmd/zapret-autotune/discover_test.go new file mode 100644 index 0000000..879aef2 --- /dev/null +++ b/cmd/zapret-autotune/discover_test.go @@ -0,0 +1,182 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "reflect" + "strings" + "testing" +) + +const discoveryFixture = `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://rr16---sn-n8v7znsz.googlevideo.com/videoplayback?signature=SECRET"}}` + +func discoveryArgs() []string { + return []string{"--video-id", "dQw4w9WgXcQ", "--proxy", "socks5://localhost:1080"} +} + +func TestDiscoverYoutubeSuccess(t *testing.T) { + var stdout, stderr bytes.Buffer + called := false + runner := func(ctx context.Context, args []string, out, diagnostics io.Writer) error { + called = true + if _, ok := ctx.Deadline(); !ok { + t.Error("request must have a deadline") + } + prefix := []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", "--noproxy", "", "--proxy", "socks5://localhost:1080", "--user-agent", playerUserAgent, "--json"} + if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(args)-1] != playerURL { + t.Fatalf("unexpected curl arguments: %q", args) + } + var payload struct { + VideoID string `json:"videoId"` + Context struct { + Client struct { + ClientName string `json:"clientName"` + ClientVersion string `json:"clientVersion"` + AndroidSDKVersion int `json:"androidSdkVersion"` + UserAgent string `json:"userAgent"` + OSName string `json:"osName"` + OSVersion string `json:"osVersion"` + } `json:"client"` + } `json:"context"` + } + if err := json.Unmarshal([]byte(args[len(prefix)]), &payload); err != nil { + t.Fatal(err) + } + client := payload.Context.Client + if payload.VideoID != "dQw4w9WgXcQ" || client.ClientName != "ANDROID" || client.ClientVersion != "21.02.35" || + client.AndroidSDKVersion != 30 || client.UserAgent != playerUserAgent || client.OSName != "Android" || client.OSVersion != "11" { + t.Fatal("unexpected player payload") + } + io.WriteString(diagnostics, "SECRET signed URL must not be printed") + _, err := io.WriteString(out, discoveryFixture) + return err + } + if code := discoverYoutube(discoveryArgs(), &stdout, &stderr, runner); code != 0 { + t.Fatalf("code %d: %s", code, &stderr) + } + if !called || stderr.Len() != 0 || stdout.String() != "{\"videoId\":\"dQw4w9WgXcQ\",\"hostnames\":[\"rr16---sn-n8v7znsz.googlevideo.com\"]}\n" { + t.Fatalf("unexpected output: stdout=%q stderr=%q", &stdout, &stderr) + } +} + +func TestDiscoverYoutubeInvalidArgs(t *testing.T) { + tests := [][]string{ + nil, {"--video-id", "dQw4w9WgXcQ"}, {"--proxy", "socks5://localhost:1080"}, + {"--video-id", "short", "--proxy", "socks5://localhost:1080"}, + {"--video-id", "dQw4w9WgXc!", "--proxy", "socks5://localhost:1080"}, + {"--SECRET", "SECRET"}, append(discoveryArgs(), "SECRET"), + } + for _, proxy := range []string{ + "http://localhost:1080", "socks5h://localhost:1080", "socks5://user:SECRET@localhost:1080", + "socks5://:1080", "socks5://localhost", "socks5://localhost:0", "socks5://localhost:65536", + "socks5://localhost:bad", "socks5://localhost:1080/path", "socks5://localhost:1080?SECRET", + "socks5://localhost:1080#SECRET", "socks5://bad_host:1080", "socks5://localhost:1080?", + } { + tests = append(tests, []string{"--video-id", "dQw4w9WgXcQ", "--proxy", proxy}) + } + for _, args := range tests { + var stdout, stderr bytes.Buffer + runner := func(context.Context, []string, io.Writer, io.Writer) error { + t.Fatal("invalid arguments must not invoke curl") + return nil + } + if code := discoverYoutube(args, &stdout, &stderr, runner); code != 2 || stdout.Len() != 0 || stderr.Len() == 0 { + t.Errorf("args %q: code=%d stdout=%q stderr=%q", args, code, &stdout, &stderr) + } + if strings.Contains(stderr.String(), "SECRET") || strings.Contains(stderr.String(), "localhost") { + t.Error("argument leaked") + } + } + for _, proxy := range []string{"socks5://localhost:1080", "socks5://127.0.0.1:1080", "socks5://[::1]:1080", "socks5://proxy.example:1080"} { + if !validDiscoveryProxy(proxy) { + t.Errorf("valid proxy rejected: %s", proxy) + } + } +} + +func TestPlayerHostnameFailsClosed(t *testing.T) { + fixtures := []string{ + `SECRET`, `{} trailing SECRET`, `null`, `{}`, `{"playabilityStatus":{"status":"LOGIN_REQUIRED","reason":"SECRET"}}`, + `{"playabilityStatus":{"status":"OK"}}`, `{"playabilityStatus":{"status":"OK"},"streamingData":null}`, + } + for _, rawURL := range []string{ + "", "http://rr.googlevideo.com/?SECRET", "//rr.googlevideo.com/?SECRET", "https://googlevideo.com/?SECRET", + "https://evilgooglevideo.com/?SECRET", "https://rr.googlevideo.com.evil.test/?SECRET", + "https://user:SECRET@rr.googlevideo.com/", "https://rr..googlevideo.com/?SECRET", + "https://-rr.googlevideo.com/?SECRET", "https://rr.googlevideo.com./?SECRET", + "https://rr.googlevideo.com:80/?SECRET", "https://rr.googlevideo.com:/?SECRET", "https://%SECRET", + } { + encoded, _ := json.Marshal(rawURL) + fixtures = append(fixtures, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":`+string(encoded)+`}}`) + } + for _, fixture := range fixtures { + var stdout, stderr bytes.Buffer + runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error { + io.WriteString(diagnostics, "SECRET") + _, err := io.WriteString(out, fixture) + return err + } + if code := discoverYoutube(discoveryArgs(), &stdout, &stderr, runner); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 { + t.Errorf("fixture %q: code=%d stdout=%q stderr=%q", fixture, code, &stdout, &stderr) + } + if strings.Contains(stderr.String(), "SECRET") || strings.Contains(stderr.String(), "https://") { + t.Error("response leaked") + } + } + upper := strings.Replace(discoveryFixture, "rr16---sn-n8v7znsz.googlevideo.com", "RR16---SN-N8V7ZNSZ.GOOGLEVIDEO.COM:443", 1) + if host, err := playerHostname([]byte(upper)); err != nil || host != "rr16---sn-n8v7znsz.googlevideo.com" { + t.Fatalf("hostname normalization: %q %v", host, err) + } +} + +func TestDiscoverYoutubeRequestFailures(t *testing.T) { + for _, mode := range []string{"runner", "stdout limit", "stderr limit", "write result"} { + t.Run(mode, func(t *testing.T) { + var stdout, stderr bytes.Buffer + runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error { + switch mode { + case "runner": + io.WriteString(out, discoveryFixture) + io.WriteString(diagnostics, "https://signed.example/?SECRET") + return errors.New("SECRET socks5://user:password@localhost:1080") + case "stdout limit": + // Ignore the write error to ensure the command still fails closed. + out.Write(bytes.Repeat([]byte("S"), playerResponseLimit+1)) + case "stderr limit": + diagnostics.Write(bytes.Repeat([]byte("S"), (64<<10)+1)) + } + _, err := io.WriteString(out, discoveryFixture) + return err + } + var dest io.Writer = &stdout + if mode == "write result" { + dest = failingWriter{} + } + if code := discoverYoutube(discoveryArgs(), dest, &stderr, runner); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, &stdout, &stderr) + } + for _, forbidden := range []string{"SECRET", "https://", "password", "localhost"} { + if strings.Contains(stderr.String(), forbidden) { + t.Errorf("leaked %s", forbidden) + } + } + }) + } +} + +func TestPlayerOutputBound(t *testing.T) { + var dst bytes.Buffer + w := &playerBoundedWriter{dst: &dst, limit: 4} + if n, err := w.Write([]byte("1234")); n != 4 || err != nil { + t.Fatal("exact limit must be accepted") + } + if n, err := w.Write([]byte("5")); n != 0 || err == nil || !w.exceeded || dst.String() != "1234" { + t.Fatal("overflow must not be retained") + } + if _, err := playerHostname(bytes.Repeat([]byte(" "), playerResponseLimit+1)); err == nil { + t.Fatal("oversized parser input accepted") + } +} diff --git a/cmd/zapret-autotune/main.go b/cmd/zapret-autotune/main.go index 0a8f6ab..16aaa38 100644 --- a/cmd/zapret-autotune/main.go +++ b/cmd/zapret-autotune/main.go @@ -12,7 +12,8 @@ Commands: status Show latest recorded tune run/proposal (not live service health) history [--limit N] List recorded tune attempts (default 20; proposals not active) strategies youtube List recorded proposals (not active), newest first - discover youtube Refuse discovery until provider/config is agreed + discover youtube --video-id ID --proxy socks5://HOST:PORT + Discover cache hostname only (no media-transfer check) check youtube Refuse checks until probe URLs/video IDs are configured and agreed tune youtube Test youtube.com frontend only; print options without applying render Select normalized offline results and print NFQWS2_OPT @@ -62,9 +63,8 @@ func run(args []string, stdout, stderr io.Writer) int { return 0 } } - if len(args) == 2 && args[0] == "discover" && args[1] == "youtube" { - fmt.Fprintln(stderr, "error: discover youtube is unavailable: YouTube provider/config is not implemented; probe URLs/video IDs and API credentials/cookie requirements have not been configured or agreed. No network requests were made; no endpoints were generated.") - return 1 + if len(args) >= 2 && args[0] == "discover" && args[1] == "youtube" { + return discoverYoutube(args[2:], stdout, stderr, runPlayerCurl) } if len(args) == 2 && args[0] == "check" && args[1] == "youtube" { fmt.Fprintln(stderr, "error: check youtube is unavailable: YouTube probe URLs/video IDs have not been configured or agreed. No network requests were made; production health was not checked.") diff --git a/cmd/zapret-autotune/main_test.go b/cmd/zapret-autotune/main_test.go index 837ff43..92b1d7a 100644 --- a/cmd/zapret-autotune/main_test.go +++ b/cmd/zapret-autotune/main_test.go @@ -18,7 +18,7 @@ func TestRun(t *testing.T) { err string }{ {"status", []string{"status"}, 0, "apply: BLOCKED", ""}, - {"discovery refused", []string{"discover", "youtube"}, 1, "", "probe URLs/video IDs and API credentials/cookie requirements"}, + {"discovery missing flags", []string{"discover", "youtube"}, 2, "", "requires --video-id"}, {"check refused", []string{"check", "youtube"}, 1, "", "YouTube probe URLs/video IDs have not been configured or agreed"}, {"unsupported check provider", []string{"check", "other"}, 2, "", usage}, {"missing check provider", []string{"check"}, 2, "", usage}, @@ -29,7 +29,7 @@ func TestRun(t *testing.T) { {"unsupported provider", []string{"discover", "other"}, 2, "", usage}, {"missing provider", []string{"discover"}, 2, "", usage}, {"extra status argument", []string{"status", "extra"}, 2, "", usage}, - {"extra discovery argument", []string{"discover", "youtube", "extra"}, 2, "", usage}, + {"extra discovery argument", []string{"discover", "youtube", "extra"}, 2, "", "requires --video-id"}, {"unsupported tune provider", []string{"tune", "other"}, 2, "", usage}, {"apply unavailable", []string{"apply"}, 2, "", usage}, {"disruption flag rejected", []string{"tune", "--allow-disruption"}, 2, "", usage}, @@ -51,9 +51,6 @@ func TestRun(t *testing.T) { if tt.name == "status" && !strings.Contains(stdout.String(), "NOT CHECKED") { t.Error("status must not claim production health") } - if tt.name == "discovery refused" && !strings.Contains(stderr.String(), "no endpoints were generated") { - t.Error("discovery must explicitly refuse endpoint generation") - } if tt.name == "check refused" { for _, want := range []string{"No network requests were made", "production health was not checked"} { if !strings.Contains(stderr.String(), want) {