374 lines
14 KiB
Go
374 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"mime"
|
|
"net/http"
|
|
"net/textproto"
|
|
"net/url"
|
|
"os/exec"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// stdin is mandatory: ephemeral signed URLs must never enter argv or diagnostics.
|
|
type subprocessRunner func(context.Context, []string, io.Reader, io.Writer, io.Writer) error
|
|
|
|
func runResourceCurl(ctx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
|
|
cmd := exec.CommandContext(ctx, "curl", args...)
|
|
cmd.Stdin, cmd.Stdout, cmd.Stderr = stdin, out, diagnostic
|
|
return cmd.Run()
|
|
}
|
|
|
|
type resourceError struct{ cause error }
|
|
|
|
func (e *resourceError) PublicError() string { return "resource qualification failed" }
|
|
func (e *resourceError) Error() string { return e.PublicError() }
|
|
func (e *resourceError) Unwrap() error { return e.cause }
|
|
|
|
func resourceFailure(ctx context.Context, err error) error {
|
|
if ctx.Err() != nil {
|
|
err = errors.Join(err, ctx.Err())
|
|
}
|
|
if !managedRecoveryTimeoutSafe(err) {
|
|
err = &managedRecoveryFatal{Err: err}
|
|
}
|
|
return &resourceError{cause: err}
|
|
}
|
|
|
|
func privateResourceURL(raw, host string) (*url.URL, error) {
|
|
if len(raw) == 0 || len(raw) > 16<<10 {
|
|
return nil, errors.New("invalid resource URL")
|
|
}
|
|
for _, c := range raw {
|
|
if c <= 32 || c == 127 {
|
|
return nil, errors.New("invalid resource URL")
|
|
}
|
|
}
|
|
u, err := url.Parse(raw)
|
|
if err != nil || u.Scheme != "https" || u.User != nil || u.Opaque != "" || u.Fragment != "" || u.Hostname() != host || (u.Port() != "" && u.Port() != "443") || strings.HasSuffix(u.Host, ":") || !discoveryHostname(host) {
|
|
return nil, errors.New("invalid resource origin")
|
|
}
|
|
return u, nil
|
|
}
|
|
|
|
func thumbnailResource(videoID string) (string, error) {
|
|
if !videoIDPattern.MatchString(videoID) {
|
|
return "", errors.New("invalid video ID")
|
|
}
|
|
return "https://i.ytimg.com/vi/" + videoID + "/hqdefault.jpg", nil
|
|
}
|
|
|
|
// Parse a fresh structural player/ytInitialData object, not HTML URL regexes.
|
|
// Callers extracting ytInitialData from a WWW page pass that JSON object here.
|
|
func avatarResource(metadata []byte) (string, error) {
|
|
if len(metadata) > playerResponseLimit {
|
|
return "", errors.New("metadata exceeds limit")
|
|
}
|
|
var root interface{}
|
|
if json.Unmarshal(metadata, &root) != nil {
|
|
return "", errors.New("invalid metadata")
|
|
}
|
|
var visit func(interface{}, int) string
|
|
visit = func(v interface{}, depth int) string {
|
|
if depth > 64 {
|
|
return ""
|
|
}
|
|
switch x := v.(type) {
|
|
case map[string]interface{}:
|
|
if raw, ok := x["url"].(string); ok {
|
|
if _, err := privateResourceURL(raw, "yt3.ggpht.com"); err == nil {
|
|
return raw
|
|
}
|
|
}
|
|
for _, child := range x {
|
|
if raw := visit(child, depth+1); raw != "" {
|
|
return raw
|
|
}
|
|
}
|
|
case []interface{}:
|
|
for _, child := range x {
|
|
if raw := visit(child, depth+1); raw != "" {
|
|
return raw
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
if raw := visit(root, 0); raw != "" {
|
|
return raw, nil
|
|
}
|
|
return "", errors.New("avatar unavailable")
|
|
}
|
|
|
|
// The WWW page is transient; decode the embedded JSON structurally and never
|
|
// scan HTML for arbitrary URLs to fetch.
|
|
func avatarResourceFromPage(page []byte) (string, error) {
|
|
if len(page) > 5<<20 {
|
|
return "", errors.New("page exceeds limit")
|
|
}
|
|
for _, marker := range []string{"var ytInitialData = ", "ytInitialData = ", "window[\"ytInitialData\"] = "} {
|
|
if i := bytes.Index(page, []byte(marker)); i >= 0 {
|
|
var object json.RawMessage
|
|
if json.NewDecoder(bytes.NewReader(page[i+len(marker):])).Decode(&object) == nil {
|
|
return avatarResource(object)
|
|
}
|
|
}
|
|
}
|
|
return "", errors.New("structural avatar metadata unavailable")
|
|
}
|
|
|
|
var resourceContentRange = regexp.MustCompile(`^bytes 0-([0-9]+)/([0-9]+)$`)
|
|
|
|
type mediaResource struct{ url, host, kind string }
|
|
type mediaCapability struct {
|
|
OrdinaryURL bool
|
|
SABRSupported bool
|
|
}
|
|
|
|
// SABR is deliberately unknown/not supported, never fabricated as readiness.
|
|
func playerMediaResource(body []byte) (mediaResource, mediaCapability, error) {
|
|
var p struct {
|
|
PlayabilityStatus struct {
|
|
Status string `json:"status"`
|
|
} `json:"playabilityStatus"`
|
|
StreamingData struct {
|
|
Formats []struct {
|
|
URL string `json:"url"`
|
|
MimeType string `json:"mimeType"`
|
|
} `json:"formats"`
|
|
AdaptiveFormats []struct {
|
|
URL string `json:"url"`
|
|
MimeType string `json:"mimeType"`
|
|
} `json:"adaptiveFormats"`
|
|
} `json:"streamingData"`
|
|
}
|
|
if len(body) > playerResponseLimit || json.Unmarshal(body, &p) != nil || p.PlayabilityStatus.Status != "OK" {
|
|
return mediaResource{}, mediaCapability{}, errors.New("invalid playable metadata")
|
|
}
|
|
for _, f := range append(p.StreamingData.Formats, p.StreamingData.AdaptiveFormats...) {
|
|
u, err := url.Parse(f.URL)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
host := u.Hostname()
|
|
if !strings.HasSuffix(host, ".googlevideo.com") {
|
|
continue
|
|
}
|
|
if _, err := privateResourceURL(f.URL, host); err != nil {
|
|
continue
|
|
}
|
|
kind, _, err := mime.ParseMediaType(f.MimeType)
|
|
if err == nil && (kind == "video/mp4" || kind == "audio/mp4") {
|
|
return mediaResource{f.URL, host, kind}, mediaCapability{OrdinaryURL: true}, nil
|
|
}
|
|
}
|
|
return mediaResource{}, mediaCapability{}, nil
|
|
}
|
|
|
|
const resourceTrailer = "\nAUTOTUNE_RESOURCE_METRICS:"
|
|
|
|
func collectResourceTransfer(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner) (assetTransfer, http.Header, error) {
|
|
fail := func(err error) (assetTransfer, http.Header, error) {
|
|
return assetTransfer{}, nil, resourceFailure(ctx, err)
|
|
}
|
|
if !validDiscoveryProxy(proxy) || runner == nil {
|
|
return fail(errors.New("invalid owned proxy or runner"))
|
|
}
|
|
if _, err := privateResourceURL(raw, host); err != nil {
|
|
return fail(err)
|
|
}
|
|
version, tlsFlag, maxTLS := uint16(0x0303), "--tlsv1.2", "1.2"
|
|
if protocol == "tls13" {
|
|
version, tlsFlag, maxTLS = 0x0304, "--tlsv1.3", "1.3"
|
|
} else if protocol != "tls12" {
|
|
return fail(errors.New("invalid TLS protocol"))
|
|
}
|
|
limit := 5 << 20
|
|
if media {
|
|
limit = 65536
|
|
}
|
|
// Qualification uses HTTP/1.1, not browser HTTP/2. --raw preserves native
|
|
// transfer framing so net/http, rather than curl and Go twice, decodes it.
|
|
args := []string{"--disable", "--silent", "--show-error", "--verbose", "--http1.1", "--raw", "--noproxy", "", "--proxy", proxy, "--proto", "=https", tlsFlag, "--tls-max", maxTLS, "--max-time", "30", "--max-filesize", strconv.Itoa(limit), "--suppress-connect-headers", "--include", "--write-out", resourceTrailer + "%{json}", "--config", "-"}
|
|
if media {
|
|
args = append(args, "--range", "0-65535")
|
|
}
|
|
// curl config quotes are not JSON quotes; reject controls above, escape only these.
|
|
config := "url = \"" + strings.NewReplacer("\\", "\\\\", "\"", "\\\"").Replace(raw) + "\"\n"
|
|
childCtx, cancel := context.WithTimeout(ctx, 35*time.Second)
|
|
defer cancel()
|
|
var output, diagnostic bytes.Buffer
|
|
out := &playerBoundedWriter{dst: &output, limit: limit + (96 << 10)}
|
|
diag := &playerBoundedWriter{dst: &diagnostic, limit: 64 << 10}
|
|
err := runner(childCtx, args, strings.NewReader(config), out, diag)
|
|
if out.exceeded || diag.exceeded {
|
|
err = errors.Join(err, errors.New("resource output exceeds limit"))
|
|
}
|
|
if err != nil || childCtx.Err() != nil {
|
|
return fail(errors.Join(err, childCtx.Err()))
|
|
}
|
|
i := bytes.LastIndex(output.Bytes(), []byte(resourceTrailer))
|
|
if i < 0 {
|
|
return fail(errors.New("missing transfer metrics"))
|
|
}
|
|
var metrics struct {
|
|
URL string `json:"url_effective"`
|
|
Verify *int `json:"ssl_verify_result"`
|
|
Redirects *int `json:"num_redirects"`
|
|
Status int `json:"http_code"`
|
|
Time float64 `json:"time_total"`
|
|
Speed float64 `json:"speed_download"`
|
|
Size float64 `json:"size_download"`
|
|
}
|
|
if json.Unmarshal(output.Bytes()[i+len(resourceTrailer):], &metrics) != nil {
|
|
return fail(errors.New("invalid transfer metrics"))
|
|
}
|
|
if _, err := privateResourceURL(metrics.URL, host); err != nil || metrics.URL != raw || metrics.Verify == nil || *metrics.Verify != 0 || metrics.Redirects == nil || *metrics.Redirects != 0 {
|
|
return fail(errors.New("resource origin verification failed"))
|
|
}
|
|
actualTLS := "SSL connection using TLSv1.2"
|
|
if version == 0x0304 {
|
|
actualTLS = "SSL connection using TLSv1.3"
|
|
}
|
|
otherTLS := "SSL connection using TLSv1.3"
|
|
if version == 0x0304 {
|
|
otherTLS = "SSL connection using TLSv1.2"
|
|
}
|
|
if !strings.Contains(diagnostic.String(), actualTLS) || strings.Contains(diagnostic.String(), otherTLS) {
|
|
return fail(errors.New("actual TLS version mismatch"))
|
|
}
|
|
wire := output.Bytes()[:i]
|
|
headerEnd := bytes.Index(wire, []byte("\r\n\r\n"))
|
|
statusEnd := bytes.Index(wire, []byte("\r\n"))
|
|
if headerEnd < 0 || headerEnd > 64<<10 || statusEnd < 0 {
|
|
return fail(errors.New("missing or oversized response headers"))
|
|
}
|
|
h, err := textproto.NewReader(bufio.NewReader(bytes.NewReader(wire[statusEnd+2 : headerEnd+4]))).ReadMIMEHeader()
|
|
if err != nil {
|
|
return fail(errors.New("invalid response headers"))
|
|
}
|
|
for _, key := range []string{"Content-Type", "Content-Length", "Transfer-Encoding", "Content-Range"} {
|
|
if len(h.Values(key)) > 1 {
|
|
return fail(errors.New("duplicate response framing headers"))
|
|
}
|
|
}
|
|
if h.Get("Transfer-Encoding") != "" && (h.Get("Transfer-Encoding") != "chunked" || h.Get("Content-Length") != "") {
|
|
return fail(errors.New("ambiguous response framing"))
|
|
}
|
|
r := bufio.NewReader(bytes.NewReader(wire))
|
|
response, err := http.ReadResponse(r, nil)
|
|
if err != nil {
|
|
return fail(errors.New("invalid response headers"))
|
|
}
|
|
defer response.Body.Close()
|
|
// CONNECT headers are suppressed by curl. Interim/duplicate responses and
|
|
// unrequested versions are outside this contract, never heuristically skipped.
|
|
if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || response.Header.Get("Content-Type") == "" {
|
|
return fail(errors.New("unsupported response contract"))
|
|
}
|
|
body, err := io.ReadAll(io.LimitReader(response.Body, int64(limit+1)))
|
|
// With --raw curl counts the wire body, including chunk framing; the image
|
|
// limit and durable byte identity apply to the fully decoded payload instead.
|
|
wireSize := len(wire) - headerEnd - 4
|
|
_, remaining := r.Peek(1)
|
|
if err != nil || remaining != io.EOF || len(body) == 0 || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status {
|
|
return fail(errors.New("incomplete or oversized resource body"))
|
|
}
|
|
for _, n := range []float64{metrics.Time, metrics.Speed} {
|
|
if n <= 0 || math.IsNaN(n) || math.IsInf(n, 0) {
|
|
return fail(errors.New("invalid resource metrics"))
|
|
}
|
|
}
|
|
contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
return fail(errors.New("invalid content type"))
|
|
}
|
|
t := assetTransfer{EffectiveHost: host, SSLVerifyResult: metrics.Verify, Redirects: *metrics.Redirects, TLSVersion: version, Status: metrics.Status, ContentType: contentType, Body: body, LatencyMS: metrics.Time * 1000, SpeedKBPS: metrics.Speed / 1024}
|
|
return t, response.Header, nil
|
|
}
|
|
|
|
// The ordinary 3/3 sampler is shared by image and media qualification.
|
|
func sampleResource(ctx context.Context, transfer func(context.Context) (assetTransfer, error)) ([]assetTransfer, error) {
|
|
var samples []assetTransfer
|
|
for i := 0; i < 3; i++ {
|
|
t, err := transfer(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
samples = append(samples, t)
|
|
}
|
|
return samples, nil
|
|
}
|
|
|
|
func collectAssetMeasurement(ctx context.Context, proxy, raw, host, protocol, args string, runner subprocessRunner) (assetMeasurement, error) {
|
|
m := assetMeasurement{Source: "image-transfer", Endpoint: host, Protocol: protocol, Args: args}
|
|
if !assetEndpoint(host) {
|
|
return m, resourceFailure(ctx, errors.New("invalid image endpoint"))
|
|
}
|
|
samples, err := sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) {
|
|
t, _, err := collectResourceTransfer(ctx, proxy, raw, host, protocol, false, runner)
|
|
if err == nil {
|
|
if t.Status != 200 {
|
|
err = errors.New("image HTTP status")
|
|
} else {
|
|
err = normalizeAssetBody(&t)
|
|
}
|
|
}
|
|
if err != nil {
|
|
return assetTransfer{}, resourceFailure(ctx, err)
|
|
}
|
|
t.Body = nil
|
|
return t, nil
|
|
})
|
|
m.Transfers = samples
|
|
return m, err
|
|
}
|
|
|
|
func collectMediaSamples(ctx context.Context, proxy string, resource mediaResource, protocol string, runner subprocessRunner) ([]assetTransfer, error) {
|
|
if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") {
|
|
return nil, resourceFailure(ctx, errors.New("ordinary media unavailable"))
|
|
}
|
|
return sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) {
|
|
t, _, err := collectMediaTransfer(ctx, proxy, resource, protocol, runner)
|
|
return t, err
|
|
})
|
|
}
|
|
|
|
// Only normalized range facts leave this collector; URLs, headers and payloads
|
|
// remain transient. Used by both cold sampling and post-warmup sampling.
|
|
func collectMediaTransfer(ctx context.Context, proxy string, resource mediaResource, protocol string, runner subprocessRunner) (assetTransfer, GooglevideoPass, error) {
|
|
if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") {
|
|
return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("ordinary media unavailable"))
|
|
}
|
|
t, h, err := collectResourceTransfer(ctx, proxy, resource.url, resource.host, protocol, true, runner)
|
|
if err != nil {
|
|
return assetTransfer{}, GooglevideoPass{}, err
|
|
}
|
|
parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range"))
|
|
var end, total int64
|
|
if len(parts) == 3 {
|
|
end, _ = strconv.ParseInt(parts[1], 10, 64)
|
|
total, _ = strconv.ParseInt(parts[2], 10, 64)
|
|
}
|
|
if t.Status != 206 || t.ContentType != resource.kind || len(parts) != 3 || end < 15 || end > 65535 || end+1 != int64(len(t.Body)) || total <= end || len(t.Body) < 16 || string(t.Body[4:8]) != "ftyp" || binary.BigEndian.Uint32(t.Body[:4]) < 16 || int(binary.BigEndian.Uint32(t.Body[:4])) > len(t.Body) {
|
|
return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("invalid media range or container"))
|
|
}
|
|
// Retain only normalized byte identity; no signed URL or payload.
|
|
t.Size, t.Format, t.BodyHash = len(t.Body), "mp4", fmt.Sprintf("%x", sha256.Sum256(t.Body))
|
|
t.Body = nil
|
|
p := GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: t.Status, RangeStart: 0, RangeEnd: end, ResourceBytes: total, Bytes: int64(t.Size), FTYP: true, LatencyMS: t.LatencyMS}
|
|
return t, p, nil
|
|
}
|