feat: add autonomous YouTube resource recovery and adaptive GGC

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-11 20:15:16 +03:00
1 parent 8826c591dc
commit b4e1e1e9ca
43 files changed
+5163 -93

No files matched your search

+373
View File
@@ -0,0 +1,373 @@
package main
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"mime"
"net/http"
"net/textproto"
"net/url"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
)
// stdin is mandatory: ephemeral signed URLs must never enter argv or diagnostics.
type subprocessRunner func(context.Context, []string, io.Reader, io.Writer, io.Writer) error
func runResourceCurl(ctx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
cmd := exec.CommandContext(ctx, "curl", args...)
cmd.Stdin, cmd.Stdout, cmd.Stderr = stdin, out, diagnostic
return cmd.Run()
}
type resourceError struct{ cause error }
func (e *resourceError) PublicError() string { return "resource qualification failed" }
func (e *resourceError) Error() string { return e.PublicError() }
func (e *resourceError) Unwrap() error { return e.cause }
func resourceFailure(ctx context.Context, err error) error {
if ctx.Err() != nil {
err = errors.Join(err, ctx.Err())
}
if !managedRecoveryTimeoutSafe(err) {
err = &managedRecoveryFatal{Err: err}
}
return &resourceError{cause: err}
}
func privateResourceURL(raw, host string) (*url.URL, error) {
if len(raw) == 0 || len(raw) > 16<<10 {
return nil, errors.New("invalid resource URL")
}
for _, c := range raw {
if c <= 32 || c == 127 {
return nil, errors.New("invalid resource URL")
}
}
u, err := url.Parse(raw)
if err != nil || u.Scheme != "https" || u.User != nil || u.Opaque != "" || u.Fragment != "" || u.Hostname() != host || (u.Port() != "" && u.Port() != "443") || strings.HasSuffix(u.Host, ":") || !discoveryHostname(host) {
return nil, errors.New("invalid resource origin")
}
return u, nil
}
func thumbnailResource(videoID string) (string, error) {
if !videoIDPattern.MatchString(videoID) {
return "", errors.New("invalid video ID")
}
return "https://i.ytimg.com/vi/" + videoID + "/hqdefault.jpg", nil
}
// Parse a fresh structural player/ytInitialData object, not HTML URL regexes.
// Callers extracting ytInitialData from a WWW page pass that JSON object here.
func avatarResource(metadata []byte) (string, error) {
if len(metadata) > playerResponseLimit {
return "", errors.New("metadata exceeds limit")
}
var root interface{}
if json.Unmarshal(metadata, &root) != nil {
return "", errors.New("invalid metadata")
}
var visit func(interface{}, int) string
visit = func(v interface{}, depth int) string {
if depth > 64 {
return ""
}
switch x := v.(type) {
case map[string]interface{}:
if raw, ok := x["url"].(string); ok {
if _, err := privateResourceURL(raw, "yt3.ggpht.com"); err == nil {
return raw
}
}
for _, child := range x {
if raw := visit(child, depth+1); raw != "" {
return raw
}
}
case []interface{}:
for _, child := range x {
if raw := visit(child, depth+1); raw != "" {
return raw
}
}
}
return ""
}
if raw := visit(root, 0); raw != "" {
return raw, nil
}
return "", errors.New("avatar unavailable")
}
// The WWW page is transient; decode the embedded JSON structurally and never
// scan HTML for arbitrary URLs to fetch.
func avatarResourceFromPage(page []byte) (string, error) {
if len(page) > 5<<20 {
return "", errors.New("page exceeds limit")
}
for _, marker := range []string{"var ytInitialData = ", "ytInitialData = ", "window[\"ytInitialData\"] = "} {
if i := bytes.Index(page, []byte(marker)); i >= 0 {
var object json.RawMessage
if json.NewDecoder(bytes.NewReader(page[i+len(marker):])).Decode(&object) == nil {
return avatarResource(object)
}
}
}
return "", errors.New("structural avatar metadata unavailable")
}
var resourceContentRange = regexp.MustCompile(`^bytes 0-([0-9]+)/([0-9]+)$`)
type mediaResource struct{ url, host, kind string }
type mediaCapability struct {
OrdinaryURL bool
SABRSupported bool
}
// SABR is deliberately unknown/not supported, never fabricated as readiness.
func playerMediaResource(body []byte) (mediaResource, mediaCapability, error) {
var p struct {
PlayabilityStatus struct {
Status string `json:"status"`
} `json:"playabilityStatus"`
StreamingData struct {
Formats []struct {
URL string `json:"url"`
MimeType string `json:"mimeType"`
} `json:"formats"`
AdaptiveFormats []struct {
URL string `json:"url"`
MimeType string `json:"mimeType"`
} `json:"adaptiveFormats"`
} `json:"streamingData"`
}
if len(body) > playerResponseLimit || json.Unmarshal(body, &p) != nil || p.PlayabilityStatus.Status != "OK" {
return mediaResource{}, mediaCapability{}, errors.New("invalid playable metadata")
}
for _, f := range append(p.StreamingData.Formats, p.StreamingData.AdaptiveFormats...) {
u, err := url.Parse(f.URL)
if err != nil {
continue
}
host := u.Hostname()
if !strings.HasSuffix(host, ".googlevideo.com") {
continue
}
if _, err := privateResourceURL(f.URL, host); err != nil {
continue
}
kind, _, err := mime.ParseMediaType(f.MimeType)
if err == nil && (kind == "video/mp4" || kind == "audio/mp4") {
return mediaResource{f.URL, host, kind}, mediaCapability{OrdinaryURL: true}, nil
}
}
return mediaResource{}, mediaCapability{}, nil
}
const resourceTrailer = "\nAUTOTUNE_RESOURCE_METRICS:"
func collectResourceTransfer(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner) (assetTransfer, http.Header, error) {
fail := func(err error) (assetTransfer, http.Header, error) {
return assetTransfer{}, nil, resourceFailure(ctx, err)
}
if !validDiscoveryProxy(proxy) || runner == nil {
return fail(errors.New("invalid owned proxy or runner"))
}
if _, err := privateResourceURL(raw, host); err != nil {
return fail(err)
}
version, tlsFlag, maxTLS := uint16(0x0303), "--tlsv1.2", "1.2"
if protocol == "tls13" {
version, tlsFlag, maxTLS = 0x0304, "--tlsv1.3", "1.3"
} else if protocol != "tls12" {
return fail(errors.New("invalid TLS protocol"))
}
limit := 5 << 20
if media {
limit = 65536
}
// Qualification uses HTTP/1.1, not browser HTTP/2. --raw preserves native
// transfer framing so net/http, rather than curl and Go twice, decodes it.
args := []string{"--disable", "--silent", "--show-error", "--verbose", "--http1.1", "--raw", "--noproxy", "", "--proxy", proxy, "--proto", "=https", tlsFlag, "--tls-max", maxTLS, "--max-time", "30", "--max-filesize", strconv.Itoa(limit), "--suppress-connect-headers", "--include", "--write-out", resourceTrailer + "%{json}", "--config", "-"}
if media {
args = append(args, "--range", "0-65535")
}
// curl config quotes are not JSON quotes; reject controls above, escape only these.
config := "url = \"" + strings.NewReplacer("\\", "\\\\", "\"", "\\\"").Replace(raw) + "\"\n"
childCtx, cancel := context.WithTimeout(ctx, 35*time.Second)
defer cancel()
var output, diagnostic bytes.Buffer
out := &playerBoundedWriter{dst: &output, limit: limit + (96 << 10)}
diag := &playerBoundedWriter{dst: &diagnostic, limit: 64 << 10}
err := runner(childCtx, args, strings.NewReader(config), out, diag)
if out.exceeded || diag.exceeded {
err = errors.Join(err, errors.New("resource output exceeds limit"))
}
if err != nil || childCtx.Err() != nil {
return fail(errors.Join(err, childCtx.Err()))
}
i := bytes.LastIndex(output.Bytes(), []byte(resourceTrailer))
if i < 0 {
return fail(errors.New("missing transfer metrics"))
}
var metrics struct {
URL string `json:"url_effective"`
Verify *int `json:"ssl_verify_result"`
Redirects *int `json:"num_redirects"`
Status int `json:"http_code"`
Time float64 `json:"time_total"`
Speed float64 `json:"speed_download"`
Size float64 `json:"size_download"`
}
if json.Unmarshal(output.Bytes()[i+len(resourceTrailer):], &metrics) != nil {
return fail(errors.New("invalid transfer metrics"))
}
if _, err := privateResourceURL(metrics.URL, host); err != nil || metrics.URL != raw || metrics.Verify == nil || *metrics.Verify != 0 || metrics.Redirects == nil || *metrics.Redirects != 0 {
return fail(errors.New("resource origin verification failed"))
}
actualTLS := "SSL connection using TLSv1.2"
if version == 0x0304 {
actualTLS = "SSL connection using TLSv1.3"
}
otherTLS := "SSL connection using TLSv1.3"
if version == 0x0304 {
otherTLS = "SSL connection using TLSv1.2"
}
if !strings.Contains(diagnostic.String(), actualTLS) || strings.Contains(diagnostic.String(), otherTLS) {
return fail(errors.New("actual TLS version mismatch"))
}
wire := output.Bytes()[:i]
headerEnd := bytes.Index(wire, []byte("\r\n\r\n"))
statusEnd := bytes.Index(wire, []byte("\r\n"))
if headerEnd < 0 || headerEnd > 64<<10 || statusEnd < 0 {
return fail(errors.New("missing or oversized response headers"))
}
h, err := textproto.NewReader(bufio.NewReader(bytes.NewReader(wire[statusEnd+2 : headerEnd+4]))).ReadMIMEHeader()
if err != nil {
return fail(errors.New("invalid response headers"))
}
for _, key := range []string{"Content-Type", "Content-Length", "Transfer-Encoding", "Content-Range"} {
if len(h.Values(key)) > 1 {
return fail(errors.New("duplicate response framing headers"))
}
}
if h.Get("Transfer-Encoding") != "" && (h.Get("Transfer-Encoding") != "chunked" || h.Get("Content-Length") != "") {
return fail(errors.New("ambiguous response framing"))
}
r := bufio.NewReader(bytes.NewReader(wire))
response, err := http.ReadResponse(r, nil)
if err != nil {
return fail(errors.New("invalid response headers"))
}
defer response.Body.Close()
// CONNECT headers are suppressed by curl. Interim/duplicate responses and
// unrequested versions are outside this contract, never heuristically skipped.
if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || response.Header.Get("Content-Type") == "" {
return fail(errors.New("unsupported response contract"))
}
body, err := io.ReadAll(io.LimitReader(response.Body, int64(limit+1)))
// With --raw curl counts the wire body, including chunk framing; the image
// limit and durable byte identity apply to the fully decoded payload instead.
wireSize := len(wire) - headerEnd - 4
_, remaining := r.Peek(1)
if err != nil || remaining != io.EOF || len(body) == 0 || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status {
return fail(errors.New("incomplete or oversized resource body"))
}
for _, n := range []float64{metrics.Time, metrics.Speed} {
if n <= 0 || math.IsNaN(n) || math.IsInf(n, 0) {
return fail(errors.New("invalid resource metrics"))
}
}
contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type"))
if err != nil {
return fail(errors.New("invalid content type"))
}
t := assetTransfer{EffectiveHost: host, SSLVerifyResult: metrics.Verify, Redirects: *metrics.Redirects, TLSVersion: version, Status: metrics.Status, ContentType: contentType, Body: body, LatencyMS: metrics.Time * 1000, SpeedKBPS: metrics.Speed / 1024}
return t, response.Header, nil
}
// The ordinary 3/3 sampler is shared by image and media qualification.
func sampleResource(ctx context.Context, transfer func(context.Context) (assetTransfer, error)) ([]assetTransfer, error) {
var samples []assetTransfer
for i := 0; i < 3; i++ {
t, err := transfer(ctx)
if err != nil {
return nil, err
}
samples = append(samples, t)
}
return samples, nil
}
func collectAssetMeasurement(ctx context.Context, proxy, raw, host, protocol, args string, runner subprocessRunner) (assetMeasurement, error) {
m := assetMeasurement{Source: "image-transfer", Endpoint: host, Protocol: protocol, Args: args}
if !assetEndpoint(host) {
return m, resourceFailure(ctx, errors.New("invalid image endpoint"))
}
samples, err := sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) {
t, _, err := collectResourceTransfer(ctx, proxy, raw, host, protocol, false, runner)
if err == nil {
if t.Status != 200 {
err = errors.New("image HTTP status")
} else {
err = normalizeAssetBody(&t)
}
}
if err != nil {
return assetTransfer{}, resourceFailure(ctx, err)
}
t.Body = nil
return t, nil
})
m.Transfers = samples
return m, err
}
func collectMediaSamples(ctx context.Context, proxy string, resource mediaResource, protocol string, runner subprocessRunner) ([]assetTransfer, error) {
if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") {
return nil, resourceFailure(ctx, errors.New("ordinary media unavailable"))
}
return sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) {
t, _, err := collectMediaTransfer(ctx, proxy, resource, protocol, runner)
return t, err
})
}
// Only normalized range facts leave this collector; URLs, headers and payloads
// remain transient. Used by both cold sampling and post-warmup sampling.
func collectMediaTransfer(ctx context.Context, proxy string, resource mediaResource, protocol string, runner subprocessRunner) (assetTransfer, GooglevideoPass, error) {
if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") {
return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("ordinary media unavailable"))
}
t, h, err := collectResourceTransfer(ctx, proxy, resource.url, resource.host, protocol, true, runner)
if err != nil {
return assetTransfer{}, GooglevideoPass{}, err
}
parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range"))
var end, total int64
if len(parts) == 3 {
end, _ = strconv.ParseInt(parts[1], 10, 64)
total, _ = strconv.ParseInt(parts[2], 10, 64)
}
if t.Status != 206 || t.ContentType != resource.kind || len(parts) != 3 || end < 15 || end > 65535 || end+1 != int64(len(t.Body)) || total <= end || len(t.Body) < 16 || string(t.Body[4:8]) != "ftyp" || binary.BigEndian.Uint32(t.Body[:4]) < 16 || int(binary.BigEndian.Uint32(t.Body[:4])) > len(t.Body) {
return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("invalid media range or container"))
}
// Retain only normalized byte identity; no signed URL or payload.
t.Size, t.Format, t.BodyHash = len(t.Body), "mp4", fmt.Sprintf("%x", sha256.Sum256(t.Body))
t.Body = nil
p := GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: t.Status, RangeStart: 0, RangeEnd: end, ResourceBytes: total, Bytes: int64(t.Size), FTYP: true, LatencyMS: t.LatencyMS}
return t, p, nil
}