fix: discover avatar metadata from trusted watch pages

Fall back only on missing homepage metadata, preserve private bounded inventory and fail-closed error classification. Record supplied field results without claiming image or family qualification.

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-12 21:21:19 +03:00
1 parent a60f62de22
commit 50a78608ce
6 files changed
+231 -29

No files matched your search

+43 -17
View File
@@ -42,6 +42,15 @@ func (e *managedResourceFailure) Unwrap() error { return e.Err }
// generic fatal wrapper at a live-health boundary. Missing resources remain
// DEGRADED; joined guard/infrastructure/cancellation failures retain their type.
func managedResourceHealthError(ctx context.Context, err error) error {
// Only a direct, known parser absence is missing inventory. Never flatten
// joined guard/infrastructure failures into an unrelated image absence.
if ctx.Err() == nil {
if failure, ok := err.(*managedResourceFailure); ok && failure.ProbeKind == "avatar" && failure.Host == "yt3.ggpht.com" {
if code, ok := failure.Err.(avatarMetadataError); ok && (code == "markerAbsent" || code == "jsonInvalid" || code == "noAllowedAvatar") {
return &managedResourceFailure{ProbeKind: failure.ProbeKind, Host: failure.Host}
}
}
}
var failure *managedResourceFailure
var collector *resourceError
if ctx.Err() == nil && errors.As(err, &failure) && (failure.ProbeKind == "media" || assetEndpoint(failure.Host) && failure.ProbeKind == managedImageProbeKind(failure.Host)) && errors.As(failure.Err, &collector) && managedResourceTransferNonpass(err, assetEndpoint(failure.Host)) {
@@ -87,24 +96,41 @@ func managedServiceResources(cfg healthConfig, player func(context.Context) ([]b
if err != nil {
i.resourceErr = &managedResourceFailure{"media", "unknown", err}
}
bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
defer cancel()
var output bytes.Buffer
out := &playerBoundedWriter{dst: &output, limit: 5 << 20}
// Fixed root origin; no redirects, credentials, user URL or diagnostics.
err = page(bounded, []string{"--disable", "--silent", "--show-error", "--fail", "--noproxy", "", "--proxy", cfg.SOCKSProxy, "--proto", "=https", "--max-time", "30", "--max-filesize", "5242880", "https://www.youtube.com/"}, out, io.Discard)
if ctx.Err() != nil {
return i, managedRecoveryContextError(ctx, err)
}
if err != nil || bounded.Err() != nil || out.exceeded {
cause := errors.Join(err, bounded.Err())
if cause == nil || out.exceeded {
cause = errors.Join(cause, errors.New("fresh root metadata request failed"))
for attempt := 0; attempt < 2; attempt++ {
origin := "https://www.youtube.com/"
if attempt == 1 {
if !videoIDPattern.MatchString(cfg.VideoID) {
return i, &managedRecoveryFatal{Err: errors.New("invalid video ID")}
}
origin = "https://www.youtube.com/watch?v=" + cfg.VideoID
}
bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
var output bytes.Buffer
out := &playerBoundedWriter{dst: &output, limit: 5 << 20}
// Fixed trusted WWW origin; no redirects, credentials or diagnostics.
err = page(bounded, []string{"--disable", "--silent", "--show-error", "--fail", "--noproxy", "", "--proxy", cfg.SOCKSProxy, "--proto", "=https", "--max-time", "30", "--max-filesize", "5242880", origin}, out, io.Discard)
requestErr := bounded.Err()
cancel()
if ctx.Err() != nil {
return i, managedRecoveryContextError(ctx, err)
}
if err != nil || requestErr != nil || out.exceeded {
cause := errors.Join(err, requestErr)
if out.exceeded {
cause = errors.Join(cause, errors.New("fresh metadata exceeds limit"))
}
return i, managedResourceHealthError(ctx, &managedResourceFailure{"avatar", "yt3.ggpht.com", resourceFailure(ctx, cause)})
}
raw, parseErr := avatarResourceFromPage(output.Bytes())
if parseErr == nil {
i.images["yt3.ggpht.com"] = raw
return i, nil
}
// Only missing structural metadata permits one fresh watch request.
var missing avatarMetadataError
if attempt == 1 || !errors.As(parseErr, &missing) || (missing != "markerAbsent" && missing != "noAllowedAvatar") {
return i, managedResourceHealthError(ctx, &managedResourceFailure{"avatar", "yt3.ggpht.com", parseErr})
}
return i, managedResourceHealthError(ctx, &managedResourceFailure{"avatar", "yt3.ggpht.com", resourceFailure(bounded, cause)})
}
if raw, e := avatarResourceFromPage(output.Bytes()); e == nil {
i.images["yt3.ggpht.com"] = raw
}
return i, nil
}
@@ -0,0 +1,149 @@
package main
import (
"context"
"errors"
"io"
"os/exec"
"strings"
"testing"
)
func TestAvatarPageClassification(t *testing.T) {
for _, tc := range []struct{ body, code string }{
{`<html></html>`, "markerAbsent"},
{`var ytInitialData = {broken};`, "jsonInvalid"},
{`var ytInitialData = {"url":"https://yt3.ggpht.com.attacker.invalid/avatar"};`, "noAllowedAvatar"},
{`var ytInitialData = {"url":"http://yt3.ggpht.com/avatar"};`, "noAllowedAvatar"},
{`var ytInitialData = {"url":"https://user@yt3.ggpht.com/avatar"};`, "noAllowedAvatar"},
{`var ytInitialData = {"url":"https://yt3.ggpht.com/avatar?private=fixture"};`, "valid"},
} {
t.Run(tc.code, func(t *testing.T) {
raw, err := avatarResourceFromPage([]byte(tc.body))
if tc.code == "valid" {
if err != nil || raw == "" {
t.Fatal("missing validated resource", err)
}
return
}
var classified avatarMetadataError
if raw != "" || !errors.As(err, &classified) || string(classified) != tc.code {
t.Fatal("unexpected classification", err)
}
})
}
}
func TestManagedAvatarInventoryFallback(t *testing.T) {
exit := func(code string) error { return exec.Command("sh", "-c", "exit "+code).Run() }
for _, tc := range []struct {
name, home, watch, video string
pageErr, watchErr error
calls int
avatar, fatal bool
}{
{name: "homeNoAvatarWatchHasValidatedURL", home: `var ytInitialData = {};`, watch: `var ytInitialData = {"url":"https://yt3.ggpht.com/fresh?private=fixture"};`, calls: 2, avatar: true},
{name: "homeAlreadyValid", home: `var ytInitialData = {"url":"https://yt3.ggpht.com/fresh"};`, calls: 1, avatar: true},
{name: "bothMissing", home: `var ytInitialData = {};`, watch: `<html></html>`, calls: 2},
{name: "malformedHome", home: `var ytInitialData = {bad};`, calls: 1},
{name: "malformedWatch", home: `var ytInitialData = {};`, watch: `var ytInitialData = {bad};`, calls: 2},
{name: "watchHTTPMissing", home: `var ytInitialData = {};`, watchErr: exit("22"), calls: 2},
{name: "watchTimeout", home: `var ytInitialData = {};`, watchErr: exit("28"), calls: 2},
{name: "watchWrongCN", home: `var ytInitialData = {};`, watchErr: exit("60"), calls: 2},
{name: "watchMixedGuard", home: `var ytInitialData = {};`, watchErr: errors.Join(exit("28"), errors.New("owned proxy guard failed")), calls: 2, fatal: true},
{name: "untrustedWatchURL", watch: `var ytInitialData = {"url":"https://yt3.ggpht.com.evil.invalid/avatar"};`, calls: 2},
{name: "invalidVideo", video: "bad&injection", calls: 1, fatal: true},
{name: "timeoutNoFallback", pageErr: exit("28"), calls: 1},
{name: "wrongCNNoFallback", pageErr: exit("60"), calls: 1},
{name: "mixedGuardNoFallback", pageErr: errors.Join(exit("28"), errors.New("owned proxy guard failed")), calls: 1, fatal: true},
{name: "oversizedHomeNoFallback", home: strings.Repeat("x", (5<<20)+1), calls: 1, fatal: true},
} {
t.Run(tc.name, func(t *testing.T) {
cfg := healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080", VideoID: "dQw4w9WgXcQ"}
if tc.video != "" {
cfg.VideoID = tc.video
}
calls := 0
resources := managedServiceResources(cfg, func(context.Context) ([]byte, error) {
return []byte(`{"playabilityStatus":{"status":"OK"},"videoDetails":{"thumbnail":{"thumbnails":[{"url":"https://i.ytimg.com/fresh"}]}},"streamingData":{"formats":[{"url":"https://fresh.googlevideo.com/private","mimeType":"video/mp4"}]}}`), nil
}, func(ctx context.Context, args []string, out, diagnostic io.Writer) error {
calls++
if _, ok := ctx.Deadline(); !ok || diagnostic != io.Discard {
t.Fatal("unbounded metadata or diagnostics")
}
joined := strings.Join(args, " ")
if strings.Contains(joined, "private=") || strings.Contains(joined, "--insecure") || strings.Contains(joined, "--location") || !strings.Contains(joined, "--proto =https") || !strings.Contains(joined, "--max-filesize 5242880") || !strings.Contains(joined, "--proxy "+cfg.SOCKSProxy) {
t.Fatal("unsafe metadata request")
}
body, origin := tc.home, "https://www.youtube.com/"
if calls == 2 {
body, origin = tc.watch, "https://www.youtube.com/watch?v="+cfg.VideoID
}
if args[len(args)-1] != origin {
t.Fatal("unexpected metadata origin")
}
if tc.pageErr != nil {
return tc.pageErr
}
if calls == 2 && tc.watchErr != nil {
return tc.watchErr
}
_, err := io.WriteString(out, body)
return err
})
i, err := resources(context.Background())
var fatal *managedRecoveryFatal
if calls != tc.calls || (i.images["yt3.ggpht.com"] != "") != tc.avatar || errors.As(err, &fatal) != tc.fatal {
t.Fatal("fallback or guard contract", calls, err)
}
if (err == nil) != tc.avatar || i.images["i.ytimg.com"] == "" || i.media.url == "" || i.resourceErr != nil {
t.Fatal("missing avatar masked or partial inventory lost", err)
}
if err != nil && strings.Contains(err.Error(), "private") {
t.Fatal("private metadata leaked")
}
})
}
}
func TestManagedAvatarMetadataHealthClassification(t *testing.T) {
for _, code := range []avatarMetadataError{"markerAbsent", "jsonInvalid", "noAllowedAvatar"} {
failure := &managedResourceFailure{"avatar", "yt3.ggpht.com", code}
err := managedResourceHealthError(context.Background(), failure)
var missing *managedResourceFailure
if !errors.As(err, &missing) || missing.Err != nil {
t.Fatal("known metadata absence not normalized", code)
}
for _, mixed := range []error{
errors.Join(failure, errors.New("owned proxy guard failed")),
&managedResourceFailure{"avatar", "yt3.ggpht.com", errors.Join(code, errors.New("collector unavailable"))},
&managedResourceFailure{"avatar", "yt3.ggpht.com", errors.Join(code, context.Canceled)},
} {
err := managedResourceHealthError(context.Background(), mixed)
if !errors.As(err, &missing) || missing.Err == nil {
t.Fatal("mixed safety failure normalized", code)
}
}
}
unknown := avatarMetadataError("unknown")
err := managedResourceHealthError(context.Background(), &managedResourceFailure{"avatar", "yt3.ggpht.com", unknown})
if !errors.Is(err, unknown) {
t.Fatal("unknown parser error normalized")
}
}
func TestManagedAvatarInventoryCancellationNoFallback(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
calls := 0
resources := managedServiceResources(healthConfig{VideoID: "dQw4w9WgXcQ"}, func(context.Context) ([]byte, error) {
return []byte(`{"playabilityStatus":{"status":"OK"}}`), nil
}, func(ctx context.Context, _ []string, _, _ io.Writer) error {
calls++
return ctx.Err()
})
i, err := resources(ctx)
if calls != 1 || !errors.Is(err, context.Canceled) || i.images["yt3.ggpht.com"] != "" {
t.Fatal("cancellation lost or fallback attempted", calls, err)
}
}
@@ -41,10 +41,11 @@ func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
}
knownTimeout := exec.Command("sh", "-c", "exit 28").Run()
if strings.HasPrefix(mode, "metadata-") {
d.resources = managedServiceResources(healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080"}, func(context.Context) ([]byte, error) {
d.resources = managedServiceResources(healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080", VideoID: "dQw4w9WgXcQ"}, func(context.Context) ([]byte, error) {
return []byte(fmt.Sprintf(`{"playabilityStatus":{"status":"OK"},"streamingData":{"formats":[{"url":%q,"mimeType":"video/mp4"}]}}`, experimentalResourceFixture().url)), nil
}, func(_ context.Context, args []string, out, diagnostic io.Writer) error {
if args[len(args)-1] != "https://www.youtube.com/" || diagnostic != io.Discard {
origin := args[len(args)-1]
if (origin != "https://www.youtube.com/" && !(mode == "metadata-malformed" && origin == "https://www.youtube.com/watch?v=dQw4w9WgXcQ")) || diagnostic != io.Discard {
t.Fatal("metadata origin/diagnostics changed")
}
switch mode {
+13 -3
View File
@@ -33,6 +33,11 @@ func runResourceCurl(ctx context.Context, args []string, stdin io.Reader, out, d
type resourceError struct{ cause error }
// Safe metadata classifications contain neither page content nor resource URLs.
type avatarMetadataError string
func (e avatarMetadataError) Error() string { return string(e) }
func (e *resourceError) PublicError() string { return "resource qualification failed" }
func (e *resourceError) Error() string { return e.PublicError() }
func (e *resourceError) Unwrap() error { return e.cause }
@@ -78,7 +83,7 @@ func avatarResource(metadata []byte) (string, error) {
}
var root interface{}
if json.Unmarshal(metadata, &root) != nil {
return "", errors.New("invalid metadata")
return "", avatarMetadataError("jsonInvalid")
}
var visit func(interface{}, int) string
visit = func(v interface{}, depth int) string {
@@ -109,7 +114,7 @@ func avatarResource(metadata []byte) (string, error) {
if raw := visit(root, 0); raw != "" {
return raw, nil
}
return "", errors.New("avatar unavailable")
return "", avatarMetadataError("noAllowedAvatar")
}
// The WWW page is transient; decode the embedded JSON structurally and never
@@ -118,15 +123,20 @@ func avatarResourceFromPage(page []byte) (string, error) {
if len(page) > 5<<20 {
return "", errors.New("page exceeds limit")
}
found := false
for _, marker := range []string{"var ytInitialData = ", "ytInitialData = ", "window[\"ytInitialData\"] = "} {
if i := bytes.Index(page, []byte(marker)); i >= 0 {
found = true
var object json.RawMessage
if json.NewDecoder(bytes.NewReader(page[i+len(marker):])).Decode(&object) == nil {
return avatarResource(object)
}
}
}
return "", errors.New("structural avatar metadata unavailable")
if found {
return "", avatarMetadataError("jsonInvalid")
}
return "", avatarMetadataError("markerAbsent")
}
var resourceContentRange = regexp.MustCompile(`^bytes 0-([0-9]+)/([0-9]+)$`)