fix: discover avatar metadata from trusted watch pages
Fall back only on missing homepage metadata, preserve private bounded inventory and fail-closed error classification. Record supplied field results without claiming image or family qualification. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
a60f62de22
commit
50a78608ce
6 files changed
+231
-29
No files matched your search
@@ -42,6 +42,15 @@ func (e *managedResourceFailure) Unwrap() error { return e.Err }
|
||||
// generic fatal wrapper at a live-health boundary. Missing resources remain
|
||||
// DEGRADED; joined guard/infrastructure/cancellation failures retain their type.
|
||||
func managedResourceHealthError(ctx context.Context, err error) error {
|
||||
// Only a direct, known parser absence is missing inventory. Never flatten
|
||||
// joined guard/infrastructure failures into an unrelated image absence.
|
||||
if ctx.Err() == nil {
|
||||
if failure, ok := err.(*managedResourceFailure); ok && failure.ProbeKind == "avatar" && failure.Host == "yt3.ggpht.com" {
|
||||
if code, ok := failure.Err.(avatarMetadataError); ok && (code == "markerAbsent" || code == "jsonInvalid" || code == "noAllowedAvatar") {
|
||||
return &managedResourceFailure{ProbeKind: failure.ProbeKind, Host: failure.Host}
|
||||
}
|
||||
}
|
||||
}
|
||||
var failure *managedResourceFailure
|
||||
var collector *resourceError
|
||||
if ctx.Err() == nil && errors.As(err, &failure) && (failure.ProbeKind == "media" || assetEndpoint(failure.Host) && failure.ProbeKind == managedImageProbeKind(failure.Host)) && errors.As(failure.Err, &collector) && managedResourceTransferNonpass(err, assetEndpoint(failure.Host)) {
|
||||
@@ -87,24 +96,41 @@ func managedServiceResources(cfg healthConfig, player func(context.Context) ([]b
|
||||
if err != nil {
|
||||
i.resourceErr = &managedResourceFailure{"media", "unknown", err}
|
||||
}
|
||||
bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
|
||||
defer cancel()
|
||||
var output bytes.Buffer
|
||||
out := &playerBoundedWriter{dst: &output, limit: 5 << 20}
|
||||
// Fixed root origin; no redirects, credentials, user URL or diagnostics.
|
||||
err = page(bounded, []string{"--disable", "--silent", "--show-error", "--fail", "--noproxy", "", "--proxy", cfg.SOCKSProxy, "--proto", "=https", "--max-time", "30", "--max-filesize", "5242880", "https://www.youtube.com/"}, out, io.Discard)
|
||||
if ctx.Err() != nil {
|
||||
return i, managedRecoveryContextError(ctx, err)
|
||||
}
|
||||
if err != nil || bounded.Err() != nil || out.exceeded {
|
||||
cause := errors.Join(err, bounded.Err())
|
||||
if cause == nil || out.exceeded {
|
||||
cause = errors.Join(cause, errors.New("fresh root metadata request failed"))
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
origin := "https://www.youtube.com/"
|
||||
if attempt == 1 {
|
||||
if !videoIDPattern.MatchString(cfg.VideoID) {
|
||||
return i, &managedRecoveryFatal{Err: errors.New("invalid video ID")}
|
||||
}
|
||||
origin = "https://www.youtube.com/watch?v=" + cfg.VideoID
|
||||
}
|
||||
bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
|
||||
var output bytes.Buffer
|
||||
out := &playerBoundedWriter{dst: &output, limit: 5 << 20}
|
||||
// Fixed trusted WWW origin; no redirects, credentials or diagnostics.
|
||||
err = page(bounded, []string{"--disable", "--silent", "--show-error", "--fail", "--noproxy", "", "--proxy", cfg.SOCKSProxy, "--proto", "=https", "--max-time", "30", "--max-filesize", "5242880", origin}, out, io.Discard)
|
||||
requestErr := bounded.Err()
|
||||
cancel()
|
||||
if ctx.Err() != nil {
|
||||
return i, managedRecoveryContextError(ctx, err)
|
||||
}
|
||||
if err != nil || requestErr != nil || out.exceeded {
|
||||
cause := errors.Join(err, requestErr)
|
||||
if out.exceeded {
|
||||
cause = errors.Join(cause, errors.New("fresh metadata exceeds limit"))
|
||||
}
|
||||
return i, managedResourceHealthError(ctx, &managedResourceFailure{"avatar", "yt3.ggpht.com", resourceFailure(ctx, cause)})
|
||||
}
|
||||
raw, parseErr := avatarResourceFromPage(output.Bytes())
|
||||
if parseErr == nil {
|
||||
i.images["yt3.ggpht.com"] = raw
|
||||
return i, nil
|
||||
}
|
||||
// Only missing structural metadata permits one fresh watch request.
|
||||
var missing avatarMetadataError
|
||||
if attempt == 1 || !errors.As(parseErr, &missing) || (missing != "markerAbsent" && missing != "noAllowedAvatar") {
|
||||
return i, managedResourceHealthError(ctx, &managedResourceFailure{"avatar", "yt3.ggpht.com", parseErr})
|
||||
}
|
||||
return i, managedResourceHealthError(ctx, &managedResourceFailure{"avatar", "yt3.ggpht.com", resourceFailure(bounded, cause)})
|
||||
}
|
||||
if raw, e := avatarResourceFromPage(output.Bytes()); e == nil {
|
||||
i.images["yt3.ggpht.com"] = raw
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAvatarPageClassification(t *testing.T) {
|
||||
for _, tc := range []struct{ body, code string }{
|
||||
{`<html></html>`, "markerAbsent"},
|
||||
{`var ytInitialData = {broken};`, "jsonInvalid"},
|
||||
{`var ytInitialData = {"url":"https://yt3.ggpht.com.attacker.invalid/avatar"};`, "noAllowedAvatar"},
|
||||
{`var ytInitialData = {"url":"http://yt3.ggpht.com/avatar"};`, "noAllowedAvatar"},
|
||||
{`var ytInitialData = {"url":"https://user@yt3.ggpht.com/avatar"};`, "noAllowedAvatar"},
|
||||
{`var ytInitialData = {"url":"https://yt3.ggpht.com/avatar?private=fixture"};`, "valid"},
|
||||
} {
|
||||
t.Run(tc.code, func(t *testing.T) {
|
||||
raw, err := avatarResourceFromPage([]byte(tc.body))
|
||||
if tc.code == "valid" {
|
||||
if err != nil || raw == "" {
|
||||
t.Fatal("missing validated resource", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
var classified avatarMetadataError
|
||||
if raw != "" || !errors.As(err, &classified) || string(classified) != tc.code {
|
||||
t.Fatal("unexpected classification", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedAvatarInventoryFallback(t *testing.T) {
|
||||
exit := func(code string) error { return exec.Command("sh", "-c", "exit "+code).Run() }
|
||||
for _, tc := range []struct {
|
||||
name, home, watch, video string
|
||||
pageErr, watchErr error
|
||||
calls int
|
||||
avatar, fatal bool
|
||||
}{
|
||||
{name: "homeNoAvatarWatchHasValidatedURL", home: `var ytInitialData = {};`, watch: `var ytInitialData = {"url":"https://yt3.ggpht.com/fresh?private=fixture"};`, calls: 2, avatar: true},
|
||||
{name: "homeAlreadyValid", home: `var ytInitialData = {"url":"https://yt3.ggpht.com/fresh"};`, calls: 1, avatar: true},
|
||||
{name: "bothMissing", home: `var ytInitialData = {};`, watch: `<html></html>`, calls: 2},
|
||||
{name: "malformedHome", home: `var ytInitialData = {bad};`, calls: 1},
|
||||
{name: "malformedWatch", home: `var ytInitialData = {};`, watch: `var ytInitialData = {bad};`, calls: 2},
|
||||
{name: "watchHTTPMissing", home: `var ytInitialData = {};`, watchErr: exit("22"), calls: 2},
|
||||
{name: "watchTimeout", home: `var ytInitialData = {};`, watchErr: exit("28"), calls: 2},
|
||||
{name: "watchWrongCN", home: `var ytInitialData = {};`, watchErr: exit("60"), calls: 2},
|
||||
{name: "watchMixedGuard", home: `var ytInitialData = {};`, watchErr: errors.Join(exit("28"), errors.New("owned proxy guard failed")), calls: 2, fatal: true},
|
||||
{name: "untrustedWatchURL", watch: `var ytInitialData = {"url":"https://yt3.ggpht.com.evil.invalid/avatar"};`, calls: 2},
|
||||
{name: "invalidVideo", video: "bad&injection", calls: 1, fatal: true},
|
||||
{name: "timeoutNoFallback", pageErr: exit("28"), calls: 1},
|
||||
{name: "wrongCNNoFallback", pageErr: exit("60"), calls: 1},
|
||||
{name: "mixedGuardNoFallback", pageErr: errors.Join(exit("28"), errors.New("owned proxy guard failed")), calls: 1, fatal: true},
|
||||
{name: "oversizedHomeNoFallback", home: strings.Repeat("x", (5<<20)+1), calls: 1, fatal: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080", VideoID: "dQw4w9WgXcQ"}
|
||||
if tc.video != "" {
|
||||
cfg.VideoID = tc.video
|
||||
}
|
||||
calls := 0
|
||||
resources := managedServiceResources(cfg, func(context.Context) ([]byte, error) {
|
||||
return []byte(`{"playabilityStatus":{"status":"OK"},"videoDetails":{"thumbnail":{"thumbnails":[{"url":"https://i.ytimg.com/fresh"}]}},"streamingData":{"formats":[{"url":"https://fresh.googlevideo.com/private","mimeType":"video/mp4"}]}}`), nil
|
||||
}, func(ctx context.Context, args []string, out, diagnostic io.Writer) error {
|
||||
calls++
|
||||
if _, ok := ctx.Deadline(); !ok || diagnostic != io.Discard {
|
||||
t.Fatal("unbounded metadata or diagnostics")
|
||||
}
|
||||
joined := strings.Join(args, " ")
|
||||
if strings.Contains(joined, "private=") || strings.Contains(joined, "--insecure") || strings.Contains(joined, "--location") || !strings.Contains(joined, "--proto =https") || !strings.Contains(joined, "--max-filesize 5242880") || !strings.Contains(joined, "--proxy "+cfg.SOCKSProxy) {
|
||||
t.Fatal("unsafe metadata request")
|
||||
}
|
||||
body, origin := tc.home, "https://www.youtube.com/"
|
||||
if calls == 2 {
|
||||
body, origin = tc.watch, "https://www.youtube.com/watch?v="+cfg.VideoID
|
||||
}
|
||||
if args[len(args)-1] != origin {
|
||||
t.Fatal("unexpected metadata origin")
|
||||
}
|
||||
if tc.pageErr != nil {
|
||||
return tc.pageErr
|
||||
}
|
||||
if calls == 2 && tc.watchErr != nil {
|
||||
return tc.watchErr
|
||||
}
|
||||
_, err := io.WriteString(out, body)
|
||||
return err
|
||||
})
|
||||
i, err := resources(context.Background())
|
||||
var fatal *managedRecoveryFatal
|
||||
if calls != tc.calls || (i.images["yt3.ggpht.com"] != "") != tc.avatar || errors.As(err, &fatal) != tc.fatal {
|
||||
t.Fatal("fallback or guard contract", calls, err)
|
||||
}
|
||||
if (err == nil) != tc.avatar || i.images["i.ytimg.com"] == "" || i.media.url == "" || i.resourceErr != nil {
|
||||
t.Fatal("missing avatar masked or partial inventory lost", err)
|
||||
}
|
||||
if err != nil && strings.Contains(err.Error(), "private") {
|
||||
t.Fatal("private metadata leaked")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedAvatarMetadataHealthClassification(t *testing.T) {
|
||||
for _, code := range []avatarMetadataError{"markerAbsent", "jsonInvalid", "noAllowedAvatar"} {
|
||||
failure := &managedResourceFailure{"avatar", "yt3.ggpht.com", code}
|
||||
err := managedResourceHealthError(context.Background(), failure)
|
||||
var missing *managedResourceFailure
|
||||
if !errors.As(err, &missing) || missing.Err != nil {
|
||||
t.Fatal("known metadata absence not normalized", code)
|
||||
}
|
||||
for _, mixed := range []error{
|
||||
errors.Join(failure, errors.New("owned proxy guard failed")),
|
||||
&managedResourceFailure{"avatar", "yt3.ggpht.com", errors.Join(code, errors.New("collector unavailable"))},
|
||||
&managedResourceFailure{"avatar", "yt3.ggpht.com", errors.Join(code, context.Canceled)},
|
||||
} {
|
||||
err := managedResourceHealthError(context.Background(), mixed)
|
||||
if !errors.As(err, &missing) || missing.Err == nil {
|
||||
t.Fatal("mixed safety failure normalized", code)
|
||||
}
|
||||
}
|
||||
}
|
||||
unknown := avatarMetadataError("unknown")
|
||||
err := managedResourceHealthError(context.Background(), &managedResourceFailure{"avatar", "yt3.ggpht.com", unknown})
|
||||
if !errors.Is(err, unknown) {
|
||||
t.Fatal("unknown parser error normalized")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedAvatarInventoryCancellationNoFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
calls := 0
|
||||
resources := managedServiceResources(healthConfig{VideoID: "dQw4w9WgXcQ"}, func(context.Context) ([]byte, error) {
|
||||
return []byte(`{"playabilityStatus":{"status":"OK"}}`), nil
|
||||
}, func(ctx context.Context, _ []string, _, _ io.Writer) error {
|
||||
calls++
|
||||
return ctx.Err()
|
||||
})
|
||||
i, err := resources(ctx)
|
||||
if calls != 1 || !errors.Is(err, context.Canceled) || i.images["yt3.ggpht.com"] != "" {
|
||||
t.Fatal("cancellation lost or fallback attempted", calls, err)
|
||||
}
|
||||
}
|
||||
@@ -41,10 +41,11 @@ func TestExperimentalProductionProbeCollectorClassification(t *testing.T) {
|
||||
}
|
||||
knownTimeout := exec.Command("sh", "-c", "exit 28").Run()
|
||||
if strings.HasPrefix(mode, "metadata-") {
|
||||
d.resources = managedServiceResources(healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080"}, func(context.Context) ([]byte, error) {
|
||||
d.resources = managedServiceResources(healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080", VideoID: "dQw4w9WgXcQ"}, func(context.Context) ([]byte, error) {
|
||||
return []byte(fmt.Sprintf(`{"playabilityStatus":{"status":"OK"},"streamingData":{"formats":[{"url":%q,"mimeType":"video/mp4"}]}}`, experimentalResourceFixture().url)), nil
|
||||
}, func(_ context.Context, args []string, out, diagnostic io.Writer) error {
|
||||
if args[len(args)-1] != "https://www.youtube.com/" || diagnostic != io.Discard {
|
||||
origin := args[len(args)-1]
|
||||
if (origin != "https://www.youtube.com/" && !(mode == "metadata-malformed" && origin == "https://www.youtube.com/watch?v=dQw4w9WgXcQ")) || diagnostic != io.Discard {
|
||||
t.Fatal("metadata origin/diagnostics changed")
|
||||
}
|
||||
switch mode {
|
||||
|
||||
@@ -33,6 +33,11 @@ func runResourceCurl(ctx context.Context, args []string, stdin io.Reader, out, d
|
||||
|
||||
type resourceError struct{ cause error }
|
||||
|
||||
// Safe metadata classifications contain neither page content nor resource URLs.
|
||||
type avatarMetadataError string
|
||||
|
||||
func (e avatarMetadataError) Error() string { return string(e) }
|
||||
|
||||
func (e *resourceError) PublicError() string { return "resource qualification failed" }
|
||||
func (e *resourceError) Error() string { return e.PublicError() }
|
||||
func (e *resourceError) Unwrap() error { return e.cause }
|
||||
@@ -78,7 +83,7 @@ func avatarResource(metadata []byte) (string, error) {
|
||||
}
|
||||
var root interface{}
|
||||
if json.Unmarshal(metadata, &root) != nil {
|
||||
return "", errors.New("invalid metadata")
|
||||
return "", avatarMetadataError("jsonInvalid")
|
||||
}
|
||||
var visit func(interface{}, int) string
|
||||
visit = func(v interface{}, depth int) string {
|
||||
@@ -109,7 +114,7 @@ func avatarResource(metadata []byte) (string, error) {
|
||||
if raw := visit(root, 0); raw != "" {
|
||||
return raw, nil
|
||||
}
|
||||
return "", errors.New("avatar unavailable")
|
||||
return "", avatarMetadataError("noAllowedAvatar")
|
||||
}
|
||||
|
||||
// The WWW page is transient; decode the embedded JSON structurally and never
|
||||
@@ -118,15 +123,20 @@ func avatarResourceFromPage(page []byte) (string, error) {
|
||||
if len(page) > 5<<20 {
|
||||
return "", errors.New("page exceeds limit")
|
||||
}
|
||||
found := false
|
||||
for _, marker := range []string{"var ytInitialData = ", "ytInitialData = ", "window[\"ytInitialData\"] = "} {
|
||||
if i := bytes.Index(page, []byte(marker)); i >= 0 {
|
||||
found = true
|
||||
var object json.RawMessage
|
||||
if json.NewDecoder(bytes.NewReader(page[i+len(marker):])).Decode(&object) == nil {
|
||||
return avatarResource(object)
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", errors.New("structural avatar metadata unavailable")
|
||||
if found {
|
||||
return "", avatarMetadataError("jsonInvalid")
|
||||
}
|
||||
return "", avatarMetadataError("markerAbsent")
|
||||
}
|
||||
|
||||
var resourceContentRange = regexp.MustCompile(`^bytes 0-([0-9]+)/([0-9]+)$`)
|
||||
|
||||
Reference in new issue
Block a user