From e270810f33704b721d71ae3a7bf667b5b955a695 Mon Sep 17 00:00:00 2001 From: Zeusina Date: Thu, 15 Oct 2026 17:25:12 +0300 Subject: [PATCH] refactor: execute adapter plans through shared core Execute the private YouTube qualification plan through core-owned scheduling while retaining worker cleanup, proof sealing and legacy policy boundaries. Preserve unsafe collector failures alongside concurrent cancellation or deadlines. Send bounded Player JSON through stdin instead of argv and update all shared runner callers. Generic controller cutover and remote runtime acceptance remain pending. Co-authored-by: GPT-6 --- RESOURCE_MONITOR.md | 9 +- RESOURCE_PLAN.md | 48 ++++ cmd/zapret-autotune/adapter_executor.go | 125 +++++++++ .../adapter_executor_parity_test.go | 148 +++++++++++ cmd/zapret-autotune/adapter_executor_test.go | 248 ++++++++++++++++++ cmd/zapret-autotune/assets_test.go | 4 +- cmd/zapret-autotune/discover.go | 11 +- cmd/zapret-autotune/discover_test.go | 12 +- cmd/zapret-autotune/googlevideo_test.go | 12 +- cmd/zapret-autotune/health_test.go | 10 +- cmd/zapret-autotune/managed_asset_verifier.go | 2 +- .../managed_avatar_inventory_test.go | 4 +- cmd/zapret-autotune/managed_cli_test.go | 4 +- cmd/zapret-autotune/managed_dns_test.go | 2 +- .../managed_experimental_probe_test.go | 2 +- .../managed_experimental_test.go | 6 +- .../managed_googlevideo_runtime.go | 61 ++--- .../managed_monitor_cli_test.go | 4 +- cmd/zapret-autotune/managed_recovery.go | 4 +- cmd/zapret-autotune/managed_service_test.go | 2 +- .../player_cancellation_test.go | 21 +- cmd/zapret-autotune/player_stdin_test.go | 74 ++++++ cmd/zapret-autotune/recover_test.go | 2 +- cmd/zapret-autotune/service_adapters_test.go | 2 +- 24 files changed, 725 insertions(+), 92 deletions(-) create mode 100644 cmd/zapret-autotune/adapter_executor.go create mode 100644 cmd/zapret-autotune/adapter_executor_parity_test.go create mode 100644 cmd/zapret-autotune/adapter_executor_test.go create mode 100644 cmd/zapret-autotune/player_stdin_test.go diff --git a/RESOURCE_MONITOR.md b/RESOURCE_MONITOR.md index cc9a67e..4d2a58c 100644 --- a/RESOURCE_MONITOR.md +++ b/RESOURCE_MONITOR.md @@ -37,8 +37,13 @@ qualify a candidate. V2 version/kind/expectations invalidate old HMAC rows/proof historic v1 fingerprints remain unchanged. MP4 prefix health is not playback or YouTube proof: three discovered CDNs, >=1024-byte producer samples, warmup, suffix authorization and adaptive policy still belong to the legacy adapter. -Trusted compiled-in kind+version service modules remain a roadmap design, not an -implemented plugin API. Local source/closure review and injected-wire checks are +The private compiled-in YouTube module and core `executeAdapterPlan[T]` bridge +now drive the existing owned qualifier's ordered warmup and steady collection. +They do not add public module configuration or change this monitor's schemas, +controller, suffix authorization or legacy SOCKS5h routing boundary. Generic +controller cutover still needs dynamic-inventory fingerprints and warmup-bearing +generic proof binding; images/avatars and full YouTube READY remain unresolved. +Local source/closure review and injected-wire checks are recorded alongside scoped managed v2 field observations below; full acceptance is not implied. diff --git a/RESOURCE_PLAN.md b/RESOURCE_PLAN.md index 12b5d03..3e3a07c 100644 --- a/RESOURCE_PLAN.md +++ b/RESOURCE_PLAN.md @@ -707,6 +707,54 @@ Player media parsing (`playerMediaResource`) and owned native qualification representative selection remain in the legacy controller; signed MP4 freshness and the host/TLS qualification plan are pure service semantics in the module. +The next private execution bridge is now wired: `verifyGooglevideo` runs the +actual owned-worker collector through `executeAdapterPlan[T]`. Core independently +pins the known module binding and exact three-host/TLS1.2+1.3, 10-attempt/120-second +warmup, three steady samples/protocol and 1024-byte policy before worker startup +and probes. Invalid, duplicate, overflowing or weakened plans fail closed. The +executor preserves host order, alternating warmups, discarded learning successes, +strict measured steady facts and negative/fatal classification. It has no worker, +goroutine, persistence or seal capability; existing core range/ftyp/TLS validation +and post-worker-join proof validation/sealing remain the authority. + +This is a **private execution bridge**, not public adapter configuration or a new +generic service/matrix proof. Dynamic-inventory fingerprints and warmup-bearing +generic proof binding are still required before generic-controller cutover. +The inherited legacy SOCKS5h boundary below is unchanged, not an all-DNS-answer +pinning guarantee. Three-CDN qualification does not itself grant suffix scope; +images/avatars and runtime acceptance remain separate and not READY. + +Player JSON now travels through bounded private stdin (`curl --json @-`), not +process argv; the shared runner passes no stdin for metadata GETs. Video IDs are +validated before execution, POST bytes and headers remain unchanged, and opaque +public errors retain typed cancellation/unsafe-output causes. This is a private +Player transport change, not generic public POST/discovery configuration. + +Bridge-local checks: an unchanged fixture run against archived `f485317` and the +wired producer has normalized proof SHA256 +`fb380324e320a78f1ceb011c59b82b9de4ff4f31f5f032f469a181292987e3b2`. +Only warmup duration, attestation time and private seals are excluded; exact +probe order and cleanup/join are checked. Baseline and focused serial/race logs: +`/tmp/opencode/adapter-executor-{baseline,focused}.log`. These are injected local +checks, not network/native acceptance or a full-suite rerun. + +Cancellation review follow-up: both warmup and steady probe boundaries now use +`managedRecoveryContextError`, preserving concurrent unsafe I/O/output overflow +alongside cancellation/deadline causes. Production collector regressions check +fatal/non-restartable classification, full cleanup/join and no attestation/seal; +plain cancellation remains timeout-safe. A pre-fix Go-overlay negative control +fails all eight mixed-failure cases; the fixed producer retains the hash above. +Actual `sudo` EUID-0 serial full Go checks passed (1806 test/subtest pass events), +as did 227 affected serial/race events, separate module race tests, vet and a +CGO-disabled build. Logs: `/tmp/opencode/adapter-executor-root-full.json`, +`adapter-executor-context-root-race.json`, `adapter-executor-context-focused.log` +and `adapter-executor-context-negative-control.log` in that same directory. +The reviewer-reported earlier `SavedFirstAndFailClosed/nonpass` freshness failure +did not recur in ten focused repeats or this full run; no freshness guards or +fixtures were changed, and its earlier cause is not established as a baseline +flake. Python checks were not rerun in this follow-up. These are local tests, +not remote runtime acceptance, a commit/deployment or a READY claim. + Core origin/TLS validation, shared image/media transfer code, bounded worker lifecycle and post-cleanup/join sealing remain authoritative. Legacy media and metadata routing retains SOCKS5h with the owned encrypted-resolver guard and diff --git a/cmd/zapret-autotune/adapter_executor.go b/cmd/zapret-autotune/adapter_executor.go new file mode 100644 index 0000000..c4b140e --- /dev/null +++ b/cmd/zapret-autotune/adapter_executor.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "errors" + "time" + + "zapret-autotune/internal/adapter" + "zapret-autotune/internal/adapter/youtube" +) + +var errAdapterPlan = errors.New("invalid private adapter execution plan") + +// This is core policy, not module-supplied caps. Adding a binding requires an +// audited execution policy here; even a correctly identified module is untrusted. +func validateAdapterPlan(binding adapter.Binding, plan adapter.Plan, resources map[string]mediaResource) error { + if binding != (youtube.Module{}).Binding() || len(plan.Hosts) != 3 || + plan.Protocols != [2]string{"tls12", "tls13"} || plan.Samples != 3 || + plan.MinBytes != 1024 || plan.WarmupLimit != 10 || plan.WarmupTimeout != 120*time.Second { + return errAdapterPlan + } + if (youtube.Module{}).ValidateHosts(plan.Hosts) != nil { + return errAdapterPlan + } + for _, host := range plan.Hosts { + resource, ok := resources[host] + if !ok || !validDomain(host) || resource.host != host { + return errAdapterPlan + } + if _, err := privateResourceURL(resource.url, host); err != nil { + return errAdapterPlan + } + if !youtube.SignedSampleFresh(youtube.Resource{URL: resource.url, Host: host, Kind: resource.kind}, time.Now(), 120*time.Second) { + return errAdapterPlan + } + } + return nil +} + +// Typed measurements stay private and ephemeral. Only core callbacks collect +// and classify them; modules cannot supply callbacks, attestations or seals. +type adapterMeasurement[T any] struct { + value T + bytes int64 +} + +type adapterHostMeasurements[T any] struct { + host string + warmupAttempts, warmupFailures int + warmupLimit int + warmupMS float64 + samples []T +} + +// executeAdapterPlan owns scheduling only. Its caller owns the worker, transport +// validation and post-join proof validation/sealing. No goroutines or persistence. +func executeAdapterPlan[T any](ctx context.Context, binding adapter.Binding, plan adapter.Plan, resources map[string]mediaResource, + probe func(context.Context, mediaResource, string) (adapterMeasurement[T], error), nonpass func(error) bool, +) ([]adapterHostMeasurements[T], error) { + if err := validateAdapterPlan(binding, plan, resources); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + hosts := make([]adapterHostMeasurements[T], len(plan.Hosts)) + for i, host := range plan.Hosts { + hosts[i].host = host + } + for i := range hosts { + h := &hosts[i] + h.warmupLimit = plan.WarmupLimit + start := time.Now() + warmCtx, cancel := context.WithTimeout(ctx, plan.WarmupTimeout) + learned := map[string]bool{} + for h.warmupAttempts < h.warmupLimit && len(learned) < len(plan.Protocols) && warmCtx.Err() == nil { + protocol := plan.Protocols[h.warmupAttempts%len(plan.Protocols)] + h.warmupAttempts++ + measurement, err := probe(warmCtx, resources[h.host], protocol) + err = managedRecoveryContextError(warmCtx, err) + if err != nil { + h.warmupFailures++ + if !nonpass(err) && (warmCtx.Err() == nil || !managedRecoveryTimeoutSafe(err)) { + cancel() + return hosts, err + } + } else if measurement.bytes < plan.MinBytes { + h.warmupFailures++ + } else { + learned[protocol] = true + } + } + h.warmupMS = float64(time.Since(start).Microseconds()) / 1000 + cancel() + if err := ctx.Err(); err != nil { + return hosts, err + } + if len(learned) != len(plan.Protocols) || h.warmupMS > float64(plan.WarmupTimeout/time.Millisecond) { + return hosts, errCandidateNonpass + } + for _, protocol := range plan.Protocols { + for n := 0; n < plan.Samples; n++ { + if err := ctx.Err(); err != nil { + return hosts, err + } + measurement, err := probe(ctx, resources[h.host], protocol) + err = managedRecoveryContextError(ctx, err) + if ctx.Err() != nil { + return hosts, err + } + if err != nil { + if nonpass(err) { + return hosts, errCandidateNonpass + } + return hosts, err + } + if measurement.bytes < plan.MinBytes { + return hosts, errCandidateNonpass + } + h.samples = append(h.samples, measurement.value) + } + } + } + return hosts, nil +} diff --git a/cmd/zapret-autotune/adapter_executor_parity_test.go b/cmd/zapret-autotune/adapter_executor_parity_test.go new file mode 100644 index 0000000..e3604f8 --- /dev/null +++ b/cmd/zapret-autotune/adapter_executor_parity_test.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "strings" + "testing" + "time" +) + +// Also runnable unchanged against archived f485317. Only wall-clock duration, +// attestation time and process-private seals are excluded from the golden hash. +func TestAdapterExecutorLegacyParity(t *testing.T) { + r, c, resources, events := googlevideoRuntimeFixture(t) + runner := r.curl + var order []string + r.curl = func(ctx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + protocol := "tls12" + if strings.Contains(strings.Join(args, " "), "--tlsv1.3") { + protocol = "tls13" + } + order = append(order, protocol) + if len(order) <= 2 { + return context.DeadlineExceeded + } + return runner(ctx, args, stdin, out, diagnostic) + } + p, err := r.verifyGooglevideo(context.Background(), c, resources) + if err != nil || p.producerSeal == nil { + t.Fatal(err, p) + } + if !strings.HasSuffix(strings.Join(*events, ","), "delete,join") { + t.Fatal(*events) + } + expected := []string{"tls12", "tls13"} + for range c.Endpoints { + expected = append(expected, "tls12", "tls13", "tls12", "tls12", "tls12", "tls13", "tls13", "tls13") + } + if !reflect.DeepEqual(order, expected) { + t.Fatal(order, expected) + } + for i := range p.Hosts { + p.Hosts[i].WarmupMS = 0 + } + p.Attestation.CheckedAt = time.Unix(1700000000, 0).UTC() + raw, err := json.Marshal(p) + if err != nil { + t.Fatal(err) + } + hash := fmt.Sprintf("%x", sha256.Sum256(raw)) + t.Log("normalized proof sha256:", hash) + const baseline = "fb380324e320a78f1ceb011c59b82b9de4ff4f31f5f032f469a181292987e3b2" + if hash != baseline { + t.Fatalf("proof parity changed: %s\n%s", hash, raw) + } +} + +func TestAdapterExecutorProductionCancellationJoinsWithoutSeal(t *testing.T) { + for _, at := range []int{1, 3} { + t.Run(fmt.Sprint(at), func(t *testing.T) { + r, c, resources, events := googlevideoRuntimeFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + runner, calls := r.curl, 0 + r.curl = func(probeCtx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + calls++ + if calls == at { + cancel() + return ctx.Err() + } + return runner(probeCtx, args, stdin, out, diagnostic) + } + p, err := r.verifyGooglevideo(ctx, c, resources) + if !errors.Is(err, context.Canceled) || calls != at || p.Attestation != nil || p.producerSeal != nil || + !strings.HasSuffix(strings.Join(*events, ","), "delete,join") { + t.Fatal(p, err, calls, *events) + } + if !managedRecoveryTimeoutSafe(err) { + t.Fatal("plain cancellation became unsafe", err) + } + }) + } +} + +func TestAdapterExecutorProductionCancellationPreservesUnsafeTransfer(t *testing.T) { + unsafeIO := errors.New("unsafe collector IO") + for _, at := range []int{1, 3} { // first warmup or first steady after both TLS versions learn + for _, deadline := range []bool{false, true} { + for _, overflow := range []bool{false, true} { + t.Run(fmt.Sprintf("probe%d/deadline%t/overflow%t", at, deadline, overflow), func(t *testing.T) { + r, c, resources, events := googlevideoRuntimeFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wantContext := context.Canceled + if deadline { + var stop context.CancelFunc + ctx, stop = context.WithTimeout(ctx, time.Second) + defer stop() + wantContext = context.DeadlineExceeded + } + runner, calls := r.curl, 0 + r.curl = func(probeCtx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + calls++ + if calls != at { + return runner(probeCtx, args, stdin, out, diagnostic) + } + if deadline { + <-ctx.Done() + } else { + cancel() + } + if overflow { + // Exercise the real bounded collector writer, not a forged fatal DTO. + _, writeErr := out.Write(make([]byte, 256<<10)) + return errors.Join(writeErr, ctx.Err()) + } + return errors.Join(unsafeIO, ctx.Err()) + } + p, err := r.verifyGooglevideo(ctx, c, resources) + var fatal *managedRecoveryFatal + if !errors.Is(err, wantContext) || !errors.As(err, &fatal) || managedRecoveryTimeoutSafe(err) || managedResourceTransferNonpass(err, false, false) { + t.Fatalf("lost mixed safety/context error: %v", err) + } + if overflow { + var collected *resourceError + if !errors.As(err, &collected) || collected.reason != reasonOutputBound { + t.Fatal("lost collector overflow", err) + } + } else if !errors.Is(err, unsafeIO) { + t.Fatal("lost unsafe IO cause", err) + } + if calls != at || p.Attestation != nil || p.producerSeal != nil || + !strings.HasSuffix(strings.Join(*events, ","), "delete,join") { + t.Fatal(p, err, calls, *events) + } + if at == 3 && (p.Hosts[0].WarmupAttempts != 2 || len(p.Hosts[0].Passes) != 0) { + t.Fatal("learning leaked into steady proof", p.Hosts) + } + }) + } + } + } +} diff --git a/cmd/zapret-autotune/adapter_executor_test.go b/cmd/zapret-autotune/adapter_executor_test.go new file mode 100644 index 0000000..792a677 --- /dev/null +++ b/cmd/zapret-autotune/adapter_executor_test.go @@ -0,0 +1,248 @@ +package main + +import ( + "context" + "errors" + "math" + "reflect" + "testing" + "time" + + "zapret-autotune/internal/adapter" + "zapret-autotune/internal/adapter/youtube" +) + +func adapterExecutorFixture(t *testing.T) (adapter.Binding, adapter.Plan, map[string]mediaResource) { + t.Helper() + hosts := []string{"rr1---sn-alpha.googlevideo.com", "rr2---sn-beta.googlevideo.com", "rr5---sn-gamma.googlevideo.com"} + resources := map[string]mediaResource{} + var samples []youtube.Resource + for _, host := range hosts { + url := "https://" + host + "/videoplayback?expire=4102444800&sig=private" + resources[host] = mediaResource{host: host, url: url, kind: "video/mp4"} + samples = append(samples, youtube.Resource{Host: host, URL: url, Kind: "video/mp4"}) + } + m := youtube.Module{} + p, err := m.PlanQualification(hosts, samples, time.Now()) + if err != nil { + t.Fatal(err) + } + return m.Binding(), p, resources +} + +func TestAdapterExecutorRejectsUntrustedPlanBeforeProbe(t *testing.T) { + mutations := map[string]func(*adapter.Binding, *adapter.Plan, map[string]mediaResource){ + "binding-kind": func(b *adapter.Binding, _ *adapter.Plan, _ map[string]mediaResource) { b.Kind = "other" }, + "binding-version": func(b *adapter.Binding, _ *adapter.Plan, _ map[string]mediaResource) { b.Version++ }, + "binding-policy": func(b *adapter.Binding, _ *adapter.Plan, _ map[string]mediaResource) { b.PolicySHA256 = "" }, + "missing-host": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Hosts = p.Hosts[:2] }, + "extra-host": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { + p.Hosts = append(p.Hosts, "rr9---sn-delta.googlevideo.com") + }, + "duplicate-host": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Hosts[1] = p.Hosts[0] }, + "host-url": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { + p.Hosts[0] = "https://example.com" + }, + "host-case": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { + p.Hosts[0] = "RR1---sn-alpha.googlevideo.com" + }, + "host-suffix": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { + p.Hosts[0] = "rr1---sn-alpha.example.com" + }, + "missing-resource": func(_ *adapter.Binding, p *adapter.Plan, r map[string]mediaResource) { delete(r, p.Hosts[0]) }, + "resource-host": func(_ *adapter.Binding, p *adapter.Plan, r map[string]mediaResource) { + v := r[p.Hosts[0]] + v.host = p.Hosts[1] + r[p.Hosts[0]] = v + }, + "resource-url": func(_ *adapter.Binding, p *adapter.Plan, r map[string]mediaResource) { + v := r[p.Hosts[0]] + v.url = "https://evil.example/" + r[p.Hosts[0]] = v + }, + "missing-protocol": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Protocols[1] = "" }, + "duplicate-protocol": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Protocols[1] = p.Protocols[0] }, + "unknown-protocol": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Protocols[1] = "http" }, + "protocol-order": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { + p.Protocols = [2]string{"tls13", "tls12"} + }, + } + for _, n := range []int{math.MinInt, -1, 0, 1, 4, math.MaxInt} { + value := n + mutations["samples-"+time.Duration(n).String()] = func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Samples = value } + } + for _, n := range []int{-1, 0, 1, 11, math.MaxInt} { + value := n + mutations["warmup-"+time.Duration(n).String()] = func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.WarmupLimit = value } + } + for _, n := range []int64{math.MinInt64, -1, 0, 1, 1025, math.MaxInt64} { + value := n + mutations["bytes-"+time.Duration(n).String()] = func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.MinBytes = value } + } + for _, n := range []time.Duration{-1, 0, time.Second, 121 * time.Second, time.Duration(math.MaxInt64)} { + value := n + mutations["timeout-"+n.String()] = func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.WarmupTimeout = value } + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + b, p, resources := adapterExecutorFixture(t) + mutate(&b, &p, resources) + _, err := executeAdapterPlan(context.Background(), b, p, resources, + func(context.Context, mediaResource, string) (adapterMeasurement[int], error) { + t.Fatal("invalid plan probed") + return adapterMeasurement[int]{}, nil + }, + func(error) bool { t.Fatal("invalid plan classified"); return false }) + if !errors.Is(err, errAdapterPlan) { + t.Fatal(err) + } + }) + } +} + +func TestAdapterExecutorOrderedWarmupAndTypedSamples(t *testing.T) { + b, p, resources := adapterExecutorFixture(t) + var order []string + results, err := executeAdapterPlan(context.Background(), b, p, resources, + func(ctx context.Context, r mediaResource, protocol string) (adapterMeasurement[int], error) { + if _, ok := ctx.Deadline(); len(order)%8 < 2 && !ok { + t.Fatal("warmup lacks deadline") + } + order = append(order, r.host+"/"+protocol) + return adapterMeasurement[int]{value: len(order), bytes: 1024}, nil + }, func(error) bool { return false }) + if err != nil { + t.Fatal(err) + } + var expected []string + for i, host := range p.Hosts { + for _, protocol := range []string{"tls12", "tls13", "tls12", "tls12", "tls12", "tls13", "tls13", "tls13"} { + expected = append(expected, host+"/"+protocol) + } + h := results[i] + base := i * 8 + if h.host != host || h.warmupAttempts != 2 || h.warmupFailures != 0 || h.warmupLimit != 10 || !reflect.DeepEqual(h.samples, []int{base + 3, base + 4, base + 5, base + 6, base + 7, base + 8}) { + t.Fatal(h) + } + } + if !reflect.DeepEqual(order, expected) { + t.Fatal(order, expected) + } +} + +func TestAdapterExecutorFailureAndCancellation(t *testing.T) { + fatal := errors.New("infrastructure failure") + negative := errors.New("known negative") + for _, mode := range []string{"warm-negative", "warm-small", "warm-fatal", "warm-mixed", "steady-negative", "steady-small", "steady-fatal", "cancel-before", "cancel-warm", "deadline-warm", "cancel-steady"} { + t.Run(mode, func(t *testing.T) { + b, p, resources := adapterExecutorFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + want := errCandidateNonpass + wantCalls := 10 + if mode == "cancel-before" { + cancel() + want = context.Canceled + wantCalls = 0 + } + if mode == "warm-fatal" { + want = fatal + wantCalls = 1 + } + if mode == "warm-mixed" { + want = fatal + wantCalls = 1 + } + if mode == "steady-negative" || mode == "steady-small" { + wantCalls = 3 + } + if mode == "steady-fatal" { + want = fatal + wantCalls = 3 + } + if mode == "cancel-warm" { + want = context.Canceled + wantCalls = 1 + } + if mode == "cancel-steady" { + want = context.Canceled + wantCalls = 3 + } + if mode == "deadline-warm" { + var stop context.CancelFunc + ctx, stop = context.WithTimeout(ctx, time.Second) + defer stop() + want = context.DeadlineExceeded + wantCalls = 1 + } + calls := 0 + hosts, err := executeAdapterPlan(ctx, b, p, resources, + func(probeCtx context.Context, _ mediaResource, _ string) (adapterMeasurement[int], error) { + calls++ + m := adapterMeasurement[int]{value: calls, bytes: 1024} + switch mode { + case "warm-negative": + return m, negative + case "warm-small": + m.bytes = 1023 + case "warm-fatal": + return m, fatal + case "warm-mixed": + return m, errors.Join(negative, fatal) + case "steady-negative": + if calls == 3 { + return m, negative + } + case "steady-small": + if calls == 3 { + m.bytes = 0 + } + case "steady-fatal": + if calls == 3 { + return m, fatal + } + case "cancel-warm": + cancel() + return m, ctx.Err() + case "deadline-warm": + <-probeCtx.Done() + return m, probeCtx.Err() + case "cancel-steady": + if calls == 3 { + cancel() + return m, ctx.Err() + } + } + return m, nil + }, func(err error) bool { return err == negative }) + if !errors.Is(err, want) || calls != wantCalls { + t.Fatal(err, calls, want, wantCalls) + } + for _, host := range hosts { + if len(host.samples) != 0 { + t.Fatal("phantom steady sample", host) + } + } + if calls == 10 && (hosts[0].warmupAttempts != 10 || hosts[0].warmupFailures != 10) { + t.Fatal(hosts) + } + }) + } +} + +type weakenedExecutorAdapter struct{ youtube.Module } + +func (m weakenedExecutorAdapter) PlanQualification(hosts []string, resources []adapter.Resource, now time.Time) (adapter.Plan, error) { + p, err := m.Module.PlanQualification(hosts, resources, now) + p.Samples = 1 + return p, err +} + +func TestAdapterExecutorProductionPreflightRejectsWeakenedModule(t *testing.T) { + replaceServiceAdapters(t, serviceAdapterRegistry{{youtube.Kind, youtube.Version}: weakenedExecutorAdapter{}}) + r, c, resources, events := googlevideoRuntimeFixture(t) + p, err := r.verifyGooglevideo(context.Background(), c, resources) + if !errors.Is(err, errAdapterPlan) || len(*events) != 0 || p.Attestation != nil || p.producerSeal != nil { + t.Fatal("module weakened production execution policy", p, err, *events) + } +} diff --git a/cmd/zapret-autotune/assets_test.go b/cmd/zapret-autotune/assets_test.go index 9653694..0f82224 100644 --- a/cmd/zapret-autotune/assets_test.go +++ b/cmd/zapret-autotune/assets_test.go @@ -271,7 +271,9 @@ func TestAssetCLIApplyPartialFailureAndStatus(t *testing.T) { t.Fatalf("requested status: %s %s", out, diagnostic) } if fail { - d.player = func(context.Context, []string, io.Writer, io.Writer) error { return errors.New("unit health failure") } + d.player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { + return errors.New("unit health failure") + } } code, _, diagnostic := callManaged(t, d, "apply", "--asset-candidate", fmt.Sprintf("%d,%d", a, b)) if (code != 0) != fail { diff --git a/cmd/zapret-autotune/discover.go b/cmd/zapret-autotune/discover.go index cec9cb3..e2cbfe8 100644 --- a/cmd/zapret-autotune/discover.go +++ b/cmd/zapret-autotune/discover.go @@ -24,7 +24,7 @@ const playerResponseLimit = 1 << 20 var videoIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{11}$`) var discoveryDNSLabel = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`) -type playerRunner func(context.Context, []string, io.Writer, io.Writer) error +type playerRunner func(context.Context, []string, io.Reader, io.Writer, io.Writer) error // Keep request diagnostics private while retaining typed causes for supervision. type playerRequestError struct{ cause error } @@ -32,8 +32,9 @@ type playerRequestError struct{ cause error } func (e *playerRequestError) Error() string { return "player request failed or exceeded output limit" } func (e *playerRequestError) Unwrap() error { return e.cause } -func runPlayerCurl(ctx context.Context, args []string, stdout, stderr io.Writer) error { +func runPlayerCurl(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { cmd := exec.CommandContext(ctx, "curl", args...) + cmd.Stdin = stdin cmd.Stdout, cmd.Stderr = stdout, stderr return cmd.Run() } @@ -151,6 +152,9 @@ func requestPlayer(videoID, proxy string, runner playerRunner) ([]byte, error) { if _, err := requiredYoutubeAdapter(); err != nil { return nil, err } + if !videoIDPattern.MatchString(videoID) { + return nil, errors.New("invalid player video ID") + } payload, _ := json.Marshal(map[string]interface{}{ "videoId": videoID, "context": map[string]interface{}{"client": map[string]interface{}{ @@ -164,8 +168,9 @@ func requestPlayer(videoID, proxy string, runner playerRunner) ([]byte, error) { output := &playerBoundedWriter{dst: &body, limit: playerResponseLimit} diagnostics := &playerBoundedWriter{dst: io.Discard, limit: 64 << 10} // Disable curlrc and NO_PROXY bypasses: the explicit proxy must be used. + // Keep the bounded JSON body off argv; curl retains its JSON POST headers. err := runner(ctx, []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", - "--noproxy", "", "--proxy", proxy, "--user-agent", playerUserAgent, "--json", string(payload), playerURL}, output, diagnostics) + "--noproxy", "", "--proxy", proxy, "--user-agent", playerUserAgent, "--json", "@-", playerURL}, bytes.NewReader(payload), output, diagnostics) if err != nil || output.exceeded || diagnostics.exceeded { if output.exceeded || diagnostics.exceeded { // An output failure remains unsafe even if the child was also canceled. diff --git a/cmd/zapret-autotune/discover_test.go b/cmd/zapret-autotune/discover_test.go index 879aef2..1ea102f 100644 --- a/cmd/zapret-autotune/discover_test.go +++ b/cmd/zapret-autotune/discover_test.go @@ -20,13 +20,13 @@ func discoveryArgs() []string { func TestDiscoverYoutubeSuccess(t *testing.T) { var stdout, stderr bytes.Buffer called := false - runner := func(ctx context.Context, args []string, out, diagnostics io.Writer) error { + runner := func(ctx context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error { called = true if _, ok := ctx.Deadline(); !ok { t.Error("request must have a deadline") } prefix := []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", "--noproxy", "", "--proxy", "socks5://localhost:1080", "--user-agent", playerUserAgent, "--json"} - if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(args)-1] != playerURL { + if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(prefix)] != "@-" || args[len(args)-1] != playerURL { t.Fatalf("unexpected curl arguments: %q", args) } var payload struct { @@ -42,7 +42,7 @@ func TestDiscoverYoutubeSuccess(t *testing.T) { } `json:"client"` } `json:"context"` } - if err := json.Unmarshal([]byte(args[len(prefix)]), &payload); err != nil { + if err := json.NewDecoder(stdin).Decode(&payload); err != nil { t.Fatal(err) } client := payload.Context.Client @@ -79,7 +79,7 @@ func TestDiscoverYoutubeInvalidArgs(t *testing.T) { } for _, args := range tests { var stdout, stderr bytes.Buffer - runner := func(context.Context, []string, io.Writer, io.Writer) error { + runner := func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { t.Fatal("invalid arguments must not invoke curl") return nil } @@ -114,7 +114,7 @@ func TestPlayerHostnameFailsClosed(t *testing.T) { } for _, fixture := range fixtures { var stdout, stderr bytes.Buffer - runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error { + runner := func(_ context.Context, _ []string, _ io.Reader, out, diagnostics io.Writer) error { io.WriteString(diagnostics, "SECRET") _, err := io.WriteString(out, fixture) return err @@ -136,7 +136,7 @@ func TestDiscoverYoutubeRequestFailures(t *testing.T) { for _, mode := range []string{"runner", "stdout limit", "stderr limit", "write result"} { t.Run(mode, func(t *testing.T) { var stdout, stderr bytes.Buffer - runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error { + runner := func(_ context.Context, _ []string, _ io.Reader, out, diagnostics io.Writer) error { switch mode { case "runner": io.WriteString(out, discoveryFixture) diff --git a/cmd/zapret-autotune/googlevideo_test.go b/cmd/zapret-autotune/googlevideo_test.go index 0f5b1dd..d9908bd 100644 --- a/cmd/zapret-autotune/googlevideo_test.go +++ b/cmd/zapret-autotune/googlevideo_test.go @@ -17,16 +17,20 @@ var cacheTestFlags = []string{"--video-id", "dQw4w9WgXcQ", "--proxy", "socks5:// func cachePlayer(t *testing.T, order *[]string) playerRunner { t.Helper() - return func(_ context.Context, args []string, out, diagnostics io.Writer) error { + return func(_ context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error { *order = append(*order, "discover") invocation := strings.Join(args, " ") - for _, want := range []string{"--proxy socks5://localhost:1080", "21.02.35", "dQw4w9WgXcQ", playerURL} { + for _, want := range []string{"--proxy socks5://localhost:1080", "21.02.35", "--json @-", playerURL} { if !strings.Contains(invocation, want) { t.Fatalf("discovery missing %q: %s", want, invocation) } } + payload, err := io.ReadAll(stdin) + if err != nil || !strings.Contains(string(payload), `"videoId":"dQw4w9WgXcQ"`) || strings.Contains(invocation, "dQw4w9WgXcQ") { + t.Fatal("player payload must be supplied only on stdin") + } fmt.Fprint(diagnostics, "https://secret.invalid/signed?token=SECRET") - _, err := fmt.Fprintf(out, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://%s/videoplayback?token=SECRET"}}`, cacheTestHost) + _, err = fmt.Fprintf(out, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://%s/videoplayback?token=SECRET"}}`, cacheTestHost) return err } } @@ -209,7 +213,7 @@ func TestTuneGooglevideoFailures(t *testing.T) { var order []string player := cachePlayer(t, &order) if mode == "discovery" { - player = func(context.Context, []string, io.Writer, io.Writer) error { + player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { order = append(order, "discover") return fmt.Errorf("https://signed.invalid/?SECRET") } diff --git a/cmd/zapret-autotune/health_test.go b/cmd/zapret-autotune/health_test.go index e09c5dc..aa6b87c 100644 --- a/cmd/zapret-autotune/health_test.go +++ b/cmd/zapret-autotune/health_test.go @@ -125,20 +125,20 @@ func TestRunHealthYoutube(t *testing.T) { t.Setenv("NO_PROXY", "*") var stdout, stderr bytes.Buffer calls := 0 - runner := func(ctx context.Context, args []string, out, diagnostics io.Writer) error { + runner := func(ctx context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error { calls++ deadline, ok := ctx.Deadline() if !ok || time.Until(deadline) <= 0 || time.Until(deadline) > 35*time.Second { t.Error("request must have bounded deadline") } prefix := []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", "--noproxy", "", "--proxy", proxy, "--user-agent", playerUserAgent, "--json"} - if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(args)-1] != playerURL { + if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(prefix)] != "@-" || args[len(args)-1] != playerURL { t.Fatalf("unexpected proxy/API arguments: %q", args) } var payload struct { VideoID string `json:"videoId"` } - if err := json.Unmarshal([]byte(args[len(prefix)]), &payload); err != nil || payload.VideoID != "abcdefghijk" { + if err := json.NewDecoder(stdin).Decode(&payload); err != nil || payload.VideoID != "abcdefghijk" { t.Fatalf("unexpected video payload: %+v %v", payload, err) } io.WriteString(diagnostics, "SECRET") @@ -158,7 +158,7 @@ func TestHealthYoutubeFailures(t *testing.T) { t.Run(mode, func(t *testing.T) { var stdout, stderr bytes.Buffer calls := 0 - runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error { + runner := func(_ context.Context, _ []string, _ io.Reader, out, diagnostics io.Writer) error { calls++ io.WriteString(diagnostics, "SECRET") switch mode { @@ -192,7 +192,7 @@ func TestHealthYoutubeFailures(t *testing.T) { func TestHealthDispatchAndConfigFailWithoutRequest(t *testing.T) { setHealthEnvironment(t) - runner := func(context.Context, []string, io.Writer, io.Writer) error { + runner := func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { t.Fatal("invalid command/config must not request or fall back") return nil } diff --git a/cmd/zapret-autotune/managed_asset_verifier.go b/cmd/zapret-autotune/managed_asset_verifier.go index 627863d..26b6b1c 100644 --- a/cmd/zapret-autotune/managed_asset_verifier.go +++ b/cmd/zapret-autotune/managed_asset_verifier.go @@ -128,7 +128,7 @@ func managedServiceResources(cfg healthConfig, player func(context.Context) ([]b var output bytes.Buffer out := &playerBoundedWriter{dst: &output, limit: 5 << 20} // Fixed trusted WWW origin; no redirects, credentials or diagnostics. - err = page(bounded, []string{"--disable", "--silent", "--show-error", "--fail", "--noproxy", "", "--proxy", proxy, "--proto", "=https", "--max-time", "30", "--max-filesize", "5242880", origin}, out, io.Discard) + err = page(bounded, []string{"--disable", "--silent", "--show-error", "--fail", "--noproxy", "", "--proxy", proxy, "--proto", "=https", "--max-time", "30", "--max-filesize", "5242880", origin}, nil, out, io.Discard) requestErr := bounded.Err() cancel() if ctx.Err() != nil { diff --git a/cmd/zapret-autotune/managed_avatar_inventory_test.go b/cmd/zapret-autotune/managed_avatar_inventory_test.go index a41158f..0cab060 100644 --- a/cmd/zapret-autotune/managed_avatar_inventory_test.go +++ b/cmd/zapret-autotune/managed_avatar_inventory_test.go @@ -66,7 +66,7 @@ func TestManagedAvatarInventoryFallback(t *testing.T) { calls := 0 resources := managedServiceResources(cfg, func(context.Context) ([]byte, error) { return []byte(`{"playabilityStatus":{"status":"OK"},"videoDetails":{"thumbnail":{"thumbnails":[{"url":"https://i.ytimg.com/fresh"}]}},"streamingData":{"formats":[{"url":"https://fresh.googlevideo.com/private","mimeType":"video/mp4"}]}}`), nil - }, func(ctx context.Context, args []string, out, diagnostic io.Writer) error { + }, func(ctx context.Context, args []string, _ io.Reader, out, diagnostic io.Writer) error { calls++ if _, ok := ctx.Deadline(); !ok || diagnostic != io.Discard { t.Fatal("unbounded metadata or diagnostics") @@ -148,7 +148,7 @@ func TestManagedAvatarInventoryCancellationNoFallback(t *testing.T) { cancel() } return []byte(`{"playabilityStatus":{"status":"OK"}}`), nil - }, func(ctx context.Context, args []string, out, _ io.Writer) error { + }, func(ctx context.Context, args []string, _ io.Reader, out, _ io.Writer) error { pages++ assertManagedDNSArgs(t, args, "socks5h://127.0.0.1:11080") if args[len(args)-1] != "https://www.youtube.com/" { diff --git a/cmd/zapret-autotune/managed_cli_test.go b/cmd/zapret-autotune/managed_cli_test.go index 654e5cb..1f217e8 100644 --- a/cmd/zapret-autotune/managed_cli_test.go +++ b/cmd/zapret-autotune/managed_cli_test.go @@ -46,7 +46,7 @@ func managedCLIFixture(t *testing.T) (managedDeps, map[string]string, *int) { } return nil }, - player: func(_ context.Context, args []string, stdout, _ io.Writer) error { + player: func(_ context.Context, args []string, _ io.Reader, stdout, _ io.Writer) error { if !strings.Contains(strings.Join(args, "|"), "--noproxy||--proxy|socks5h://127.0.0.1:11080|") { t.Fatalf("unsafe health argv: %q", args) } @@ -212,7 +212,7 @@ func TestManagedCLIApplyAndRollback(t *testing.T) { } switch mode { case "health": - d.player = func(_ context.Context, _ []string, stdout, _ io.Writer) error { + d.player = func(_ context.Context, _ []string, _ io.Reader, stdout, _ io.Writer) error { _, err := io.WriteString(stdout, `{"playabilityStatus":{"status":"ERROR","secret":"do-not-print"}}`) return err } diff --git a/cmd/zapret-autotune/managed_dns_test.go b/cmd/zapret-autotune/managed_dns_test.go index 3ef5b89..3078e32 100644 --- a/cmd/zapret-autotune/managed_dns_test.go +++ b/cmd/zapret-autotune/managed_dns_test.go @@ -48,7 +48,7 @@ func TestManagedDNSProductionPlayerAndGuard(t *testing.T) { return nil } requests := 0 - target.player = func(_ context.Context, args []string, out, _ io.Writer) error { + target.player = func(_ context.Context, args []string, _ io.Reader, out, _ io.Writer) error { assertManagedDNSArgs(t, args, "socks5h://127.0.0.1:11080") if args[len(args)-1] != playerURL { t.Fatal("Player hostname changed", args) diff --git a/cmd/zapret-autotune/managed_experimental_probe_test.go b/cmd/zapret-autotune/managed_experimental_probe_test.go index 255c057..5df0ce1 100644 --- a/cmd/zapret-autotune/managed_experimental_probe_test.go +++ b/cmd/zapret-autotune/managed_experimental_probe_test.go @@ -43,7 +43,7 @@ func TestExperimentalProductionProbeCollectorClassification(t *testing.T) { if strings.HasPrefix(mode, "metadata-") { d.resources = managedServiceResources(healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080", VideoID: "dQw4w9WgXcQ"}, func(context.Context) ([]byte, error) { return []byte(fmt.Sprintf(`{"playabilityStatus":{"status":"OK"},"streamingData":{"formats":[{"url":%q,"mimeType":"video/mp4"}]}}`, experimentalResourceFixture().url)), nil - }, func(_ context.Context, args []string, out, diagnostic io.Writer) error { + }, func(_ context.Context, args []string, _ io.Reader, out, diagnostic io.Writer) error { origin := args[len(args)-1] if (origin != "https://www.youtube.com/" && !(mode == "metadata-malformed" && origin == "https://www.youtube.com/watch?v=dQw4w9WgXcQ")) || diagnostic != io.Discard { t.Fatal("metadata origin/diagnostics changed") diff --git a/cmd/zapret-autotune/managed_experimental_test.go b/cmd/zapret-autotune/managed_experimental_test.go index c9b9833..0a21a4c 100644 --- a/cmd/zapret-autotune/managed_experimental_test.go +++ b/cmd/zapret-autotune/managed_experimental_test.go @@ -132,7 +132,7 @@ func TestExperimentalSuffixCLIExplicit(t *testing.T) { func TestExperimentalSuffixCLIRejectsBypassBeforeWork(t *testing.T) { d, _, _ := managedCLIFixture(t) d.lookup = func(string) (string, bool) { t.Fatal("bypass reached runtime work"); return "", false } - d.player = func(context.Context, []string, io.Writer, io.Writer) error { + d.player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { t.Fatal("bypass requested Player") return nil } @@ -280,7 +280,7 @@ func TestExperimentalSuffixCLIRejectsInheritedPolicy(t *testing.T) { if err := m.Apply(db, []int64{front, asset, id}, GooglevideoScopeExperimental); err != nil { t.Fatal(err) } - d.player = func(context.Context, []string, io.Writer, io.Writer) error { + d.player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { t.Fatal("inherited bypass requested Player") return nil } @@ -319,7 +319,7 @@ func TestExperimentalSuffixCLIInterleavedPolicyChange(t *testing.T) { } return db, nil } - d.player = func(context.Context, []string, io.Writer, io.Writer) error { + d.player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { t.Fatal("racing CLI requested Player") return nil } diff --git a/cmd/zapret-autotune/managed_googlevideo_runtime.go b/cmd/zapret-autotune/managed_googlevideo_runtime.go index f255e3b..a6c075d 100644 --- a/cmd/zapret-autotune/managed_googlevideo_runtime.go +++ b/cmd/zapret-autotune/managed_googlevideo_runtime.go @@ -123,57 +123,26 @@ func (r managedAssetRuntime) verifyGooglevideo(ctx context.Context, c storedCand if err != nil { return p, errGooglevideoInsufficientSamples } + if !reflect.DeepEqual(plan.Hosts, c.Endpoints) { + return p, errAdapterPlan + } + if err := validateAdapterPlan(binding, plan, byHost); err != nil { + return p, err + } for _, host := range plan.Hosts { p.Hosts = append(p.Hosts, GooglevideoHostProof{Host: host}) } err = r.withOwnedWorker(ctx, c.Endpoints, c.Args, func(probeCtx context.Context) error { - for i := range p.Hosts { - h := &p.Hosts[i] - h.WarmupLimit = plan.WarmupLimit - start := time.Now() - warmCtx, cancel := context.WithTimeout(probeCtx, plan.WarmupTimeout) - learned := map[string]bool{} - for h.WarmupAttempts < h.WarmupLimit && len(learned) < len(plan.Protocols) && warmCtx.Err() == nil { - protocol := plan.Protocols[h.WarmupAttempts%len(plan.Protocols)] - h.WarmupAttempts++ - _, pass, err := collectMediaTransfer(warmCtx, r.proxy, byHost[h.Host], protocol, r.curl) - if err != nil { - h.WarmupFailures++ - if !managedGooglevideoTransferNonpass(err) && warmCtx.Err() == nil { - cancel() - return err - } - } else if pass.Bytes < plan.MinBytes { - h.WarmupFailures++ - } else { - learned[protocol] = true - } - } - h.WarmupMS = float64(time.Since(start).Microseconds()) / 1000 - cancel() - if probeCtx.Err() != nil { - return probeCtx.Err() - } - if len(learned) != len(plan.Protocols) || h.WarmupMS > float64(plan.WarmupTimeout/time.Millisecond) { - return errCandidateNonpass - } - for _, protocol := range plan.Protocols { - for n := 0; n < plan.Samples; n++ { - _, pass, err := collectMediaTransfer(probeCtx, r.proxy, byHost[h.Host], protocol, r.curl) - if err != nil { - if managedGooglevideoTransferNonpass(err) { - return errCandidateNonpass - } - return err - } - if pass.Bytes < plan.MinBytes { - return errCandidateNonpass - } - h.Passes = append(h.Passes, pass) - } - } + hosts, err := executeAdapterPlan(probeCtx, binding, plan, byHost, + func(ctx context.Context, resource mediaResource, protocol string) (adapterMeasurement[GooglevideoPass], error) { + _, pass, err := collectMediaTransfer(ctx, r.proxy, resource, protocol, r.curl) + return adapterMeasurement[GooglevideoPass]{value: pass, bytes: pass.Bytes}, err + }, managedGooglevideoTransferNonpass) + for i, h := range hosts { + p.Hosts[i] = GooglevideoHostProof{Host: h.host, WarmupLimit: h.warmupLimit, + WarmupAttempts: h.warmupAttempts, WarmupFailures: h.warmupFailures, WarmupMS: h.warmupMS, Passes: h.samples} } - return nil + return err }) if err != nil { return p, err diff --git a/cmd/zapret-autotune/managed_monitor_cli_test.go b/cmd/zapret-autotune/managed_monitor_cli_test.go index 2a6c1d0..c9939fa 100644 --- a/cmd/zapret-autotune/managed_monitor_cli_test.go +++ b/cmd/zapret-autotune/managed_monitor_cli_test.go @@ -352,7 +352,7 @@ func TestManagedMonitorProductionAdapterBinding(t *testing.T) { recovery, _, _ := recoveryFixture(t, false) target, _, _ := managedCLIFixture(t) target.directory = recovery.transaction.Dir - target.player = func(ctx context.Context, args []string, out, _ io.Writer) error { + target.player = func(ctx context.Context, args []string, _ io.Reader, out, _ io.Writer) error { if ctx.Err() != nil { t.Fatal(ctx.Err()) } @@ -580,7 +580,7 @@ func TestManagedMonitorCLIBindingAndCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() requests := 0 - d.player = func(ctx context.Context, args []string, out, _ io.Writer) error { + d.player = func(ctx context.Context, args []string, _ io.Reader, out, _ io.Writer) error { requests++ if !strings.Contains(strings.Join(args, "|"), "--noproxy||--proxy|socks5h://127.0.0.1:11080|") { t.Fatal(args) diff --git a/cmd/zapret-autotune/managed_recovery.go b/cmd/zapret-autotune/managed_recovery.go index 3b4f3d9..9933526 100644 --- a/cmd/zapret-autotune/managed_recovery.go +++ b/cmd/zapret-autotune/managed_recovery.go @@ -73,8 +73,8 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, } bounded, cancel := context.WithTimeout(ctx, 35*time.Second) defer cancel() - body, err := requestPlayer(videoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error { - return target.player(bounded, args, out, diagnostics) + body, err := requestPlayer(videoID, cfg.SOCKSProxy, func(_ context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error { + return target.player(bounded, args, stdin, out, diagnostics) }) if bounded.Err() != nil { return nil, managedRecoveryContextError(ctx, errors.Join(err, bounded.Err())) diff --git a/cmd/zapret-autotune/managed_service_test.go b/cmd/zapret-autotune/managed_service_test.go index ad1c7d1..56731c7 100644 --- a/cmd/zapret-autotune/managed_service_test.go +++ b/cmd/zapret-autotune/managed_service_test.go @@ -146,7 +146,7 @@ func TestManagedServiceFreshPrivateInventory(t *testing.T) { cfg := healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080"} resources := managedServiceResources(cfg, func(context.Context) ([]byte, error) { return []byte(`{"playabilityStatus":{"status":"OK"},"videoDetails":{"thumbnail":{"thumbnails":[{"url":"https://i.ytimg.com/fresh?private=thumbnail"}]}},"streamingData":{}}`), nil - }, func(_ context.Context, args []string, out, _ io.Writer) error { + }, func(_ context.Context, args []string, _ io.Reader, out, _ io.Writer) error { if args[len(args)-1] != "https://www.youtube.com/" || strings.Contains(strings.Join(args, " "), "private=") { t.Fatal(args) } diff --git a/cmd/zapret-autotune/player_cancellation_test.go b/cmd/zapret-autotune/player_cancellation_test.go index d10db1f..ed45017 100644 --- a/cmd/zapret-autotune/player_cancellation_test.go +++ b/cmd/zapret-autotune/player_cancellation_test.go @@ -34,7 +34,7 @@ func (w playerCancelWriter) Write(p []byte) (int, error) { func canceledPlayerProcess(t *testing.T, cancel context.CancelFunc, extra error, overflow string) playerRunner { t.Helper() - return func(ctx context.Context, _ []string, out, diagnostics io.Writer) error { + return func(ctx context.Context, _ []string, _ io.Reader, out, diagnostics io.Writer) error { cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestPlayerCancellationChild$") cmd.Env = append(os.Environ(), "AUTOTUNE_PLAYER_CANCEL_CHILD=1") cmd.Stdout = playerCancelWriter{cancel: cancel} @@ -52,8 +52,8 @@ func TestPlayerRequestCancellationCause(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() runner := canceledPlayerProcess(t, cancel, nil, "") - _, err := requestPlayer("abcdefghijk", "socks5://127.0.0.1:11080", func(_ context.Context, args []string, out, diagnostics io.Writer) error { - return runner(ctx, args, out, diagnostics) + _, err := requestPlayer("abcdefghijk", "socks5://127.0.0.1:11080", func(_ context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error { + return runner(ctx, args, stdin, out, diagnostics) }) var child *exec.ExitError if !errors.As(err, &child) || !managedRecoveryTimeoutSafe(err) { @@ -67,12 +67,17 @@ func TestPlayerRequestCancellationCause(t *testing.T) { func TestPlayerRequestPublicErrorsOpaque(t *testing.T) { const secret = "https://user:password@example.com/video?signature=private" cause := errors.New(secret) - runner := func(_ context.Context, _ []string, _, diagnostics io.Writer) error { + runner := func(_ context.Context, args []string, stdin io.Reader, _, diagnostics io.Writer) error { + payload, err := io.ReadAll(stdin) + if err != nil || !bytes.Contains(payload, []byte("abcdefghijk")) || strings.Contains(strings.Join(args, " "), "abcdefghijk") { + t.Fatal("private POST body missing from stdin or exposed in argv") + } io.WriteString(diagnostics, secret) - return cause + diagnostics.Write(payload) + return errors.Join(cause, errors.New(string(payload))) } _, err := requestPlayer("abcdefghijk", "socks5://127.0.0.1:11080", runner) - if !errors.Is(err, cause) || strings.Contains(err.Error(), secret) || managedRecoveryTimeoutSafe(err) { + if !errors.Is(err, cause) || strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), "abcdefghijk") || managedRecoveryTimeoutSafe(err) { t.Fatal("opaque error lost its unsafe cause or exposed it") } t.Setenv("ZAPRET_AUTOTUNE_VIDEO_ID", "abcdefghijk") @@ -84,7 +89,7 @@ func TestPlayerRequestPublicErrorsOpaque(t *testing.T) { } else { code = discoverYoutube([]string{"--video-id", "abcdefghijk", "--proxy", "socks5://127.0.0.1:11080"}, &out, &diagnostics, runner) } - if code != 1 || out.Len() != 0 || strings.Contains(diagnostics.String(), secret) || !strings.Contains(diagnostics.String(), "player request failed") { + if code != 1 || out.Len() != 0 || strings.Contains(diagnostics.String(), secret) || strings.Contains(diagnostics.String(), "abcdefghijk") || !strings.Contains(diagnostics.String(), "player request failed") { t.Fatalf("%s: code=%d diagnostics=%q", command, code, diagnostics.String()) } } @@ -141,7 +146,7 @@ func TestManagedMonitorSIGTERMChild(t *testing.T) { env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db" ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM) defer cancel() - d.player = func(ctx context.Context, _ []string, _, diagnostics io.Writer) error { + d.player = func(ctx context.Context, _ []string, _ io.Reader, _, diagnostics io.Writer) error { cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestPlayerCancellationChild$") cmd.Env = append(os.Environ(), "AUTOTUNE_PLAYER_CANCEL_CHILD=1") cmd.Stdout, cmd.Stderr = os.Stdout, diagnostics diff --git a/cmd/zapret-autotune/player_stdin_test.go b/cmd/zapret-autotune/player_stdin_test.go new file mode 100644 index 0000000..ab05e77 --- /dev/null +++ b/cmd/zapret-autotune/player_stdin_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func playerCurlStub(t *testing.T, script string) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "curl"), []byte("#!/bin/sh\n"+script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + return dir +} + +func TestPlayerPOSTPrivateStdin(t *testing.T) { + dir := playerCurlStub(t, `printf '%s\n' "$@" > "$PLAYER_CAPTURE.args" +cat > "$PLAYER_CAPTURE.body" +printf '%s' "$PLAYER_RESPONSE" +`) + t.Setenv("PLAYER_CAPTURE", filepath.Join(dir, "request")) + t.Setenv("PLAYER_RESPONSE", discoveryFixture) + body, err := requestPlayer("SECRET12345", "socks5://127.0.0.1:11080", runPlayerCurl) + if err != nil || string(body) != discoveryFixture { + t.Fatal("real runner did not preserve the response", err) + } + args, err := os.ReadFile(filepath.Join(dir, "request.args")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(args), "SECRET12345") || !strings.Contains(string(args), "--json\n@-\n") { + t.Fatal("POST body must travel only on stdin") + } + payload, err := os.ReadFile(filepath.Join(dir, "request.body")) + const want = `{"context":{"client":{"androidSdkVersion":30,"clientName":"ANDROID","clientVersion":"21.02.35","osName":"Android","osVersion":"11","userAgent":"com.google.youtube/21.02.35(Linux;U;Android11)gzip"}},"videoId":"SECRET12345"}` + if err != nil || string(payload) != want { + t.Fatal("POST bytes changed", err) + } +} + +func TestPlayerPOSTInputBound(t *testing.T) { + for _, id := range []string{"", strings.Repeat("S", playerResponseLimit+1), "SECRET\n1234"} { + _, err := requestPlayer(id, "socks5://127.0.0.1:11080", func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { + t.Fatal("invalid input invoked curl") + return nil + }) + if err == nil || strings.Contains(err.Error(), "SECRET") { + t.Fatal("invalid POST input must fail privately before execution") + } + } +} + +func TestPlayerCurlStdinCancellation(t *testing.T) { + playerCurlStub(t, "cat > /dev/null\nprintf ready\nexec sleep 60\n") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var diagnostics bytes.Buffer + err := runPlayerCurl(ctx, []string{"--json", "@-"}, strings.NewReader(`{"videoId":"SECRET12345"}`), playerCancelWriter{cancel: cancel}, &diagnostics) + if err == nil || !errors.Is(ctx.Err(), context.Canceled) || !managedRecoveryTimeoutSafe(err) { + t.Fatal("real curl runner must join a canceled child", err) + } + if strings.Contains(diagnostics.String(), "SECRET12345") { + t.Fatal("stdin leaked into diagnostics") + } +} diff --git a/cmd/zapret-autotune/recover_test.go b/cmd/zapret-autotune/recover_test.go index b5ecbbc..8d19c45 100644 --- a/cmd/zapret-autotune/recover_test.go +++ b/cmd/zapret-autotune/recover_test.go @@ -66,7 +66,7 @@ func TestRecoverGooglevideo(t *testing.T) { var order, checked []string player := cachePlayer(t, &order) if mode == "discovery-error" { - player = func(context.Context, []string, io.Writer, io.Writer) error { + player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { order = append(order, "discover") return fmt.Errorf("SECRET discovery failure") } diff --git a/cmd/zapret-autotune/service_adapters_test.go b/cmd/zapret-autotune/service_adapters_test.go index a9362ed..b7eadb0 100644 --- a/cmd/zapret-autotune/service_adapters_test.go +++ b/cmd/zapret-autotune/service_adapters_test.go @@ -185,7 +185,7 @@ func TestServiceAdapterDiscoveryPreflight(t *testing.T) { modules[serviceAdapterKey{youtube.Kind, youtube.Version}] = changedPolicyServiceAdapter{youtube.Module{}} } calls := 0 - player := func(context.Context, []string, io.Writer, io.Writer) error { calls++; return nil } + player := func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { calls++; return nil } request := func(context.Context) ([]byte, error) { calls++; return nil, nil } target, _, _ := managedCLIFixture(t) target.player = player