diff --git a/RESOURCE_MONITOR.md b/RESOURCE_MONITOR.md index cc9a67e..4d2a58c 100644 --- a/RESOURCE_MONITOR.md +++ b/RESOURCE_MONITOR.md @@ -37,8 +37,13 @@ qualify a candidate. V2 version/kind/expectations invalidate old HMAC rows/proof historic v1 fingerprints remain unchanged. MP4 prefix health is not playback or YouTube proof: three discovered CDNs, >=1024-byte producer samples, warmup, suffix authorization and adaptive policy still belong to the legacy adapter. -Trusted compiled-in kind+version service modules remain a roadmap design, not an -implemented plugin API. Local source/closure review and injected-wire checks are +The private compiled-in YouTube module and core `executeAdapterPlan[T]` bridge +now drive the existing owned qualifier's ordered warmup and steady collection. +They do not add public module configuration or change this monitor's schemas, +controller, suffix authorization or legacy SOCKS5h routing boundary. Generic +controller cutover still needs dynamic-inventory fingerprints and warmup-bearing +generic proof binding; images/avatars and full YouTube READY remain unresolved. +Local source/closure review and injected-wire checks are recorded alongside scoped managed v2 field observations below; full acceptance is not implied. diff --git a/RESOURCE_PLAN.md b/RESOURCE_PLAN.md index 12b5d03..3e3a07c 100644 --- a/RESOURCE_PLAN.md +++ b/RESOURCE_PLAN.md @@ -707,6 +707,54 @@ Player media parsing (`playerMediaResource`) and owned native qualification representative selection remain in the legacy controller; signed MP4 freshness and the host/TLS qualification plan are pure service semantics in the module. +The next private execution bridge is now wired: `verifyGooglevideo` runs the +actual owned-worker collector through `executeAdapterPlan[T]`. Core independently +pins the known module binding and exact three-host/TLS1.2+1.3, 10-attempt/120-second +warmup, three steady samples/protocol and 1024-byte policy before worker startup +and probes. Invalid, duplicate, overflowing or weakened plans fail closed. The +executor preserves host order, alternating warmups, discarded learning successes, +strict measured steady facts and negative/fatal classification. It has no worker, +goroutine, persistence or seal capability; existing core range/ftyp/TLS validation +and post-worker-join proof validation/sealing remain the authority. + +This is a **private execution bridge**, not public adapter configuration or a new +generic service/matrix proof. Dynamic-inventory fingerprints and warmup-bearing +generic proof binding are still required before generic-controller cutover. +The inherited legacy SOCKS5h boundary below is unchanged, not an all-DNS-answer +pinning guarantee. Three-CDN qualification does not itself grant suffix scope; +images/avatars and runtime acceptance remain separate and not READY. + +Player JSON now travels through bounded private stdin (`curl --json @-`), not +process argv; the shared runner passes no stdin for metadata GETs. Video IDs are +validated before execution, POST bytes and headers remain unchanged, and opaque +public errors retain typed cancellation/unsafe-output causes. This is a private +Player transport change, not generic public POST/discovery configuration. + +Bridge-local checks: an unchanged fixture run against archived `f485317` and the +wired producer has normalized proof SHA256 +`fb380324e320a78f1ceb011c59b82b9de4ff4f31f5f032f469a181292987e3b2`. +Only warmup duration, attestation time and private seals are excluded; exact +probe order and cleanup/join are checked. Baseline and focused serial/race logs: +`/tmp/opencode/adapter-executor-{baseline,focused}.log`. These are injected local +checks, not network/native acceptance or a full-suite rerun. + +Cancellation review follow-up: both warmup and steady probe boundaries now use +`managedRecoveryContextError`, preserving concurrent unsafe I/O/output overflow +alongside cancellation/deadline causes. Production collector regressions check +fatal/non-restartable classification, full cleanup/join and no attestation/seal; +plain cancellation remains timeout-safe. A pre-fix Go-overlay negative control +fails all eight mixed-failure cases; the fixed producer retains the hash above. +Actual `sudo` EUID-0 serial full Go checks passed (1806 test/subtest pass events), +as did 227 affected serial/race events, separate module race tests, vet and a +CGO-disabled build. Logs: `/tmp/opencode/adapter-executor-root-full.json`, +`adapter-executor-context-root-race.json`, `adapter-executor-context-focused.log` +and `adapter-executor-context-negative-control.log` in that same directory. +The reviewer-reported earlier `SavedFirstAndFailClosed/nonpass` freshness failure +did not recur in ten focused repeats or this full run; no freshness guards or +fixtures were changed, and its earlier cause is not established as a baseline +flake. Python checks were not rerun in this follow-up. These are local tests, +not remote runtime acceptance, a commit/deployment or a READY claim. + Core origin/TLS validation, shared image/media transfer code, bounded worker lifecycle and post-cleanup/join sealing remain authoritative. Legacy media and metadata routing retains SOCKS5h with the owned encrypted-resolver guard and diff --git a/cmd/zapret-autotune/adapter_executor.go b/cmd/zapret-autotune/adapter_executor.go new file mode 100644 index 0000000..c4b140e --- /dev/null +++ b/cmd/zapret-autotune/adapter_executor.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "errors" + "time" + + "zapret-autotune/internal/adapter" + "zapret-autotune/internal/adapter/youtube" +) + +var errAdapterPlan = errors.New("invalid private adapter execution plan") + +// This is core policy, not module-supplied caps. Adding a binding requires an +// audited execution policy here; even a correctly identified module is untrusted. +func validateAdapterPlan(binding adapter.Binding, plan adapter.Plan, resources map[string]mediaResource) error { + if binding != (youtube.Module{}).Binding() || len(plan.Hosts) != 3 || + plan.Protocols != [2]string{"tls12", "tls13"} || plan.Samples != 3 || + plan.MinBytes != 1024 || plan.WarmupLimit != 10 || plan.WarmupTimeout != 120*time.Second { + return errAdapterPlan + } + if (youtube.Module{}).ValidateHosts(plan.Hosts) != nil { + return errAdapterPlan + } + for _, host := range plan.Hosts { + resource, ok := resources[host] + if !ok || !validDomain(host) || resource.host != host { + return errAdapterPlan + } + if _, err := privateResourceURL(resource.url, host); err != nil { + return errAdapterPlan + } + if !youtube.SignedSampleFresh(youtube.Resource{URL: resource.url, Host: host, Kind: resource.kind}, time.Now(), 120*time.Second) { + return errAdapterPlan + } + } + return nil +} + +// Typed measurements stay private and ephemeral. Only core callbacks collect +// and classify them; modules cannot supply callbacks, attestations or seals. +type adapterMeasurement[T any] struct { + value T + bytes int64 +} + +type adapterHostMeasurements[T any] struct { + host string + warmupAttempts, warmupFailures int + warmupLimit int + warmupMS float64 + samples []T +} + +// executeAdapterPlan owns scheduling only. Its caller owns the worker, transport +// validation and post-join proof validation/sealing. No goroutines or persistence. +func executeAdapterPlan[T any](ctx context.Context, binding adapter.Binding, plan adapter.Plan, resources map[string]mediaResource, + probe func(context.Context, mediaResource, string) (adapterMeasurement[T], error), nonpass func(error) bool, +) ([]adapterHostMeasurements[T], error) { + if err := validateAdapterPlan(binding, plan, resources); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + hosts := make([]adapterHostMeasurements[T], len(plan.Hosts)) + for i, host := range plan.Hosts { + hosts[i].host = host + } + for i := range hosts { + h := &hosts[i] + h.warmupLimit = plan.WarmupLimit + start := time.Now() + warmCtx, cancel := context.WithTimeout(ctx, plan.WarmupTimeout) + learned := map[string]bool{} + for h.warmupAttempts < h.warmupLimit && len(learned) < len(plan.Protocols) && warmCtx.Err() == nil { + protocol := plan.Protocols[h.warmupAttempts%len(plan.Protocols)] + h.warmupAttempts++ + measurement, err := probe(warmCtx, resources[h.host], protocol) + err = managedRecoveryContextError(warmCtx, err) + if err != nil { + h.warmupFailures++ + if !nonpass(err) && (warmCtx.Err() == nil || !managedRecoveryTimeoutSafe(err)) { + cancel() + return hosts, err + } + } else if measurement.bytes < plan.MinBytes { + h.warmupFailures++ + } else { + learned[protocol] = true + } + } + h.warmupMS = float64(time.Since(start).Microseconds()) / 1000 + cancel() + if err := ctx.Err(); err != nil { + return hosts, err + } + if len(learned) != len(plan.Protocols) || h.warmupMS > float64(plan.WarmupTimeout/time.Millisecond) { + return hosts, errCandidateNonpass + } + for _, protocol := range plan.Protocols { + for n := 0; n < plan.Samples; n++ { + if err := ctx.Err(); err != nil { + return hosts, err + } + measurement, err := probe(ctx, resources[h.host], protocol) + err = managedRecoveryContextError(ctx, err) + if ctx.Err() != nil { + return hosts, err + } + if err != nil { + if nonpass(err) { + return hosts, errCandidateNonpass + } + return hosts, err + } + if measurement.bytes < plan.MinBytes { + return hosts, errCandidateNonpass + } + h.samples = append(h.samples, measurement.value) + } + } + } + return hosts, nil +} diff --git a/cmd/zapret-autotune/adapter_executor_parity_test.go b/cmd/zapret-autotune/adapter_executor_parity_test.go new file mode 100644 index 0000000..e3604f8 --- /dev/null +++ b/cmd/zapret-autotune/adapter_executor_parity_test.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "strings" + "testing" + "time" +) + +// Also runnable unchanged against archived f485317. Only wall-clock duration, +// attestation time and process-private seals are excluded from the golden hash. +func TestAdapterExecutorLegacyParity(t *testing.T) { + r, c, resources, events := googlevideoRuntimeFixture(t) + runner := r.curl + var order []string + r.curl = func(ctx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + protocol := "tls12" + if strings.Contains(strings.Join(args, " "), "--tlsv1.3") { + protocol = "tls13" + } + order = append(order, protocol) + if len(order) <= 2 { + return context.DeadlineExceeded + } + return runner(ctx, args, stdin, out, diagnostic) + } + p, err := r.verifyGooglevideo(context.Background(), c, resources) + if err != nil || p.producerSeal == nil { + t.Fatal(err, p) + } + if !strings.HasSuffix(strings.Join(*events, ","), "delete,join") { + t.Fatal(*events) + } + expected := []string{"tls12", "tls13"} + for range c.Endpoints { + expected = append(expected, "tls12", "tls13", "tls12", "tls12", "tls12", "tls13", "tls13", "tls13") + } + if !reflect.DeepEqual(order, expected) { + t.Fatal(order, expected) + } + for i := range p.Hosts { + p.Hosts[i].WarmupMS = 0 + } + p.Attestation.CheckedAt = time.Unix(1700000000, 0).UTC() + raw, err := json.Marshal(p) + if err != nil { + t.Fatal(err) + } + hash := fmt.Sprintf("%x", sha256.Sum256(raw)) + t.Log("normalized proof sha256:", hash) + const baseline = "fb380324e320a78f1ceb011c59b82b9de4ff4f31f5f032f469a181292987e3b2" + if hash != baseline { + t.Fatalf("proof parity changed: %s\n%s", hash, raw) + } +} + +func TestAdapterExecutorProductionCancellationJoinsWithoutSeal(t *testing.T) { + for _, at := range []int{1, 3} { + t.Run(fmt.Sprint(at), func(t *testing.T) { + r, c, resources, events := googlevideoRuntimeFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + runner, calls := r.curl, 0 + r.curl = func(probeCtx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + calls++ + if calls == at { + cancel() + return ctx.Err() + } + return runner(probeCtx, args, stdin, out, diagnostic) + } + p, err := r.verifyGooglevideo(ctx, c, resources) + if !errors.Is(err, context.Canceled) || calls != at || p.Attestation != nil || p.producerSeal != nil || + !strings.HasSuffix(strings.Join(*events, ","), "delete,join") { + t.Fatal(p, err, calls, *events) + } + if !managedRecoveryTimeoutSafe(err) { + t.Fatal("plain cancellation became unsafe", err) + } + }) + } +} + +func TestAdapterExecutorProductionCancellationPreservesUnsafeTransfer(t *testing.T) { + unsafeIO := errors.New("unsafe collector IO") + for _, at := range []int{1, 3} { // first warmup or first steady after both TLS versions learn + for _, deadline := range []bool{false, true} { + for _, overflow := range []bool{false, true} { + t.Run(fmt.Sprintf("probe%d/deadline%t/overflow%t", at, deadline, overflow), func(t *testing.T) { + r, c, resources, events := googlevideoRuntimeFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wantContext := context.Canceled + if deadline { + var stop context.CancelFunc + ctx, stop = context.WithTimeout(ctx, time.Second) + defer stop() + wantContext = context.DeadlineExceeded + } + runner, calls := r.curl, 0 + r.curl = func(probeCtx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + calls++ + if calls != at { + return runner(probeCtx, args, stdin, out, diagnostic) + } + if deadline { + <-ctx.Done() + } else { + cancel() + } + if overflow { + // Exercise the real bounded collector writer, not a forged fatal DTO. + _, writeErr := out.Write(make([]byte, 256<<10)) + return errors.Join(writeErr, ctx.Err()) + } + return errors.Join(unsafeIO, ctx.Err()) + } + p, err := r.verifyGooglevideo(ctx, c, resources) + var fatal *managedRecoveryFatal + if !errors.Is(err, wantContext) || !errors.As(err, &fatal) || managedRecoveryTimeoutSafe(err) || managedResourceTransferNonpass(err, false, false) { + t.Fatalf("lost mixed safety/context error: %v", err) + } + if overflow { + var collected *resourceError + if !errors.As(err, &collected) || collected.reason != reasonOutputBound { + t.Fatal("lost collector overflow", err) + } + } else if !errors.Is(err, unsafeIO) { + t.Fatal("lost unsafe IO cause", err) + } + if calls != at || p.Attestation != nil || p.producerSeal != nil || + !strings.HasSuffix(strings.Join(*events, ","), "delete,join") { + t.Fatal(p, err, calls, *events) + } + if at == 3 && (p.Hosts[0].WarmupAttempts != 2 || len(p.Hosts[0].Passes) != 0) { + t.Fatal("learning leaked into steady proof", p.Hosts) + } + }) + } + } + } +} diff --git a/cmd/zapret-autotune/adapter_executor_test.go b/cmd/zapret-autotune/adapter_executor_test.go new file mode 100644 index 0000000..792a677 --- /dev/null +++ b/cmd/zapret-autotune/adapter_executor_test.go @@ -0,0 +1,248 @@ +package main + +import ( + "context" + "errors" + "math" + "reflect" + "testing" + "time" + + "zapret-autotune/internal/adapter" + "zapret-autotune/internal/adapter/youtube" +) + +func adapterExecutorFixture(t *testing.T) (adapter.Binding, adapter.Plan, map[string]mediaResource) { + t.Helper() + hosts := []string{"rr1---sn-alpha.googlevideo.com", "rr2---sn-beta.googlevideo.com", "rr5---sn-gamma.googlevideo.com"} + resources := map[string]mediaResource{} + var samples []youtube.Resource + for _, host := range hosts { + url := "https://" + host + "/videoplayback?expire=4102444800&sig=private" + resources[host] = mediaResource{host: host, url: url, kind: "video/mp4"} + samples = append(samples, youtube.Resource{Host: host, URL: url, Kind: "video/mp4"}) + } + m := youtube.Module{} + p, err := m.PlanQualification(hosts, samples, time.Now()) + if err != nil { + t.Fatal(err) + } + return m.Binding(), p, resources +} + +func TestAdapterExecutorRejectsUntrustedPlanBeforeProbe(t *testing.T) { + mutations := map[string]func(*adapter.Binding, *adapter.Plan, map[string]mediaResource){ + "binding-kind": func(b *adapter.Binding, _ *adapter.Plan, _ map[string]mediaResource) { b.Kind = "other" }, + "binding-version": func(b *adapter.Binding, _ *adapter.Plan, _ map[string]mediaResource) { b.Version++ }, + "binding-policy": func(b *adapter.Binding, _ *adapter.Plan, _ map[string]mediaResource) { b.PolicySHA256 = "" }, + "missing-host": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Hosts = p.Hosts[:2] }, + "extra-host": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { + p.Hosts = append(p.Hosts, "rr9---sn-delta.googlevideo.com") + }, + "duplicate-host": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Hosts[1] = p.Hosts[0] }, + "host-url": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { + p.Hosts[0] = "https://example.com" + }, + "host-case": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { + p.Hosts[0] = "RR1---sn-alpha.googlevideo.com" + }, + "host-suffix": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { + p.Hosts[0] = "rr1---sn-alpha.example.com" + }, + "missing-resource": func(_ *adapter.Binding, p *adapter.Plan, r map[string]mediaResource) { delete(r, p.Hosts[0]) }, + "resource-host": func(_ *adapter.Binding, p *adapter.Plan, r map[string]mediaResource) { + v := r[p.Hosts[0]] + v.host = p.Hosts[1] + r[p.Hosts[0]] = v + }, + "resource-url": func(_ *adapter.Binding, p *adapter.Plan, r map[string]mediaResource) { + v := r[p.Hosts[0]] + v.url = "https://evil.example/" + r[p.Hosts[0]] = v + }, + "missing-protocol": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Protocols[1] = "" }, + "duplicate-protocol": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Protocols[1] = p.Protocols[0] }, + "unknown-protocol": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Protocols[1] = "http" }, + "protocol-order": func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { + p.Protocols = [2]string{"tls13", "tls12"} + }, + } + for _, n := range []int{math.MinInt, -1, 0, 1, 4, math.MaxInt} { + value := n + mutations["samples-"+time.Duration(n).String()] = func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.Samples = value } + } + for _, n := range []int{-1, 0, 1, 11, math.MaxInt} { + value := n + mutations["warmup-"+time.Duration(n).String()] = func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.WarmupLimit = value } + } + for _, n := range []int64{math.MinInt64, -1, 0, 1, 1025, math.MaxInt64} { + value := n + mutations["bytes-"+time.Duration(n).String()] = func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.MinBytes = value } + } + for _, n := range []time.Duration{-1, 0, time.Second, 121 * time.Second, time.Duration(math.MaxInt64)} { + value := n + mutations["timeout-"+n.String()] = func(_ *adapter.Binding, p *adapter.Plan, _ map[string]mediaResource) { p.WarmupTimeout = value } + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + b, p, resources := adapterExecutorFixture(t) + mutate(&b, &p, resources) + _, err := executeAdapterPlan(context.Background(), b, p, resources, + func(context.Context, mediaResource, string) (adapterMeasurement[int], error) { + t.Fatal("invalid plan probed") + return adapterMeasurement[int]{}, nil + }, + func(error) bool { t.Fatal("invalid plan classified"); return false }) + if !errors.Is(err, errAdapterPlan) { + t.Fatal(err) + } + }) + } +} + +func TestAdapterExecutorOrderedWarmupAndTypedSamples(t *testing.T) { + b, p, resources := adapterExecutorFixture(t) + var order []string + results, err := executeAdapterPlan(context.Background(), b, p, resources, + func(ctx context.Context, r mediaResource, protocol string) (adapterMeasurement[int], error) { + if _, ok := ctx.Deadline(); len(order)%8 < 2 && !ok { + t.Fatal("warmup lacks deadline") + } + order = append(order, r.host+"/"+protocol) + return adapterMeasurement[int]{value: len(order), bytes: 1024}, nil + }, func(error) bool { return false }) + if err != nil { + t.Fatal(err) + } + var expected []string + for i, host := range p.Hosts { + for _, protocol := range []string{"tls12", "tls13", "tls12", "tls12", "tls12", "tls13", "tls13", "tls13"} { + expected = append(expected, host+"/"+protocol) + } + h := results[i] + base := i * 8 + if h.host != host || h.warmupAttempts != 2 || h.warmupFailures != 0 || h.warmupLimit != 10 || !reflect.DeepEqual(h.samples, []int{base + 3, base + 4, base + 5, base + 6, base + 7, base + 8}) { + t.Fatal(h) + } + } + if !reflect.DeepEqual(order, expected) { + t.Fatal(order, expected) + } +} + +func TestAdapterExecutorFailureAndCancellation(t *testing.T) { + fatal := errors.New("infrastructure failure") + negative := errors.New("known negative") + for _, mode := range []string{"warm-negative", "warm-small", "warm-fatal", "warm-mixed", "steady-negative", "steady-small", "steady-fatal", "cancel-before", "cancel-warm", "deadline-warm", "cancel-steady"} { + t.Run(mode, func(t *testing.T) { + b, p, resources := adapterExecutorFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + want := errCandidateNonpass + wantCalls := 10 + if mode == "cancel-before" { + cancel() + want = context.Canceled + wantCalls = 0 + } + if mode == "warm-fatal" { + want = fatal + wantCalls = 1 + } + if mode == "warm-mixed" { + want = fatal + wantCalls = 1 + } + if mode == "steady-negative" || mode == "steady-small" { + wantCalls = 3 + } + if mode == "steady-fatal" { + want = fatal + wantCalls = 3 + } + if mode == "cancel-warm" { + want = context.Canceled + wantCalls = 1 + } + if mode == "cancel-steady" { + want = context.Canceled + wantCalls = 3 + } + if mode == "deadline-warm" { + var stop context.CancelFunc + ctx, stop = context.WithTimeout(ctx, time.Second) + defer stop() + want = context.DeadlineExceeded + wantCalls = 1 + } + calls := 0 + hosts, err := executeAdapterPlan(ctx, b, p, resources, + func(probeCtx context.Context, _ mediaResource, _ string) (adapterMeasurement[int], error) { + calls++ + m := adapterMeasurement[int]{value: calls, bytes: 1024} + switch mode { + case "warm-negative": + return m, negative + case "warm-small": + m.bytes = 1023 + case "warm-fatal": + return m, fatal + case "warm-mixed": + return m, errors.Join(negative, fatal) + case "steady-negative": + if calls == 3 { + return m, negative + } + case "steady-small": + if calls == 3 { + m.bytes = 0 + } + case "steady-fatal": + if calls == 3 { + return m, fatal + } + case "cancel-warm": + cancel() + return m, ctx.Err() + case "deadline-warm": + <-probeCtx.Done() + return m, probeCtx.Err() + case "cancel-steady": + if calls == 3 { + cancel() + return m, ctx.Err() + } + } + return m, nil + }, func(err error) bool { return err == negative }) + if !errors.Is(err, want) || calls != wantCalls { + t.Fatal(err, calls, want, wantCalls) + } + for _, host := range hosts { + if len(host.samples) != 0 { + t.Fatal("phantom steady sample", host) + } + } + if calls == 10 && (hosts[0].warmupAttempts != 10 || hosts[0].warmupFailures != 10) { + t.Fatal(hosts) + } + }) + } +} + +type weakenedExecutorAdapter struct{ youtube.Module } + +func (m weakenedExecutorAdapter) PlanQualification(hosts []string, resources []adapter.Resource, now time.Time) (adapter.Plan, error) { + p, err := m.Module.PlanQualification(hosts, resources, now) + p.Samples = 1 + return p, err +} + +func TestAdapterExecutorProductionPreflightRejectsWeakenedModule(t *testing.T) { + replaceServiceAdapters(t, serviceAdapterRegistry{{youtube.Kind, youtube.Version}: weakenedExecutorAdapter{}}) + r, c, resources, events := googlevideoRuntimeFixture(t) + p, err := r.verifyGooglevideo(context.Background(), c, resources) + if !errors.Is(err, errAdapterPlan) || len(*events) != 0 || p.Attestation != nil || p.producerSeal != nil { + t.Fatal("module weakened production execution policy", p, err, *events) + } +} diff --git a/cmd/zapret-autotune/assets_test.go b/cmd/zapret-autotune/assets_test.go index 9653694..0f82224 100644 --- a/cmd/zapret-autotune/assets_test.go +++ b/cmd/zapret-autotune/assets_test.go @@ -271,7 +271,9 @@ func TestAssetCLIApplyPartialFailureAndStatus(t *testing.T) { t.Fatalf("requested status: %s %s", out, diagnostic) } if fail { - d.player = func(context.Context, []string, io.Writer, io.Writer) error { return errors.New("unit health failure") } + d.player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { + return errors.New("unit health failure") + } } code, _, diagnostic := callManaged(t, d, "apply", "--asset-candidate", fmt.Sprintf("%d,%d", a, b)) if (code != 0) != fail { diff --git a/cmd/zapret-autotune/discover.go b/cmd/zapret-autotune/discover.go index cec9cb3..e2cbfe8 100644 --- a/cmd/zapret-autotune/discover.go +++ b/cmd/zapret-autotune/discover.go @@ -24,7 +24,7 @@ const playerResponseLimit = 1 << 20 var videoIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{11}$`) var discoveryDNSLabel = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`) -type playerRunner func(context.Context, []string, io.Writer, io.Writer) error +type playerRunner func(context.Context, []string, io.Reader, io.Writer, io.Writer) error // Keep request diagnostics private while retaining typed causes for supervision. type playerRequestError struct{ cause error } @@ -32,8 +32,9 @@ type playerRequestError struct{ cause error } func (e *playerRequestError) Error() string { return "player request failed or exceeded output limit" } func (e *playerRequestError) Unwrap() error { return e.cause } -func runPlayerCurl(ctx context.Context, args []string, stdout, stderr io.Writer) error { +func runPlayerCurl(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { cmd := exec.CommandContext(ctx, "curl", args...) + cmd.Stdin = stdin cmd.Stdout, cmd.Stderr = stdout, stderr return cmd.Run() } @@ -151,6 +152,9 @@ func requestPlayer(videoID, proxy string, runner playerRunner) ([]byte, error) { if _, err := requiredYoutubeAdapter(); err != nil { return nil, err } + if !videoIDPattern.MatchString(videoID) { + return nil, errors.New("invalid player video ID") + } payload, _ := json.Marshal(map[string]interface{}{ "videoId": videoID, "context": map[string]interface{}{"client": map[string]interface{}{ @@ -164,8 +168,9 @@ func requestPlayer(videoID, proxy string, runner playerRunner) ([]byte, error) { output := &playerBoundedWriter{dst: &body, limit: playerResponseLimit} diagnostics := &playerBoundedWriter{dst: io.Discard, limit: 64 << 10} // Disable curlrc and NO_PROXY bypasses: the explicit proxy must be used. + // Keep the bounded JSON body off argv; curl retains its JSON POST headers. err := runner(ctx, []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", - "--noproxy", "", "--proxy", proxy, "--user-agent", playerUserAgent, "--json", string(payload), playerURL}, output, diagnostics) + "--noproxy", "", "--proxy", proxy, "--user-agent", playerUserAgent, "--json", "@-", playerURL}, bytes.NewReader(payload), output, diagnostics) if err != nil || output.exceeded || diagnostics.exceeded { if output.exceeded || diagnostics.exceeded { // An output failure remains unsafe even if the child was also canceled. diff --git a/cmd/zapret-autotune/discover_test.go b/cmd/zapret-autotune/discover_test.go index 879aef2..1ea102f 100644 --- a/cmd/zapret-autotune/discover_test.go +++ b/cmd/zapret-autotune/discover_test.go @@ -20,13 +20,13 @@ func discoveryArgs() []string { func TestDiscoverYoutubeSuccess(t *testing.T) { var stdout, stderr bytes.Buffer called := false - runner := func(ctx context.Context, args []string, out, diagnostics io.Writer) error { + runner := func(ctx context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error { called = true if _, ok := ctx.Deadline(); !ok { t.Error("request must have a deadline") } prefix := []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", "--noproxy", "", "--proxy", "socks5://localhost:1080", "--user-agent", playerUserAgent, "--json"} - if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(args)-1] != playerURL { + if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(prefix)] != "@-" || args[len(args)-1] != playerURL { t.Fatalf("unexpected curl arguments: %q", args) } var payload struct { @@ -42,7 +42,7 @@ func TestDiscoverYoutubeSuccess(t *testing.T) { } `json:"client"` } `json:"context"` } - if err := json.Unmarshal([]byte(args[len(prefix)]), &payload); err != nil { + if err := json.NewDecoder(stdin).Decode(&payload); err != nil { t.Fatal(err) } client := payload.Context.Client @@ -79,7 +79,7 @@ func TestDiscoverYoutubeInvalidArgs(t *testing.T) { } for _, args := range tests { var stdout, stderr bytes.Buffer - runner := func(context.Context, []string, io.Writer, io.Writer) error { + runner := func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { t.Fatal("invalid arguments must not invoke curl") return nil } @@ -114,7 +114,7 @@ func TestPlayerHostnameFailsClosed(t *testing.T) { } for _, fixture := range fixtures { var stdout, stderr bytes.Buffer - runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error { + runner := func(_ context.Context, _ []string, _ io.Reader, out, diagnostics io.Writer) error { io.WriteString(diagnostics, "SECRET") _, err := io.WriteString(out, fixture) return err @@ -136,7 +136,7 @@ func TestDiscoverYoutubeRequestFailures(t *testing.T) { for _, mode := range []string{"runner", "stdout limit", "stderr limit", "write result"} { t.Run(mode, func(t *testing.T) { var stdout, stderr bytes.Buffer - runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error { + runner := func(_ context.Context, _ []string, _ io.Reader, out, diagnostics io.Writer) error { switch mode { case "runner": io.WriteString(out, discoveryFixture) diff --git a/cmd/zapret-autotune/googlevideo_test.go b/cmd/zapret-autotune/googlevideo_test.go index 0f5b1dd..d9908bd 100644 --- a/cmd/zapret-autotune/googlevideo_test.go +++ b/cmd/zapret-autotune/googlevideo_test.go @@ -17,16 +17,20 @@ var cacheTestFlags = []string{"--video-id", "dQw4w9WgXcQ", "--proxy", "socks5:// func cachePlayer(t *testing.T, order *[]string) playerRunner { t.Helper() - return func(_ context.Context, args []string, out, diagnostics io.Writer) error { + return func(_ context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error { *order = append(*order, "discover") invocation := strings.Join(args, " ") - for _, want := range []string{"--proxy socks5://localhost:1080", "21.02.35", "dQw4w9WgXcQ", playerURL} { + for _, want := range []string{"--proxy socks5://localhost:1080", "21.02.35", "--json @-", playerURL} { if !strings.Contains(invocation, want) { t.Fatalf("discovery missing %q: %s", want, invocation) } } + payload, err := io.ReadAll(stdin) + if err != nil || !strings.Contains(string(payload), `"videoId":"dQw4w9WgXcQ"`) || strings.Contains(invocation, "dQw4w9WgXcQ") { + t.Fatal("player payload must be supplied only on stdin") + } fmt.Fprint(diagnostics, "https://secret.invalid/signed?token=SECRET") - _, err := fmt.Fprintf(out, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://%s/videoplayback?token=SECRET"}}`, cacheTestHost) + _, err = fmt.Fprintf(out, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://%s/videoplayback?token=SECRET"}}`, cacheTestHost) return err } } @@ -209,7 +213,7 @@ func TestTuneGooglevideoFailures(t *testing.T) { var order []string player := cachePlayer(t, &order) if mode == "discovery" { - player = func(context.Context, []string, io.Writer, io.Writer) error { + player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { order = append(order, "discover") return fmt.Errorf("https://signed.invalid/?SECRET") } diff --git a/cmd/zapret-autotune/health_test.go b/cmd/zapret-autotune/health_test.go index e09c5dc..aa6b87c 100644 --- a/cmd/zapret-autotune/health_test.go +++ b/cmd/zapret-autotune/health_test.go @@ -125,20 +125,20 @@ func TestRunHealthYoutube(t *testing.T) { t.Setenv("NO_PROXY", "*") var stdout, stderr bytes.Buffer calls := 0 - runner := func(ctx context.Context, args []string, out, diagnostics io.Writer) error { + runner := func(ctx context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error { calls++ deadline, ok := ctx.Deadline() if !ok || time.Until(deadline) <= 0 || time.Until(deadline) > 35*time.Second { t.Error("request must have bounded deadline") } prefix := []string{"--disable", "--silent", "--show-error", "--fail", "--max-time", "30", "--noproxy", "", "--proxy", proxy, "--user-agent", playerUserAgent, "--json"} - if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(args)-1] != playerURL { + if len(args) != len(prefix)+2 || !reflect.DeepEqual(args[:len(prefix)], prefix) || args[len(prefix)] != "@-" || args[len(args)-1] != playerURL { t.Fatalf("unexpected proxy/API arguments: %q", args) } var payload struct { VideoID string `json:"videoId"` } - if err := json.Unmarshal([]byte(args[len(prefix)]), &payload); err != nil || payload.VideoID != "abcdefghijk" { + if err := json.NewDecoder(stdin).Decode(&payload); err != nil || payload.VideoID != "abcdefghijk" { t.Fatalf("unexpected video payload: %+v %v", payload, err) } io.WriteString(diagnostics, "SECRET") @@ -158,7 +158,7 @@ func TestHealthYoutubeFailures(t *testing.T) { t.Run(mode, func(t *testing.T) { var stdout, stderr bytes.Buffer calls := 0 - runner := func(_ context.Context, _ []string, out, diagnostics io.Writer) error { + runner := func(_ context.Context, _ []string, _ io.Reader, out, diagnostics io.Writer) error { calls++ io.WriteString(diagnostics, "SECRET") switch mode { @@ -192,7 +192,7 @@ func TestHealthYoutubeFailures(t *testing.T) { func TestHealthDispatchAndConfigFailWithoutRequest(t *testing.T) { setHealthEnvironment(t) - runner := func(context.Context, []string, io.Writer, io.Writer) error { + runner := func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { t.Fatal("invalid command/config must not request or fall back") return nil } diff --git a/cmd/zapret-autotune/managed_asset_verifier.go b/cmd/zapret-autotune/managed_asset_verifier.go index 627863d..26b6b1c 100644 --- a/cmd/zapret-autotune/managed_asset_verifier.go +++ b/cmd/zapret-autotune/managed_asset_verifier.go @@ -128,7 +128,7 @@ func managedServiceResources(cfg healthConfig, player func(context.Context) ([]b var output bytes.Buffer out := &playerBoundedWriter{dst: &output, limit: 5 << 20} // Fixed trusted WWW origin; no redirects, credentials or diagnostics. - err = page(bounded, []string{"--disable", "--silent", "--show-error", "--fail", "--noproxy", "", "--proxy", proxy, "--proto", "=https", "--max-time", "30", "--max-filesize", "5242880", origin}, out, io.Discard) + err = page(bounded, []string{"--disable", "--silent", "--show-error", "--fail", "--noproxy", "", "--proxy", proxy, "--proto", "=https", "--max-time", "30", "--max-filesize", "5242880", origin}, nil, out, io.Discard) requestErr := bounded.Err() cancel() if ctx.Err() != nil { diff --git a/cmd/zapret-autotune/managed_avatar_inventory_test.go b/cmd/zapret-autotune/managed_avatar_inventory_test.go index a41158f..0cab060 100644 --- a/cmd/zapret-autotune/managed_avatar_inventory_test.go +++ b/cmd/zapret-autotune/managed_avatar_inventory_test.go @@ -66,7 +66,7 @@ func TestManagedAvatarInventoryFallback(t *testing.T) { calls := 0 resources := managedServiceResources(cfg, func(context.Context) ([]byte, error) { return []byte(`{"playabilityStatus":{"status":"OK"},"videoDetails":{"thumbnail":{"thumbnails":[{"url":"https://i.ytimg.com/fresh"}]}},"streamingData":{"formats":[{"url":"https://fresh.googlevideo.com/private","mimeType":"video/mp4"}]}}`), nil - }, func(ctx context.Context, args []string, out, diagnostic io.Writer) error { + }, func(ctx context.Context, args []string, _ io.Reader, out, diagnostic io.Writer) error { calls++ if _, ok := ctx.Deadline(); !ok || diagnostic != io.Discard { t.Fatal("unbounded metadata or diagnostics") @@ -148,7 +148,7 @@ func TestManagedAvatarInventoryCancellationNoFallback(t *testing.T) { cancel() } return []byte(`{"playabilityStatus":{"status":"OK"}}`), nil - }, func(ctx context.Context, args []string, out, _ io.Writer) error { + }, func(ctx context.Context, args []string, _ io.Reader, out, _ io.Writer) error { pages++ assertManagedDNSArgs(t, args, "socks5h://127.0.0.1:11080") if args[len(args)-1] != "https://www.youtube.com/" { diff --git a/cmd/zapret-autotune/managed_cli_test.go b/cmd/zapret-autotune/managed_cli_test.go index 654e5cb..1f217e8 100644 --- a/cmd/zapret-autotune/managed_cli_test.go +++ b/cmd/zapret-autotune/managed_cli_test.go @@ -46,7 +46,7 @@ func managedCLIFixture(t *testing.T) (managedDeps, map[string]string, *int) { } return nil }, - player: func(_ context.Context, args []string, stdout, _ io.Writer) error { + player: func(_ context.Context, args []string, _ io.Reader, stdout, _ io.Writer) error { if !strings.Contains(strings.Join(args, "|"), "--noproxy||--proxy|socks5h://127.0.0.1:11080|") { t.Fatalf("unsafe health argv: %q", args) } @@ -212,7 +212,7 @@ func TestManagedCLIApplyAndRollback(t *testing.T) { } switch mode { case "health": - d.player = func(_ context.Context, _ []string, stdout, _ io.Writer) error { + d.player = func(_ context.Context, _ []string, _ io.Reader, stdout, _ io.Writer) error { _, err := io.WriteString(stdout, `{"playabilityStatus":{"status":"ERROR","secret":"do-not-print"}}`) return err } diff --git a/cmd/zapret-autotune/managed_dns_test.go b/cmd/zapret-autotune/managed_dns_test.go index 3ef5b89..3078e32 100644 --- a/cmd/zapret-autotune/managed_dns_test.go +++ b/cmd/zapret-autotune/managed_dns_test.go @@ -48,7 +48,7 @@ func TestManagedDNSProductionPlayerAndGuard(t *testing.T) { return nil } requests := 0 - target.player = func(_ context.Context, args []string, out, _ io.Writer) error { + target.player = func(_ context.Context, args []string, _ io.Reader, out, _ io.Writer) error { assertManagedDNSArgs(t, args, "socks5h://127.0.0.1:11080") if args[len(args)-1] != playerURL { t.Fatal("Player hostname changed", args) diff --git a/cmd/zapret-autotune/managed_experimental_probe_test.go b/cmd/zapret-autotune/managed_experimental_probe_test.go index 255c057..5df0ce1 100644 --- a/cmd/zapret-autotune/managed_experimental_probe_test.go +++ b/cmd/zapret-autotune/managed_experimental_probe_test.go @@ -43,7 +43,7 @@ func TestExperimentalProductionProbeCollectorClassification(t *testing.T) { if strings.HasPrefix(mode, "metadata-") { d.resources = managedServiceResources(healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080", VideoID: "dQw4w9WgXcQ"}, func(context.Context) ([]byte, error) { return []byte(fmt.Sprintf(`{"playabilityStatus":{"status":"OK"},"streamingData":{"formats":[{"url":%q,"mimeType":"video/mp4"}]}}`, experimentalResourceFixture().url)), nil - }, func(_ context.Context, args []string, out, diagnostic io.Writer) error { + }, func(_ context.Context, args []string, _ io.Reader, out, diagnostic io.Writer) error { origin := args[len(args)-1] if (origin != "https://www.youtube.com/" && !(mode == "metadata-malformed" && origin == "https://www.youtube.com/watch?v=dQw4w9WgXcQ")) || diagnostic != io.Discard { t.Fatal("metadata origin/diagnostics changed") diff --git a/cmd/zapret-autotune/managed_experimental_test.go b/cmd/zapret-autotune/managed_experimental_test.go index c9b9833..0a21a4c 100644 --- a/cmd/zapret-autotune/managed_experimental_test.go +++ b/cmd/zapret-autotune/managed_experimental_test.go @@ -132,7 +132,7 @@ func TestExperimentalSuffixCLIExplicit(t *testing.T) { func TestExperimentalSuffixCLIRejectsBypassBeforeWork(t *testing.T) { d, _, _ := managedCLIFixture(t) d.lookup = func(string) (string, bool) { t.Fatal("bypass reached runtime work"); return "", false } - d.player = func(context.Context, []string, io.Writer, io.Writer) error { + d.player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { t.Fatal("bypass requested Player") return nil } @@ -280,7 +280,7 @@ func TestExperimentalSuffixCLIRejectsInheritedPolicy(t *testing.T) { if err := m.Apply(db, []int64{front, asset, id}, GooglevideoScopeExperimental); err != nil { t.Fatal(err) } - d.player = func(context.Context, []string, io.Writer, io.Writer) error { + d.player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { t.Fatal("inherited bypass requested Player") return nil } @@ -319,7 +319,7 @@ func TestExperimentalSuffixCLIInterleavedPolicyChange(t *testing.T) { } return db, nil } - d.player = func(context.Context, []string, io.Writer, io.Writer) error { + d.player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { t.Fatal("racing CLI requested Player") return nil } diff --git a/cmd/zapret-autotune/managed_googlevideo_runtime.go b/cmd/zapret-autotune/managed_googlevideo_runtime.go index f255e3b..a6c075d 100644 --- a/cmd/zapret-autotune/managed_googlevideo_runtime.go +++ b/cmd/zapret-autotune/managed_googlevideo_runtime.go @@ -123,57 +123,26 @@ func (r managedAssetRuntime) verifyGooglevideo(ctx context.Context, c storedCand if err != nil { return p, errGooglevideoInsufficientSamples } + if !reflect.DeepEqual(plan.Hosts, c.Endpoints) { + return p, errAdapterPlan + } + if err := validateAdapterPlan(binding, plan, byHost); err != nil { + return p, err + } for _, host := range plan.Hosts { p.Hosts = append(p.Hosts, GooglevideoHostProof{Host: host}) } err = r.withOwnedWorker(ctx, c.Endpoints, c.Args, func(probeCtx context.Context) error { - for i := range p.Hosts { - h := &p.Hosts[i] - h.WarmupLimit = plan.WarmupLimit - start := time.Now() - warmCtx, cancel := context.WithTimeout(probeCtx, plan.WarmupTimeout) - learned := map[string]bool{} - for h.WarmupAttempts < h.WarmupLimit && len(learned) < len(plan.Protocols) && warmCtx.Err() == nil { - protocol := plan.Protocols[h.WarmupAttempts%len(plan.Protocols)] - h.WarmupAttempts++ - _, pass, err := collectMediaTransfer(warmCtx, r.proxy, byHost[h.Host], protocol, r.curl) - if err != nil { - h.WarmupFailures++ - if !managedGooglevideoTransferNonpass(err) && warmCtx.Err() == nil { - cancel() - return err - } - } else if pass.Bytes < plan.MinBytes { - h.WarmupFailures++ - } else { - learned[protocol] = true - } - } - h.WarmupMS = float64(time.Since(start).Microseconds()) / 1000 - cancel() - if probeCtx.Err() != nil { - return probeCtx.Err() - } - if len(learned) != len(plan.Protocols) || h.WarmupMS > float64(plan.WarmupTimeout/time.Millisecond) { - return errCandidateNonpass - } - for _, protocol := range plan.Protocols { - for n := 0; n < plan.Samples; n++ { - _, pass, err := collectMediaTransfer(probeCtx, r.proxy, byHost[h.Host], protocol, r.curl) - if err != nil { - if managedGooglevideoTransferNonpass(err) { - return errCandidateNonpass - } - return err - } - if pass.Bytes < plan.MinBytes { - return errCandidateNonpass - } - h.Passes = append(h.Passes, pass) - } - } + hosts, err := executeAdapterPlan(probeCtx, binding, plan, byHost, + func(ctx context.Context, resource mediaResource, protocol string) (adapterMeasurement[GooglevideoPass], error) { + _, pass, err := collectMediaTransfer(ctx, r.proxy, resource, protocol, r.curl) + return adapterMeasurement[GooglevideoPass]{value: pass, bytes: pass.Bytes}, err + }, managedGooglevideoTransferNonpass) + for i, h := range hosts { + p.Hosts[i] = GooglevideoHostProof{Host: h.host, WarmupLimit: h.warmupLimit, + WarmupAttempts: h.warmupAttempts, WarmupFailures: h.warmupFailures, WarmupMS: h.warmupMS, Passes: h.samples} } - return nil + return err }) if err != nil { return p, err diff --git a/cmd/zapret-autotune/managed_monitor_cli_test.go b/cmd/zapret-autotune/managed_monitor_cli_test.go index 2a6c1d0..c9939fa 100644 --- a/cmd/zapret-autotune/managed_monitor_cli_test.go +++ b/cmd/zapret-autotune/managed_monitor_cli_test.go @@ -352,7 +352,7 @@ func TestManagedMonitorProductionAdapterBinding(t *testing.T) { recovery, _, _ := recoveryFixture(t, false) target, _, _ := managedCLIFixture(t) target.directory = recovery.transaction.Dir - target.player = func(ctx context.Context, args []string, out, _ io.Writer) error { + target.player = func(ctx context.Context, args []string, _ io.Reader, out, _ io.Writer) error { if ctx.Err() != nil { t.Fatal(ctx.Err()) } @@ -580,7 +580,7 @@ func TestManagedMonitorCLIBindingAndCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() requests := 0 - d.player = func(ctx context.Context, args []string, out, _ io.Writer) error { + d.player = func(ctx context.Context, args []string, _ io.Reader, out, _ io.Writer) error { requests++ if !strings.Contains(strings.Join(args, "|"), "--noproxy||--proxy|socks5h://127.0.0.1:11080|") { t.Fatal(args) diff --git a/cmd/zapret-autotune/managed_recovery.go b/cmd/zapret-autotune/managed_recovery.go index 3b4f3d9..9933526 100644 --- a/cmd/zapret-autotune/managed_recovery.go +++ b/cmd/zapret-autotune/managed_recovery.go @@ -73,8 +73,8 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, } bounded, cancel := context.WithTimeout(ctx, 35*time.Second) defer cancel() - body, err := requestPlayer(videoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error { - return target.player(bounded, args, out, diagnostics) + body, err := requestPlayer(videoID, cfg.SOCKSProxy, func(_ context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error { + return target.player(bounded, args, stdin, out, diagnostics) }) if bounded.Err() != nil { return nil, managedRecoveryContextError(ctx, errors.Join(err, bounded.Err())) diff --git a/cmd/zapret-autotune/managed_service_test.go b/cmd/zapret-autotune/managed_service_test.go index ad1c7d1..56731c7 100644 --- a/cmd/zapret-autotune/managed_service_test.go +++ b/cmd/zapret-autotune/managed_service_test.go @@ -146,7 +146,7 @@ func TestManagedServiceFreshPrivateInventory(t *testing.T) { cfg := healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080"} resources := managedServiceResources(cfg, func(context.Context) ([]byte, error) { return []byte(`{"playabilityStatus":{"status":"OK"},"videoDetails":{"thumbnail":{"thumbnails":[{"url":"https://i.ytimg.com/fresh?private=thumbnail"}]}},"streamingData":{}}`), nil - }, func(_ context.Context, args []string, out, _ io.Writer) error { + }, func(_ context.Context, args []string, _ io.Reader, out, _ io.Writer) error { if args[len(args)-1] != "https://www.youtube.com/" || strings.Contains(strings.Join(args, " "), "private=") { t.Fatal(args) } diff --git a/cmd/zapret-autotune/player_cancellation_test.go b/cmd/zapret-autotune/player_cancellation_test.go index d10db1f..ed45017 100644 --- a/cmd/zapret-autotune/player_cancellation_test.go +++ b/cmd/zapret-autotune/player_cancellation_test.go @@ -34,7 +34,7 @@ func (w playerCancelWriter) Write(p []byte) (int, error) { func canceledPlayerProcess(t *testing.T, cancel context.CancelFunc, extra error, overflow string) playerRunner { t.Helper() - return func(ctx context.Context, _ []string, out, diagnostics io.Writer) error { + return func(ctx context.Context, _ []string, _ io.Reader, out, diagnostics io.Writer) error { cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestPlayerCancellationChild$") cmd.Env = append(os.Environ(), "AUTOTUNE_PLAYER_CANCEL_CHILD=1") cmd.Stdout = playerCancelWriter{cancel: cancel} @@ -52,8 +52,8 @@ func TestPlayerRequestCancellationCause(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() runner := canceledPlayerProcess(t, cancel, nil, "") - _, err := requestPlayer("abcdefghijk", "socks5://127.0.0.1:11080", func(_ context.Context, args []string, out, diagnostics io.Writer) error { - return runner(ctx, args, out, diagnostics) + _, err := requestPlayer("abcdefghijk", "socks5://127.0.0.1:11080", func(_ context.Context, args []string, stdin io.Reader, out, diagnostics io.Writer) error { + return runner(ctx, args, stdin, out, diagnostics) }) var child *exec.ExitError if !errors.As(err, &child) || !managedRecoveryTimeoutSafe(err) { @@ -67,12 +67,17 @@ func TestPlayerRequestCancellationCause(t *testing.T) { func TestPlayerRequestPublicErrorsOpaque(t *testing.T) { const secret = "https://user:password@example.com/video?signature=private" cause := errors.New(secret) - runner := func(_ context.Context, _ []string, _, diagnostics io.Writer) error { + runner := func(_ context.Context, args []string, stdin io.Reader, _, diagnostics io.Writer) error { + payload, err := io.ReadAll(stdin) + if err != nil || !bytes.Contains(payload, []byte("abcdefghijk")) || strings.Contains(strings.Join(args, " "), "abcdefghijk") { + t.Fatal("private POST body missing from stdin or exposed in argv") + } io.WriteString(diagnostics, secret) - return cause + diagnostics.Write(payload) + return errors.Join(cause, errors.New(string(payload))) } _, err := requestPlayer("abcdefghijk", "socks5://127.0.0.1:11080", runner) - if !errors.Is(err, cause) || strings.Contains(err.Error(), secret) || managedRecoveryTimeoutSafe(err) { + if !errors.Is(err, cause) || strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), "abcdefghijk") || managedRecoveryTimeoutSafe(err) { t.Fatal("opaque error lost its unsafe cause or exposed it") } t.Setenv("ZAPRET_AUTOTUNE_VIDEO_ID", "abcdefghijk") @@ -84,7 +89,7 @@ func TestPlayerRequestPublicErrorsOpaque(t *testing.T) { } else { code = discoverYoutube([]string{"--video-id", "abcdefghijk", "--proxy", "socks5://127.0.0.1:11080"}, &out, &diagnostics, runner) } - if code != 1 || out.Len() != 0 || strings.Contains(diagnostics.String(), secret) || !strings.Contains(diagnostics.String(), "player request failed") { + if code != 1 || out.Len() != 0 || strings.Contains(diagnostics.String(), secret) || strings.Contains(diagnostics.String(), "abcdefghijk") || !strings.Contains(diagnostics.String(), "player request failed") { t.Fatalf("%s: code=%d diagnostics=%q", command, code, diagnostics.String()) } } @@ -141,7 +146,7 @@ func TestManagedMonitorSIGTERMChild(t *testing.T) { env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db" ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM) defer cancel() - d.player = func(ctx context.Context, _ []string, _, diagnostics io.Writer) error { + d.player = func(ctx context.Context, _ []string, _ io.Reader, _, diagnostics io.Writer) error { cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestPlayerCancellationChild$") cmd.Env = append(os.Environ(), "AUTOTUNE_PLAYER_CANCEL_CHILD=1") cmd.Stdout, cmd.Stderr = os.Stdout, diagnostics diff --git a/cmd/zapret-autotune/player_stdin_test.go b/cmd/zapret-autotune/player_stdin_test.go new file mode 100644 index 0000000..ab05e77 --- /dev/null +++ b/cmd/zapret-autotune/player_stdin_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func playerCurlStub(t *testing.T, script string) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "curl"), []byte("#!/bin/sh\n"+script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + return dir +} + +func TestPlayerPOSTPrivateStdin(t *testing.T) { + dir := playerCurlStub(t, `printf '%s\n' "$@" > "$PLAYER_CAPTURE.args" +cat > "$PLAYER_CAPTURE.body" +printf '%s' "$PLAYER_RESPONSE" +`) + t.Setenv("PLAYER_CAPTURE", filepath.Join(dir, "request")) + t.Setenv("PLAYER_RESPONSE", discoveryFixture) + body, err := requestPlayer("SECRET12345", "socks5://127.0.0.1:11080", runPlayerCurl) + if err != nil || string(body) != discoveryFixture { + t.Fatal("real runner did not preserve the response", err) + } + args, err := os.ReadFile(filepath.Join(dir, "request.args")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(args), "SECRET12345") || !strings.Contains(string(args), "--json\n@-\n") { + t.Fatal("POST body must travel only on stdin") + } + payload, err := os.ReadFile(filepath.Join(dir, "request.body")) + const want = `{"context":{"client":{"androidSdkVersion":30,"clientName":"ANDROID","clientVersion":"21.02.35","osName":"Android","osVersion":"11","userAgent":"com.google.youtube/21.02.35(Linux;U;Android11)gzip"}},"videoId":"SECRET12345"}` + if err != nil || string(payload) != want { + t.Fatal("POST bytes changed", err) + } +} + +func TestPlayerPOSTInputBound(t *testing.T) { + for _, id := range []string{"", strings.Repeat("S", playerResponseLimit+1), "SECRET\n1234"} { + _, err := requestPlayer(id, "socks5://127.0.0.1:11080", func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { + t.Fatal("invalid input invoked curl") + return nil + }) + if err == nil || strings.Contains(err.Error(), "SECRET") { + t.Fatal("invalid POST input must fail privately before execution") + } + } +} + +func TestPlayerCurlStdinCancellation(t *testing.T) { + playerCurlStub(t, "cat > /dev/null\nprintf ready\nexec sleep 60\n") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var diagnostics bytes.Buffer + err := runPlayerCurl(ctx, []string{"--json", "@-"}, strings.NewReader(`{"videoId":"SECRET12345"}`), playerCancelWriter{cancel: cancel}, &diagnostics) + if err == nil || !errors.Is(ctx.Err(), context.Canceled) || !managedRecoveryTimeoutSafe(err) { + t.Fatal("real curl runner must join a canceled child", err) + } + if strings.Contains(diagnostics.String(), "SECRET12345") { + t.Fatal("stdin leaked into diagnostics") + } +} diff --git a/cmd/zapret-autotune/recover_test.go b/cmd/zapret-autotune/recover_test.go index b5ecbbc..8d19c45 100644 --- a/cmd/zapret-autotune/recover_test.go +++ b/cmd/zapret-autotune/recover_test.go @@ -66,7 +66,7 @@ func TestRecoverGooglevideo(t *testing.T) { var order, checked []string player := cachePlayer(t, &order) if mode == "discovery-error" { - player = func(context.Context, []string, io.Writer, io.Writer) error { + player = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { order = append(order, "discover") return fmt.Errorf("SECRET discovery failure") } diff --git a/cmd/zapret-autotune/service_adapters_test.go b/cmd/zapret-autotune/service_adapters_test.go index a9362ed..b7eadb0 100644 --- a/cmd/zapret-autotune/service_adapters_test.go +++ b/cmd/zapret-autotune/service_adapters_test.go @@ -185,7 +185,7 @@ func TestServiceAdapterDiscoveryPreflight(t *testing.T) { modules[serviceAdapterKey{youtube.Kind, youtube.Version}] = changedPolicyServiceAdapter{youtube.Module{}} } calls := 0 - player := func(context.Context, []string, io.Writer, io.Writer) error { calls++; return nil } + player := func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { calls++; return nil } request := func(context.Context) ([]byte, error) { calls++; return nil, nil } target, _, _ := managedCLIFixture(t) target.player = player