From 45d28e37330069ddf0efa3120fec8a558d37ba7f Mon Sep 17 00:00:00 2001 From: Zeusina Date: Thu, 15 Oct 2026 12:04:28 +0300 Subject: [PATCH] refactor: share strict resource validation through generic core Use private typed generic, image and MP4 response contracts in the existing bounded HTTPS execution pipeline. Preserve generic mismatch measurements, strict legacy DTOs and producer behavior; reject overflowing Content-Range integers before normalization. Add injected-wire adapter parity and direct-core regressions. Local targeted EUID-0/race, full Go and config fixtures passed; native acceptance remains pending. The unrelated long Python supervision timeout remains unresolved. No public schema, suffix-learning or controller migration. Co-authored-by: GPT-6 --- RESOURCE_PLAN.md | 25 +++ cmd/zapret-autotune/resource_check.go | 14 +- cmd/zapret-autotune/resource_check_test.go | 4 +- cmd/zapret-autotune/resource_collector.go | 124 +++++++---- cmd/zapret-autotune/resource_contract_test.go | 127 ++++++++++++ .../resource_core_parity_test.go | 193 ++++++++++++++++++ cmd/zapret-autotune/resource_qualify_test.go | 2 +- 7 files changed, 439 insertions(+), 50 deletions(-) create mode 100644 cmd/zapret-autotune/resource_contract_test.go create mode 100644 cmd/zapret-autotune/resource_core_parity_test.go diff --git a/RESOURCE_PLAN.md b/RESOURCE_PLAN.md index 1056892..6327e85 100644 --- a/RESOURCE_PLAN.md +++ b/RESOURCE_PLAN.md @@ -1,5 +1,30 @@ # Resource compatibility planner and isolated qualifier +## First collector migration (locally verified/reviewed; remote acceptance pending) + +Private typed response contracts now select generic GET/HEAD, full decoded image, +or bounded MP4 range validation inside the existing shared HTTPS transfer core. +The redundant legacy forwarding wrapper and media boolean are removed; image and +media adapters retain their measurements and GooglevideoPass DTOs. Generic v1 +status/MIME/byte expectations use the same validation entry, with a distinct +completed-mismatch result retaining the same measured facts. No public Range config, schema, +proof importer, registry or dependency was added. + +Injected-wire tests exercise the real generic producer and legacy callers, both +TLS versions, normalized byte/hash/timing facts, private stdin/pinning, origin, +redirect, framing, bounds and strict range/ftyp/image decoder rejection. Direct +core tests ensure strict validation is not an unused legacy-only helper. +MP4 requests use Range `0-65535` (a 64 KiB upper bound, not an exact response +size requirement). The core accepts validated 16–65536-byte ranges; the existing +producer additionally requires at least 1024 measured bytes. Content-Range end +and total must parse without overflow, with total greater than end; rejected or +unknown totals cannot produce a media PASS DTO. +This is collector migration only: legacy owned-worker production, three-CDN +sampling/warmup, native suffix adaptive learning and image/avatar readiness gates +remain unchanged. Generic matrix/controller/journal/profile replacement still +requires typed discovery/range proof and suffix/adaptive-learning capability parity. +No remote result, service READY or full controller migration is claimed here. + `plan` remains **proposal-only** and never changes runtime or config. Explicit generic startup apply/replay and the per-PID-1 failure latch have passed bounded remote acceptance with independent privileged verification (see below). diff --git a/cmd/zapret-autotune/resource_check.go b/cmd/zapret-autotune/resource_check.go index c4af956..acd796b 100644 --- a/cmd/zapret-autotune/resource_check.go +++ b/cmd/zapret-autotune/resource_check.go @@ -402,22 +402,20 @@ func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpe return result, qualificationOperation(err, stagePostDNSGuard) } pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443") - options := resourceTransferOptions{generic: true, method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n"} - transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, false, d.runner, options) - if d.qualification && err != nil { + options := resourceTransferOptions{method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n", expect: &r.Expect} + transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, d.runner, options) + if d.qualification && err != nil && err != errResourceExpectation { knownTimeout := probeCtx.Err() != nil && managedRecoveryTimeoutSafe(err) if !knownTimeout && !resourceRejectionOnly(err) && !managedAssetTransferNonpass(err) { return result, qualificationOperation(err, stageTransfer) } } result.Outcome = "transfer_failed" - if err == nil { + if err == nil || err == errResourceExpectation { result.Bytes, result.TimeMS, result.Status = len(transfer.Body), transfer.LatencyMS, transfer.Status result.Outcome = "expectation_failed" - for _, status := range r.Expect.Statuses { - if status == transfer.Status && len(transfer.Body) >= r.Expect.MinBytes && (r.Expect.ContentType == "" || transfer.ContentType == r.Expect.ContentType) { - result.Outcome = "pass" - } + if err == nil { + result.Outcome = "pass" } } } diff --git a/cmd/zapret-autotune/resource_check_test.go b/cmd/zapret-autotune/resource_check_test.go index 744ed5b..243ee55 100644 --- a/cmd/zapret-autotune/resource_check_test.go +++ b/cmd/zapret-autotune/resource_check_test.go @@ -365,7 +365,7 @@ func TestResourceCheckTransferContract(t *testing.T) { configuredWire("https://example.com/", method, "tls12", mode, out, diag) return nil } - transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://example.com/", "example.com", "tls12", false, runner, resourceTransferOptions{generic: true, method: method, limit: 100, timeout: time.Second}) + transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://example.com/", "example.com", "tls12", runner, resourceTransferOptions{method: method, limit: 100, timeout: time.Second}) valid := mode == "pass" || mode == "chunked" || method == "HEAD" && mode == "truncated" if (err == nil) != valid { t.Fatalf("err=%v", err) @@ -484,7 +484,7 @@ func TestResourceCheckCurlSOCKSPinnedDestination(t *testing.T) { }() ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) defer cancel() - _, _, err = collectResourceTransferOptions(ctx, "socks5h://"+listener.Addr().String(), "https://example.com/private?secret=value", "example.com", "tls12", false, runResourceCurl, resourceTransferOptions{generic: true, method: "GET", limit: 100, timeout: 3 * time.Second, config: "connect-to = \"example.com:443:93.184.216.34:443\"\n"}) + _, _, err = collectResourceTransferOptions(ctx, "socks5h://"+listener.Addr().String(), "https://example.com/private?secret=value", "example.com", "tls12", runResourceCurl, resourceTransferOptions{method: "GET", limit: 100, timeout: 3 * time.Second, config: "connect-to = \"example.com:443:93.184.216.34:443\"\n"}) if err == nil { t.Fatal("fixture unexpectedly transferred") } diff --git a/cmd/zapret-autotune/resource_collector.go b/cmd/zapret-autotune/resource_collector.go index d0c7a85..2c46cfa 100644 --- a/cmd/zapret-autotune/resource_collector.go +++ b/cmd/zapret-autotune/resource_collector.go @@ -229,26 +229,39 @@ func playerMediaResource(body []byte) (mediaResource, mediaCapability, error) { const resourceTrailer = "\nAUTOTUNE_RESOURCE_METRICS:" -func collectResourceTransfer(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner) (assetTransfer, http.Header, error) { - return collectResourceTransferOptions(ctx, proxy, raw, host, protocol, media, runner, resourceTransferOptions{}) -} +type resourceResponseKind uint8 + +// A completed generic mismatch retains measured facts, unlike unsafe transfers. +var errResourceExpectation = errors.New("resource expectation failed") + +const ( + resourceResponseGeneric resourceResponseKind = iota + resourceResponseImage + resourceResponseMP4 +) type resourceTransferOptions struct { method string limit int timeout time.Duration // Private curl config only: never placed in argv or public evidence. - config string - generic bool + config string + response resourceResponseKind + mediaType string + expect *resourceCheckExpect } -func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner, options resourceTransferOptions) (assetTransfer, http.Header, error) { +func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, protocol string, runner subprocessRunner, options resourceTransferOptions) (assetTransfer, http.Header, error) { + generic := options.response == resourceResponseGeneric reason := reasonOrigin fail := func(err error) (assetTransfer, http.Header, error) { failure := resourceFailure(ctx, err) failure.(*resourceError).reason = reason return assetTransfer{}, nil, failure } + if options.response > resourceResponseMP4 || (!generic && options.method != "GET") || (options.response == resourceResponseMP4 && options.mediaType != "video/mp4" && options.mediaType != "audio/mp4") { + return fail(errors.New("invalid response contract")) + } proxy, proxyErr := managedProbeProxy(proxy) if proxyErr != nil || runner == nil { return fail(errors.New("invalid owned proxy or runner")) @@ -263,19 +276,19 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto return fail(errors.New("invalid TLS protocol")) } limit := 5 << 20 - if media { + if options.response == resourceResponseMP4 { limit = 65536 } - if options.generic { + if generic { limit = options.limit } // Qualification uses HTTP/1.1, not browser HTTP/2. --raw preserves native // transfer framing so net/http, rather than curl and Go twice, decodes it. args := []string{"--disable", "--silent", "--show-error", "--verbose", "--http1.1", "--raw", "--noproxy", "", "--proxy", proxy, "--proto", "=https", tlsFlag, "--tls-max", maxTLS, "--max-time", "30", "--max-filesize", strconv.Itoa(limit), "--suppress-connect-headers", "--include", "--write-out", resourceTrailer + "%{json}", "--config", "-"} - if media { + if options.response == resourceResponseMP4 { args = append(args, "--range", "0-65535") } - if options.generic { + if generic { if (options.method != "GET" && options.method != "HEAD") || options.limit < 1 || options.limit > 5<<20 || options.timeout < 100*time.Millisecond || options.timeout > 30*time.Second { return fail(errors.New("invalid transfer options")) } @@ -288,7 +301,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto config := "url = \"" + strings.NewReplacer("\\", "\\\\", "\"", "\\\"").Replace(raw) + "\"\n" config += options.config timeout := 35 * time.Second - if options.generic { + if generic { timeout = options.timeout } childCtx, cancel := context.WithTimeout(ctx, timeout) @@ -366,10 +379,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto return fail(errors.New("ambiguous response framing")) } r := bufio.NewReader(bytes.NewReader(wire)) - var request *http.Request - if options.generic { - request = &http.Request{Method: options.method} - } + request := &http.Request{Method: options.method} response, err := http.ReadResponse(r, request) if err != nil { return fail(errors.New("invalid response headers")) @@ -377,10 +387,10 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto defer response.Body.Close() // CONNECT headers are suppressed by curl. Interim/duplicate responses and // unrequested versions are outside this contract, never heuristically skipped. - if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || (!options.generic && response.Header.Get("Content-Type") == "") { + if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || (!generic && response.Header.Get("Content-Type") == "") { return fail(errors.New("unsupported response contract")) } - if options.generic && response.StatusCode >= 300 && response.StatusCode < 400 { + if generic && response.StatusCode >= 300 && response.StatusCode < 400 { return fail(&resourcePeerResponseRejection{error: errors.New("redirect rejected")}) } reason = reasonBody @@ -389,27 +399,74 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto // limit and durable byte identity apply to the fully decoded payload instead. wireSize := len(wire) - headerEnd - 4 _, remaining := r.Peek(1) - if err != nil || remaining != io.EOF || (!options.generic && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status { + if err != nil || remaining != io.EOF || (!generic && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status { return fail(errors.New("incomplete or oversized resource body")) } reason = reasonMetrics for _, n := range []float64{metrics.Time, metrics.Speed} { - if n < 0 || (!options.generic && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) { + if n < 0 || (!generic && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) { return fail(errors.New("invalid resource metrics")) } } - if options.generic && metrics.Time > options.timeout.Seconds()+1 { + if generic && metrics.Time > options.timeout.Seconds()+1 { return fail(errors.New("resource timing exceeds budget")) } reason = reasonMIME contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type")) - if err != nil && (!options.generic || response.Header.Get("Content-Type") != "") { + if err != nil && (!generic || response.Header.Get("Content-Type") != "") { return fail(errors.New("invalid content type")) } t := assetTransfer{EffectiveHost: host, SSLVerifyResult: metrics.Verify, Redirects: *metrics.Redirects, TLSVersion: version, Status: metrics.Status, ContentType: contentType, Body: body, LatencyMS: metrics.Time * 1000, SpeedKBPS: metrics.Speed / 1024} + if err := options.validateResponse(&t, response.Header); err != nil { + if err == errResourceExpectation { + return t, response.Header, err + } + return assetTransfer{}, nil, resourceFailure(ctx, err) + } return t, response.Header, nil } +// Adapter contracts run only after the common origin/TLS/framing/bounds checks. +// Generic v1 deliberately has no range or decoded-image expectation. +func (options resourceTransferOptions) validateResponse(t *assetTransfer, h http.Header) error { + switch options.response { + case resourceResponseGeneric: + if options.expect == nil { + return nil + } + e := options.expect + for _, status := range e.Statuses { + if t.Status == status && len(t.Body) >= e.MinBytes && len(t.Body) <= e.MaxBytes && (e.ContentType == "" || t.ContentType == e.ContentType) { + return nil + } + } + return errResourceExpectation + case resourceResponseImage: + if t.Status != 200 { + return errors.New("image HTTP status") + } + return normalizeAssetBody(t) + case resourceResponseMP4: + parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range")) + var end, total int64 + if len(parts) == 3 { + var endErr, totalErr error + end, endErr = strconv.ParseInt(parts[1], 10, 64) + total, totalErr = strconv.ParseInt(parts[2], 10, 64) + if endErr != nil || totalErr != nil { + return errors.New("invalid media range or container") + } + } + if t.Status != 206 || t.ContentType != options.mediaType || len(parts) != 3 || end < 15 || end > 65535 || end+1 != int64(len(t.Body)) || total <= end || len(t.Body) < 16 || string(t.Body[4:8]) != "ftyp" || binary.BigEndian.Uint32(t.Body[:4]) < 16 || int(binary.BigEndian.Uint32(t.Body[:4])) > len(t.Body) { + return errors.New("invalid media range or container") + } + t.Size, t.Format, t.BodyHash = len(t.Body), "mp4", fmt.Sprintf("%x", sha256.Sum256(t.Body)) + return nil + default: + return errors.New("invalid response contract") + } +} + // The ordinary 3/3 sampler is shared by image and media qualification. func sampleResource(ctx context.Context, transfer func(context.Context) (assetTransfer, error)) ([]assetTransfer, error) { var samples []assetTransfer @@ -429,14 +486,7 @@ func collectAssetMeasurement(ctx context.Context, proxy, raw, host, protocol, ar return m, resourceFailure(ctx, errors.New("invalid image endpoint")) } samples, err := sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) { - t, _, err := collectResourceTransfer(ctx, proxy, raw, host, protocol, false, runner) - if err == nil { - if t.Status != 200 { - err = errors.New("image HTTP status") - } else { - err = normalizeAssetBody(&t) - } - } + t, _, err := collectResourceTransferOptions(ctx, proxy, raw, host, protocol, runner, resourceTransferOptions{method: "GET", response: resourceResponseImage}) if err != nil { return assetTransfer{}, resourceFailure(ctx, err) } @@ -463,22 +513,18 @@ func collectMediaTransfer(ctx context.Context, proxy string, resource mediaResou if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") { return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("ordinary media unavailable")) } - t, h, err := collectResourceTransfer(ctx, proxy, resource.url, resource.host, protocol, true, runner) + t, h, err := collectResourceTransferOptions(ctx, proxy, resource.url, resource.host, protocol, runner, resourceTransferOptions{method: "GET", response: resourceResponseMP4, mediaType: resource.kind}) if err != nil { return assetTransfer{}, GooglevideoPass{}, err } - parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range")) - var end, total int64 - if len(parts) == 3 { - end, _ = strconv.ParseInt(parts[1], 10, 64) - total, _ = strconv.ParseInt(parts[2], 10, 64) - } - if t.Status != 206 || t.ContentType != resource.kind || len(parts) != 3 || end < 15 || end > 65535 || end+1 != int64(len(t.Body)) || total <= end || len(t.Body) < 16 || string(t.Body[4:8]) != "ftyp" || binary.BigEndian.Uint32(t.Body[:4]) < 16 || int(binary.BigEndian.Uint32(t.Body[:4])) > len(t.Body) { + // The shared core has validated and normalized the complete range contract. + _, totalText, _ := strings.Cut(h.Get("Content-Range"), "/") + total, err := strconv.ParseInt(totalText, 10, 64) + if err != nil { return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("invalid media range or container")) } // Retain only normalized byte identity; no signed URL or payload. - t.Size, t.Format, t.BodyHash = len(t.Body), "mp4", fmt.Sprintf("%x", sha256.Sum256(t.Body)) t.Body = nil - p := GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: t.Status, RangeStart: 0, RangeEnd: end, ResourceBytes: total, Bytes: int64(t.Size), FTYP: true, LatencyMS: t.LatencyMS} + p := GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: t.Status, RangeStart: 0, RangeEnd: int64(t.Size) - 1, ResourceBytes: total, Bytes: int64(t.Size), FTYP: true, LatencyMS: t.LatencyMS} return t, p, nil } diff --git a/cmd/zapret-autotune/resource_contract_test.go b/cmd/zapret-autotune/resource_contract_test.go new file mode 100644 index 0000000..78417d9 --- /dev/null +++ b/cmd/zapret-autotune/resource_contract_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "reflect" + "strings" + "testing" + "time" +) + +func TestResourceCoreMediaRangeIntegers(t *testing.T) { + for _, tc := range []struct { + name, contentRange string + total int64 + }{ + {"ordinary", "bytes 0-23/100", 100}, + {"max-valid", "bytes 0-23/9223372036854775807", 9223372036854775807}, + {"total-overflow", "bytes 0-23/9223372036854775808", 0}, + {"total-huge", "bytes 0-23/999999999999999999999999999999999999", 0}, + {"end-overflow", "bytes 0-9223372036854775808/9223372036854775807", 0}, + {"nonnumeric", "bytes 0-23/abc", 0}, + {"unknown", "bytes 0-23/*", 0}, + {"signed", "bytes 0-23/+100", 0}, + {"negative", "bytes 0-23/-100", 0}, + {"signed-end", "bytes 0-+23/100", 0}, + {"negative-end", "bytes 0--23/100", 0}, + {"total-equals-end", "bytes 0-23/23", 0}, + {"total-below-end", "bytes 0-23/22", 0}, + {"end-body-mismatch", "bytes 0-24/100", 0}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := "https://rr1.unit.googlevideo.com/fresh?sig=private-secret" + runner := func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error { + var wire bytes.Buffer + resourceWire(raw, "media", &wire, diag) + fmt.Fprint(out, strings.Replace(wire.String(), "bytes 0-23/100", tc.contentRange, 1)) + return nil + } + transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5://127.0.0.1:1080", raw, "rr1.unit.googlevideo.com", "tls12", runner, resourceTransferOptions{method: "GET", response: resourceResponseMP4, mediaType: "video/mp4"}) + valid := tc.total != 0 + if (err == nil) != valid || !valid && !reflect.DeepEqual(transfer, assetTransfer{}) { + t.Fatalf("core valid=%v transfer=%+v err=%v", valid, transfer, err) + } + transfer, pass, err := collectMediaTransfer(context.Background(), "socks5://127.0.0.1:1080", mediaResource{raw, "rr1.unit.googlevideo.com", "video/mp4"}, "tls12", runner) + if (err == nil) != valid { + t.Fatalf("adapter valid=%v err=%v", valid, err) + } + if valid { + if pass.ResourceBytes != tc.total || pass.Bytes != 24 || pass.RangeEnd != 23 || transfer.Size != 24 { + t.Fatalf("incorrect normalized facts: %+v %+v", transfer, pass) + } + } else if !reflect.DeepEqual(transfer, assetTransfer{}) || !reflect.DeepEqual(pass, GooglevideoPass{}) { + t.Fatalf("rejected range produced evidence: %+v %+v", transfer, pass) + } + }) + } +} + +// Unlike the adapter golden test, these assertions call the shared execution +// entry directly: strict validation must not be left in a legacy-only branch. +func TestResourceCoreTypedContracts(t *testing.T) { + for _, mode := range []string{"pass", "status", "start", "end", "total", "malformed", "ftyp", "box", "decoder"} { + for _, kind := range []resourceResponseKind{resourceResponseGeneric, resourceResponseImage, resourceResponseMP4} { + t.Run(fmt.Sprintf("%d/%s", kind, mode), func(t *testing.T) { + options := resourceTransferOptions{method: "GET", limit: 1024, timeout: time.Second, response: kind, mediaType: "video/mp4"} + raw := "https://i.ytimg.com/fresh?sig=private-secret" + runner := func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error { + var wire bytes.Buffer + wireMode := "" + if kind == resourceResponseMP4 { + wireMode = "media" + } + if mode == "decoder" && kind != resourceResponseMP4 { + wireMode = "type" + } + resourceWire(raw, wireMode, &wire, diag) + data := wire.String() + switch mode { + case "status": + data = strings.ReplaceAll(data, "206", "200") + if kind != resourceResponseMP4 { + data = strings.ReplaceAll(data, "200", "201") + } + case "start": + data = strings.Replace(data, "bytes 0-23/100", "bytes 1-24/100", 1) + case "end": + data = strings.Replace(data, "bytes 0-23/100", "bytes 0-24/100", 1) + case "total": + data = strings.Replace(data, "bytes 0-23/100", "bytes 0-23/23", 1) + case "malformed": + data = strings.Replace(data, "bytes 0-23/100", "bytes 0-23/*", 1) + case "ftyp": + data = strings.Replace(data, "ftyp", "xxxx", 1) + case "box": + data = strings.Replace(data, string([]byte{0, 0, 0, 24}), string([]byte{0, 0, 0, 25}), 1) + } + fmt.Fprint(out, data) + return nil + } + transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5://127.0.0.1:1080", raw, "i.ytimg.com", "tls12", runner, options) + valid := kind == resourceResponseGeneric || mode == "pass" || kind == resourceResponseImage && mode != "decoder" && mode != "status" || kind == resourceResponseMP4 && mode == "decoder" + if (err == nil) != valid { + t.Fatalf("valid=%v err=%v", valid, err) + } + if valid && kind != resourceResponseGeneric && (transfer.Size != len(transfer.Body) || transfer.BodyHash == "" || transfer.Format == "") { + t.Fatal("strict normalization not performed by core") + } + }) + } + } +} + +func TestResourceCoreGenericExpectationFacts(t *testing.T) { + raw := "https://i.ytimg.com/private" + runner := func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error { + resourceWire(raw, "", out, diag) + return nil + } + expect := resourceCheckExpect{Statuses: []int{201}, MaxBytes: 1024} + transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5://127.0.0.1:1080", raw, "i.ytimg.com", "tls12", runner, resourceTransferOptions{method: "GET", limit: 1024, timeout: time.Second, expect: &expect}) + if err != errResourceExpectation || transfer.Status != 200 || len(transfer.Body) != len(resourceFixture()) || transfer.LatencyMS != 100 { + t.Fatalf("core did not validate generic expectation with intact facts: %+v %v", transfer, err) + } +} diff --git a/cmd/zapret-autotune/resource_core_parity_test.go b/cmd/zapret-autotune/resource_core_parity_test.go new file mode 100644 index 0000000..bc8a1d5 --- /dev/null +++ b/cmd/zapret-autotune/resource_core_parity_test.go @@ -0,0 +1,193 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net" + "reflect" + "strings" + "testing" +) + +// Exercise the actual legacy adapters and generic producer with the same wire +// mutations. This test also runs unchanged against the pre-migration source. +func TestResourceCoreAdapterParity(t *testing.T) { + for _, adapter := range []string{"image", "media", "GET", "HEAD"} { + for _, protocol := range []string{"tls12", "tls13"} { + for _, mode := range []string{"pass", "verify", "origin", "redirect", "tls", "framing", "oversize", "range", "ftyp", "decoder"} { + t.Run(adapter+"/"+protocol+"/"+mode, func(t *testing.T) { + host := "i.ytimg.com" + if adapter == "media" { + host = "rr1.unit.googlevideo.com" + } + raw := "https://" + host + "/fresh?sig=private-secret" + calls := 0 + runner := func(_ context.Context, args []string, stdin io.Reader, out, diag io.Writer) error { + calls++ + argv := strings.Join(args, " ") + for _, required := range []string{"--disable --silent", "--http1.1 --raw", "--noproxy --proxy socks5h://127.0.0.1:1080", "--proto =https", "--suppress-connect-headers", "--config -"} { + if !strings.Contains(argv, required) { + t.Fatalf("missing %s", required) + } + } + if strings.Contains(argv, "private-secret") || strings.Contains(argv, host) || strings.Contains(argv, "--insecure") || strings.Contains(argv, "--location") { + t.Fatal("unsafe argv") + } + if strings.Contains(argv, "--range 0-65535") != (adapter == "media") || strings.Contains(argv, "--head") != (adapter == "HEAD") { + t.Fatal("method/range changed") + } + config, _ := io.ReadAll(stdin) + if !strings.Contains(string(config), raw) { + t.Fatal("URL not in private stdin") + } + if adapter == "GET" || adapter == "HEAD" { + if !strings.Contains(string(config), `connect-to = "i.ytimg.com:443:93.184.216.34:443"`) || !strings.Contains(string(config), "Authorization: private-header") { + t.Fatal("private header or CONNECT pin lost") + } + } + var wire, diagnostic bytes.Buffer + wireMode := "" + if adapter == "media" { + wireMode = "media" + } + if mode == "decoder" && adapter == "image" { + wireMode = "type" + } + resourceWire(raw, wireMode, &wire, &diagnostic) + data := wire.String() + if adapter == "HEAD" { + start := strings.Index(data, "\r\n\r\n") + 4 + end := strings.LastIndex(data, resourceTrailer) + data = data[:start] + data[end:] + data = strings.Replace(data, fmt.Sprintf(`"size_download":%d`, len(resourceFixture())), `"size_download":0`, 1) + } + switch mode { + case "verify": + data = strings.Replace(data, `"ssl_verify_result":0`, `"ssl_verify_result":60`, 1) + case "origin": + data = strings.Replace(data, `"url_effective":"https://`, `"url_effective":"https://wrong.`, 1) + case "redirect": + data = strings.Replace(data, `"num_redirects":0`, `"num_redirects":1`, 1) + case "framing": + data = strings.Replace(data, "Content-Length:", "Content-Length: 1\r\nContent-Length:", 1) + case "oversize": + _, err := out.Write(make([]byte, 6<<20)) + return err + case "range": + data = strings.Replace(data, "bytes 0-23/100", "bytes 1-24/100", 1) + case "ftyp": + data = strings.Replace(data, "ftyp", "xxxx", 1) + } + fmt.Fprint(out, data) + actual := "TLSv1.2" + if protocol == "tls13" { + actual = "TLSv1.3" + } + if mode == "tls" { + actual = "TLSv1.1" + } + fmt.Fprintln(diag, "SSL connection using "+actual) + return nil + } + valid := mode == "pass" || mode == "range" && adapter != "media" || mode == "ftyp" && adapter != "media" || mode == "decoder" && adapter != "image" + var err error + var transfer assetTransfer + switch adapter { + case "image": + var m assetMeasurement + m, err = collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", raw, host, protocol, "candidate", runner) + if err == nil { + transfer = m.Transfers[0] + if calls != 3 || m.Source != "image-transfer" || m.Endpoint != host || m.Protocol != protocol || m.Args != "candidate" { + t.Fatal("legacy measurement changed") + } + } + case "media": + var pass GooglevideoPass + transfer, pass, err = collectMediaTransfer(context.Background(), "socks5://127.0.0.1:1080", mediaResource{raw, host, "video/mp4"}, protocol, runner) + if err == nil && !reflect.DeepEqual(pass, GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: 206, RangeStart: 0, RangeEnd: 23, ResourceBytes: 100, Bytes: 24, FTYP: true, LatencyMS: 100}) { + t.Fatalf("legacy pass changed: %+v", pass) + } + default: + d := resourceCheckDeps{guard: func() error { return nil }, resolve: func(context.Context, string, string) ([]net.IP, error) { + return []net.IP{net.ParseIP("93.184.216.34")}, nil + }, runner: runner} + r := resourceCheckSpec{ID: "r", Group: "g", URL: raw, host: host, Method: adapter, TimeoutMS: 1000, Expect: resourceCheckExpect{Statuses: []int{200}, ContentType: "image/png", MaxBytes: 1024}, privateConfig: "header = \"Authorization: private-header\"\n"} + result, e := runResourceProtocol(context.Background(), "s", r, protocol, "dns", "socks5://127.0.0.1:1080", d) + err = e + if valid { + wantBytes := len(resourceFixture()) + if adapter == "HEAD" { + wantBytes = 0 + } + if result != (resourceCheckResult{Service: "s", Resource: "r", Group: "g", TLS: protocol, Outcome: "pass", Status: 200, Bytes: wantBytes, TimeMS: 100}) { + t.Fatalf("generic result changed: %+v", result) + } + } else if result.Outcome == "pass" { + t.Fatal("unsafe generic pass") + } + if err == nil && !valid { + err = fmt.Errorf("rejected transfer") + } + } + if (err == nil) != valid { + t.Fatalf("valid=%v err=%v", valid, err) + } + if valid && (adapter == "image" || adapter == "media") { + body, format, status, mime := resourceFixture(), "png", 200, "image/png" + if adapter == "media" { + body, format, status, mime = []byte{0, 0, 0, 24, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm', 0, 0, 0, 0, 'i', 's', 'o', 'm', 'm', 'p', '4', '2'}, "mp4", 206, "video/mp4" + } + version := uint16(0x0303) + if protocol == "tls13" { + version = 0x0304 + } + verify := 0 + want := assetTransfer{EffectiveHost: host, SSLVerifyResult: &verify, TLSVersion: version, Status: status, ContentType: mime, Size: len(body), Format: format, BodyHash: fmt.Sprintf("%x", sha256.Sum256(body)), LatencyMS: 100, SpeedKBPS: 10000.0 / 1024} + if !reflect.DeepEqual(transfer, want) { + t.Fatalf("normalized facts changed: got %+v want %+v", transfer, want) + } + encoded, _ := json.Marshal(transfer) + if strings.Contains(string(encoded), "private-secret") { + t.Fatal("private evidence") + } + } + }) + } + } + } +} + +func TestResourceCoreGenericMismatchParity(t *testing.T) { + for _, field := range []string{"status", "mime", "bytes"} { + for _, qualification := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/qualification=%v", field, qualification), func(t *testing.T) { + raw := "https://i.ytimg.com/fresh?sig=private-secret" + r := resourceCheckSpec{ID: "r", Group: "g", URL: raw, host: "i.ytimg.com", Method: "GET", TimeoutMS: 1000, Expect: resourceCheckExpect{Statuses: []int{200}, ContentType: "image/png", MaxBytes: 1024}} + switch field { + case "status": + r.Expect.Statuses = []int{201} + case "mime": + r.Expect.ContentType = "image/jpeg" + case "bytes": + r.Expect.MinBytes = 1000 + } + d := resourceCheckDeps{qualification: qualification, guard: func() error { return nil }, resolve: func(context.Context, string, string) ([]net.IP, error) { + return []net.IP{net.ParseIP("93.184.216.34")}, nil + }, runner: func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error { + resourceWire(raw, "", out, diag) + return nil + }} + result, err := runResourceProtocol(context.Background(), "s", r, "tls12", "dns", "socks5://127.0.0.1:1080", d) + want := resourceCheckResult{Service: "s", Resource: "r", Group: "g", TLS: "tls12", Outcome: "expectation_failed", Bytes: len(resourceFixture()), TimeMS: 100, Status: 200} + if err != nil || result != want { + t.Fatalf("result=%+v err=%v", result, err) + } + }) + } + } +} diff --git a/cmd/zapret-autotune/resource_qualify_test.go b/cmd/zapret-autotune/resource_qualify_test.go index bba5758..730a9d4 100644 --- a/cmd/zapret-autotune/resource_qualify_test.go +++ b/cmd/zapret-autotune/resource_qualify_test.go @@ -310,7 +310,7 @@ func TestResourceQualifierTypedPeerResponseBoundary(t *testing.T) { configuredWire("https://a.example/", "GET", "tls12", mode, out, diag) return nil } - _, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://a.example/", "a.example", "tls12", false, runner, resourceTransferOptions{generic: true, method: "GET", limit: 100, timeout: time.Second}) + _, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://a.example/", "a.example", "tls12", runner, resourceTransferOptions{method: "GET", limit: 100, timeout: time.Second}) var peer *resourcePeerResponseRejection var fatal *managedRecoveryFatal if mode == "redirect" {