diff --git a/RESOURCE_PLAN.md b/RESOURCE_PLAN.md index 1056892..6327e85 100644 --- a/RESOURCE_PLAN.md +++ b/RESOURCE_PLAN.md @@ -1,5 +1,30 @@ # Resource compatibility planner and isolated qualifier +## First collector migration (locally verified/reviewed; remote acceptance pending) + +Private typed response contracts now select generic GET/HEAD, full decoded image, +or bounded MP4 range validation inside the existing shared HTTPS transfer core. +The redundant legacy forwarding wrapper and media boolean are removed; image and +media adapters retain their measurements and GooglevideoPass DTOs. Generic v1 +status/MIME/byte expectations use the same validation entry, with a distinct +completed-mismatch result retaining the same measured facts. No public Range config, schema, +proof importer, registry or dependency was added. + +Injected-wire tests exercise the real generic producer and legacy callers, both +TLS versions, normalized byte/hash/timing facts, private stdin/pinning, origin, +redirect, framing, bounds and strict range/ftyp/image decoder rejection. Direct +core tests ensure strict validation is not an unused legacy-only helper. +MP4 requests use Range `0-65535` (a 64 KiB upper bound, not an exact response +size requirement). The core accepts validated 16–65536-byte ranges; the existing +producer additionally requires at least 1024 measured bytes. Content-Range end +and total must parse without overflow, with total greater than end; rejected or +unknown totals cannot produce a media PASS DTO. +This is collector migration only: legacy owned-worker production, three-CDN +sampling/warmup, native suffix adaptive learning and image/avatar readiness gates +remain unchanged. Generic matrix/controller/journal/profile replacement still +requires typed discovery/range proof and suffix/adaptive-learning capability parity. +No remote result, service READY or full controller migration is claimed here. + `plan` remains **proposal-only** and never changes runtime or config. Explicit generic startup apply/replay and the per-PID-1 failure latch have passed bounded remote acceptance with independent privileged verification (see below). diff --git a/cmd/zapret-autotune/resource_check.go b/cmd/zapret-autotune/resource_check.go index c4af956..acd796b 100644 --- a/cmd/zapret-autotune/resource_check.go +++ b/cmd/zapret-autotune/resource_check.go @@ -402,22 +402,20 @@ func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpe return result, qualificationOperation(err, stagePostDNSGuard) } pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443") - options := resourceTransferOptions{generic: true, method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n"} - transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, false, d.runner, options) - if d.qualification && err != nil { + options := resourceTransferOptions{method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n", expect: &r.Expect} + transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, d.runner, options) + if d.qualification && err != nil && err != errResourceExpectation { knownTimeout := probeCtx.Err() != nil && managedRecoveryTimeoutSafe(err) if !knownTimeout && !resourceRejectionOnly(err) && !managedAssetTransferNonpass(err) { return result, qualificationOperation(err, stageTransfer) } } result.Outcome = "transfer_failed" - if err == nil { + if err == nil || err == errResourceExpectation { result.Bytes, result.TimeMS, result.Status = len(transfer.Body), transfer.LatencyMS, transfer.Status result.Outcome = "expectation_failed" - for _, status := range r.Expect.Statuses { - if status == transfer.Status && len(transfer.Body) >= r.Expect.MinBytes && (r.Expect.ContentType == "" || transfer.ContentType == r.Expect.ContentType) { - result.Outcome = "pass" - } + if err == nil { + result.Outcome = "pass" } } } diff --git a/cmd/zapret-autotune/resource_check_test.go b/cmd/zapret-autotune/resource_check_test.go index 744ed5b..243ee55 100644 --- a/cmd/zapret-autotune/resource_check_test.go +++ b/cmd/zapret-autotune/resource_check_test.go @@ -365,7 +365,7 @@ func TestResourceCheckTransferContract(t *testing.T) { configuredWire("https://example.com/", method, "tls12", mode, out, diag) return nil } - transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://example.com/", "example.com", "tls12", false, runner, resourceTransferOptions{generic: true, method: method, limit: 100, timeout: time.Second}) + transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://example.com/", "example.com", "tls12", runner, resourceTransferOptions{method: method, limit: 100, timeout: time.Second}) valid := mode == "pass" || mode == "chunked" || method == "HEAD" && mode == "truncated" if (err == nil) != valid { t.Fatalf("err=%v", err) @@ -484,7 +484,7 @@ func TestResourceCheckCurlSOCKSPinnedDestination(t *testing.T) { }() ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) defer cancel() - _, _, err = collectResourceTransferOptions(ctx, "socks5h://"+listener.Addr().String(), "https://example.com/private?secret=value", "example.com", "tls12", false, runResourceCurl, resourceTransferOptions{generic: true, method: "GET", limit: 100, timeout: 3 * time.Second, config: "connect-to = \"example.com:443:93.184.216.34:443\"\n"}) + _, _, err = collectResourceTransferOptions(ctx, "socks5h://"+listener.Addr().String(), "https://example.com/private?secret=value", "example.com", "tls12", runResourceCurl, resourceTransferOptions{method: "GET", limit: 100, timeout: 3 * time.Second, config: "connect-to = \"example.com:443:93.184.216.34:443\"\n"}) if err == nil { t.Fatal("fixture unexpectedly transferred") } diff --git a/cmd/zapret-autotune/resource_collector.go b/cmd/zapret-autotune/resource_collector.go index d0c7a85..2c46cfa 100644 --- a/cmd/zapret-autotune/resource_collector.go +++ b/cmd/zapret-autotune/resource_collector.go @@ -229,26 +229,39 @@ func playerMediaResource(body []byte) (mediaResource, mediaCapability, error) { const resourceTrailer = "\nAUTOTUNE_RESOURCE_METRICS:" -func collectResourceTransfer(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner) (assetTransfer, http.Header, error) { - return collectResourceTransferOptions(ctx, proxy, raw, host, protocol, media, runner, resourceTransferOptions{}) -} +type resourceResponseKind uint8 + +// A completed generic mismatch retains measured facts, unlike unsafe transfers. +var errResourceExpectation = errors.New("resource expectation failed") + +const ( + resourceResponseGeneric resourceResponseKind = iota + resourceResponseImage + resourceResponseMP4 +) type resourceTransferOptions struct { method string limit int timeout time.Duration // Private curl config only: never placed in argv or public evidence. - config string - generic bool + config string + response resourceResponseKind + mediaType string + expect *resourceCheckExpect } -func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner, options resourceTransferOptions) (assetTransfer, http.Header, error) { +func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, protocol string, runner subprocessRunner, options resourceTransferOptions) (assetTransfer, http.Header, error) { + generic := options.response == resourceResponseGeneric reason := reasonOrigin fail := func(err error) (assetTransfer, http.Header, error) { failure := resourceFailure(ctx, err) failure.(*resourceError).reason = reason return assetTransfer{}, nil, failure } + if options.response > resourceResponseMP4 || (!generic && options.method != "GET") || (options.response == resourceResponseMP4 && options.mediaType != "video/mp4" && options.mediaType != "audio/mp4") { + return fail(errors.New("invalid response contract")) + } proxy, proxyErr := managedProbeProxy(proxy) if proxyErr != nil || runner == nil { return fail(errors.New("invalid owned proxy or runner")) @@ -263,19 +276,19 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto return fail(errors.New("invalid TLS protocol")) } limit := 5 << 20 - if media { + if options.response == resourceResponseMP4 { limit = 65536 } - if options.generic { + if generic { limit = options.limit } // Qualification uses HTTP/1.1, not browser HTTP/2. --raw preserves native // transfer framing so net/http, rather than curl and Go twice, decodes it. args := []string{"--disable", "--silent", "--show-error", "--verbose", "--http1.1", "--raw", "--noproxy", "", "--proxy", proxy, "--proto", "=https", tlsFlag, "--tls-max", maxTLS, "--max-time", "30", "--max-filesize", strconv.Itoa(limit), "--suppress-connect-headers", "--include", "--write-out", resourceTrailer + "%{json}", "--config", "-"} - if media { + if options.response == resourceResponseMP4 { args = append(args, "--range", "0-65535") } - if options.generic { + if generic { if (options.method != "GET" && options.method != "HEAD") || options.limit < 1 || options.limit > 5<<20 || options.timeout < 100*time.Millisecond || options.timeout > 30*time.Second { return fail(errors.New("invalid transfer options")) } @@ -288,7 +301,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto config := "url = \"" + strings.NewReplacer("\\", "\\\\", "\"", "\\\"").Replace(raw) + "\"\n" config += options.config timeout := 35 * time.Second - if options.generic { + if generic { timeout = options.timeout } childCtx, cancel := context.WithTimeout(ctx, timeout) @@ -366,10 +379,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto return fail(errors.New("ambiguous response framing")) } r := bufio.NewReader(bytes.NewReader(wire)) - var request *http.Request - if options.generic { - request = &http.Request{Method: options.method} - } + request := &http.Request{Method: options.method} response, err := http.ReadResponse(r, request) if err != nil { return fail(errors.New("invalid response headers")) @@ -377,10 +387,10 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto defer response.Body.Close() // CONNECT headers are suppressed by curl. Interim/duplicate responses and // unrequested versions are outside this contract, never heuristically skipped. - if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || (!options.generic && response.Header.Get("Content-Type") == "") { + if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || (!generic && response.Header.Get("Content-Type") == "") { return fail(errors.New("unsupported response contract")) } - if options.generic && response.StatusCode >= 300 && response.StatusCode < 400 { + if generic && response.StatusCode >= 300 && response.StatusCode < 400 { return fail(&resourcePeerResponseRejection{error: errors.New("redirect rejected")}) } reason = reasonBody @@ -389,27 +399,74 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto // limit and durable byte identity apply to the fully decoded payload instead. wireSize := len(wire) - headerEnd - 4 _, remaining := r.Peek(1) - if err != nil || remaining != io.EOF || (!options.generic && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status { + if err != nil || remaining != io.EOF || (!generic && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status { return fail(errors.New("incomplete or oversized resource body")) } reason = reasonMetrics for _, n := range []float64{metrics.Time, metrics.Speed} { - if n < 0 || (!options.generic && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) { + if n < 0 || (!generic && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) { return fail(errors.New("invalid resource metrics")) } } - if options.generic && metrics.Time > options.timeout.Seconds()+1 { + if generic && metrics.Time > options.timeout.Seconds()+1 { return fail(errors.New("resource timing exceeds budget")) } reason = reasonMIME contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type")) - if err != nil && (!options.generic || response.Header.Get("Content-Type") != "") { + if err != nil && (!generic || response.Header.Get("Content-Type") != "") { return fail(errors.New("invalid content type")) } t := assetTransfer{EffectiveHost: host, SSLVerifyResult: metrics.Verify, Redirects: *metrics.Redirects, TLSVersion: version, Status: metrics.Status, ContentType: contentType, Body: body, LatencyMS: metrics.Time * 1000, SpeedKBPS: metrics.Speed / 1024} + if err := options.validateResponse(&t, response.Header); err != nil { + if err == errResourceExpectation { + return t, response.Header, err + } + return assetTransfer{}, nil, resourceFailure(ctx, err) + } return t, response.Header, nil } +// Adapter contracts run only after the common origin/TLS/framing/bounds checks. +// Generic v1 deliberately has no range or decoded-image expectation. +func (options resourceTransferOptions) validateResponse(t *assetTransfer, h http.Header) error { + switch options.response { + case resourceResponseGeneric: + if options.expect == nil { + return nil + } + e := options.expect + for _, status := range e.Statuses { + if t.Status == status && len(t.Body) >= e.MinBytes && len(t.Body) <= e.MaxBytes && (e.ContentType == "" || t.ContentType == e.ContentType) { + return nil + } + } + return errResourceExpectation + case resourceResponseImage: + if t.Status != 200 { + return errors.New("image HTTP status") + } + return normalizeAssetBody(t) + case resourceResponseMP4: + parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range")) + var end, total int64 + if len(parts) == 3 { + var endErr, totalErr error + end, endErr = strconv.ParseInt(parts[1], 10, 64) + total, totalErr = strconv.ParseInt(parts[2], 10, 64) + if endErr != nil || totalErr != nil { + return errors.New("invalid media range or container") + } + } + if t.Status != 206 || t.ContentType != options.mediaType || len(parts) != 3 || end < 15 || end > 65535 || end+1 != int64(len(t.Body)) || total <= end || len(t.Body) < 16 || string(t.Body[4:8]) != "ftyp" || binary.BigEndian.Uint32(t.Body[:4]) < 16 || int(binary.BigEndian.Uint32(t.Body[:4])) > len(t.Body) { + return errors.New("invalid media range or container") + } + t.Size, t.Format, t.BodyHash = len(t.Body), "mp4", fmt.Sprintf("%x", sha256.Sum256(t.Body)) + return nil + default: + return errors.New("invalid response contract") + } +} + // The ordinary 3/3 sampler is shared by image and media qualification. func sampleResource(ctx context.Context, transfer func(context.Context) (assetTransfer, error)) ([]assetTransfer, error) { var samples []assetTransfer @@ -429,14 +486,7 @@ func collectAssetMeasurement(ctx context.Context, proxy, raw, host, protocol, ar return m, resourceFailure(ctx, errors.New("invalid image endpoint")) } samples, err := sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) { - t, _, err := collectResourceTransfer(ctx, proxy, raw, host, protocol, false, runner) - if err == nil { - if t.Status != 200 { - err = errors.New("image HTTP status") - } else { - err = normalizeAssetBody(&t) - } - } + t, _, err := collectResourceTransferOptions(ctx, proxy, raw, host, protocol, runner, resourceTransferOptions{method: "GET", response: resourceResponseImage}) if err != nil { return assetTransfer{}, resourceFailure(ctx, err) } @@ -463,22 +513,18 @@ func collectMediaTransfer(ctx context.Context, proxy string, resource mediaResou if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") { return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("ordinary media unavailable")) } - t, h, err := collectResourceTransfer(ctx, proxy, resource.url, resource.host, protocol, true, runner) + t, h, err := collectResourceTransferOptions(ctx, proxy, resource.url, resource.host, protocol, runner, resourceTransferOptions{method: "GET", response: resourceResponseMP4, mediaType: resource.kind}) if err != nil { return assetTransfer{}, GooglevideoPass{}, err } - parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range")) - var end, total int64 - if len(parts) == 3 { - end, _ = strconv.ParseInt(parts[1], 10, 64) - total, _ = strconv.ParseInt(parts[2], 10, 64) - } - if t.Status != 206 || t.ContentType != resource.kind || len(parts) != 3 || end < 15 || end > 65535 || end+1 != int64(len(t.Body)) || total <= end || len(t.Body) < 16 || string(t.Body[4:8]) != "ftyp" || binary.BigEndian.Uint32(t.Body[:4]) < 16 || int(binary.BigEndian.Uint32(t.Body[:4])) > len(t.Body) { + // The shared core has validated and normalized the complete range contract. + _, totalText, _ := strings.Cut(h.Get("Content-Range"), "/") + total, err := strconv.ParseInt(totalText, 10, 64) + if err != nil { return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("invalid media range or container")) } // Retain only normalized byte identity; no signed URL or payload. - t.Size, t.Format, t.BodyHash = len(t.Body), "mp4", fmt.Sprintf("%x", sha256.Sum256(t.Body)) t.Body = nil - p := GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: t.Status, RangeStart: 0, RangeEnd: end, ResourceBytes: total, Bytes: int64(t.Size), FTYP: true, LatencyMS: t.LatencyMS} + p := GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: t.Status, RangeStart: 0, RangeEnd: int64(t.Size) - 1, ResourceBytes: total, Bytes: int64(t.Size), FTYP: true, LatencyMS: t.LatencyMS} return t, p, nil } diff --git a/cmd/zapret-autotune/resource_contract_test.go b/cmd/zapret-autotune/resource_contract_test.go new file mode 100644 index 0000000..78417d9 --- /dev/null +++ b/cmd/zapret-autotune/resource_contract_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "reflect" + "strings" + "testing" + "time" +) + +func TestResourceCoreMediaRangeIntegers(t *testing.T) { + for _, tc := range []struct { + name, contentRange string + total int64 + }{ + {"ordinary", "bytes 0-23/100", 100}, + {"max-valid", "bytes 0-23/9223372036854775807", 9223372036854775807}, + {"total-overflow", "bytes 0-23/9223372036854775808", 0}, + {"total-huge", "bytes 0-23/999999999999999999999999999999999999", 0}, + {"end-overflow", "bytes 0-9223372036854775808/9223372036854775807", 0}, + {"nonnumeric", "bytes 0-23/abc", 0}, + {"unknown", "bytes 0-23/*", 0}, + {"signed", "bytes 0-23/+100", 0}, + {"negative", "bytes 0-23/-100", 0}, + {"signed-end", "bytes 0-+23/100", 0}, + {"negative-end", "bytes 0--23/100", 0}, + {"total-equals-end", "bytes 0-23/23", 0}, + {"total-below-end", "bytes 0-23/22", 0}, + {"end-body-mismatch", "bytes 0-24/100", 0}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := "https://rr1.unit.googlevideo.com/fresh?sig=private-secret" + runner := func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error { + var wire bytes.Buffer + resourceWire(raw, "media", &wire, diag) + fmt.Fprint(out, strings.Replace(wire.String(), "bytes 0-23/100", tc.contentRange, 1)) + return nil + } + transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5://127.0.0.1:1080", raw, "rr1.unit.googlevideo.com", "tls12", runner, resourceTransferOptions{method: "GET", response: resourceResponseMP4, mediaType: "video/mp4"}) + valid := tc.total != 0 + if (err == nil) != valid || !valid && !reflect.DeepEqual(transfer, assetTransfer{}) { + t.Fatalf("core valid=%v transfer=%+v err=%v", valid, transfer, err) + } + transfer, pass, err := collectMediaTransfer(context.Background(), "socks5://127.0.0.1:1080", mediaResource{raw, "rr1.unit.googlevideo.com", "video/mp4"}, "tls12", runner) + if (err == nil) != valid { + t.Fatalf("adapter valid=%v err=%v", valid, err) + } + if valid { + if pass.ResourceBytes != tc.total || pass.Bytes != 24 || pass.RangeEnd != 23 || transfer.Size != 24 { + t.Fatalf("incorrect normalized facts: %+v %+v", transfer, pass) + } + } else if !reflect.DeepEqual(transfer, assetTransfer{}) || !reflect.DeepEqual(pass, GooglevideoPass{}) { + t.Fatalf("rejected range produced evidence: %+v %+v", transfer, pass) + } + }) + } +} + +// Unlike the adapter golden test, these assertions call the shared execution +// entry directly: strict validation must not be left in a legacy-only branch. +func TestResourceCoreTypedContracts(t *testing.T) { + for _, mode := range []string{"pass", "status", "start", "end", "total", "malformed", "ftyp", "box", "decoder"} { + for _, kind := range []resourceResponseKind{resourceResponseGeneric, resourceResponseImage, resourceResponseMP4} { + t.Run(fmt.Sprintf("%d/%s", kind, mode), func(t *testing.T) { + options := resourceTransferOptions{method: "GET", limit: 1024, timeout: time.Second, response: kind, mediaType: "video/mp4"} + raw := "https://i.ytimg.com/fresh?sig=private-secret" + runner := func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error { + var wire bytes.Buffer + wireMode := "" + if kind == resourceResponseMP4 { + wireMode = "media" + } + if mode == "decoder" && kind != resourceResponseMP4 { + wireMode = "type" + } + resourceWire(raw, wireMode, &wire, diag) + data := wire.String() + switch mode { + case "status": + data = strings.ReplaceAll(data, "206", "200") + if kind != resourceResponseMP4 { + data = strings.ReplaceAll(data, "200", "201") + } + case "start": + data = strings.Replace(data, "bytes 0-23/100", "bytes 1-24/100", 1) + case "end": + data = strings.Replace(data, "bytes 0-23/100", "bytes 0-24/100", 1) + case "total": + data = strings.Replace(data, "bytes 0-23/100", "bytes 0-23/23", 1) + case "malformed": + data = strings.Replace(data, "bytes 0-23/100", "bytes 0-23/*", 1) + case "ftyp": + data = strings.Replace(data, "ftyp", "xxxx", 1) + case "box": + data = strings.Replace(data, string([]byte{0, 0, 0, 24}), string([]byte{0, 0, 0, 25}), 1) + } + fmt.Fprint(out, data) + return nil + } + transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5://127.0.0.1:1080", raw, "i.ytimg.com", "tls12", runner, options) + valid := kind == resourceResponseGeneric || mode == "pass" || kind == resourceResponseImage && mode != "decoder" && mode != "status" || kind == resourceResponseMP4 && mode == "decoder" + if (err == nil) != valid { + t.Fatalf("valid=%v err=%v", valid, err) + } + if valid && kind != resourceResponseGeneric && (transfer.Size != len(transfer.Body) || transfer.BodyHash == "" || transfer.Format == "") { + t.Fatal("strict normalization not performed by core") + } + }) + } + } +} + +func TestResourceCoreGenericExpectationFacts(t *testing.T) { + raw := "https://i.ytimg.com/private" + runner := func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error { + resourceWire(raw, "", out, diag) + return nil + } + expect := resourceCheckExpect{Statuses: []int{201}, MaxBytes: 1024} + transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5://127.0.0.1:1080", raw, "i.ytimg.com", "tls12", runner, resourceTransferOptions{method: "GET", limit: 1024, timeout: time.Second, expect: &expect}) + if err != errResourceExpectation || transfer.Status != 200 || len(transfer.Body) != len(resourceFixture()) || transfer.LatencyMS != 100 { + t.Fatalf("core did not validate generic expectation with intact facts: %+v %v", transfer, err) + } +} diff --git a/cmd/zapret-autotune/resource_core_parity_test.go b/cmd/zapret-autotune/resource_core_parity_test.go new file mode 100644 index 0000000..bc8a1d5 --- /dev/null +++ b/cmd/zapret-autotune/resource_core_parity_test.go @@ -0,0 +1,193 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net" + "reflect" + "strings" + "testing" +) + +// Exercise the actual legacy adapters and generic producer with the same wire +// mutations. This test also runs unchanged against the pre-migration source. +func TestResourceCoreAdapterParity(t *testing.T) { + for _, adapter := range []string{"image", "media", "GET", "HEAD"} { + for _, protocol := range []string{"tls12", "tls13"} { + for _, mode := range []string{"pass", "verify", "origin", "redirect", "tls", "framing", "oversize", "range", "ftyp", "decoder"} { + t.Run(adapter+"/"+protocol+"/"+mode, func(t *testing.T) { + host := "i.ytimg.com" + if adapter == "media" { + host = "rr1.unit.googlevideo.com" + } + raw := "https://" + host + "/fresh?sig=private-secret" + calls := 0 + runner := func(_ context.Context, args []string, stdin io.Reader, out, diag io.Writer) error { + calls++ + argv := strings.Join(args, " ") + for _, required := range []string{"--disable --silent", "--http1.1 --raw", "--noproxy --proxy socks5h://127.0.0.1:1080", "--proto =https", "--suppress-connect-headers", "--config -"} { + if !strings.Contains(argv, required) { + t.Fatalf("missing %s", required) + } + } + if strings.Contains(argv, "private-secret") || strings.Contains(argv, host) || strings.Contains(argv, "--insecure") || strings.Contains(argv, "--location") { + t.Fatal("unsafe argv") + } + if strings.Contains(argv, "--range 0-65535") != (adapter == "media") || strings.Contains(argv, "--head") != (adapter == "HEAD") { + t.Fatal("method/range changed") + } + config, _ := io.ReadAll(stdin) + if !strings.Contains(string(config), raw) { + t.Fatal("URL not in private stdin") + } + if adapter == "GET" || adapter == "HEAD" { + if !strings.Contains(string(config), `connect-to = "i.ytimg.com:443:93.184.216.34:443"`) || !strings.Contains(string(config), "Authorization: private-header") { + t.Fatal("private header or CONNECT pin lost") + } + } + var wire, diagnostic bytes.Buffer + wireMode := "" + if adapter == "media" { + wireMode = "media" + } + if mode == "decoder" && adapter == "image" { + wireMode = "type" + } + resourceWire(raw, wireMode, &wire, &diagnostic) + data := wire.String() + if adapter == "HEAD" { + start := strings.Index(data, "\r\n\r\n") + 4 + end := strings.LastIndex(data, resourceTrailer) + data = data[:start] + data[end:] + data = strings.Replace(data, fmt.Sprintf(`"size_download":%d`, len(resourceFixture())), `"size_download":0`, 1) + } + switch mode { + case "verify": + data = strings.Replace(data, `"ssl_verify_result":0`, `"ssl_verify_result":60`, 1) + case "origin": + data = strings.Replace(data, `"url_effective":"https://`, `"url_effective":"https://wrong.`, 1) + case "redirect": + data = strings.Replace(data, `"num_redirects":0`, `"num_redirects":1`, 1) + case "framing": + data = strings.Replace(data, "Content-Length:", "Content-Length: 1\r\nContent-Length:", 1) + case "oversize": + _, err := out.Write(make([]byte, 6<<20)) + return err + case "range": + data = strings.Replace(data, "bytes 0-23/100", "bytes 1-24/100", 1) + case "ftyp": + data = strings.Replace(data, "ftyp", "xxxx", 1) + } + fmt.Fprint(out, data) + actual := "TLSv1.2" + if protocol == "tls13" { + actual = "TLSv1.3" + } + if mode == "tls" { + actual = "TLSv1.1" + } + fmt.Fprintln(diag, "SSL connection using "+actual) + return nil + } + valid := mode == "pass" || mode == "range" && adapter != "media" || mode == "ftyp" && adapter != "media" || mode == "decoder" && adapter != "image" + var err error + var transfer assetTransfer + switch adapter { + case "image": + var m assetMeasurement + m, err = collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", raw, host, protocol, "candidate", runner) + if err == nil { + transfer = m.Transfers[0] + if calls != 3 || m.Source != "image-transfer" || m.Endpoint != host || m.Protocol != protocol || m.Args != "candidate" { + t.Fatal("legacy measurement changed") + } + } + case "media": + var pass GooglevideoPass + transfer, pass, err = collectMediaTransfer(context.Background(), "socks5://127.0.0.1:1080", mediaResource{raw, host, "video/mp4"}, protocol, runner) + if err == nil && !reflect.DeepEqual(pass, GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: 206, RangeStart: 0, RangeEnd: 23, ResourceBytes: 100, Bytes: 24, FTYP: true, LatencyMS: 100}) { + t.Fatalf("legacy pass changed: %+v", pass) + } + default: + d := resourceCheckDeps{guard: func() error { return nil }, resolve: func(context.Context, string, string) ([]net.IP, error) { + return []net.IP{net.ParseIP("93.184.216.34")}, nil + }, runner: runner} + r := resourceCheckSpec{ID: "r", Group: "g", URL: raw, host: host, Method: adapter, TimeoutMS: 1000, Expect: resourceCheckExpect{Statuses: []int{200}, ContentType: "image/png", MaxBytes: 1024}, privateConfig: "header = \"Authorization: private-header\"\n"} + result, e := runResourceProtocol(context.Background(), "s", r, protocol, "dns", "socks5://127.0.0.1:1080", d) + err = e + if valid { + wantBytes := len(resourceFixture()) + if adapter == "HEAD" { + wantBytes = 0 + } + if result != (resourceCheckResult{Service: "s", Resource: "r", Group: "g", TLS: protocol, Outcome: "pass", Status: 200, Bytes: wantBytes, TimeMS: 100}) { + t.Fatalf("generic result changed: %+v", result) + } + } else if result.Outcome == "pass" { + t.Fatal("unsafe generic pass") + } + if err == nil && !valid { + err = fmt.Errorf("rejected transfer") + } + } + if (err == nil) != valid { + t.Fatalf("valid=%v err=%v", valid, err) + } + if valid && (adapter == "image" || adapter == "media") { + body, format, status, mime := resourceFixture(), "png", 200, "image/png" + if adapter == "media" { + body, format, status, mime = []byte{0, 0, 0, 24, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm', 0, 0, 0, 0, 'i', 's', 'o', 'm', 'm', 'p', '4', '2'}, "mp4", 206, "video/mp4" + } + version := uint16(0x0303) + if protocol == "tls13" { + version = 0x0304 + } + verify := 0 + want := assetTransfer{EffectiveHost: host, SSLVerifyResult: &verify, TLSVersion: version, Status: status, ContentType: mime, Size: len(body), Format: format, BodyHash: fmt.Sprintf("%x", sha256.Sum256(body)), LatencyMS: 100, SpeedKBPS: 10000.0 / 1024} + if !reflect.DeepEqual(transfer, want) { + t.Fatalf("normalized facts changed: got %+v want %+v", transfer, want) + } + encoded, _ := json.Marshal(transfer) + if strings.Contains(string(encoded), "private-secret") { + t.Fatal("private evidence") + } + } + }) + } + } + } +} + +func TestResourceCoreGenericMismatchParity(t *testing.T) { + for _, field := range []string{"status", "mime", "bytes"} { + for _, qualification := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/qualification=%v", field, qualification), func(t *testing.T) { + raw := "https://i.ytimg.com/fresh?sig=private-secret" + r := resourceCheckSpec{ID: "r", Group: "g", URL: raw, host: "i.ytimg.com", Method: "GET", TimeoutMS: 1000, Expect: resourceCheckExpect{Statuses: []int{200}, ContentType: "image/png", MaxBytes: 1024}} + switch field { + case "status": + r.Expect.Statuses = []int{201} + case "mime": + r.Expect.ContentType = "image/jpeg" + case "bytes": + r.Expect.MinBytes = 1000 + } + d := resourceCheckDeps{qualification: qualification, guard: func() error { return nil }, resolve: func(context.Context, string, string) ([]net.IP, error) { + return []net.IP{net.ParseIP("93.184.216.34")}, nil + }, runner: func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error { + resourceWire(raw, "", out, diag) + return nil + }} + result, err := runResourceProtocol(context.Background(), "s", r, "tls12", "dns", "socks5://127.0.0.1:1080", d) + want := resourceCheckResult{Service: "s", Resource: "r", Group: "g", TLS: "tls12", Outcome: "expectation_failed", Bytes: len(resourceFixture()), TimeMS: 100, Status: 200} + if err != nil || result != want { + t.Fatalf("result=%+v err=%v", result, err) + } + }) + } + } +} diff --git a/cmd/zapret-autotune/resource_qualify_test.go b/cmd/zapret-autotune/resource_qualify_test.go index bba5758..730a9d4 100644 --- a/cmd/zapret-autotune/resource_qualify_test.go +++ b/cmd/zapret-autotune/resource_qualify_test.go @@ -310,7 +310,7 @@ func TestResourceQualifierTypedPeerResponseBoundary(t *testing.T) { configuredWire("https://a.example/", "GET", "tls12", mode, out, diag) return nil } - _, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://a.example/", "a.example", "tls12", false, runner, resourceTransferOptions{generic: true, method: "GET", limit: 100, timeout: time.Second}) + _, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://a.example/", "a.example", "tls12", runner, resourceTransferOptions{method: "GET", limit: 100, timeout: time.Second}) var peer *resourcePeerResponseRejection var fatal *managedRecoveryFatal if mode == "redirect" {