refactor: share strict resource validation through generic core
Use private typed generic, image and MP4 response contracts in the existing bounded HTTPS execution pipeline. Preserve generic mismatch measurements, strict legacy DTOs and producer behavior; reject overflowing Content-Range integers before normalization. Add injected-wire adapter parity and direct-core regressions. Local targeted EUID-0/race, full Go and config fixtures passed; native acceptance remains pending. The unrelated long Python supervision timeout remains unresolved. No public schema, suffix-learning or controller migration. Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
2149a4abc8
commit
45d28e3733
7 files changed
+439
-50
No files matched your search
@@ -402,22 +402,20 @@ func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpe
|
||||
return result, qualificationOperation(err, stagePostDNSGuard)
|
||||
}
|
||||
pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443")
|
||||
options := resourceTransferOptions{generic: true, method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n"}
|
||||
transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, false, d.runner, options)
|
||||
if d.qualification && err != nil {
|
||||
options := resourceTransferOptions{method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n", expect: &r.Expect}
|
||||
transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, d.runner, options)
|
||||
if d.qualification && err != nil && err != errResourceExpectation {
|
||||
knownTimeout := probeCtx.Err() != nil && managedRecoveryTimeoutSafe(err)
|
||||
if !knownTimeout && !resourceRejectionOnly(err) && !managedAssetTransferNonpass(err) {
|
||||
return result, qualificationOperation(err, stageTransfer)
|
||||
}
|
||||
}
|
||||
result.Outcome = "transfer_failed"
|
||||
if err == nil {
|
||||
if err == nil || err == errResourceExpectation {
|
||||
result.Bytes, result.TimeMS, result.Status = len(transfer.Body), transfer.LatencyMS, transfer.Status
|
||||
result.Outcome = "expectation_failed"
|
||||
for _, status := range r.Expect.Statuses {
|
||||
if status == transfer.Status && len(transfer.Body) >= r.Expect.MinBytes && (r.Expect.ContentType == "" || transfer.ContentType == r.Expect.ContentType) {
|
||||
result.Outcome = "pass"
|
||||
}
|
||||
if err == nil {
|
||||
result.Outcome = "pass"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -365,7 +365,7 @@ func TestResourceCheckTransferContract(t *testing.T) {
|
||||
configuredWire("https://example.com/", method, "tls12", mode, out, diag)
|
||||
return nil
|
||||
}
|
||||
transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://example.com/", "example.com", "tls12", false, runner, resourceTransferOptions{generic: true, method: method, limit: 100, timeout: time.Second})
|
||||
transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://example.com/", "example.com", "tls12", runner, resourceTransferOptions{method: method, limit: 100, timeout: time.Second})
|
||||
valid := mode == "pass" || mode == "chunked" || method == "HEAD" && mode == "truncated"
|
||||
if (err == nil) != valid {
|
||||
t.Fatalf("err=%v", err)
|
||||
@@ -484,7 +484,7 @@ func TestResourceCheckCurlSOCKSPinnedDestination(t *testing.T) {
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
|
||||
defer cancel()
|
||||
_, _, err = collectResourceTransferOptions(ctx, "socks5h://"+listener.Addr().String(), "https://example.com/private?secret=value", "example.com", "tls12", false, runResourceCurl, resourceTransferOptions{generic: true, method: "GET", limit: 100, timeout: 3 * time.Second, config: "connect-to = \"example.com:443:93.184.216.34:443\"\n"})
|
||||
_, _, err = collectResourceTransferOptions(ctx, "socks5h://"+listener.Addr().String(), "https://example.com/private?secret=value", "example.com", "tls12", runResourceCurl, resourceTransferOptions{method: "GET", limit: 100, timeout: 3 * time.Second, config: "connect-to = \"example.com:443:93.184.216.34:443\"\n"})
|
||||
if err == nil {
|
||||
t.Fatal("fixture unexpectedly transferred")
|
||||
}
|
||||
|
||||
@@ -229,26 +229,39 @@ func playerMediaResource(body []byte) (mediaResource, mediaCapability, error) {
|
||||
|
||||
const resourceTrailer = "\nAUTOTUNE_RESOURCE_METRICS:"
|
||||
|
||||
func collectResourceTransfer(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner) (assetTransfer, http.Header, error) {
|
||||
return collectResourceTransferOptions(ctx, proxy, raw, host, protocol, media, runner, resourceTransferOptions{})
|
||||
}
|
||||
type resourceResponseKind uint8
|
||||
|
||||
// A completed generic mismatch retains measured facts, unlike unsafe transfers.
|
||||
var errResourceExpectation = errors.New("resource expectation failed")
|
||||
|
||||
const (
|
||||
resourceResponseGeneric resourceResponseKind = iota
|
||||
resourceResponseImage
|
||||
resourceResponseMP4
|
||||
)
|
||||
|
||||
type resourceTransferOptions struct {
|
||||
method string
|
||||
limit int
|
||||
timeout time.Duration
|
||||
// Private curl config only: never placed in argv or public evidence.
|
||||
config string
|
||||
generic bool
|
||||
config string
|
||||
response resourceResponseKind
|
||||
mediaType string
|
||||
expect *resourceCheckExpect
|
||||
}
|
||||
|
||||
func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner, options resourceTransferOptions) (assetTransfer, http.Header, error) {
|
||||
func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, protocol string, runner subprocessRunner, options resourceTransferOptions) (assetTransfer, http.Header, error) {
|
||||
generic := options.response == resourceResponseGeneric
|
||||
reason := reasonOrigin
|
||||
fail := func(err error) (assetTransfer, http.Header, error) {
|
||||
failure := resourceFailure(ctx, err)
|
||||
failure.(*resourceError).reason = reason
|
||||
return assetTransfer{}, nil, failure
|
||||
}
|
||||
if options.response > resourceResponseMP4 || (!generic && options.method != "GET") || (options.response == resourceResponseMP4 && options.mediaType != "video/mp4" && options.mediaType != "audio/mp4") {
|
||||
return fail(errors.New("invalid response contract"))
|
||||
}
|
||||
proxy, proxyErr := managedProbeProxy(proxy)
|
||||
if proxyErr != nil || runner == nil {
|
||||
return fail(errors.New("invalid owned proxy or runner"))
|
||||
@@ -263,19 +276,19 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
|
||||
return fail(errors.New("invalid TLS protocol"))
|
||||
}
|
||||
limit := 5 << 20
|
||||
if media {
|
||||
if options.response == resourceResponseMP4 {
|
||||
limit = 65536
|
||||
}
|
||||
if options.generic {
|
||||
if generic {
|
||||
limit = options.limit
|
||||
}
|
||||
// Qualification uses HTTP/1.1, not browser HTTP/2. --raw preserves native
|
||||
// transfer framing so net/http, rather than curl and Go twice, decodes it.
|
||||
args := []string{"--disable", "--silent", "--show-error", "--verbose", "--http1.1", "--raw", "--noproxy", "", "--proxy", proxy, "--proto", "=https", tlsFlag, "--tls-max", maxTLS, "--max-time", "30", "--max-filesize", strconv.Itoa(limit), "--suppress-connect-headers", "--include", "--write-out", resourceTrailer + "%{json}", "--config", "-"}
|
||||
if media {
|
||||
if options.response == resourceResponseMP4 {
|
||||
args = append(args, "--range", "0-65535")
|
||||
}
|
||||
if options.generic {
|
||||
if generic {
|
||||
if (options.method != "GET" && options.method != "HEAD") || options.limit < 1 || options.limit > 5<<20 || options.timeout < 100*time.Millisecond || options.timeout > 30*time.Second {
|
||||
return fail(errors.New("invalid transfer options"))
|
||||
}
|
||||
@@ -288,7 +301,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
|
||||
config := "url = \"" + strings.NewReplacer("\\", "\\\\", "\"", "\\\"").Replace(raw) + "\"\n"
|
||||
config += options.config
|
||||
timeout := 35 * time.Second
|
||||
if options.generic {
|
||||
if generic {
|
||||
timeout = options.timeout
|
||||
}
|
||||
childCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
@@ -366,10 +379,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
|
||||
return fail(errors.New("ambiguous response framing"))
|
||||
}
|
||||
r := bufio.NewReader(bytes.NewReader(wire))
|
||||
var request *http.Request
|
||||
if options.generic {
|
||||
request = &http.Request{Method: options.method}
|
||||
}
|
||||
request := &http.Request{Method: options.method}
|
||||
response, err := http.ReadResponse(r, request)
|
||||
if err != nil {
|
||||
return fail(errors.New("invalid response headers"))
|
||||
@@ -377,10 +387,10 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
|
||||
defer response.Body.Close()
|
||||
// CONNECT headers are suppressed by curl. Interim/duplicate responses and
|
||||
// unrequested versions are outside this contract, never heuristically skipped.
|
||||
if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || (!options.generic && response.Header.Get("Content-Type") == "") {
|
||||
if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || (!generic && response.Header.Get("Content-Type") == "") {
|
||||
return fail(errors.New("unsupported response contract"))
|
||||
}
|
||||
if options.generic && response.StatusCode >= 300 && response.StatusCode < 400 {
|
||||
if generic && response.StatusCode >= 300 && response.StatusCode < 400 {
|
||||
return fail(&resourcePeerResponseRejection{error: errors.New("redirect rejected")})
|
||||
}
|
||||
reason = reasonBody
|
||||
@@ -389,27 +399,74 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto
|
||||
// limit and durable byte identity apply to the fully decoded payload instead.
|
||||
wireSize := len(wire) - headerEnd - 4
|
||||
_, remaining := r.Peek(1)
|
||||
if err != nil || remaining != io.EOF || (!options.generic && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status {
|
||||
if err != nil || remaining != io.EOF || (!generic && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status {
|
||||
return fail(errors.New("incomplete or oversized resource body"))
|
||||
}
|
||||
reason = reasonMetrics
|
||||
for _, n := range []float64{metrics.Time, metrics.Speed} {
|
||||
if n < 0 || (!options.generic && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) {
|
||||
if n < 0 || (!generic && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) {
|
||||
return fail(errors.New("invalid resource metrics"))
|
||||
}
|
||||
}
|
||||
if options.generic && metrics.Time > options.timeout.Seconds()+1 {
|
||||
if generic && metrics.Time > options.timeout.Seconds()+1 {
|
||||
return fail(errors.New("resource timing exceeds budget"))
|
||||
}
|
||||
reason = reasonMIME
|
||||
contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type"))
|
||||
if err != nil && (!options.generic || response.Header.Get("Content-Type") != "") {
|
||||
if err != nil && (!generic || response.Header.Get("Content-Type") != "") {
|
||||
return fail(errors.New("invalid content type"))
|
||||
}
|
||||
t := assetTransfer{EffectiveHost: host, SSLVerifyResult: metrics.Verify, Redirects: *metrics.Redirects, TLSVersion: version, Status: metrics.Status, ContentType: contentType, Body: body, LatencyMS: metrics.Time * 1000, SpeedKBPS: metrics.Speed / 1024}
|
||||
if err := options.validateResponse(&t, response.Header); err != nil {
|
||||
if err == errResourceExpectation {
|
||||
return t, response.Header, err
|
||||
}
|
||||
return assetTransfer{}, nil, resourceFailure(ctx, err)
|
||||
}
|
||||
return t, response.Header, nil
|
||||
}
|
||||
|
||||
// Adapter contracts run only after the common origin/TLS/framing/bounds checks.
|
||||
// Generic v1 deliberately has no range or decoded-image expectation.
|
||||
func (options resourceTransferOptions) validateResponse(t *assetTransfer, h http.Header) error {
|
||||
switch options.response {
|
||||
case resourceResponseGeneric:
|
||||
if options.expect == nil {
|
||||
return nil
|
||||
}
|
||||
e := options.expect
|
||||
for _, status := range e.Statuses {
|
||||
if t.Status == status && len(t.Body) >= e.MinBytes && len(t.Body) <= e.MaxBytes && (e.ContentType == "" || t.ContentType == e.ContentType) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errResourceExpectation
|
||||
case resourceResponseImage:
|
||||
if t.Status != 200 {
|
||||
return errors.New("image HTTP status")
|
||||
}
|
||||
return normalizeAssetBody(t)
|
||||
case resourceResponseMP4:
|
||||
parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range"))
|
||||
var end, total int64
|
||||
if len(parts) == 3 {
|
||||
var endErr, totalErr error
|
||||
end, endErr = strconv.ParseInt(parts[1], 10, 64)
|
||||
total, totalErr = strconv.ParseInt(parts[2], 10, 64)
|
||||
if endErr != nil || totalErr != nil {
|
||||
return errors.New("invalid media range or container")
|
||||
}
|
||||
}
|
||||
if t.Status != 206 || t.ContentType != options.mediaType || len(parts) != 3 || end < 15 || end > 65535 || end+1 != int64(len(t.Body)) || total <= end || len(t.Body) < 16 || string(t.Body[4:8]) != "ftyp" || binary.BigEndian.Uint32(t.Body[:4]) < 16 || int(binary.BigEndian.Uint32(t.Body[:4])) > len(t.Body) {
|
||||
return errors.New("invalid media range or container")
|
||||
}
|
||||
t.Size, t.Format, t.BodyHash = len(t.Body), "mp4", fmt.Sprintf("%x", sha256.Sum256(t.Body))
|
||||
return nil
|
||||
default:
|
||||
return errors.New("invalid response contract")
|
||||
}
|
||||
}
|
||||
|
||||
// The ordinary 3/3 sampler is shared by image and media qualification.
|
||||
func sampleResource(ctx context.Context, transfer func(context.Context) (assetTransfer, error)) ([]assetTransfer, error) {
|
||||
var samples []assetTransfer
|
||||
@@ -429,14 +486,7 @@ func collectAssetMeasurement(ctx context.Context, proxy, raw, host, protocol, ar
|
||||
return m, resourceFailure(ctx, errors.New("invalid image endpoint"))
|
||||
}
|
||||
samples, err := sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) {
|
||||
t, _, err := collectResourceTransfer(ctx, proxy, raw, host, protocol, false, runner)
|
||||
if err == nil {
|
||||
if t.Status != 200 {
|
||||
err = errors.New("image HTTP status")
|
||||
} else {
|
||||
err = normalizeAssetBody(&t)
|
||||
}
|
||||
}
|
||||
t, _, err := collectResourceTransferOptions(ctx, proxy, raw, host, protocol, runner, resourceTransferOptions{method: "GET", response: resourceResponseImage})
|
||||
if err != nil {
|
||||
return assetTransfer{}, resourceFailure(ctx, err)
|
||||
}
|
||||
@@ -463,22 +513,18 @@ func collectMediaTransfer(ctx context.Context, proxy string, resource mediaResou
|
||||
if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") {
|
||||
return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("ordinary media unavailable"))
|
||||
}
|
||||
t, h, err := collectResourceTransfer(ctx, proxy, resource.url, resource.host, protocol, true, runner)
|
||||
t, h, err := collectResourceTransferOptions(ctx, proxy, resource.url, resource.host, protocol, runner, resourceTransferOptions{method: "GET", response: resourceResponseMP4, mediaType: resource.kind})
|
||||
if err != nil {
|
||||
return assetTransfer{}, GooglevideoPass{}, err
|
||||
}
|
||||
parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range"))
|
||||
var end, total int64
|
||||
if len(parts) == 3 {
|
||||
end, _ = strconv.ParseInt(parts[1], 10, 64)
|
||||
total, _ = strconv.ParseInt(parts[2], 10, 64)
|
||||
}
|
||||
if t.Status != 206 || t.ContentType != resource.kind || len(parts) != 3 || end < 15 || end > 65535 || end+1 != int64(len(t.Body)) || total <= end || len(t.Body) < 16 || string(t.Body[4:8]) != "ftyp" || binary.BigEndian.Uint32(t.Body[:4]) < 16 || int(binary.BigEndian.Uint32(t.Body[:4])) > len(t.Body) {
|
||||
// The shared core has validated and normalized the complete range contract.
|
||||
_, totalText, _ := strings.Cut(h.Get("Content-Range"), "/")
|
||||
total, err := strconv.ParseInt(totalText, 10, 64)
|
||||
if err != nil {
|
||||
return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("invalid media range or container"))
|
||||
}
|
||||
// Retain only normalized byte identity; no signed URL or payload.
|
||||
t.Size, t.Format, t.BodyHash = len(t.Body), "mp4", fmt.Sprintf("%x", sha256.Sum256(t.Body))
|
||||
t.Body = nil
|
||||
p := GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: t.Status, RangeStart: 0, RangeEnd: end, ResourceBytes: total, Bytes: int64(t.Size), FTYP: true, LatencyMS: t.LatencyMS}
|
||||
p := GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: t.Status, RangeStart: 0, RangeEnd: int64(t.Size) - 1, ResourceBytes: total, Bytes: int64(t.Size), FTYP: true, LatencyMS: t.LatencyMS}
|
||||
return t, p, nil
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestResourceCoreMediaRangeIntegers(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, contentRange string
|
||||
total int64
|
||||
}{
|
||||
{"ordinary", "bytes 0-23/100", 100},
|
||||
{"max-valid", "bytes 0-23/9223372036854775807", 9223372036854775807},
|
||||
{"total-overflow", "bytes 0-23/9223372036854775808", 0},
|
||||
{"total-huge", "bytes 0-23/999999999999999999999999999999999999", 0},
|
||||
{"end-overflow", "bytes 0-9223372036854775808/9223372036854775807", 0},
|
||||
{"nonnumeric", "bytes 0-23/abc", 0},
|
||||
{"unknown", "bytes 0-23/*", 0},
|
||||
{"signed", "bytes 0-23/+100", 0},
|
||||
{"negative", "bytes 0-23/-100", 0},
|
||||
{"signed-end", "bytes 0-+23/100", 0},
|
||||
{"negative-end", "bytes 0--23/100", 0},
|
||||
{"total-equals-end", "bytes 0-23/23", 0},
|
||||
{"total-below-end", "bytes 0-23/22", 0},
|
||||
{"end-body-mismatch", "bytes 0-24/100", 0},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
raw := "https://rr1.unit.googlevideo.com/fresh?sig=private-secret"
|
||||
runner := func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error {
|
||||
var wire bytes.Buffer
|
||||
resourceWire(raw, "media", &wire, diag)
|
||||
fmt.Fprint(out, strings.Replace(wire.String(), "bytes 0-23/100", tc.contentRange, 1))
|
||||
return nil
|
||||
}
|
||||
transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5://127.0.0.1:1080", raw, "rr1.unit.googlevideo.com", "tls12", runner, resourceTransferOptions{method: "GET", response: resourceResponseMP4, mediaType: "video/mp4"})
|
||||
valid := tc.total != 0
|
||||
if (err == nil) != valid || !valid && !reflect.DeepEqual(transfer, assetTransfer{}) {
|
||||
t.Fatalf("core valid=%v transfer=%+v err=%v", valid, transfer, err)
|
||||
}
|
||||
transfer, pass, err := collectMediaTransfer(context.Background(), "socks5://127.0.0.1:1080", mediaResource{raw, "rr1.unit.googlevideo.com", "video/mp4"}, "tls12", runner)
|
||||
if (err == nil) != valid {
|
||||
t.Fatalf("adapter valid=%v err=%v", valid, err)
|
||||
}
|
||||
if valid {
|
||||
if pass.ResourceBytes != tc.total || pass.Bytes != 24 || pass.RangeEnd != 23 || transfer.Size != 24 {
|
||||
t.Fatalf("incorrect normalized facts: %+v %+v", transfer, pass)
|
||||
}
|
||||
} else if !reflect.DeepEqual(transfer, assetTransfer{}) || !reflect.DeepEqual(pass, GooglevideoPass{}) {
|
||||
t.Fatalf("rejected range produced evidence: %+v %+v", transfer, pass)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Unlike the adapter golden test, these assertions call the shared execution
|
||||
// entry directly: strict validation must not be left in a legacy-only branch.
|
||||
func TestResourceCoreTypedContracts(t *testing.T) {
|
||||
for _, mode := range []string{"pass", "status", "start", "end", "total", "malformed", "ftyp", "box", "decoder"} {
|
||||
for _, kind := range []resourceResponseKind{resourceResponseGeneric, resourceResponseImage, resourceResponseMP4} {
|
||||
t.Run(fmt.Sprintf("%d/%s", kind, mode), func(t *testing.T) {
|
||||
options := resourceTransferOptions{method: "GET", limit: 1024, timeout: time.Second, response: kind, mediaType: "video/mp4"}
|
||||
raw := "https://i.ytimg.com/fresh?sig=private-secret"
|
||||
runner := func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error {
|
||||
var wire bytes.Buffer
|
||||
wireMode := ""
|
||||
if kind == resourceResponseMP4 {
|
||||
wireMode = "media"
|
||||
}
|
||||
if mode == "decoder" && kind != resourceResponseMP4 {
|
||||
wireMode = "type"
|
||||
}
|
||||
resourceWire(raw, wireMode, &wire, diag)
|
||||
data := wire.String()
|
||||
switch mode {
|
||||
case "status":
|
||||
data = strings.ReplaceAll(data, "206", "200")
|
||||
if kind != resourceResponseMP4 {
|
||||
data = strings.ReplaceAll(data, "200", "201")
|
||||
}
|
||||
case "start":
|
||||
data = strings.Replace(data, "bytes 0-23/100", "bytes 1-24/100", 1)
|
||||
case "end":
|
||||
data = strings.Replace(data, "bytes 0-23/100", "bytes 0-24/100", 1)
|
||||
case "total":
|
||||
data = strings.Replace(data, "bytes 0-23/100", "bytes 0-23/23", 1)
|
||||
case "malformed":
|
||||
data = strings.Replace(data, "bytes 0-23/100", "bytes 0-23/*", 1)
|
||||
case "ftyp":
|
||||
data = strings.Replace(data, "ftyp", "xxxx", 1)
|
||||
case "box":
|
||||
data = strings.Replace(data, string([]byte{0, 0, 0, 24}), string([]byte{0, 0, 0, 25}), 1)
|
||||
}
|
||||
fmt.Fprint(out, data)
|
||||
return nil
|
||||
}
|
||||
transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5://127.0.0.1:1080", raw, "i.ytimg.com", "tls12", runner, options)
|
||||
valid := kind == resourceResponseGeneric || mode == "pass" || kind == resourceResponseImage && mode != "decoder" && mode != "status" || kind == resourceResponseMP4 && mode == "decoder"
|
||||
if (err == nil) != valid {
|
||||
t.Fatalf("valid=%v err=%v", valid, err)
|
||||
}
|
||||
if valid && kind != resourceResponseGeneric && (transfer.Size != len(transfer.Body) || transfer.BodyHash == "" || transfer.Format == "") {
|
||||
t.Fatal("strict normalization not performed by core")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceCoreGenericExpectationFacts(t *testing.T) {
|
||||
raw := "https://i.ytimg.com/private"
|
||||
runner := func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error {
|
||||
resourceWire(raw, "", out, diag)
|
||||
return nil
|
||||
}
|
||||
expect := resourceCheckExpect{Statuses: []int{201}, MaxBytes: 1024}
|
||||
transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5://127.0.0.1:1080", raw, "i.ytimg.com", "tls12", runner, resourceTransferOptions{method: "GET", limit: 1024, timeout: time.Second, expect: &expect})
|
||||
if err != errResourceExpectation || transfer.Status != 200 || len(transfer.Body) != len(resourceFixture()) || transfer.LatencyMS != 100 {
|
||||
t.Fatalf("core did not validate generic expectation with intact facts: %+v %v", transfer, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Exercise the actual legacy adapters and generic producer with the same wire
|
||||
// mutations. This test also runs unchanged against the pre-migration source.
|
||||
func TestResourceCoreAdapterParity(t *testing.T) {
|
||||
for _, adapter := range []string{"image", "media", "GET", "HEAD"} {
|
||||
for _, protocol := range []string{"tls12", "tls13"} {
|
||||
for _, mode := range []string{"pass", "verify", "origin", "redirect", "tls", "framing", "oversize", "range", "ftyp", "decoder"} {
|
||||
t.Run(adapter+"/"+protocol+"/"+mode, func(t *testing.T) {
|
||||
host := "i.ytimg.com"
|
||||
if adapter == "media" {
|
||||
host = "rr1.unit.googlevideo.com"
|
||||
}
|
||||
raw := "https://" + host + "/fresh?sig=private-secret"
|
||||
calls := 0
|
||||
runner := func(_ context.Context, args []string, stdin io.Reader, out, diag io.Writer) error {
|
||||
calls++
|
||||
argv := strings.Join(args, " ")
|
||||
for _, required := range []string{"--disable --silent", "--http1.1 --raw", "--noproxy --proxy socks5h://127.0.0.1:1080", "--proto =https", "--suppress-connect-headers", "--config -"} {
|
||||
if !strings.Contains(argv, required) {
|
||||
t.Fatalf("missing %s", required)
|
||||
}
|
||||
}
|
||||
if strings.Contains(argv, "private-secret") || strings.Contains(argv, host) || strings.Contains(argv, "--insecure") || strings.Contains(argv, "--location") {
|
||||
t.Fatal("unsafe argv")
|
||||
}
|
||||
if strings.Contains(argv, "--range 0-65535") != (adapter == "media") || strings.Contains(argv, "--head") != (adapter == "HEAD") {
|
||||
t.Fatal("method/range changed")
|
||||
}
|
||||
config, _ := io.ReadAll(stdin)
|
||||
if !strings.Contains(string(config), raw) {
|
||||
t.Fatal("URL not in private stdin")
|
||||
}
|
||||
if adapter == "GET" || adapter == "HEAD" {
|
||||
if !strings.Contains(string(config), `connect-to = "i.ytimg.com:443:93.184.216.34:443"`) || !strings.Contains(string(config), "Authorization: private-header") {
|
||||
t.Fatal("private header or CONNECT pin lost")
|
||||
}
|
||||
}
|
||||
var wire, diagnostic bytes.Buffer
|
||||
wireMode := ""
|
||||
if adapter == "media" {
|
||||
wireMode = "media"
|
||||
}
|
||||
if mode == "decoder" && adapter == "image" {
|
||||
wireMode = "type"
|
||||
}
|
||||
resourceWire(raw, wireMode, &wire, &diagnostic)
|
||||
data := wire.String()
|
||||
if adapter == "HEAD" {
|
||||
start := strings.Index(data, "\r\n\r\n") + 4
|
||||
end := strings.LastIndex(data, resourceTrailer)
|
||||
data = data[:start] + data[end:]
|
||||
data = strings.Replace(data, fmt.Sprintf(`"size_download":%d`, len(resourceFixture())), `"size_download":0`, 1)
|
||||
}
|
||||
switch mode {
|
||||
case "verify":
|
||||
data = strings.Replace(data, `"ssl_verify_result":0`, `"ssl_verify_result":60`, 1)
|
||||
case "origin":
|
||||
data = strings.Replace(data, `"url_effective":"https://`, `"url_effective":"https://wrong.`, 1)
|
||||
case "redirect":
|
||||
data = strings.Replace(data, `"num_redirects":0`, `"num_redirects":1`, 1)
|
||||
case "framing":
|
||||
data = strings.Replace(data, "Content-Length:", "Content-Length: 1\r\nContent-Length:", 1)
|
||||
case "oversize":
|
||||
_, err := out.Write(make([]byte, 6<<20))
|
||||
return err
|
||||
case "range":
|
||||
data = strings.Replace(data, "bytes 0-23/100", "bytes 1-24/100", 1)
|
||||
case "ftyp":
|
||||
data = strings.Replace(data, "ftyp", "xxxx", 1)
|
||||
}
|
||||
fmt.Fprint(out, data)
|
||||
actual := "TLSv1.2"
|
||||
if protocol == "tls13" {
|
||||
actual = "TLSv1.3"
|
||||
}
|
||||
if mode == "tls" {
|
||||
actual = "TLSv1.1"
|
||||
}
|
||||
fmt.Fprintln(diag, "SSL connection using "+actual)
|
||||
return nil
|
||||
}
|
||||
valid := mode == "pass" || mode == "range" && adapter != "media" || mode == "ftyp" && adapter != "media" || mode == "decoder" && adapter != "image"
|
||||
var err error
|
||||
var transfer assetTransfer
|
||||
switch adapter {
|
||||
case "image":
|
||||
var m assetMeasurement
|
||||
m, err = collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", raw, host, protocol, "candidate", runner)
|
||||
if err == nil {
|
||||
transfer = m.Transfers[0]
|
||||
if calls != 3 || m.Source != "image-transfer" || m.Endpoint != host || m.Protocol != protocol || m.Args != "candidate" {
|
||||
t.Fatal("legacy measurement changed")
|
||||
}
|
||||
}
|
||||
case "media":
|
||||
var pass GooglevideoPass
|
||||
transfer, pass, err = collectMediaTransfer(context.Background(), "socks5://127.0.0.1:1080", mediaResource{raw, host, "video/mp4"}, protocol, runner)
|
||||
if err == nil && !reflect.DeepEqual(pass, GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: 206, RangeStart: 0, RangeEnd: 23, ResourceBytes: 100, Bytes: 24, FTYP: true, LatencyMS: 100}) {
|
||||
t.Fatalf("legacy pass changed: %+v", pass)
|
||||
}
|
||||
default:
|
||||
d := resourceCheckDeps{guard: func() error { return nil }, resolve: func(context.Context, string, string) ([]net.IP, error) {
|
||||
return []net.IP{net.ParseIP("93.184.216.34")}, nil
|
||||
}, runner: runner}
|
||||
r := resourceCheckSpec{ID: "r", Group: "g", URL: raw, host: host, Method: adapter, TimeoutMS: 1000, Expect: resourceCheckExpect{Statuses: []int{200}, ContentType: "image/png", MaxBytes: 1024}, privateConfig: "header = \"Authorization: private-header\"\n"}
|
||||
result, e := runResourceProtocol(context.Background(), "s", r, protocol, "dns", "socks5://127.0.0.1:1080", d)
|
||||
err = e
|
||||
if valid {
|
||||
wantBytes := len(resourceFixture())
|
||||
if adapter == "HEAD" {
|
||||
wantBytes = 0
|
||||
}
|
||||
if result != (resourceCheckResult{Service: "s", Resource: "r", Group: "g", TLS: protocol, Outcome: "pass", Status: 200, Bytes: wantBytes, TimeMS: 100}) {
|
||||
t.Fatalf("generic result changed: %+v", result)
|
||||
}
|
||||
} else if result.Outcome == "pass" {
|
||||
t.Fatal("unsafe generic pass")
|
||||
}
|
||||
if err == nil && !valid {
|
||||
err = fmt.Errorf("rejected transfer")
|
||||
}
|
||||
}
|
||||
if (err == nil) != valid {
|
||||
t.Fatalf("valid=%v err=%v", valid, err)
|
||||
}
|
||||
if valid && (adapter == "image" || adapter == "media") {
|
||||
body, format, status, mime := resourceFixture(), "png", 200, "image/png"
|
||||
if adapter == "media" {
|
||||
body, format, status, mime = []byte{0, 0, 0, 24, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm', 0, 0, 0, 0, 'i', 's', 'o', 'm', 'm', 'p', '4', '2'}, "mp4", 206, "video/mp4"
|
||||
}
|
||||
version := uint16(0x0303)
|
||||
if protocol == "tls13" {
|
||||
version = 0x0304
|
||||
}
|
||||
verify := 0
|
||||
want := assetTransfer{EffectiveHost: host, SSLVerifyResult: &verify, TLSVersion: version, Status: status, ContentType: mime, Size: len(body), Format: format, BodyHash: fmt.Sprintf("%x", sha256.Sum256(body)), LatencyMS: 100, SpeedKBPS: 10000.0 / 1024}
|
||||
if !reflect.DeepEqual(transfer, want) {
|
||||
t.Fatalf("normalized facts changed: got %+v want %+v", transfer, want)
|
||||
}
|
||||
encoded, _ := json.Marshal(transfer)
|
||||
if strings.Contains(string(encoded), "private-secret") {
|
||||
t.Fatal("private evidence")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceCoreGenericMismatchParity(t *testing.T) {
|
||||
for _, field := range []string{"status", "mime", "bytes"} {
|
||||
for _, qualification := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/qualification=%v", field, qualification), func(t *testing.T) {
|
||||
raw := "https://i.ytimg.com/fresh?sig=private-secret"
|
||||
r := resourceCheckSpec{ID: "r", Group: "g", URL: raw, host: "i.ytimg.com", Method: "GET", TimeoutMS: 1000, Expect: resourceCheckExpect{Statuses: []int{200}, ContentType: "image/png", MaxBytes: 1024}}
|
||||
switch field {
|
||||
case "status":
|
||||
r.Expect.Statuses = []int{201}
|
||||
case "mime":
|
||||
r.Expect.ContentType = "image/jpeg"
|
||||
case "bytes":
|
||||
r.Expect.MinBytes = 1000
|
||||
}
|
||||
d := resourceCheckDeps{qualification: qualification, guard: func() error { return nil }, resolve: func(context.Context, string, string) ([]net.IP, error) {
|
||||
return []net.IP{net.ParseIP("93.184.216.34")}, nil
|
||||
}, runner: func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error {
|
||||
resourceWire(raw, "", out, diag)
|
||||
return nil
|
||||
}}
|
||||
result, err := runResourceProtocol(context.Background(), "s", r, "tls12", "dns", "socks5://127.0.0.1:1080", d)
|
||||
want := resourceCheckResult{Service: "s", Resource: "r", Group: "g", TLS: "tls12", Outcome: "expectation_failed", Bytes: len(resourceFixture()), TimeMS: 100, Status: 200}
|
||||
if err != nil || result != want {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -310,7 +310,7 @@ func TestResourceQualifierTypedPeerResponseBoundary(t *testing.T) {
|
||||
configuredWire("https://a.example/", "GET", "tls12", mode, out, diag)
|
||||
return nil
|
||||
}
|
||||
_, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://a.example/", "a.example", "tls12", false, runner, resourceTransferOptions{generic: true, method: "GET", limit: 100, timeout: time.Second})
|
||||
_, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", "https://a.example/", "a.example", "tls12", runner, resourceTransferOptions{method: "GET", limit: 100, timeout: time.Second})
|
||||
var peer *resourcePeerResponseRejection
|
||||
var fatal *managedRecoveryFatal
|
||||
if mode == "redirect" {
|
||||
|
||||
Reference in new issue
Block a user