From cd05e75189f90c32e1346536c50f0973e7105cb3 Mon Sep 17 00:00:00 2001 From: Zeusina Date: Thu, 15 Oct 2026 15:07:54 +0300 Subject: [PATCH] feat: add typed generic image and MP4 checks Preserve strict v1 semantics and fingerprints; bind v2 typed expectations to a new proof namespace. Reuse bounded shared collection across checks, monitoring, qualification and apply health. Document prefix-only MP4 and deferred public GIF/plugin/runtime boundaries. Co-authored-by: GPT-6 --- RESOURCE_MONITOR.md | 24 +- RESOURCE_PLAN.md | 37 +- cmd/zapret-autotune/resource_check.go | 57 ++- cmd/zapret-autotune/resource_check_test.go | 2 +- cmd/zapret-autotune/resource_collector.go | 57 ++- cmd/zapret-autotune/resource_monitor.go | 3 + cmd/zapret-autotune/resource_v2_test.go | 468 +++++++++++++++++++++ resource-check-v2.example.json | 30 ++ 8 files changed, 654 insertions(+), 24 deletions(-) create mode 100644 cmd/zapret-autotune/resource_v2_test.go create mode 100644 resource-check-v2.example.json diff --git a/RESOURCE_MONITOR.md b/RESOURCE_MONITOR.md index 134fb96..fb309e9 100644 --- a/RESOURCE_MONITOR.md +++ b/RESOURCE_MONITOR.md @@ -1,6 +1,6 @@ # Periodic configured resource checks -`managed monitor --resource-config FILE` runs resource-config v1 periodically. +`managed monitor --resource-config FILE` runs resource-config v1 or v2 periodically. PID 1 selects the same mode when `ZAPRET_AUTOTUNE_RESOURCE_CONFIG` is nonempty; unset preserves the existing YouTube monitor. The modes share one singleton lock. Supply an absolute container-visible config path (for example mount a @@ -19,9 +19,29 @@ not playback health, generic recovery or whole-production READY: HEALTHY is base passed native full/race checks, independent security review and isolated remote fault acceptance for `4efc066` plus ownership-status fix `5c85f3f`. See [PROGRESS.md](PROGRESS.md) for supplied evidence and its narrow scope. -Version1 is bounded to 16 services / 64 resources with stable configured IDs; +Both versions are bounded to 16 services / 64 resources with stable configured IDs; valid configuration/state does not imply an automatic strategy assignment. +V1 remains HTTP GET/HEAD only, with explicit `kind: http`. See +`resource-check-v2.example.json` for v2: omitted kind defaults to `http`; all +expectations explicitly declare statuses/content_type/min_bytes/max_bytes. +`image` is GET-only, [200], PNG/JPEG with full bounded decode (5 MiB maximum, +16 MiPixels); public GIF is deferred pending an aggregate frame/pixel budget, +without changing legacy GIF adapters. `mp4_range` is GET-only, [206], audio/mp4 +or video/mp4, 16–65536 bytes, checked Content-Range and ftyp. Range stays fixed +at 0-65535; a smaller max_bytes bounds the actual response, not the requested end. +Operator Range/Accept-Encoding headers remain forbidden in both versions. +Timeouts (100–30000 ms) and declared byte limits apply to every kind. Typed +wire-valid semantic failures degrade checks; malformed/unsafe execution cannot +qualify a candidate. V2 version/kind/expectations invalidate old HMAC rows/proofs; +historic v1 fingerprints remain unchanged. MP4 prefix health is not playback or +YouTube proof: three discovered CDNs, >=1024-byte producer samples, warmup, +suffix authorization and adaptive policy still belong to the legacy adapter. +Trusted compiled-in kind+version service modules remain a roadmap design, not an +implemented plugin API. Local v2 checks are covered by injected-wire tests; +Independent local source/closure review passed; managed remote acceptance is +still pending. + Invalid resource JSON initially or after reload is BLOCKED, with no DNS or transfers until repaired. Bad static target role, proxy or DNS settings still fail startup validation; resource JSON is deliberately not part of diff --git a/RESOURCE_PLAN.md b/RESOURCE_PLAN.md index 6a8412a..075f462 100644 --- a/RESOURCE_PLAN.md +++ b/RESOURCE_PLAN.md @@ -63,15 +63,42 @@ no owned containers/networks were created. Root-private artifacts, keys, manifes and hashes remain retained. Selected foreign identity/running/start facts matched before/after, not arbitrary foreign application-data hashes. -### Next minimal generic capability (not implemented) +### Public typed checks (v2; remote acceptance pending) -Add versioned declarative typed image/media expectations and validators for full -decoded bodies and strict Range proof; do not change v1 GET/HEAD semantics. Then -let the YouTube adapter supply qualified exact endpoints/dynamic inventory into +`resource-check-v2.example.json` adds closed `kind` values `http`, `image` and +`mp4_range`; omitted v2 kind canonicalizes to `http`. V1 still requires explicit +`http` and retains its serialization/fingerprints. V2 requires explicit statuses, +content_type, min_bytes and max_bytes. HTTP retains GET/HEAD semantics; typed +checks require GET. Image requires exactly [200] and image/png or image/jpeg, +full decoding, positive dimensions at most 16 MiPixels and a configured body +ceiling at most 5 MiB. Public GIF/omitted image MIME are deliberately deferred +until a decoder with an aggregate frame/pixel budget exists; legacy GIF remains. +MP4 requires exactly [206], video/mp4 or audio/mp4, and 16 <= min <= max <= 65536. +The core always requests bytes 0-65535, even with a smaller max_bytes; actual +returned bytes must fit that smaller bound and the validated Content-Range/ftyp. +This proves a bounded MP4 prefix, **not playback or YouTube qualification**. + +All kinds use configured 100–30000 ms timeouts and byte ceilings, private stdin, +strict framing/TLS/origin and all-answer DNS policy through the same public check, +periodic monitor, isolated M=3 qualifier and apply-health path. Wire-valid typed +semantic failures are complete expectation negatives; unsafe output, malformed +metrics/framing, cancellation and ownership failures cannot become benign proof. +V2 source and row HMACs bind version, normalized kind and expectations, so old +plain-HTTP evidence/state cannot promote typed checks. V1 historical journal +reconstruction is unchanged. No public proof import or module framework exists. + +Local verification for this v2 slice: full Go tests (`-p 1 -parallel 1`), affected +resource race tests, vet, CGO-disabled build, four managed-target Python fixtures +and shell syntax passed. Current and archived `45d28e3` each passed the existing +86 legacy adapter/generic-mismatch parity subtests. Privileged tests ran as EUID 0 +inside the current user namespace (UID 0 maps to host UID 100000), not host-initial +root. These are local injected-wire/fixture results, not remote network acceptance. + +Next let the YouTube adapter supply qualified exact endpoints/dynamic inventory into the generic registry, preserving fresh discovery and the three-CDN contract. Keep measured endpoint proof separate from explicit suffix/adaptive-learning authorization: both capabilities are prerequisites to complete managed migration. -No proposal framework, legacy deletion or full controller/journal/profile migration +No plugin framework, legacy deletion or full controller/journal/profile migration is implied. Full runtime acceptance is pending; avatars and legacy exit78 remain unresolved. No production READY claim. diff --git a/cmd/zapret-autotune/resource_check.go b/cmd/zapret-autotune/resource_check.go index acd796b..e3631a8 100644 --- a/cmd/zapret-autotune/resource_check.go +++ b/cmd/zapret-autotune/resource_check.go @@ -124,9 +124,35 @@ func parseResourceChecks(data []byte, lookup func(string) (string, bool)) (resou } d = json.NewDecoder(bytes.NewReader(data)) d.DisallowUnknownFields() - if d.Decode(&cfg) != nil || cfg.Version != 1 || len(cfg.Services) == 0 || len(cfg.Services) > 16 { + if d.Decode(&cfg) != nil || (cfg.Version != 1 && cfg.Version != 2) || len(cfg.Services) == 0 || len(cfg.Services) > 16 { return fail() } + if cfg.Version == 2 { + // Presence matters: an omitted minimum must not silently become zero. + var fields struct { + Services []struct { + Resources []struct { + Kind *string `json:"kind"` + Expect struct { + Statuses *[]int `json:"statuses"` + ContentType *string `json:"content_type"` + MinBytes *int `json:"min_bytes"` + MaxBytes *int `json:"max_bytes"` + } `json:"expect"` + } `json:"resources"` + } `json:"services"` + } + if json.Unmarshal(data, &fields) != nil { + return fail() + } + for _, s := range fields.Services { + for _, r := range s.Resources { + if r.Kind != nil && *r.Kind == "" || r.Expect.Statuses == nil || r.Expect.ContentType == nil || *r.Expect.ContentType == "" || r.Expect.MinBytes == nil || r.Expect.MaxBytes == nil { + return fail() + } + } + } + } services, total := map[string]bool{}, 0 for si := range cfg.Services { s := &cfg.Services[si] @@ -137,8 +163,11 @@ func parseResourceChecks(data []byte, lookup func(string) (string, bool)) (resou ids := map[string]bool{} for ri := range s.Resources { r := &s.Resources[ri] + if cfg.Version == 2 && r.Kind == "" { + r.Kind = "http" + } total++ - if total > 64 || !resourceID.MatchString(r.ID) || ids[r.ID] || !resourceID.MatchString(r.Group) || r.Kind != "http" { + if total > 64 || !resourceID.MatchString(r.ID) || ids[r.ID] || !resourceID.MatchString(r.Group) || (r.Kind != "http" && (cfg.Version != 2 || r.Kind != "image" && r.Kind != "mp4_range")) { return fail() } ids[r.ID] = true @@ -213,6 +242,24 @@ func parseResourceChecks(data []byte, lookup func(string) (string, bool)) (resou return fail() } } + if r.Kind != "http" { + status := 200 + if r.Kind == "mp4_range" { + status = 206 + } + if r.Method != "GET" || len(r.Expect.Statuses) != 1 || r.Expect.Statuses[0] != status { + return fail() + } + if r.Kind == "image" { + // ponytail: public GIF awaits an aggregate frame/pixel-bounded decoder; + // legacy image adapters retain their existing GIF contract. + if r.Expect.ContentType != "image/png" && r.Expect.ContentType != "image/jpeg" { + return fail() + } + } else if (r.Expect.ContentType != "video/mp4" && r.Expect.ContentType != "audio/mp4") || r.Expect.MinBytes < 16 || r.Expect.MaxBytes > 65536 { + return fail() + } + } if len(r.HeadersEnv) > 16 { return fail() } @@ -403,6 +450,12 @@ func runResourceProtocol(ctx context.Context, service string, r resourceCheckSpe } pin := r.host + ":443:" + net.JoinHostPort(ips[0].String(), "443") options := resourceTransferOptions{method: r.Method, limit: r.Expect.MaxBytes, timeout: time.Duration(r.TimeoutMS) * time.Millisecond, config: r.privateConfig + "connect-to = " + privateCurlValue(pin) + "\n", expect: &r.Expect} + switch r.Kind { + case "image": + options.response = resourceResponseImage + case "mp4_range": + options.response, options.mediaType = resourceResponseMP4, r.Expect.ContentType + } transfer, _, err := collectResourceTransferOptions(probeCtx, proxy, r.URL, r.host, protocol, d.runner, options) if d.qualification && err != nil && err != errResourceExpectation { knownTimeout := probeCtx.Err() != nil && managedRecoveryTimeoutSafe(err) diff --git a/cmd/zapret-autotune/resource_check_test.go b/cmd/zapret-autotune/resource_check_test.go index 243ee55..619290e 100644 --- a/cmd/zapret-autotune/resource_check_test.go +++ b/cmd/zapret-autotune/resource_check_test.go @@ -94,7 +94,7 @@ func TestResourceCheckSchema(t *testing.T) { t.Fatal(err) } for _, data := range []string{ - resourceCheckExample + ` {}`, strings.Replace(resourceCheckExample, `"version":1`, `"version":2`, 1), + resourceCheckExample + ` {}`, strings.Replace(resourceCheckExample, `"version":1`, `"version":3`, 1), strings.Replace(resourceCheckExample, `"version":1`, `"version":1,"version":1`, 1), strings.Replace(resourceCheckExample, `"version":1`, `"unknown":1,"version":1`, 1), strings.Replace(resourceCheckExample, `"id":"first"`, `"id":"First /"`, 1), diff --git a/cmd/zapret-autotune/resource_collector.go b/cmd/zapret-autotune/resource_collector.go index 2c46cfa..477457c 100644 --- a/cmd/zapret-autotune/resource_collector.go +++ b/cmd/zapret-autotune/resource_collector.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "image" "io" "math" "mime" @@ -253,6 +254,7 @@ type resourceTransferOptions struct { func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, protocol string, runner subprocessRunner, options resourceTransferOptions) (assetTransfer, http.Header, error) { generic := options.response == resourceResponseGeneric + configured := generic || options.expect != nil reason := reasonOrigin fail := func(err error) (assetTransfer, http.Header, error) { failure := resourceFailure(ctx, err) @@ -279,7 +281,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto if options.response == resourceResponseMP4 { limit = 65536 } - if generic { + if configured { limit = options.limit } // Qualification uses HTTP/1.1, not browser HTTP/2. --raw preserves native @@ -288,10 +290,13 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto if options.response == resourceResponseMP4 { args = append(args, "--range", "0-65535") } - if generic { + if configured { if (options.method != "GET" && options.method != "HEAD") || options.limit < 1 || options.limit > 5<<20 || options.timeout < 100*time.Millisecond || options.timeout > 30*time.Second { return fail(errors.New("invalid transfer options")) } + if options.response == resourceResponseMP4 && options.limit > 65536 { + return fail(errors.New("invalid transfer options")) + } args = append(args, "--max-time", strconv.FormatFloat(options.timeout.Seconds(), 'f', 3, 64)) if options.method == "HEAD" { args = append(args, "--head", "--max-filesize", "0") @@ -301,7 +306,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto config := "url = \"" + strings.NewReplacer("\\", "\\\\", "\"", "\\\"").Replace(raw) + "\"\n" config += options.config timeout := 35 * time.Second - if generic { + if configured { timeout = options.timeout } childCtx, cancel := context.WithTimeout(ctx, timeout) @@ -387,7 +392,7 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto defer response.Body.Close() // CONNECT headers are suppressed by curl. Interim/duplicate responses and // unrequested versions are outside this contract, never heuristically skipped. - if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || (!generic && response.Header.Get("Content-Type") == "") { + if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || (!configured && response.Header.Get("Content-Type") == "") { return fail(errors.New("unsupported response contract")) } if generic && response.StatusCode >= 300 && response.StatusCode < 400 { @@ -399,24 +404,32 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto // limit and durable byte identity apply to the fully decoded payload instead. wireSize := len(wire) - headerEnd - 4 _, remaining := r.Peek(1) - if err != nil || remaining != io.EOF || (!generic && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status { + if err != nil || remaining != io.EOF || (!configured && len(body) == 0) || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status { return fail(errors.New("incomplete or oversized resource body")) } reason = reasonMetrics for _, n := range []float64{metrics.Time, metrics.Speed} { - if n < 0 || (!generic && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) { + if n < 0 || (!configured && n == 0) || math.IsNaN(n) || math.IsInf(n, 0) { return fail(errors.New("invalid resource metrics")) } } - if generic && metrics.Time > options.timeout.Seconds()+1 { + if configured && metrics.Time > options.timeout.Seconds()+1 { return fail(errors.New("resource timing exceeds budget")) } reason = reasonMIME contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type")) - if err != nil && (!generic || response.Header.Get("Content-Type") != "") { - return fail(errors.New("invalid content type")) + if err != nil { + contentType = "" } t := assetTransfer{EffectiveHost: host, SSLVerifyResult: metrics.Verify, Redirects: *metrics.Redirects, TLSVersion: version, Status: metrics.Status, ContentType: contentType, Body: body, LatencyMS: metrics.Time * 1000, SpeedKBPS: metrics.Speed / 1024} + if err != nil && options.expect != nil && (options.response == resourceResponseImage || options.response == resourceResponseMP4) { + // A malformed semantic MIME value is a completed typed mismatch only + // after the common wire/security checks; never retain the raw value. + return t, response.Header, errResourceExpectation + } + if err != nil && (!configured || response.Header.Get("Content-Type") != "") { + return fail(errors.New("invalid content type")) + } if err := options.validateResponse(&t, response.Header); err != nil { if err == errResourceExpectation { return t, response.Header, err @@ -429,22 +442,38 @@ func collectResourceTransferOptions(ctx context.Context, proxy, raw, host, proto // Adapter contracts run only after the common origin/TLS/framing/bounds checks. // Generic v1 deliberately has no range or decoded-image expectation. func (options resourceTransferOptions) validateResponse(t *assetTransfer, h http.Header) error { - switch options.response { - case resourceResponseGeneric: - if options.expect == nil { - return nil + if err := options.validateContract(t, h); err != nil { + if options.expect != nil { + return errResourceExpectation } - e := options.expect + return err + } + if e := options.expect; e != nil { for _, status := range e.Statuses { if t.Status == status && len(t.Body) >= e.MinBytes && len(t.Body) <= e.MaxBytes && (e.ContentType == "" || t.ContentType == e.ContentType) { return nil } } return errResourceExpectation + } + return nil +} + +func (options resourceTransferOptions) validateContract(t *assetTransfer, h http.Header) error { + switch options.response { + case resourceResponseGeneric: + return nil case resourceResponseImage: if t.Status != 200 { return errors.New("image HTTP status") } + if options.expect != nil { + // Reject GIF before the legacy full decoder can allocate its frames. + _, format, err := image.DecodeConfig(bytes.NewReader(t.Body)) + if err != nil || (format != "png" && format != "jpeg") { + return errors.New("invalid public image format") + } + } return normalizeAssetBody(t) case resourceResponseMP4: parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range")) diff --git a/cmd/zapret-autotune/resource_monitor.go b/cmd/zapret-autotune/resource_monitor.go index 15cec29..fe397c0 100644 --- a/cmd/zapret-autotune/resource_monitor.go +++ b/cmd/zapret-autotune/resource_monitor.go @@ -74,6 +74,9 @@ func readResourceMonitorSource(path string, lookup func(string) (string, bool), func resourceRowFingerprint(key []byte, source resourceMonitorSource, r resourceCheckSpec) string { b, _ := json.Marshal(r) b, _ = json.Marshal([]string{string(b), r.privateConfig, source.server, source.proxy}) + if source.config.Version == 2 { + b, _ = json.Marshal([]string{"resource-check-v2", string(b)}) + } return resourceMAC(key, b) } func reconcileResourceRows(key []byte, source resourceMonitorSource, prior []resourceMonitorRow) []resourceMonitorRow { diff --git a/cmd/zapret-autotune/resource_v2_test.go b/cmd/zapret-autotune/resource_v2_test.go new file mode 100644 index 0000000..aaea3e9 --- /dev/null +++ b/cmd/zapret-autotune/resource_v2_test.go @@ -0,0 +1,468 @@ +package main + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "image" + "image/gif" + "image/jpeg" + "io" + "net" + "strconv" + "strings" + "testing" + "time" +) + +const resourceV2Example = `{"version":2,"services":[{"id":"media","resources":[{"id":"image","group":"images","kind":"image","url":"https://example.com/image?SECRET","method":"GET","tls":"both","timeout_ms":700,"expect":{"statuses":[200],"content_type":"image/png","min_bytes":1,"max_bytes":1024},"headers_env":{"Authorization":"CHECK_TOKEN"}},{"id":"prefix","group":"media","kind":"mp4_range","url":"https://example.com/prefix?SECRET","method":"GET","tls":"both","timeout_ms":700,"expect":{"statuses":[206],"content_type":"video/mp4","min_bytes":16,"max_bytes":65536}}]}]}` + +func resourceV2Source(t *testing.T) resourceMonitorSource { + t.Helper() + cfg, err := parseResourceChecks([]byte(resourceV2Example), resourceCheckLookup) + if err != nil { + t.Fatal(err) + } + return resourceMonitorSource{config: cfg, server: "127.0.0.1:5353", proxy: "socks5h://127.0.0.1:1080", fingerprint: strings.Repeat("a", 64)} +} + +func TestResourceV2Schema(t *testing.T) { + for _, tc := range []struct{ from, to string }{ + {`"version":2`, `"version":3`}, {`"version":2`, `"version":1`}, + {`"kind":"image"`, `"kind":"unknown"`}, {`"method":"GET"`, `"method":"HEAD"`}, + {`"kind":"image"`, `"kind":""`}, + {`"statuses":[200]`, `"statuses":[200,201]`}, {`"statuses":[206]`, `"statuses":[200]`}, + {`"image/png"`, `"image/gif"`}, {`"image/png"`, `"video/mp4"`}, + {`"video/mp4"`, `"image/png"`}, {`"min_bytes":16`, `"min_bytes":15`}, + {`"max_bytes":65536`, `"max_bytes":65537`}, {`"min_bytes":1,`, ``}, + {`"content_type":"image/png",`, ``}, {`"content_type":"image/png"`, `"content_type":null`}, + {`"Authorization"`, `"rAnGe"`}, {`"Authorization"`, `"Accept-Encoding"`}, + {`"version":2`, `"version":2,"version":2`}, {`"kind":"image"`, `"module_version":1,"kind":"image"`}, + } { + if _, err := parseResourceChecks([]byte(strings.Replace(resourceV2Example, tc.from, tc.to, 1)), resourceCheckLookup); err == nil { + t.Fatalf("accepted %s -> %s", tc.from, tc.to) + } + } + legacyMissing := strings.Replace(resourceCheckExample, `"kind":"http",`, "", 1) + if _, err := parseResourceChecks([]byte(legacyMissing), resourceCheckLookup); err == nil { + t.Fatal("v1 defaulted kind") + } + for _, kind := range []string{"", `"kind":"http",`} { + raw := strings.ReplaceAll(resourceCheckExample, `"version":1`, `"version":2`) + raw = strings.ReplaceAll(raw, `"kind":"http",`, kind) + raw = strings.Replace(raw, `"statuses":[200],"min_bytes":0`, `"statuses":[200],"content_type":"text/plain","min_bytes":0`, 1) + cfg, err := parseResourceChecks([]byte(raw), resourceCheckLookup) + if err != nil || cfg.Services[0].Resources[0].Kind != "http" { + t.Fatal("v2 default/explicit http", err) + } + } +} + +// Real HTTP/1.1 framing and curl JSON, but no network and no child secrets. +func resourceV2Checks(t *testing.T, mode string, calls *int) resourceCheckDeps { + t.Helper() + return resourceCheckDeps{qualification: true, guard: func() error { return nil }, resolve: func(context.Context, string, string) ([]net.IP, error) { return []net.IP{net.ParseIP("8.8.8.8")}, nil }, runner: func(ctx context.Context, args []string, stdin io.Reader, out, diag io.Writer) error { + (*calls)++ + config, err := io.ReadAll(stdin) + if err != nil { + return err + } + argv := strings.Join(args, " ") + if strings.Contains(argv, "SECRET") || strings.Contains(argv, "private-token") || !bytes.Contains(config, []byte("connect-to =")) { + t.Fatal("secret/pin boundary") + } + if !strings.Contains(argv, "--max-time 0.700") { + t.Fatal("configured curl timeout", argv) + } + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > 700*time.Millisecond { + t.Fatal("configured context timeout") + } + raw, err := strconv.Unquote(strings.TrimPrefix(strings.Split(string(config), "\n")[0], "url = ")) + if err != nil { + return err + } + body, mimeType, status, extra := resourceFixture(), "image/png", 200, "" + if strings.Contains(raw, "/prefix") { + if !strings.Contains(argv, "--range 0-65535") { + t.Fatal("fixed core range missing") + } + size := 24 + if mode == "min-range" { + size = 16 + } + if mode == "max-range" { + size = 65536 + } + body = make([]byte, size) + binary.BigEndian.PutUint32(body, 16) + copy(body[4:], "ftypisom") + mimeType, status = "video/mp4", 206 + extra = fmt.Sprintf("Content-Range: bytes 0-%d/100000\r\n", size-1) + if mode == "bad-range" { + extra = "Content-Range: bytes 0-23/9223372036854775808\r\n" + } + if mode == "bad-body" { + copy(body[4:], "xxxx") + } + } else { + if !bytes.Contains(config, []byte("Authorization: Bearer private-token")) { + t.Fatal("secret header not on stdin") + } + if mode == "jpeg" { + var b bytes.Buffer + if err := jpeg.Encode(&b, image.NewRGBA(image.Rect(0, 0, 1, 1)), nil); err != nil { + return err + } + body, mimeType = b.Bytes(), "image/jpeg" + } + if mode == "gif" { + var b bytes.Buffer + if err := gif.Encode(&b, image.NewRGBA(image.Rect(0, 0, 1, 1)), nil); err != nil { + return err + } + body = b.Bytes() + } + if mode == "bad-body" { + body = []byte("not an image") + } + if mode == "truncated" { + body = body[:len(body)/2] + } + } + if mode == "status" { + status = 500 + } + if mode == "mime" { + mimeType = "text/html" + } + if mode == "malformed-mime" { + mimeType += "; broken" + } + fmt.Fprintf(out, "HTTP/1.1 %d result\r\nContent-Type: %s\r\nContent-Length: %d\r\n%s\r\n", status, mimeType, len(body), extra) + out.Write(body) + metrics, _ := json.Marshal(map[string]interface{}{"url_effective": raw, "ssl_verify_result": 0, "num_redirects": 0, "http_code": status, "time_total": 0.1, "speed_download": 10000, "size_download": len(body)}) + if mode == "metrics" { + metrics = []byte("invalid") + } + fmt.Fprint(out, resourceTrailer, string(metrics)) + version := "1.2" + if strings.Contains(argv, "--tlsv1.3") { + version = "1.3" + } + fmt.Fprintln(diag, "SSL connection using TLSv"+version) + if mode == "mixed" { + return errors.Join(errors.New("execution failed"), errors.New("invalid media range or container")) + } + return nil + }} +} + +func TestResourceV2SharedPublicPaths(t *testing.T) { + for _, mode := range []string{"pass", "jpeg", "min-range", "max-range", "bad-body", "bad-range", "status", "mime", "malformed-mime", "gif", "truncated", "metrics", "mixed"} { + t.Run(mode, func(t *testing.T) { + source := resourceV2Source(t) + if mode == "jpeg" { + source.config.Services[0].Resources[0].Expect.ContentType = "image/jpeg" + } + calls := 0 + checks := resourceV2Checks(t, mode, &calls) + negative := mode == "bad-body" || mode == "bad-range" || mode == "status" || mode == "mime" || mode == "malformed-mime" || mode == "gif" || mode == "truncated" + unknown := mode == "metrics" || mode == "mixed" + for _, spec := range source.config.Services[0].Resources { + for _, protocol := range []string{"tls12", "tls13"} { + result, err := runResourceProtocol(context.Background(), "media", spec, protocol, source.server, source.proxy, checks) + if unknown { + if err == nil { + t.Fatal("infrastructure became negative") + } + continue + } + want := "pass" + if negative && (mode != "bad-range" || spec.Kind == "mp4_range") && (mode != "gif" && mode != "truncated" || spec.Kind == "image") { + want = "expectation_failed" + } + if err != nil || result.Outcome != want || result.Bytes == 0 { + t.Fatalf("%s %s: %+v %v", spec.Kind, protocol, result, err) + } + if mode == "malformed-mime" { + status, size := 200, len(resourceFixture()) + if spec.Kind == "mp4_range" { + status, size = 206, 24 + } + if result.Status != status || result.Bytes != size || result.TimeMS != 100 { + t.Fatalf("lost malformed-MIME facts: %+v", result) + } + } + } + } + if unknown { + return + } + for _, spec := range source.config.Services[0].Resources { + _, err := probeResourceMonitor(context.Background(), "media", spec, source, func() error { return nil }, checks) + if err != nil { + t.Fatal(err) + } + } + snapshot := &resourceApplySnapshot{source: source, guard: func() error { return nil }} + status, err := resourceApplyHealth(context.Background(), snapshot, checks) + if (!negative && (err != nil || status != "OK")) || (negative && err == nil) { + t.Fatal("apply", status, err) + } + }) + } +} + +func TestResourceV2QualifierCompletion(t *testing.T) { + for _, mode := range []string{"pass", "bad-body", "bad-range", "malformed-mime", "metrics"} { + t.Run(mode, func(t *testing.T) { + q, store, registry, _, _ := qualifierFixture(t) + source := resourceV2Source(t) + q.source = func() (resourceMonitorSource, error) { return source, nil } + calls := 0 + q.checks = resourceV2Checks(t, mode, &calls) + proof, err := q.complete(context.Background(), store, registry) + if mode == "metrics" { + if err == nil { + t.Fatal("sealed unknown infrastructure") + } + return + } + if err != nil || calls != 12 { + t.Fatalf("completion %d %v", calls, err) + } + for _, cell := range proof.matrix.Runs[0].Checks { + want := "pass" + if mode == "bad-body" || mode == "malformed-mime" || mode == "bad-range" && cell.Resource == "prefix" { + want = "expectation_failed" + } + if cell.Attempts != 3 || cell.Outcome != want { + t.Fatalf("incomplete/wrong cell %+v", cell) + } + } + }) + } +} + +func TestResourceCoreMalformedMIMEClassification(t *testing.T) { + for _, kind := range []resourceResponseKind{resourceResponseGeneric, resourceResponseImage, resourceResponseMP4} { + for _, configured := range []bool{false, true} { + t.Run(fmt.Sprintf("kind-%d/configured-%t", kind, configured), func(t *testing.T) { + raw := "https://i.ytimg.com/private?SECRET" + options := resourceTransferOptions{method: "GET", response: kind, mediaType: "video/mp4", limit: 1024, timeout: time.Second} + if configured { + options.expect = &resourceCheckExpect{Statuses: []int{200}, MaxBytes: 1024} + } + runner := func(_ context.Context, _ []string, _ io.Reader, out, diag io.Writer) error { + var wire bytes.Buffer + mode := "" + if kind == resourceResponseMP4 { + mode = "media" + } + resourceWire(raw, mode, &wire, diag) + data := strings.Replace(wire.String(), "Content-Type: image/png\r\n", "Content-Type: image/png; broken\r\n", 1) + data = strings.Replace(data, "Content-Type: video/mp4\r\n", "Content-Type: video/mp4; broken\r\n", 1) + _, err := io.WriteString(out, data) + return err + } + transfer, _, err := collectResourceTransferOptions(context.Background(), "socks5h://127.0.0.1:1080", raw, "i.ytimg.com", "tls12", runner, options) + if configured && kind != resourceResponseGeneric { + status, size := 200, len(resourceFixture()) + if kind == resourceResponseMP4 { + status, size = 206, 24 + } + if err != errResourceExpectation || transfer.Status != status || len(transfer.Body) != size || transfer.LatencyMS != 100 || transfer.ContentType != "" { + t.Fatalf("typed facts/classification: %+v %v", transfer, err) + } + } else { + var fatal *managedRecoveryFatal + if !errors.As(err, &fatal) || err == errResourceExpectation || resourceRejectionOnly(err) || transfer.Status != 0 || len(transfer.Body) != 0 { + t.Fatalf("generic/private malformed MIME relaxed: %+v %v", transfer, err) + } + var resource *resourceError + if !errors.As(err, &resource) || resource.reason != reasonMIME || fatal.Err.Error() != "invalid content type" { + t.Fatalf("legacy MIME failure changed: %v", err) + } + } + }) + } + } +} + +func TestResourceV2FingerprintCompatibility(t *testing.T) { + key := []byte("fingerprint-test-key") + cfg, err := parseResourceChecks([]byte(resourceCheckExample), resourceCheckLookup) + if err != nil { + t.Fatal(err) + } + source := resourceMonitorSource{config: cfg, server: "dns", proxy: "proxy"} + spec := cfg.Services[0].Resources[0] + b, _ := json.Marshal(spec) + b, _ = json.Marshal([]string{string(b), spec.privateConfig, source.server, source.proxy}) + old := resourceMAC(key, b) + if resourceRowFingerprint(key, source, spec) != old { + t.Fatal("historic v1 serialization changed") + } + source.config.Version = 2 + v2 := resourceRowFingerprint(key, source, spec) + if v2 == old { + t.Fatal("v2 reused v1 proof namespace") + } + for _, mutate := range []func(*resourceCheckSpec){func(r *resourceCheckSpec) { r.Kind = "image" }, func(r *resourceCheckSpec) { r.Expect.MinBytes++ }, func(r *resourceCheckSpec) { r.Expect.MaxBytes++ }, func(r *resourceCheckSpec) { r.Expect.Statuses = []int{201} }} { + r := spec + mutate(&r) + if resourceRowFingerprint(key, source, r) == v2 { + t.Fatal("semantic change reused row proof") + } + } + prior := []resourceMonitorRow{{Service: cfg.Services[0].ID, Resource: spec.ID, Fingerprint: old, Health: "OK", Outcome: "pass"}} + rows := reconcileResourceRows(key, source, prior) + if rows[0].Health != "UNKNOWN" { + t.Fatal("v1 health survived v2 semantics") + } +} + +func TestResourceV2ConfiguredBounds(t *testing.T) { + for _, kind := range []string{"image", "mp4_range"} { + for _, mode := range []string{"exact", "minimum", "overflow"} { + t.Run(kind+"/"+mode, func(t *testing.T) { + source := resourceV2Source(t) + spec := source.config.Services[0].Resources[0] + size := len(resourceFixture()) + if kind == "mp4_range" { + spec = source.config.Services[0].Resources[1] + size = 24 + } + spec.Expect.MinBytes, spec.Expect.MaxBytes = size, size + if mode == "minimum" { + spec.Expect.MinBytes++ + spec.Expect.MaxBytes++ + } + if mode == "overflow" { + spec.Expect.MinBytes-- + spec.Expect.MaxBytes-- + } + calls := 0 + checks := resourceV2Checks(t, "pass", &calls) + runner := checks.runner + checks.runner = func(ctx context.Context, args []string, stdin io.Reader, out, diag io.Writer) error { + for i, a := range args { + if a == "--max-filesize" && args[i+1] != strconv.Itoa(spec.Expect.MaxBytes) { + t.Fatal("ignored configured limit", args) + } + } + return runner(ctx, args, stdin, out, diag) + } + result, err := runResourceProtocol(context.Background(), "media", spec, "tls12", source.server, source.proxy, checks) + switch mode { + case "exact": + if err != nil || result.Outcome != "pass" { + t.Fatal(result, err) + } + case "minimum": + if err != nil || result.Outcome != "expectation_failed" { + t.Fatal(result, err) + } + case "overflow": + if err == nil { + t.Fatal("body-bound violation became completed expectation", result) + } + } + }) + } + } +} + +func TestResourceV2CancelCannotSeal(t *testing.T) { + q, store, registry, _, _ := qualifierFixture(t) + source := resourceV2Source(t) + q.source = func() (resourceMonitorSource, error) { return source, nil } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + joined := false + calls := 0 + q.checks = resourceV2Checks(t, "pass", &calls) + q.checks.runner = func(child context.Context, _ []string, _ io.Reader, _ io.Writer, _ io.Writer) error { + cancel() + <-child.Done() + joined = true + return child.Err() + } + proof, err := q.complete(ctx, store, registry) + if err == nil || !joined || store.saveCompleted(proof) == nil { + t.Fatal("cancelled batch acquired proof", err, joined) + } +} + +func TestResourceV2OldMatrixCannotPromote(t *testing.T) { + source, key, registry, matrix, now := planFixture(t) + if len(deriveResourcePlan(source, key, registry, matrix, now).Assignments) == 0 { + t.Fatal("invalid baseline fixture") + } + source.config.Version = 2 + for si := range source.config.Services { + for ri := range source.config.Services[si].Resources { + source.config.Services[si].Resources[ri].Kind = "image" + } + } + // Even retaining the old source MAC cannot bypass per-cell semantic binding. + if len(deriveResourcePlan(source, key, registry, matrix, now).Assignments) != 0 { + t.Fatal("old HTTP matrix promoted typed images") + } +} + +func TestResourceV2CLIAndSourceBinding(t *testing.T) { + for _, mode := range []string{"pass", "bad-body"} { + t.Run(mode, func(t *testing.T) { + calls := 0 + checks := resourceV2Checks(t, mode, &calls) + checks.lookup = resourceCheckLookup + var out, diag bytes.Buffer + code := resourceCheckCLI(context.Background(), resourceCheckFile(t, resourceV2Example), &out, &diag, checks) + want := 0 + if mode != "pass" { + want = 1 + } + if code != want || calls != 4 || bytes.Contains(out.Bytes(), []byte("SECRET")) || bytes.Contains(out.Bytes(), []byte("private-token")) || diag.Len() != 0 { + t.Fatal(code, calls, out.String(), diag.String()) + } + }) + } + calls := 0 + checks := resourceV2Checks(t, "pass", &calls) + checks.lookup = resourceCheckLookup + checks.resolve = func(context.Context, string, string) ([]net.IP, error) { + t.Fatal("invalid typed HEAD reached DNS") + return nil, nil + } + var out, diag bytes.Buffer + if resourceCheckCLI(context.Background(), resourceCheckFile(t, strings.Replace(resourceV2Example, `"method":"GET"`, `"method":"HEAD"`, 1)), &out, &diag, checks) != 2 || calls != 0 { + t.Fatal("invalid typed config executed") + } + legacy, key := resourceMonitorFixture(t) + b, _ := json.Marshal(legacy.config) + parts := []string{string(b), legacy.server, legacy.proxy} + for _, s := range legacy.config.Services { + for _, r := range s.Resources { + parts = append(parts, r.privateConfig) + } + } + b, _ = json.Marshal(parts) + if legacy.fingerprint != resourceMAC(key, b) { + t.Fatal("v1 source serialization changed") + } + raw := strings.Replace(resourceCheckExample, `"version":1`, `"version":2`, 1) + raw = strings.Replace(raw, `"statuses":[200],"min_bytes":0`, `"statuses":[200],"content_type":"text/plain","min_bytes":0`, 1) + v2, err := readResourceMonitorSource(resourceCheckFile(t, raw), resourceCheckLookup, key) + if err != nil || v2.fingerprint == legacy.fingerprint { + t.Fatal("version unbound source", err) + } + implicit, err := readResourceMonitorSource(resourceCheckFile(t, strings.ReplaceAll(raw, `"kind":"http",`, "")), resourceCheckLookup, key) + if err != nil || implicit.fingerprint != v2.fingerprint { + t.Fatal("default kind not canonical", err) + } +} diff --git a/resource-check-v2.example.json b/resource-check-v2.example.json new file mode 100644 index 0000000..3255095 --- /dev/null +++ b/resource-check-v2.example.json @@ -0,0 +1,30 @@ +{ + "version": 2, + "services": [ + { + "id": "example", + "resources": [ + { + "id": "image", + "group": "images", + "kind": "image", + "url_env": "RESOURCE_IMAGE_URL", + "method": "GET", + "tls": "both", + "timeout_ms": 10000, + "expect": {"statuses": [200], "content_type": "image/png", "min_bytes": 1, "max_bytes": 5242880} + }, + { + "id": "prefix", + "group": "media", + "kind": "mp4_range", + "url_env": "RESOURCE_MP4_URL", + "method": "GET", + "tls": "both", + "timeout_ms": 10000, + "expect": {"statuses": [206], "content_type": "video/mp4", "min_bytes": 16, "max_bytes": 65536} + } + ] + } + ] +}