diff --git a/Dockerfile.target b/Dockerfile.target index 06e7535..16ca177 100644 --- a/Dockerfile.target +++ b/Dockerfile.target @@ -17,6 +17,7 @@ RUN if [ ! -e /opt/zapret2/lua ] && [ ! -L /opt/zapret2/lua ]; then \ fi \ && test -d /opt/zapret2/lua \ && command -v setsid \ + && (ss-server -h 2>&1 | grep -q -- '-d ') \ && mkdir -p /etc/zapret-autotune /var/lib/zapret-autotune/target \ && printf 'zapret-autotune-managed-v1\n' > /etc/zapret-autotune/target-id \ && chmod 700 /var/lib/zapret-autotune /var/lib/zapret-autotune/target diff --git a/MANAGED_TARGET.md b/MANAGED_TARGET.md index 07c0adc..26b160f 100644 --- a/MANAGED_TARGET.md +++ b/MANAGED_TARGET.md @@ -251,6 +251,10 @@ signed URLs; use `docker compose ... config --quiet` for validation. | --- | --- | | `ZAPRET_AUTOTUNE_IMAGE` | Local build/test default `zapret-autotune-managed-target:local`; production promotion requires a managed-target `repository@sha256:` | | `SS_PASSWORD` | Required independent hex-only secret, at least 32 characters; deployment README generates 64 | +| `SS_DNS_SERVER` | Empty (unchanged resolver); optional literal IPv4/IPv6 for target-only native ss-server DNS, validated before launch | +| `ZAPRET_AUTOTUNE_YOUTUBE_SERVICE` | `0` (legacy) or opt-in `1`; explicitly forwarded by Compose | +| `ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE` | `cold` default or `adaptive` (requires service `1`); explicitly forwarded | +| `ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS` | JSON array of 1..5 unique VideoIDs; defaults to only the primary ID; explicitly forwarded | | `ZAPRET_AUTOTUNE_HOST_SOCKS_PORT` | `11080`; choose an available TCP/UDP port 1–65535, published only on `127.0.0.1` | | `SOCKS_PORT` | Supplied deployment fixes internal listener to `1080`; not a host-port override | | `ZAPRET_AUTOTUNE_SOCKS_PROXY` | Fixed `socks5://127.0.0.1:1080` inside this target, no credentials/direct fallback | @@ -262,13 +266,26 @@ signed URLs; use `docker compose ... config --quiet` for validation. The monitor's three policy flags override corresponding environment defaults; explicit empty/invalid environment settings are rejected. The entrypoint accepts -no arguments. The supplied Compose environment does not forward the video/policy -settings from `.env`: add the desired existing variables to `target.environment` +no arguments. Compose forwards the primary/sample VideoIDs and service/mode settings. +It does not forward the interval/threshold/cooldown policy +settings from `.env`: add those existing variables to `target.environment` in your deployment copy/override if needed. Keep all fixed identity/path settings unchanged. No public SS listener is published; Xray configuration is external and operator-owned, optionally using the host's loopback SOCKS port. Do not modify an existing Xray deployment as part of bootstrap. +Current service/adaptive hardware acceptance is **pending**, independent of the +recorded legacy RC PASS. Adaptive means post-warmup, not cold: first connections +may fail and native learning repeats after restart. At least three real unique +signed ordinary media host samples are required; multiple VideoIDs may yield one +CDN host and leave `DEGRADED` with no authorized suffix. Sampled thumbnail/avatar +routes do not prove availability of the whole thumbnail pool. The reported +`8.8.8.8` DNS diagnostic corrected a tampered avatar, not all thumbnail routes; +native DNS correction is not full asset availability proof. Target DNS uses +validated native `-d` argv (nonsecret), with installed-option support checked +at image build; passwords remain private JSON only. No DoH bridge, permanent +Google IP pins, global host DNS edits or external startup manager are introduced. + ### Supervision and failure handling The container entrypoint is PID 1 and owns all technical child restart control. diff --git a/cmd/zapret-autotune/assets.go b/cmd/zapret-autotune/assets.go new file mode 100644 index 0000000..ef4659b --- /dev/null +++ b/cmd/zapret-autotune/assets.go @@ -0,0 +1,238 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/json" + "fmt" + "image" + "image/gif" + _ "image/jpeg" + _ "image/png" + "io" + "math" + "sort" + "strings" + "time" +) + +// Rebuild only the two bounded candidate tables, inside migrate's transaction. +// Refuse unexpected objects rather than silently discarding custom constraints. +func migrateAssetCandidates(ctx context.Context, conn *sql.Conn) error { + var actual []string + rows, err := conn.QueryContext(ctx, `SELECT sql FROM sqlite_master WHERE tbl_name IN ('strategy_candidates', 'candidate_evidence') AND sql IS NOT NULL ORDER BY sql`) + if err != nil { + return err + } + for rows.Next() { + var s string + if err := rows.Scan(&s); err != nil { + rows.Close() + return err + } + actual = append(actual, strings.TrimSpace(s)) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + var expected []string + for _, s := range strings.Split(strings.TrimSpace(candidateSchema), ";\n") { + if !strings.HasPrefix(s, "PRAGMA") { + expected = append(expected, strings.TrimSpace(s)) + } + } + sort.Strings(expected) + if strings.Join(actual, "\n") != strings.Join(expected, "\n") { + return fmt.Errorf("unsupported candidate schema; asset migration refused") + } + if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE asset_candidate_backup AS SELECT * FROM strategy_candidates; +CREATE TEMP TABLE asset_evidence_backup AS SELECT * FROM candidate_evidence; +DROP TABLE candidate_evidence; +DROP TABLE strategy_candidates;`); err != nil { + return err + } + schema := strings.Replace(candidateSchema, "('frontend', 'googlevideo')", "('frontend', 'googlevideo', 'assets')", 1) + if _, err := conn.ExecContext(ctx, schema); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `CREATE TRIGGER asset_candidate_scope BEFORE INSERT ON strategy_candidates + WHEN NEW.strategy_group = 'assets' AND (json_array_length(NEW.endpoints) > 2 + OR EXISTS (SELECT 1 FROM json_each(NEW.endpoints) WHERE type != 'text' OR value NOT IN ('i.ytimg.com', 'yt3.ggpht.com')) + OR (SELECT count(DISTINCT value) FROM json_each(NEW.endpoints)) != json_array_length(NEW.endpoints)) + BEGIN SELECT RAISE(ABORT, 'invalid asset service inventory'); END;`); err != nil { + return err + } + _, err = conn.ExecContext(ctx, `INSERT INTO strategy_candidates SELECT * FROM asset_candidate_backup; +INSERT INTO candidate_evidence SELECT * FROM asset_evidence_backup; +DROP TABLE asset_candidate_backup; +DROP TABLE asset_evidence_backup; +PRAGMA user_version = 6;`) + return err +} + +// No URLs: the caller supplies actual response bodies, never a blockcheck report. +// Three transfers per exact TLS version are required; no probing is wired here. +type assetTransfer struct { + EffectiveHost string `json:"effective_host"` + SSLVerifyResult *int `json:"ssl_verify_result"` + Redirects int `json:"redirects"` + ContentType string `json:"content_type"` + BodyHash string `json:"body_sha256"` + Format string `json:"format"` + Size int `json:"size"` + TLSVersion uint16 `json:"tls_version"` + Status int `json:"status"` + Body []byte `json:"-"` + LatencyMS float64 `json:"latency_ms"` + SpeedKBPS float64 `json:"speed_kbps"` +} + +type assetMeasurement struct { + Source string `json:"source"` + Endpoint string `json:"endpoint"` + Protocol string `json:"protocol"` + Args string `json:"args"` + Transfers []assetTransfer `json:"transfers"` +} + +func (t *assetTransfer) UnmarshalJSON(raw []byte) error { + type fields assetTransfer + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return err + } + for _, key := range []string{"effective_host", "ssl_verify_result", "redirects", "content_type", "body_sha256", "format", "size", "tls_version", "status", "latency_ms", "speed_kbps"} { + if value, ok := keys[key]; !ok || bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + return fmt.Errorf("missing image transfer field") + } + } + var value fields + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + if err := d.Decode(&value); err != nil { + return err + } + *t = assetTransfer(value) + return nil +} + +func saveAssetEvidence(tx *sql.Tx, id int64, m assetMeasurement, checked time.Time) error { + if m.Source != "image-transfer" || !assetEndpoint(m.Endpoint) || (m.Protocol != "tls12" && m.Protocol != "tls13") || checked.IsZero() || len(m.Transfers) != 3 { + return fmt.Errorf("invalid image-transfer evidence") + } + var group, args string + if err := tx.QueryRow(`SELECT strategy_group, args FROM strategy_candidates WHERE id = ?`, id).Scan(&group, &args); err != nil { + return err + } + if group != "assets" || m.Args != args { + return fmt.Errorf("asset evidence definition mismatch") + } + latency, speed, err := validateAssetTransfers(m) + if err != nil { + return err + } + for i := range m.Transfers { + if err := normalizeAssetBody(&m.Transfers[i]); err != nil { + return err + } + } + raw, err := json.Marshal(m) + if err != nil { + return err + } + _, err = tx.Exec(`INSERT INTO candidate_evidence (candidate_id, endpoint, protocol, passes_ok, passes_total, success_rate, coverage, median_latency_ms, median_speed_kbps, checked_at, raw_measurement) VALUES (?, ?, ?, 3, 3, 1, 1, ?, ?, ?, ?)`, id, m.Endpoint, m.Protocol, latency, speed, checked.UTC().Format(time.RFC3339Nano), string(raw)) + return err +} + +func validateAssetTransfers(m assetMeasurement) (float64, float64, error) { + if m.Source != "image-transfer" || !assetEndpoint(m.Endpoint) || (m.Protocol != "tls12" && m.Protocol != "tls13") || len(m.Transfers) != 3 { + return 0, 0, fmt.Errorf("invalid image-transfer evidence") + } + tlsVersion := uint16(0x0303) + if m.Protocol == "tls13" { + tlsVersion = 0x0304 + } + var latency, speed []float64 + for _, transfer := range m.Transfers { + if transfer.TLSVersion != tlsVersion || transfer.Status != 200 || transfer.EffectiveHost != m.Endpoint || transfer.SSLVerifyResult == nil || *transfer.SSLVerifyResult != 0 || transfer.Redirects != 0 { + return 0, 0, fmt.Errorf("image transfer must match TLS and HTTP 200 with a bounded image body") + } + if err := normalizeAssetBody(&transfer); err != nil { + return 0, 0, err + } + for _, n := range []float64{transfer.LatencyMS, transfer.SpeedKBPS} { + if n < 0 || math.IsNaN(n) || math.IsInf(n, 0) { + return 0, 0, fmt.Errorf("invalid image transfer metric") + } + } + latency = append(latency, transfer.LatencyMS) + speed = append(speed, transfer.SpeedKBPS) + } + sort.Float64s(latency) + sort.Float64s(speed) + return latency[1], speed[1], nil +} + +// Bodies are transient. Stored evidence carries only verified normalized facts. +func normalizeAssetBody(t *assetTransfer) error { + if len(t.Body) != 0 { + if len(t.Body) > 5<<20 { + return fmt.Errorf("image body exceeds limit") + } + cfg, format, err := image.DecodeConfig(bytes.NewReader(t.Body)) + if err != nil || cfg.Width <= 0 || cfg.Height <= 0 || cfg.Width > (16<<20)/cfg.Height { + return fmt.Errorf("invalid image dimensions") + } + if format == "gif" { + if _, err := gif.DecodeAll(bytes.NewReader(t.Body)); err != nil { + return fmt.Errorf("invalid GIF body") + } + } else if _, _, err := image.Decode(bytes.NewReader(t.Body)); err != nil { + return fmt.Errorf("invalid image body") + } + t.Format, t.Size, t.BodyHash = format, len(t.Body), fmt.Sprintf("%x", sha256.Sum256(t.Body)) + } + want := map[string]string{"png": "image/png", "jpeg": "image/jpeg", "gif": "image/gif"}[t.Format] + if want == "" || t.ContentType != want || t.Size <= 0 || t.Size > 5<<20 || len(t.BodyHash) != 64 || strings.Trim(t.BodyHash, "0123456789abcdef") != "" { + return fmt.Errorf("invalid normalized image evidence") + } + return nil +} + +// Revalidate asset bodies on reads too: corrupted raw evidence cannot qualify. +func validateStoredAssetEvidence(db *sql.DB, c storedCandidate) (bool, error) { + rows, err := db.Query(`SELECT endpoint, protocol, median_latency_ms, median_speed_kbps, checked_at, raw_measurement FROM candidate_evidence WHERE candidate_id = ?`, c.ID) + if err != nil { + return false, err + } + defer rows.Close() + for rows.Next() { + var endpoint, protocol, checkedAt, raw string + var latency, speed float64 + if err := rows.Scan(&endpoint, &protocol, &latency, &speed, &checkedAt, &raw); err != nil { + return false, err + } + checked, err := time.Parse(time.RFC3339Nano, checkedAt) + if err != nil || checked.IsZero() { + return false, nil + } + var m assetMeasurement + d := json.NewDecoder(strings.NewReader(raw)) + d.DisallowUnknownFields() + if err := d.Decode(&m); err != nil { + return false, nil + } + if err := d.Decode(new(interface{})); err != io.EOF { + return false, nil + } + l, s, err := validateAssetTransfers(m) + if err != nil || m.Endpoint != endpoint || m.Protocol != protocol || m.Args != c.Args || l != latency || s != speed { + return false, nil + } + } + return true, rows.Err() +} diff --git a/cmd/zapret-autotune/assets_test.go b/cmd/zapret-autotune/assets_test.go new file mode 100644 index 0000000..9653694 --- /dev/null +++ b/cmd/zapret-autotune/assets_test.go @@ -0,0 +1,315 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "image" + "image/png" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +// Recreate an actual pre-assets schema, not just a misleading version number. +func restoreV5CandidateSchema(t *testing.T, db *sql.DB) { + t.Helper() + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + _, err = tx.Exec(`CREATE TEMP TABLE old_candidates AS SELECT id, source_run_id, strategy_group, endpoints, args, fingerprint FROM strategy_candidates; +CREATE TEMP TABLE old_evidence AS SELECT * FROM candidate_evidence; +DROP TABLE googlevideo_scope_evidence; +DROP TABLE candidate_evidence; DROP TABLE strategy_candidates;` + candidateSchema + ` +INSERT INTO strategy_candidates SELECT * FROM old_candidates; +INSERT INTO candidate_evidence SELECT * FROM old_evidence; +DROP TABLE old_candidates; DROP TABLE old_evidence; +PRAGMA user_version = 5;`) + if err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } +} + +// Synthetic unit fixture only: no claim that any endpoint was reachable. +func unitImageMeasurement(t *testing.T, host, protocol, args string) assetMeasurement { + t.Helper() + var body bytes.Buffer + if err := png.Encode(&body, image.NewRGBA(image.Rect(0, 0, 1, 1))); err != nil { + t.Fatal(err) + } + v := uint16(0x0303) + if protocol == "tls13" { + v = 0x0304 + } + m := assetMeasurement{Source: "image-transfer", Endpoint: host, Protocol: protocol, Args: args} + for i := 0; i < 3; i++ { + zero := 0 + m.Transfers = append(m.Transfers, assetTransfer{EffectiveHost: host, SSLVerifyResult: &zero, ContentType: "image/png", TLSVersion: v, Status: 200, Body: body.Bytes(), LatencyMS: 10, SpeedKBPS: 100}) + } + return m +} + +func unitAssetCandidate(t *testing.T, db *sql.DB, host, args string, qualify bool) int64 { + t.Helper() + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + id, err := insertStrategyCandidate(tx, "assets", []string{host}, args, nil) + if err != nil { + t.Fatal(err) + } + if qualify { + for _, protocol := range []string{"tls12", "tls13"} { + if err := saveAssetEvidence(tx, id, unitImageMeasurement(t, host, protocol, args), time.Now()); err != nil { + t.Fatal(err) + } + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + return id +} + +func TestAssetMigrationLegacyBytesAndRollback(t *testing.T) { + db := testStore(t) + saveProposal(t, db, "--lua-desync=fake") + restoreV5CandidateSchema(t, db) + before, err := readStrategyCandidate(db, 1) + if err != nil { + t.Fatal(err) + } + config, err := renderManagedConfig(db, []int64{1}) + if err != nil { + t.Fatal(err) + } + var evidence string + if err := db.QueryRow(`SELECT group_concat(raw_measurement) FROM candidate_evidence`).Scan(&evidence); err != nil { + t.Fatal(err) + } + if err := migrate(db); err != nil { + t.Fatal(err) + } + after, err := readStrategyCandidate(db, 1) + if err != nil || !reflect.DeepEqual(before, after) { + t.Fatalf("changed candidate: %+v %v", after, err) + } + rendered, err := renderManagedConfig(db, []int64{1}) + if err != nil || !bytes.Equal(config, rendered) || managedHash(config) != managedHash(rendered) { + t.Fatalf("legacy bytes changed: %v", err) + } + var afterEvidence string + if err := db.QueryRow(`SELECT group_concat(raw_measurement) FROM candidate_evidence`).Scan(&afterEvidence); err != nil || evidence != afterEvidence { + t.Fatalf("evidence changed: %v", err) + } + if _, err := db.Exec(`UPDATE strategy_candidates SET args = '--lua-desync=multisplit'`); err == nil { + t.Fatal("immutable trigger lost") + } + restoreV5CandidateSchema(t, db) + // A foreign reference forces failure after the migration starts rebuilding. + if _, err := db.Exec(`CREATE TABLE candidate_external_ref (id INTEGER REFERENCES strategy_candidates(id)); INSERT INTO candidate_external_ref VALUES (1);`); err != nil { + t.Fatal(err) + } + if err := migrate(db); err == nil { + t.Fatal("expected atomic migration failure") + } + var version int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { + t.Fatalf("version=%d %v", version, err) + } + rendered, err = renderManagedConfig(db, []int64{1}) + if err != nil || !bytes.Equal(config, rendered) { + t.Fatalf("rollback lost legacy bytes: %v", err) + } + if _, err := db.Exec(`DROP TABLE candidate_external_ref; CREATE TRIGGER custom_candidate BEFORE INSERT ON strategy_candidates BEGIN SELECT 1; END;`); err != nil { + t.Fatal(err) + } + if err := migrate(db); err == nil { + t.Fatal("unknown schema silently adopted") + } +} + +func TestAssetEvidenceAndDisjointRender(t *testing.T) { + db := testStore(t) + a := unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true) + b := unitAssetCandidate(t, db, "yt3.ggpht.com", "--lua-desync=multisplit", true) + config, err := renderManagedConfig(db, []int64{a, b}) + if err != nil || !strings.Contains(string(config), "--hostlist-domains=^i.ytimg.com --lua-desync=fake --new") || !strings.Contains(string(config), "--hostlist-domains=^yt3.ggpht.com --lua-desync=multisplit") { + t.Fatalf("render: %s %v", config, err) + } + ids, err := qualifiedCandidateIDs(db, "assets") + if err != nil || !reflect.DeepEqual(ids, []int64{b, a}) { + t.Fatalf("IDs %v %v", ids, err) + } + if _, err := db.Exec(`UPDATE candidate_evidence SET checked_at = 'fixture' WHERE candidate_id = ? AND protocol = 'tls12'`, a); err != nil { + t.Fatal(err) + } + if _, err := renderManagedConfig(db, []int64{a}); err == nil { + t.Fatal("invalid checked timestamp qualified") + } + if _, err := db.Exec(`UPDATE candidate_evidence SET checked_at = ? WHERE candidate_id = ? AND protocol = 'tls12'`, time.Now().UTC().Format(time.RFC3339Nano), a); err != nil { + t.Fatal(err) + } + overlap := unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=multisplit", true) + if _, err := renderManagedConfig(db, []int64{a, overlap}); err == nil { + t.Fatal("accepted overlap") + } + missing := unitAssetCandidate(t, db, "yt3.ggpht.com", "--lua-desync=fake", false) + if _, err := renderManagedConfig(db, []int64{a, missing}); err == nil { + t.Fatal("accepted missing evidence") + } + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + for _, host := range []string{"ytimg.com", "other.ytimg.com", "*.ytimg.com", "youtube.com"} { + if _, err := insertStrategyCandidate(tx, "assets", []string{host}, "--lua-desync=fake", nil); err == nil { + t.Fatalf("accepted %s", host) + } + } + if err := saveCandidateEvidence(tx, missing, "yt3.ggpht.com", "tls12", []byte(`{}`), time.Now()); err == nil { + t.Fatal("accepted blockcheck for assets") + } + for _, damage := range []string{"status", "magic", "tls", "count", "source", "scope"} { + m := unitImageMeasurement(t, "yt3.ggpht.com", "tls12", "--lua-desync=fake") + switch damage { + case "status": + m.Transfers[0].Status = 400 + case "magic": + m.Transfers[0].Body = []byte("not an image") + case "tls": + m.Transfers[0].TLSVersion = 0x0304 + case "count": + m.Transfers = m.Transfers[:2] + case "source": + m.Source = "blockcheck" + case "scope": + m.Endpoint = "i.ytimg.com" + } + if err := saveAssetEvidence(tx, missing, m, time.Now()); err == nil { + t.Fatalf("accepted %s", damage) + } + } + tx.Rollback() + m := unitImageMeasurement(t, "yt3.ggpht.com", "tls13", "--lua-desync=multisplit") + m.Transfers[0].Status = 400 + raw, _ := json.Marshal(m) + if _, err := db.Exec(`UPDATE candidate_evidence SET raw_measurement = ? WHERE candidate_id = ? AND protocol = 'tls13'`, string(raw), b); err != nil { + t.Fatal(err) + } + if _, err := renderManagedConfig(db, []int64{b}); err == nil { + t.Fatal("corrupt HTTP400 evidence qualified") + } + if _, err := db.Exec("PRAGMA user_version = 8"); err != nil { + t.Fatal(err) + } + if _, err := renderManagedConfig(db, []int64{a}); err == nil { + t.Fatal("unknown schema rendered") + } +} + +func TestAssetCLIApplyPartialFailureAndStatus(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(fmt.Sprint(fail), func(t *testing.T) { + d, env, _ := managedCLIFixture(t) + env["ZAPRET_AUTOTUNE_VIDEO_ID"] = "dQw4w9WgXcQ" + path := filepath.Join(t.TempDir(), "assets.db") + db, err := openStore(path) + if err != nil { + t.Fatal(err) + } + saveProposal(t, db, "--lua-desync=fake") + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + ggc, err := insertStrategyCandidate(tx, "googlevideo", []string{"rr1.unit.googlevideo.com"}, "--lua-desync=fake", nil) + if err != nil { + tx.Rollback() + t.Fatal(err) + } + var report struct { + Strategies []json.RawMessage `json:"strategies"` + } + if err := json.Unmarshal([]byte(combinedReportFixture("--lua-desync=fake")), &report); err != nil { + tx.Rollback() + t.Fatal(err) + } + for i, protocol := range []string{"tls12", "tls13"} { + if err := saveCandidateEvidence(tx, ggc, "rr1.unit.googlevideo.com", protocol, report.Strategies[i], time.Now()); err != nil { + tx.Rollback() + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + a := unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true) + b := unitAssetCandidate(t, db, "yt3.ggpht.com", "--lua-desync=multisplit", true) + db.Close() + d.openDB = func() (*sql.DB, error) { return openHistoryStore(path) } + initManagedCLI(t, d) + if code, out, diagnostic := callManaged(t, d, "status"); code != 0 || strings.Contains(out, "assets") { + t.Fatalf("legacy status: %s %s", out, diagnostic) + } + if code, out, diagnostic := callManaged(t, d, "status", "--assets"); code != 0 || !strings.Contains(out, "unresolved_groups=frontend,googlevideo,assets") { + t.Fatalf("requested status: %s %s", out, diagnostic) + } + if fail { + d.player = func(context.Context, []string, io.Writer, io.Writer) error { return errors.New("unit health failure") } + } + code, _, diagnostic := callManaged(t, d, "apply", "--asset-candidate", fmt.Sprintf("%d,%d", a, b)) + if (code != 0) != fail { + t.Fatalf("apply=%d %s", code, diagnostic) + } + config, err := os.ReadFile(filepath.Join(d.directory, "config")) + if err != nil { + t.Fatal(err) + } + if fail && string(config) != managedBaseline { + t.Fatal("failure did not restore baseline") + } + if !fail { + if code, out, diagnostic := callManaged(t, d, "status"); code != 0 || !strings.Contains(out, "groups=assets") || strings.Contains(out, "unresolved_groups=frontend,googlevideo,assets") { + t.Fatalf("asset status: %s %s", out, diagnostic) + } + if code, _, diagnostic := callManaged(t, d, "apply", "--asset-candidate", fmt.Sprint(a)); code != 0 { + t.Fatalf("partial apply: %s", diagnostic) + } + if code, out, diagnostic := callManaged(t, d, "status"); code != 0 || !strings.Contains(out, "unresolved_groups=frontend,googlevideo,assets") { + t.Fatalf("partial status: %s %s", out, diagnostic) + } + if code, _, diagnostic := callManaged(t, d, "apply", "--frontend-candidate", "1", "--googlevideo-candidate", fmt.Sprint(ggc), "--asset-candidate", fmt.Sprint(a), "--asset-candidate", fmt.Sprint(b)); code != 0 { + t.Fatalf("four candidate apply: %s", diagnostic) + } + if code, out, diagnostic := callManaged(t, d, "active-ids"); code != 0 || !strings.Contains(out, fmt.Sprintf(`"active_candidate_ids":[1,%d,%d,%d]`, ggc, a, b)) || !strings.Contains(out, `"unresolved_groups":null`) { + t.Fatalf("four IDs: %s %s", out, diagnostic) + } + } + }) + } + for _, args := range [][]string{{"apply", "--asset-candidate", "1,1"}, {"apply", "--asset-candidate", "1,"}, {"apply", "--asset-candidate", "1,2,3"}, {"apply", "--frontend-candidate", "1,2"}, {"apply", "--frontend-candidate", "1", "--asset-candidate", "1"}} { + if _, _, err := managedArguments(args); err == nil { + t.Fatalf("accepted %v", args) + } + } + _, ids, err := managedArguments([]string{"apply", "--asset-candidate", "3", "--googlevideo-candidate", "2", "--asset-candidate", "4", "--frontend-candidate", "1"}) + if err != nil || !reflect.DeepEqual(ids, []int64{1, 2, 3, 4}) { + t.Fatalf("IDs %v %v", ids, err) + } +} diff --git a/cmd/zapret-autotune/candidates.go b/cmd/zapret-autotune/candidates.go index e9c9173..2a4aa15 100644 --- a/cmd/zapret-autotune/candidates.go +++ b/cmd/zapret-autotune/candidates.go @@ -5,6 +5,8 @@ import ( "database/sql" "encoding/json" "fmt" + "io" + "math" "strings" "time" ) @@ -47,7 +49,17 @@ CREATE TRIGGER evidence_scope_update BEFORE UPDATE ON candidate_evidence BEGIN SELECT RAISE(ABORT, 'evidence endpoint outside candidate scope'); END; PRAGMA user_version = 3;` -func insertStrategyCandidate(tx *sql.Tx, group string, endpoints []string, args string, sourceRun interface{}) (int64, error) { +func insertStrategyCandidate(tx *sql.Tx, group string, endpoints []string, args string, sourceRun interface{}, modes ...QualificationMode) (int64, error) { + mode := QualificationCold + if len(modes) > 1 { + return 0, fmt.Errorf("one qualification mode required") + } + if len(modes) == 1 { + mode = modes[0] + } + if mode != QualificationCold && (mode != QualificationAdaptive || group != "googlevideo") { + return 0, fmt.Errorf("invalid qualification mode/group") + } c := candidate{Group: group, Domains: append([]string(nil), endpoints...), Args: args} if err := c.validateMeasurements(); err != nil { return 0, err @@ -65,7 +77,13 @@ func insertStrategyCandidate(tx *sql.Tx, group string, endpoints []string, args if err != nil { return 0, err } - result, err := tx.Exec(`INSERT INTO strategy_candidates (source_run_id, strategy_group, endpoints, args, fingerprint) VALUES (?, ?, ?, ?, ?)`, sourceRun, group, string(scope), args, fingerprint) + query := `INSERT INTO strategy_candidates (source_run_id, strategy_group, endpoints, args, fingerprint) VALUES (?, ?, ?, ?, ?)` + values := []interface{}{sourceRun, group, string(scope), args, fingerprint} + if mode == QualificationAdaptive { + query = `INSERT INTO strategy_candidates (source_run_id, strategy_group, endpoints, args, fingerprint, qualification_mode) VALUES (?, ?, ?, ?, ?, ?)` + values = append(values, mode) + } + result, err := tx.Exec(query, values...) if err != nil { return 0, err } @@ -74,6 +92,19 @@ func insertStrategyCandidate(tx *sql.Tx, group string, endpoints []string, args // Evidence is tied to the immutable definition, never inferred from scan output. func saveCandidateEvidence(tx *sql.Tx, id int64, endpoint, protocol string, raw []byte, checked time.Time) error { + var version int + if err := tx.QueryRow("PRAGMA user_version").Scan(&version); err != nil { + return err + } + if version >= 7 { + var mode QualificationMode + if err := tx.QueryRow("SELECT qualification_mode FROM strategy_candidates WHERE id = ?", id).Scan(&mode); err != nil { + return err + } + if mode != QualificationCold { + return fmt.Errorf("adaptive evidence must not use cold metrics") + } + } if !validDomain(endpoint) || (protocol != "tls12" && protocol != "tls13") || checked.IsZero() { return fmt.Errorf("invalid evidence endpoint, protocol or checked timestamp") } @@ -82,6 +113,9 @@ func saveCandidateEvidence(tx *sql.Tx, id int64, endpoint, protocol string, raw if err := tx.QueryRow(`SELECT strategy_group, args FROM strategy_candidates WHERE id = ?`, id).Scan(&group, &args); err != nil { return err } + if group == "assets" { + return fmt.Errorf("assets require image-transfer evidence") + } var fields map[string]json.RawMessage if err := json.Unmarshal(raw, &fields); err != nil { return err @@ -119,6 +153,7 @@ type storedCandidate struct { Group, Args, Fingerprint string Endpoints []string Qualified bool + Mode QualificationMode } // Qualification precedes LIMIT: at most ten immutable definitions per trial. @@ -138,18 +173,41 @@ const candidateQualifiedSQL = `NOT EXISTS (SELECT 1 FROM json_each(c.endpoints) (SELECT 'tls12' AS protocol UNION ALL SELECT 'tls13') p WHERE NOT EXISTS (SELECT 1 FROM candidate_evidence r WHERE r.candidate_id = c.id AND r.endpoint = e.value AND r.protocol = p.protocol AND r.passes_ok = 3 - AND r.passes_total = 3 AND r.success_rate = 1 AND r.coverage = 1))` + AND r.passes_total = 3 AND r.success_rate = 1 AND r.coverage = 1 + AND (c.strategy_group != 'assets' OR (json_extract(r.raw_measurement, '$.source') = 'image-transfer' + AND json_extract(r.raw_measurement, '$.endpoint') = e.value + AND json_extract(r.raw_measurement, '$.protocol') = p.protocol + AND json_extract(r.raw_measurement, '$.args') = c.args + AND json_array_length(r.raw_measurement, '$.transfers') = 3))))` // Returns detached snapshots of immutable definitions, not active state. func listQualifiedCandidates(db *sql.DB, group string, limit int) ([]storedCandidate, error) { - if group != "frontend" && group != "googlevideo" { + if group != "frontend" && group != "googlevideo" && group != "assets" { return nil, fmt.Errorf("unknown candidate group") } if limit < 1 || limit > 10 { return nil, fmt.Errorf("candidate limit must be between 1 and 10") } - rows, err := db.Query(`SELECT id, strategy_group, args, fingerprint, endpoints FROM strategy_candidates c - WHERE strategy_group = ? AND `+candidateQualifiedSQL+` ORDER BY id DESC LIMIT ?`, group, limit) + query := `SELECT id, strategy_group, args, fingerprint, endpoints FROM strategy_candidates c + WHERE strategy_group = ? AND ` + candidateQualifiedSQL + ` ORDER BY id DESC` + var version int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil { + return nil, err + } + if version > 7 { + return nil, fmt.Errorf("unsupported qualification schema") + } + if version == 7 { + query = `SELECT id, strategy_group, args, fingerprint, endpoints FROM strategy_candidates c + WHERE strategy_group = ? AND ((qualification_mode = 'cold' AND ` + candidateQualifiedSQL + `) OR qualification_mode = 'adaptive') ORDER BY id DESC` + } + queryArgs := []interface{}{group} + // Asset bodies must validate before LIMIT, not just their SQL summaries. + if group == "frontend" { + query += " LIMIT ?" + queryArgs = append(queryArgs, limit) + } + rows, err := db.Query(query, queryArgs...) if err != nil { return nil, err } @@ -165,17 +223,51 @@ func listQualifiedCandidates(db *sql.DB, group string, limit int) ([]storedCandi return nil, err } c.Qualified = true + c.Mode = QualificationCold result = append(result, c) } if err := rows.Err(); err != nil { return nil, err } - return result, rows.Close() + if err := rows.Close(); err != nil { + return nil, err + } + if group == "assets" || group == "googlevideo" { + var validated []storedCandidate + for _, c := range result { + ok := true + var err error + if group == "assets" { + ok, err = validateStoredAssetEvidence(db, c) + } else { + c.Mode, err = readQualificationMode(db, c.ID) + ok = c.Mode == QualificationCold + if err == nil && c.Mode == QualificationAdaptive { + ok, err = validateStoredAdaptiveGooglevideoEvidence(db, c) + } + } + if err != nil { + return nil, err + } + if ok { + validated = append(validated, c) + if len(validated) == limit { + break + } + } + } + return validated, nil + } + return result, nil } // One shared qualification rule for the complete endpoint × TLS matrix. // Missing or failed evidence is never a qualification; this is not active state. func readStrategyCandidate(db *sql.DB, id int64) (storedCandidate, error) { + return readStrategyCandidatePolicy(db, id, true) +} + +func readStrategyCandidatePolicy(db *sql.DB, id int64, fresh bool) (storedCandidate, error) { var c storedCandidate var scope string err := db.QueryRow(`SELECT id, strategy_group, args, fingerprint, endpoints, @@ -185,5 +277,251 @@ func readStrategyCandidate(db *sql.DB, id int64) (storedCandidate, error) { return c, err } err = json.Unmarshal([]byte(scope), &c.Endpoints) + if err == nil { + c.Mode, err = readQualificationMode(db, id) + if c.Mode != QualificationCold { + c.Qualified = false + if err == nil && c.Mode == QualificationAdaptive { + c.Qualified, err = validateAdaptiveGooglevideoEvidence(db, c, fresh) + } + } + } + if err == nil && c.Qualified && c.Group == "assets" { + c.Qualified, err = validateStoredAssetEvidence(db, c) + } return c, err } + +type QualificationMode string + +const QualificationCold QualificationMode = "cold" +const QualificationAdaptive QualificationMode = "adaptive" +const NativeCircularPostWarmup = "native-circular-post-warmup" + +// Additive migration: no candidate/evidence/asset/owner table rebuilds. +const googlevideoQualificationSchema = ` +ALTER TABLE strategy_candidates ADD COLUMN qualification_mode TEXT NOT NULL DEFAULT 'cold' + CHECK (qualification_mode = 'cold' OR (qualification_mode = 'adaptive' AND strategy_group = 'googlevideo')); +CREATE TABLE googlevideo_scope_evidence ( + candidate_id INTEGER PRIMARY KEY REFERENCES strategy_candidates(id), + proof TEXT NOT NULL CHECK (json_valid(proof)) +); +PRAGMA user_version = 7;` + +func readQualificationMode(db *sql.DB, id int64) (QualificationMode, error) { + var version int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil { + return "", err + } + if version < 7 { + return QualificationCold, nil + } + if version != 7 { + return "", fmt.Errorf("unsupported qualification schema") + } + var mode QualificationMode + err := db.QueryRow("SELECT qualification_mode FROM strategy_candidates WHERE id = ?", id).Scan(&mode) + if err == nil && mode != QualificationCold && mode != QualificationAdaptive { + err = fmt.Errorf("unknown qualification mode") + } + return mode, err +} + +// Collector DTO: normalized facts only. No URL, query, body, or headers can be +// persisted. HTTP root/404 connectivity is deliberately not media evidence. +type GooglevideoPass struct { + Protocol string `json:"protocol"` + VerifiedTLS bool `json:"verified_tls"` + Status int `json:"status"` + RangeStart int64 `json:"range_start"` + RangeEnd int64 `json:"range_end"` + ResourceBytes int64 `json:"resource_bytes"` + Bytes int64 `json:"bytes"` + FTYP bool `json:"ftyp"` + LatencyMS float64 `json:"latency_ms"` +} + +type GooglevideoHostProof struct { + Host string `json:"host"` + WarmupAttempts int `json:"warmup_attempts"` + WarmupFailures int `json:"warmup_failures"` + WarmupLimit int `json:"warmup_limit"` + WarmupMS float64 `json:"warmup_ms"` + Passes []GooglevideoPass `json:"steady_passes"` +} + +type GooglevideoQualificationProof struct { + Version int `json:"version"` + Mode QualificationMode `json:"mode"` + SourceKind string `json:"source_kind"` + Args string `json:"args"` + Hosts []GooglevideoHostProof `json:"hosts"` + Attestation *GooglevideoRuntimeAttestation `json:"attestation,omitempty"` + producerSeal *googlevideoProducerSeal +} + +func (p GooglevideoQualificationProof) validate(c storedCandidate) error { + if err := p.validateStored(c); err != nil { + return err + } + if p.Mode == QualificationAdaptive { + return p.Attestation.validate(c) + } + return nil +} + +func (p GooglevideoQualificationProof) validateStored(c storedCandidate) error { + if p.Version != 1 || p.Mode != c.Mode || p.Args != c.Args || len(p.Hosts) != len(c.Endpoints) || len(p.Hosts) < 3 { + return fmt.Errorf("invalid representative proof definition") + } + if (p.Mode == QualificationCold && p.SourceKind != "media-range") || (p.Mode == QualificationAdaptive && p.SourceKind != NativeCircularPostWarmup) || (p.Mode != QualificationCold && p.Mode != QualificationAdaptive) { + return fmt.Errorf("invalid proof source/mode") + } + if err := (&candidate{Args: p.Args}).validateArgs(); err != nil { + return err + } + if err := validateGooglevideoHosts(c.Endpoints); err != nil { + return err + } + if p.Mode == QualificationCold && p.Attestation != nil { + return fmt.Errorf("cold evidence cannot contain native attestation") + } + wanted, seen, roots, prefixes := map[string]bool{}, map[string]bool{}, map[string]bool{}, map[string]bool{} + for _, host := range c.Endpoints { + wanted[host] = true + } + for _, h := range p.Hosts { + prefix, root, ok := strings.Cut(strings.TrimSuffix(h.Host, ".googlevideo.com"), "---sn-") + if !ok || !strings.HasSuffix(h.Host, ".googlevideo.com") || !validDomain(h.Host) || h.Host != strings.ToLower(h.Host) || !wanted[h.Host] || seen[h.Host] || root == "" || len(prefix) < 3 || !strings.HasPrefix(prefix, "rr") || strings.Trim(prefix[2:], "0123456789") != "" { + return fmt.Errorf("invalid exact GGC sample") + } + seen[h.Host], roots[strings.Split(root, "-")[0]], prefixes[prefix] = true, true, true + if math.IsNaN(h.WarmupMS) || math.IsInf(h.WarmupMS, 0) || h.WarmupMS < 0 || h.WarmupMS > 120000 || h.WarmupAttempts < 0 || h.WarmupFailures < 0 || h.WarmupFailures > h.WarmupAttempts || h.WarmupLimit < 0 || h.WarmupLimit > 10 || h.WarmupAttempts > h.WarmupLimit { + return fmt.Errorf("unbounded warmup") + } + if p.Mode == QualificationCold && (h.WarmupAttempts != 0 || h.WarmupFailures != 0 || h.WarmupLimit != 0 || h.WarmupMS != 0) { + return fmt.Errorf("cold evidence cannot hide warmup") + } + if p.Mode == QualificationAdaptive && (h.WarmupLimit == 0 || h.WarmupAttempts < 2 || h.WarmupFailures > h.WarmupAttempts-2 || h.WarmupMS <= 0) { + return fmt.Errorf("adaptive requires bounded warmup evidence") + } + counts := map[string]int{} + for _, pass := range h.Passes { + if (pass.Protocol != "tls12" && pass.Protocol != "tls13") || !pass.VerifiedTLS || pass.Status != 206 || pass.RangeStart != 0 || pass.RangeEnd < 1023 || pass.ResourceBytes <= pass.RangeEnd || pass.Bytes != pass.RangeEnd+1 || !pass.FTYP || math.IsNaN(pass.LatencyMS) || math.IsInf(pass.LatencyMS, 0) || pass.LatencyMS <= 0 { + return fmt.Errorf("verified ranged media transfer required") + } + counts[pass.Protocol]++ + } + if counts["tls12"] != 3 || counts["tls13"] != 3 { + return fmt.Errorf("three actual steady transfers per TLS protocol required") + } + } + if len(roots) < 2 || len(prefixes) < 2 { + return fmt.Errorf("diverse GGC root groups and host prefixes required") + } + if p.Mode == QualificationAdaptive { + return p.Attestation.validateStored(c) + } + return nil +} + +func validateGooglevideoSuffixEvidence(db *sql.DB, c storedCandidate) error { + return validateGooglevideoSuffixEvidencePolicy(db, c, true) +} + +func validateGooglevideoSuffixEvidencePolicy(db *sql.DB, c storedCandidate, fresh bool) error { + if c.Group != "googlevideo" || !c.Qualified { + return fmt.Errorf("qualified googlevideo candidate required") + } + proof, err := readGooglevideoProof(db, c.ID) + if err != nil { + return err + } + if fresh { + return proof.validate(c) + } + return proof.validateStored(c) +} + +func readGooglevideoProof(db *sql.DB, id int64) (GooglevideoQualificationProof, error) { + var proof GooglevideoQualificationProof + var raw string + if err := db.QueryRow("SELECT proof FROM googlevideo_scope_evidence WHERE candidate_id = ?", id).Scan(&raw); err != nil { + return proof, err + } + return decodeGooglevideoProof(raw) +} + +func decodeGooglevideoProof(raw string) (GooglevideoQualificationProof, error) { + var proof GooglevideoQualificationProof + d := json.NewDecoder(strings.NewReader(raw)) + d.DisallowUnknownFields() + if err := d.Decode(&proof); err != nil { + return proof, err + } + if err := d.Decode(new(interface{})); err != io.EOF { + return proof, fmt.Errorf("unexpected proof data") + } + return proof, nil +} + +func validateStoredAdaptiveGooglevideoEvidence(db *sql.DB, c storedCandidate) (bool, error) { + return validateAdaptiveGooglevideoEvidence(db, c, true) +} + +func validateAdaptiveGooglevideoEvidence(db *sql.DB, c storedCandidate, fresh bool) (bool, error) { + if c.Group != "googlevideo" || c.Mode != QualificationAdaptive { + return false, nil + } + var raw string + if err := db.QueryRow("SELECT proof FROM googlevideo_scope_evidence WHERE candidate_id = ?", c.ID).Scan(&raw); err != nil { + if err == sql.ErrNoRows { + return false, nil + } + return false, err + } + // Malformed/stale proof is unqualified, not a reason to accept cold metrics. + proof, err := decodeGooglevideoProof(raw) + if err != nil { + return false, nil + } + if !fresh { + return proof.validateStored(c) == nil, nil + } + return proof.validate(c) == nil, nil +} + +// One transactional insert, no cold candidate_evidence summaries. A private +// producer seal is required as well as valid DTO facts and stored definition. +func saveAdaptiveGooglevideoEvidence(tx *sql.Tx, id int64, proof GooglevideoQualificationProof) error { + if proof.producerSeal == nil || *proof.producerSeal != googlevideoProofSeal(proof) { + return fmt.Errorf("completed owned collector attestation required") + } + var c storedCandidate + var scope string + if err := tx.QueryRow("SELECT strategy_group, args, fingerprint, endpoints, qualification_mode FROM strategy_candidates WHERE id = ?", id).Scan(&c.Group, &c.Args, &c.Fingerprint, &scope, &c.Mode); err != nil { + return err + } + if err := json.Unmarshal([]byte(scope), &c.Endpoints); err != nil { + return err + } + if c.Group != "googlevideo" || c.Mode != QualificationAdaptive { + return fmt.Errorf("adaptive definition required") + } + if err := proof.validate(c); err != nil { + return err + } + var coldRows int + if err := tx.QueryRow("SELECT count(*) FROM candidate_evidence WHERE candidate_id = ?", id).Scan(&coldRows); err != nil { + return err + } + if coldRows != 0 { + return fmt.Errorf("adaptive evidence cannot contain cold rows") + } + raw, err := json.Marshal(proof) + if err != nil { + return err + } + _, err = tx.Exec("INSERT INTO googlevideo_scope_evidence (candidate_id, proof) VALUES (?, ?)", id, string(raw)) + return err +} diff --git a/cmd/zapret-autotune/candidates_test.go b/cmd/zapret-autotune/candidates_test.go index a14698c..97ac079 100644 --- a/cmd/zapret-autotune/candidates_test.go +++ b/cmd/zapret-autotune/candidates_test.go @@ -58,7 +58,7 @@ func populatedV2Store(t *testing.T) *sql.DB { if err := migrate(db); err != nil { t.Fatal(err) } - if _, err := db.Exec(`DROP TABLE monitor_state; DROP TABLE candidate_evidence; DROP TABLE strategy_candidates; PRAGMA user_version = 2;`); err != nil { + if _, err := db.Exec(`DROP TABLE monitor_state; DROP TABLE googlevideo_scope_evidence; DROP TABLE candidate_evidence; DROP TABLE strategy_candidates; PRAGMA user_version = 2;`); err != nil { t.Fatal(err) } return db @@ -99,7 +99,7 @@ func TestPopulatedV2CandidateMigration(t *testing.T) { } } var version int - if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 7 { t.Fatalf("version=%d err=%v", version, err) } if before != legacySnapshot(t, db) { diff --git a/cmd/zapret-autotune/config.go b/cmd/zapret-autotune/config.go index 054d17f..5312cda 100644 --- a/cmd/zapret-autotune/config.go +++ b/cmd/zapret-autotune/config.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "net" "net/url" @@ -28,12 +29,18 @@ func parseHealthConfig(lookup func(string) (string, bool)) (healthConfig, error) SOCKSProxy: value("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5://127.0.0.1:11080"), VideoID: value("ZAPRET_AUTOTUNE_VIDEO_ID", "dQw4w9WgXcQ"), } + if _, err := managedServiceEnabled(lookup); err != nil { + return healthConfig{}, err + } if !validLoopbackSOCKS(cfg.SOCKSProxy) { return healthConfig{}, fmt.Errorf("ZAPRET_AUTOTUNE_SOCKS_PROXY requires socks5://LOOPBACK:PORT without credentials") } if !videoIDPattern.MatchString(cfg.VideoID) { return healthConfig{}, fmt.Errorf("ZAPRET_AUTOTUNE_VIDEO_ID requires an 11-character video ID") } + if _, _, err := managedGooglevideoConfig(lookup, cfg.VideoID); err != nil { + return healthConfig{}, err + } for _, setting := range []struct { key string fallback string @@ -56,6 +63,46 @@ func parseHealthConfig(lookup func(string) (string, bool)) (healthConfig, error) return cfg, nil } +func managedGooglevideoConfig(lookup func(string) (string, bool), primary string) (QualificationMode, []string, error) { + mode := QualificationCold + if raw, ok := lookup("ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE"); ok { + mode = QualificationMode(raw) + } + if mode != QualificationCold && mode != QualificationAdaptive { + return "", nil, fmt.Errorf("ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE requires cold or adaptive") + } + service, _ := managedServiceEnabled(lookup) + if mode == QualificationAdaptive && !service { + return "", nil, fmt.Errorf("adaptive googlevideo requires ZAPRET_AUTOTUNE_YOUTUBE_SERVICE=1") + } + // Only the configured primary is sampled by default; no implicit public playlist. + ids := []string{primary} + if raw, ok := lookup("ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS"); ok { + if json.Unmarshal([]byte(raw), &ids) != nil || len(ids) == 0 || len(ids) > 5 { + return "", nil, fmt.Errorf("GOOGLEVIDEO_VIDEO_IDS requires a JSON array of 1..5 video IDs") + } + seen := map[string]bool{} + for _, id := range ids { + if !videoIDPattern.MatchString(id) || seen[id] { + return "", nil, fmt.Errorf("GOOGLEVIDEO_VIDEO_IDS requires unique 11-character video IDs") + } + seen[id] = true + } + } + return mode, ids, nil +} + +func managedServiceEnabled(lookup func(string) (string, bool)) (bool, error) { + value, exists := lookup("ZAPRET_AUTOTUNE_YOUTUBE_SERVICE") + if !exists { + return false, nil + } + if value != "0" && value != "1" { + return false, fmt.Errorf("ZAPRET_AUTOTUNE_YOUTUBE_SERVICE requires 0 or 1") + } + return value == "1", nil +} + func validLoopbackSOCKS(raw string) bool { if !validDiscoveryProxy(raw) { return false diff --git a/cmd/zapret-autotune/fingerprint_test.go b/cmd/zapret-autotune/fingerprint_test.go index 403fc8c..e7efa96 100644 --- a/cmd/zapret-autotune/fingerprint_test.go +++ b/cmd/zapret-autotune/fingerprint_test.go @@ -117,7 +117,7 @@ func TestPopulatedV1FingerprintMigration(t *testing.T) { } assertRowCounts(t, db, 3, 4) var version int - if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 7 { t.Fatalf("version=%d err=%v", version, err) } for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} { diff --git a/cmd/zapret-autotune/googlevideo_scope_test.go b/cmd/zapret-autotune/googlevideo_scope_test.go new file mode 100644 index 0000000..842ca62 --- /dev/null +++ b/cmd/zapret-autotune/googlevideo_scope_test.go @@ -0,0 +1,312 @@ +package main + +import ( + "bytes" + "database/sql" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" +) + +func scopeFixture(t *testing.T, db *sql.DB) (int64, GooglevideoQualificationProof) { + t.Helper() + hosts := []string{"rr1---sn-alpha.googlevideo.com", "rr2---sn-beta.googlevideo.com", "rr5---sn-gamma.googlevideo.com"} + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + id, err := insertStrategyCandidate(tx, "googlevideo", hosts, "--lua-desync=fake", nil) + if err != nil { + t.Fatal(err) + } + p := GooglevideoQualificationProof{Version: 1, Mode: QualificationCold, SourceKind: "media-range", Args: "--lua-desync=fake"} + for _, host := range hosts { + h := GooglevideoHostProof{Host: host} + for _, protocol := range []string{"tls12", "tls13"} { + if _, err := tx.Exec(`INSERT INTO candidate_evidence VALUES (?, ?, ?, 3, 3, 1, 1, 1, 1, 'fixture', '{}')`, id, host, protocol); err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + h.Passes = append(h.Passes, GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: 206, RangeEnd: 1023, ResourceBytes: 4096, Bytes: 1024, FTYP: true, LatencyMS: 10}) + } + } + p.Hosts = append(p.Hosts, h) + } + raw, err := json.Marshal(p) + if err != nil { + t.Fatal(err) + } + if _, err = tx.Exec("INSERT INTO googlevideo_scope_evidence VALUES (?, ?)", id, string(raw)); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + return id, p +} + +func TestGooglevideoScopeTransactional(t *testing.T) { + db := testStore(t) + id, _ := scopeFixture(t, db) + assets := []int64{unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true), unitAssetCandidate(t, db, "yt3.ggpht.com", "--lua-desync=fake", true)} + ids := append([]int64{id}, assets...) + exact, err := renderManagedConfig(db, ids) + if err != nil { + t.Fatal(err) + } + explicit, err := renderManagedConfigScope(db, ids, GooglevideoScopeExact) + if err != nil || !bytes.Equal(exact, explicit) { + t.Fatalf("legacy bytes changed: %v", err) + } + suffix, err := renderManagedConfigScope(db, ids, GooglevideoScopeSuffix) + if err != nil || !strings.Contains(string(suffix), "--hostlist-domains=googlevideo.com ") { + t.Fatalf("suffix=%s %v", suffix, err) + } + if _, err := renderManagedConfigScope(db, []int64{id}, GooglevideoScopeSuffix); err == nil { + t.Fatal("missing assets accepted") + } + m, _ := managedFixture(t) + if err := m.Apply(db, ids, GooglevideoScopeSuffix); err != nil { + t.Fatal(err) + } + j, err := m.load() + if err != nil || j.Version != 3 || j.Scope != GooglevideoScopeSuffix || !reflect.DeepEqual(j.Candidates, ids) { + t.Fatalf("journal=%+v %v", j, err) + } + if err := m.Apply(db, ids); err != nil { + t.Fatal(err) + } // omitted policy preserves suffix + assertManagedContent(t, m, string(suffix)) + m.Health = func() (string, error) { return "FAIL", nil } + if err := m.Apply(db, []int64{id}, GooglevideoScopeExact); err == nil { + t.Fatal("health rejection accepted") + } + j, err = m.load() + if err != nil || j.Scope != GooglevideoScopeSuffix || !reflect.DeepEqual(j.Candidates, ids) || j.PendingScope != "" { + t.Fatalf("rollback=%+v %v", j, err) + } + assertManagedContent(t, m, string(suffix)) + m.Health = func() (string, error) { return "OK", nil } + m.fault = func(s string) error { + if s == "replaced" { + return errors.New("death") + } + return nil + } + if err := m.Apply(db, []int64{id}, GooglevideoScopeExact); err == nil { + t.Fatal("fault absent") + } + j, err = m.load() + if err != nil || j.Scope != GooglevideoScopeSuffix || j.PendingScope != GooglevideoScopeExact { + t.Fatalf("pending=%+v %v", j, err) + } + m.fault = nil + if err := m.Reconcile(db); err != nil { + t.Fatal(err) + } + j, err = m.load() + if err != nil || j.Scope != GooglevideoScopeSuffix || !reflect.DeepEqual(j.Candidates, ids) { + t.Fatalf("reconcile=%+v %v", j, err) + } + assertManagedContent(t, m, string(suffix)) + j.Scope = "future" + if err := m.journal(j); err != nil { + t.Fatal(err) + } + if _, err := m.load(); err == nil { + t.Fatal("unknown scope accepted") + } +} + +func TestGooglevideoProofGate(t *testing.T) { + db := testStore(t) + id, base := scopeFixture(t, db) + c, err := readStrategyCandidate(db, id) + if err != nil { + t.Fatal(err) + } + if err := base.validate(c); err != nil { + t.Fatal(err) + } + for _, name := range []string{"samples", "warmup", "404", "tls", "range", "ftyp", "steady", "diversity", "args", "version"} { + t.Run(name, func(t *testing.T) { + raw, _ := json.Marshal(base) + var p GooglevideoQualificationProof + if err := json.Unmarshal(raw, &p); err != nil { + t.Fatal(err) + } + switch name { + case "samples": + p.Hosts = p.Hosts[:2] + case "warmup": + p.Mode = QualificationAdaptive + p.SourceKind = NativeCircularPostWarmup + c.Mode = QualificationAdaptive + p.Hosts[0].WarmupAttempts = 11 + p.Hosts[0].WarmupLimit = 10 + case "404": + p.Hosts[0].Passes[0].Status = 404 + case "tls": + p.Hosts[0].Passes[0].VerifiedTLS = false + case "range": + p.Hosts[0].Passes[0].ResourceBytes = 0 + case "ftyp": + p.Hosts[0].Passes[0].FTYP = false + case "steady": + p.Hosts[0].Passes = p.Hosts[0].Passes[:5] + case "diversity": + for i := range p.Hosts { + p.Hosts[i].Host = "rr1---sn-alpha.googlevideo.com" + } + case "args": + p.Args = "--lua-desync=multisplit" + case "version": + p.Version = 2 + } + if err := p.validate(c); err == nil { + t.Fatal("invalid proof accepted") + } + c.Mode = QualificationCold + }) + } + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + adaptive, err := insertStrategyCandidate(tx, "googlevideo", c.Endpoints, c.Args, nil, QualificationAdaptive) + if err != nil { + t.Fatal(err) + } + for _, host := range c.Endpoints { + for _, protocol := range []string{"tls12", "tls13"} { + if _, err := tx.Exec(`INSERT INTO candidate_evidence VALUES (?, ?, ?, 3, 3, 1, 1, 1, 1, 'fixture', '{}')`, adaptive, host, protocol); err != nil { + t.Fatal(err) + } + } + } + if err := saveAdaptiveGooglevideoEvidence(tx, adaptive, base); err == nil { + t.Fatal("unwired collector accepted") + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + a, err := readStrategyCandidate(db, adaptive) + if err != nil || a.Qualified || a.Mode != QualificationAdaptive { + t.Fatalf("cold metrics qualified adaptive: %+v %v", a, err) + } + qualified, err := listQualifiedCandidates(db, "googlevideo", 1) + if err != nil || len(qualified) != 1 || qualified[0].ID != id { + t.Fatalf("limit filtered incorrectly: %+v %v", qualified, err) + } + if _, err := renderManagedConfig(db, []int64{adaptive}); err == nil { + t.Fatal("adaptive applied without collector attestation") + } +} + +func TestGooglevideoV6MigrationPreservesBytes(t *testing.T) { + db := testStore(t) + id := managedCandidateFixture(t, db, "googlevideo", "rr5---sn-alpha.googlevideo.com", "--lua-desync=fake", true) + before, err := renderManagedConfig(db, []int64{id}) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("DROP TABLE googlevideo_scope_evidence; ALTER TABLE strategy_candidates DROP COLUMN qualification_mode; PRAGMA user_version=6;"); err != nil { + t.Fatal(err) + } + legacy, err := renderManagedConfig(db, []int64{id}) + if err != nil || !bytes.Equal(before, legacy) { + t.Fatalf("v6=%s %v", legacy, err) + } + if err := migrate(db); err != nil { + t.Fatal(err) + } + after, err := renderManagedConfig(db, []int64{id}) + if err != nil || !bytes.Equal(before, after) || managedHash(before) != managedHash(after) { + t.Fatalf("migration changed legacy: %v", err) + } + if _, err := db.Exec("UPDATE strategy_candidates SET qualification_mode='adaptive' WHERE id=?", id); err == nil { + t.Fatal("immutable definition trigger lost") + } +} + +func TestGooglevideoScopeCLIArguments(t *testing.T) { + for _, scope := range []string{"exact", "suffix"} { + if _, ids, err := managedArguments([]string{"apply", "--googlevideo-candidate", "1", "--googlevideo-scope", scope}); err != nil || len(ids) != 1 { + t.Fatalf("%s %v", scope, err) + } + } + for _, args := range [][]string{{"apply", "--googlevideo-scope", "suffix"}, {"apply", "--googlevideo-candidate", "1", "--googlevideo-scope", "future"}, {"apply", "--googlevideo-candidate", "1", "--googlevideo-scope", "suffix", "--googlevideo-scope", "exact"}} { + if _, _, err := managedArguments(args); err == nil { + t.Fatalf("accepted %v", args) + } + } +} + +func TestGooglevideoScopeRejectsNonMediaAndSecrets(t *testing.T) { + db := testStore(t) + id, proof := scopeFixture(t, db) + ids := []int64{id, unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true), unitAssetCandidate(t, db, "yt3.ggpht.com", "--lua-desync=fake", true)} + proof.Hosts[0].Passes[0].Status = 404 + raw, err := json.Marshal(proof) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("UPDATE googlevideo_scope_evidence SET proof=? WHERE candidate_id=?", string(raw), id); err != nil { + t.Fatal(err) + } + if _, err := renderManagedConfigScope(db, ids, GooglevideoScopeSuffix); err == nil { + t.Fatal("root 404 passed media gate") + } + proof.Hosts[0].Passes[0].Status = 206 + raw, err = json.Marshal(proof) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + fields["signed_url"] = json.RawMessage(`"https://secret.invalid/videoplayback?token=private"`) + raw, err = json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("UPDATE googlevideo_scope_evidence SET proof=? WHERE candidate_id=?", string(raw), id); err != nil { + t.Fatal(err) + } + if _, err := renderManagedConfigScope(db, ids, GooglevideoScopeSuffix); err == nil { + t.Fatal("secret-bearing evidence accepted") + } + if _, err := renderManagedConfig(db, ids); err != nil { + t.Fatalf("legacy exact evidence changed: %v", err) + } +} + +func TestGooglevideoV6MigrationAtomicFailure(t *testing.T) { + db := testStore(t) + id := unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true) + before, err := renderManagedConfig(db, []int64{id}) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("DROP TABLE googlevideo_scope_evidence; ALTER TABLE strategy_candidates DROP COLUMN qualification_mode; CREATE TABLE googlevideo_scope_evidence (conflict INTEGER); PRAGMA user_version=6;"); err != nil { + t.Fatal(err) + } + if err := migrate(db); err == nil { + t.Fatal("conflicting migration accepted") + } + var version, columns int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 6 { + t.Fatalf("version=%d %v", version, err) + } + if err := db.QueryRow("SELECT count(*) FROM pragma_table_info('strategy_candidates') WHERE name='qualification_mode'").Scan(&columns); err != nil || columns != 0 { + t.Fatalf("partial column migration: %d %v", columns, err) + } + after, err := renderManagedConfig(db, []int64{id}) + if err != nil || !bytes.Equal(before, after) { + t.Fatalf("assets changed on failed migration: %v", err) + } +} diff --git a/cmd/zapret-autotune/googlevideo_test.go b/cmd/zapret-autotune/googlevideo_test.go index ff709dc..0f5b1dd 100644 --- a/cmd/zapret-autotune/googlevideo_test.go +++ b/cmd/zapret-autotune/googlevideo_test.go @@ -154,7 +154,7 @@ fi if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil { t.Fatal(err) } - if evidence != 2 || legacy != 0 || version != 5 { + if evidence != 2 || legacy != 0 || version != 7 { t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version) } for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} { diff --git a/cmd/zapret-autotune/health_test.go b/cmd/zapret-autotune/health_test.go index 0cebd4e..1fd6f5d 100644 --- a/cmd/zapret-autotune/health_test.go +++ b/cmd/zapret-autotune/health_test.go @@ -21,8 +21,8 @@ func configLookup(values map[string]string) func(string) (string, bool) { func TestHealthConfigDefaultsAndOverrides(t *testing.T) { cfg, err := parseHealthConfig(configLookup(nil)) - want := healthConfig{"socks5://127.0.0.1:11080", "dQw4w9WgXcQ", 5 * time.Minute, 3, 30 * time.Minute} - if err != nil || cfg != want { + want := healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080", VideoID: "dQw4w9WgXcQ", HealthInterval: 5 * time.Minute, FailureThreshold: 3, RecoveryCooldown: 30 * time.Minute} + if err != nil || !reflect.DeepEqual(cfg, want) { t.Fatalf("defaults = %+v, %v; want %+v", cfg, err, want) } values := map[string]string{ @@ -33,8 +33,8 @@ func TestHealthConfigDefaultsAndOverrides(t *testing.T) { "ZAPRET_AUTOTUNE_RECOVERY_COOLDOWN": "1h", } cfg, err = parseHealthConfig(configLookup(values)) - want = healthConfig{"socks5://[::1]:12080", "abcdefghijk", 90 * time.Second, 5, time.Hour} - if err != nil || cfg != want { + want = healthConfig{SOCKSProxy: "socks5://[::1]:12080", VideoID: "abcdefghijk", HealthInterval: 90 * time.Second, FailureThreshold: 5, RecoveryCooldown: time.Hour} + if err != nil || !reflect.DeepEqual(cfg, want) { t.Fatalf("overrides = %+v, %v; want %+v", cfg, err, want) } for _, proxy := range []string{"socks5://localhost:1", "socks5://LOCALHOST:65535", "socks5://127.0.0.2:11080", "socks5://[::1]:11080"} { diff --git a/cmd/zapret-autotune/main.go b/cmd/zapret-autotune/main.go index fdaf509..d53ec3d 100644 --- a/cmd/zapret-autotune/main.go +++ b/cmd/zapret-autotune/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "io" + "net" "os" ) @@ -35,6 +36,17 @@ func run(args []string, stdout, stderr io.Writer) int { } func runWithPlayer(args []string, stdout, stderr io.Writer, player playerRunner) int { + if len(args) == 1 && args[0] == "target-config-check" { + if _, err := parseHealthConfig(os.LookupEnv); err != nil { + fmt.Fprintln(stderr, "configerror: invalid target health/service configuration") + return 64 + } + if dns := os.Getenv("SS_DNS_SERVER"); dns != "" && net.ParseIP(dns) == nil { + fmt.Fprintln(stderr, "configerror: SS_DNS_SERVER requires a literal IPv4 or IPv6 address") + return 64 + } + return 0 + } if len(args) > 0 && args[0] == "managed" { return managedCLI(args[1:], stdout, stderr, managedDependencies(player)) } diff --git a/cmd/zapret-autotune/managed_asset_runtime.go b/cmd/zapret-autotune/managed_asset_runtime.go new file mode 100644 index 0000000..621c247 --- /dev/null +++ b/cmd/zapret-autotune/managed_asset_runtime.go @@ -0,0 +1,354 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strconv" + "strings" + "syscall" + "time" +) + +const managedAssetQueue = 200 // Serialized recovery never runs blockcheck concurrently. +const managedAssetTable = "zapret_autotune_asset_probe" + +type managedAssetWorker struct { + pid int + done <-chan error + stop func() error +} +type managedAssetRuntime struct { + guard func() error + nft func(context.Context, string) (string, error) + start func([]string) (managedAssetWorker, error) + queue func(int) (bool, error) + curl subprocessRunner + proxy string +} + +type managedAssetCleanupFailure struct{ Err error } + +func (e *managedAssetCleanupFailure) Error() string { return "owned asset runtime cleanup failed" } +func (e *managedAssetCleanupFailure) Unwrap() error { return e.Err } + +func newManagedAssetVerifier(target managedDeps) func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error) { + return newManagedAssetRuntime(target).verify +} + +func newManagedAssetRuntime(target managedDeps) managedAssetRuntime { + proxy, _ := target.lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY") + r := managedAssetRuntime{proxy: proxy, curl: runResourceCurl} + r.guard = func() error { + if _, err := managedRuntimeGuard(target); err != nil { + return err + } + status, err := os.ReadFile("/proc/self/status") + if err != nil { + return err + } + for _, line := range strings.Split(string(status), "\n") { + if strings.HasPrefix(line, "CapEff:") { + bits, err := strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(line, "CapEff:")), 16, 64) + if err == nil && bits&(1<<5) != 0 && bits&(1<<12) != 0 { + return nil + } + } + } + return errors.New("asset verifier requires CAP_KILL and CAP_NET_ADMIN") + } + r.nft = func(ctx context.Context, input string) (string, error) { + cmd := exec.CommandContext(ctx, "nft", "-f", "-") + cmd.Stdin = strings.NewReader(input) + var out strings.Builder + cmd.Stdout, cmd.Stderr = &out, io.Discard + err := cmd.Run() + return out.String(), err + } + r.start = func(args []string) (managedAssetWorker, error) { + // No daemon or Setpgid: retain monitor session ownership. + cmd := exec.Command("/opt/zapret2/nfq2/nfqws2", args...) + cmd.Stdout, cmd.Stderr = io.Discard, io.Discard + if err := cmd.Start(); err != nil { + return managedAssetWorker{}, err + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + return managedAssetWorker{pid: cmd.Process.Pid, done: done, stop: func() error { + err := cmd.Process.Kill() + if errors.Is(err, os.ErrProcessDone) { + return errors.New("asset worker exited before owned stop") + } + return err + }}, nil + } + r.queue = managedAssetQueueReady + return r +} + +// Queue peer portid is a netlink socket id, not necessarily a process id. +func managedAssetQueueReady(pid int) (bool, error) { + queues, err := os.ReadFile("/proc/net/netfilter/nfnetlink_queue") + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + peer := "" + for _, line := range strings.Split(string(queues), "\n") { + f := strings.Fields(line) + if len(f) >= 2 && f[0] == strconv.Itoa(managedAssetQueue) { + peer = f[1] + } + } + if peer == "" { + return false, nil + } + if pid == 0 { + return true, nil + } + if err := syscall.Kill(pid, 0); err != nil { + return false, err + } + netlink, err := os.ReadFile("/proc/net/netlink") + if err != nil { + return false, err + } + inodes := map[string]bool{} + for _, line := range strings.Split(string(netlink), "\n") { + f := strings.Fields(line) + if len(f) >= 10 && f[1] == "12" && f[2] == peer { + inodes["socket:["+f[9]+"]"] = true + } + } + fds, err := os.ReadDir(fmt.Sprintf("/proc/%d/fd", pid)) + if err != nil { + return false, err + } + for _, fd := range fds { + link, err := os.Readlink(fmt.Sprintf("/proc/%d/fd/%s", pid, fd.Name())) + if err == nil && inodes[link] { + return true, nil + } + } + return false, errors.New("asset queue belongs to another process") +} + +const managedAssetRules = `create table inet zapret_autotune_asset_probe +add chain inet zapret_autotune_asset_probe output { type filter hook output priority -151; policy accept; } +add chain inet zapret_autotune_asset_probe input { type filter hook input priority -151; policy accept; } +add rule inet zapret_autotune_asset_probe output meta mark & 0x10000000 == 0 tcp dport 443 queue num 200 +add rule inet zapret_autotune_asset_probe input meta mark & 0x10000000 == 0 tcp sport 443 queue num 200 +` + +func (r managedAssetRuntime) verify(ctx context.Context, c storedCandidate, inventory managedServiceInventory) (measurements []assetMeasurement, err error) { + if c.Group != "assets" || len(c.Endpoints) == 0 || len(c.Endpoints) > 2 || strings.TrimSpace(c.Args) == "" { + return nil, &managedRecoveryFatal{Err: errors.New("invalid asset candidate")} + } + if e := (&candidate{Args: c.Args}).validateArgs(); e != nil { + return nil, &managedRecoveryFatal{Err: e} + } + for _, arg := range strings.Fields(c.Args) { + if strings.HasPrefix(arg, "--lua-init=") && arg != "--lua-init=@/opt/zapret2/lua/zapret-lib.lua" && arg != "--lua-init=@/opt/zapret2/lua/zapret-antidpi.lua" { + return nil, &managedRecoveryFatal{Err: errors.New("asset verifier only permits fixed Lua init files")} + } + } + seen := map[string]bool{} + for _, host := range c.Endpoints { + if !assetEndpoint(host) || seen[host] { + return nil, &managedRecoveryFatal{Err: errors.New("invalid asset endpoints")} + } + seen[host] = true + if _, e := privateResourceURL(inventory.images[host], host); e != nil { + return nil, e + } + } + if e := ctx.Err(); e != nil { + return nil, e + } + err = r.withOwnedWorker(ctx, c.Endpoints, c.Args, func(probeCtx context.Context) error { + for _, host := range c.Endpoints { + for _, protocol := range []string{"tls12", "tls13"} { + m, e := collectAssetMeasurement(probeCtx, r.proxy, inventory.images[host], host, protocol, c.Args, r.curl) + if e != nil { + if ctx.Err() != nil { + return ctx.Err() + } + if managedAssetTransferNonpass(e) { + return errCandidateNonpass + } + return e + } + measurements = append(measurements, m) + } + } + return nil + }) + if err != nil { + return nil, err + } + return measurements, nil +} + +// Shared, serialized session lifecycle: only an atomically created table is +// owned, and success is returned only after queue removal and worker join. +func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []string, strategyArgs string, capture func(context.Context) error) (err error) { + if err := ctx.Err(); err != nil { + return err + } + if e := r.guard(); e != nil { + return &managedRecoveryFatal{Err: e} + } + tables, e := r.nft(ctx, "list tables\n") + if e != nil { + return &managedRecoveryFatal{Err: e} + } + if strings.Contains(tables, "table inet "+managedAssetTable) { + return &managedRecoveryFatal{Err: errors.New("asset nft table already exists")} + } + occupied, e := r.queue(0) + if e != nil || occupied { + return &managedRecoveryFatal{Err: errors.Join(e, errors.New("asset queue unavailable"))} + } + args := managedProbeWorkerArgs(endpoints, strategyArgs) + worker, e := r.start(args) + if e != nil { + return &managedRecoveryFatal{Err: e} + } + joined, table := false, false + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + var cleanupErr error + // Remove queues before killing the consumer. + if table { + _, cleanupErr = r.nft(cleanup, "delete table inet "+managedAssetTable+"\n") + } + if !joined { + select { + case e := <-worker.done: + joined = true + cleanupErr = errors.Join(cleanupErr, e, errors.New("asset worker exited unexpectedly")) + default: + } + } + if !joined { + cleanupErr = errors.Join(cleanupErr, worker.stop()) + select { + case e := <-worker.done: + joined = true + if e != nil { + var exit *exec.ExitError + if !errors.As(e, &exit) || !exit.ProcessState.Sys().(syscall.WaitStatus).Signaled() || exit.ProcessState.Sys().(syscall.WaitStatus).Signal() != syscall.SIGKILL { + cleanupErr = errors.Join(cleanupErr, e) + } + } + case <-cleanup.Done(): + cleanupErr = errors.Join(cleanupErr, errors.New("asset worker join timeout")) + } + } + if cleanupErr != nil { + err = &managedRecoveryFatal{Err: errors.Join(err, &managedAssetCleanupFailure{Err: cleanupErr})} + } + }() + readyCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + tick := time.NewTicker(25 * time.Millisecond) + defer tick.Stop() + for { + select { + case e := <-worker.done: + joined = true + return &managedRecoveryFatal{Err: errors.Join(e, errors.New("asset worker exited before READY"))} + default: + } + ready, e := r.queue(worker.pid) + if e != nil { + return &managedRecoveryFatal{Err: e} + } + if ready { + break + } + select { + case <-readyCtx.Done(): + if ctx.Err() != nil { + return ctx.Err() + } + return &managedRecoveryFatal{Err: errors.New("asset queue READY timeout")} + case <-tick.C: + } + } + // nft's transaction is atomic; failed creation owns no state to delete. + if _, e := r.nft(ctx, managedAssetRules); e != nil { + return &managedRecoveryFatal{Err: e} + } + table = true + // Recheck ownership after installing the queue and before the first capture. + if ready, e := r.queue(worker.pid); e != nil || !ready { + return &managedRecoveryFatal{Err: errors.Join(e, errors.New("owned queue lost before capture"))} + } + probeCtx, stopProbe := context.WithCancel(ctx) + watchDone := make(chan struct{}) + exited := make(chan error, 1) + go func() { + defer close(watchDone) + select { + case e := <-worker.done: + exited <- errors.Join(e, errors.New("asset worker exited during probe")) + stopProbe() + case <-probeCtx.Done(): + } + }() + defer func() { + stopProbe() + <-watchDone + select { + case e := <-exited: + joined = true + err = &managedRecoveryFatal{Err: errors.Join(err, e)} + default: + } + }() + return capture(probeCtx) +} + +func managedProbeWorkerArgs(endpoints []string, strategyArgs string) []string { + args := []string{"--qnum=200", "--fwmark=0x10000000", "--lua-init=@/opt/zapret2/lua/zapret-lib.lua", "--lua-init=@/opt/zapret2/lua/zapret-antidpi.lua", "--filter-tcp=443", "--hostlist-domains=" + strings.Join(endpoints, ",")} + return append(args, strings.Fields(strategyArgs)...) +} + +// Collector errors conservatively carry a fatal wrapper. Only known transport +// failures and genuine rejected responses are nonpasses; malformed evidence, +// missing executables, unknown errors and worker failures remain infrastructure. +func managedAssetTransferNonpass(err error) bool { + if err == nil { + return false + } + switch e := err.(type) { + case *exec.ExitError: + switch e.ExitCode() { + case 6, 7, 28, 35, 51, 52, 55, 56, 60: + return true + } + return false + case interface{ Unwrap() []error }: + children := e.Unwrap() + if len(children) == 0 { + return false + } + for _, child := range children { + if !managedAssetTransferNonpass(child) { + return false + } + } + return true + case interface{ Unwrap() error }: + return managedAssetTransferNonpass(e.Unwrap()) + default: + return err == context.DeadlineExceeded || err.Error() == "resource origin verification failed" || err.Error() == "image HTTP status" + } +} diff --git a/cmd/zapret-autotune/managed_asset_runtime_test.go b/cmd/zapret-autotune/managed_asset_runtime_test.go new file mode 100644 index 0000000..6148c8e --- /dev/null +++ b/cmd/zapret-autotune/managed_asset_runtime_test.go @@ -0,0 +1,354 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func assetRuntimeFixture(t *testing.T) (managedAssetRuntime, *[]string, chan error) { + t.Helper() + events := []string{} + done := make(chan error, 1) + r := managedAssetRuntime{proxy: "socks5://127.0.0.1:1080", guard: func() error { return nil }} + r.nft = func(_ context.Context, input string) (string, error) { + if input == "list tables\n" { + events = append(events, "inventory") + return "", nil + } + if strings.HasPrefix(input, "delete") { + events = append(events, "delete") + return "", nil + } + if len(events) == 0 || events[len(events)-1] != "ready" { + t.Fatal("queue activated before READY", events) + } + if !strings.Contains(input, "hook input") || !strings.Contains(input, "hook output") || !strings.Contains(input, "meta mark & 0x10000000 == 0") { + t.Fatal(input) + } + events = append(events, "activate") + return "", nil + } + r.queue = func(pid int) (bool, error) { + if pid == 0 { + return false, nil + } + events = append(events, "ready") + return true, nil + } + r.start = func(args []string) (managedAssetWorker, error) { + if strings.Contains(strings.Join(args, " "), "--daemon") { + t.Fatal(args) + } + events = append(events, "start") + return managedAssetWorker{pid: 123, done: done, stop: func() error { events = append(events, "join"); done <- nil; return nil }}, nil + } + r.curl = func(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + events = append(events, "curl") + config, err := io.ReadAll(stdin) + if err != nil { + return err + } + raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(config), "url = "), "\n")) + if err != nil { + return err + } + var diag bytes.Buffer + resourceWire(raw, "", out, &diag) + tls := diag.String() + if strings.Contains(strings.Join(args, " "), "--tlsv1.3") { + tls = strings.ReplaceAll(tls, "TLSv1.2", "TLSv1.3") + } + _, err = io.WriteString(diagnostic, tls) + return err + } + return r, &events, done +} + +func TestManagedAssetRuntimeOwnedCollector(t *testing.T) { + d, front, _ := recoveryFixture(t, true) + if err := d.transaction.Apply(d.db, []int64{front}); err != nil { + t.Fatal(err) + } + before := map[string][]byte{} + entries, err := os.ReadDir(d.transaction.Dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + data, err := os.ReadFile(filepath.Join(d.transaction.Dir, entry.Name())) + if err != nil { + t.Fatal(err) + } + before[entry.Name()] = data + } + r, events, _ := assetRuntimeFixture(t) + c := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com", "yt3.ggpht.com"}, Args: managedAssetSeeds[0]} + m, err := r.verify(context.Background(), c, managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}}) + if err != nil || len(m) != 4 { + t.Fatal(m, err) + } + count := 0 + for _, e := range *events { + if e == "curl" { + count++ + } + } + if count != 12 || strings.Join((*events)[len(*events)-2:], ",") != "delete,join" { + t.Fatal(*events) + } + for _, measurement := range m { + if measurement.Args != c.Args { + t.Fatal("unbound strategy") + } + if _, _, err := validateAssetTransfers(measurement); err != nil { + t.Fatal(err) + } + } + for name, data := range before { + after, err := os.ReadFile(filepath.Join(d.transaction.Dir, name)) + if err != nil || !bytes.Equal(data, after) { + t.Fatal("probe mutated managed artifact", name, err) + } + } + assertActiveCandidates(t, d.transaction, d.db, []int64{front}) +} + +func TestManagedAssetRuntimeFailedAtomicNFTInstallNeverDeletesForeignTable(t *testing.T) { + for _, mode := range []string{"preexisting", "raced-create", "install-failure"} { + t.Run(mode, func(t *testing.T) { + r, events, _ := assetRuntimeFixture(t) + r.nft = func(_ context.Context, input string) (string, error) { + if input == "list tables\n" { + if mode == "preexisting" { + return "table inet " + managedAssetTable, nil + } + return "", nil + } + if strings.HasPrefix(input, "delete") { + t.Fatal("deleted foreign table after failed atomic install") + } + if !strings.HasPrefix(input, "create table inet "+managedAssetTable+"\n") { + t.Fatal("table creation must reject racing existing table", input) + } + return "", errors.New(mode) + } + c := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com"}, Args: managedAssetSeeds[0]} + m, err := r.verify(context.Background(), c, managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}}) + var fatal *managedRecoveryFatal + if len(m) != 0 || !errors.As(err, &fatal) || errors.Is(err, errCandidateNonpass) { + t.Fatal(m, err) + } + if mode == "preexisting" { + if len(*events) != 0 { + t.Fatal("started worker over preexisting table", *events) + } + } else if strings.Join(*events, ",") != "start,ready,join" { + t.Fatal("failed install did not join owned worker", *events) + } + }) + } +} + +func TestManagedAssetRuntimeCancelFailureAndCleanup(t *testing.T) { + for _, mode := range []string{"cancel-ready", "cancel-curl", "worker", "nonpass", "cleanup"} { + t.Run(mode, func(t *testing.T) { + r, events, done := assetRuntimeFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if mode == "cancel-ready" { + r.queue = func(pid int) (bool, error) { + if pid != 0 { + cancel() + } + return false, nil + } + } + if mode == "cancel-curl" { + r.curl = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { + cancel() + return context.Canceled + } + } + if mode == "worker" { + r.curl = func(ctx context.Context, _ []string, _ io.Reader, _, _ io.Writer) error { + done <- errors.New("worker failure") + <-ctx.Done() + return ctx.Err() + } + } + if mode == "nonpass" { + r.curl = func(_ context.Context, _ []string, _ io.Reader, out, diagnostic io.Writer) error { + resourceWire("https://i.ytimg.com/image", "cn", out, diagnostic) + return nil + } + } + if mode == "cleanup" { + nft := r.nft + r.nft = func(ctx context.Context, input string) (string, error) { + out, err := nft(ctx, input) + if strings.HasPrefix(input, "delete") { + err = errors.New("cleanup failed") + } + return out, err + } + } + c := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com"}, Args: managedAssetSeeds[0]} + m, err := r.verify(ctx, c, managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}}) + if err == nil || len(m) != 0 { + t.Fatal(m, err) + } + var fatal *managedRecoveryFatal + if (mode == "worker" || mode == "cleanup") && !errors.As(err, &fatal) { + t.Fatal("missing fatal", err) + } + if mode == "nonpass" && !errors.Is(err, errCandidateNonpass) { + t.Fatal(err) + } + if strings.HasPrefix(mode, "cancel") && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if mode == "cancel-ready" && (*events)[len(*events)-1] != "join" { + t.Fatal(*events) + } + if mode != "cancel-ready" && !strings.Contains(strings.Join(*events, ","), "delete") { + t.Fatal(*events) + } + }) + } +} + +func TestManagedAssetSeedsFallbackAndNoQualification(t *testing.T) { + for _, pass := range []bool{true, false} { + t.Run(strconv.FormatBool(pass), func(t *testing.T) { + d, front, _ := recoveryFixture(t, true) + avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true) + ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true) + if err := d.transaction.Apply(d.db, []int64{front, avatar, ggc}); err != nil { + t.Fatal(err) + } + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}}, nil + } + calls := 0 + d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) { + calls++ + if !pass || calls == 1 { + return nil, errCandidateNonpass + } + return []assetMeasurement{unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args)}, nil + } + err := repairManagedAssets(context.Background(), d, "i.ytimg.com") + if !pass { + if !errors.Is(err, errNoQualifiedStrategy) || calls != len(managedAssetSeeds) { + t.Fatal(calls, err) + } + assertActiveCandidates(t, d.transaction, d.db, []int64{front, avatar, ggc}) + } else { + if err != nil || calls != 2 { + t.Fatal(calls, err) + } + ids, err := d.transaction.ActiveCandidates(d.db) + if err != nil || len(ids) != 4 { + t.Fatal(ids, err) + } + for _, required := range []int64{front, avatar, ggc} { + found := false + for _, id := range ids { + found = found || id == required + } + if !found { + t.Fatal(ids) + } + } + } + }) + } +} + +func TestManagedAssetPartialOverlapAndStageHealth(t *testing.T) { + d, front, _ := recoveryFixture(t, true) + old := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com", "yt3.ggpht.com"}, Args: "--lua-desync=fake"} + id, err := saveManagedAssetCandidate(d.db, old, []assetMeasurement{ + unitImageMeasurement(t, "i.ytimg.com", "tls12", old.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", old.Args), + unitImageMeasurement(t, "yt3.ggpht.com", "tls12", old.Args), unitImageMeasurement(t, "yt3.ggpht.com", "tls13", old.Args), + }) + if err != nil { + t.Fatal(err) + } + unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=multisplit", true) + if err := d.transaction.Apply(d.db, []int64{front, id}); err != nil { + t.Fatal(err) + } + r, _, _ := assetRuntimeFixture(t) + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}, proxy: r.proxy, runner: r.curl, resourceErr: &managedResourceFailure{"media", "unknown", nil}}, nil + } + d.frontHealth = func(context.Context) (string, error) { return "OK", nil } + d.health = func(context.Context) (string, error) { + t.Fatal("unrelated media gated asset stage") + return "DEGRADED", nil + } + d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) { + if c.ID != 0 { + return nil, errCandidateNonpass + } + if len(c.Endpoints) != 2 { + t.Fatal("partial overlap lost sibling", c) + } + return []assetMeasurement{ + unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args), + unitImageMeasurement(t, "yt3.ggpht.com", "tls12", c.Args), unitImageMeasurement(t, "yt3.ggpht.com", "tls13", c.Args), + }, nil + } + if err := repairManagedAssets(context.Background(), d, "i.ytimg.com"); err != nil { + t.Fatal(err) + } + ids, err := d.transaction.ActiveCandidates(d.db) + if err != nil || len(ids) != 2 { + t.Fatal(ids, err) + } + replacement, err := readStrategyCandidate(d.db, ids[1]) + if err != nil || len(replacement.Endpoints) != 2 { + t.Fatal(replacement, err) + } +} + +func TestManagedAssetMalformedAndUnsafeCleanupNeverRestore(t *testing.T) { + for _, unsafe := range []bool{false, true} { + d, front, events := recoveryFixture(t, true) + if err := d.transaction.Apply(d.db, []int64{front}); err != nil { + t.Fatal(err) + } + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}}, nil + } + calls := 0 + d.verifyAssets = func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error) { + calls++ + err := errors.New("malformed collector metrics") + if unsafe { + return nil, &managedRecoveryFatal{Err: &managedAssetCleanupFailure{Err: err}} + } + return nil, &managedRecoveryFatal{Err: err} + } + err := repairManagedAssets(context.Background(), d, "i.ytimg.com") + var fatal *managedRecoveryFatal + if !errors.As(err, &fatal) || errors.Is(err, errNoQualifiedStrategy) || calls != 1 { + t.Fatal(calls, err) + } + if unsafe && (*events)[len(*events)-1] != "stop" { + t.Fatal("unsafe restoration", *events) + } + assertActiveCandidates(t, d.transaction, d.db, []int64{front}) + } +} diff --git a/cmd/zapret-autotune/managed_asset_verifier.go b/cmd/zapret-autotune/managed_asset_verifier.go new file mode 100644 index 0000000..50d3072 --- /dev/null +++ b/cmd/zapret-autotune/managed_asset_verifier.go @@ -0,0 +1,468 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "time" +) + +var errManagedAssetVerifierUnwired = errors.New("owned temporary asset verifier unavailable") + +// Dependency-only construction fails closed. The production monitor adapter +// replaces this callback with newManagedAssetVerifier, never a root/404 probe. +func unwiredManagedAssetVerifier(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error) { + return nil, errManagedAssetVerifierUnwired +} + +type managedResourceFailure struct { + ProbeKind, Host string + Err error +} + +func (e *managedResourceFailure) Error() string { return e.PublicError() } +func (e *managedResourceFailure) PublicError() string { + host := e.Host + if !discoveryHostname(host) { + host = "unknown" + } + kind := e.ProbeKind + if kind != "thumbnail" && kind != "avatar" && kind != "media" && kind != "front" { + kind = "unknown" + } + return "DEGRADED: " + kind + " " + host +} +func (e *managedResourceFailure) Unwrap() error { return e.Err } + +// Ephemeral inventory: never serialize URLs, page bodies or Player responses. +type managedServiceInventory struct { + images map[string]string + media mediaResource + resourceErr error + proxy string + runner subprocessRunner +} + +func managedServiceResources(cfg healthConfig, player func(context.Context) ([]byte, error), page playerRunner) func(context.Context) (managedServiceInventory, error) { + return func(ctx context.Context) (managedServiceInventory, error) { + i := managedServiceInventory{images: make(map[string]string), proxy: cfg.SOCKSProxy, runner: runResourceCurl} + body, err := player(ctx) + if err != nil { + return i, &managedResourceFailure{"front", managedFrontendHost, err} + } + var p struct { + VideoDetails struct { + Thumbnail struct { + Thumbnails []struct { + URL string `json:"url"` + } `json:"thumbnails"` + } `json:"thumbnail"` + } `json:"videoDetails"` + } + if json.Unmarshal(body, &p) == nil { + for _, t := range p.VideoDetails.Thumbnail.Thumbnails { + if _, e := privateResourceURL(t.URL, "i.ytimg.com"); e == nil { + i.images["i.ytimg.com"] = t.URL + break + } + } + } + i.media, _, err = playerMediaResource(body) + if err != nil { + i.resourceErr = &managedResourceFailure{"media", "unknown", err} + } + bounded, cancel := context.WithTimeout(ctx, 35*time.Second) + defer cancel() + var output bytes.Buffer + out := &playerBoundedWriter{dst: &output, limit: 5 << 20} + // Fixed root origin; no redirects, credentials, user URL or diagnostics. + err = page(bounded, []string{"--disable", "--silent", "--show-error", "--fail", "--noproxy", "", "--proxy", cfg.SOCKSProxy, "--proto", "=https", "--max-time", "30", "--max-filesize", "5242880", "https://www.youtube.com/"}, out, io.Discard) + if ctx.Err() != nil { + return i, managedRecoveryContextError(ctx, err) + } + if err != nil || bounded.Err() != nil || out.exceeded { + return i, &managedResourceFailure{"avatar", "yt3.ggpht.com", resourceFailure(bounded, errors.Join(err, bounded.Err(), errors.New("fresh root metadata request failed")))} + } + if raw, e := avatarResourceFromPage(output.Bytes()); e == nil { + i.images["yt3.ggpht.com"] = raw + } + return i, nil + } +} + +func managedServiceHealth(proxy string, front func(context.Context) (string, error), inventory func(context.Context) (managedServiceInventory, error), runner subprocessRunner) func(context.Context) (string, error) { + return func(ctx context.Context) (string, error) { + status, err := front(ctx) + if err != nil || status != "OK" { + return "DEGRADED", &managedResourceFailure{"front", managedFrontendHost, err} + } + i, err := inventory(ctx) + var fatal *managedRecoveryFatal + if ctx.Err() != nil || errors.As(err, &fatal) { + return "DEGRADED", err + } + missing := err + for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} { + kind := "thumbnail" + if host == "yt3.ggpht.com" { + kind = "avatar" + } + if i.images[host] == "" { + if missing == nil { + missing = &managedResourceFailure{kind, host, nil} + } + continue + } + for _, protocol := range []string{"tls12", "tls13"} { + if _, err := collectAssetMeasurement(ctx, proxy, i.images[host], host, protocol, "", runner); err != nil { + // At the live-health boundary, known bad image bodies/origins + // are resource damage. Preserve mixed guard/cancellation failures. + if ctx.Err() == nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) && managedAssetHealthNonpass(err) { + err = nil + } + return "DEGRADED", &managedResourceFailure{kind, host, err} + } + } + } + if missing != nil { + return "DEGRADED", missing + } + if i.resourceErr != nil { + return "DEGRADED", i.resourceErr + } + if i.media.url == "" { + return "DEGRADED", &managedResourceFailure{"media", "unknown", nil} + } // SABR unknown, not ready. + for _, protocol := range []string{"tls12", "tls13"} { + if _, err := collectMediaSamples(ctx, proxy, i.media, protocol, runner); err != nil { + return "DEGRADED", &managedResourceFailure{"media", i.media.host, err} + } + } + return "OK", nil + } +} + +func managedFullDesiredSet(db *sql.DB, current, replacements []int64) ([]int64, error) { + replaceGroups := map[string]bool{} + for _, id := range replacements { + c, err := readStrategyCandidate(db, id) + if err != nil { + return nil, err + } + replaceGroups[c.Group] = true + } + var desired []int64 + for _, id := range current { + c, err := readStrategyCandidate(db, id) + if err != nil { + return nil, err + } + if !replaceGroups[c.Group] { + desired = append(desired, id) + } + } + return append(desired, replacements...), nil +} + +func repairManagedAssets(ctx context.Context, d managedRecoveryDeps, host string) (err error) { + if d.resources == nil || d.verifyAssets == nil { + return errManagedAssetVerifierUnwired + } + i, err := d.resources(ctx) + var fatal *managedRecoveryFatal + if ctx.Err() != nil || errors.As(err, &fatal) { + return managedRecoveryContextError(ctx, err) + } + if err != nil && i.images[host] == "" { + return err + } + if i.images[host] == "" { + return &managedResourceFailure{managedImageProbeKind(host), host, nil} + } + m := *d.transaction + current, err := m.ActiveCandidates(d.db) + if err != nil { + return &managedRecoveryFatal{Err: err} + } + saved, err := listQualifiedCandidates(d.db, "assets", 10) + if err != nil { + return &managedRecoveryFatal{Err: err} + } + // Saved definitions first, then a bounded readonly empirical seed inventory. + // Include overlapping siblings in seeds; never discard a partial host set. + endpoints := []string{host} + for _, id := range current { + old, e := readStrategyCandidate(d.db, id) + if e != nil { + return &managedRecoveryFatal{Err: e} + } + if old.Group == "assets" { + for _, a := range old.Endpoints { + if a == host { + endpoints = append([]string(nil), old.Endpoints...) + } + } + } + } + for _, args := range managedAssetSeeds { + saved = append(saved, storedCandidate{Group: "assets", Endpoints: endpoints, Args: args}) + } + for _, c := range saved { + covers := false + for _, endpoint := range c.Endpoints { + covers = covers || endpoint == host + } + if !covers { + continue + } + var desired []int64 + for _, id := range current { + old, e := readStrategyCandidate(d.db, id) + if e != nil { + return &managedRecoveryFatal{Err: e} + } + overlap := false + for _, a := range old.Endpoints { + for _, b := range c.Endpoints { + overlap = overlap || a == b + } + } + if old.Group != "assets" || !overlap { + desired = append(desired, id) + } + // A partially overlapping multi-host candidate cannot lose its sibling. + if old.Group == "assets" && overlap { + for _, a := range old.Endpoints { + found := false + for _, b := range c.Endpoints { + found = found || a == b + } + if !found { + covers = false + } + } + } + } + if !covers { + continue + } + if e := (&candidate{Args: c.Args}).validateArgs(); e != nil { + return &managedRecoveryFatal{Err: e} + } + // All temporary workers must be joined by Verify before restoration/apply. + e := d.run(ctx, managedZapretScript, "stop") + var measurements []assetMeasurement + if e == nil { + measurements, e = d.verifyAssets(ctx, c, i) + } + var unsafeCleanup *managedAssetCleanupFailure + if errors.As(e, &unsafeCleanup) { + // Never reactivate the permanent service over an unjoined worker or + // residual owned queue. The fatal monitor exit retains last-good files. + return e + } + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + restoreErr := d.run(cleanup, managedZapretScript, "start") + cancel() + if restoreErr != nil { + return &managedRecoveryFatal{Err: errors.Join(e, restoreErr)} + } + if e != nil { + if errors.As(e, &fatal) { + return e + } + if ctx.Err() != nil || errors.Is(e, context.Canceled) { + return managedRecoveryContextError(ctx, e) + } + if errors.Is(e, errCandidateNonpass) { + continue + } + return e + } + if e = ctx.Err(); e != nil { + return e + } + id, e := saveManagedAssetCandidate(d.db, c, measurements) + if e != nil { + return &managedRecoveryFatal{Err: e} + } + m.Health = func() (string, error) { + // Asset-stage apply must not be vetoed by unrelated media damage. + // Global service health still gates READY after recovery. + var status string + var healthErr error + if d.frontHealth == nil { + status, healthErr = d.health(ctx) + } else { + status, healthErr = managedAssetApplyHealth(ctx, d, c.Endpoints) + } + var failure *managedResourceFailure + var fatal *managedRecoveryFatal + if ctx.Err() != nil { + return "DEGRADED", managedRecoveryContextError(ctx, healthErr) + } + // Apply treats a non-OK status with no error as a rejection. Never + // erase a safety failure or cancellation joined to ordinary damage. + if !errors.As(healthErr, &fatal) && !errors.Is(healthErr, context.Canceled) && !errors.Is(healthErr, context.DeadlineExceeded) && errors.As(healthErr, &failure) { + return "DEGRADED", nil + } + return status, healthErr + } + m.Reload = func() error { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return d.run(cleanup, managedZapretScript, "restart") + } + e = m.Apply(d.db, append(desired, id)) + if e == nil { + return nil + } + var rejection *managedHealthRejection + var rollback *managedRollbackError + if errors.As(e, &fatal) || errors.As(e, &rollback) { + return &managedRecoveryFatal{Err: e} + } + if ctx.Err() != nil || errors.Is(e, context.Canceled) || errors.Is(e, context.DeadlineExceeded) { + return managedRecoveryContextError(ctx, e) + } + if !errors.As(e, &rejection) { + return &managedRecoveryFatal{Err: e} + } + } + return errNoQualifiedStrategy +} + +var managedAssetSeeds = [...]string{ + "--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld", + "--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld", + "--payload=tls_client_hello --lua-desync=multidisorder:pos=1,midsld", + "--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:tcp_md5:repeats=1 --lua-desync=multidisorder:pos=1,midsld", + "--payload=tls_client_hello --lua-desync=multisplit:pos=1,midsld", +} + +func managedAssetApplyHealth(ctx context.Context, d managedRecoveryDeps, endpoints []string) (string, error) { + status, err := d.frontHealth(ctx) + if err != nil || status != "OK" { + return "DEGRADED", err + } + i, err := d.resources(ctx) + var fatal *managedRecoveryFatal + if ctx.Err() != nil || errors.As(err, &fatal) { + return "DEGRADED", managedRecoveryContextError(ctx, err) + } + if err != nil { + for _, host := range endpoints { + if i.images[host] == "" { + return "DEGRADED", err + } + } + } + // Production's normal service collector uses the same guarded SOCKS proxy. + // An adapter installs this callback with a candidate-independent collector. + if i.resourceErr != nil && len(i.images) == 0 { + return "DEGRADED", i.resourceErr + } + return managedAssetInventoryHealth(ctx, i, endpoints) +} + +func managedAssetInventoryHealth(ctx context.Context, i managedServiceInventory, endpoints []string) (string, error) { + if i.runner == nil || i.proxy == "" { + return "DEGRADED", errors.New("asset health collector unavailable") + } + for _, host := range endpoints { + if i.images[host] == "" { + return "DEGRADED", &managedResourceFailure{managedImageProbeKind(host), host, nil} + } + for _, protocol := range []string{"tls12", "tls13"} { + if _, err := collectAssetMeasurement(ctx, i.proxy, i.images[host], host, protocol, "", i.runner); err != nil { + if ctx.Err() != nil { + return "DEGRADED", managedRecoveryContextError(ctx, err) + } + if managedAssetHealthNonpass(err) { + return "DEGRADED", nil + } + return "DEGRADED", &managedResourceFailure{managedImageProbeKind(host), host, err} + } + } + } + return "OK", nil +} + +// Collector wrappers are conservatively fatal. At the post-activation health +// boundary, rejected origins/bodies are ordinary damage, not infrastructure. +// Every joined leaf must be a known nonpass; mixed safety failures stay fatal. +func managedAssetHealthNonpass(err error) bool { + switch e := err.(type) { + case interface{ Unwrap() []error }: + children := e.Unwrap() + if len(children) == 0 { + return false + } + for _, child := range children { + if !managedAssetHealthNonpass(child) { + return false + } + } + return true + case interface{ Unwrap() error }: + return managedAssetHealthNonpass(e.Unwrap()) + default: + if err == nil || err == context.Canceled { + return false + } + return managedAssetTransferNonpass(err) || err.Error() == "image body exceeds limit" || err.Error() == "invalid image dimensions" || err.Error() == "invalid image body" + } +} + +func managedImageProbeKind(host string) string { + if host == "i.ytimg.com" { + return "thumbnail" + } + if host == "yt3.ggpht.com" { + return "avatar" + } + return "unknown" +} + +func saveManagedAssetCandidate(db *sql.DB, c storedCandidate, measurements []assetMeasurement) (int64, error) { + if len(measurements) != 2*len(c.Endpoints) { + return 0, fmt.Errorf("incomplete asset verification matrix") + } + seen := map[string]bool{} + for _, m := range measurements { + member := false + for _, host := range c.Endpoints { + member = member || host == m.Endpoint + } + key := m.Endpoint + "/" + m.Protocol + if !member || seen[key] || m.Args != c.Args { + return 0, fmt.Errorf("asset verification matrix definition mismatch") + } + seen[key] = true + if _, _, err := validateAssetTransfers(m); err != nil { + return 0, err + } + } + tx, err := db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + id, err := insertStrategyCandidate(tx, "assets", c.Endpoints, c.Args, nil) + if err != nil { + return 0, err + } + for _, m := range measurements { + if err := saveAssetEvidence(tx, id, m, time.Now()); err != nil { + return 0, err + } + } + if err := tx.Commit(); err != nil { + return 0, err + } + return id, nil +} diff --git a/cmd/zapret-autotune/managed_cli.go b/cmd/zapret-autotune/managed_cli.go index 5cb2545..bd935d9 100644 --- a/cmd/zapret-autotune/managed_cli.go +++ b/cmd/zapret-autotune/managed_cli.go @@ -6,6 +6,7 @@ import ( "database/sql" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" @@ -55,32 +56,55 @@ func managedDependencies(player playerRunner) managedDeps { } func managedArguments(args []string) (string, []int64, error) { - invalid := fmt.Errorf("managed accepts init, reconcile, status, active-ids, or apply with one or both candidate flags (full desired set); no path flags") + invalid := fmt.Errorf("managed accepts init, reconcile, status [--assets], active-ids, or apply with frontend/googlevideo/asset candidate flags (full desired set); no path flags") + if len(args) == 2 && args[0] == "status" && args[1] == "--assets" { + return "status", nil, nil + } if len(args) == 1 && (args[0] == "init" || args[0] == "reconcile" || args[0] == "status" || args[0] == "active-ids") { return args[0], nil, nil } - if (len(args) != 3 && len(args) != 5) || args[0] != "apply" { + if len(args) < 3 || len(args)%2 != 1 || args[0] != "apply" { return "", nil, invalid } - selected := map[string]int64{} + selected := map[string][]int64{} + seen := map[int64]bool{} + scopeSeen := false for i := 1; i < len(args); i += 2 { key := args[i] - if selected[key] != 0 || (key != "--frontend-candidate" && key != "--googlevideo-candidate") || args[i+1] == "" || strings.Trim(args[i+1], "0123456789") != "" { + if key == "--googlevideo-scope" { + if scopeSeen || (args[i+1] != "exact" && args[i+1] != "suffix") { + return "", nil, invalid + } + scopeSeen = true + continue + } + if (len(selected[key]) != 0 && key != "--asset-candidate") || (key != "--frontend-candidate" && key != "--googlevideo-candidate" && key != "--asset-candidate") { return "", nil, invalid } - id, err := strconv.ParseInt(args[i+1], 10, 64) - if err != nil || id <= 0 { + values := strings.Split(args[i+1], ",") + if key != "--asset-candidate" && len(values) != 1 { + return "", nil, invalid + } + for _, value := range values { + if value == "" || strings.Trim(value, "0123456789") != "" { + return "", nil, invalid + } + id, err := strconv.ParseInt(value, 10, 64) + if err != nil || id <= 0 || seen[id] { + return "", nil, invalid + } + seen[id] = true + selected[key] = append(selected[key], id) + } + if len(selected["--asset-candidate"]) > 2 { return "", nil, invalid } - selected[key] = id } var ids []int64 - for _, key := range []string{"--frontend-candidate", "--googlevideo-candidate"} { - if id := selected[key]; id != 0 { - ids = append(ids, id) - } + for _, key := range []string{"--frontend-candidate", "--googlevideo-candidate", "--asset-candidate"} { + ids = append(ids, selected[key]...) } - if len(ids) == 2 && ids[0] == ids[1] { + if len(ids) == 0 || (scopeSeen && len(selected["--googlevideo-candidate"]) == 0) { return "", nil, invalid } return "apply", ids, nil @@ -196,7 +220,7 @@ func managedCheckIdentity(m *managedTransaction) error { } // Status and supervisor reads share the same locked, DB-validated snapshot. -func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB, error), activeOnly bool) error { +func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB, error), activeOnly bool, requestAssets ...bool) error { return m.locked(func() error { j, err := m.load() if err != nil { @@ -213,23 +237,29 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB, return errManagedDrift } groups := map[string]bool{} - known := j.State == "committed" && j.Version == 2 && string(b) == managedBaseline && len(j.Candidates) == 0 - if j.State == "committed" && j.Version == 2 && len(j.Candidates) != 0 { + assetHosts := map[string]bool{} + known := j.State == "committed" && j.Version >= 2 && string(b) == managedBaseline && len(j.Candidates) == 0 + if j.State == "committed" && j.Version >= 2 && len(j.Candidates) != 0 { db, err := openDB() if db != nil { defer db.Close() } if err == nil && db != nil { - rendered, err := renderManagedConfig(db, j.Candidates) + rendered, err := renderStoredManagedConfigScope(db, j.Candidates, j.Scope) if err == nil && managedHash(rendered) == j.Hash { known = true for _, id := range j.Candidates { - c, err := readStrategyCandidate(db, id) + c, err := readStrategyCandidatePolicy(db, id, false) if err != nil || !c.Qualified { known = false break } groups[c.Group] = true + if c.Group == "assets" { + for _, host := range c.Endpoints { + assetHosts[host] = true + } + } } } } @@ -247,6 +277,12 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB, missing = append(missing, group) } } + if groups["assets"] { + active = append(active, "assets") + } + if (groups["assets"] || (len(requestAssets) != 0 && requestAssets[0])) && (!known || len(assetHosts) != 2) { + missing = append(missing, "assets") + } if activeOnly { if !known { return fmt.Errorf("active candidate IDs unresolved; deployment preserved by hash") @@ -268,6 +304,9 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB, reportIDs = j.Candidates } _, err = fmt.Fprintf(out, "managed-test generation=%d hash=%s state=%s groups=%s active_candidate_ids=%v unresolved_groups=%s\nLive health: NOT CHECKED (Player API is not GGC/video playback)\n", j.Generation, j.Hash, j.State, groupText, reportIDs, strings.Join(missing, ",")) + if err == nil && j.Version == 3 { + _, err = fmt.Fprintf(out, "Googlevideo scope: %s (suffix is authorized extrapolation, not all-host qualification)\n", j.Scope) + } return err }) } @@ -329,18 +368,32 @@ func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int { if command == "reconcile" { err = managedPrestartReconcile(m, d) } else if command == "status" || command == "active-ids" { - err = managedReport(m, stdout, d.openDB, command == "active-ids") + err = managedReport(m, stdout, d.openDB, command == "active-ids", len(args) == 2 && args[1] == "--assets") } else { expected := map[int64]string{} + var scopes []GooglevideoScope for i := 1; i < len(args); i += 2 { - id, _ := strconv.ParseInt(args[i+1], 10, 64) - expected[id] = strings.TrimSuffix(strings.TrimPrefix(args[i], "--"), "-candidate") + if args[i] == "--googlevideo-scope" { + scopes = append(scopes, GooglevideoScope(args[i+1])) + continue + } + group := strings.TrimSuffix(strings.TrimPrefix(args[i], "--"), "-candidate") + if group == "asset" { + group = "assets" + } + for _, value := range strings.Split(args[i+1], ",") { + id, _ := strconv.ParseInt(value, 10, 64) + expected[id] = group + } } - err = managedActivate(m, d, ids, expected) + err = managedActivate(m, d, ids, expected, scopes...) } } if err != nil { fmt.Fprintln(stderr, "error:", err) + if errors.Is(err, errManagedScopeUnresolved) { + return 78 + } return 1 } if command == "init" || command == "reconcile" || command == "apply" { @@ -380,7 +433,7 @@ func managedPrestartReconcile(m *managedTransaction, d managedDeps) (err error) return err } baselineHash := managedHash([]byte(managedBaseline)) - if j.Version != 2 || (len(j.Candidates) == 0 && j.Hash != baselineHash) || (j.State == "prepared" && len(j.PendingCandidates) == 0 && j.Next != baselineHash) { + if j.Version < 2 || (len(j.Candidates) == 0 && j.Hash != baselineHash) || (j.State == "prepared" && len(j.PendingCandidates) == 0 && j.Next != baselineHash) { return fmt.Errorf("startup candidate state unresolved; refusing activation") } return nil @@ -395,7 +448,9 @@ func managedPrestartReconcile(m *managedTransaction, d managedDeps) (err error) return err } -func managedActivate(m *managedTransaction, d managedDeps, ids []int64, expected map[int64]string) error { +var errManagedScopeUnresolved = errors.New("managed deployment policy unresolved; explicit reconcile or fresh managed state required") + +func managedActivate(m *managedTransaction, d managedDeps, ids []int64, expected map[int64]string, scopes ...GooglevideoScope) error { info, err := os.Stat(d.script) if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0111 == 0 { return fmt.Errorf("target zapret daemon init executable unavailable") @@ -430,6 +485,9 @@ func managedActivate(m *managedTransaction, d managedDeps, ids []int64, expected return fmt.Errorf("candidate database missing") } defer db.Close() + if _, err := m.ActiveCandidates(db); err != nil { + return fmt.Errorf("%w: %v", errManagedScopeUnresolved, err) + } for _, id := range ids { group := expected[id] c, err := readStrategyCandidate(db, id) @@ -437,11 +495,8 @@ func managedActivate(m *managedTransaction, d managedDeps, ids []int64, expected return fmt.Errorf("invalid %s candidate ID", group) } } - if _, err := renderManagedConfig(db, ids); err != nil { - return err - } if err := m.Reconcile(db); err != nil { return err } - return m.Apply(db, ids) + return m.Apply(db, ids, scopes...) } diff --git a/cmd/zapret-autotune/managed_config.go b/cmd/zapret-autotune/managed_config.go index 82a6b74..afd6a00 100644 --- a/cmd/zapret-autotune/managed_config.go +++ b/cmd/zapret-autotune/managed_config.go @@ -15,25 +15,64 @@ const managedFirewallConfig = "INIT_APPLY_FW=1\n" + // renderManagedConfig accepts identifiers, not caller-supplied strategies or scope. // Qualification is the existing complete endpoint × TLS evidence rule. func renderManagedConfig(db *sql.DB, ids []int64) ([]byte, error) { + return renderManagedConfigScope(db, ids, GooglevideoScopeExact) +} + +type GooglevideoScope string + +const ( + GooglevideoScopeExact GooglevideoScope = "exact" + GooglevideoScopeSuffix GooglevideoScope = "suffix" +) + +func (s GooglevideoScope) valid() bool { + return s == "" || s == GooglevideoScopeExact || s == GooglevideoScopeSuffix +} + +func renderManagedConfigScope(db *sql.DB, ids []int64, scope GooglevideoScope) ([]byte, error) { + return renderManagedConfigPolicy(db, ids, scope, true) +} + +// Only journal verification may reconstruct bytes without fresh qualification. +func renderStoredManagedConfigScope(db *sql.DB, ids []int64, scope GooglevideoScope) ([]byte, error) { + return renderManagedConfigPolicy(db, ids, scope, false) +} + +func renderManagedConfigPolicy(db *sql.DB, ids []int64, scope GooglevideoScope, fresh bool) ([]byte, error) { + if !scope.valid() { + return nil, fmt.Errorf("unknown googlevideo scope %q", scope) + } if db == nil { return nil, fmt.Errorf("candidate definitions database required") } + var version int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil { + return nil, err + } + if version < 3 || version > 7 { + return nil, fmt.Errorf("unsupported candidate schema version %d", version) + } var profiles []string groups := map[string]bool{} seen := map[int64]bool{} + assets := map[string]bool{} + var otherScopes []string for _, id := range ids { if id <= 0 || seen[id] { return nil, fmt.Errorf("duplicate candidate %d", id) } seen[id] = true - c, err := readStrategyCandidate(db, id) + c, err := readStrategyCandidatePolicy(db, id, fresh) if err != nil { return nil, err } if !c.Qualified { return nil, fmt.Errorf("candidate %d is not qualified", id) } - if groups[c.Group] { + if c.Group == "assets" && version < 6 { + return nil, fmt.Errorf("assets require schema 6") + } + if groups[c.Group] && c.Group != "assets" { return nil, fmt.Errorf("duplicate candidate group %q", c.Group) } v := candidate{Group: c.Group, Domains: append([]string(nil), c.Endpoints...), Args: c.Args} @@ -46,6 +85,16 @@ func renderManagedConfig(db *sql.DB, ids []int64) ([]byte, error) { } domains := append([]string(nil), v.Domains...) for i, host := range domains { + if c.Group != "assets" { + otherScopes = append(otherScopes, host) + } + if c.Group == "assets" { + if assets[host] { + return nil, fmt.Errorf("overlapping asset endpoint") + } + assets[host] = true + domains[i] = "^" + host + } if c.Group == "googlevideo" { if !strings.HasSuffix(host, ".googlevideo.com") { return nil, fmt.Errorf("GGC scope must be an exact googlevideo host") @@ -53,9 +102,26 @@ func renderManagedConfig(db *sql.DB, ids []int64) ([]byte, error) { domains[i] = "^" + host } } + if c.Group == "googlevideo" && scope == GooglevideoScopeSuffix { + if version < 7 || validateGooglevideoSuffixEvidencePolicy(db, c, fresh) != nil { + return nil, fmt.Errorf("suffix requires representative qualified hosts and verified media evidence") + } + // Authorized extrapolation, not evidence that all suffix hosts passed. + domains = []string{"googlevideo.com"} + } groups[c.Group] = true profiles = append(profiles, "--filter-tcp=443 --filter-l7=tls --hostlist-domains="+strings.Join(domains, ",")+" "+c.Args) } + if scope == GooglevideoScopeSuffix && (!groups["googlevideo"] || len(assets) != 2) { + return nil, fmt.Errorf("suffix requires googlevideo and both qualified asset profiles") + } + for host := range assets { + for _, scope := range otherScopes { + if host == scope || strings.HasSuffix(host, "."+scope) { + return nil, fmt.Errorf("asset endpoint overlaps another profile") + } + } + } if len(profiles) == 0 { return nil, fmt.Errorf("qualified candidate required") } diff --git a/cmd/zapret-autotune/managed_googlevideo_recovery.go b/cmd/zapret-autotune/managed_googlevideo_recovery.go new file mode 100644 index 0000000..f414a5b --- /dev/null +++ b/cmd/zapret-autotune/managed_googlevideo_recovery.go @@ -0,0 +1,208 @@ +package main + +import ( + "context" + "errors" + "sort" + "strconv" + "time" +) + +// Ordinary unresolved state, not a target/ownership guard failure. +type InsufficientAdaptiveSamples struct{} + +func (*InsufficientAdaptiveSamples) Error() string { + return "DEGRADED: insufficient fresh ordinary signed GGC samples" +} +func (*InsufficientAdaptiveSamples) Unwrap() []error { + return []error{errGooglevideoInsufficientSamples, errNoQualifiedStrategy} +} + +func managedGooglevideoSamples(ids []string, request func(context.Context, string) ([]byte, error)) func(context.Context) ([]mediaResource, error) { + return func(ctx context.Context) ([]mediaResource, error) { + byHost := map[string]mediaResource{} + for _, id := range ids { + body, err := request(ctx, id) + var fatal *managedRecoveryFatal + if ctx.Err() != nil || errors.As(err, &fatal) { + return nil, managedRecoveryContextError(ctx, err) + } + if err != nil { + continue + } + r, _, err := playerMediaResource(body) + if err != nil { + continue + } // Missing/SABR is unknown, never fabricated. + u, err := privateResourceURL(r.url, r.host) + if err != nil || u.Path != "/videoplayback" { + continue + } + q := u.Query() + expires, err := strconv.ParseInt(q.Get("expire"), 10, 64) + if err != nil || expires <= time.Now().Unix()+120 || (q.Get("sig") == "" && q.Get("signature") == "" && q.Get("lsig") == "") { + continue + } + byHost[r.host] = r + } + hosts := make([]string, 0, len(byHost)) + for host := range byHost { + hosts = append(hosts, host) + } + sort.Strings(hosts) + if validateGooglevideoHosts(hosts) != nil { + return nil, &InsufficientAdaptiveSamples{} + } + resources := make([]mediaResource, 0, len(hosts)) + for _, host := range hosts { + resources = append(resources, byHost[host]) + } + return resources, nil + } +} + +func managedAdaptiveAssetsQualified(d managedRecoveryDeps, ids []int64) (bool, error) { + covered := map[string]bool{} + for _, id := range ids { + c, err := readStrategyCandidate(d.db, id) + if err != nil { + return false, err + } + if c.Group == "assets" && c.Qualified { + for _, host := range c.Endpoints { + covered[host] = true + } + } + } + return covered["i.ytimg.com"] && covered["yt3.ggpht.com"], nil +} + +func recoverManagedGooglevideo(ctx context.Context, d managedRecoveryDeps) (int64, error) { + if !d.service || d.mediaSamples == nil || d.verifyGooglevideo == nil { + return 0, &InsufficientAdaptiveSamples{} + } + m := *d.transaction + current, err := m.ActiveCandidates(d.db) + if err != nil { + return 0, &managedRecoveryFatal{Err: err} + } + ok, err := managedAdaptiveAssetsQualified(d, current) + if err != nil { + return 0, &managedRecoveryFatal{Err: err} + } + if !ok { + return 0, &managedResourceFailure{"thumbnail", "unknown", nil} + } + status, err := managedAssetApplyHealth(ctx, d, []string{"i.ytimg.com", "yt3.ggpht.com"}) + if err != nil { + return 0, err + } + if status != "OK" { + return 0, &managedHealthRejection{Status: status} + } + saved, err := listQualifiedCandidates(d.db, "googlevideo", 10) + if err != nil { + return 0, &managedRecoveryFatal{Err: err} + } + // Revalidate saved native definitions first. Old proof never authorizes apply. + trials := []storedCandidate{} + for _, c := range saved { + if c.Mode == QualificationAdaptive && validateNativeGooglevideoArgs(c.Args) == nil { + trials = append(trials, c) + } + } + trials = append(trials, storedCandidate{Args: managedGooglevideoArgs}) + for _, c := range trials { + resources, err := d.mediaSamples(ctx) // Fresh on every stage, URLs stay private. + if err != nil { + return 0, err + } + c.ID, c.Group, c.Mode = 0, "googlevideo", QualificationAdaptive + c.Endpoints = nil + for _, r := range resources { + c.Endpoints = append(c.Endpoints, r.host) + } + if validateGooglevideoHosts(c.Endpoints) != nil { + return 0, &InsufficientAdaptiveSamples{} + } + err = d.run(ctx, managedZapretScript, "stop") + var proof GooglevideoQualificationProof + if err == nil { + proof, err = d.verifyGooglevideo(ctx, c, resources) + } + var unsafe *managedAssetCleanupFailure + if errors.As(err, &unsafe) { + return 0, err + } + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + restoreErr := d.run(cleanup, managedZapretScript, "start") + cancel() + if restoreErr != nil { + return 0, &managedRecoveryFatal{Err: errors.Join(err, restoreErr)} + } + var fatal *managedRecoveryFatal + if ctx.Err() != nil || errors.As(err, &fatal) { + return 0, managedRecoveryContextError(ctx, err) + } + if errors.Is(err, errGooglevideoInsufficientSamples) { + return 0, &InsufficientAdaptiveSamples{} + } + if errors.Is(err, errCandidateNonpass) { + continue + } + if err != nil { + return 0, managedRecoveryContextError(ctx, err) + } + if err := ctx.Err(); err != nil { + return 0, err + } + tx, err := d.db.Begin() + if err != nil { + return 0, &managedRecoveryFatal{Err: err} + } + id, err := insertStrategyCandidate(tx, "googlevideo", c.Endpoints, c.Args, nil, QualificationAdaptive) + if err == nil { + err = saveAdaptiveGooglevideoEvidence(tx, id, proof) + } + if err == nil { + err = tx.Commit() + } else { + _ = tx.Rollback() + } + if err != nil { + return 0, &managedRecoveryFatal{Err: err} + } + desired, err := managedFullDesiredSet(d.db, current, []int64{id}) + if err != nil { + return 0, &managedRecoveryFatal{Err: err} + } + m.Reload = func() error { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return d.run(cleanup, managedZapretScript, "restart") + } + m.Health = func() (string, error) { + status, err := d.health(ctx) // Fresh media byte health, not hostname-only. + var failure *managedResourceFailure + var fatal *managedRecoveryFatal + if ctx.Err() != nil { + return "DEGRADED", managedRecoveryContextError(ctx, err) + } + if errors.As(err, &failure) && !errors.As(err, &fatal) && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + return "DEGRADED", nil + } + return status, err + } + err = m.Apply(d.db, desired, GooglevideoScopeSuffix) + var rollback *managedRollbackError + var rejection *managedHealthRejection + if err != nil && !errors.As(err, &rejection) && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + return 0, &managedRecoveryFatal{Err: err} + } + if errors.As(err, &rollback) { + return 0, &managedRecoveryFatal{Err: err} + } + return id, err + } + return 0, errNoQualifiedStrategy +} diff --git a/cmd/zapret-autotune/managed_googlevideo_recovery_test.go b/cmd/zapret-autotune/managed_googlevideo_recovery_test.go new file mode 100644 index 0000000..6959484 --- /dev/null +++ b/cmd/zapret-autotune/managed_googlevideo_recovery_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "image" + "image/png" + "io" + "reflect" + "strconv" + "strings" + "testing" +) + +func TestManagedAdaptiveConfig(t *testing.T) { + for _, tc := range []struct { + mode, service, ids string + valid bool + }{ + {"", "", "", true}, {"adaptive", "1", `["dQw4w9WgXcQ","aqz-KE-bpKQ","jNQXAC9IVRw"]`, true}, + {"adaptive", "0", "", false}, {"unknown", "1", "", false}, + {"adaptive", "1", `[]`, false}, {"adaptive", "1", `["bad"]`, false}, + {"adaptive", "1", `["dQw4w9WgXcQ","dQw4w9WgXcQ"]`, false}, + {"adaptive", "1", `["dQw4w9WgXcQ","aqz-KE-bpKQ","jNQXAC9IVRw","aaaaaaaaaaa","bbbbbbbbbbb","ccccccccccc"]`, false}, + } { + lookup := func(key string) (string, bool) { + v := map[string]string{"ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE": tc.mode, "ZAPRET_AUTOTUNE_YOUTUBE_SERVICE": tc.service, "ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS": tc.ids}[key] + return v, v != "" + } + cfg, err := parseHealthConfig(lookup) + if (err == nil) != tc.valid { + t.Fatalf("%+v: %v", tc, err) + } + mode, ids, _ := managedGooglevideoConfig(lookup, cfg.VideoID) + if tc.mode == "" && (mode != QualificationCold || !reflect.DeepEqual(ids, []string{cfg.VideoID})) { + t.Fatal(cfg) + } + } +} + +func TestManagedAdaptiveRecoveryProductionPath(t *testing.T) { + for _, scenario := range []string{"qualified", "one-host", "unqualified-assets", "rollback"} { + t.Run(scenario, func(t *testing.T) { + d, front, events := recoveryFixture(t, true) + r, _, resources, _ := googlevideoRuntimeFixture(t) + d.service, d.adaptive = true, true + d.frontHealth = d.health + assets := []int64{unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", scenario != "unqualified-assets"), unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)} + old := []int64{front} + if scenario != "unqualified-assets" { + old = append(old, assets...) + } + if err := d.transaction.Apply(d.db, old); err != nil { + t.Fatal(err) + } + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/sample.jpg", "yt3.ggpht.com": "https://yt3.ggpht.com/sample.jpg"}, proxy: r.proxy, runner: adaptiveImageRunner}, nil + } + d.mediaSamples = func(context.Context) ([]mediaResource, error) { + if scenario == "one-host" { + return resources[:1], nil + } + return resources, nil + } + d.verifyGooglevideo = r.verifyGooglevideo + if scenario == "rollback" { + d.health = func(context.Context) (string, error) { + return "DEGRADED", &managedResourceFailure{"media", resources[0].host, nil} + } + } + result, err := recoverManagedTarget(context.Background(), d) + if err != nil { + t.Fatal(err) + } + if scenario != "qualified" { + if result.GGCError == nil { + t.Fatal("unqualified stage succeeded") + } + assertActiveCandidates(t, d.transaction, d.db, old) + b, _ := d.transaction.read("config") + if strings.Contains(string(b), "--hostlist-domains=googlevideo.com") { + t.Fatal("unproven suffix") + } + return + } + if result.GGCError != nil || result.GooglevideoID == 0 { + t.Fatal(result) + } + c, err := readStrategyCandidate(d.db, result.GooglevideoID) + if err != nil || !c.Qualified || c.Mode != QualificationAdaptive { + t.Fatal(c, err) + } + p, err := readGooglevideoProof(d.db, c.ID) + if err != nil || p.SourceKind != NativeCircularPostWarmup { + t.Fatal(p, err) + } + j, err := d.transaction.load() + if err != nil || j.Scope != GooglevideoScopeSuffix { + t.Fatal(j, err) + } + assertActiveCandidates(t, d.transaction, d.db, append(old, c.ID)) + if strings.Join(*events, ",") != "stop,start,restart" { + t.Fatal("restore must precede apply", *events) + } + }) + } +} + +func adaptiveImageRunner(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + b, err := io.ReadAll(stdin) + if err != nil { + return err + } + raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(b), "url = "), "\n")) + if err != nil { + return err + } + var body bytes.Buffer + if err := png.Encode(&body, image.NewRGBA(image.Rect(0, 0, 32, 32))); err != nil { + return err + } + fmt.Fprintf(out, "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n", body.Len()) + _, _ = out.Write(body.Bytes()) + metrics, _ := json.Marshal(map[string]interface{}{"url_effective": raw, "ssl_verify_result": 0, "num_redirects": 0, "http_code": 200, "time_total": 0.1, "speed_download": 10000, "size_download": body.Len()}) + fmt.Fprint(out, resourceTrailer, string(metrics)) + tls := "TLSv1.2" + if strings.Contains(strings.Join(args, " "), "--tlsv1.3") { + tls = "TLSv1.3" + } + fmt.Fprintln(diagnostic, "SSL connection using", tls) + return nil +} + +func TestInsufficientAdaptiveSamplesCooldown(t *testing.T) { + err := &InsufficientAdaptiveSamples{} + if !errors.Is(err, errNoQualifiedStrategy) || !errors.Is(err, errGooglevideoInsufficientSamples) { + t.Fatal(err) + } +} diff --git a/cmd/zapret-autotune/managed_googlevideo_runtime.go b/cmd/zapret-autotune/managed_googlevideo_runtime.go new file mode 100644 index 0000000..663ca91 --- /dev/null +++ b/cmd/zapret-autotune/managed_googlevideo_runtime.go @@ -0,0 +1,216 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "strconv" + "strings" + "time" +) + +// ponytail: one audited composite only. New native variants need an explicit +// semantic allowlist and regressions, not the generic shell-safe args validator. +const managedGooglevideoArgs = "--lua-init=@/opt/zapret2/lua/zapret-auto.lua --payload=all --in-range=-s34228 --lua-desync=circular:fails=1:retrans=2 --payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1:strategy=1 --lua-desync=multidisorder:pos=1,midsld:strategy=1 --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN:strategy=2" + +// Source inspected from the existing prod-derived prototype image. The default +// detector uses incoming seq >4096 for success and outgoing retrans <=32768; +// circular orchestrates only tagged instances, with contiguous strategy IDs. +// Incoming range must exceed 4096 (default x0-x0 cannot learn success). +const managedGooglevideoLuaHash = "aacfde0c95c3058f8e95f5d7d244398bdc03ebf846a8f17322129fb543366a3d" +const managedGooglevideoCollector = "owned-nfq-media-range-v1" + +// Proof freshness is deliberately bounded to one day; expired proofs require a +// new owned run, never a refresh of their timestamp or cold summary rows. +const managedGooglevideoProofTTL = 24 * time.Hour + +var errGooglevideoInsufficientSamples = errors.New("insufficient fresh ordinary signed GGC samples") + +type GooglevideoRuntimeAttestation struct { + Collector string `json:"collector"` + LuaSHA256 string `json:"lua_sha256"` + RulesSHA256 string `json:"rules_sha256"` + WorkerArgs []string `json:"worker_args"` + CheckedAt time.Time `json:"checked_at"` +} + +// Not serialized: DTO construction or unmarshalling cannot authorize a write. +// This binds the completed producer's exact facts, including failed warmups. +type googlevideoProducerSeal [32]byte + +func googlevideoProofSeal(p GooglevideoQualificationProof) googlevideoProducerSeal { + raw, _ := json.Marshal(p) + return sha256.Sum256(raw) +} + +func validateNativeGooglevideoArgs(args string) error { + if !reflect.DeepEqual(strings.Fields(args), strings.Fields(managedGooglevideoArgs)) { + return errors.New("unattested native circular semantics") + } + return nil +} + +func (a *GooglevideoRuntimeAttestation) validate(c storedCandidate) error { + if err := a.validateStored(c); err != nil { + return err + } + if time.Since(a.CheckedAt) > managedGooglevideoProofTTL { + return errors.New("stale owned native attestation") + } + return nil +} + +// Age affects new qualification, not reconstruction of journal-owned bytes. +func (a *GooglevideoRuntimeAttestation) validateStored(c storedCandidate) error { + if a == nil || a.Collector != managedGooglevideoCollector || a.LuaSHA256 != managedGooglevideoLuaHash || a.RulesSHA256 != fmt.Sprintf("%x", sha256.Sum256([]byte(managedAssetRules))) || !reflect.DeepEqual(a.WorkerArgs, managedProbeWorkerArgs(c.Endpoints, c.Args)) || a.CheckedAt.IsZero() || time.Since(a.CheckedAt) < -time.Minute { + return errors.New("missing, stale or mismatched owned native attestation") + } + return validateNativeGooglevideoArgs(c.Args) +} + +// Caller gathers these resources from fresh Player queries for configured video +// samples. Never synthesize hosts by rewriting a signed URL. One-host inventory +// is an explicit unresolved result, before any worker or nft state is touched. +func newManagedGooglevideoVerifier(target managedDeps) func(context.Context, storedCandidate, []mediaResource) (GooglevideoQualificationProof, error) { + r := newManagedAssetRuntime(target) + guard := r.guard + r.guard = func() error { + if err := guard(); err != nil { + return err + } + lua, err := os.ReadFile("/opt/zapret2/lua/zapret-auto.lua") + if err != nil || fmt.Sprintf("%x", sha256.Sum256(lua)) != managedGooglevideoLuaHash { + return errors.New("unverified native automation Lua source") + } + return nil + } + return r.verifyGooglevideo +} + +func (r managedAssetRuntime) verifyGooglevideo(ctx context.Context, c storedCandidate, resources []mediaResource) (GooglevideoQualificationProof, error) { + p := GooglevideoQualificationProof{Version: 1, Mode: QualificationAdaptive, SourceKind: NativeCircularPostWarmup, Args: c.Args} + if c.Group != "googlevideo" || c.Mode != QualificationAdaptive { + return p, errors.New("adaptive googlevideo definition required") + } + if err := validateNativeGooglevideoArgs(c.Args); err != nil { + return p, err + } + byHost := map[string]mediaResource{} + for _, resource := range resources { + u, err := privateResourceURL(resource.url, resource.host) + if err != nil || u.Path != "/videoplayback" || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") { + return p, errGooglevideoInsufficientSamples + } + q := u.Query() + expires, err := strconv.ParseInt(q.Get("expire"), 10, 64) + if err != nil || expires <= time.Now().Unix()+120 || (q.Get("sig") == "" && q.Get("signature") == "" && q.Get("lsig") == "") { + return p, errGooglevideoInsufficientSamples + } + byHost[resource.host] = resource + } + // Validate representative host diversity using the shared gate before capture. + for _, host := range c.Endpoints { + if _, ok := byHost[host]; !ok { + return p, errGooglevideoInsufficientSamples + } + p.Hosts = append(p.Hosts, GooglevideoHostProof{Host: host}) + } + if err := validateGooglevideoHosts(c.Endpoints); err != nil { + return p, errGooglevideoInsufficientSamples + } + err := r.withOwnedWorker(ctx, c.Endpoints, c.Args, func(probeCtx context.Context) error { + for i := range p.Hosts { + h := &p.Hosts[i] + h.WarmupLimit = 10 + start := time.Now() + warmCtx, cancel := context.WithTimeout(probeCtx, 120*time.Second) + learned := map[string]bool{} + for h.WarmupAttempts < h.WarmupLimit && len(learned) < 2 && warmCtx.Err() == nil { + protocol := []string{"tls12", "tls13"}[h.WarmupAttempts%2] + h.WarmupAttempts++ + _, pass, err := collectMediaTransfer(warmCtx, r.proxy, byHost[h.Host], protocol, r.curl) + if err != nil { + h.WarmupFailures++ + if !managedGooglevideoTransferNonpass(err) && warmCtx.Err() == nil { + cancel() + return err + } + } else if pass.Bytes < 1024 { + h.WarmupFailures++ + } else { + learned[protocol] = true + } + } + h.WarmupMS = float64(time.Since(start).Microseconds()) / 1000 + cancel() + if probeCtx.Err() != nil { + return probeCtx.Err() + } + if len(learned) != 2 || h.WarmupMS > 120000 { + return errCandidateNonpass + } + for _, protocol := range []string{"tls12", "tls13"} { + for n := 0; n < 3; n++ { + _, pass, err := collectMediaTransfer(probeCtx, r.proxy, byHost[h.Host], protocol, r.curl) + if err != nil { + if managedGooglevideoTransferNonpass(err) { + return errCandidateNonpass + } + return err + } + if pass.Bytes < 1024 { + return errCandidateNonpass + } + h.Passes = append(h.Passes, pass) + } + } + } + return nil + }) + if err != nil { + return p, err + } + // Nothing attests until the owned consumer has been stopped and joined. + p.Attestation = &GooglevideoRuntimeAttestation{Collector: managedGooglevideoCollector, LuaSHA256: managedGooglevideoLuaHash, RulesSHA256: fmt.Sprintf("%x", sha256.Sum256([]byte(managedAssetRules))), WorkerArgs: managedProbeWorkerArgs(c.Endpoints, c.Args), CheckedAt: time.Now().UTC()} + if err := p.validate(c); err != nil { + return p, err + } + seal := googlevideoProofSeal(p) + p.producerSeal = &seal + return p, nil +} + +func managedGooglevideoTransferNonpass(err error) bool { + if managedAssetTransferNonpass(err) { + return true + } + if err == nil { + return false + } + if err.Error() == "invalid media range or container" { + return true + } + if e, ok := err.(interface{ Unwrap() error }); ok { + return managedGooglevideoTransferNonpass(e.Unwrap()) + } + return false +} + +func validateGooglevideoHosts(hosts []string) error { + seen, roots, prefixes := map[string]bool{}, map[string]bool{}, map[string]bool{} + for _, host := range hosts { + prefix, root, ok := strings.Cut(strings.TrimSuffix(host, ".googlevideo.com"), "---sn-") + if !ok || !strings.HasSuffix(host, ".googlevideo.com") || !validDomain(host) || host != strings.ToLower(host) || seen[host] || root == "" || len(prefix) < 3 || !strings.HasPrefix(prefix, "rr") || strings.Trim(prefix[2:], "0123456789") != "" { + return errGooglevideoInsufficientSamples + } + seen[host], roots[strings.Split(root, "-")[0]], prefixes[prefix] = true, true, true + } + if len(hosts) < 3 || len(roots) < 2 || len(prefixes) < 2 { + return errGooglevideoInsufficientSamples + } + return nil +} diff --git a/cmd/zapret-autotune/managed_googlevideo_runtime_test.go b/cmd/zapret-autotune/managed_googlevideo_runtime_test.go new file mode 100644 index 0000000..21469a0 --- /dev/null +++ b/cmd/zapret-autotune/managed_googlevideo_runtime_test.go @@ -0,0 +1,296 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "strings" + "testing" + "time" +) + +// No network: fake owned lifecycle and wire replies exercise the producer, not +// hardware readiness. The production factory adds the real guard/Lua hash gate. +func googlevideoRuntimeFixture(t *testing.T) (managedAssetRuntime, storedCandidate, []mediaResource, *[]string) { + t.Helper() + r, events, _ := assetRuntimeFixture(t) + c := storedCandidate{Group: "googlevideo", Mode: QualificationAdaptive, Args: managedGooglevideoArgs, Endpoints: []string{"rr1---sn-alpha.googlevideo.com", "rr2---sn-beta.googlevideo.com", "rr5---sn-gamma.googlevideo.com"}} + var resources []mediaResource + for _, host := range c.Endpoints { + resources = append(resources, mediaResource{url: fmt.Sprintf("https://%s/videoplayback?expire=%d&sig=private-secret", host, time.Now().Add(time.Hour).Unix()), host: host, kind: "video/mp4"}) + } + r.curl = func(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + *events = append(*events, "media") + if strings.Contains(strings.Join(args, " "), "private-secret") || !strings.Contains(strings.Join(args, " "), "--range 0-65535") { + t.Fatal("private URL in argv or no range") + } + b, err := io.ReadAll(stdin) + if err != nil { + return err + } + raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(b), "url = "), "\n")) + if err != nil { + return err + } + body := make([]byte, 1024) + copy(body, []byte{0, 0, 0, 24, 'f', 't', 'y', 'p'}) + fmt.Fprint(out, "HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp4\r\nContent-Length: 1024\r\nContent-Range: bytes 0-1023/4096\r\n\r\n") + out.Write(body) + metrics, _ := json.Marshal(map[string]interface{}{"url_effective": raw, "ssl_verify_result": 0, "num_redirects": 0, "http_code": 206, "time_total": 0.1, "speed_download": 10000, "size_download": len(body)}) + fmt.Fprint(out, resourceTrailer, string(metrics)) + tls := "TLSv1.2" + if strings.Contains(strings.Join(args, " "), "--tlsv1.3") { + tls = "TLSv1.3" + } + fmt.Fprintln(diagnostic, "SSL connection using", tls) + return nil + } + return r, c, resources, events +} + +func TestManagedGooglevideoOwnedPostWarmupProducer(t *testing.T) { + r, c, resources, events := googlevideoRuntimeFixture(t) + runner, calls := r.curl, 0 + r.curl = func(ctx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + calls++ + if calls <= 2 { + return context.DeadlineExceeded + } + return runner(ctx, args, stdin, out, diagnostic) + } + p, err := r.verifyGooglevideo(context.Background(), c, resources) + if err != nil || p.producerSeal == nil || p.validate(c) != nil { + t.Fatal(p, err) + } + if calls != 26 || p.Hosts[0].WarmupAttempts != 4 || p.Hosts[0].WarmupFailures != 2 || len(p.Hosts[0].Passes) != 6 { + t.Fatal("hidden warmup", p.Hosts, calls) + } + joined := strings.Join(*events, ",") + if strings.Count(joined, "start") != 1 || !strings.HasSuffix(joined, "delete,join") { + t.Fatal(joined) + } + raw, _ := json.Marshal(p) + if strings.Contains(string(raw), "private-secret") || strings.Contains(string(raw), "videoplayback") { + t.Fatal("durable URL leak") + } + for _, h := range p.Hosts { + for _, pass := range h.Passes { + if pass.RangeStart != 0 || pass.RangeEnd != 1023 || pass.ResourceBytes != 4096 { + t.Fatal(pass) + } + } + } +} + +func TestManagedGooglevideoRefusesUnresolvedAndUnknownSemantics(t *testing.T) { + for _, mode := range []string{"missing", "one-host", "same-root", "rewritten", "expired", "unsigned", "no-incoming", "blocking-incoming", "detector-payload", "detector", "stage-gap", "ignored-stage", "unknown-init", "unknown-key", "out-range"} { + t.Run(mode, func(t *testing.T) { + r, c, resources, events := googlevideoRuntimeFixture(t) + switch mode { + case "missing": + resources = nil + case "one-host": + resources = resources[:1] + case "same-root": + c.Endpoints[1] = "rr2---sn-alpha.googlevideo.com" + case "rewritten": + resources[0].host = c.Endpoints[1] + case "expired": + resources[0].url = "https://" + resources[0].host + "/videoplayback?expire=1&sig=private-secret" + case "unsigned": + resources[0].url = strings.Split(resources[0].url, "&sig=")[0] + case "no-incoming": + c.Args = strings.Replace(c.Args, " --in-range=-s34228", "", 1) + case "blocking-incoming": + c.Args = strings.Replace(c.Args, "--in-range=-s34228", "--in-range=0", 1) + case "detector-payload": + c.Args = strings.Replace(c.Args, "--payload=all", "--payload=tls_client_hello", 1) + case "detector": + c.Args = strings.Replace(c.Args, "circular:fails=1", "circular:success_detector=cond_true:fails=1", 1) + case "stage-gap": + c.Args = strings.Replace(c.Args, "strategy=2", "strategy=3", 1) + case "ignored-stage": + c.Args = strings.Replace(c.Args, ":strategy=2", "", 1) + case "unknown-init": + c.Args += " --lua-init=@/opt/zapret2/lua/unknown.lua" + case "unknown-key": + c.Args = strings.Replace(c.Args, "circular:fails=1", "circular:unknown=1:fails=1", 1) + case "out-range": + c.Args += " --out-range=0" + } + p, err := r.verifyGooglevideo(context.Background(), c, resources) + if err == nil || p.producerSeal != nil || len(*events) != 0 { + t.Fatal("captured unresolved inventory/config", p, err, *events) + } + }) + } +} + +func TestManagedGooglevideoWarmupBoundAndCleanupAttestation(t *testing.T) { + for _, mode := range []string{"nonpass", "root404", "cleanup", "queue-lost", "worker-exit"} { + t.Run(mode, func(t *testing.T) { + r, c, resources, events := googlevideoRuntimeFixture(t) + calls := 0 + if mode == "nonpass" { + r.curl = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { + calls++ + return context.DeadlineExceeded + } + } + if mode == "root404" { + r.curl = func(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + calls++ + b, err := io.ReadAll(stdin) + if err != nil { + return err + } + raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(b), "url = "), "\n")) + if err != nil { + return err + } + var diag bytes.Buffer + resourceWire(raw, "root", out, &diag) + tls := diag.String() + if strings.Contains(strings.Join(args, " "), "--tlsv1.3") { + tls = strings.ReplaceAll(tls, "TLSv1.2", "TLSv1.3") + } + io.WriteString(diagnostic, tls) + return nil + } + } + if mode == "cleanup" { + nft := r.nft + r.nft = func(ctx context.Context, input string) (string, error) { + if strings.HasPrefix(input, "delete") { + return "", errors.New("failed") + } + return nft(ctx, input) + } + } + if mode == "queue-lost" { + queue, n := r.queue, 0 + r.queue = func(pid int) (bool, error) { + n++ + if n == 3 { + return false, nil + } + return queue(pid) + } + } + if mode == "worker-exit" { + start := r.start + r.start = func(args []string) (managedAssetWorker, error) { + w, err := start(args) + ch := make(chan error, 1) + ch <- errors.New("dead") + w.done = ch + return w, err + } + } + p, err := r.verifyGooglevideo(context.Background(), c, resources) + if err == nil || p.Attestation != nil || p.producerSeal != nil { + t.Fatal("attested failed lifecycle", p, err) + } + if (mode == "nonpass" || mode == "root404") && (calls != 10 || p.Hosts[0].WarmupFailures != 10 || len(p.Hosts[0].Passes) != 0 || !strings.HasSuffix(strings.Join(*events, ","), "delete,join")) { + t.Fatal(p, calls, *events) + } + }) + } +} + +func TestManagedGooglevideoAdaptiveStorageAndReadGate(t *testing.T) { + r, c, resources, _ := googlevideoRuntimeFixture(t) + p, err := r.verifyGooglevideo(context.Background(), c, resources) + if err != nil { + t.Fatal(err) + } + db := testStore(t) + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + id, err := insertStrategyCandidate(tx, c.Group, c.Endpoints, c.Args, nil, c.Mode) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(p) + var dto GooglevideoQualificationProof + if err := json.Unmarshal(raw, &dto); err != nil { + t.Fatal(err) + } + if dto.validate(c) != nil { + t.Fatal("DTO should be structurally valid") + } + if err := saveAdaptiveGooglevideoEvidence(tx, id, dto); err == nil { + t.Fatal("DTO alone authorized a write") + } + p.Hosts[0].WarmupFailures++ + if err := saveAdaptiveGooglevideoEvidence(tx, id, p); err == nil { + t.Fatal("modified producer proof accepted") + } + p.Hosts[0].WarmupFailures-- + if err := saveAdaptiveGooglevideoEvidence(tx, id, p); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + var n int + if err := db.QueryRow("SELECT count(*) FROM candidate_evidence WHERE candidate_id=?", id).Scan(&n); err != nil || n != 0 { + t.Fatal("manufactured cold metrics", n, err) + } + stored, err := readStrategyCandidate(db, id) + if err != nil || !stored.Qualified || stored.Mode != QualificationAdaptive { + t.Fatal(stored, err) + } + list, err := listQualifiedCandidates(db, "googlevideo", 1) + if err != nil || len(list) != 1 || list[0].ID != id { + t.Fatal(list, err) + } + if err := validateGooglevideoSuffixEvidence(db, stored); err != nil { + t.Fatal(err) + } + for _, mode := range []string{"malformed", "stale", "args", "attestation", "hidden-warmup", "media", "secret"} { + t.Run(mode, func(t *testing.T) { + proof, err := decodeGooglevideoProof(string(raw)) + if err != nil { + t.Fatal(err) + } + switch mode { + case "stale": + proof.Attestation.CheckedAt = time.Now().Add(-25 * time.Hour) + case "args": + proof.Args = "--lua-desync=fake" + case "attestation": + proof.Attestation = nil + case "hidden-warmup": + proof.Hosts[0].WarmupAttempts = 0 + case "media": + proof.Hosts[0].Passes[0].Status = 404 + } + bad, _ := json.Marshal(proof) + if mode == "malformed" { + bad = []byte(`{"version":"bad"}`) + } + if mode == "secret" { + bad = append(bad[:len(bad)-1], []byte(`,"signed_url":"private-secret"}`)...) + } + if _, err := db.Exec("UPDATE googlevideo_scope_evidence SET proof=? WHERE candidate_id=?", string(bad), id); err != nil { + t.Fatal(err) + } + stored, err := readStrategyCandidate(db, id) + if err != nil || stored.Qualified { + t.Fatal(stored, err) + } + list, err := listQualifiedCandidates(db, "googlevideo", 1) + if err != nil || len(list) != 0 { + t.Fatal(list, err) + } + }) + } +} diff --git a/cmd/zapret-autotune/managed_googlevideo_samples_test.go b/cmd/zapret-autotune/managed_googlevideo_samples_test.go new file mode 100644 index 0000000..bf42643 --- /dev/null +++ b/cmd/zapret-autotune/managed_googlevideo_samples_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" +) + +func TestManagedGooglevideoFreshPrivateSamples(t *testing.T) { + ids := []string{"dQw4w9WgXcQ", "aqz-KE-bpKQ", "jNQXAC9IVRw"} + hosts := []string{"rr1---sn-alpha.googlevideo.com", "rr2---sn-beta.googlevideo.com", "rr5---sn-gamma.googlevideo.com"} + for _, scenario := range []string{"diverse", "duplicate", "expired", "sabr"} { + t.Run(scenario, func(t *testing.T) { + calls := 0 + collect := managedGooglevideoSamples(ids, func(_ context.Context, id string) ([]byte, error) { + i := calls % len(ids) + calls++ + if id != ids[i] { + t.Fatal("configured ID changed", id) + } + if scenario == "sabr" { + return []byte(`{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"private-secret"}}`), nil + } + host := hosts[i] + if scenario == "duplicate" { + host = hosts[0] + } + expires := time.Now().Add(time.Hour).Unix() + if scenario == "expired" { + expires = time.Now().Unix() + 30 + } + url, _ := json.Marshal(fmt.Sprintf("https://%s/videoplayback?expire=%d&sig=private-secret", host, expires)) + return []byte(fmt.Sprintf(`{"playabilityStatus":{"status":"OK"},"streamingData":{"adaptiveFormats":[{"mimeType":"video/mp4","url":%s}]}}`, url)), nil + }) + for n := 0; n < 2; n++ { + resources, err := collect(context.Background()) + if scenario != "diverse" { + var insufficient *InsufficientAdaptiveSamples + if !errors.As(err, &insufficient) || strings.Contains(err.Error(), "private-secret") || len(resources) != 0 { + t.Fatal("unresolved samples accepted or leaked", resources, err) + } + continue + } + var actual []string + for _, r := range resources { + actual = append(actual, r.host) + if !strings.Contains(r.url, "sig=private-secret") { + t.Fatal("signed URL changed") + } + } + if err != nil || !reflect.DeepEqual(actual, hosts) { + t.Fatal(actual, err) + } + } + if calls != 2*len(ids) { + t.Fatal("samples were reused", calls) + } + }) + } +} diff --git a/cmd/zapret-autotune/managed_monitor_cli.go b/cmd/zapret-autotune/managed_monitor_cli.go index 09a68e0..e41ea35 100644 --- a/cmd/zapret-autotune/managed_monitor_cli.go +++ b/cmd/zapret-autotune/managed_monitor_cli.go @@ -111,13 +111,27 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr now: time.Now, wait: waitManagedMonitor, reportRecoveryError: func(error) { fmt.Fprintln(stderr, "managed recovery failed (details suppressed)") }, } + if recovery.service { + monitor.reportProbeError = func(err error) { + var failure *managedResourceFailure + if errors.As(err, &failure) { + fmt.Fprintln(stderr, failure.PublicError()) + } else { + fmt.Fprintln(stderr, "DEGRADED: service scope unresolved") + } + } + } if err := wireManagedRecovery(&monitor, recovery, func(error) { fmt.Fprintln(stderr, "managed GGC unresolved; monitoring continues (not playback health)") }); err != nil { fmt.Fprintln(stderr, "error: managed monitor wiring refused") return 78 } - fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health") + if recovery.service { + fmt.Fprintln(stdout, "managed monitor: service resource checks; SABR unknown; owned temporary asset verifier") + } else { + fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health") + } err = supervise(ctx, monitor) if err != nil { var fatal *managedRecoveryFatal @@ -174,7 +188,14 @@ func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand defer cancel() return command(bounded, path, args...) } - return managedRecoveryDependencies(target, db, scan, check, run) + d, err := managedRecoveryDependencies(target, db, scan, check, run) + if err == nil && d.service { + d.verifyAssets = newManagedAssetVerifier(target) + if d.adaptive { + d.verifyGooglevideo = newManagedGooglevideoVerifier(target) + } + } + return d, err } // Joined cancellation plus failed cleanup/DB/lock close is not a graceful exit. diff --git a/cmd/zapret-autotune/managed_monitor_cli_test.go b/cmd/zapret-autotune/managed_monitor_cli_test.go index 7c25243..a846578 100644 --- a/cmd/zapret-autotune/managed_monitor_cli_test.go +++ b/cmd/zapret-autotune/managed_monitor_cli_test.go @@ -26,10 +26,18 @@ func TestManagedPrestartHelper(t *testing.T) { for len(args) > 0 && args[0] != "--" { args = args[1:] } - if len(args) < 3 { + if len(args) < 2 { os.Exit(1) } - args = args[2:] // -- managed COMMAND + args = args[1:] // -- COMMAND ... + if len(args) == 1 && args[0] == "target-config-check" { + // Use the production validator, not a fixture-only success response. + os.Exit(run(args, os.Stdout, os.Stderr)) + } + if len(args) < 2 || args[0] != "managed" { + os.Exit(1) + } + args = args[1:] // managed COMMAND if args[0] == "monitor" { os.Exit(78) // Stop after initial activation; no scans or network. } @@ -41,7 +49,7 @@ func TestManagedPrestartHelper(t *testing.T) { } func TestManagedEntrypointPrestartReconciliation(t *testing.T) { - for _, mode := range []string{"fresh", "baseline", "committed", "prepared", "drift", "unknown", "legacy", "corrupt-backup", "unqualified", "reload-failure"} { + for _, mode := range []string{"fresh", "baseline", "committed", "prepared", "drift", "unknown", "legacy", "corrupt-backup", "unqualified", "reload-failure", "invalid-dns", "invalid-service"} { t.Run(mode, func(t *testing.T) { d, env, _ := managedCLIFixture(t) env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db" @@ -125,7 +133,7 @@ func TestManagedEntrypointPrestartReconciliation(t *testing.T) { t.Fatal(err) } write("awk", "#!/bin/sh\nexit 0\n") - child := "#!/bin/sh\nsleeper=\ntrap '[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; exit 0' TERM INT\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n" + child := "#!/bin/sh\n[ -z \"${SS_PASSWORD-}\" ] || exit 1\n[ \"$#\" -eq 2 ] || { [ \"$#\" -eq 4 ] && [ \"$3\" = -d ] && [ \"$4\" = \"$SS_DNS_SERVER\" ]; } || exit 1\n: > \"$FIXTURE_CHILD_STARTED\"\nsleeper=\ntrap '[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; exit 0' TERM INT\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n" write("ss-server", child) write("ss-local", child) log := filepath.Join(root, "log") @@ -156,11 +164,34 @@ func TestManagedEntrypointPrestartReconciliation(t *testing.T) { // The CLI guard sees fixed production bindings; only fake daemon reads // use the fixture config, via its own script rather than CLI path flags. cmd.Env = append(cmd.Env, "FIXTURE_CONFIG="+filepath.Join(d.directory, "config")) + childStarted := filepath.Join(root, "child-started") + cmd.Env = append(cmd.Env, "FIXTURE_CHILD_STARTED="+childStarted, "SS_DNS_SERVER=8.8.8.8") + wantExit := 78 + if mode == "invalid-dns" { + cmd.Env = append(cmd.Env, "SS_DNS_SERVER=8.8.8.8\n--invalid") + wantExit = 64 + } + if mode == "invalid-service" { + cmd.Env = append(cmd.Env, "ZAPRET_AUTOTUNE_YOUTUBE_SERVICE=invalid") + wantExit = 64 + } output, err := cmd.CombinedOutput() var exit *exec.ExitError - if ctx.Err() != nil || !errors.As(err, &exit) || exit.ExitCode() != 78 { + if ctx.Err() != nil || !errors.As(err, &exit) || exit.ExitCode() != wantExit { t.Fatal(err, ctx.Err(), string(output)) } + if bytes.Contains(output, []byte(strings.Repeat("a", 32))) { + t.Fatal("startup exposed password") + } + if wantExit == 64 { + if _, err := os.Stat(childStarted); !errors.Is(err, os.ErrNotExist) { + t.Fatal("invalid configuration launched a service", err) + } + files, err := os.ReadDir(ss) + if err != nil || len(files) != 0 || !strings.Contains(string(output), "configerror:") { + t.Fatal("invalid configuration did not fail before config creation", err, string(output)) + } + } b, _ := os.ReadFile(log) allowed := mode == "fresh" || mode == "baseline" || mode == "committed" || mode == "prepared" if !allowed && mode != "reload-failure" && len(b) != 0 { diff --git a/cmd/zapret-autotune/managed_recovery.go b/cmd/zapret-autotune/managed_recovery.go index 4a6b31c..3befc3d 100644 --- a/cmd/zapret-autotune/managed_recovery.go +++ b/cmd/zapret-autotune/managed_recovery.go @@ -30,6 +30,15 @@ type managedRecoveryDeps struct { health func(context.Context) (string, error) run func(context.Context, string, ...string) error jobTimeout time.Duration // Zero uses the production budget; tests may shorten it. + service bool + frontHealth func(context.Context) (string, error) + resources func(context.Context) (managedServiceInventory, error) + // Verify owns temporary raw-config activation and must join/restore before + // returning. It must not touch permanent config or the transaction journal. + verifyAssets func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error) + adaptive bool + mediaSamples func(context.Context) ([]mediaResource, error) + verifyGooglevideo func(context.Context, storedCandidate, []mediaResource) (GooglevideoQualificationProof, error) } // Target-local adapter: reuses bounded Player requests without invoking a CLI. @@ -47,7 +56,7 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, return d, err } d.transaction = &managedTransaction{Dir: target.directory, Name: "config"} - request := func(ctx context.Context) ([]byte, error) { + requestVideo := func(ctx context.Context, videoID string) ([]byte, error) { if err := ctx.Err(); err != nil { return nil, err } @@ -56,7 +65,7 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, } bounded, cancel := context.WithTimeout(ctx, 35*time.Second) defer cancel() - body, err := requestPlayer(cfg.VideoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error { + body, err := requestPlayer(videoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error { return target.player(bounded, args, out, diagnostics) }) if bounded.Err() != nil { @@ -64,6 +73,15 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, } return body, err } + request := func(ctx context.Context) ([]byte, error) { return requestVideo(ctx, cfg.VideoID) } + mode, videoIDs, err := managedGooglevideoConfig(target.lookup, cfg.VideoID) + if err != nil { + return d, err + } + d.adaptive = mode == QualificationAdaptive + if d.adaptive { + d.mediaSamples = managedGooglevideoSamples(videoIDs, requestVideo) + } d.health = func(ctx context.Context) (string, error) { body, err := request(ctx) if err != nil { @@ -95,6 +113,16 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, } return playerHostname(body) } + d.service, err = managedServiceEnabled(target.lookup) + if err != nil { + return d, err + } + if d.service { + d.frontHealth = d.health + d.resources = managedServiceResources(cfg, request, target.player) + d.health = managedServiceHealth(cfg.SOCKSProxy, d.frontHealth, d.resources, runResourceCurl) + d.verifyAssets = unwiredManagedAssetVerifier + } return d, nil } @@ -199,6 +227,17 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma if err := ctx.Err(); err != nil { return result, err } + if d.service { + _, healthErr := d.health(ctx) + var fatal *managedRecoveryFatal + if ctx.Err() != nil || errors.As(healthErr, &fatal) || errors.Is(healthErr, context.Canceled) || errors.Is(healthErr, context.DeadlineExceeded) { + return result, managedRecoveryContextError(ctx, healthErr) + } + var failure *managedResourceFailure + if errors.As(healthErr, &failure) && assetEndpoint(failure.Host) { + return result, repairManagedAssets(ctx, d, failure.Host) + } + } m := *d.transaction // Rollback must restart even after job cancellation; independently bounded. var cleanupCtx context.Context @@ -221,7 +260,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma return nil } m.Health = func() (string, error) { - status, err := d.health(ctx) + health := d.health + if d.service && d.frontHealth != nil { + health = d.frontHealth + } + // Commit frontend/GGC progress independently; the monitor's overall + // service probe still requires media and both images before READY. + status, err := health(ctx) if ctx.Err() != nil { return "", managedRecoveryContextError(ctx, err) } @@ -262,6 +307,16 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma if err := ctx.Err(); err != nil { return err } + if d.service { + current, e := m.ActiveCandidates(d.db) + if e != nil { + return &managedRecoveryFatal{Err: e} + } + ids, e = managedFullDesiredSet(d.db, current, ids) + if e != nil { + return &managedRecoveryFatal{Err: e} + } + } cleanupReload := m.Reload first := true m.Reload = func() error { @@ -301,7 +356,11 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma } healthy := false if result.FrontendID != 0 { - status, e := d.health(ctx) + frontHealth := d.health + if d.service && d.frontHealth != nil { + frontHealth = d.frontHealth + } + status, e := frontHealth(ctx) e = managedRecoveryContextError(ctx, e) var fatal *managedRecoveryFatal if errors.As(e, &fatal) || errors.Is(e, context.Canceled) { @@ -372,11 +431,19 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma if e != nil { return result, e } - if len(current) != 1 || current[0] != result.FrontendID { + if !d.service && (len(current) != 1 || current[0] != result.FrontendID) { if e := apply([]int64{result.FrontendID}); e != nil { return result, e } } + if d.adaptive { + result.GooglevideoID, result.GGCError = recoverManagedGooglevideo(ctx, d) + var fatal *managedRecoveryFatal + if errors.As(result.GGCError, &fatal) || ctx.Err() != nil { + return result, result.GGCError + } + return result, nil + } host, e := d.discover(ctx) e = managedRecoveryContextError(ctx, e) var discoveryFatal *managedRecoveryFatal @@ -578,6 +645,9 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe result, err := recoverManagedTarget(ctx, recovery) if result.GGCError != nil { reportGGC(result.GGCError) + if recovery.adaptive && err == nil { + return result.GGCError + } } // The supervisor reports ordinary recovery failures and retries only // after its durable cooldown. Safety failures (rollback/cleanup/drift) @@ -667,6 +737,15 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe if err != nil { return &managedRecoveryFatal{Err: err} } + if recovery.adaptive && candidate.Group == "googlevideo" && candidate.Mode == QualificationAdaptive && candidate.Qualified { + journal, e := transaction.load() + if e != nil { + return &managedRecoveryFatal{Err: e} + } + if journal.Scope == GooglevideoScopeSuffix { + return nil + } + } if candidate.Group == "googlevideo" && len(candidate.Endpoints) == 1 && candidate.Endpoints[0] == host { return nil } diff --git a/cmd/zapret-autotune/managed_recovery_regression_test.go b/cmd/zapret-autotune/managed_recovery_regression_test.go new file mode 100644 index 0000000..569139f --- /dev/null +++ b/cmd/zapret-autotune/managed_recovery_regression_test.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" + "time" +) + +func TestManagedExpiredAdaptiveJournalRecovery(t *testing.T) { + for _, scenario := range []string{"committed", "prepared", "rollback"} { + t.Run(scenario, func(t *testing.T) { + d, front, _ := recoveryFixture(t, true) + r, c, resources, _ := googlevideoRuntimeFixture(t) + proof, err := r.verifyGooglevideo(context.Background(), c, resources) + if err != nil { + t.Fatal(err) + } + tx, err := d.db.Begin() + if err != nil { + t.Fatal(err) + } + id, err := insertStrategyCandidate(tx, c.Group, c.Endpoints, c.Args, nil, c.Mode) + if err == nil { + err = saveAdaptiveGooglevideoEvidence(tx, id, proof) + } + if err != nil { + _ = tx.Rollback() + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + ids := []int64{front, unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true), unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true), id} + if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); err != nil { + t.Fatal(err) + } + before, err := d.transaction.read("config") + if err != nil { + t.Fatal(err) + } + if scenario == "prepared" { + crash := errors.New("process died after replacement") + d.transaction.fault = func(point string) error { + if point == "replaced" { + return crash + } + return nil + } + if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); !errors.Is(err, crash) { + t.Fatal(err) + } + d.transaction.fault = nil + } + proof.Attestation.CheckedAt = time.Now().Add(-25 * time.Hour) + raw, err := json.Marshal(proof) + if err != nil { + t.Fatal(err) + } + if _, err := d.db.Exec("UPDATE googlevideo_scope_evidence SET proof=? WHERE candidate_id=?", string(raw), id); err != nil { + t.Fatal(err) + } + if err := d.transaction.Reconcile(d.db); err != nil { + t.Fatal("aged committed/pending refs prevented reconciliation", err) + } + assertActiveCandidates(t, d.transaction, d.db, ids) + stored, err := readStrategyCandidate(d.db, id) + if err != nil || stored.Qualified { + t.Fatal("aged proof authorized fresh qualification", stored, err) + } + if err := validateGooglevideoSuffixEvidence(d.db, stored); err == nil { + t.Fatal("aged proof authorized new suffix") + } + if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); err == nil { + t.Fatal("aged proof authorized new apply") + } + if err := d.transaction.applyRenderedDB(before, d.db, ids...); err == nil { + t.Fatal("pre-rendered bytes bypassed freshness") + } + d.service, d.adaptive = true, true + d.frontHealth = d.health + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/sample.jpg", "yt3.ggpht.com": "https://yt3.ggpht.com/sample.jpg"}, proxy: r.proxy, runner: adaptiveImageRunner}, nil + } + samples, verifies := 0, 0 + d.mediaSamples = func(context.Context) ([]mediaResource, error) { + samples++ + return resources, nil + } + d.verifyGooglevideo = func(ctx context.Context, c storedCandidate, resources []mediaResource) (GooglevideoQualificationProof, error) { + verifies++ + return r.verifyGooglevideo(ctx, c, resources) + } + if scenario == "rollback" { + d.health = func(context.Context) (string, error) { + return "DEGRADED", &managedResourceFailure{"media", resources[0].host, nil} + } + } + result, err := recoverManagedTarget(context.Background(), d) + if err != nil || samples != 1 || verifies != 1 { + t.Fatal("aged active state blocked fresh owned verification", result, samples, verifies, err) + } + if scenario == "rollback" { + if result.GGCError == nil { + t.Fatal("unhealthy apply accepted") + } + assertActiveCandidates(t, d.transaction, d.db, ids) + after, err := d.transaction.read("config") + if err != nil || !reflect.DeepEqual(before, after) { + t.Fatal("aged rollback changed committed bytes", err) + } + } else { + if result.GGCError != nil || result.GooglevideoID <= id { + t.Fatal("fresh revalidation did not replace expired proof", result) + } + ids[len(ids)-1] = result.GooglevideoID + assertActiveCandidates(t, d.transaction, d.db, ids) + } + // Expiry alone is safe; malformed/invalid historical proof is not. + activeID := ids[len(ids)-1] + if _, err := d.db.Exec("UPDATE googlevideo_scope_evidence SET proof='{}' WHERE candidate_id=?", activeID); err != nil { + t.Fatal(err) + } + if _, err := d.transaction.ActiveCandidates(d.db); err == nil { + t.Fatal("invalid committed proof accepted") + } + if err := d.transaction.Reconcile(d.db); err == nil { + t.Fatal("invalid committed proof reconciled") + } + }) + } +} + +func TestManagedServiceDispatchSafetyPrecedesImageRepair(t *testing.T) { + for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} { + for _, failure := range []error{&managedRecoveryFatal{Err: errors.New("guard failed")}, context.Canceled, context.DeadlineExceeded} { + t.Run(host+"/"+failure.Error(), func(t *testing.T) { + d, front, events := recoveryFixture(t, true) + d.service = true + mixed := errors.Join(&managedResourceFailure{managedImageProbeKind(host), host, nil}, failure) + d.health = func(context.Context) (string, error) { return "DEGRADED", mixed } + d.resources = func(context.Context) (managedServiceInventory, error) { + t.Fatal("mixed safety failure dispatched image repair") + return managedServiceInventory{}, nil + } + _, err := recoverManagedTarget(context.Background(), d) + if !errors.Is(err, failure) || len(*events) != 0 { + t.Fatal("joined safety failure swallowed/retried", err, *events) + } + assertActiveCandidates(t, d.transaction, d.db, []int64{front}) + }) + } + } +} diff --git a/cmd/zapret-autotune/managed_service_test.go b/cmd/zapret-autotune/managed_service_test.go new file mode 100644 index 0000000..02dc2ff --- /dev/null +++ b/cmd/zapret-autotune/managed_service_test.go @@ -0,0 +1,350 @@ +package main + +import ( + "context" + "errors" + "io" + "strings" + "testing" +) + +func TestManagedServiceOptIn(t *testing.T) { + for _, value := range []string{"", "0", "1", "true", "secret"} { + enabled, err := managedServiceEnabled(func(key string) (string, bool) { return value, key == "ZAPRET_AUTOTUNE_YOUTUBE_SERVICE" && value != "" }) + valid := value == "" || value == "0" || value == "1" + if (err == nil) != valid || (valid && enabled != (value == "1")) { + t.Fatalf("value %q: %v %v", value, enabled, err) + } + } +} + +func TestManagedServiceMissingResourcesNeverGreen(t *testing.T) { + front := func(context.Context) (string, error) { return "OK", nil } + for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} { + i := managedServiceInventory{images: map[string]string{}} + if host == "yt3.ggpht.com" { + i.images["i.ytimg.com"] = "https://i.ytimg.com/test" + } + // The first missing image must fail before any resource process is spawned. + if host == "yt3.ggpht.com" { + continue + } + health := managedServiceHealth("socks5://127.0.0.1:11080", front, func(context.Context) (managedServiceInventory, error) { return i, nil }, func(context.Context, []string, io.Reader, io.Writer, io.Writer) error { + t.Fatal("unexpected transfer") + return nil + }) + status, err := health(context.Background()) + var failure *managedResourceFailure + if status != "DEGRADED" || !errors.As(err, &failure) || failure.Host != host { + t.Fatalf("%s %v", status, err) + } + } + if public := (&managedResourceFailure{"https://secret/?token=x", "secret/?token=x", errors.New("private")}).PublicError(); strings.Contains(public, "secret") || strings.Contains(public, "private") { + t.Fatal(public) + } +} + +func TestManagedServiceFullDesiredSet(t *testing.T) { + d, front, _ := recoveryFixture(t, true) + ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true) + thumb := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true) + avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true) + newFront := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=multisplit", true) + ids, err := managedFullDesiredSet(d.db, []int64{front, ggc, thumb, avatar}, []int64{newFront}) + if err != nil { + t.Fatal(err) + } + if len(ids) != 4 || ids[0] != ggc || ids[1] != thumb || ids[2] != avatar || ids[3] != newFront { + t.Fatal(ids) + } +} + +func TestManagedServiceSavedAssetRepairPreservesFullSet(t *testing.T) { + d, front, events := recoveryFixture(t, true) + ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true) + old := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true) + avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true) + saved := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=multisplit", true) + if err := d.transaction.Apply(d.db, []int64{front, ggc, old, avatar}); err != nil { + t.Fatal(err) + } + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/fresh"}}, nil + } + joined := false + d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) { + if c.ID != saved || len(*events) == 0 || (*events)[len(*events)-1] != "stop" { + t.Fatal("not saved-first under downtime", c.ID, *events) + } + joined = true + return []assetMeasurement{unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args)}, nil + } + d.health = func(context.Context) (string, error) { + if !joined { + t.Fatal("apply before verifier joined") + } + return "OK", nil + } + if err := repairManagedAssets(context.Background(), d, "i.ytimg.com"); err != nil { + t.Fatal(err) + } + ids, err := d.transaction.ActiveCandidates(d.db) + if err != nil || len(ids) != 4 { + t.Fatal(ids, err) + } + for _, required := range []int64{front, ggc, avatar} { + found := false + for _, id := range ids { + found = found || id == required + } + if !found { + t.Fatal("lost sibling", required, ids) + } + } +} + +func TestManagedServiceAssetRollbackAndCancellation(t *testing.T) { + for _, cancelJob := range []bool{false, true} { + t.Run(map[bool]string{false: "false-media-rollback", true: "cancel-joins"}[cancelJob], func(t *testing.T) { + d, front, events := recoveryFixture(t, true) + old := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true) + avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true) + if err := d.transaction.Apply(d.db, []int64{front, old, avatar}); err != nil { + t.Fatal(err) + } + unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=multisplit", true) + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/fresh"}}, nil + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + joined := false + d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) { + joined = true + if cancelJob { + cancel() + return nil, context.Canceled + } + return []assetMeasurement{unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args)}, nil + } + d.health = func(context.Context) (string, error) { + return "DEGRADED", &managedResourceFailure{"media", "fresh.googlevideo.com", nil} + } + err := repairManagedAssets(ctx, d, "i.ytimg.com") + if err == nil || !joined { + t.Fatal("unverified success or unjoined verifier", err) + } + if cancelJob && (!errors.Is(err, context.Canceled) || (*events)[len(*events)-1] != "start") { + t.Fatal("missing bounded restoration", err, *events) + } + assertActiveCandidates(t, d.transaction, d.db, []int64{front, old, avatar}) + }) + } +} + +func TestManagedServiceFreshPrivateInventory(t *testing.T) { + cfg := healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080"} + resources := managedServiceResources(cfg, func(context.Context) ([]byte, error) { + return []byte(`{"playabilityStatus":{"status":"OK"},"videoDetails":{"thumbnail":{"thumbnails":[{"url":"https://i.ytimg.com/fresh?private=thumbnail"}]}},"streamingData":{}}`), nil + }, func(_ context.Context, args []string, out, _ io.Writer) error { + if args[len(args)-1] != "https://www.youtube.com/" || strings.Contains(strings.Join(args, " "), "private=") { + t.Fatal(args) + } + _, err := io.WriteString(out, `var ytInitialData = {"avatar":{"url":"https://yt3.ggpht.com/fresh?private=avatar"}};`) + return err + }) + i, err := resources(context.Background()) + if err != nil || len(i.images) != 2 || i.media.url != "" { + t.Fatal("fresh inventory or SABR unknown contract", err) + } +} + +func TestManagedServicePartialInventoryChecksAndRepairsAvailableImage(t *testing.T) { + for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} { + t.Run(host, func(t *testing.T) { + d, front, _ := recoveryFixture(t, true) + ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true) + thumb := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true) + avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true) + if err := d.transaction.Apply(d.db, []int64{front, ggc, thumb, avatar}); err != nil { + t.Fatal(err) + } + r, _, _ := assetRuntimeFixture(t) + d.service = true + d.frontHealth = func(context.Context) (string, error) { return "OK", nil } + missingHost := "yt3.ggpht.com" + if host == missingHost { + missingHost = "i.ytimg.com" + } + missing := &managedResourceFailure{managedImageProbeKind(missingHost), missingHost, errors.New("private metadata unavailable")} + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{host: "https://" + host + "/image?token=private"}, proxy: r.proxy, runner: r.curl, resourceErr: &managedResourceFailure{"media", "unknown", nil}}, missing + } + repaired, checked := false, false + d.health = managedServiceHealth(r.proxy, d.frontHealth, d.resources, func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error { + checked = true + if !repaired { + resourceWire("https://"+host+"/image?token=private", "cn", out, diag) + return nil + } + return r.curl(ctx, args, in, out, diag) + }) + status, err := d.health(context.Background()) + var failure *managedResourceFailure + if status != "DEGRADED" || !checked || !errors.As(err, &failure) || failure.Host != host || failure.ProbeKind != managedImageProbeKind(host) || strings.Contains(err.Error(), "private") { + t.Fatal(status, checked, err) + } + d.verifyAssets = func(ctx context.Context, c storedCandidate, i managedServiceInventory) ([]assetMeasurement, error) { + m, err := r.verify(ctx, c, i) + repaired = err == nil + return m, err + } + if _, err := recoverManagedTarget(context.Background(), d); err != nil || !repaired { + t.Fatal("partial metadata blocked independent asset repair", err) + } + ids, err := d.transaction.ActiveCandidates(d.db) + if err != nil || len(ids) != 4 { + t.Fatal("lost full desired set", ids, err) + } + status, err = d.health(context.Background()) + if status != "DEGRADED" || !errors.As(err, &failure) || failure.Host != missingHost { + t.Fatal("missing resources became green", status, err) + } + }) + } +} + +func TestManagedServiceBootstrapCommitsFrontendBeforeAssetRepair(t *testing.T) { + d, _, _ := recoveryFixture(t, false) + d.service = true + r, _, _ := assetRuntimeFixture(t) + frontChecks := 0 + d.frontHealth = func(context.Context) (string, error) { + frontChecks++ + if frontChecks == 1 { + return "DEGRADED", nil + } + return "OK", nil + } + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}, proxy: r.proxy, runner: r.curl, resourceErr: &managedResourceFailure{"media", "unknown", nil}}, nil + } + repaired := false + d.health = managedServiceHealth(r.proxy, d.frontHealth, d.resources, func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error { + if !repaired { + resourceWire("https://i.ytimg.com/image", "cn", out, diag) + return nil + } + return r.curl(ctx, args, in, out, diag) + }) + result, err := recoverManagedTarget(context.Background(), d) + if err != nil || result.FrontendID == 0 || result.GooglevideoID == 0 { + t.Fatal("broken thumbnail vetoed frontend commit", result, err) + } + assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID}) + d.verifyAssets = func(ctx context.Context, c storedCandidate, i managedServiceInventory) ([]assetMeasurement, error) { + m, err := r.verify(ctx, c, i) + repaired = err == nil + return m, err + } + if _, err := recoverManagedTarget(context.Background(), d); err != nil || !repaired { + t.Fatal("thumbnail repair blocked by missing media", err) + } + ids, err := d.transaction.ActiveCandidates(d.db) + if err != nil || len(ids) != 3 || ids[0] != result.FrontendID || ids[1] != result.GooglevideoID { + t.Fatal("asset commit lost frontend/GGC", ids, err) + } + status, err := d.health(context.Background()) + var failure *managedResourceFailure + if status != "DEGRADED" || !errors.As(err, &failure) || failure.ProbeKind != "media" { + t.Fatal("missing media became service-ready", status, err) + } +} + +func TestManagedServiceAssetLocalHealthFailureRollsBack(t *testing.T) { + d, front, _ := recoveryFixture(t, true) + old := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true) + avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true) + ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true) + prior := []int64{front, ggc, old, avatar} + if err := d.transaction.Apply(d.db, prior); err != nil { + t.Fatal(err) + } + before, err := d.transaction.read("config") + if err != nil { + t.Fatal(err) + } + r, _, _ := assetRuntimeFixture(t) + d.frontHealth = func(context.Context) (string, error) { return "OK", nil } + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}, proxy: r.proxy, runner: func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error { + input, err := io.ReadAll(in) + if err != nil { + return err + } + if strings.Contains(string(input), "i.ytimg.com") { + resourceWire("https://i.ytimg.com/image", "truncated", out, diag) + return nil + } + return r.curl(ctx, args, strings.NewReader(string(input)), out, diag) + }}, nil + } + i, _ := d.resources(context.Background()) + if status, err := managedAssetInventoryHealth(context.Background(), i, []string{"yt3.ggpht.com"}); status != "OK" || err != nil { + t.Fatal("old sibling must remain healthy", status, err) + } + d.verifyAssets = r.verify + if err := repairManagedAssets(context.Background(), d, "i.ytimg.com"); !errors.Is(err, errNoQualifiedStrategy) { + t.Fatal(err) + } + assertActiveCandidates(t, d.transaction, d.db, prior) + after, err := d.transaction.read("config") + if err != nil || string(before) != string(after) { + t.Fatal("rejected asset changed prior config", err) + } +} + +func TestManagedAssetHealthNonpassPreservesMixedSafetyFailures(t *testing.T) { + badBody := &managedRecoveryFatal{Err: errors.New("invalid image body")} + for _, tc := range []struct { + err error + nonpass bool + }{ + {badBody, true}, + {errors.Join(badBody, &managedRecoveryFatal{Err: errors.New("guard failed")}), false}, + {errors.Join(badBody, context.Canceled), false}, + {&managedAssetCleanupFailure{Err: errors.New("cleanup failed")}, false}, + } { + if managedAssetHealthNonpass(tc.err) != tc.nonpass { + t.Fatal(tc.err) + } + } +} + +func TestManagedAssetRepairMixedFatalNeverRetries(t *testing.T) { + for _, stage := range []string{"verify", "health"} { + t.Run(stage, func(t *testing.T) { + d, front, _ := recoveryFixture(t, true) + r, _, _ := assetRuntimeFixture(t) + d.resources = func(context.Context) (managedServiceInventory, error) { + return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}, proxy: r.proxy, runner: r.curl}, nil + } + mixed := errors.Join(&managedRecoveryFatal{Err: errors.New("guard failed")}, errCandidateNonpass, &managedResourceFailure{"thumbnail", "i.ytimg.com", nil}) + calls := 0 + d.verifyAssets = func(ctx context.Context, c storedCandidate, i managedServiceInventory) ([]assetMeasurement, error) { + calls++ + if stage == "verify" { + return nil, mixed + } + return r.verify(ctx, c, i) + } + d.frontHealth = func(context.Context) (string, error) { return "DEGRADED", mixed } + err := repairManagedAssets(context.Background(), d, "i.ytimg.com") + var fatal *managedRecoveryFatal + if !errors.As(err, &fatal) || calls != 1 { + t.Fatal("mixed safety failure retried as rejection", calls, err) + } + assertActiveCandidates(t, d.transaction, d.db, []int64{front}) + }) + } +} diff --git a/cmd/zapret-autotune/managed_transaction.go b/cmd/zapret-autotune/managed_transaction.go index 8c84bcf..90c0067 100644 --- a/cmd/zapret-autotune/managed_transaction.go +++ b/cmd/zapret-autotune/managed_transaction.go @@ -30,14 +30,16 @@ func (e *managedRollbackError) Error() string { func (e *managedRollbackError) Unwrap() error { return e.Cause } type managedJournal struct { - Version int `json:"version"` - Generation uint64 `json:"generation"` - ID string `json:"transaction_id"` - State string `json:"state"` - Hash string `json:"committed_hash"` - Next string `json:"pending_hash,omitempty"` - Candidates []int64 `json:"committed_candidate_ids,omitempty"` - PendingCandidates []int64 `json:"pending_candidate_ids,omitempty"` + Version int `json:"version"` + Generation uint64 `json:"generation"` + ID string `json:"transaction_id"` + State string `json:"state"` + Hash string `json:"committed_hash"` + Next string `json:"pending_hash,omitempty"` + Candidates []int64 `json:"committed_candidate_ids,omitempty"` + PendingCandidates []int64 `json:"pending_candidate_ids,omitempty"` + Scope GooglevideoScope `json:"committed_googlevideo_scope,omitempty"` + PendingScope GooglevideoScope `json:"pending_googlevideo_scope,omitempty"` } // Hooks take no commands or arguments. The target adapter binds Reload @@ -189,7 +191,9 @@ func (m *managedTransaction) load() (managedJournal, error) { return err == nil && len(b) == 32 && strings.ToLower(s) == s } validIDs := func(ids []int64) bool { - if len(ids) > 2 { + // Two legacy groups plus at most two disjoint inventory-bound assets. + // Group and scope policy remains in the DB-qualified renderer. + if len(ids) > 4 { return false } seen := map[int64]bool{} @@ -201,7 +205,7 @@ func (m *managedTransaction) load() (managedJournal, error) { } return true } - if (j.Version != 1 && j.Version != 2) || j.ID == "" || !validHash(j.Hash) || (j.State != "committed" && j.State != "prepared") || (j.State == "prepared" && !validHash(j.Next)) || (j.State == "committed" && (j.Next != "" || len(j.PendingCandidates) != 0)) || !validIDs(j.Candidates) || !validIDs(j.PendingCandidates) || (j.Version == 1 && (len(j.Candidates) != 0 || len(j.PendingCandidates) != 0)) { + if (j.Version != 1 && j.Version != 2 && j.Version != 3) || !j.Scope.valid() || !j.PendingScope.valid() || (j.Version < 3 && (j.Scope != "" || j.PendingScope != "")) || (j.Version == 3 && (j.Scope == "" || (j.State == "prepared" && j.PendingScope == ""))) || j.ID == "" || !validHash(j.Hash) || (j.State != "committed" && j.State != "prepared") || (j.State == "prepared" && !validHash(j.Next)) || (j.State == "committed" && (j.Next != "" || len(j.PendingCandidates) != 0 || j.PendingScope != "")) || !validIDs(j.Candidates) || !validIDs(j.PendingCandidates) || (j.Version == 1 && (len(j.Candidates) != 0 || len(j.PendingCandidates) != 0)) { return j, fmt.Errorf("invalid managed journal") } return j, nil @@ -232,13 +236,16 @@ func (m *managedTransaction) checkpoint(point string) error { return nil } -func (m *managedTransaction) Apply(db *sql.DB, ids []int64) error { +func (m *managedTransaction) Apply(db *sql.DB, ids []int64, scopes ...GooglevideoScope) error { ids = append([]int64(nil), ids...) - b, err := renderManagedConfig(db, ids) - if err != nil { - return err + if db == nil || len(scopes) > 1 || (len(scopes) == 1 && (!scopes[0].valid() || scopes[0] == "")) { + return fmt.Errorf("candidate database and optional explicit scope required") } - return m.applyRenderedDB(b, db, ids...) + var scope *GooglevideoScope + if len(scopes) == 1 { + scope = &scopes[0] + } + return m.applyRenderedPolicy(nil, db, scope, ids...) } // ActiveCandidates uses the journal as active-state authority. The DB supplies @@ -269,7 +276,7 @@ func (m *managedTransaction) ActiveCandidates(db *sql.DB) ([]int64, error) { if db == nil { return fmt.Errorf("candidate database required") } - rendered, err := renderManagedConfig(db, j.Candidates) + rendered, err := renderStoredManagedConfigScope(db, j.Candidates, j.Scope) if err != nil { return err } @@ -286,6 +293,9 @@ func (m *managedTransaction) ActiveCandidates(db *sql.DB) ([]int64, error) { func committedManagedJournal(j managedJournal) managedJournal { j.State, j.Hash, j.Next = "committed", j.Next, "" j.Candidates, j.PendingCandidates = j.PendingCandidates, nil + if j.Version == 3 { + j.Scope, j.PendingScope = j.PendingScope, "" + } return j } @@ -306,14 +316,18 @@ func (m *managedTransaction) applyRendered(b []byte, ids ...int64) error { return m.applyRenderedDB(b, nil, ids...) } -func validateManagedRefs(db *sql.DB, ids []int64, hash string) error { +func validateManagedRefs(db *sql.DB, ids []int64, hash string, scopes ...GooglevideoScope) error { if len(ids) == 0 { return nil } if db == nil { return fmt.Errorf("candidate database required") } - b, err := renderManagedConfig(db, ids) + var scope GooglevideoScope + if len(scopes) != 0 { + scope = scopes[0] + } + b, err := renderStoredManagedConfigScope(db, ids, scope) if err != nil || managedHash(b) != hash { return fmt.Errorf("managed candidate state unresolved") } @@ -321,6 +335,10 @@ func validateManagedRefs(db *sql.DB, ids []int64, hash string) error { } func (m *managedTransaction) applyRenderedDB(b []byte, db *sql.DB, ids ...int64) error { + return m.applyRenderedPolicy(b, db, nil, ids...) +} + +func (m *managedTransaction) applyRenderedPolicy(b []byte, db *sql.DB, requested *GooglevideoScope, ids ...int64) error { return m.locked(func() error { j, err := m.load() if err != nil { @@ -329,10 +347,28 @@ func (m *managedTransaction) applyRenderedDB(b []byte, db *sql.DB, ids ...int64) if j.State != "committed" { return fmt.Errorf("incomplete transaction: reconcile first") } - if err := validateManagedRefs(db, j.Candidates, j.Hash); err != nil { + if err := validateManagedRefs(db, j.Candidates, j.Hash, j.Scope); err != nil { return err } - if err := validateManagedRefs(db, ids, managedHash(b)); err != nil { + scope := j.Scope + if scope == "" { + scope = GooglevideoScopeExact + } + if requested != nil { + scope = *requested + } + if b == nil { + b, err = renderManagedConfigScope(db, ids, scope) + if err != nil { + return err + } + } else if len(ids) != 0 { + fresh, err := renderManagedConfigScope(db, ids, scope) + if err != nil || managedHash(fresh) != managedHash(b) { + return fmt.Errorf("fresh qualified candidate state required") + } + } + if err := validateManagedRefs(db, ids, managedHash(b), scope); err != nil { return err } prior, err := m.read(m.Name) @@ -374,7 +410,15 @@ func (m *managedTransaction) applyRenderedDB(b []byte, db *sql.DB, ids ...int64) return err } j.Generation++ - j.Version = 2 + if requested != nil || j.Version == 3 { + j.Version = 3 + if j.Scope == "" { + j.Scope = GooglevideoScopeExact + } + j.PendingScope = scope + } else { + j.Version = 2 + } j.ID, j.State, j.Next = hex.EncodeToString(id), "prepared", managedHash(b) j.PendingCandidates = append([]int64(nil), ids...) if err := m.journal(j); err != nil { @@ -487,6 +531,7 @@ func (m *managedTransaction) restore(j managedJournal) error { } j.State, j.Next = "committed", "" j.PendingCandidates = nil + j.PendingScope = "" return m.journal(j) } @@ -502,10 +547,10 @@ func (m *managedTransaction) Reconcile(databases ...*sql.DB) error { if len(databases) != 0 { db = databases[0] } - if err := validateManagedRefs(db, j.Candidates, j.Hash); err != nil { + if err := validateManagedRefs(db, j.Candidates, j.Hash, j.Scope); err != nil { return err } - if err := validateManagedRefs(db, j.PendingCandidates, j.Next); err != nil { + if err := validateManagedRefs(db, j.PendingCandidates, j.Next, j.PendingScope); err != nil { return err } if j.State == "prepared" { diff --git a/cmd/zapret-autotune/monitor_runtime.go b/cmd/zapret-autotune/monitor_runtime.go index 6369dbc..1b7cc13 100644 --- a/cmd/zapret-autotune/monitor_runtime.go +++ b/cmd/zapret-autotune/monitor_runtime.go @@ -39,6 +39,7 @@ type managedMonitorDeps struct { now func() time.Time wait func(context.Context, time.Duration) error reportRecoveryError func(error) + reportProbeError func(error) } // A startup reconciliation can be consumed once, never during live recovery. @@ -155,6 +156,9 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) { return err } probeErr := d.probe(ctx) + if probeErr != nil && d.reportProbeError != nil { + d.reportProbeError(probeErr) + } var fatalProbe *managedRecoveryFatal if errors.As(probeErr, &fatalProbe) { return probeErr diff --git a/cmd/zapret-autotune/monitor_test.go b/cmd/zapret-autotune/monitor_test.go index 1dacb5e..a2be997 100644 --- a/cmd/zapret-autotune/monitor_test.go +++ b/cmd/zapret-autotune/monitor_test.go @@ -287,6 +287,7 @@ func monitorMigrationSnapshot(t *testing.T, db *sql.DB) string { func TestMonitorV3MigrationPreservesHistory(t *testing.T) { db := testStore(t) saveProposal(t, db, "--lua-desync=fake") + restoreV5CandidateSchema(t, db) if _, err := db.Exec(`INSERT INTO googlevideo_tune_runs (started_at, finished_at, state, error) VALUES ('start', 'finish', 'FAILED', 'original failure'); DROP TABLE monitor_state; PRAGMA user_version = 3;`); err != nil { @@ -299,14 +300,16 @@ func TestMonitorV3MigrationPreservesHistory(t *testing.T) { } } var version, count int - if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 7 { t.Fatalf("version=%d err=%v", version, err) } if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 { t.Fatalf("monitor count=%d err=%v", count, err) } - if after := monitorMigrationSnapshot(t, db); before != after { - t.Fatal("v3 history/candidate data changed") + // The additive v7 candidate column contributes one trailing cold default; + // compare the legacy row projection, not the new field's presence. + if after := strings.ReplaceAll(monitorMigrationSnapshot(t, db), `, "cold"}`, `}`); before != after { + t.Fatalf("v3 history/candidate data changed:\nbefore=%s\nafter=%s", before, after) } } @@ -391,6 +394,7 @@ func TestMonitorCrashReconciliation(t *testing.T) { func TestMonitorV4MigrationPreservesUnverifiableLease(t *testing.T) { db := testStore(t) + restoreV5CandidateSchema(t, db) if _, err := db.Exec("DROP TABLE monitor_state;" + monitorSchema); err != nil { t.Fatal(err) } @@ -408,7 +412,7 @@ func TestMonitorV4MigrationPreservesUnverifiableLease(t *testing.T) { } } var version int - if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 7 { t.Fatalf("version=%d err=%v", version, err) } clear, err := reconcileMonitorRecovery(db, "service", time.Minute, func(int) (int64, error) { diff --git a/cmd/zapret-autotune/render.go b/cmd/zapret-autotune/render.go index fe7d88a..86c0873 100644 --- a/cmd/zapret-autotune/render.go +++ b/cmd/zapret-autotune/render.go @@ -79,7 +79,7 @@ func (c *candidate) validate() error { } func (c *candidate) validateMeasurements() error { - if c.Group != "frontend" && c.Group != "googlevideo" { + if c.Group != "frontend" && c.Group != "googlevideo" && c.Group != "assets" { return fmt.Errorf("unknown group") } if len(c.Domains) == 0 { @@ -90,6 +90,9 @@ func (c *candidate) validateMeasurements() error { return fmt.Errorf("invalid DNS hostname") } c.Domains[i] = strings.ToLower(domain) + if c.Group == "assets" && !assetEndpoint(c.Domains[i]) { + return fmt.Errorf("asset endpoint outside service inventory") + } } sort.Strings(c.Domains) for _, metric := range []float64{c.SuccessRate, c.Coverage, c.MedianLatency, c.MedianSpeed} { @@ -106,6 +109,10 @@ func (c *candidate) validateMeasurements() error { return nil } +func assetEndpoint(host string) bool { + return host == "i.ytimg.com" || host == "yt3.ggpht.com" +} + func (c *candidate) validateArgs() error { // Only shell-inert, unquoted option values can be rendered verbatim. for _, ch := range c.Args { diff --git a/cmd/zapret-autotune/resource_collector.go b/cmd/zapret-autotune/resource_collector.go new file mode 100644 index 0000000..b4d636f --- /dev/null +++ b/cmd/zapret-autotune/resource_collector.go @@ -0,0 +1,373 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "mime" + "net/http" + "net/textproto" + "net/url" + "os/exec" + "regexp" + "strconv" + "strings" + "time" +) + +// stdin is mandatory: ephemeral signed URLs must never enter argv or diagnostics. +type subprocessRunner func(context.Context, []string, io.Reader, io.Writer, io.Writer) error + +func runResourceCurl(ctx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + cmd := exec.CommandContext(ctx, "curl", args...) + cmd.Stdin, cmd.Stdout, cmd.Stderr = stdin, out, diagnostic + return cmd.Run() +} + +type resourceError struct{ cause error } + +func (e *resourceError) PublicError() string { return "resource qualification failed" } +func (e *resourceError) Error() string { return e.PublicError() } +func (e *resourceError) Unwrap() error { return e.cause } + +func resourceFailure(ctx context.Context, err error) error { + if ctx.Err() != nil { + err = errors.Join(err, ctx.Err()) + } + if !managedRecoveryTimeoutSafe(err) { + err = &managedRecoveryFatal{Err: err} + } + return &resourceError{cause: err} +} + +func privateResourceURL(raw, host string) (*url.URL, error) { + if len(raw) == 0 || len(raw) > 16<<10 { + return nil, errors.New("invalid resource URL") + } + for _, c := range raw { + if c <= 32 || c == 127 { + return nil, errors.New("invalid resource URL") + } + } + u, err := url.Parse(raw) + if err != nil || u.Scheme != "https" || u.User != nil || u.Opaque != "" || u.Fragment != "" || u.Hostname() != host || (u.Port() != "" && u.Port() != "443") || strings.HasSuffix(u.Host, ":") || !discoveryHostname(host) { + return nil, errors.New("invalid resource origin") + } + return u, nil +} + +func thumbnailResource(videoID string) (string, error) { + if !videoIDPattern.MatchString(videoID) { + return "", errors.New("invalid video ID") + } + return "https://i.ytimg.com/vi/" + videoID + "/hqdefault.jpg", nil +} + +// Parse a fresh structural player/ytInitialData object, not HTML URL regexes. +// Callers extracting ytInitialData from a WWW page pass that JSON object here. +func avatarResource(metadata []byte) (string, error) { + if len(metadata) > playerResponseLimit { + return "", errors.New("metadata exceeds limit") + } + var root interface{} + if json.Unmarshal(metadata, &root) != nil { + return "", errors.New("invalid metadata") + } + var visit func(interface{}, int) string + visit = func(v interface{}, depth int) string { + if depth > 64 { + return "" + } + switch x := v.(type) { + case map[string]interface{}: + if raw, ok := x["url"].(string); ok { + if _, err := privateResourceURL(raw, "yt3.ggpht.com"); err == nil { + return raw + } + } + for _, child := range x { + if raw := visit(child, depth+1); raw != "" { + return raw + } + } + case []interface{}: + for _, child := range x { + if raw := visit(child, depth+1); raw != "" { + return raw + } + } + } + return "" + } + if raw := visit(root, 0); raw != "" { + return raw, nil + } + return "", errors.New("avatar unavailable") +} + +// The WWW page is transient; decode the embedded JSON structurally and never +// scan HTML for arbitrary URLs to fetch. +func avatarResourceFromPage(page []byte) (string, error) { + if len(page) > 5<<20 { + return "", errors.New("page exceeds limit") + } + for _, marker := range []string{"var ytInitialData = ", "ytInitialData = ", "window[\"ytInitialData\"] = "} { + if i := bytes.Index(page, []byte(marker)); i >= 0 { + var object json.RawMessage + if json.NewDecoder(bytes.NewReader(page[i+len(marker):])).Decode(&object) == nil { + return avatarResource(object) + } + } + } + return "", errors.New("structural avatar metadata unavailable") +} + +var resourceContentRange = regexp.MustCompile(`^bytes 0-([0-9]+)/([0-9]+)$`) + +type mediaResource struct{ url, host, kind string } +type mediaCapability struct { + OrdinaryURL bool + SABRSupported bool +} + +// SABR is deliberately unknown/not supported, never fabricated as readiness. +func playerMediaResource(body []byte) (mediaResource, mediaCapability, error) { + var p struct { + PlayabilityStatus struct { + Status string `json:"status"` + } `json:"playabilityStatus"` + StreamingData struct { + Formats []struct { + URL string `json:"url"` + MimeType string `json:"mimeType"` + } `json:"formats"` + AdaptiveFormats []struct { + URL string `json:"url"` + MimeType string `json:"mimeType"` + } `json:"adaptiveFormats"` + } `json:"streamingData"` + } + if len(body) > playerResponseLimit || json.Unmarshal(body, &p) != nil || p.PlayabilityStatus.Status != "OK" { + return mediaResource{}, mediaCapability{}, errors.New("invalid playable metadata") + } + for _, f := range append(p.StreamingData.Formats, p.StreamingData.AdaptiveFormats...) { + u, err := url.Parse(f.URL) + if err != nil { + continue + } + host := u.Hostname() + if !strings.HasSuffix(host, ".googlevideo.com") { + continue + } + if _, err := privateResourceURL(f.URL, host); err != nil { + continue + } + kind, _, err := mime.ParseMediaType(f.MimeType) + if err == nil && (kind == "video/mp4" || kind == "audio/mp4") { + return mediaResource{f.URL, host, kind}, mediaCapability{OrdinaryURL: true}, nil + } + } + return mediaResource{}, mediaCapability{}, nil +} + +const resourceTrailer = "\nAUTOTUNE_RESOURCE_METRICS:" + +func collectResourceTransfer(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner) (assetTransfer, http.Header, error) { + fail := func(err error) (assetTransfer, http.Header, error) { + return assetTransfer{}, nil, resourceFailure(ctx, err) + } + if !validDiscoveryProxy(proxy) || runner == nil { + return fail(errors.New("invalid owned proxy or runner")) + } + if _, err := privateResourceURL(raw, host); err != nil { + return fail(err) + } + version, tlsFlag, maxTLS := uint16(0x0303), "--tlsv1.2", "1.2" + if protocol == "tls13" { + version, tlsFlag, maxTLS = 0x0304, "--tlsv1.3", "1.3" + } else if protocol != "tls12" { + return fail(errors.New("invalid TLS protocol")) + } + limit := 5 << 20 + if media { + limit = 65536 + } + // Qualification uses HTTP/1.1, not browser HTTP/2. --raw preserves native + // transfer framing so net/http, rather than curl and Go twice, decodes it. + args := []string{"--disable", "--silent", "--show-error", "--verbose", "--http1.1", "--raw", "--noproxy", "", "--proxy", proxy, "--proto", "=https", tlsFlag, "--tls-max", maxTLS, "--max-time", "30", "--max-filesize", strconv.Itoa(limit), "--suppress-connect-headers", "--include", "--write-out", resourceTrailer + "%{json}", "--config", "-"} + if media { + args = append(args, "--range", "0-65535") + } + // curl config quotes are not JSON quotes; reject controls above, escape only these. + config := "url = \"" + strings.NewReplacer("\\", "\\\\", "\"", "\\\"").Replace(raw) + "\"\n" + childCtx, cancel := context.WithTimeout(ctx, 35*time.Second) + defer cancel() + var output, diagnostic bytes.Buffer + out := &playerBoundedWriter{dst: &output, limit: limit + (96 << 10)} + diag := &playerBoundedWriter{dst: &diagnostic, limit: 64 << 10} + err := runner(childCtx, args, strings.NewReader(config), out, diag) + if out.exceeded || diag.exceeded { + err = errors.Join(err, errors.New("resource output exceeds limit")) + } + if err != nil || childCtx.Err() != nil { + return fail(errors.Join(err, childCtx.Err())) + } + i := bytes.LastIndex(output.Bytes(), []byte(resourceTrailer)) + if i < 0 { + return fail(errors.New("missing transfer metrics")) + } + var metrics struct { + URL string `json:"url_effective"` + Verify *int `json:"ssl_verify_result"` + Redirects *int `json:"num_redirects"` + Status int `json:"http_code"` + Time float64 `json:"time_total"` + Speed float64 `json:"speed_download"` + Size float64 `json:"size_download"` + } + if json.Unmarshal(output.Bytes()[i+len(resourceTrailer):], &metrics) != nil { + return fail(errors.New("invalid transfer metrics")) + } + if _, err := privateResourceURL(metrics.URL, host); err != nil || metrics.URL != raw || metrics.Verify == nil || *metrics.Verify != 0 || metrics.Redirects == nil || *metrics.Redirects != 0 { + return fail(errors.New("resource origin verification failed")) + } + actualTLS := "SSL connection using TLSv1.2" + if version == 0x0304 { + actualTLS = "SSL connection using TLSv1.3" + } + otherTLS := "SSL connection using TLSv1.3" + if version == 0x0304 { + otherTLS = "SSL connection using TLSv1.2" + } + if !strings.Contains(diagnostic.String(), actualTLS) || strings.Contains(diagnostic.String(), otherTLS) { + return fail(errors.New("actual TLS version mismatch")) + } + wire := output.Bytes()[:i] + headerEnd := bytes.Index(wire, []byte("\r\n\r\n")) + statusEnd := bytes.Index(wire, []byte("\r\n")) + if headerEnd < 0 || headerEnd > 64<<10 || statusEnd < 0 { + return fail(errors.New("missing or oversized response headers")) + } + h, err := textproto.NewReader(bufio.NewReader(bytes.NewReader(wire[statusEnd+2 : headerEnd+4]))).ReadMIMEHeader() + if err != nil { + return fail(errors.New("invalid response headers")) + } + for _, key := range []string{"Content-Type", "Content-Length", "Transfer-Encoding", "Content-Range"} { + if len(h.Values(key)) > 1 { + return fail(errors.New("duplicate response framing headers")) + } + } + if h.Get("Transfer-Encoding") != "" && (h.Get("Transfer-Encoding") != "chunked" || h.Get("Content-Length") != "") { + return fail(errors.New("ambiguous response framing")) + } + r := bufio.NewReader(bytes.NewReader(wire)) + response, err := http.ReadResponse(r, nil) + if err != nil { + return fail(errors.New("invalid response headers")) + } + defer response.Body.Close() + // CONNECT headers are suppressed by curl. Interim/duplicate responses and + // unrequested versions are outside this contract, never heuristically skipped. + if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || response.Header.Get("Content-Type") == "" { + return fail(errors.New("unsupported response contract")) + } + body, err := io.ReadAll(io.LimitReader(response.Body, int64(limit+1))) + // With --raw curl counts the wire body, including chunk framing; the image + // limit and durable byte identity apply to the fully decoded payload instead. + wireSize := len(wire) - headerEnd - 4 + _, remaining := r.Peek(1) + if err != nil || remaining != io.EOF || len(body) == 0 || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status { + return fail(errors.New("incomplete or oversized resource body")) + } + for _, n := range []float64{metrics.Time, metrics.Speed} { + if n <= 0 || math.IsNaN(n) || math.IsInf(n, 0) { + return fail(errors.New("invalid resource metrics")) + } + } + contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type")) + if err != nil { + return fail(errors.New("invalid content type")) + } + t := assetTransfer{EffectiveHost: host, SSLVerifyResult: metrics.Verify, Redirects: *metrics.Redirects, TLSVersion: version, Status: metrics.Status, ContentType: contentType, Body: body, LatencyMS: metrics.Time * 1000, SpeedKBPS: metrics.Speed / 1024} + return t, response.Header, nil +} + +// The ordinary 3/3 sampler is shared by image and media qualification. +func sampleResource(ctx context.Context, transfer func(context.Context) (assetTransfer, error)) ([]assetTransfer, error) { + var samples []assetTransfer + for i := 0; i < 3; i++ { + t, err := transfer(ctx) + if err != nil { + return nil, err + } + samples = append(samples, t) + } + return samples, nil +} + +func collectAssetMeasurement(ctx context.Context, proxy, raw, host, protocol, args string, runner subprocessRunner) (assetMeasurement, error) { + m := assetMeasurement{Source: "image-transfer", Endpoint: host, Protocol: protocol, Args: args} + if !assetEndpoint(host) { + return m, resourceFailure(ctx, errors.New("invalid image endpoint")) + } + samples, err := sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) { + t, _, err := collectResourceTransfer(ctx, proxy, raw, host, protocol, false, runner) + if err == nil { + if t.Status != 200 { + err = errors.New("image HTTP status") + } else { + err = normalizeAssetBody(&t) + } + } + if err != nil { + return assetTransfer{}, resourceFailure(ctx, err) + } + t.Body = nil + return t, nil + }) + m.Transfers = samples + return m, err +} + +func collectMediaSamples(ctx context.Context, proxy string, resource mediaResource, protocol string, runner subprocessRunner) ([]assetTransfer, error) { + if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") { + return nil, resourceFailure(ctx, errors.New("ordinary media unavailable")) + } + return sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) { + t, _, err := collectMediaTransfer(ctx, proxy, resource, protocol, runner) + return t, err + }) +} + +// Only normalized range facts leave this collector; URLs, headers and payloads +// remain transient. Used by both cold sampling and post-warmup sampling. +func collectMediaTransfer(ctx context.Context, proxy string, resource mediaResource, protocol string, runner subprocessRunner) (assetTransfer, GooglevideoPass, error) { + if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") { + return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("ordinary media unavailable")) + } + t, h, err := collectResourceTransfer(ctx, proxy, resource.url, resource.host, protocol, true, runner) + if err != nil { + return assetTransfer{}, GooglevideoPass{}, err + } + parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range")) + var end, total int64 + if len(parts) == 3 { + end, _ = strconv.ParseInt(parts[1], 10, 64) + total, _ = strconv.ParseInt(parts[2], 10, 64) + } + if t.Status != 206 || t.ContentType != resource.kind || len(parts) != 3 || end < 15 || end > 65535 || end+1 != int64(len(t.Body)) || total <= end || len(t.Body) < 16 || string(t.Body[4:8]) != "ftyp" || binary.BigEndian.Uint32(t.Body[:4]) < 16 || int(binary.BigEndian.Uint32(t.Body[:4])) > len(t.Body) { + return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("invalid media range or container")) + } + // Retain only normalized byte identity; no signed URL or payload. + t.Size, t.Format, t.BodyHash = len(t.Body), "mp4", fmt.Sprintf("%x", sha256.Sum256(t.Body)) + t.Body = nil + p := GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: t.Status, RangeStart: 0, RangeEnd: end, ResourceBytes: total, Bytes: int64(t.Size), FTYP: true, LatencyMS: t.LatencyMS} + return t, p, nil +} diff --git a/cmd/zapret-autotune/resource_collector_test.go b/cmd/zapret-autotune/resource_collector_test.go new file mode 100644 index 0000000..5f2d910 --- /dev/null +++ b/cmd/zapret-autotune/resource_collector_test.go @@ -0,0 +1,310 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "image" + "image/png" + "io" + "os" + "os/exec" + "strconv" + "strings" + "testing" +) + +func resourceFixture() []byte { + var b bytes.Buffer + if err := png.Encode(&b, image.NewRGBA(image.Rect(0, 0, 1, 1))); err != nil { + panic(err) + } + return b.Bytes() +} + +func resourceWire(raw, mode string, out, diagnostic io.Writer) { + body, contentType, status := resourceFixture(), "image/png", 200 + header := "" + if mode == "media" || mode == "bad-range" { + body = []byte{0, 0, 0, 24, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm', 0, 0, 0, 0, 'i', 's', 'o', 'm', 'm', 'p', '4', '2'} + contentType, status, header = "video/mp4", 206, "Content-Range: bytes 0-23/100\r\n" + if mode == "bad-range" { + header = "Content-Range: bytes 1-24/100\r\n" + } + } + verify, redirects, effective := 0, 0, raw + switch mode { + case "root": + body, contentType, status = []byte("not found"), "text/html", 404 + case "html": + body, contentType, status = []byte("bad request"), "text/html", 400 + case "cn": + verify = 60 + case "redirect": + status, redirects = 302, 1 + case "host": + effective = "https://other.ytimg.com/resource" + case "truncated": + body = body[:len(body)/2] + case "type": + contentType = "image/jpeg" + } + fmt.Fprintf(out, "HTTP/1.1 %d result\r\nContent-Type: %s\r\nContent-Length: %d\r\n%s\r\n", status, contentType, len(body), header) + out.Write(body) + metrics, _ := json.Marshal(map[string]interface{}{"url_effective": effective, "ssl_verify_result": verify, "num_redirects": redirects, "http_code": status, "time_total": 0.1, "speed_download": 10000, "size_download": len(body)}) + fmt.Fprint(out, resourceTrailer, string(metrics)) + fmt.Fprintln(diagnostic, "* SSL connection using TLSv1.2 / test") +} + +// A real child exercises the stdin/argv protocol, without any socket or curl. +func TestResourceChild(t *testing.T) { + if os.Getenv("AUTOTUNE_RESOURCE_CHILD") != "1" { + return + } + b, err := io.ReadAll(io.LimitReader(os.Stdin, 32<<10)) + if err != nil { + os.Exit(3) + } + raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(b), "url = "), "\n")) + if err != nil || strings.ContainsAny(raw, "\r\n") { + os.Exit(4) + } + resourceWire(raw, "", os.Stdout, os.Stderr) + os.Exit(0) +} + +func TestResourcePrivateChildAndEvidence(t *testing.T) { + raw := "https://yt3.ggpht.com/avatar?signature=private-secret"e=\"&slash=\\" + calls := 0 + runner := func(ctx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + calls++ + joined := strings.Join(args, " ") + if strings.Contains(joined, "private-secret") || strings.Contains(joined, "https://") || strings.Contains(joined, "-k") || args[0] != "--disable" { + t.Fatalf("unsafe argv: %v", args) + } + if !strings.Contains(joined, "--noproxy --proxy socks5://127.0.0.1:1080") || !strings.Contains(joined, "--tlsv1.2 --tls-max 1.2") { + t.Fatal("missing explicit routing/TLS") + } + if !strings.Contains(joined, "--http1.1 --raw") || !strings.Contains(joined, "--suppress-connect-headers") { + t.Fatal("missing native HTTP/1.1 raw contract") + } + cmd := exec.CommandContext(ctx, os.Args[0], append([]string{"-test.run=^TestResourceChild$", "--"}, args...)...) + cmd.Env = append(os.Environ(), "AUTOTUNE_RESOURCE_CHILD=1") + cmd.Stdin, cmd.Stdout, cmd.Stderr = stdin, out, diagnostic + return cmd.Run() + } + m, err := collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", raw, "yt3.ggpht.com", "tls12", "--lua-desync=fake", runner) + if err != nil || calls != 3 { + t.Fatalf("calls=%d err=%v", calls, err) + } + if _, _, err := validateAssetTransfers(m); err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(m) + if strings.Contains(string(b), "private-secret") || strings.Contains(string(b), "body\"") || len(m.Transfers[0].Body) != 0 || m.Transfers[0].Size != len(resourceFixture()) { + t.Fatal("durable evidence leaks payload or misses byte identity") + } +} + +func TestResourceNativeFraming(t *testing.T) { + raw := "https://yt3.ggpht.com/avatar?signature=private-secret" + for _, mode := range []string{"length", "chunked", "close", "normalized-chunked", "h2", "empty", "truncated-header", "truncated-length", "truncated-chunk", "duplicate-length", "duplicate-type", "duplicate-transfer", "mixed-framing", "interim", "duplicate-response", "extra-bytes"} { + t.Run(mode, func(t *testing.T) { + payload := resourceFixture() + // A delimiter inside a body must not steal the final private footer. + payload = append(payload, []byte(resourceTrailer+`{"url_effective":"private-body-marker"}`)...) + header := fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n", len(payload)) + body := payload + switch mode { + case "chunked", "normalized-chunked", "truncated-chunk": + header = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nTransfer-Encoding: chunked\r\n\r\n" + if mode != "normalized-chunked" { + var wire bytes.Buffer + fmt.Fprintf(&wire, "%x\r\n", len(payload[:20])) + wire.Write(payload[:20]) + fmt.Fprint(&wire, "\r\n") + fmt.Fprintf(&wire, "%x\r\n", len(payload[20:])) + wire.Write(payload[20:]) + fmt.Fprint(&wire, "\r\n0\r\n\r\n") + body = wire.Bytes() + if mode == "truncated-chunk" { + body = body[:len(body)-5] + } + } + case "close": + header = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nConnection: close\r\n\r\n" + case "h2": + header = strings.Replace(header, "HTTP/1.1 200 OK", "HTTP/2 200", 1) + case "empty": + header, body = "", nil + case "truncated-header": + header = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\n" + case "truncated-length": + body = payload[:len(payload)-1] + case "duplicate-length": + header = strings.Replace(header, "Content-Length:", fmt.Sprintf("Content-Length: %d\r\nContent-Length:", len(payload)), 1) + case "duplicate-type": + header = strings.Replace(header, "Content-Type:", "Content-Type: image/png\r\nContent-Type:", 1) + case "duplicate-transfer": + header = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nTransfer-Encoding: chunked\r\nTransfer-Encoding: chunked\r\n\r\n" + case "mixed-framing": + header = strings.Replace(header, "Content-Length:", "Transfer-Encoding: chunked\r\nContent-Length:", 1) + case "interim": + header = "HTTP/1.1 100 Continue\r\n\r\n" + header + case "duplicate-response": + header = "HTTP/1.1 200 Connection established\r\n\r\n" + header + case "extra-bytes": + body = append(append([]byte(nil), payload...), []byte("unframed bytes")...) + } + runner := func(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error { + if !strings.Contains(strings.Join(args, " "), "--http1.1 --raw") { + t.Fatal("HTTP version/framing not pinned") + } + fmt.Fprint(out, header) + out.Write(body) + metrics, _ := json.Marshal(map[string]interface{}{"url_effective": raw, "ssl_verify_result": 0, "num_redirects": 0, "http_code": 200, "time_total": 0.1, "speed_download": 10000, "size_download": len(body)}) + fmt.Fprint(out, resourceTrailer, string(metrics)) + fmt.Fprintln(diagnostic, "SSL connection using TLSv1.2") + return nil + } + m, err := collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", raw, "yt3.ggpht.com", "tls12", "", runner) + valid := mode == "length" || mode == "chunked" || mode == "close" + if (err == nil) != valid { + t.Fatalf("valid=%v error=%v", valid, err) + } + if err != nil && strings.Contains(err.Error(), "private-secret") { + t.Fatal("private footer leaked") + } + if valid && (len(m.Transfers) != 3 || m.Transfers[0].Size != len(payload)) { + t.Fatal("wrong decoded body identity") + } + }) + } +} + +func TestResourceRejectionsAndCancellation(t *testing.T) { + for _, mode := range []string{"root", "html", "cn", "redirect", "host", "truncated", "type", "tls", "overflow", "unknown", "mixed", "cancel"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + runner := func(_ context.Context, _ []string, stdin io.Reader, out, diagnostic io.Writer) error { + if mode == "cancel" { + cancel() + return context.Canceled + } + if mode == "unknown" { + return errors.New("private-secret") + } + if mode == "mixed" { + cancel() + return errors.Join(context.Canceled, errors.New("private-secret")) + } + if mode == "overflow" { + _, err := out.Write(make([]byte, 6<<20)) + return err + } + resourceWire("https://yt3.ggpht.com/avatar?signature=private-secret", mode, out, diagnostic) + if mode == "tls" { + fmt.Fprintln(diagnostic, "SSL connection using TLSv1.3") + } + return nil + } + _, err := collectAssetMeasurement(ctx, "socks5://127.0.0.1:1080", "https://yt3.ggpht.com/avatar?signature=private-secret", "yt3.ggpht.com", "tls12", "", runner) + if err == nil || strings.Contains(err.Error(), "private-secret") { + t.Fatal("accepted failure or exposed secret", err) + } + var fatal *managedRecoveryFatal + if errors.As(err, &fatal) != (mode != "cancel") { + t.Fatal("unsafe cancellation classification", err) + } + if (mode == "cancel" || mode == "mixed") && !errors.Is(err, context.Canceled) { + t.Fatal("lost cancellation") + } + }) + } +} + +func TestResourceDiscoveryAndMedia(t *testing.T) { + if raw, err := avatarResourceFromPage([]byte(``)); err != nil || raw != "https://yt3.ggpht.com/fresh" { + t.Fatal("WWW structural parsing", err) + } + for _, raw := range []string{"http://yt3.ggpht.com/a", "https://yt3.ggpht.com.evil/a", "https://user@yt3.ggpht.com/a", "https://yt3.ggpht.com:444/a", "https://yt3.ggpht.com/a\nurl=bad"} { + if _, err := privateResourceURL(raw, "yt3.ggpht.com"); err == nil { + t.Fatal("accepted unsafe URL", raw) + } + } + if _, err := thumbnailResource("invalid"); err == nil { + t.Fatal("invalid thumbnail ID") + } + if raw, err := avatarResource([]byte(`{"owner":{"thumbnail":{"thumbnails":[{"url":"https://yt3.ggpht.com/fresh"}]}}}`)); err != nil || raw != "https://yt3.ggpht.com/fresh" { + t.Fatal("avatar parsing", err) + } + resource, capability, err := playerMediaResource([]byte(`{"playabilityStatus":{"status":"OK"},"streamingData":{"adaptiveFormats":[{"url":"https://rr1.unit.googlevideo.com/videoplayback?sig=private-secret","mimeType":"video/mp4; codecs=\"avc1\""}]}}`)) + if err != nil || !capability.OrdinaryURL || capability.SABRSupported { + t.Fatal("media capability", err) + } + for _, mode := range []string{"media", "bad-range", "root"} { + runner := func(_ context.Context, args []string, _ io.Reader, out, diagnostic io.Writer) error { + if !strings.Contains(strings.Join(args, " "), "--range 0-65535") { + t.Fatal("missing bounded range") + } + resourceWire(resource.url, mode, out, diagnostic) + return nil + } + samples, err := collectMediaSamples(context.Background(), "socks5://127.0.0.1:1080", resource, "tls12", runner) + if (err == nil) != (mode == "media") { + t.Fatal(mode, err) + } + if err == nil && (len(samples) != 3 || samples[0].Body != nil || samples[0].Format != "mp4") { + t.Fatal("invalid normalized media samples") + } + } + _, capability, err = playerMediaResource([]byte(`{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://rr1.unit.googlevideo.com/sabr?sig=private"}}`)) + if err != nil || capability.OrdinaryURL || capability.SABRSupported { + t.Fatal("SABR falsely green") + } +} + +func TestResourceTLS13AndStrictDTO(t *testing.T) { + raw := "https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg" + runner := func(_ context.Context, args []string, _ io.Reader, out, diagnostic io.Writer) error { + if !strings.Contains(strings.Join(args, " "), "--tlsv1.3 --tls-max 1.3") { + t.Fatal("TLS13 not pinned") + } + var private bytes.Buffer + resourceWire(raw, "", out, &private) + fmt.Fprintln(diagnostic, "* SSL connection using TLSv1.3 / test") + return nil + } + m, err := collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", raw, "i.ytimg.com", "tls13", "", runner) + if err != nil { + t.Fatal(err) + } + if _, _, err := validateAssetTransfers(m); err != nil { + t.Fatal(err) + } + encoded, _ := json.Marshal(m.Transfers[0]) + var fields map[string]json.RawMessage + json.Unmarshal(encoded, &fields) + for key := range fields { + copy := make(map[string]json.RawMessage) + for k, v := range fields { + if k != key { + copy[k] = v + } + } + b, _ := json.Marshal(copy) + var transfer assetTransfer + if json.Unmarshal(b, &transfer) == nil { + t.Fatal("missing required field accepted", key) + } + } + fields["url"] = json.RawMessage(`"https://signed.invalid/?signature=private"`) + b, _ := json.Marshal(fields) + var transfer assetTransfer + if json.Unmarshal(b, &transfer) == nil { + t.Fatal("unknown private field accepted") + } +} diff --git a/cmd/zapret-autotune/store.go b/cmd/zapret-autotune/store.go index 113db2d..dfa96c8 100644 --- a/cmd/zapret-autotune/store.go +++ b/cmd/zapret-autotune/store.go @@ -114,7 +114,7 @@ func migrate(db *sql.DB) error { if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil { return err } - if version > 5 { + if version > 7 { return fmt.Errorf("unsupported database schema version %d", version) } if version == 0 { @@ -241,6 +241,16 @@ PRAGMA user_version = 1;`) return err } } + if version < 6 { + if err := migrateAssetCandidates(ctx, conn); err != nil { + return err + } + } + if version < 7 { + if _, err := conn.ExecContext(ctx, googlevideoQualificationSchema); err != nil { + return err + } + } _, err = conn.ExecContext(ctx, "COMMIT") return err } diff --git a/cmd/zapret-autotune/store_test.go b/cmd/zapret-autotune/store_test.go index e9efe61..04169e3 100644 --- a/cmd/zapret-autotune/store_test.go +++ b/cmd/zapret-autotune/store_test.go @@ -224,7 +224,7 @@ func TestConcurrentFirstOpens(t *testing.T) { } for _, db := range stores { var version, objects int - if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 7 { t.Fatalf("version=%d err=%v", version, err) } if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE @@ -377,7 +377,7 @@ func TestSchemaConstraintsAndIncompleteEvidence(t *testing.T) { func TestUnsupportedSchemaVersion(t *testing.T) { db := testStore(t) - if _, err := db.Exec("PRAGMA user_version = 6"); err != nil { + if _, err := db.Exec("PRAGMA user_version = 8"); err != nil { t.Fatal(err) } if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") { diff --git a/cmd/zapret-autotune/target_config_test.go b/cmd/zapret-autotune/target_config_test.go new file mode 100644 index 0000000..a3d209d --- /dev/null +++ b/cmd/zapret-autotune/target_config_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "bytes" + "strings" + "testing" +) + +func TestTargetConfigCheck(t *testing.T) { + t.Setenv("ZAPRET_AUTOTUNE_YOUTUBE_SERVICE", "0") + t.Setenv("ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE", "cold") + t.Setenv("ZAPRET_AUTOTUNE_VIDEO_ID", "dQw4w9WgXcQ") + t.Setenv("ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS", `["dQw4w9WgXcQ"]`) + t.Setenv("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5://127.0.0.1:1080") + for _, dns := range []string{"", "8.8.8.8", "2001:4860:4860::8888", "::1", "resolver.example", "8.8.8.8:53", "[::1]", "fe80::1%eth0", "-d 8.8.8.8", "8.8.8.8\n", "\x00", "1.2.3.999"} { + t.Run(strings.ReplaceAll(dns, "\x00", "NUL"), func(t *testing.T) { + t.Setenv("SS_DNS_SERVER", strings.ReplaceAll(dns, "\x00", "\r")) // Env cannot contain NUL. + var out, errOut bytes.Buffer + code := run([]string{"target-config-check"}, &out, &errOut) + valid := dns == "" || dns == "8.8.8.8" || dns == "2001:4860:4860::8888" || dns == "::1" + if (code == 0) != valid || out.Len() != 0 || (!valid && !strings.HasPrefix(errOut.String(), "configerror:")) { + t.Fatalf("code=%d stdout=%q stderr=%q", code, out.String(), errOut.String()) + } + }) + } + t.Setenv("SS_DNS_SERVER", "") + for key, value := range map[string]string{ + "ZAPRET_AUTOTUNE_YOUTUBE_SERVICE": "true", + "ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE": "adaptive", + "ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS": `[]`, + } { + t.Run(key, func(t *testing.T) { + t.Setenv(key, value) + var out, errOut bytes.Buffer + if run([]string{"target-config-check"}, &out, &errOut) != 64 || out.Len() != 0 || !strings.HasPrefix(errOut.String(), "configerror:") { + t.Fatal(out.String(), errOut.String()) + } + }) + } +} diff --git a/deploy/managed-target/.env.example b/deploy/managed-target/.env.example index f9292a9..3c46dd9 100644 --- a/deploy/managed-target/.env.example +++ b/deploy/managed-target/.env.example @@ -1 +1,9 @@ ZAPRET_AUTOTUNE_HOST_SOCKS_PORT=11080 +# Legacy Player API/exact-host behavior remains the default. +ZAPRET_AUTOTUNE_YOUTUBE_SERVICE=0 +ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE=cold +ZAPRET_AUTOTUNE_VIDEO_ID=dQw4w9WgXcQ +# Optional JSON array: 1..5 unique 11-character VideoIDs (not signed URLs). +# ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS='["dQw4w9WgXcQ","aqz-KE-bpKQ","jNQXAC9IVRw"]' +# Optional target ss-server native DNS; empty retains the base resolver behavior. +SS_DNS_SERVER= diff --git a/deploy/managed-target/README.md b/deploy/managed-target/README.md index 835c2f3..61a25f7 100644 --- a/deploy/managed-target/README.md +++ b/deploy/managed-target/README.md @@ -40,11 +40,45 @@ with a tag or local default. Offline static guard (from repository root): `python3 deploy/managed-target/check_static.py`. +Offline real pre-launch validation and private Compose forwarding fixtures: +`python3 deploy/managed-target/check_config.py` (requires local Go and Compose). Isolated, accelerated shell supervision fixtures (no network): `python3 deploy/managed-target/check_supervision.py`. ## In-container supervision +### Opt-in service/adaptive and target-native DNS + +Compose forwards `ZAPRET_AUTOTUNE_YOUTUBE_SERVICE` (`0` or `1`, default `0`), +`ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE` (`cold` or `adaptive`, default `cold`), +`ZAPRET_AUTOTUNE_VIDEO_ID` (default `dQw4w9WgXcQ`) and +`ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS` (JSON array of 1..5 unique VideoIDs; +default only the primary ID). Explicit empty/invalid settings are rejected, +not silently replaced. Adaptive requires service `1`. See `.env.example`. +Service `0` preserves legacy Player API/exact-host behavior. + +Adaptive qualification is **post-warmup**, distinct from cold qualification; +initial connections may fail and native learning must repeat after restart. +It requires at least **three real unique signed ordinary media host samples**. +Three sample videos may all resolve to the same CDN host: that means +`DEGRADED`, no authorized googlevideo suffix, not fabricated coverage. Service +asset checks cover sampled thumbnail/avatar routes, not the whole thumbnail pool. +The current service/adaptive hardware acceptance gate is **pending**; the old +clean-run PASS below does not qualify these new modes. + +Optional `SS_DNS_SERVER` is an unbracketed literal IPv4/IPv6 address only (for +example `8.8.8.8` or `2001:4860:4860::8888`); empty preserves existing resolver +behavior. No domain, port, zone, control characters or option strings are +accepted. Validation runs before service launch and reports opaque `configerror` +without echoing input. DNS applies only to this target's native `ss-server`. +The image build checks installed `ss-server -h` for native `-d ` support; +the validated nonsecret address is passed as a separate `-d` argv value, not +eval or an assumed JSON key. Passwords still go only to private JSON configs. +The reported `8.8.8.8` diagnostic corrected a tampered avatar response, but does +not fix/prove all thumbnail routes. Native DNS correction is **not full asset +availability proof**. No DoH bridge, permanent Google IP pins or global host DNS +changes are supplied. PID 1 remains the sole startup/recovery owner. + Compose has no automatic restart. PID 1 restarts the entire owned runtime only for technical failures, with three starts per rolling six hours (including initial startup) and cancellable 15/30-second cooldowns. Clean typed recovery timeouts, diff --git a/deploy/managed-target/check_config.py b/deploy/managed-target/check_config.py new file mode 100644 index 0000000..21634e3 --- /dev/null +++ b/deploy/managed-target/check_config.py @@ -0,0 +1,65 @@ +"""Offline entrypoint/config fixtures: python3 deploy/managed-target/check_config.py.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +env = dict(os.environ, GOPROXY='off', GOTOOLCHAIN='local') +with tempfile.TemporaryDirectory(prefix='target-config-') as tmp: + root = Path(tmp) + binary = root / 'autotune' + subprocess.run(['go', 'build', '-o', str(binary), './cmd/zapret-autotune'], + cwd=ROOT, env=env, check=True) + private = root / 'private' + private.mkdir() + # Exercise only the real entrypoint's pre-launch region; no services/network. + script = (ROOT / 'managed-target-entrypoint.sh').read_text().split('\nserver_pid=')[0] + script = script.replace('/usr/local/bin/zapret-autotune', str(binary)) + script = script.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={private}') + start = script.index("awk '$2 ==") + end = script.index('chmod 0700 "$config_dir"', start) + script = script[:start] + script[end:] + entry = root / 'entry.sh' + entry.write_text(script) + base = dict(env, SS_PASSWORD='ab' * 32, + ZAPRET_AUTOTUNE_YOUTUBE_SERVICE='0', ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE='cold', + ZAPRET_AUTOTUNE_VIDEO_ID='dQw4w9WgXcQ', + ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS='["dQw4w9WgXcQ"]', + ZAPRET_AUTOTUNE_SOCKS_PROXY='socks5://127.0.0.1:1080') + for dns, valid in (('', True), ('8.8.8.8', True), ('2001:4860:4860::8888', True), + ('example.org', False), ('8.8.8.8\n', False), ('-d 8.8.8.8', False)): + for path in private.iterdir(): + path.unlink() + result = subprocess.run(['sh', str(entry)], env=dict(base, SS_DNS_SERVER=dns), + capture_output=True, timeout=5) + assert result.returncode == (0 if valid else 64), result.stderr + assert not result.stdout and base['SS_PASSWORD'].encode() not in result.stderr + if not valid: + assert b'configerror:' in result.stderr and not list(private.iterdir()) + else: + for name in ('server', 'local'): + path = private / (name + '.json') + assert path.stat().st_mode & 0o777 == 0o600 + assert json.loads(path.read_text())['password'] == base['SS_PASSWORD'] + # Render privately, never print the config/password; config does not deploy. + compose_env = {k: v for k, v in base.items() if not k.startswith('ZAPRET_AUTOTUNE_')} + for overrides, service, mode, ids in ( + ({}, '0', 'cold', ['dQw4w9WgXcQ']), + ({'ZAPRET_AUTOTUNE_VIDEO_ID': 'aqz-KE-bpKQ'}, '0', 'cold', ['aqz-KE-bpKQ']), + ({'ZAPRET_AUTOTUNE_YOUTUBE_SERVICE': '1', 'ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE': 'adaptive', + 'ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS': '["dQw4w9WgXcQ","aqz-KE-bpKQ","jNQXAC9IVRw"]', + 'SS_DNS_SERVER': '8.8.8.8'}, '1', 'adaptive', ['dQw4w9WgXcQ', 'aqz-KE-bpKQ', 'jNQXAC9IVRw']), + ): + result = subprocess.run(['docker', 'compose', '--env-file', '/dev/null', '-f', + str(ROOT / 'deploy/managed-target/compose.yaml'), + 'config', '--format', 'json'], env=dict(compose_env, **overrides), + capture_output=True, timeout=10) + assert result.returncode == 0, 'Compose config failed (output suppressed)' + settings = json.loads(result.stdout)['services']['target']['environment'] + assert settings['ZAPRET_AUTOTUNE_YOUTUBE_SERVICE'] == service + assert settings['ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE'] == mode + assert json.loads(settings['ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS']) == ids + assert settings['SS_DNS_SERVER'] == overrides.get('SS_DNS_SERVER', '') +print('target config fixtures passed') diff --git a/deploy/managed-target/check_static.py b/deploy/managed-target/check_static.py index d67d398..4f842af 100644 --- a/deploy/managed-target/check_static.py +++ b/deploy/managed-target/check_static.py @@ -19,6 +19,17 @@ assert " source: ./target-state\n" in compose assert " target: /var/lib/zapret-autotune/target\n" in compose assert " create_host_path: false\n" in compose assert "SS_PASSWORD: ${SS_PASSWORD:?Set SS_PASSWORD in the deployment environment}" in compose +for setting in ( + 'SS_DNS_SERVER: ${SS_DNS_SERVER-}', + 'ZAPRET_AUTOTUNE_YOUTUBE_SERVICE: ${ZAPRET_AUTOTUNE_YOUTUBE_SERVICE-0}', + 'ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE: ${ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE-cold}', + 'ZAPRET_AUTOTUNE_VIDEO_ID: ${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}', + "ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS: '${ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS-[\"${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}\"]}'", +): + assert setting in compose, setting +assert entrypoint.index('target-config-check || exit 64') < entrypoint.index('cat > "$config_dir/server.json"') +assert 'set -- "$@" -d "$SS_DNS_SERVER"' in entrypoint +assert 'eval ' not in entrypoint assert "secrets:" not in compose assert "ss_password=${SS_PASSWORD-}" in entrypoint assert "unset SS_PASSWORD" in entrypoint diff --git a/deploy/managed-target/check_supervision.py b/deploy/managed-target/check_supervision.py index 61816f1..07ad5b5 100644 --- a/deploy/managed-target/check_supervision.py +++ b/deploy/managed-target/check_supervision.py @@ -73,7 +73,9 @@ if name in ('ss-server', 'ss-local'): with (root / 'children').open('a') as f: f.write(str(os.getpid()) + '\n') assert 'SS_PASSWORD' not in os.environ - assert sys.argv[1] == '-c' and len(sys.argv) == 3 + assert sys.argv[1] == '-c' + dns = os.environ.get('SS_DNS_SERVER', '') if name == 'ss-server' else '' + assert sys.argv[3:] == (['-d', dns] if dns else []) assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600 log(name) if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail') and increment('ss_runs') == 1: @@ -92,6 +94,8 @@ if name == 'zapret2': sys.exit(1) sys.exit(0) assert name == 'autotune' +if sys.argv[1:] == ['target-config-check']: + sys.exit(0) command = sys.argv[2] if command == 'init': log('init') @@ -140,7 +144,7 @@ elif command == 'monitor': ''' -def run(scenario, expected): +def run(scenario, expected, dns=''): with tempfile.TemporaryDirectory(prefix='supervision-') as tmp: root = Path(tmp) binary = root / 'bin' @@ -165,7 +169,8 @@ def run(scenario, expected): entry = root / 'entry.sh' entry.write_text(text) env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp, - SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config')) + SCENARIO=scenario, SS_PASSWORD='ab' * 32, SS_DNS_SERVER=dns, + ZAPRET_CONFIG=str(root / 'config')) peer = None if scenario == 'orphans': # Local fixture is not PID 1. Adopt orphan workers here so the test @@ -224,6 +229,9 @@ events = run('cancel', 0) assert events.count('start') == 1 events = run('prestartfail', 78) assert 'start' not in events and 'stop' not in events +for dns in ('8.8.8.8', '2001:4860:4860::8888'): + events = run('fatal64', 64, dns) + assert events.count('ss-server') == 1 and events.count('ss-local') == 1 def check_watchdog_traps(shell): diff --git a/deploy/managed-target/compose.yaml b/deploy/managed-target/compose.yaml index 30b398c..2089231 100644 --- a/deploy/managed-target/compose.yaml +++ b/deploy/managed-target/compose.yaml @@ -11,6 +11,11 @@ services: cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID, KILL] environment: SS_PASSWORD: ${SS_PASSWORD:?Set SS_PASSWORD in the deployment environment} + SS_DNS_SERVER: ${SS_DNS_SERVER-} + ZAPRET_AUTOTUNE_YOUTUBE_SERVICE: ${ZAPRET_AUTOTUNE_YOUTUBE_SERVICE-0} + ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE: ${ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE-cold} + ZAPRET_AUTOTUNE_VIDEO_ID: ${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ} + ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS: '${ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS-["${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}"]}' SOCKS_PORT: "1080" ZAPRET_AUTOTUNE_TARGET_ROLE: managed-test ZAPRET_AUTOTUNE_TARGET_ID: zapret-autotune-managed-v1 diff --git a/managed-target-entrypoint.sh b/managed-target-entrypoint.sh index a9a44a8..d0330a3 100644 --- a/managed-target-entrypoint.sh +++ b/managed-target-entrypoint.sh @@ -12,6 +12,8 @@ case "$ss_password" in echo 'error: target secret must contain only hexadecimal characters' >&2; exit 64 ;; esac [ "${#ss_password}" -ge 32 ] || { echo 'error: target secret must be at least 32 hex characters' >&2; exit 64; } +# Validate before creating configs or launching any service. Errors never echo input. +/usr/local/bin/zapret-autotune target-config-check || exit 64 config_dir=/run/zapret-autotune-ss awk '$2 == "/run/zapret-autotune-ss" && $3 == "tmpfs" { found=1 } END { exit !found }' /proc/mounts || { echo 'error: target Shadowsocks config directory must be a dedicated tmpfs' >&2; exit 78; @@ -163,7 +165,10 @@ trap 'exit 0' INT TERM # Never invoke /opt/entrypoint.sh: zapret must start AFTER reconciliation. run_cycle() { -ss-server -c "$config_dir/server.json" & +set -- ss-server -c "$config_dir/server.json" +# DNS is not a secret. Native -d avoids assuming support for a JSON DNS key. +[ -z "${SS_DNS_SERVER-}" ] || set -- "$@" -d "$SS_DNS_SERVER" +"$@" & server_pid=$! ss-local -c "$config_dir/local.json" & local_pid=$!