feat: add autonomous YouTube resource recovery and adaptive GGC

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-11 20:15:16 +03:00
1 parent 8826c591dc
commit b4e1e1e9ca
43 files changed
+5153 -83

No files matched your search

+1
View File
@@ -17,6 +17,7 @@ RUN if [ ! -e /opt/zapret2/lua ] && [ ! -L /opt/zapret2/lua ]; then \
fi \ fi \
&& test -d /opt/zapret2/lua \ && test -d /opt/zapret2/lua \
&& command -v setsid \ && command -v setsid \
&& (ss-server -h 2>&1 | grep -q -- '-d <addr>') \
&& mkdir -p /etc/zapret-autotune /var/lib/zapret-autotune/target \ && mkdir -p /etc/zapret-autotune /var/lib/zapret-autotune/target \
&& printf 'zapret-autotune-managed-v1\n' > /etc/zapret-autotune/target-id \ && printf 'zapret-autotune-managed-v1\n' > /etc/zapret-autotune/target-id \
&& chmod 700 /var/lib/zapret-autotune /var/lib/zapret-autotune/target && chmod 700 /var/lib/zapret-autotune /var/lib/zapret-autotune/target
+19 -2
View File
@@ -251,6 +251,10 @@ signed URLs; use `docker compose ... config --quiet` for validation.
| --- | --- | | --- | --- |
| `ZAPRET_AUTOTUNE_IMAGE` | Local build/test default `zapret-autotune-managed-target:local`; production promotion requires a managed-target `repository@sha256:<digest>` | | `ZAPRET_AUTOTUNE_IMAGE` | Local build/test default `zapret-autotune-managed-target:local`; production promotion requires a managed-target `repository@sha256:<digest>` |
| `SS_PASSWORD` | Required independent hex-only secret, at least 32 characters; deployment README generates 64 | | `SS_PASSWORD` | Required independent hex-only secret, at least 32 characters; deployment README generates 64 |
| `SS_DNS_SERVER` | Empty (unchanged resolver); optional literal IPv4/IPv6 for target-only native ss-server DNS, validated before launch |
| `ZAPRET_AUTOTUNE_YOUTUBE_SERVICE` | `0` (legacy) or opt-in `1`; explicitly forwarded by Compose |
| `ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE` | `cold` default or `adaptive` (requires service `1`); explicitly forwarded |
| `ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS` | JSON array of 1..5 unique VideoIDs; defaults to only the primary ID; explicitly forwarded |
| `ZAPRET_AUTOTUNE_HOST_SOCKS_PORT` | `11080`; choose an available TCP/UDP port 1–65535, published only on `127.0.0.1` | | `ZAPRET_AUTOTUNE_HOST_SOCKS_PORT` | `11080`; choose an available TCP/UDP port 1–65535, published only on `127.0.0.1` |
| `SOCKS_PORT` | Supplied deployment fixes internal listener to `1080`; not a host-port override | | `SOCKS_PORT` | Supplied deployment fixes internal listener to `1080`; not a host-port override |
| `ZAPRET_AUTOTUNE_SOCKS_PROXY` | Fixed `socks5://127.0.0.1:1080` inside this target, no credentials/direct fallback | | `ZAPRET_AUTOTUNE_SOCKS_PROXY` | Fixed `socks5://127.0.0.1:1080` inside this target, no credentials/direct fallback |
@@ -262,13 +266,26 @@ signed URLs; use `docker compose ... config --quiet` for validation.
The monitor's three policy flags override corresponding environment defaults; The monitor's three policy flags override corresponding environment defaults;
explicit empty/invalid environment settings are rejected. The entrypoint accepts explicit empty/invalid environment settings are rejected. The entrypoint accepts
no arguments. The supplied Compose environment does not forward the video/policy no arguments. Compose forwards the primary/sample VideoIDs and service/mode settings.
settings from `.env`: add the desired existing variables to `target.environment` It does not forward the interval/threshold/cooldown policy
settings from `.env`: add those existing variables to `target.environment`
in your deployment copy/override if needed. Keep all fixed identity/path settings in your deployment copy/override if needed. Keep all fixed identity/path settings
unchanged. No public SS listener is published; Xray configuration is external and unchanged. No public SS listener is published; Xray configuration is external and
operator-owned, optionally using the host's loopback SOCKS port. Do not modify an operator-owned, optionally using the host's loopback SOCKS port. Do not modify an
existing Xray deployment as part of bootstrap. existing Xray deployment as part of bootstrap.
Current service/adaptive hardware acceptance is **pending**, independent of the
recorded legacy RC PASS. Adaptive means post-warmup, not cold: first connections
may fail and native learning repeats after restart. At least three real unique
signed ordinary media host samples are required; multiple VideoIDs may yield one
CDN host and leave `DEGRADED` with no authorized suffix. Sampled thumbnail/avatar
routes do not prove availability of the whole thumbnail pool. The reported
`8.8.8.8` DNS diagnostic corrected a tampered avatar, not all thumbnail routes;
native DNS correction is not full asset availability proof. Target DNS uses
validated native `-d` argv (nonsecret), with installed-option support checked
at image build; passwords remain private JSON only. No DoH bridge, permanent
Google IP pins, global host DNS edits or external startup manager are introduced.
### Supervision and failure handling ### Supervision and failure handling
The container entrypoint is PID 1 and owns all technical child restart control. The container entrypoint is PID 1 and owns all technical child restart control.
+238
View File
@@ -0,0 +1,238 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/json"
"fmt"
"image"
"image/gif"
_ "image/jpeg"
_ "image/png"
"io"
"math"
"sort"
"strings"
"time"
)
// Rebuild only the two bounded candidate tables, inside migrate's transaction.
// Refuse unexpected objects rather than silently discarding custom constraints.
func migrateAssetCandidates(ctx context.Context, conn *sql.Conn) error {
var actual []string
rows, err := conn.QueryContext(ctx, `SELECT sql FROM sqlite_master WHERE tbl_name IN ('strategy_candidates', 'candidate_evidence') AND sql IS NOT NULL ORDER BY sql`)
if err != nil {
return err
}
for rows.Next() {
var s string
if err := rows.Scan(&s); err != nil {
rows.Close()
return err
}
actual = append(actual, strings.TrimSpace(s))
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
var expected []string
for _, s := range strings.Split(strings.TrimSpace(candidateSchema), ";\n") {
if !strings.HasPrefix(s, "PRAGMA") {
expected = append(expected, strings.TrimSpace(s))
}
}
sort.Strings(expected)
if strings.Join(actual, "\n") != strings.Join(expected, "\n") {
return fmt.Errorf("unsupported candidate schema; asset migration refused")
}
if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE asset_candidate_backup AS SELECT * FROM strategy_candidates;
CREATE TEMP TABLE asset_evidence_backup AS SELECT * FROM candidate_evidence;
DROP TABLE candidate_evidence;
DROP TABLE strategy_candidates;`); err != nil {
return err
}
schema := strings.Replace(candidateSchema, "('frontend', 'googlevideo')", "('frontend', 'googlevideo', 'assets')", 1)
if _, err := conn.ExecContext(ctx, schema); err != nil {
return err
}
if _, err := conn.ExecContext(ctx, `CREATE TRIGGER asset_candidate_scope BEFORE INSERT ON strategy_candidates
WHEN NEW.strategy_group = 'assets' AND (json_array_length(NEW.endpoints) > 2
OR EXISTS (SELECT 1 FROM json_each(NEW.endpoints) WHERE type != 'text' OR value NOT IN ('i.ytimg.com', 'yt3.ggpht.com'))
OR (SELECT count(DISTINCT value) FROM json_each(NEW.endpoints)) != json_array_length(NEW.endpoints))
BEGIN SELECT RAISE(ABORT, 'invalid asset service inventory'); END;`); err != nil {
return err
}
_, err = conn.ExecContext(ctx, `INSERT INTO strategy_candidates SELECT * FROM asset_candidate_backup;
INSERT INTO candidate_evidence SELECT * FROM asset_evidence_backup;
DROP TABLE asset_candidate_backup;
DROP TABLE asset_evidence_backup;
PRAGMA user_version = 6;`)
return err
}
// No URLs: the caller supplies actual response bodies, never a blockcheck report.
// Three transfers per exact TLS version are required; no probing is wired here.
type assetTransfer struct {
EffectiveHost string `json:"effective_host"`
SSLVerifyResult *int `json:"ssl_verify_result"`
Redirects int `json:"redirects"`
ContentType string `json:"content_type"`
BodyHash string `json:"body_sha256"`
Format string `json:"format"`
Size int `json:"size"`
TLSVersion uint16 `json:"tls_version"`
Status int `json:"status"`
Body []byte `json:"-"`
LatencyMS float64 `json:"latency_ms"`
SpeedKBPS float64 `json:"speed_kbps"`
}
type assetMeasurement struct {
Source string `json:"source"`
Endpoint string `json:"endpoint"`
Protocol string `json:"protocol"`
Args string `json:"args"`
Transfers []assetTransfer `json:"transfers"`
}
func (t *assetTransfer) UnmarshalJSON(raw []byte) error {
type fields assetTransfer
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
return err
}
for _, key := range []string{"effective_host", "ssl_verify_result", "redirects", "content_type", "body_sha256", "format", "size", "tls_version", "status", "latency_ms", "speed_kbps"} {
if value, ok := keys[key]; !ok || bytes.Equal(bytes.TrimSpace(value), []byte("null")) {
return fmt.Errorf("missing image transfer field")
}
}
var value fields
d := json.NewDecoder(bytes.NewReader(raw))
d.DisallowUnknownFields()
if err := d.Decode(&value); err != nil {
return err
}
*t = assetTransfer(value)
return nil
}
func saveAssetEvidence(tx *sql.Tx, id int64, m assetMeasurement, checked time.Time) error {
if m.Source != "image-transfer" || !assetEndpoint(m.Endpoint) || (m.Protocol != "tls12" && m.Protocol != "tls13") || checked.IsZero() || len(m.Transfers) != 3 {
return fmt.Errorf("invalid image-transfer evidence")
}
var group, args string
if err := tx.QueryRow(`SELECT strategy_group, args FROM strategy_candidates WHERE id = ?`, id).Scan(&group, &args); err != nil {
return err
}
if group != "assets" || m.Args != args {
return fmt.Errorf("asset evidence definition mismatch")
}
latency, speed, err := validateAssetTransfers(m)
if err != nil {
return err
}
for i := range m.Transfers {
if err := normalizeAssetBody(&m.Transfers[i]); err != nil {
return err
}
}
raw, err := json.Marshal(m)
if err != nil {
return err
}
_, err = tx.Exec(`INSERT INTO candidate_evidence (candidate_id, endpoint, protocol, passes_ok, passes_total, success_rate, coverage, median_latency_ms, median_speed_kbps, checked_at, raw_measurement) VALUES (?, ?, ?, 3, 3, 1, 1, ?, ?, ?, ?)`, id, m.Endpoint, m.Protocol, latency, speed, checked.UTC().Format(time.RFC3339Nano), string(raw))
return err
}
func validateAssetTransfers(m assetMeasurement) (float64, float64, error) {
if m.Source != "image-transfer" || !assetEndpoint(m.Endpoint) || (m.Protocol != "tls12" && m.Protocol != "tls13") || len(m.Transfers) != 3 {
return 0, 0, fmt.Errorf("invalid image-transfer evidence")
}
tlsVersion := uint16(0x0303)
if m.Protocol == "tls13" {
tlsVersion = 0x0304
}
var latency, speed []float64
for _, transfer := range m.Transfers {
if transfer.TLSVersion != tlsVersion || transfer.Status != 200 || transfer.EffectiveHost != m.Endpoint || transfer.SSLVerifyResult == nil || *transfer.SSLVerifyResult != 0 || transfer.Redirects != 0 {
return 0, 0, fmt.Errorf("image transfer must match TLS and HTTP 200 with a bounded image body")
}
if err := normalizeAssetBody(&transfer); err != nil {
return 0, 0, err
}
for _, n := range []float64{transfer.LatencyMS, transfer.SpeedKBPS} {
if n < 0 || math.IsNaN(n) || math.IsInf(n, 0) {
return 0, 0, fmt.Errorf("invalid image transfer metric")
}
}
latency = append(latency, transfer.LatencyMS)
speed = append(speed, transfer.SpeedKBPS)
}
sort.Float64s(latency)
sort.Float64s(speed)
return latency[1], speed[1], nil
}
// Bodies are transient. Stored evidence carries only verified normalized facts.
func normalizeAssetBody(t *assetTransfer) error {
if len(t.Body) != 0 {
if len(t.Body) > 5<<20 {
return fmt.Errorf("image body exceeds limit")
}
cfg, format, err := image.DecodeConfig(bytes.NewReader(t.Body))
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 || cfg.Width > (16<<20)/cfg.Height {
return fmt.Errorf("invalid image dimensions")
}
if format == "gif" {
if _, err := gif.DecodeAll(bytes.NewReader(t.Body)); err != nil {
return fmt.Errorf("invalid GIF body")
}
} else if _, _, err := image.Decode(bytes.NewReader(t.Body)); err != nil {
return fmt.Errorf("invalid image body")
}
t.Format, t.Size, t.BodyHash = format, len(t.Body), fmt.Sprintf("%x", sha256.Sum256(t.Body))
}
want := map[string]string{"png": "image/png", "jpeg": "image/jpeg", "gif": "image/gif"}[t.Format]
if want == "" || t.ContentType != want || t.Size <= 0 || t.Size > 5<<20 || len(t.BodyHash) != 64 || strings.Trim(t.BodyHash, "0123456789abcdef") != "" {
return fmt.Errorf("invalid normalized image evidence")
}
return nil
}
// Revalidate asset bodies on reads too: corrupted raw evidence cannot qualify.
func validateStoredAssetEvidence(db *sql.DB, c storedCandidate) (bool, error) {
rows, err := db.Query(`SELECT endpoint, protocol, median_latency_ms, median_speed_kbps, checked_at, raw_measurement FROM candidate_evidence WHERE candidate_id = ?`, c.ID)
if err != nil {
return false, err
}
defer rows.Close()
for rows.Next() {
var endpoint, protocol, checkedAt, raw string
var latency, speed float64
if err := rows.Scan(&endpoint, &protocol, &latency, &speed, &checkedAt, &raw); err != nil {
return false, err
}
checked, err := time.Parse(time.RFC3339Nano, checkedAt)
if err != nil || checked.IsZero() {
return false, nil
}
var m assetMeasurement
d := json.NewDecoder(strings.NewReader(raw))
d.DisallowUnknownFields()
if err := d.Decode(&m); err != nil {
return false, nil
}
if err := d.Decode(new(interface{})); err != io.EOF {
return false, nil
}
l, s, err := validateAssetTransfers(m)
if err != nil || m.Endpoint != endpoint || m.Protocol != protocol || m.Args != c.Args || l != latency || s != speed {
return false, nil
}
}
return true, rows.Err()
}
+315
View File
@@ -0,0 +1,315 @@
package main
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"image"
"image/png"
"io"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
// Recreate an actual pre-assets schema, not just a misleading version number.
func restoreV5CandidateSchema(t *testing.T, db *sql.DB) {
t.Helper()
tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
_, err = tx.Exec(`CREATE TEMP TABLE old_candidates AS SELECT id, source_run_id, strategy_group, endpoints, args, fingerprint FROM strategy_candidates;
CREATE TEMP TABLE old_evidence AS SELECT * FROM candidate_evidence;
DROP TABLE googlevideo_scope_evidence;
DROP TABLE candidate_evidence; DROP TABLE strategy_candidates;` + candidateSchema + `
INSERT INTO strategy_candidates SELECT * FROM old_candidates;
INSERT INTO candidate_evidence SELECT * FROM old_evidence;
DROP TABLE old_candidates; DROP TABLE old_evidence;
PRAGMA user_version = 5;`)
if err != nil {
t.Fatal(err)
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
}
// Synthetic unit fixture only: no claim that any endpoint was reachable.
func unitImageMeasurement(t *testing.T, host, protocol, args string) assetMeasurement {
t.Helper()
var body bytes.Buffer
if err := png.Encode(&body, image.NewRGBA(image.Rect(0, 0, 1, 1))); err != nil {
t.Fatal(err)
}
v := uint16(0x0303)
if protocol == "tls13" {
v = 0x0304
}
m := assetMeasurement{Source: "image-transfer", Endpoint: host, Protocol: protocol, Args: args}
for i := 0; i < 3; i++ {
zero := 0
m.Transfers = append(m.Transfers, assetTransfer{EffectiveHost: host, SSLVerifyResult: &zero, ContentType: "image/png", TLSVersion: v, Status: 200, Body: body.Bytes(), LatencyMS: 10, SpeedKBPS: 100})
}
return m
}
func unitAssetCandidate(t *testing.T, db *sql.DB, host, args string, qualify bool) int64 {
t.Helper()
tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
id, err := insertStrategyCandidate(tx, "assets", []string{host}, args, nil)
if err != nil {
t.Fatal(err)
}
if qualify {
for _, protocol := range []string{"tls12", "tls13"} {
if err := saveAssetEvidence(tx, id, unitImageMeasurement(t, host, protocol, args), time.Now()); err != nil {
t.Fatal(err)
}
}
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
return id
}
func TestAssetMigrationLegacyBytesAndRollback(t *testing.T) {
db := testStore(t)
saveProposal(t, db, "--lua-desync=fake")
restoreV5CandidateSchema(t, db)
before, err := readStrategyCandidate(db, 1)
if err != nil {
t.Fatal(err)
}
config, err := renderManagedConfig(db, []int64{1})
if err != nil {
t.Fatal(err)
}
var evidence string
if err := db.QueryRow(`SELECT group_concat(raw_measurement) FROM candidate_evidence`).Scan(&evidence); err != nil {
t.Fatal(err)
}
if err := migrate(db); err != nil {
t.Fatal(err)
}
after, err := readStrategyCandidate(db, 1)
if err != nil || !reflect.DeepEqual(before, after) {
t.Fatalf("changed candidate: %+v %v", after, err)
}
rendered, err := renderManagedConfig(db, []int64{1})
if err != nil || !bytes.Equal(config, rendered) || managedHash(config) != managedHash(rendered) {
t.Fatalf("legacy bytes changed: %v", err)
}
var afterEvidence string
if err := db.QueryRow(`SELECT group_concat(raw_measurement) FROM candidate_evidence`).Scan(&afterEvidence); err != nil || evidence != afterEvidence {
t.Fatalf("evidence changed: %v", err)
}
if _, err := db.Exec(`UPDATE strategy_candidates SET args = '--lua-desync=multisplit'`); err == nil {
t.Fatal("immutable trigger lost")
}
restoreV5CandidateSchema(t, db)
// A foreign reference forces failure after the migration starts rebuilding.
if _, err := db.Exec(`CREATE TABLE candidate_external_ref (id INTEGER REFERENCES strategy_candidates(id)); INSERT INTO candidate_external_ref VALUES (1);`); err != nil {
t.Fatal(err)
}
if err := migrate(db); err == nil {
t.Fatal("expected atomic migration failure")
}
var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d %v", version, err)
}
rendered, err = renderManagedConfig(db, []int64{1})
if err != nil || !bytes.Equal(config, rendered) {
t.Fatalf("rollback lost legacy bytes: %v", err)
}
if _, err := db.Exec(`DROP TABLE candidate_external_ref; CREATE TRIGGER custom_candidate BEFORE INSERT ON strategy_candidates BEGIN SELECT 1; END;`); err != nil {
t.Fatal(err)
}
if err := migrate(db); err == nil {
t.Fatal("unknown schema silently adopted")
}
}
func TestAssetEvidenceAndDisjointRender(t *testing.T) {
db := testStore(t)
a := unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true)
b := unitAssetCandidate(t, db, "yt3.ggpht.com", "--lua-desync=multisplit", true)
config, err := renderManagedConfig(db, []int64{a, b})
if err != nil || !strings.Contains(string(config), "--hostlist-domains=^i.ytimg.com --lua-desync=fake --new") || !strings.Contains(string(config), "--hostlist-domains=^yt3.ggpht.com --lua-desync=multisplit") {
t.Fatalf("render: %s %v", config, err)
}
ids, err := qualifiedCandidateIDs(db, "assets")
if err != nil || !reflect.DeepEqual(ids, []int64{b, a}) {
t.Fatalf("IDs %v %v", ids, err)
}
if _, err := db.Exec(`UPDATE candidate_evidence SET checked_at = 'fixture' WHERE candidate_id = ? AND protocol = 'tls12'`, a); err != nil {
t.Fatal(err)
}
if _, err := renderManagedConfig(db, []int64{a}); err == nil {
t.Fatal("invalid checked timestamp qualified")
}
if _, err := db.Exec(`UPDATE candidate_evidence SET checked_at = ? WHERE candidate_id = ? AND protocol = 'tls12'`, time.Now().UTC().Format(time.RFC3339Nano), a); err != nil {
t.Fatal(err)
}
overlap := unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=multisplit", true)
if _, err := renderManagedConfig(db, []int64{a, overlap}); err == nil {
t.Fatal("accepted overlap")
}
missing := unitAssetCandidate(t, db, "yt3.ggpht.com", "--lua-desync=fake", false)
if _, err := renderManagedConfig(db, []int64{a, missing}); err == nil {
t.Fatal("accepted missing evidence")
}
tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
for _, host := range []string{"ytimg.com", "other.ytimg.com", "*.ytimg.com", "youtube.com"} {
if _, err := insertStrategyCandidate(tx, "assets", []string{host}, "--lua-desync=fake", nil); err == nil {
t.Fatalf("accepted %s", host)
}
}
if err := saveCandidateEvidence(tx, missing, "yt3.ggpht.com", "tls12", []byte(`{}`), time.Now()); err == nil {
t.Fatal("accepted blockcheck for assets")
}
for _, damage := range []string{"status", "magic", "tls", "count", "source", "scope"} {
m := unitImageMeasurement(t, "yt3.ggpht.com", "tls12", "--lua-desync=fake")
switch damage {
case "status":
m.Transfers[0].Status = 400
case "magic":
m.Transfers[0].Body = []byte("not an image")
case "tls":
m.Transfers[0].TLSVersion = 0x0304
case "count":
m.Transfers = m.Transfers[:2]
case "source":
m.Source = "blockcheck"
case "scope":
m.Endpoint = "i.ytimg.com"
}
if err := saveAssetEvidence(tx, missing, m, time.Now()); err == nil {
t.Fatalf("accepted %s", damage)
}
}
tx.Rollback()
m := unitImageMeasurement(t, "yt3.ggpht.com", "tls13", "--lua-desync=multisplit")
m.Transfers[0].Status = 400
raw, _ := json.Marshal(m)
if _, err := db.Exec(`UPDATE candidate_evidence SET raw_measurement = ? WHERE candidate_id = ? AND protocol = 'tls13'`, string(raw), b); err != nil {
t.Fatal(err)
}
if _, err := renderManagedConfig(db, []int64{b}); err == nil {
t.Fatal("corrupt HTTP400 evidence qualified")
}
if _, err := db.Exec("PRAGMA user_version = 8"); err != nil {
t.Fatal(err)
}
if _, err := renderManagedConfig(db, []int64{a}); err == nil {
t.Fatal("unknown schema rendered")
}
}
func TestAssetCLIApplyPartialFailureAndStatus(t *testing.T) {
for _, fail := range []bool{false, true} {
t.Run(fmt.Sprint(fail), func(t *testing.T) {
d, env, _ := managedCLIFixture(t)
env["ZAPRET_AUTOTUNE_VIDEO_ID"] = "dQw4w9WgXcQ"
path := filepath.Join(t.TempDir(), "assets.db")
db, err := openStore(path)
if err != nil {
t.Fatal(err)
}
saveProposal(t, db, "--lua-desync=fake")
tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
ggc, err := insertStrategyCandidate(tx, "googlevideo", []string{"rr1.unit.googlevideo.com"}, "--lua-desync=fake", nil)
if err != nil {
tx.Rollback()
t.Fatal(err)
}
var report struct {
Strategies []json.RawMessage `json:"strategies"`
}
if err := json.Unmarshal([]byte(combinedReportFixture("--lua-desync=fake")), &report); err != nil {
tx.Rollback()
t.Fatal(err)
}
for i, protocol := range []string{"tls12", "tls13"} {
if err := saveCandidateEvidence(tx, ggc, "rr1.unit.googlevideo.com", protocol, report.Strategies[i], time.Now()); err != nil {
tx.Rollback()
t.Fatal(err)
}
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
a := unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true)
b := unitAssetCandidate(t, db, "yt3.ggpht.com", "--lua-desync=multisplit", true)
db.Close()
d.openDB = func() (*sql.DB, error) { return openHistoryStore(path) }
initManagedCLI(t, d)
if code, out, diagnostic := callManaged(t, d, "status"); code != 0 || strings.Contains(out, "assets") {
t.Fatalf("legacy status: %s %s", out, diagnostic)
}
if code, out, diagnostic := callManaged(t, d, "status", "--assets"); code != 0 || !strings.Contains(out, "unresolved_groups=frontend,googlevideo,assets") {
t.Fatalf("requested status: %s %s", out, diagnostic)
}
if fail {
d.player = func(context.Context, []string, io.Writer, io.Writer) error { return errors.New("unit health failure") }
}
code, _, diagnostic := callManaged(t, d, "apply", "--asset-candidate", fmt.Sprintf("%d,%d", a, b))
if (code != 0) != fail {
t.Fatalf("apply=%d %s", code, diagnostic)
}
config, err := os.ReadFile(filepath.Join(d.directory, "config"))
if err != nil {
t.Fatal(err)
}
if fail && string(config) != managedBaseline {
t.Fatal("failure did not restore baseline")
}
if !fail {
if code, out, diagnostic := callManaged(t, d, "status"); code != 0 || !strings.Contains(out, "groups=assets") || strings.Contains(out, "unresolved_groups=frontend,googlevideo,assets") {
t.Fatalf("asset status: %s %s", out, diagnostic)
}
if code, _, diagnostic := callManaged(t, d, "apply", "--asset-candidate", fmt.Sprint(a)); code != 0 {
t.Fatalf("partial apply: %s", diagnostic)
}
if code, out, diagnostic := callManaged(t, d, "status"); code != 0 || !strings.Contains(out, "unresolved_groups=frontend,googlevideo,assets") {
t.Fatalf("partial status: %s %s", out, diagnostic)
}
if code, _, diagnostic := callManaged(t, d, "apply", "--frontend-candidate", "1", "--googlevideo-candidate", fmt.Sprint(ggc), "--asset-candidate", fmt.Sprint(a), "--asset-candidate", fmt.Sprint(b)); code != 0 {
t.Fatalf("four candidate apply: %s", diagnostic)
}
if code, out, diagnostic := callManaged(t, d, "active-ids"); code != 0 || !strings.Contains(out, fmt.Sprintf(`"active_candidate_ids":[1,%d,%d,%d]`, ggc, a, b)) || !strings.Contains(out, `"unresolved_groups":null`) {
t.Fatalf("four IDs: %s %s", out, diagnostic)
}
}
})
}
for _, args := range [][]string{{"apply", "--asset-candidate", "1,1"}, {"apply", "--asset-candidate", "1,"}, {"apply", "--asset-candidate", "1,2,3"}, {"apply", "--frontend-candidate", "1,2"}, {"apply", "--frontend-candidate", "1", "--asset-candidate", "1"}} {
if _, _, err := managedArguments(args); err == nil {
t.Fatalf("accepted %v", args)
}
}
_, ids, err := managedArguments([]string{"apply", "--asset-candidate", "3", "--googlevideo-candidate", "2", "--asset-candidate", "4", "--frontend-candidate", "1"})
if err != nil || !reflect.DeepEqual(ids, []int64{1, 2, 3, 4}) {
t.Fatalf("IDs %v %v", ids, err)
}
}
+345 -7
View File
@@ -5,6 +5,8 @@ import (
"database/sql" "database/sql"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io"
"math"
"strings" "strings"
"time" "time"
) )
@@ -47,7 +49,17 @@ CREATE TRIGGER evidence_scope_update BEFORE UPDATE ON candidate_evidence
BEGIN SELECT RAISE(ABORT, 'evidence endpoint outside candidate scope'); END; BEGIN SELECT RAISE(ABORT, 'evidence endpoint outside candidate scope'); END;
PRAGMA user_version = 3;` PRAGMA user_version = 3;`
func insertStrategyCandidate(tx *sql.Tx, group string, endpoints []string, args string, sourceRun interface{}) (int64, error) { func insertStrategyCandidate(tx *sql.Tx, group string, endpoints []string, args string, sourceRun interface{}, modes ...QualificationMode) (int64, error) {
mode := QualificationCold
if len(modes) > 1 {
return 0, fmt.Errorf("one qualification mode required")
}
if len(modes) == 1 {
mode = modes[0]
}
if mode != QualificationCold && (mode != QualificationAdaptive || group != "googlevideo") {
return 0, fmt.Errorf("invalid qualification mode/group")
}
c := candidate{Group: group, Domains: append([]string(nil), endpoints...), Args: args} c := candidate{Group: group, Domains: append([]string(nil), endpoints...), Args: args}
if err := c.validateMeasurements(); err != nil { if err := c.validateMeasurements(); err != nil {
return 0, err return 0, err
@@ -65,7 +77,13 @@ func insertStrategyCandidate(tx *sql.Tx, group string, endpoints []string, args
if err != nil { if err != nil {
return 0, err return 0, err
} }
result, err := tx.Exec(`INSERT INTO strategy_candidates (source_run_id, strategy_group, endpoints, args, fingerprint) VALUES (?, ?, ?, ?, ?)`, sourceRun, group, string(scope), args, fingerprint) query := `INSERT INTO strategy_candidates (source_run_id, strategy_group, endpoints, args, fingerprint) VALUES (?, ?, ?, ?, ?)`
values := []interface{}{sourceRun, group, string(scope), args, fingerprint}
if mode == QualificationAdaptive {
query = `INSERT INTO strategy_candidates (source_run_id, strategy_group, endpoints, args, fingerprint, qualification_mode) VALUES (?, ?, ?, ?, ?, ?)`
values = append(values, mode)
}
result, err := tx.Exec(query, values...)
if err != nil { if err != nil {
return 0, err return 0, err
} }
@@ -74,6 +92,19 @@ func insertStrategyCandidate(tx *sql.Tx, group string, endpoints []string, args
// Evidence is tied to the immutable definition, never inferred from scan output. // Evidence is tied to the immutable definition, never inferred from scan output.
func saveCandidateEvidence(tx *sql.Tx, id int64, endpoint, protocol string, raw []byte, checked time.Time) error { func saveCandidateEvidence(tx *sql.Tx, id int64, endpoint, protocol string, raw []byte, checked time.Time) error {
var version int
if err := tx.QueryRow("PRAGMA user_version").Scan(&version); err != nil {
return err
}
if version >= 7 {
var mode QualificationMode
if err := tx.QueryRow("SELECT qualification_mode FROM strategy_candidates WHERE id = ?", id).Scan(&mode); err != nil {
return err
}
if mode != QualificationCold {
return fmt.Errorf("adaptive evidence must not use cold metrics")
}
}
if !validDomain(endpoint) || (protocol != "tls12" && protocol != "tls13") || checked.IsZero() { if !validDomain(endpoint) || (protocol != "tls12" && protocol != "tls13") || checked.IsZero() {
return fmt.Errorf("invalid evidence endpoint, protocol or checked timestamp") return fmt.Errorf("invalid evidence endpoint, protocol or checked timestamp")
} }
@@ -82,6 +113,9 @@ func saveCandidateEvidence(tx *sql.Tx, id int64, endpoint, protocol string, raw
if err := tx.QueryRow(`SELECT strategy_group, args FROM strategy_candidates WHERE id = ?`, id).Scan(&group, &args); err != nil { if err := tx.QueryRow(`SELECT strategy_group, args FROM strategy_candidates WHERE id = ?`, id).Scan(&group, &args); err != nil {
return err return err
} }
if group == "assets" {
return fmt.Errorf("assets require image-transfer evidence")
}
var fields map[string]json.RawMessage var fields map[string]json.RawMessage
if err := json.Unmarshal(raw, &fields); err != nil { if err := json.Unmarshal(raw, &fields); err != nil {
return err return err
@@ -119,6 +153,7 @@ type storedCandidate struct {
Group, Args, Fingerprint string Group, Args, Fingerprint string
Endpoints []string Endpoints []string
Qualified bool Qualified bool
Mode QualificationMode
} }
// Qualification precedes LIMIT: at most ten immutable definitions per trial. // Qualification precedes LIMIT: at most ten immutable definitions per trial.
@@ -138,18 +173,41 @@ const candidateQualifiedSQL = `NOT EXISTS (SELECT 1 FROM json_each(c.endpoints)
(SELECT 'tls12' AS protocol UNION ALL SELECT 'tls13') p (SELECT 'tls12' AS protocol UNION ALL SELECT 'tls13') p
WHERE NOT EXISTS (SELECT 1 FROM candidate_evidence r WHERE r.candidate_id = c.id WHERE NOT EXISTS (SELECT 1 FROM candidate_evidence r WHERE r.candidate_id = c.id
AND r.endpoint = e.value AND r.protocol = p.protocol AND r.passes_ok = 3 AND r.endpoint = e.value AND r.protocol = p.protocol AND r.passes_ok = 3
AND r.passes_total = 3 AND r.success_rate = 1 AND r.coverage = 1))` AND r.passes_total = 3 AND r.success_rate = 1 AND r.coverage = 1
AND (c.strategy_group != 'assets' OR (json_extract(r.raw_measurement, '$.source') = 'image-transfer'
AND json_extract(r.raw_measurement, '$.endpoint') = e.value
AND json_extract(r.raw_measurement, '$.protocol') = p.protocol
AND json_extract(r.raw_measurement, '$.args') = c.args
AND json_array_length(r.raw_measurement, '$.transfers') = 3))))`
// Returns detached snapshots of immutable definitions, not active state. // Returns detached snapshots of immutable definitions, not active state.
func listQualifiedCandidates(db *sql.DB, group string, limit int) ([]storedCandidate, error) { func listQualifiedCandidates(db *sql.DB, group string, limit int) ([]storedCandidate, error) {
if group != "frontend" && group != "googlevideo" { if group != "frontend" && group != "googlevideo" && group != "assets" {
return nil, fmt.Errorf("unknown candidate group") return nil, fmt.Errorf("unknown candidate group")
} }
if limit < 1 || limit > 10 { if limit < 1 || limit > 10 {
return nil, fmt.Errorf("candidate limit must be between 1 and 10") return nil, fmt.Errorf("candidate limit must be between 1 and 10")
} }
rows, err := db.Query(`SELECT id, strategy_group, args, fingerprint, endpoints FROM strategy_candidates c query := `SELECT id, strategy_group, args, fingerprint, endpoints FROM strategy_candidates c
WHERE strategy_group = ? AND `+candidateQualifiedSQL+` ORDER BY id DESC LIMIT ?`, group, limit) WHERE strategy_group = ? AND ` + candidateQualifiedSQL + ` ORDER BY id DESC`
var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil {
return nil, err
}
if version > 7 {
return nil, fmt.Errorf("unsupported qualification schema")
}
if version == 7 {
query = `SELECT id, strategy_group, args, fingerprint, endpoints FROM strategy_candidates c
WHERE strategy_group = ? AND ((qualification_mode = 'cold' AND ` + candidateQualifiedSQL + `) OR qualification_mode = 'adaptive') ORDER BY id DESC`
}
queryArgs := []interface{}{group}
// Asset bodies must validate before LIMIT, not just their SQL summaries.
if group == "frontend" {
query += " LIMIT ?"
queryArgs = append(queryArgs, limit)
}
rows, err := db.Query(query, queryArgs...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -165,17 +223,51 @@ func listQualifiedCandidates(db *sql.DB, group string, limit int) ([]storedCandi
return nil, err return nil, err
} }
c.Qualified = true c.Qualified = true
c.Mode = QualificationCold
result = append(result, c) result = append(result, c)
} }
if err := rows.Err(); err != nil { if err := rows.Err(); err != nil {
return nil, err return nil, err
} }
return result, rows.Close() if err := rows.Close(); err != nil {
return nil, err
}
if group == "assets" || group == "googlevideo" {
var validated []storedCandidate
for _, c := range result {
ok := true
var err error
if group == "assets" {
ok, err = validateStoredAssetEvidence(db, c)
} else {
c.Mode, err = readQualificationMode(db, c.ID)
ok = c.Mode == QualificationCold
if err == nil && c.Mode == QualificationAdaptive {
ok, err = validateStoredAdaptiveGooglevideoEvidence(db, c)
}
}
if err != nil {
return nil, err
}
if ok {
validated = append(validated, c)
if len(validated) == limit {
break
}
}
}
return validated, nil
}
return result, nil
} }
// One shared qualification rule for the complete endpoint × TLS matrix. // One shared qualification rule for the complete endpoint × TLS matrix.
// Missing or failed evidence is never a qualification; this is not active state. // Missing or failed evidence is never a qualification; this is not active state.
func readStrategyCandidate(db *sql.DB, id int64) (storedCandidate, error) { func readStrategyCandidate(db *sql.DB, id int64) (storedCandidate, error) {
return readStrategyCandidatePolicy(db, id, true)
}
func readStrategyCandidatePolicy(db *sql.DB, id int64, fresh bool) (storedCandidate, error) {
var c storedCandidate var c storedCandidate
var scope string var scope string
err := db.QueryRow(`SELECT id, strategy_group, args, fingerprint, endpoints, err := db.QueryRow(`SELECT id, strategy_group, args, fingerprint, endpoints,
@@ -185,5 +277,251 @@ func readStrategyCandidate(db *sql.DB, id int64) (storedCandidate, error) {
return c, err return c, err
} }
err = json.Unmarshal([]byte(scope), &c.Endpoints) err = json.Unmarshal([]byte(scope), &c.Endpoints)
if err == nil {
c.Mode, err = readQualificationMode(db, id)
if c.Mode != QualificationCold {
c.Qualified = false
if err == nil && c.Mode == QualificationAdaptive {
c.Qualified, err = validateAdaptiveGooglevideoEvidence(db, c, fresh)
}
}
}
if err == nil && c.Qualified && c.Group == "assets" {
c.Qualified, err = validateStoredAssetEvidence(db, c)
}
return c, err return c, err
} }
type QualificationMode string
const QualificationCold QualificationMode = "cold"
const QualificationAdaptive QualificationMode = "adaptive"
const NativeCircularPostWarmup = "native-circular-post-warmup"
// Additive migration: no candidate/evidence/asset/owner table rebuilds.
const googlevideoQualificationSchema = `
ALTER TABLE strategy_candidates ADD COLUMN qualification_mode TEXT NOT NULL DEFAULT 'cold'
CHECK (qualification_mode = 'cold' OR (qualification_mode = 'adaptive' AND strategy_group = 'googlevideo'));
CREATE TABLE googlevideo_scope_evidence (
candidate_id INTEGER PRIMARY KEY REFERENCES strategy_candidates(id),
proof TEXT NOT NULL CHECK (json_valid(proof))
);
PRAGMA user_version = 7;`
func readQualificationMode(db *sql.DB, id int64) (QualificationMode, error) {
var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil {
return "", err
}
if version < 7 {
return QualificationCold, nil
}
if version != 7 {
return "", fmt.Errorf("unsupported qualification schema")
}
var mode QualificationMode
err := db.QueryRow("SELECT qualification_mode FROM strategy_candidates WHERE id = ?", id).Scan(&mode)
if err == nil && mode != QualificationCold && mode != QualificationAdaptive {
err = fmt.Errorf("unknown qualification mode")
}
return mode, err
}
// Collector DTO: normalized facts only. No URL, query, body, or headers can be
// persisted. HTTP root/404 connectivity is deliberately not media evidence.
type GooglevideoPass struct {
Protocol string `json:"protocol"`
VerifiedTLS bool `json:"verified_tls"`
Status int `json:"status"`
RangeStart int64 `json:"range_start"`
RangeEnd int64 `json:"range_end"`
ResourceBytes int64 `json:"resource_bytes"`
Bytes int64 `json:"bytes"`
FTYP bool `json:"ftyp"`
LatencyMS float64 `json:"latency_ms"`
}
type GooglevideoHostProof struct {
Host string `json:"host"`
WarmupAttempts int `json:"warmup_attempts"`
WarmupFailures int `json:"warmup_failures"`
WarmupLimit int `json:"warmup_limit"`
WarmupMS float64 `json:"warmup_ms"`
Passes []GooglevideoPass `json:"steady_passes"`
}
type GooglevideoQualificationProof struct {
Version int `json:"version"`
Mode QualificationMode `json:"mode"`
SourceKind string `json:"source_kind"`
Args string `json:"args"`
Hosts []GooglevideoHostProof `json:"hosts"`
Attestation *GooglevideoRuntimeAttestation `json:"attestation,omitempty"`
producerSeal *googlevideoProducerSeal
}
func (p GooglevideoQualificationProof) validate(c storedCandidate) error {
if err := p.validateStored(c); err != nil {
return err
}
if p.Mode == QualificationAdaptive {
return p.Attestation.validate(c)
}
return nil
}
func (p GooglevideoQualificationProof) validateStored(c storedCandidate) error {
if p.Version != 1 || p.Mode != c.Mode || p.Args != c.Args || len(p.Hosts) != len(c.Endpoints) || len(p.Hosts) < 3 {
return fmt.Errorf("invalid representative proof definition")
}
if (p.Mode == QualificationCold && p.SourceKind != "media-range") || (p.Mode == QualificationAdaptive && p.SourceKind != NativeCircularPostWarmup) || (p.Mode != QualificationCold && p.Mode != QualificationAdaptive) {
return fmt.Errorf("invalid proof source/mode")
}
if err := (&candidate{Args: p.Args}).validateArgs(); err != nil {
return err
}
if err := validateGooglevideoHosts(c.Endpoints); err != nil {
return err
}
if p.Mode == QualificationCold && p.Attestation != nil {
return fmt.Errorf("cold evidence cannot contain native attestation")
}
wanted, seen, roots, prefixes := map[string]bool{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
for _, host := range c.Endpoints {
wanted[host] = true
}
for _, h := range p.Hosts {
prefix, root, ok := strings.Cut(strings.TrimSuffix(h.Host, ".googlevideo.com"), "---sn-")
if !ok || !strings.HasSuffix(h.Host, ".googlevideo.com") || !validDomain(h.Host) || h.Host != strings.ToLower(h.Host) || !wanted[h.Host] || seen[h.Host] || root == "" || len(prefix) < 3 || !strings.HasPrefix(prefix, "rr") || strings.Trim(prefix[2:], "0123456789") != "" {
return fmt.Errorf("invalid exact GGC sample")
}
seen[h.Host], roots[strings.Split(root, "-")[0]], prefixes[prefix] = true, true, true
if math.IsNaN(h.WarmupMS) || math.IsInf(h.WarmupMS, 0) || h.WarmupMS < 0 || h.WarmupMS > 120000 || h.WarmupAttempts < 0 || h.WarmupFailures < 0 || h.WarmupFailures > h.WarmupAttempts || h.WarmupLimit < 0 || h.WarmupLimit > 10 || h.WarmupAttempts > h.WarmupLimit {
return fmt.Errorf("unbounded warmup")
}
if p.Mode == QualificationCold && (h.WarmupAttempts != 0 || h.WarmupFailures != 0 || h.WarmupLimit != 0 || h.WarmupMS != 0) {
return fmt.Errorf("cold evidence cannot hide warmup")
}
if p.Mode == QualificationAdaptive && (h.WarmupLimit == 0 || h.WarmupAttempts < 2 || h.WarmupFailures > h.WarmupAttempts-2 || h.WarmupMS <= 0) {
return fmt.Errorf("adaptive requires bounded warmup evidence")
}
counts := map[string]int{}
for _, pass := range h.Passes {
if (pass.Protocol != "tls12" && pass.Protocol != "tls13") || !pass.VerifiedTLS || pass.Status != 206 || pass.RangeStart != 0 || pass.RangeEnd < 1023 || pass.ResourceBytes <= pass.RangeEnd || pass.Bytes != pass.RangeEnd+1 || !pass.FTYP || math.IsNaN(pass.LatencyMS) || math.IsInf(pass.LatencyMS, 0) || pass.LatencyMS <= 0 {
return fmt.Errorf("verified ranged media transfer required")
}
counts[pass.Protocol]++
}
if counts["tls12"] != 3 || counts["tls13"] != 3 {
return fmt.Errorf("three actual steady transfers per TLS protocol required")
}
}
if len(roots) < 2 || len(prefixes) < 2 {
return fmt.Errorf("diverse GGC root groups and host prefixes required")
}
if p.Mode == QualificationAdaptive {
return p.Attestation.validateStored(c)
}
return nil
}
func validateGooglevideoSuffixEvidence(db *sql.DB, c storedCandidate) error {
return validateGooglevideoSuffixEvidencePolicy(db, c, true)
}
func validateGooglevideoSuffixEvidencePolicy(db *sql.DB, c storedCandidate, fresh bool) error {
if c.Group != "googlevideo" || !c.Qualified {
return fmt.Errorf("qualified googlevideo candidate required")
}
proof, err := readGooglevideoProof(db, c.ID)
if err != nil {
return err
}
if fresh {
return proof.validate(c)
}
return proof.validateStored(c)
}
func readGooglevideoProof(db *sql.DB, id int64) (GooglevideoQualificationProof, error) {
var proof GooglevideoQualificationProof
var raw string
if err := db.QueryRow("SELECT proof FROM googlevideo_scope_evidence WHERE candidate_id = ?", id).Scan(&raw); err != nil {
return proof, err
}
return decodeGooglevideoProof(raw)
}
func decodeGooglevideoProof(raw string) (GooglevideoQualificationProof, error) {
var proof GooglevideoQualificationProof
d := json.NewDecoder(strings.NewReader(raw))
d.DisallowUnknownFields()
if err := d.Decode(&proof); err != nil {
return proof, err
}
if err := d.Decode(new(interface{})); err != io.EOF {
return proof, fmt.Errorf("unexpected proof data")
}
return proof, nil
}
func validateStoredAdaptiveGooglevideoEvidence(db *sql.DB, c storedCandidate) (bool, error) {
return validateAdaptiveGooglevideoEvidence(db, c, true)
}
func validateAdaptiveGooglevideoEvidence(db *sql.DB, c storedCandidate, fresh bool) (bool, error) {
if c.Group != "googlevideo" || c.Mode != QualificationAdaptive {
return false, nil
}
var raw string
if err := db.QueryRow("SELECT proof FROM googlevideo_scope_evidence WHERE candidate_id = ?", c.ID).Scan(&raw); err != nil {
if err == sql.ErrNoRows {
return false, nil
}
return false, err
}
// Malformed/stale proof is unqualified, not a reason to accept cold metrics.
proof, err := decodeGooglevideoProof(raw)
if err != nil {
return false, nil
}
if !fresh {
return proof.validateStored(c) == nil, nil
}
return proof.validate(c) == nil, nil
}
// One transactional insert, no cold candidate_evidence summaries. A private
// producer seal is required as well as valid DTO facts and stored definition.
func saveAdaptiveGooglevideoEvidence(tx *sql.Tx, id int64, proof GooglevideoQualificationProof) error {
if proof.producerSeal == nil || *proof.producerSeal != googlevideoProofSeal(proof) {
return fmt.Errorf("completed owned collector attestation required")
}
var c storedCandidate
var scope string
if err := tx.QueryRow("SELECT strategy_group, args, fingerprint, endpoints, qualification_mode FROM strategy_candidates WHERE id = ?", id).Scan(&c.Group, &c.Args, &c.Fingerprint, &scope, &c.Mode); err != nil {
return err
}
if err := json.Unmarshal([]byte(scope), &c.Endpoints); err != nil {
return err
}
if c.Group != "googlevideo" || c.Mode != QualificationAdaptive {
return fmt.Errorf("adaptive definition required")
}
if err := proof.validate(c); err != nil {
return err
}
var coldRows int
if err := tx.QueryRow("SELECT count(*) FROM candidate_evidence WHERE candidate_id = ?", id).Scan(&coldRows); err != nil {
return err
}
if coldRows != 0 {
return fmt.Errorf("adaptive evidence cannot contain cold rows")
}
raw, err := json.Marshal(proof)
if err != nil {
return err
}
_, err = tx.Exec("INSERT INTO googlevideo_scope_evidence (candidate_id, proof) VALUES (?, ?)", id, string(raw))
return err
}
+2 -2
View File
@@ -58,7 +58,7 @@ func populatedV2Store(t *testing.T) *sql.DB {
if err := migrate(db); err != nil { if err := migrate(db); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := db.Exec(`DROP TABLE monitor_state; DROP TABLE candidate_evidence; DROP TABLE strategy_candidates; PRAGMA user_version = 2;`); err != nil { if _, err := db.Exec(`DROP TABLE monitor_state; DROP TABLE googlevideo_scope_evidence; DROP TABLE candidate_evidence; DROP TABLE strategy_candidates; PRAGMA user_version = 2;`); err != nil {
t.Fatal(err) t.Fatal(err)
} }
return db return db
@@ -99,7 +99,7 @@ func TestPopulatedV2CandidateMigration(t *testing.T) {
} }
} }
var version int var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 7 {
t.Fatalf("version=%d err=%v", version, err) t.Fatalf("version=%d err=%v", version, err)
} }
if before != legacySnapshot(t, db) { if before != legacySnapshot(t, db) {
+47
View File
@@ -1,6 +1,7 @@
package main package main
import ( import (
"encoding/json"
"fmt" "fmt"
"net" "net"
"net/url" "net/url"
@@ -28,12 +29,18 @@ func parseHealthConfig(lookup func(string) (string, bool)) (healthConfig, error)
SOCKSProxy: value("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5://127.0.0.1:11080"), SOCKSProxy: value("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5://127.0.0.1:11080"),
VideoID: value("ZAPRET_AUTOTUNE_VIDEO_ID", "dQw4w9WgXcQ"), VideoID: value("ZAPRET_AUTOTUNE_VIDEO_ID", "dQw4w9WgXcQ"),
} }
if _, err := managedServiceEnabled(lookup); err != nil {
return healthConfig{}, err
}
if !validLoopbackSOCKS(cfg.SOCKSProxy) { if !validLoopbackSOCKS(cfg.SOCKSProxy) {
return healthConfig{}, fmt.Errorf("ZAPRET_AUTOTUNE_SOCKS_PROXY requires socks5://LOOPBACK:PORT without credentials") return healthConfig{}, fmt.Errorf("ZAPRET_AUTOTUNE_SOCKS_PROXY requires socks5://LOOPBACK:PORT without credentials")
} }
if !videoIDPattern.MatchString(cfg.VideoID) { if !videoIDPattern.MatchString(cfg.VideoID) {
return healthConfig{}, fmt.Errorf("ZAPRET_AUTOTUNE_VIDEO_ID requires an 11-character video ID") return healthConfig{}, fmt.Errorf("ZAPRET_AUTOTUNE_VIDEO_ID requires an 11-character video ID")
} }
if _, _, err := managedGooglevideoConfig(lookup, cfg.VideoID); err != nil {
return healthConfig{}, err
}
for _, setting := range []struct { for _, setting := range []struct {
key string key string
fallback string fallback string
@@ -56,6 +63,46 @@ func parseHealthConfig(lookup func(string) (string, bool)) (healthConfig, error)
return cfg, nil return cfg, nil
} }
func managedGooglevideoConfig(lookup func(string) (string, bool), primary string) (QualificationMode, []string, error) {
mode := QualificationCold
if raw, ok := lookup("ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE"); ok {
mode = QualificationMode(raw)
}
if mode != QualificationCold && mode != QualificationAdaptive {
return "", nil, fmt.Errorf("ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE requires cold or adaptive")
}
service, _ := managedServiceEnabled(lookup)
if mode == QualificationAdaptive && !service {
return "", nil, fmt.Errorf("adaptive googlevideo requires ZAPRET_AUTOTUNE_YOUTUBE_SERVICE=1")
}
// Only the configured primary is sampled by default; no implicit public playlist.
ids := []string{primary}
if raw, ok := lookup("ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS"); ok {
if json.Unmarshal([]byte(raw), &ids) != nil || len(ids) == 0 || len(ids) > 5 {
return "", nil, fmt.Errorf("GOOGLEVIDEO_VIDEO_IDS requires a JSON array of 1..5 video IDs")
}
seen := map[string]bool{}
for _, id := range ids {
if !videoIDPattern.MatchString(id) || seen[id] {
return "", nil, fmt.Errorf("GOOGLEVIDEO_VIDEO_IDS requires unique 11-character video IDs")
}
seen[id] = true
}
}
return mode, ids, nil
}
func managedServiceEnabled(lookup func(string) (string, bool)) (bool, error) {
value, exists := lookup("ZAPRET_AUTOTUNE_YOUTUBE_SERVICE")
if !exists {
return false, nil
}
if value != "0" && value != "1" {
return false, fmt.Errorf("ZAPRET_AUTOTUNE_YOUTUBE_SERVICE requires 0 or 1")
}
return value == "1", nil
}
func validLoopbackSOCKS(raw string) bool { func validLoopbackSOCKS(raw string) bool {
if !validDiscoveryProxy(raw) { if !validDiscoveryProxy(raw) {
return false return false
+1 -1
View File
@@ -117,7 +117,7 @@ func TestPopulatedV1FingerprintMigration(t *testing.T) {
} }
assertRowCounts(t, db, 3, 4) assertRowCounts(t, db, 3, 4)
var version int var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 7 {
t.Fatalf("version=%d err=%v", version, err) t.Fatalf("version=%d err=%v", version, err)
} }
for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} { for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} {
@@ -0,0 +1,312 @@
package main
import (
"bytes"
"database/sql"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
)
func scopeFixture(t *testing.T, db *sql.DB) (int64, GooglevideoQualificationProof) {
t.Helper()
hosts := []string{"rr1---sn-alpha.googlevideo.com", "rr2---sn-beta.googlevideo.com", "rr5---sn-gamma.googlevideo.com"}
tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
id, err := insertStrategyCandidate(tx, "googlevideo", hosts, "--lua-desync=fake", nil)
if err != nil {
t.Fatal(err)
}
p := GooglevideoQualificationProof{Version: 1, Mode: QualificationCold, SourceKind: "media-range", Args: "--lua-desync=fake"}
for _, host := range hosts {
h := GooglevideoHostProof{Host: host}
for _, protocol := range []string{"tls12", "tls13"} {
if _, err := tx.Exec(`INSERT INTO candidate_evidence VALUES (?, ?, ?, 3, 3, 1, 1, 1, 1, 'fixture', '{}')`, id, host, protocol); err != nil {
t.Fatal(err)
}
for i := 0; i < 3; i++ {
h.Passes = append(h.Passes, GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: 206, RangeEnd: 1023, ResourceBytes: 4096, Bytes: 1024, FTYP: true, LatencyMS: 10})
}
}
p.Hosts = append(p.Hosts, h)
}
raw, err := json.Marshal(p)
if err != nil {
t.Fatal(err)
}
if _, err = tx.Exec("INSERT INTO googlevideo_scope_evidence VALUES (?, ?)", id, string(raw)); err != nil {
t.Fatal(err)
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
return id, p
}
func TestGooglevideoScopeTransactional(t *testing.T) {
db := testStore(t)
id, _ := scopeFixture(t, db)
assets := []int64{unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true), unitAssetCandidate(t, db, "yt3.ggpht.com", "--lua-desync=fake", true)}
ids := append([]int64{id}, assets...)
exact, err := renderManagedConfig(db, ids)
if err != nil {
t.Fatal(err)
}
explicit, err := renderManagedConfigScope(db, ids, GooglevideoScopeExact)
if err != nil || !bytes.Equal(exact, explicit) {
t.Fatalf("legacy bytes changed: %v", err)
}
suffix, err := renderManagedConfigScope(db, ids, GooglevideoScopeSuffix)
if err != nil || !strings.Contains(string(suffix), "--hostlist-domains=googlevideo.com ") {
t.Fatalf("suffix=%s %v", suffix, err)
}
if _, err := renderManagedConfigScope(db, []int64{id}, GooglevideoScopeSuffix); err == nil {
t.Fatal("missing assets accepted")
}
m, _ := managedFixture(t)
if err := m.Apply(db, ids, GooglevideoScopeSuffix); err != nil {
t.Fatal(err)
}
j, err := m.load()
if err != nil || j.Version != 3 || j.Scope != GooglevideoScopeSuffix || !reflect.DeepEqual(j.Candidates, ids) {
t.Fatalf("journal=%+v %v", j, err)
}
if err := m.Apply(db, ids); err != nil {
t.Fatal(err)
} // omitted policy preserves suffix
assertManagedContent(t, m, string(suffix))
m.Health = func() (string, error) { return "FAIL", nil }
if err := m.Apply(db, []int64{id}, GooglevideoScopeExact); err == nil {
t.Fatal("health rejection accepted")
}
j, err = m.load()
if err != nil || j.Scope != GooglevideoScopeSuffix || !reflect.DeepEqual(j.Candidates, ids) || j.PendingScope != "" {
t.Fatalf("rollback=%+v %v", j, err)
}
assertManagedContent(t, m, string(suffix))
m.Health = func() (string, error) { return "OK", nil }
m.fault = func(s string) error {
if s == "replaced" {
return errors.New("death")
}
return nil
}
if err := m.Apply(db, []int64{id}, GooglevideoScopeExact); err == nil {
t.Fatal("fault absent")
}
j, err = m.load()
if err != nil || j.Scope != GooglevideoScopeSuffix || j.PendingScope != GooglevideoScopeExact {
t.Fatalf("pending=%+v %v", j, err)
}
m.fault = nil
if err := m.Reconcile(db); err != nil {
t.Fatal(err)
}
j, err = m.load()
if err != nil || j.Scope != GooglevideoScopeSuffix || !reflect.DeepEqual(j.Candidates, ids) {
t.Fatalf("reconcile=%+v %v", j, err)
}
assertManagedContent(t, m, string(suffix))
j.Scope = "future"
if err := m.journal(j); err != nil {
t.Fatal(err)
}
if _, err := m.load(); err == nil {
t.Fatal("unknown scope accepted")
}
}
func TestGooglevideoProofGate(t *testing.T) {
db := testStore(t)
id, base := scopeFixture(t, db)
c, err := readStrategyCandidate(db, id)
if err != nil {
t.Fatal(err)
}
if err := base.validate(c); err != nil {
t.Fatal(err)
}
for _, name := range []string{"samples", "warmup", "404", "tls", "range", "ftyp", "steady", "diversity", "args", "version"} {
t.Run(name, func(t *testing.T) {
raw, _ := json.Marshal(base)
var p GooglevideoQualificationProof
if err := json.Unmarshal(raw, &p); err != nil {
t.Fatal(err)
}
switch name {
case "samples":
p.Hosts = p.Hosts[:2]
case "warmup":
p.Mode = QualificationAdaptive
p.SourceKind = NativeCircularPostWarmup
c.Mode = QualificationAdaptive
p.Hosts[0].WarmupAttempts = 11
p.Hosts[0].WarmupLimit = 10
case "404":
p.Hosts[0].Passes[0].Status = 404
case "tls":
p.Hosts[0].Passes[0].VerifiedTLS = false
case "range":
p.Hosts[0].Passes[0].ResourceBytes = 0
case "ftyp":
p.Hosts[0].Passes[0].FTYP = false
case "steady":
p.Hosts[0].Passes = p.Hosts[0].Passes[:5]
case "diversity":
for i := range p.Hosts {
p.Hosts[i].Host = "rr1---sn-alpha.googlevideo.com"
}
case "args":
p.Args = "--lua-desync=multisplit"
case "version":
p.Version = 2
}
if err := p.validate(c); err == nil {
t.Fatal("invalid proof accepted")
}
c.Mode = QualificationCold
})
}
tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
adaptive, err := insertStrategyCandidate(tx, "googlevideo", c.Endpoints, c.Args, nil, QualificationAdaptive)
if err != nil {
t.Fatal(err)
}
for _, host := range c.Endpoints {
for _, protocol := range []string{"tls12", "tls13"} {
if _, err := tx.Exec(`INSERT INTO candidate_evidence VALUES (?, ?, ?, 3, 3, 1, 1, 1, 1, 'fixture', '{}')`, adaptive, host, protocol); err != nil {
t.Fatal(err)
}
}
}
if err := saveAdaptiveGooglevideoEvidence(tx, adaptive, base); err == nil {
t.Fatal("unwired collector accepted")
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
a, err := readStrategyCandidate(db, adaptive)
if err != nil || a.Qualified || a.Mode != QualificationAdaptive {
t.Fatalf("cold metrics qualified adaptive: %+v %v", a, err)
}
qualified, err := listQualifiedCandidates(db, "googlevideo", 1)
if err != nil || len(qualified) != 1 || qualified[0].ID != id {
t.Fatalf("limit filtered incorrectly: %+v %v", qualified, err)
}
if _, err := renderManagedConfig(db, []int64{adaptive}); err == nil {
t.Fatal("adaptive applied without collector attestation")
}
}
func TestGooglevideoV6MigrationPreservesBytes(t *testing.T) {
db := testStore(t)
id := managedCandidateFixture(t, db, "googlevideo", "rr5---sn-alpha.googlevideo.com", "--lua-desync=fake", true)
before, err := renderManagedConfig(db, []int64{id})
if err != nil {
t.Fatal(err)
}
if _, err := db.Exec("DROP TABLE googlevideo_scope_evidence; ALTER TABLE strategy_candidates DROP COLUMN qualification_mode; PRAGMA user_version=6;"); err != nil {
t.Fatal(err)
}
legacy, err := renderManagedConfig(db, []int64{id})
if err != nil || !bytes.Equal(before, legacy) {
t.Fatalf("v6=%s %v", legacy, err)
}
if err := migrate(db); err != nil {
t.Fatal(err)
}
after, err := renderManagedConfig(db, []int64{id})
if err != nil || !bytes.Equal(before, after) || managedHash(before) != managedHash(after) {
t.Fatalf("migration changed legacy: %v", err)
}
if _, err := db.Exec("UPDATE strategy_candidates SET qualification_mode='adaptive' WHERE id=?", id); err == nil {
t.Fatal("immutable definition trigger lost")
}
}
func TestGooglevideoScopeCLIArguments(t *testing.T) {
for _, scope := range []string{"exact", "suffix"} {
if _, ids, err := managedArguments([]string{"apply", "--googlevideo-candidate", "1", "--googlevideo-scope", scope}); err != nil || len(ids) != 1 {
t.Fatalf("%s %v", scope, err)
}
}
for _, args := range [][]string{{"apply", "--googlevideo-scope", "suffix"}, {"apply", "--googlevideo-candidate", "1", "--googlevideo-scope", "future"}, {"apply", "--googlevideo-candidate", "1", "--googlevideo-scope", "suffix", "--googlevideo-scope", "exact"}} {
if _, _, err := managedArguments(args); err == nil {
t.Fatalf("accepted %v", args)
}
}
}
func TestGooglevideoScopeRejectsNonMediaAndSecrets(t *testing.T) {
db := testStore(t)
id, proof := scopeFixture(t, db)
ids := []int64{id, unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true), unitAssetCandidate(t, db, "yt3.ggpht.com", "--lua-desync=fake", true)}
proof.Hosts[0].Passes[0].Status = 404
raw, err := json.Marshal(proof)
if err != nil {
t.Fatal(err)
}
if _, err := db.Exec("UPDATE googlevideo_scope_evidence SET proof=? WHERE candidate_id=?", string(raw), id); err != nil {
t.Fatal(err)
}
if _, err := renderManagedConfigScope(db, ids, GooglevideoScopeSuffix); err == nil {
t.Fatal("root 404 passed media gate")
}
proof.Hosts[0].Passes[0].Status = 206
raw, err = json.Marshal(proof)
if err != nil {
t.Fatal(err)
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(raw, &fields); err != nil {
t.Fatal(err)
}
fields["signed_url"] = json.RawMessage(`"https://secret.invalid/videoplayback?token=private"`)
raw, err = json.Marshal(fields)
if err != nil {
t.Fatal(err)
}
if _, err := db.Exec("UPDATE googlevideo_scope_evidence SET proof=? WHERE candidate_id=?", string(raw), id); err != nil {
t.Fatal(err)
}
if _, err := renderManagedConfigScope(db, ids, GooglevideoScopeSuffix); err == nil {
t.Fatal("secret-bearing evidence accepted")
}
if _, err := renderManagedConfig(db, ids); err != nil {
t.Fatalf("legacy exact evidence changed: %v", err)
}
}
func TestGooglevideoV6MigrationAtomicFailure(t *testing.T) {
db := testStore(t)
id := unitAssetCandidate(t, db, "i.ytimg.com", "--lua-desync=fake", true)
before, err := renderManagedConfig(db, []int64{id})
if err != nil {
t.Fatal(err)
}
if _, err := db.Exec("DROP TABLE googlevideo_scope_evidence; ALTER TABLE strategy_candidates DROP COLUMN qualification_mode; CREATE TABLE googlevideo_scope_evidence (conflict INTEGER); PRAGMA user_version=6;"); err != nil {
t.Fatal(err)
}
if err := migrate(db); err == nil {
t.Fatal("conflicting migration accepted")
}
var version, columns int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 6 {
t.Fatalf("version=%d %v", version, err)
}
if err := db.QueryRow("SELECT count(*) FROM pragma_table_info('strategy_candidates') WHERE name='qualification_mode'").Scan(&columns); err != nil || columns != 0 {
t.Fatalf("partial column migration: %d %v", columns, err)
}
after, err := renderManagedConfig(db, []int64{id})
if err != nil || !bytes.Equal(before, after) {
t.Fatalf("assets changed on failed migration: %v", err)
}
}
+1 -1
View File
@@ -154,7 +154,7 @@ fi
if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil { if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if evidence != 2 || legacy != 0 || version != 5 { if evidence != 2 || legacy != 0 || version != 7 {
t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version) t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version)
} }
for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} { for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} {
+4 -4
View File
@@ -21,8 +21,8 @@ func configLookup(values map[string]string) func(string) (string, bool) {
func TestHealthConfigDefaultsAndOverrides(t *testing.T) { func TestHealthConfigDefaultsAndOverrides(t *testing.T) {
cfg, err := parseHealthConfig(configLookup(nil)) cfg, err := parseHealthConfig(configLookup(nil))
want := healthConfig{"socks5://127.0.0.1:11080", "dQw4w9WgXcQ", 5 * time.Minute, 3, 30 * time.Minute} want := healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080", VideoID: "dQw4w9WgXcQ", HealthInterval: 5 * time.Minute, FailureThreshold: 3, RecoveryCooldown: 30 * time.Minute}
if err != nil || cfg != want { if err != nil || !reflect.DeepEqual(cfg, want) {
t.Fatalf("defaults = %+v, %v; want %+v", cfg, err, want) t.Fatalf("defaults = %+v, %v; want %+v", cfg, err, want)
} }
values := map[string]string{ values := map[string]string{
@@ -33,8 +33,8 @@ func TestHealthConfigDefaultsAndOverrides(t *testing.T) {
"ZAPRET_AUTOTUNE_RECOVERY_COOLDOWN": "1h", "ZAPRET_AUTOTUNE_RECOVERY_COOLDOWN": "1h",
} }
cfg, err = parseHealthConfig(configLookup(values)) cfg, err = parseHealthConfig(configLookup(values))
want = healthConfig{"socks5://[::1]:12080", "abcdefghijk", 90 * time.Second, 5, time.Hour} want = healthConfig{SOCKSProxy: "socks5://[::1]:12080", VideoID: "abcdefghijk", HealthInterval: 90 * time.Second, FailureThreshold: 5, RecoveryCooldown: time.Hour}
if err != nil || cfg != want { if err != nil || !reflect.DeepEqual(cfg, want) {
t.Fatalf("overrides = %+v, %v; want %+v", cfg, err, want) t.Fatalf("overrides = %+v, %v; want %+v", cfg, err, want)
} }
for _, proxy := range []string{"socks5://localhost:1", "socks5://LOCALHOST:65535", "socks5://127.0.0.2:11080", "socks5://[::1]:11080"} { for _, proxy := range []string{"socks5://localhost:1", "socks5://LOCALHOST:65535", "socks5://127.0.0.2:11080", "socks5://[::1]:11080"} {
+12
View File
@@ -3,6 +3,7 @@ package main
import ( import (
"fmt" "fmt"
"io" "io"
"net"
"os" "os"
) )
@@ -35,6 +36,17 @@ func run(args []string, stdout, stderr io.Writer) int {
} }
func runWithPlayer(args []string, stdout, stderr io.Writer, player playerRunner) int { func runWithPlayer(args []string, stdout, stderr io.Writer, player playerRunner) int {
if len(args) == 1 && args[0] == "target-config-check" {
if _, err := parseHealthConfig(os.LookupEnv); err != nil {
fmt.Fprintln(stderr, "configerror: invalid target health/service configuration")
return 64
}
if dns := os.Getenv("SS_DNS_SERVER"); dns != "" && net.ParseIP(dns) == nil {
fmt.Fprintln(stderr, "configerror: SS_DNS_SERVER requires a literal IPv4 or IPv6 address")
return 64
}
return 0
}
if len(args) > 0 && args[0] == "managed" { if len(args) > 0 && args[0] == "managed" {
return managedCLI(args[1:], stdout, stderr, managedDependencies(player)) return managedCLI(args[1:], stdout, stderr, managedDependencies(player))
} }
@@ -0,0 +1,354 @@
package main
import (
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"strconv"
"strings"
"syscall"
"time"
)
const managedAssetQueue = 200 // Serialized recovery never runs blockcheck concurrently.
const managedAssetTable = "zapret_autotune_asset_probe"
type managedAssetWorker struct {
pid int
done <-chan error
stop func() error
}
type managedAssetRuntime struct {
guard func() error
nft func(context.Context, string) (string, error)
start func([]string) (managedAssetWorker, error)
queue func(int) (bool, error)
curl subprocessRunner
proxy string
}
type managedAssetCleanupFailure struct{ Err error }
func (e *managedAssetCleanupFailure) Error() string { return "owned asset runtime cleanup failed" }
func (e *managedAssetCleanupFailure) Unwrap() error { return e.Err }
func newManagedAssetVerifier(target managedDeps) func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error) {
return newManagedAssetRuntime(target).verify
}
func newManagedAssetRuntime(target managedDeps) managedAssetRuntime {
proxy, _ := target.lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
r := managedAssetRuntime{proxy: proxy, curl: runResourceCurl}
r.guard = func() error {
if _, err := managedRuntimeGuard(target); err != nil {
return err
}
status, err := os.ReadFile("/proc/self/status")
if err != nil {
return err
}
for _, line := range strings.Split(string(status), "\n") {
if strings.HasPrefix(line, "CapEff:") {
bits, err := strconv.ParseUint(strings.TrimSpace(strings.TrimPrefix(line, "CapEff:")), 16, 64)
if err == nil && bits&(1<<5) != 0 && bits&(1<<12) != 0 {
return nil
}
}
}
return errors.New("asset verifier requires CAP_KILL and CAP_NET_ADMIN")
}
r.nft = func(ctx context.Context, input string) (string, error) {
cmd := exec.CommandContext(ctx, "nft", "-f", "-")
cmd.Stdin = strings.NewReader(input)
var out strings.Builder
cmd.Stdout, cmd.Stderr = &out, io.Discard
err := cmd.Run()
return out.String(), err
}
r.start = func(args []string) (managedAssetWorker, error) {
// No daemon or Setpgid: retain monitor session ownership.
cmd := exec.Command("/opt/zapret2/nfq2/nfqws2", args...)
cmd.Stdout, cmd.Stderr = io.Discard, io.Discard
if err := cmd.Start(); err != nil {
return managedAssetWorker{}, err
}
done := make(chan error, 1)
go func() { done <- cmd.Wait() }()
return managedAssetWorker{pid: cmd.Process.Pid, done: done, stop: func() error {
err := cmd.Process.Kill()
if errors.Is(err, os.ErrProcessDone) {
return errors.New("asset worker exited before owned stop")
}
return err
}}, nil
}
r.queue = managedAssetQueueReady
return r
}
// Queue peer portid is a netlink socket id, not necessarily a process id.
func managedAssetQueueReady(pid int) (bool, error) {
queues, err := os.ReadFile("/proc/net/netfilter/nfnetlink_queue")
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
if err != nil {
return false, err
}
peer := ""
for _, line := range strings.Split(string(queues), "\n") {
f := strings.Fields(line)
if len(f) >= 2 && f[0] == strconv.Itoa(managedAssetQueue) {
peer = f[1]
}
}
if peer == "" {
return false, nil
}
if pid == 0 {
return true, nil
}
if err := syscall.Kill(pid, 0); err != nil {
return false, err
}
netlink, err := os.ReadFile("/proc/net/netlink")
if err != nil {
return false, err
}
inodes := map[string]bool{}
for _, line := range strings.Split(string(netlink), "\n") {
f := strings.Fields(line)
if len(f) >= 10 && f[1] == "12" && f[2] == peer {
inodes["socket:["+f[9]+"]"] = true
}
}
fds, err := os.ReadDir(fmt.Sprintf("/proc/%d/fd", pid))
if err != nil {
return false, err
}
for _, fd := range fds {
link, err := os.Readlink(fmt.Sprintf("/proc/%d/fd/%s", pid, fd.Name()))
if err == nil && inodes[link] {
return true, nil
}
}
return false, errors.New("asset queue belongs to another process")
}
const managedAssetRules = `create table inet zapret_autotune_asset_probe
add chain inet zapret_autotune_asset_probe output { type filter hook output priority -151; policy accept; }
add chain inet zapret_autotune_asset_probe input { type filter hook input priority -151; policy accept; }
add rule inet zapret_autotune_asset_probe output meta mark & 0x10000000 == 0 tcp dport 443 queue num 200
add rule inet zapret_autotune_asset_probe input meta mark & 0x10000000 == 0 tcp sport 443 queue num 200
`
func (r managedAssetRuntime) verify(ctx context.Context, c storedCandidate, inventory managedServiceInventory) (measurements []assetMeasurement, err error) {
if c.Group != "assets" || len(c.Endpoints) == 0 || len(c.Endpoints) > 2 || strings.TrimSpace(c.Args) == "" {
return nil, &managedRecoveryFatal{Err: errors.New("invalid asset candidate")}
}
if e := (&candidate{Args: c.Args}).validateArgs(); e != nil {
return nil, &managedRecoveryFatal{Err: e}
}
for _, arg := range strings.Fields(c.Args) {
if strings.HasPrefix(arg, "--lua-init=") && arg != "--lua-init=@/opt/zapret2/lua/zapret-lib.lua" && arg != "--lua-init=@/opt/zapret2/lua/zapret-antidpi.lua" {
return nil, &managedRecoveryFatal{Err: errors.New("asset verifier only permits fixed Lua init files")}
}
}
seen := map[string]bool{}
for _, host := range c.Endpoints {
if !assetEndpoint(host) || seen[host] {
return nil, &managedRecoveryFatal{Err: errors.New("invalid asset endpoints")}
}
seen[host] = true
if _, e := privateResourceURL(inventory.images[host], host); e != nil {
return nil, e
}
}
if e := ctx.Err(); e != nil {
return nil, e
}
err = r.withOwnedWorker(ctx, c.Endpoints, c.Args, func(probeCtx context.Context) error {
for _, host := range c.Endpoints {
for _, protocol := range []string{"tls12", "tls13"} {
m, e := collectAssetMeasurement(probeCtx, r.proxy, inventory.images[host], host, protocol, c.Args, r.curl)
if e != nil {
if ctx.Err() != nil {
return ctx.Err()
}
if managedAssetTransferNonpass(e) {
return errCandidateNonpass
}
return e
}
measurements = append(measurements, m)
}
}
return nil
})
if err != nil {
return nil, err
}
return measurements, nil
}
// Shared, serialized session lifecycle: only an atomically created table is
// owned, and success is returned only after queue removal and worker join.
func (r managedAssetRuntime) withOwnedWorker(ctx context.Context, endpoints []string, strategyArgs string, capture func(context.Context) error) (err error) {
if err := ctx.Err(); err != nil {
return err
}
if e := r.guard(); e != nil {
return &managedRecoveryFatal{Err: e}
}
tables, e := r.nft(ctx, "list tables\n")
if e != nil {
return &managedRecoveryFatal{Err: e}
}
if strings.Contains(tables, "table inet "+managedAssetTable) {
return &managedRecoveryFatal{Err: errors.New("asset nft table already exists")}
}
occupied, e := r.queue(0)
if e != nil || occupied {
return &managedRecoveryFatal{Err: errors.Join(e, errors.New("asset queue unavailable"))}
}
args := managedProbeWorkerArgs(endpoints, strategyArgs)
worker, e := r.start(args)
if e != nil {
return &managedRecoveryFatal{Err: e}
}
joined, table := false, false
defer func() {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
var cleanupErr error
// Remove queues before killing the consumer.
if table {
_, cleanupErr = r.nft(cleanup, "delete table inet "+managedAssetTable+"\n")
}
if !joined {
select {
case e := <-worker.done:
joined = true
cleanupErr = errors.Join(cleanupErr, e, errors.New("asset worker exited unexpectedly"))
default:
}
}
if !joined {
cleanupErr = errors.Join(cleanupErr, worker.stop())
select {
case e := <-worker.done:
joined = true
if e != nil {
var exit *exec.ExitError
if !errors.As(e, &exit) || !exit.ProcessState.Sys().(syscall.WaitStatus).Signaled() || exit.ProcessState.Sys().(syscall.WaitStatus).Signal() != syscall.SIGKILL {
cleanupErr = errors.Join(cleanupErr, e)
}
}
case <-cleanup.Done():
cleanupErr = errors.Join(cleanupErr, errors.New("asset worker join timeout"))
}
}
if cleanupErr != nil {
err = &managedRecoveryFatal{Err: errors.Join(err, &managedAssetCleanupFailure{Err: cleanupErr})}
}
}()
readyCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
tick := time.NewTicker(25 * time.Millisecond)
defer tick.Stop()
for {
select {
case e := <-worker.done:
joined = true
return &managedRecoveryFatal{Err: errors.Join(e, errors.New("asset worker exited before READY"))}
default:
}
ready, e := r.queue(worker.pid)
if e != nil {
return &managedRecoveryFatal{Err: e}
}
if ready {
break
}
select {
case <-readyCtx.Done():
if ctx.Err() != nil {
return ctx.Err()
}
return &managedRecoveryFatal{Err: errors.New("asset queue READY timeout")}
case <-tick.C:
}
}
// nft's transaction is atomic; failed creation owns no state to delete.
if _, e := r.nft(ctx, managedAssetRules); e != nil {
return &managedRecoveryFatal{Err: e}
}
table = true
// Recheck ownership after installing the queue and before the first capture.
if ready, e := r.queue(worker.pid); e != nil || !ready {
return &managedRecoveryFatal{Err: errors.Join(e, errors.New("owned queue lost before capture"))}
}
probeCtx, stopProbe := context.WithCancel(ctx)
watchDone := make(chan struct{})
exited := make(chan error, 1)
go func() {
defer close(watchDone)
select {
case e := <-worker.done:
exited <- errors.Join(e, errors.New("asset worker exited during probe"))
stopProbe()
case <-probeCtx.Done():
}
}()
defer func() {
stopProbe()
<-watchDone
select {
case e := <-exited:
joined = true
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
default:
}
}()
return capture(probeCtx)
}
func managedProbeWorkerArgs(endpoints []string, strategyArgs string) []string {
args := []string{"--qnum=200", "--fwmark=0x10000000", "--lua-init=@/opt/zapret2/lua/zapret-lib.lua", "--lua-init=@/opt/zapret2/lua/zapret-antidpi.lua", "--filter-tcp=443", "--hostlist-domains=" + strings.Join(endpoints, ",")}
return append(args, strings.Fields(strategyArgs)...)
}
// Collector errors conservatively carry a fatal wrapper. Only known transport
// failures and genuine rejected responses are nonpasses; malformed evidence,
// missing executables, unknown errors and worker failures remain infrastructure.
func managedAssetTransferNonpass(err error) bool {
if err == nil {
return false
}
switch e := err.(type) {
case *exec.ExitError:
switch e.ExitCode() {
case 6, 7, 28, 35, 51, 52, 55, 56, 60:
return true
}
return false
case interface{ Unwrap() []error }:
children := e.Unwrap()
if len(children) == 0 {
return false
}
for _, child := range children {
if !managedAssetTransferNonpass(child) {
return false
}
}
return true
case interface{ Unwrap() error }:
return managedAssetTransferNonpass(e.Unwrap())
default:
return err == context.DeadlineExceeded || err.Error() == "resource origin verification failed" || err.Error() == "image HTTP status"
}
}
@@ -0,0 +1,354 @@
package main
import (
"bytes"
"context"
"errors"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
)
func assetRuntimeFixture(t *testing.T) (managedAssetRuntime, *[]string, chan error) {
t.Helper()
events := []string{}
done := make(chan error, 1)
r := managedAssetRuntime{proxy: "socks5://127.0.0.1:1080", guard: func() error { return nil }}
r.nft = func(_ context.Context, input string) (string, error) {
if input == "list tables\n" {
events = append(events, "inventory")
return "", nil
}
if strings.HasPrefix(input, "delete") {
events = append(events, "delete")
return "", nil
}
if len(events) == 0 || events[len(events)-1] != "ready" {
t.Fatal("queue activated before READY", events)
}
if !strings.Contains(input, "hook input") || !strings.Contains(input, "hook output") || !strings.Contains(input, "meta mark & 0x10000000 == 0") {
t.Fatal(input)
}
events = append(events, "activate")
return "", nil
}
r.queue = func(pid int) (bool, error) {
if pid == 0 {
return false, nil
}
events = append(events, "ready")
return true, nil
}
r.start = func(args []string) (managedAssetWorker, error) {
if strings.Contains(strings.Join(args, " "), "--daemon") {
t.Fatal(args)
}
events = append(events, "start")
return managedAssetWorker{pid: 123, done: done, stop: func() error { events = append(events, "join"); done <- nil; return nil }}, nil
}
r.curl = func(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
events = append(events, "curl")
config, err := io.ReadAll(stdin)
if err != nil {
return err
}
raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(config), "url = "), "\n"))
if err != nil {
return err
}
var diag bytes.Buffer
resourceWire(raw, "", out, &diag)
tls := diag.String()
if strings.Contains(strings.Join(args, " "), "--tlsv1.3") {
tls = strings.ReplaceAll(tls, "TLSv1.2", "TLSv1.3")
}
_, err = io.WriteString(diagnostic, tls)
return err
}
return r, &events, done
}
func TestManagedAssetRuntimeOwnedCollector(t *testing.T) {
d, front, _ := recoveryFixture(t, true)
if err := d.transaction.Apply(d.db, []int64{front}); err != nil {
t.Fatal(err)
}
before := map[string][]byte{}
entries, err := os.ReadDir(d.transaction.Dir)
if err != nil {
t.Fatal(err)
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
data, err := os.ReadFile(filepath.Join(d.transaction.Dir, entry.Name()))
if err != nil {
t.Fatal(err)
}
before[entry.Name()] = data
}
r, events, _ := assetRuntimeFixture(t)
c := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com", "yt3.ggpht.com"}, Args: managedAssetSeeds[0]}
m, err := r.verify(context.Background(), c, managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}})
if err != nil || len(m) != 4 {
t.Fatal(m, err)
}
count := 0
for _, e := range *events {
if e == "curl" {
count++
}
}
if count != 12 || strings.Join((*events)[len(*events)-2:], ",") != "delete,join" {
t.Fatal(*events)
}
for _, measurement := range m {
if measurement.Args != c.Args {
t.Fatal("unbound strategy")
}
if _, _, err := validateAssetTransfers(measurement); err != nil {
t.Fatal(err)
}
}
for name, data := range before {
after, err := os.ReadFile(filepath.Join(d.transaction.Dir, name))
if err != nil || !bytes.Equal(data, after) {
t.Fatal("probe mutated managed artifact", name, err)
}
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
}
func TestManagedAssetRuntimeFailedAtomicNFTInstallNeverDeletesForeignTable(t *testing.T) {
for _, mode := range []string{"preexisting", "raced-create", "install-failure"} {
t.Run(mode, func(t *testing.T) {
r, events, _ := assetRuntimeFixture(t)
r.nft = func(_ context.Context, input string) (string, error) {
if input == "list tables\n" {
if mode == "preexisting" {
return "table inet " + managedAssetTable, nil
}
return "", nil
}
if strings.HasPrefix(input, "delete") {
t.Fatal("deleted foreign table after failed atomic install")
}
if !strings.HasPrefix(input, "create table inet "+managedAssetTable+"\n") {
t.Fatal("table creation must reject racing existing table", input)
}
return "", errors.New(mode)
}
c := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com"}, Args: managedAssetSeeds[0]}
m, err := r.verify(context.Background(), c, managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}})
var fatal *managedRecoveryFatal
if len(m) != 0 || !errors.As(err, &fatal) || errors.Is(err, errCandidateNonpass) {
t.Fatal(m, err)
}
if mode == "preexisting" {
if len(*events) != 0 {
t.Fatal("started worker over preexisting table", *events)
}
} else if strings.Join(*events, ",") != "start,ready,join" {
t.Fatal("failed install did not join owned worker", *events)
}
})
}
}
func TestManagedAssetRuntimeCancelFailureAndCleanup(t *testing.T) {
for _, mode := range []string{"cancel-ready", "cancel-curl", "worker", "nonpass", "cleanup"} {
t.Run(mode, func(t *testing.T) {
r, events, done := assetRuntimeFixture(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if mode == "cancel-ready" {
r.queue = func(pid int) (bool, error) {
if pid != 0 {
cancel()
}
return false, nil
}
}
if mode == "cancel-curl" {
r.curl = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error {
cancel()
return context.Canceled
}
}
if mode == "worker" {
r.curl = func(ctx context.Context, _ []string, _ io.Reader, _, _ io.Writer) error {
done <- errors.New("worker failure")
<-ctx.Done()
return ctx.Err()
}
}
if mode == "nonpass" {
r.curl = func(_ context.Context, _ []string, _ io.Reader, out, diagnostic io.Writer) error {
resourceWire("https://i.ytimg.com/image", "cn", out, diagnostic)
return nil
}
}
if mode == "cleanup" {
nft := r.nft
r.nft = func(ctx context.Context, input string) (string, error) {
out, err := nft(ctx, input)
if strings.HasPrefix(input, "delete") {
err = errors.New("cleanup failed")
}
return out, err
}
}
c := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com"}, Args: managedAssetSeeds[0]}
m, err := r.verify(ctx, c, managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}})
if err == nil || len(m) != 0 {
t.Fatal(m, err)
}
var fatal *managedRecoveryFatal
if (mode == "worker" || mode == "cleanup") && !errors.As(err, &fatal) {
t.Fatal("missing fatal", err)
}
if mode == "nonpass" && !errors.Is(err, errCandidateNonpass) {
t.Fatal(err)
}
if strings.HasPrefix(mode, "cancel") && !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if mode == "cancel-ready" && (*events)[len(*events)-1] != "join" {
t.Fatal(*events)
}
if mode != "cancel-ready" && !strings.Contains(strings.Join(*events, ","), "delete") {
t.Fatal(*events)
}
})
}
}
func TestManagedAssetSeedsFallbackAndNoQualification(t *testing.T) {
for _, pass := range []bool{true, false} {
t.Run(strconv.FormatBool(pass), func(t *testing.T) {
d, front, _ := recoveryFixture(t, true)
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
if err := d.transaction.Apply(d.db, []int64{front, avatar, ggc}); err != nil {
t.Fatal(err)
}
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}}, nil
}
calls := 0
d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) {
calls++
if !pass || calls == 1 {
return nil, errCandidateNonpass
}
return []assetMeasurement{unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args)}, nil
}
err := repairManagedAssets(context.Background(), d, "i.ytimg.com")
if !pass {
if !errors.Is(err, errNoQualifiedStrategy) || calls != len(managedAssetSeeds) {
t.Fatal(calls, err)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front, avatar, ggc})
} else {
if err != nil || calls != 2 {
t.Fatal(calls, err)
}
ids, err := d.transaction.ActiveCandidates(d.db)
if err != nil || len(ids) != 4 {
t.Fatal(ids, err)
}
for _, required := range []int64{front, avatar, ggc} {
found := false
for _, id := range ids {
found = found || id == required
}
if !found {
t.Fatal(ids)
}
}
}
})
}
}
func TestManagedAssetPartialOverlapAndStageHealth(t *testing.T) {
d, front, _ := recoveryFixture(t, true)
old := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com", "yt3.ggpht.com"}, Args: "--lua-desync=fake"}
id, err := saveManagedAssetCandidate(d.db, old, []assetMeasurement{
unitImageMeasurement(t, "i.ytimg.com", "tls12", old.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", old.Args),
unitImageMeasurement(t, "yt3.ggpht.com", "tls12", old.Args), unitImageMeasurement(t, "yt3.ggpht.com", "tls13", old.Args),
})
if err != nil {
t.Fatal(err)
}
unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=multisplit", true)
if err := d.transaction.Apply(d.db, []int64{front, id}); err != nil {
t.Fatal(err)
}
r, _, _ := assetRuntimeFixture(t)
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}, proxy: r.proxy, runner: r.curl, resourceErr: &managedResourceFailure{"media", "unknown", nil}}, nil
}
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
d.health = func(context.Context) (string, error) {
t.Fatal("unrelated media gated asset stage")
return "DEGRADED", nil
}
d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) {
if c.ID != 0 {
return nil, errCandidateNonpass
}
if len(c.Endpoints) != 2 {
t.Fatal("partial overlap lost sibling", c)
}
return []assetMeasurement{
unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args),
unitImageMeasurement(t, "yt3.ggpht.com", "tls12", c.Args), unitImageMeasurement(t, "yt3.ggpht.com", "tls13", c.Args),
}, nil
}
if err := repairManagedAssets(context.Background(), d, "i.ytimg.com"); err != nil {
t.Fatal(err)
}
ids, err := d.transaction.ActiveCandidates(d.db)
if err != nil || len(ids) != 2 {
t.Fatal(ids, err)
}
replacement, err := readStrategyCandidate(d.db, ids[1])
if err != nil || len(replacement.Endpoints) != 2 {
t.Fatal(replacement, err)
}
}
func TestManagedAssetMalformedAndUnsafeCleanupNeverRestore(t *testing.T) {
for _, unsafe := range []bool{false, true} {
d, front, events := recoveryFixture(t, true)
if err := d.transaction.Apply(d.db, []int64{front}); err != nil {
t.Fatal(err)
}
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}}, nil
}
calls := 0
d.verifyAssets = func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error) {
calls++
err := errors.New("malformed collector metrics")
if unsafe {
return nil, &managedRecoveryFatal{Err: &managedAssetCleanupFailure{Err: err}}
}
return nil, &managedRecoveryFatal{Err: err}
}
err := repairManagedAssets(context.Background(), d, "i.ytimg.com")
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) || errors.Is(err, errNoQualifiedStrategy) || calls != 1 {
t.Fatal(calls, err)
}
if unsafe && (*events)[len(*events)-1] != "stop" {
t.Fatal("unsafe restoration", *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
}
}
@@ -0,0 +1,468 @@
package main
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"time"
)
var errManagedAssetVerifierUnwired = errors.New("owned temporary asset verifier unavailable")
// Dependency-only construction fails closed. The production monitor adapter
// replaces this callback with newManagedAssetVerifier, never a root/404 probe.
func unwiredManagedAssetVerifier(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error) {
return nil, errManagedAssetVerifierUnwired
}
type managedResourceFailure struct {
ProbeKind, Host string
Err error
}
func (e *managedResourceFailure) Error() string { return e.PublicError() }
func (e *managedResourceFailure) PublicError() string {
host := e.Host
if !discoveryHostname(host) {
host = "unknown"
}
kind := e.ProbeKind
if kind != "thumbnail" && kind != "avatar" && kind != "media" && kind != "front" {
kind = "unknown"
}
return "DEGRADED: " + kind + " " + host
}
func (e *managedResourceFailure) Unwrap() error { return e.Err }
// Ephemeral inventory: never serialize URLs, page bodies or Player responses.
type managedServiceInventory struct {
images map[string]string
media mediaResource
resourceErr error
proxy string
runner subprocessRunner
}
func managedServiceResources(cfg healthConfig, player func(context.Context) ([]byte, error), page playerRunner) func(context.Context) (managedServiceInventory, error) {
return func(ctx context.Context) (managedServiceInventory, error) {
i := managedServiceInventory{images: make(map[string]string), proxy: cfg.SOCKSProxy, runner: runResourceCurl}
body, err := player(ctx)
if err != nil {
return i, &managedResourceFailure{"front", managedFrontendHost, err}
}
var p struct {
VideoDetails struct {
Thumbnail struct {
Thumbnails []struct {
URL string `json:"url"`
} `json:"thumbnails"`
} `json:"thumbnail"`
} `json:"videoDetails"`
}
if json.Unmarshal(body, &p) == nil {
for _, t := range p.VideoDetails.Thumbnail.Thumbnails {
if _, e := privateResourceURL(t.URL, "i.ytimg.com"); e == nil {
i.images["i.ytimg.com"] = t.URL
break
}
}
}
i.media, _, err = playerMediaResource(body)
if err != nil {
i.resourceErr = &managedResourceFailure{"media", "unknown", err}
}
bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
defer cancel()
var output bytes.Buffer
out := &playerBoundedWriter{dst: &output, limit: 5 << 20}
// Fixed root origin; no redirects, credentials, user URL or diagnostics.
err = page(bounded, []string{"--disable", "--silent", "--show-error", "--fail", "--noproxy", "", "--proxy", cfg.SOCKSProxy, "--proto", "=https", "--max-time", "30", "--max-filesize", "5242880", "https://www.youtube.com/"}, out, io.Discard)
if ctx.Err() != nil {
return i, managedRecoveryContextError(ctx, err)
}
if err != nil || bounded.Err() != nil || out.exceeded {
return i, &managedResourceFailure{"avatar", "yt3.ggpht.com", resourceFailure(bounded, errors.Join(err, bounded.Err(), errors.New("fresh root metadata request failed")))}
}
if raw, e := avatarResourceFromPage(output.Bytes()); e == nil {
i.images["yt3.ggpht.com"] = raw
}
return i, nil
}
}
func managedServiceHealth(proxy string, front func(context.Context) (string, error), inventory func(context.Context) (managedServiceInventory, error), runner subprocessRunner) func(context.Context) (string, error) {
return func(ctx context.Context) (string, error) {
status, err := front(ctx)
if err != nil || status != "OK" {
return "DEGRADED", &managedResourceFailure{"front", managedFrontendHost, err}
}
i, err := inventory(ctx)
var fatal *managedRecoveryFatal
if ctx.Err() != nil || errors.As(err, &fatal) {
return "DEGRADED", err
}
missing := err
for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} {
kind := "thumbnail"
if host == "yt3.ggpht.com" {
kind = "avatar"
}
if i.images[host] == "" {
if missing == nil {
missing = &managedResourceFailure{kind, host, nil}
}
continue
}
for _, protocol := range []string{"tls12", "tls13"} {
if _, err := collectAssetMeasurement(ctx, proxy, i.images[host], host, protocol, "", runner); err != nil {
// At the live-health boundary, known bad image bodies/origins
// are resource damage. Preserve mixed guard/cancellation failures.
if ctx.Err() == nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) && managedAssetHealthNonpass(err) {
err = nil
}
return "DEGRADED", &managedResourceFailure{kind, host, err}
}
}
}
if missing != nil {
return "DEGRADED", missing
}
if i.resourceErr != nil {
return "DEGRADED", i.resourceErr
}
if i.media.url == "" {
return "DEGRADED", &managedResourceFailure{"media", "unknown", nil}
} // SABR unknown, not ready.
for _, protocol := range []string{"tls12", "tls13"} {
if _, err := collectMediaSamples(ctx, proxy, i.media, protocol, runner); err != nil {
return "DEGRADED", &managedResourceFailure{"media", i.media.host, err}
}
}
return "OK", nil
}
}
func managedFullDesiredSet(db *sql.DB, current, replacements []int64) ([]int64, error) {
replaceGroups := map[string]bool{}
for _, id := range replacements {
c, err := readStrategyCandidate(db, id)
if err != nil {
return nil, err
}
replaceGroups[c.Group] = true
}
var desired []int64
for _, id := range current {
c, err := readStrategyCandidate(db, id)
if err != nil {
return nil, err
}
if !replaceGroups[c.Group] {
desired = append(desired, id)
}
}
return append(desired, replacements...), nil
}
func repairManagedAssets(ctx context.Context, d managedRecoveryDeps, host string) (err error) {
if d.resources == nil || d.verifyAssets == nil {
return errManagedAssetVerifierUnwired
}
i, err := d.resources(ctx)
var fatal *managedRecoveryFatal
if ctx.Err() != nil || errors.As(err, &fatal) {
return managedRecoveryContextError(ctx, err)
}
if err != nil && i.images[host] == "" {
return err
}
if i.images[host] == "" {
return &managedResourceFailure{managedImageProbeKind(host), host, nil}
}
m := *d.transaction
current, err := m.ActiveCandidates(d.db)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
saved, err := listQualifiedCandidates(d.db, "assets", 10)
if err != nil {
return &managedRecoveryFatal{Err: err}
}
// Saved definitions first, then a bounded readonly empirical seed inventory.
// Include overlapping siblings in seeds; never discard a partial host set.
endpoints := []string{host}
for _, id := range current {
old, e := readStrategyCandidate(d.db, id)
if e != nil {
return &managedRecoveryFatal{Err: e}
}
if old.Group == "assets" {
for _, a := range old.Endpoints {
if a == host {
endpoints = append([]string(nil), old.Endpoints...)
}
}
}
}
for _, args := range managedAssetSeeds {
saved = append(saved, storedCandidate{Group: "assets", Endpoints: endpoints, Args: args})
}
for _, c := range saved {
covers := false
for _, endpoint := range c.Endpoints {
covers = covers || endpoint == host
}
if !covers {
continue
}
var desired []int64
for _, id := range current {
old, e := readStrategyCandidate(d.db, id)
if e != nil {
return &managedRecoveryFatal{Err: e}
}
overlap := false
for _, a := range old.Endpoints {
for _, b := range c.Endpoints {
overlap = overlap || a == b
}
}
if old.Group != "assets" || !overlap {
desired = append(desired, id)
}
// A partially overlapping multi-host candidate cannot lose its sibling.
if old.Group == "assets" && overlap {
for _, a := range old.Endpoints {
found := false
for _, b := range c.Endpoints {
found = found || a == b
}
if !found {
covers = false
}
}
}
}
if !covers {
continue
}
if e := (&candidate{Args: c.Args}).validateArgs(); e != nil {
return &managedRecoveryFatal{Err: e}
}
// All temporary workers must be joined by Verify before restoration/apply.
e := d.run(ctx, managedZapretScript, "stop")
var measurements []assetMeasurement
if e == nil {
measurements, e = d.verifyAssets(ctx, c, i)
}
var unsafeCleanup *managedAssetCleanupFailure
if errors.As(e, &unsafeCleanup) {
// Never reactivate the permanent service over an unjoined worker or
// residual owned queue. The fatal monitor exit retains last-good files.
return e
}
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
restoreErr := d.run(cleanup, managedZapretScript, "start")
cancel()
if restoreErr != nil {
return &managedRecoveryFatal{Err: errors.Join(e, restoreErr)}
}
if e != nil {
if errors.As(e, &fatal) {
return e
}
if ctx.Err() != nil || errors.Is(e, context.Canceled) {
return managedRecoveryContextError(ctx, e)
}
if errors.Is(e, errCandidateNonpass) {
continue
}
return e
}
if e = ctx.Err(); e != nil {
return e
}
id, e := saveManagedAssetCandidate(d.db, c, measurements)
if e != nil {
return &managedRecoveryFatal{Err: e}
}
m.Health = func() (string, error) {
// Asset-stage apply must not be vetoed by unrelated media damage.
// Global service health still gates READY after recovery.
var status string
var healthErr error
if d.frontHealth == nil {
status, healthErr = d.health(ctx)
} else {
status, healthErr = managedAssetApplyHealth(ctx, d, c.Endpoints)
}
var failure *managedResourceFailure
var fatal *managedRecoveryFatal
if ctx.Err() != nil {
return "DEGRADED", managedRecoveryContextError(ctx, healthErr)
}
// Apply treats a non-OK status with no error as a rejection. Never
// erase a safety failure or cancellation joined to ordinary damage.
if !errors.As(healthErr, &fatal) && !errors.Is(healthErr, context.Canceled) && !errors.Is(healthErr, context.DeadlineExceeded) && errors.As(healthErr, &failure) {
return "DEGRADED", nil
}
return status, healthErr
}
m.Reload = func() error {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
return d.run(cleanup, managedZapretScript, "restart")
}
e = m.Apply(d.db, append(desired, id))
if e == nil {
return nil
}
var rejection *managedHealthRejection
var rollback *managedRollbackError
if errors.As(e, &fatal) || errors.As(e, &rollback) {
return &managedRecoveryFatal{Err: e}
}
if ctx.Err() != nil || errors.Is(e, context.Canceled) || errors.Is(e, context.DeadlineExceeded) {
return managedRecoveryContextError(ctx, e)
}
if !errors.As(e, &rejection) {
return &managedRecoveryFatal{Err: e}
}
}
return errNoQualifiedStrategy
}
var managedAssetSeeds = [...]string{
"--payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld",
"--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:tcp_ts=-1000:repeats=1 --lua-desync=multidisorder:pos=1,midsld",
"--payload=tls_client_hello --lua-desync=multidisorder:pos=1,midsld",
"--payload=tls_client_hello --lua-desync=hostfakesplit:disorder_after:tcp_md5:repeats=1 --lua-desync=multidisorder:pos=1,midsld",
"--payload=tls_client_hello --lua-desync=multisplit:pos=1,midsld",
}
func managedAssetApplyHealth(ctx context.Context, d managedRecoveryDeps, endpoints []string) (string, error) {
status, err := d.frontHealth(ctx)
if err != nil || status != "OK" {
return "DEGRADED", err
}
i, err := d.resources(ctx)
var fatal *managedRecoveryFatal
if ctx.Err() != nil || errors.As(err, &fatal) {
return "DEGRADED", managedRecoveryContextError(ctx, err)
}
if err != nil {
for _, host := range endpoints {
if i.images[host] == "" {
return "DEGRADED", err
}
}
}
// Production's normal service collector uses the same guarded SOCKS proxy.
// An adapter installs this callback with a candidate-independent collector.
if i.resourceErr != nil && len(i.images) == 0 {
return "DEGRADED", i.resourceErr
}
return managedAssetInventoryHealth(ctx, i, endpoints)
}
func managedAssetInventoryHealth(ctx context.Context, i managedServiceInventory, endpoints []string) (string, error) {
if i.runner == nil || i.proxy == "" {
return "DEGRADED", errors.New("asset health collector unavailable")
}
for _, host := range endpoints {
if i.images[host] == "" {
return "DEGRADED", &managedResourceFailure{managedImageProbeKind(host), host, nil}
}
for _, protocol := range []string{"tls12", "tls13"} {
if _, err := collectAssetMeasurement(ctx, i.proxy, i.images[host], host, protocol, "", i.runner); err != nil {
if ctx.Err() != nil {
return "DEGRADED", managedRecoveryContextError(ctx, err)
}
if managedAssetHealthNonpass(err) {
return "DEGRADED", nil
}
return "DEGRADED", &managedResourceFailure{managedImageProbeKind(host), host, err}
}
}
}
return "OK", nil
}
// Collector wrappers are conservatively fatal. At the post-activation health
// boundary, rejected origins/bodies are ordinary damage, not infrastructure.
// Every joined leaf must be a known nonpass; mixed safety failures stay fatal.
func managedAssetHealthNonpass(err error) bool {
switch e := err.(type) {
case interface{ Unwrap() []error }:
children := e.Unwrap()
if len(children) == 0 {
return false
}
for _, child := range children {
if !managedAssetHealthNonpass(child) {
return false
}
}
return true
case interface{ Unwrap() error }:
return managedAssetHealthNonpass(e.Unwrap())
default:
if err == nil || err == context.Canceled {
return false
}
return managedAssetTransferNonpass(err) || err.Error() == "image body exceeds limit" || err.Error() == "invalid image dimensions" || err.Error() == "invalid image body"
}
}
func managedImageProbeKind(host string) string {
if host == "i.ytimg.com" {
return "thumbnail"
}
if host == "yt3.ggpht.com" {
return "avatar"
}
return "unknown"
}
func saveManagedAssetCandidate(db *sql.DB, c storedCandidate, measurements []assetMeasurement) (int64, error) {
if len(measurements) != 2*len(c.Endpoints) {
return 0, fmt.Errorf("incomplete asset verification matrix")
}
seen := map[string]bool{}
for _, m := range measurements {
member := false
for _, host := range c.Endpoints {
member = member || host == m.Endpoint
}
key := m.Endpoint + "/" + m.Protocol
if !member || seen[key] || m.Args != c.Args {
return 0, fmt.Errorf("asset verification matrix definition mismatch")
}
seen[key] = true
if _, _, err := validateAssetTransfers(m); err != nil {
return 0, err
}
}
tx, err := db.Begin()
if err != nil {
return 0, err
}
defer tx.Rollback()
id, err := insertStrategyCandidate(tx, "assets", c.Endpoints, c.Args, nil)
if err != nil {
return 0, err
}
for _, m := range measurements {
if err := saveAssetEvidence(tx, id, m, time.Now()); err != nil {
return 0, err
}
}
if err := tx.Commit(); err != nil {
return 0, err
}
return id, nil
}
+82 -27
View File
@@ -6,6 +6,7 @@ import (
"database/sql" "database/sql"
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"os" "os"
@@ -55,32 +56,55 @@ func managedDependencies(player playerRunner) managedDeps {
} }
func managedArguments(args []string) (string, []int64, error) { func managedArguments(args []string) (string, []int64, error) {
invalid := fmt.Errorf("managed accepts init, reconcile, status, active-ids, or apply with one or both candidate flags (full desired set); no path flags") invalid := fmt.Errorf("managed accepts init, reconcile, status [--assets], active-ids, or apply with frontend/googlevideo/asset candidate flags (full desired set); no path flags")
if len(args) == 2 && args[0] == "status" && args[1] == "--assets" {
return "status", nil, nil
}
if len(args) == 1 && (args[0] == "init" || args[0] == "reconcile" || args[0] == "status" || args[0] == "active-ids") { if len(args) == 1 && (args[0] == "init" || args[0] == "reconcile" || args[0] == "status" || args[0] == "active-ids") {
return args[0], nil, nil return args[0], nil, nil
} }
if (len(args) != 3 && len(args) != 5) || args[0] != "apply" { if len(args) < 3 || len(args)%2 != 1 || args[0] != "apply" {
return "", nil, invalid return "", nil, invalid
} }
selected := map[string]int64{} selected := map[string][]int64{}
seen := map[int64]bool{}
scopeSeen := false
for i := 1; i < len(args); i += 2 { for i := 1; i < len(args); i += 2 {
key := args[i] key := args[i]
if selected[key] != 0 || (key != "--frontend-candidate" && key != "--googlevideo-candidate") || args[i+1] == "" || strings.Trim(args[i+1], "0123456789") != "" { if key == "--googlevideo-scope" {
if scopeSeen || (args[i+1] != "exact" && args[i+1] != "suffix") {
return "", nil, invalid return "", nil, invalid
} }
id, err := strconv.ParseInt(args[i+1], 10, 64) scopeSeen = true
if err != nil || id <= 0 { continue
}
if (len(selected[key]) != 0 && key != "--asset-candidate") || (key != "--frontend-candidate" && key != "--googlevideo-candidate" && key != "--asset-candidate") {
return "", nil, invalid
}
values := strings.Split(args[i+1], ",")
if key != "--asset-candidate" && len(values) != 1 {
return "", nil, invalid
}
for _, value := range values {
if value == "" || strings.Trim(value, "0123456789") != "" {
return "", nil, invalid
}
id, err := strconv.ParseInt(value, 10, 64)
if err != nil || id <= 0 || seen[id] {
return "", nil, invalid
}
seen[id] = true
selected[key] = append(selected[key], id)
}
if len(selected["--asset-candidate"]) > 2 {
return "", nil, invalid return "", nil, invalid
} }
selected[key] = id
} }
var ids []int64 var ids []int64
for _, key := range []string{"--frontend-candidate", "--googlevideo-candidate"} { for _, key := range []string{"--frontend-candidate", "--googlevideo-candidate", "--asset-candidate"} {
if id := selected[key]; id != 0 { ids = append(ids, selected[key]...)
ids = append(ids, id)
} }
} if len(ids) == 0 || (scopeSeen && len(selected["--googlevideo-candidate"]) == 0) {
if len(ids) == 2 && ids[0] == ids[1] {
return "", nil, invalid return "", nil, invalid
} }
return "apply", ids, nil return "apply", ids, nil
@@ -196,7 +220,7 @@ func managedCheckIdentity(m *managedTransaction) error {
} }
// Status and supervisor reads share the same locked, DB-validated snapshot. // Status and supervisor reads share the same locked, DB-validated snapshot.
func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB, error), activeOnly bool) error { func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB, error), activeOnly bool, requestAssets ...bool) error {
return m.locked(func() error { return m.locked(func() error {
j, err := m.load() j, err := m.load()
if err != nil { if err != nil {
@@ -213,23 +237,29 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB,
return errManagedDrift return errManagedDrift
} }
groups := map[string]bool{} groups := map[string]bool{}
known := j.State == "committed" && j.Version == 2 && string(b) == managedBaseline && len(j.Candidates) == 0 assetHosts := map[string]bool{}
if j.State == "committed" && j.Version == 2 && len(j.Candidates) != 0 { known := j.State == "committed" && j.Version >= 2 && string(b) == managedBaseline && len(j.Candidates) == 0
if j.State == "committed" && j.Version >= 2 && len(j.Candidates) != 0 {
db, err := openDB() db, err := openDB()
if db != nil { if db != nil {
defer db.Close() defer db.Close()
} }
if err == nil && db != nil { if err == nil && db != nil {
rendered, err := renderManagedConfig(db, j.Candidates) rendered, err := renderStoredManagedConfigScope(db, j.Candidates, j.Scope)
if err == nil && managedHash(rendered) == j.Hash { if err == nil && managedHash(rendered) == j.Hash {
known = true known = true
for _, id := range j.Candidates { for _, id := range j.Candidates {
c, err := readStrategyCandidate(db, id) c, err := readStrategyCandidatePolicy(db, id, false)
if err != nil || !c.Qualified { if err != nil || !c.Qualified {
known = false known = false
break break
} }
groups[c.Group] = true groups[c.Group] = true
if c.Group == "assets" {
for _, host := range c.Endpoints {
assetHosts[host] = true
}
}
} }
} }
} }
@@ -247,6 +277,12 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB,
missing = append(missing, group) missing = append(missing, group)
} }
} }
if groups["assets"] {
active = append(active, "assets")
}
if (groups["assets"] || (len(requestAssets) != 0 && requestAssets[0])) && (!known || len(assetHosts) != 2) {
missing = append(missing, "assets")
}
if activeOnly { if activeOnly {
if !known { if !known {
return fmt.Errorf("active candidate IDs unresolved; deployment preserved by hash") return fmt.Errorf("active candidate IDs unresolved; deployment preserved by hash")
@@ -268,6 +304,9 @@ func managedReport(m *managedTransaction, out io.Writer, openDB func() (*sql.DB,
reportIDs = j.Candidates reportIDs = j.Candidates
} }
_, err = fmt.Fprintf(out, "managed-test generation=%d hash=%s state=%s groups=%s active_candidate_ids=%v unresolved_groups=%s\nLive health: NOT CHECKED (Player API is not GGC/video playback)\n", j.Generation, j.Hash, j.State, groupText, reportIDs, strings.Join(missing, ",")) _, err = fmt.Fprintf(out, "managed-test generation=%d hash=%s state=%s groups=%s active_candidate_ids=%v unresolved_groups=%s\nLive health: NOT CHECKED (Player API is not GGC/video playback)\n", j.Generation, j.Hash, j.State, groupText, reportIDs, strings.Join(missing, ","))
if err == nil && j.Version == 3 {
_, err = fmt.Fprintf(out, "Googlevideo scope: %s (suffix is authorized extrapolation, not all-host qualification)\n", j.Scope)
}
return err return err
}) })
} }
@@ -329,18 +368,32 @@ func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
if command == "reconcile" { if command == "reconcile" {
err = managedPrestartReconcile(m, d) err = managedPrestartReconcile(m, d)
} else if command == "status" || command == "active-ids" { } else if command == "status" || command == "active-ids" {
err = managedReport(m, stdout, d.openDB, command == "active-ids") err = managedReport(m, stdout, d.openDB, command == "active-ids", len(args) == 2 && args[1] == "--assets")
} else { } else {
expected := map[int64]string{} expected := map[int64]string{}
var scopes []GooglevideoScope
for i := 1; i < len(args); i += 2 { for i := 1; i < len(args); i += 2 {
id, _ := strconv.ParseInt(args[i+1], 10, 64) if args[i] == "--googlevideo-scope" {
expected[id] = strings.TrimSuffix(strings.TrimPrefix(args[i], "--"), "-candidate") scopes = append(scopes, GooglevideoScope(args[i+1]))
continue
} }
err = managedActivate(m, d, ids, expected) group := strings.TrimSuffix(strings.TrimPrefix(args[i], "--"), "-candidate")
if group == "asset" {
group = "assets"
}
for _, value := range strings.Split(args[i+1], ",") {
id, _ := strconv.ParseInt(value, 10, 64)
expected[id] = group
}
}
err = managedActivate(m, d, ids, expected, scopes...)
} }
} }
if err != nil { if err != nil {
fmt.Fprintln(stderr, "error:", err) fmt.Fprintln(stderr, "error:", err)
if errors.Is(err, errManagedScopeUnresolved) {
return 78
}
return 1 return 1
} }
if command == "init" || command == "reconcile" || command == "apply" { if command == "init" || command == "reconcile" || command == "apply" {
@@ -380,7 +433,7 @@ func managedPrestartReconcile(m *managedTransaction, d managedDeps) (err error)
return err return err
} }
baselineHash := managedHash([]byte(managedBaseline)) baselineHash := managedHash([]byte(managedBaseline))
if j.Version != 2 || (len(j.Candidates) == 0 && j.Hash != baselineHash) || (j.State == "prepared" && len(j.PendingCandidates) == 0 && j.Next != baselineHash) { if j.Version < 2 || (len(j.Candidates) == 0 && j.Hash != baselineHash) || (j.State == "prepared" && len(j.PendingCandidates) == 0 && j.Next != baselineHash) {
return fmt.Errorf("startup candidate state unresolved; refusing activation") return fmt.Errorf("startup candidate state unresolved; refusing activation")
} }
return nil return nil
@@ -395,7 +448,9 @@ func managedPrestartReconcile(m *managedTransaction, d managedDeps) (err error)
return err return err
} }
func managedActivate(m *managedTransaction, d managedDeps, ids []int64, expected map[int64]string) error { var errManagedScopeUnresolved = errors.New("managed deployment policy unresolved; explicit reconcile or fresh managed state required")
func managedActivate(m *managedTransaction, d managedDeps, ids []int64, expected map[int64]string, scopes ...GooglevideoScope) error {
info, err := os.Stat(d.script) info, err := os.Stat(d.script)
if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0111 == 0 { if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0111 == 0 {
return fmt.Errorf("target zapret daemon init executable unavailable") return fmt.Errorf("target zapret daemon init executable unavailable")
@@ -430,6 +485,9 @@ func managedActivate(m *managedTransaction, d managedDeps, ids []int64, expected
return fmt.Errorf("candidate database missing") return fmt.Errorf("candidate database missing")
} }
defer db.Close() defer db.Close()
if _, err := m.ActiveCandidates(db); err != nil {
return fmt.Errorf("%w: %v", errManagedScopeUnresolved, err)
}
for _, id := range ids { for _, id := range ids {
group := expected[id] group := expected[id]
c, err := readStrategyCandidate(db, id) c, err := readStrategyCandidate(db, id)
@@ -437,11 +495,8 @@ func managedActivate(m *managedTransaction, d managedDeps, ids []int64, expected
return fmt.Errorf("invalid %s candidate ID", group) return fmt.Errorf("invalid %s candidate ID", group)
} }
} }
if _, err := renderManagedConfig(db, ids); err != nil {
return err
}
if err := m.Reconcile(db); err != nil { if err := m.Reconcile(db); err != nil {
return err return err
} }
return m.Apply(db, ids) return m.Apply(db, ids, scopes...)
} }
+68 -2
View File
@@ -15,25 +15,64 @@ const managedFirewallConfig = "INIT_APPLY_FW=1\n" +
// renderManagedConfig accepts identifiers, not caller-supplied strategies or scope. // renderManagedConfig accepts identifiers, not caller-supplied strategies or scope.
// Qualification is the existing complete endpoint × TLS evidence rule. // Qualification is the existing complete endpoint × TLS evidence rule.
func renderManagedConfig(db *sql.DB, ids []int64) ([]byte, error) { func renderManagedConfig(db *sql.DB, ids []int64) ([]byte, error) {
return renderManagedConfigScope(db, ids, GooglevideoScopeExact)
}
type GooglevideoScope string
const (
GooglevideoScopeExact GooglevideoScope = "exact"
GooglevideoScopeSuffix GooglevideoScope = "suffix"
)
func (s GooglevideoScope) valid() bool {
return s == "" || s == GooglevideoScopeExact || s == GooglevideoScopeSuffix
}
func renderManagedConfigScope(db *sql.DB, ids []int64, scope GooglevideoScope) ([]byte, error) {
return renderManagedConfigPolicy(db, ids, scope, true)
}
// Only journal verification may reconstruct bytes without fresh qualification.
func renderStoredManagedConfigScope(db *sql.DB, ids []int64, scope GooglevideoScope) ([]byte, error) {
return renderManagedConfigPolicy(db, ids, scope, false)
}
func renderManagedConfigPolicy(db *sql.DB, ids []int64, scope GooglevideoScope, fresh bool) ([]byte, error) {
if !scope.valid() {
return nil, fmt.Errorf("unknown googlevideo scope %q", scope)
}
if db == nil { if db == nil {
return nil, fmt.Errorf("candidate definitions database required") return nil, fmt.Errorf("candidate definitions database required")
} }
var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil {
return nil, err
}
if version < 3 || version > 7 {
return nil, fmt.Errorf("unsupported candidate schema version %d", version)
}
var profiles []string var profiles []string
groups := map[string]bool{} groups := map[string]bool{}
seen := map[int64]bool{} seen := map[int64]bool{}
assets := map[string]bool{}
var otherScopes []string
for _, id := range ids { for _, id := range ids {
if id <= 0 || seen[id] { if id <= 0 || seen[id] {
return nil, fmt.Errorf("duplicate candidate %d", id) return nil, fmt.Errorf("duplicate candidate %d", id)
} }
seen[id] = true seen[id] = true
c, err := readStrategyCandidate(db, id) c, err := readStrategyCandidatePolicy(db, id, fresh)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if !c.Qualified { if !c.Qualified {
return nil, fmt.Errorf("candidate %d is not qualified", id) return nil, fmt.Errorf("candidate %d is not qualified", id)
} }
if groups[c.Group] { if c.Group == "assets" && version < 6 {
return nil, fmt.Errorf("assets require schema 6")
}
if groups[c.Group] && c.Group != "assets" {
return nil, fmt.Errorf("duplicate candidate group %q", c.Group) return nil, fmt.Errorf("duplicate candidate group %q", c.Group)
} }
v := candidate{Group: c.Group, Domains: append([]string(nil), c.Endpoints...), Args: c.Args} v := candidate{Group: c.Group, Domains: append([]string(nil), c.Endpoints...), Args: c.Args}
@@ -46,6 +85,16 @@ func renderManagedConfig(db *sql.DB, ids []int64) ([]byte, error) {
} }
domains := append([]string(nil), v.Domains...) domains := append([]string(nil), v.Domains...)
for i, host := range domains { for i, host := range domains {
if c.Group != "assets" {
otherScopes = append(otherScopes, host)
}
if c.Group == "assets" {
if assets[host] {
return nil, fmt.Errorf("overlapping asset endpoint")
}
assets[host] = true
domains[i] = "^" + host
}
if c.Group == "googlevideo" { if c.Group == "googlevideo" {
if !strings.HasSuffix(host, ".googlevideo.com") { if !strings.HasSuffix(host, ".googlevideo.com") {
return nil, fmt.Errorf("GGC scope must be an exact googlevideo host") return nil, fmt.Errorf("GGC scope must be an exact googlevideo host")
@@ -53,9 +102,26 @@ func renderManagedConfig(db *sql.DB, ids []int64) ([]byte, error) {
domains[i] = "^" + host domains[i] = "^" + host
} }
} }
if c.Group == "googlevideo" && scope == GooglevideoScopeSuffix {
if version < 7 || validateGooglevideoSuffixEvidencePolicy(db, c, fresh) != nil {
return nil, fmt.Errorf("suffix requires representative qualified hosts and verified media evidence")
}
// Authorized extrapolation, not evidence that all suffix hosts passed.
domains = []string{"googlevideo.com"}
}
groups[c.Group] = true groups[c.Group] = true
profiles = append(profiles, "--filter-tcp=443 --filter-l7=tls --hostlist-domains="+strings.Join(domains, ",")+" "+c.Args) profiles = append(profiles, "--filter-tcp=443 --filter-l7=tls --hostlist-domains="+strings.Join(domains, ",")+" "+c.Args)
} }
if scope == GooglevideoScopeSuffix && (!groups["googlevideo"] || len(assets) != 2) {
return nil, fmt.Errorf("suffix requires googlevideo and both qualified asset profiles")
}
for host := range assets {
for _, scope := range otherScopes {
if host == scope || strings.HasSuffix(host, "."+scope) {
return nil, fmt.Errorf("asset endpoint overlaps another profile")
}
}
}
if len(profiles) == 0 { if len(profiles) == 0 {
return nil, fmt.Errorf("qualified candidate required") return nil, fmt.Errorf("qualified candidate required")
} }
@@ -0,0 +1,208 @@
package main
import (
"context"
"errors"
"sort"
"strconv"
"time"
)
// Ordinary unresolved state, not a target/ownership guard failure.
type InsufficientAdaptiveSamples struct{}
func (*InsufficientAdaptiveSamples) Error() string {
return "DEGRADED: insufficient fresh ordinary signed GGC samples"
}
func (*InsufficientAdaptiveSamples) Unwrap() []error {
return []error{errGooglevideoInsufficientSamples, errNoQualifiedStrategy}
}
func managedGooglevideoSamples(ids []string, request func(context.Context, string) ([]byte, error)) func(context.Context) ([]mediaResource, error) {
return func(ctx context.Context) ([]mediaResource, error) {
byHost := map[string]mediaResource{}
for _, id := range ids {
body, err := request(ctx, id)
var fatal *managedRecoveryFatal
if ctx.Err() != nil || errors.As(err, &fatal) {
return nil, managedRecoveryContextError(ctx, err)
}
if err != nil {
continue
}
r, _, err := playerMediaResource(body)
if err != nil {
continue
} // Missing/SABR is unknown, never fabricated.
u, err := privateResourceURL(r.url, r.host)
if err != nil || u.Path != "/videoplayback" {
continue
}
q := u.Query()
expires, err := strconv.ParseInt(q.Get("expire"), 10, 64)
if err != nil || expires <= time.Now().Unix()+120 || (q.Get("sig") == "" && q.Get("signature") == "" && q.Get("lsig") == "") {
continue
}
byHost[r.host] = r
}
hosts := make([]string, 0, len(byHost))
for host := range byHost {
hosts = append(hosts, host)
}
sort.Strings(hosts)
if validateGooglevideoHosts(hosts) != nil {
return nil, &InsufficientAdaptiveSamples{}
}
resources := make([]mediaResource, 0, len(hosts))
for _, host := range hosts {
resources = append(resources, byHost[host])
}
return resources, nil
}
}
func managedAdaptiveAssetsQualified(d managedRecoveryDeps, ids []int64) (bool, error) {
covered := map[string]bool{}
for _, id := range ids {
c, err := readStrategyCandidate(d.db, id)
if err != nil {
return false, err
}
if c.Group == "assets" && c.Qualified {
for _, host := range c.Endpoints {
covered[host] = true
}
}
}
return covered["i.ytimg.com"] && covered["yt3.ggpht.com"], nil
}
func recoverManagedGooglevideo(ctx context.Context, d managedRecoveryDeps) (int64, error) {
if !d.service || d.mediaSamples == nil || d.verifyGooglevideo == nil {
return 0, &InsufficientAdaptiveSamples{}
}
m := *d.transaction
current, err := m.ActiveCandidates(d.db)
if err != nil {
return 0, &managedRecoveryFatal{Err: err}
}
ok, err := managedAdaptiveAssetsQualified(d, current)
if err != nil {
return 0, &managedRecoveryFatal{Err: err}
}
if !ok {
return 0, &managedResourceFailure{"thumbnail", "unknown", nil}
}
status, err := managedAssetApplyHealth(ctx, d, []string{"i.ytimg.com", "yt3.ggpht.com"})
if err != nil {
return 0, err
}
if status != "OK" {
return 0, &managedHealthRejection{Status: status}
}
saved, err := listQualifiedCandidates(d.db, "googlevideo", 10)
if err != nil {
return 0, &managedRecoveryFatal{Err: err}
}
// Revalidate saved native definitions first. Old proof never authorizes apply.
trials := []storedCandidate{}
for _, c := range saved {
if c.Mode == QualificationAdaptive && validateNativeGooglevideoArgs(c.Args) == nil {
trials = append(trials, c)
}
}
trials = append(trials, storedCandidate{Args: managedGooglevideoArgs})
for _, c := range trials {
resources, err := d.mediaSamples(ctx) // Fresh on every stage, URLs stay private.
if err != nil {
return 0, err
}
c.ID, c.Group, c.Mode = 0, "googlevideo", QualificationAdaptive
c.Endpoints = nil
for _, r := range resources {
c.Endpoints = append(c.Endpoints, r.host)
}
if validateGooglevideoHosts(c.Endpoints) != nil {
return 0, &InsufficientAdaptiveSamples{}
}
err = d.run(ctx, managedZapretScript, "stop")
var proof GooglevideoQualificationProof
if err == nil {
proof, err = d.verifyGooglevideo(ctx, c, resources)
}
var unsafe *managedAssetCleanupFailure
if errors.As(err, &unsafe) {
return 0, err
}
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
restoreErr := d.run(cleanup, managedZapretScript, "start")
cancel()
if restoreErr != nil {
return 0, &managedRecoveryFatal{Err: errors.Join(err, restoreErr)}
}
var fatal *managedRecoveryFatal
if ctx.Err() != nil || errors.As(err, &fatal) {
return 0, managedRecoveryContextError(ctx, err)
}
if errors.Is(err, errGooglevideoInsufficientSamples) {
return 0, &InsufficientAdaptiveSamples{}
}
if errors.Is(err, errCandidateNonpass) {
continue
}
if err != nil {
return 0, managedRecoveryContextError(ctx, err)
}
if err := ctx.Err(); err != nil {
return 0, err
}
tx, err := d.db.Begin()
if err != nil {
return 0, &managedRecoveryFatal{Err: err}
}
id, err := insertStrategyCandidate(tx, "googlevideo", c.Endpoints, c.Args, nil, QualificationAdaptive)
if err == nil {
err = saveAdaptiveGooglevideoEvidence(tx, id, proof)
}
if err == nil {
err = tx.Commit()
} else {
_ = tx.Rollback()
}
if err != nil {
return 0, &managedRecoveryFatal{Err: err}
}
desired, err := managedFullDesiredSet(d.db, current, []int64{id})
if err != nil {
return 0, &managedRecoveryFatal{Err: err}
}
m.Reload = func() error {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
return d.run(cleanup, managedZapretScript, "restart")
}
m.Health = func() (string, error) {
status, err := d.health(ctx) // Fresh media byte health, not hostname-only.
var failure *managedResourceFailure
var fatal *managedRecoveryFatal
if ctx.Err() != nil {
return "DEGRADED", managedRecoveryContextError(ctx, err)
}
if errors.As(err, &failure) && !errors.As(err, &fatal) && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
return "DEGRADED", nil
}
return status, err
}
err = m.Apply(d.db, desired, GooglevideoScopeSuffix)
var rollback *managedRollbackError
var rejection *managedHealthRejection
if err != nil && !errors.As(err, &rejection) && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
return 0, &managedRecoveryFatal{Err: err}
}
if errors.As(err, &rollback) {
return 0, &managedRecoveryFatal{Err: err}
}
return id, err
}
return 0, errNoQualifiedStrategy
}
@@ -0,0 +1,142 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"image"
"image/png"
"io"
"reflect"
"strconv"
"strings"
"testing"
)
func TestManagedAdaptiveConfig(t *testing.T) {
for _, tc := range []struct {
mode, service, ids string
valid bool
}{
{"", "", "", true}, {"adaptive", "1", `["dQw4w9WgXcQ","aqz-KE-bpKQ","jNQXAC9IVRw"]`, true},
{"adaptive", "0", "", false}, {"unknown", "1", "", false},
{"adaptive", "1", `[]`, false}, {"adaptive", "1", `["bad"]`, false},
{"adaptive", "1", `["dQw4w9WgXcQ","dQw4w9WgXcQ"]`, false},
{"adaptive", "1", `["dQw4w9WgXcQ","aqz-KE-bpKQ","jNQXAC9IVRw","aaaaaaaaaaa","bbbbbbbbbbb","ccccccccccc"]`, false},
} {
lookup := func(key string) (string, bool) {
v := map[string]string{"ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE": tc.mode, "ZAPRET_AUTOTUNE_YOUTUBE_SERVICE": tc.service, "ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS": tc.ids}[key]
return v, v != ""
}
cfg, err := parseHealthConfig(lookup)
if (err == nil) != tc.valid {
t.Fatalf("%+v: %v", tc, err)
}
mode, ids, _ := managedGooglevideoConfig(lookup, cfg.VideoID)
if tc.mode == "" && (mode != QualificationCold || !reflect.DeepEqual(ids, []string{cfg.VideoID})) {
t.Fatal(cfg)
}
}
}
func TestManagedAdaptiveRecoveryProductionPath(t *testing.T) {
for _, scenario := range []string{"qualified", "one-host", "unqualified-assets", "rollback"} {
t.Run(scenario, func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
r, _, resources, _ := googlevideoRuntimeFixture(t)
d.service, d.adaptive = true, true
d.frontHealth = d.health
assets := []int64{unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", scenario != "unqualified-assets"), unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)}
old := []int64{front}
if scenario != "unqualified-assets" {
old = append(old, assets...)
}
if err := d.transaction.Apply(d.db, old); err != nil {
t.Fatal(err)
}
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/sample.jpg", "yt3.ggpht.com": "https://yt3.ggpht.com/sample.jpg"}, proxy: r.proxy, runner: adaptiveImageRunner}, nil
}
d.mediaSamples = func(context.Context) ([]mediaResource, error) {
if scenario == "one-host" {
return resources[:1], nil
}
return resources, nil
}
d.verifyGooglevideo = r.verifyGooglevideo
if scenario == "rollback" {
d.health = func(context.Context) (string, error) {
return "DEGRADED", &managedResourceFailure{"media", resources[0].host, nil}
}
}
result, err := recoverManagedTarget(context.Background(), d)
if err != nil {
t.Fatal(err)
}
if scenario != "qualified" {
if result.GGCError == nil {
t.Fatal("unqualified stage succeeded")
}
assertActiveCandidates(t, d.transaction, d.db, old)
b, _ := d.transaction.read("config")
if strings.Contains(string(b), "--hostlist-domains=googlevideo.com") {
t.Fatal("unproven suffix")
}
return
}
if result.GGCError != nil || result.GooglevideoID == 0 {
t.Fatal(result)
}
c, err := readStrategyCandidate(d.db, result.GooglevideoID)
if err != nil || !c.Qualified || c.Mode != QualificationAdaptive {
t.Fatal(c, err)
}
p, err := readGooglevideoProof(d.db, c.ID)
if err != nil || p.SourceKind != NativeCircularPostWarmup {
t.Fatal(p, err)
}
j, err := d.transaction.load()
if err != nil || j.Scope != GooglevideoScopeSuffix {
t.Fatal(j, err)
}
assertActiveCandidates(t, d.transaction, d.db, append(old, c.ID))
if strings.Join(*events, ",") != "stop,start,restart" {
t.Fatal("restore must precede apply", *events)
}
})
}
}
func adaptiveImageRunner(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
b, err := io.ReadAll(stdin)
if err != nil {
return err
}
raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(b), "url = "), "\n"))
if err != nil {
return err
}
var body bytes.Buffer
if err := png.Encode(&body, image.NewRGBA(image.Rect(0, 0, 32, 32))); err != nil {
return err
}
fmt.Fprintf(out, "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n", body.Len())
_, _ = out.Write(body.Bytes())
metrics, _ := json.Marshal(map[string]interface{}{"url_effective": raw, "ssl_verify_result": 0, "num_redirects": 0, "http_code": 200, "time_total": 0.1, "speed_download": 10000, "size_download": body.Len()})
fmt.Fprint(out, resourceTrailer, string(metrics))
tls := "TLSv1.2"
if strings.Contains(strings.Join(args, " "), "--tlsv1.3") {
tls = "TLSv1.3"
}
fmt.Fprintln(diagnostic, "SSL connection using", tls)
return nil
}
func TestInsufficientAdaptiveSamplesCooldown(t *testing.T) {
err := &InsufficientAdaptiveSamples{}
if !errors.Is(err, errNoQualifiedStrategy) || !errors.Is(err, errGooglevideoInsufficientSamples) {
t.Fatal(err)
}
}
@@ -0,0 +1,216 @@
package main
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"os"
"reflect"
"strconv"
"strings"
"time"
)
// ponytail: one audited composite only. New native variants need an explicit
// semantic allowlist and regressions, not the generic shell-safe args validator.
const managedGooglevideoArgs = "--lua-init=@/opt/zapret2/lua/zapret-auto.lua --payload=all --in-range=-s34228 --lua-desync=circular:fails=1:retrans=2 --payload=tls_client_hello --lua-desync=fake:blob=fake_default_tls:tcp_ts=-1000:repeats=1:strategy=1 --lua-desync=multidisorder:pos=1,midsld:strategy=1 --lua-desync=fakeddisorder:pos=midsld:tcp_flags_set=SYN:strategy=2"
// Source inspected from the existing prod-derived prototype image. The default
// detector uses incoming seq >4096 for success and outgoing retrans <=32768;
// circular orchestrates only tagged instances, with contiguous strategy IDs.
// Incoming range must exceed 4096 (default x0-x0 cannot learn success).
const managedGooglevideoLuaHash = "aacfde0c95c3058f8e95f5d7d244398bdc03ebf846a8f17322129fb543366a3d"
const managedGooglevideoCollector = "owned-nfq-media-range-v1"
// Proof freshness is deliberately bounded to one day; expired proofs require a
// new owned run, never a refresh of their timestamp or cold summary rows.
const managedGooglevideoProofTTL = 24 * time.Hour
var errGooglevideoInsufficientSamples = errors.New("insufficient fresh ordinary signed GGC samples")
type GooglevideoRuntimeAttestation struct {
Collector string `json:"collector"`
LuaSHA256 string `json:"lua_sha256"`
RulesSHA256 string `json:"rules_sha256"`
WorkerArgs []string `json:"worker_args"`
CheckedAt time.Time `json:"checked_at"`
}
// Not serialized: DTO construction or unmarshalling cannot authorize a write.
// This binds the completed producer's exact facts, including failed warmups.
type googlevideoProducerSeal [32]byte
func googlevideoProofSeal(p GooglevideoQualificationProof) googlevideoProducerSeal {
raw, _ := json.Marshal(p)
return sha256.Sum256(raw)
}
func validateNativeGooglevideoArgs(args string) error {
if !reflect.DeepEqual(strings.Fields(args), strings.Fields(managedGooglevideoArgs)) {
return errors.New("unattested native circular semantics")
}
return nil
}
func (a *GooglevideoRuntimeAttestation) validate(c storedCandidate) error {
if err := a.validateStored(c); err != nil {
return err
}
if time.Since(a.CheckedAt) > managedGooglevideoProofTTL {
return errors.New("stale owned native attestation")
}
return nil
}
// Age affects new qualification, not reconstruction of journal-owned bytes.
func (a *GooglevideoRuntimeAttestation) validateStored(c storedCandidate) error {
if a == nil || a.Collector != managedGooglevideoCollector || a.LuaSHA256 != managedGooglevideoLuaHash || a.RulesSHA256 != fmt.Sprintf("%x", sha256.Sum256([]byte(managedAssetRules))) || !reflect.DeepEqual(a.WorkerArgs, managedProbeWorkerArgs(c.Endpoints, c.Args)) || a.CheckedAt.IsZero() || time.Since(a.CheckedAt) < -time.Minute {
return errors.New("missing, stale or mismatched owned native attestation")
}
return validateNativeGooglevideoArgs(c.Args)
}
// Caller gathers these resources from fresh Player queries for configured video
// samples. Never synthesize hosts by rewriting a signed URL. One-host inventory
// is an explicit unresolved result, before any worker or nft state is touched.
func newManagedGooglevideoVerifier(target managedDeps) func(context.Context, storedCandidate, []mediaResource) (GooglevideoQualificationProof, error) {
r := newManagedAssetRuntime(target)
guard := r.guard
r.guard = func() error {
if err := guard(); err != nil {
return err
}
lua, err := os.ReadFile("/opt/zapret2/lua/zapret-auto.lua")
if err != nil || fmt.Sprintf("%x", sha256.Sum256(lua)) != managedGooglevideoLuaHash {
return errors.New("unverified native automation Lua source")
}
return nil
}
return r.verifyGooglevideo
}
func (r managedAssetRuntime) verifyGooglevideo(ctx context.Context, c storedCandidate, resources []mediaResource) (GooglevideoQualificationProof, error) {
p := GooglevideoQualificationProof{Version: 1, Mode: QualificationAdaptive, SourceKind: NativeCircularPostWarmup, Args: c.Args}
if c.Group != "googlevideo" || c.Mode != QualificationAdaptive {
return p, errors.New("adaptive googlevideo definition required")
}
if err := validateNativeGooglevideoArgs(c.Args); err != nil {
return p, err
}
byHost := map[string]mediaResource{}
for _, resource := range resources {
u, err := privateResourceURL(resource.url, resource.host)
if err != nil || u.Path != "/videoplayback" || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") {
return p, errGooglevideoInsufficientSamples
}
q := u.Query()
expires, err := strconv.ParseInt(q.Get("expire"), 10, 64)
if err != nil || expires <= time.Now().Unix()+120 || (q.Get("sig") == "" && q.Get("signature") == "" && q.Get("lsig") == "") {
return p, errGooglevideoInsufficientSamples
}
byHost[resource.host] = resource
}
// Validate representative host diversity using the shared gate before capture.
for _, host := range c.Endpoints {
if _, ok := byHost[host]; !ok {
return p, errGooglevideoInsufficientSamples
}
p.Hosts = append(p.Hosts, GooglevideoHostProof{Host: host})
}
if err := validateGooglevideoHosts(c.Endpoints); err != nil {
return p, errGooglevideoInsufficientSamples
}
err := r.withOwnedWorker(ctx, c.Endpoints, c.Args, func(probeCtx context.Context) error {
for i := range p.Hosts {
h := &p.Hosts[i]
h.WarmupLimit = 10
start := time.Now()
warmCtx, cancel := context.WithTimeout(probeCtx, 120*time.Second)
learned := map[string]bool{}
for h.WarmupAttempts < h.WarmupLimit && len(learned) < 2 && warmCtx.Err() == nil {
protocol := []string{"tls12", "tls13"}[h.WarmupAttempts%2]
h.WarmupAttempts++
_, pass, err := collectMediaTransfer(warmCtx, r.proxy, byHost[h.Host], protocol, r.curl)
if err != nil {
h.WarmupFailures++
if !managedGooglevideoTransferNonpass(err) && warmCtx.Err() == nil {
cancel()
return err
}
} else if pass.Bytes < 1024 {
h.WarmupFailures++
} else {
learned[protocol] = true
}
}
h.WarmupMS = float64(time.Since(start).Microseconds()) / 1000
cancel()
if probeCtx.Err() != nil {
return probeCtx.Err()
}
if len(learned) != 2 || h.WarmupMS > 120000 {
return errCandidateNonpass
}
for _, protocol := range []string{"tls12", "tls13"} {
for n := 0; n < 3; n++ {
_, pass, err := collectMediaTransfer(probeCtx, r.proxy, byHost[h.Host], protocol, r.curl)
if err != nil {
if managedGooglevideoTransferNonpass(err) {
return errCandidateNonpass
}
return err
}
if pass.Bytes < 1024 {
return errCandidateNonpass
}
h.Passes = append(h.Passes, pass)
}
}
}
return nil
})
if err != nil {
return p, err
}
// Nothing attests until the owned consumer has been stopped and joined.
p.Attestation = &GooglevideoRuntimeAttestation{Collector: managedGooglevideoCollector, LuaSHA256: managedGooglevideoLuaHash, RulesSHA256: fmt.Sprintf("%x", sha256.Sum256([]byte(managedAssetRules))), WorkerArgs: managedProbeWorkerArgs(c.Endpoints, c.Args), CheckedAt: time.Now().UTC()}
if err := p.validate(c); err != nil {
return p, err
}
seal := googlevideoProofSeal(p)
p.producerSeal = &seal
return p, nil
}
func managedGooglevideoTransferNonpass(err error) bool {
if managedAssetTransferNonpass(err) {
return true
}
if err == nil {
return false
}
if err.Error() == "invalid media range or container" {
return true
}
if e, ok := err.(interface{ Unwrap() error }); ok {
return managedGooglevideoTransferNonpass(e.Unwrap())
}
return false
}
func validateGooglevideoHosts(hosts []string) error {
seen, roots, prefixes := map[string]bool{}, map[string]bool{}, map[string]bool{}
for _, host := range hosts {
prefix, root, ok := strings.Cut(strings.TrimSuffix(host, ".googlevideo.com"), "---sn-")
if !ok || !strings.HasSuffix(host, ".googlevideo.com") || !validDomain(host) || host != strings.ToLower(host) || seen[host] || root == "" || len(prefix) < 3 || !strings.HasPrefix(prefix, "rr") || strings.Trim(prefix[2:], "0123456789") != "" {
return errGooglevideoInsufficientSamples
}
seen[host], roots[strings.Split(root, "-")[0]], prefixes[prefix] = true, true, true
}
if len(hosts) < 3 || len(roots) < 2 || len(prefixes) < 2 {
return errGooglevideoInsufficientSamples
}
return nil
}
@@ -0,0 +1,296 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"strconv"
"strings"
"testing"
"time"
)
// No network: fake owned lifecycle and wire replies exercise the producer, not
// hardware readiness. The production factory adds the real guard/Lua hash gate.
func googlevideoRuntimeFixture(t *testing.T) (managedAssetRuntime, storedCandidate, []mediaResource, *[]string) {
t.Helper()
r, events, _ := assetRuntimeFixture(t)
c := storedCandidate{Group: "googlevideo", Mode: QualificationAdaptive, Args: managedGooglevideoArgs, Endpoints: []string{"rr1---sn-alpha.googlevideo.com", "rr2---sn-beta.googlevideo.com", "rr5---sn-gamma.googlevideo.com"}}
var resources []mediaResource
for _, host := range c.Endpoints {
resources = append(resources, mediaResource{url: fmt.Sprintf("https://%s/videoplayback?expire=%d&sig=private-secret", host, time.Now().Add(time.Hour).Unix()), host: host, kind: "video/mp4"})
}
r.curl = func(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
*events = append(*events, "media")
if strings.Contains(strings.Join(args, " "), "private-secret") || !strings.Contains(strings.Join(args, " "), "--range 0-65535") {
t.Fatal("private URL in argv or no range")
}
b, err := io.ReadAll(stdin)
if err != nil {
return err
}
raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(b), "url = "), "\n"))
if err != nil {
return err
}
body := make([]byte, 1024)
copy(body, []byte{0, 0, 0, 24, 'f', 't', 'y', 'p'})
fmt.Fprint(out, "HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp4\r\nContent-Length: 1024\r\nContent-Range: bytes 0-1023/4096\r\n\r\n")
out.Write(body)
metrics, _ := json.Marshal(map[string]interface{}{"url_effective": raw, "ssl_verify_result": 0, "num_redirects": 0, "http_code": 206, "time_total": 0.1, "speed_download": 10000, "size_download": len(body)})
fmt.Fprint(out, resourceTrailer, string(metrics))
tls := "TLSv1.2"
if strings.Contains(strings.Join(args, " "), "--tlsv1.3") {
tls = "TLSv1.3"
}
fmt.Fprintln(diagnostic, "SSL connection using", tls)
return nil
}
return r, c, resources, events
}
func TestManagedGooglevideoOwnedPostWarmupProducer(t *testing.T) {
r, c, resources, events := googlevideoRuntimeFixture(t)
runner, calls := r.curl, 0
r.curl = func(ctx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
calls++
if calls <= 2 {
return context.DeadlineExceeded
}
return runner(ctx, args, stdin, out, diagnostic)
}
p, err := r.verifyGooglevideo(context.Background(), c, resources)
if err != nil || p.producerSeal == nil || p.validate(c) != nil {
t.Fatal(p, err)
}
if calls != 26 || p.Hosts[0].WarmupAttempts != 4 || p.Hosts[0].WarmupFailures != 2 || len(p.Hosts[0].Passes) != 6 {
t.Fatal("hidden warmup", p.Hosts, calls)
}
joined := strings.Join(*events, ",")
if strings.Count(joined, "start") != 1 || !strings.HasSuffix(joined, "delete,join") {
t.Fatal(joined)
}
raw, _ := json.Marshal(p)
if strings.Contains(string(raw), "private-secret") || strings.Contains(string(raw), "videoplayback") {
t.Fatal("durable URL leak")
}
for _, h := range p.Hosts {
for _, pass := range h.Passes {
if pass.RangeStart != 0 || pass.RangeEnd != 1023 || pass.ResourceBytes != 4096 {
t.Fatal(pass)
}
}
}
}
func TestManagedGooglevideoRefusesUnresolvedAndUnknownSemantics(t *testing.T) {
for _, mode := range []string{"missing", "one-host", "same-root", "rewritten", "expired", "unsigned", "no-incoming", "blocking-incoming", "detector-payload", "detector", "stage-gap", "ignored-stage", "unknown-init", "unknown-key", "out-range"} {
t.Run(mode, func(t *testing.T) {
r, c, resources, events := googlevideoRuntimeFixture(t)
switch mode {
case "missing":
resources = nil
case "one-host":
resources = resources[:1]
case "same-root":
c.Endpoints[1] = "rr2---sn-alpha.googlevideo.com"
case "rewritten":
resources[0].host = c.Endpoints[1]
case "expired":
resources[0].url = "https://" + resources[0].host + "/videoplayback?expire=1&sig=private-secret"
case "unsigned":
resources[0].url = strings.Split(resources[0].url, "&sig=")[0]
case "no-incoming":
c.Args = strings.Replace(c.Args, " --in-range=-s34228", "", 1)
case "blocking-incoming":
c.Args = strings.Replace(c.Args, "--in-range=-s34228", "--in-range=0", 1)
case "detector-payload":
c.Args = strings.Replace(c.Args, "--payload=all", "--payload=tls_client_hello", 1)
case "detector":
c.Args = strings.Replace(c.Args, "circular:fails=1", "circular:success_detector=cond_true:fails=1", 1)
case "stage-gap":
c.Args = strings.Replace(c.Args, "strategy=2", "strategy=3", 1)
case "ignored-stage":
c.Args = strings.Replace(c.Args, ":strategy=2", "", 1)
case "unknown-init":
c.Args += " --lua-init=@/opt/zapret2/lua/unknown.lua"
case "unknown-key":
c.Args = strings.Replace(c.Args, "circular:fails=1", "circular:unknown=1:fails=1", 1)
case "out-range":
c.Args += " --out-range=0"
}
p, err := r.verifyGooglevideo(context.Background(), c, resources)
if err == nil || p.producerSeal != nil || len(*events) != 0 {
t.Fatal("captured unresolved inventory/config", p, err, *events)
}
})
}
}
func TestManagedGooglevideoWarmupBoundAndCleanupAttestation(t *testing.T) {
for _, mode := range []string{"nonpass", "root404", "cleanup", "queue-lost", "worker-exit"} {
t.Run(mode, func(t *testing.T) {
r, c, resources, events := googlevideoRuntimeFixture(t)
calls := 0
if mode == "nonpass" {
r.curl = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error {
calls++
return context.DeadlineExceeded
}
}
if mode == "root404" {
r.curl = func(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
calls++
b, err := io.ReadAll(stdin)
if err != nil {
return err
}
raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(b), "url = "), "\n"))
if err != nil {
return err
}
var diag bytes.Buffer
resourceWire(raw, "root", out, &diag)
tls := diag.String()
if strings.Contains(strings.Join(args, " "), "--tlsv1.3") {
tls = strings.ReplaceAll(tls, "TLSv1.2", "TLSv1.3")
}
io.WriteString(diagnostic, tls)
return nil
}
}
if mode == "cleanup" {
nft := r.nft
r.nft = func(ctx context.Context, input string) (string, error) {
if strings.HasPrefix(input, "delete") {
return "", errors.New("failed")
}
return nft(ctx, input)
}
}
if mode == "queue-lost" {
queue, n := r.queue, 0
r.queue = func(pid int) (bool, error) {
n++
if n == 3 {
return false, nil
}
return queue(pid)
}
}
if mode == "worker-exit" {
start := r.start
r.start = func(args []string) (managedAssetWorker, error) {
w, err := start(args)
ch := make(chan error, 1)
ch <- errors.New("dead")
w.done = ch
return w, err
}
}
p, err := r.verifyGooglevideo(context.Background(), c, resources)
if err == nil || p.Attestation != nil || p.producerSeal != nil {
t.Fatal("attested failed lifecycle", p, err)
}
if (mode == "nonpass" || mode == "root404") && (calls != 10 || p.Hosts[0].WarmupFailures != 10 || len(p.Hosts[0].Passes) != 0 || !strings.HasSuffix(strings.Join(*events, ","), "delete,join")) {
t.Fatal(p, calls, *events)
}
})
}
}
func TestManagedGooglevideoAdaptiveStorageAndReadGate(t *testing.T) {
r, c, resources, _ := googlevideoRuntimeFixture(t)
p, err := r.verifyGooglevideo(context.Background(), c, resources)
if err != nil {
t.Fatal(err)
}
db := testStore(t)
tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
id, err := insertStrategyCandidate(tx, c.Group, c.Endpoints, c.Args, nil, c.Mode)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(p)
var dto GooglevideoQualificationProof
if err := json.Unmarshal(raw, &dto); err != nil {
t.Fatal(err)
}
if dto.validate(c) != nil {
t.Fatal("DTO should be structurally valid")
}
if err := saveAdaptiveGooglevideoEvidence(tx, id, dto); err == nil {
t.Fatal("DTO alone authorized a write")
}
p.Hosts[0].WarmupFailures++
if err := saveAdaptiveGooglevideoEvidence(tx, id, p); err == nil {
t.Fatal("modified producer proof accepted")
}
p.Hosts[0].WarmupFailures--
if err := saveAdaptiveGooglevideoEvidence(tx, id, p); err != nil {
t.Fatal(err)
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
var n int
if err := db.QueryRow("SELECT count(*) FROM candidate_evidence WHERE candidate_id=?", id).Scan(&n); err != nil || n != 0 {
t.Fatal("manufactured cold metrics", n, err)
}
stored, err := readStrategyCandidate(db, id)
if err != nil || !stored.Qualified || stored.Mode != QualificationAdaptive {
t.Fatal(stored, err)
}
list, err := listQualifiedCandidates(db, "googlevideo", 1)
if err != nil || len(list) != 1 || list[0].ID != id {
t.Fatal(list, err)
}
if err := validateGooglevideoSuffixEvidence(db, stored); err != nil {
t.Fatal(err)
}
for _, mode := range []string{"malformed", "stale", "args", "attestation", "hidden-warmup", "media", "secret"} {
t.Run(mode, func(t *testing.T) {
proof, err := decodeGooglevideoProof(string(raw))
if err != nil {
t.Fatal(err)
}
switch mode {
case "stale":
proof.Attestation.CheckedAt = time.Now().Add(-25 * time.Hour)
case "args":
proof.Args = "--lua-desync=fake"
case "attestation":
proof.Attestation = nil
case "hidden-warmup":
proof.Hosts[0].WarmupAttempts = 0
case "media":
proof.Hosts[0].Passes[0].Status = 404
}
bad, _ := json.Marshal(proof)
if mode == "malformed" {
bad = []byte(`{"version":"bad"}`)
}
if mode == "secret" {
bad = append(bad[:len(bad)-1], []byte(`,"signed_url":"private-secret"}`)...)
}
if _, err := db.Exec("UPDATE googlevideo_scope_evidence SET proof=? WHERE candidate_id=?", string(bad), id); err != nil {
t.Fatal(err)
}
stored, err := readStrategyCandidate(db, id)
if err != nil || stored.Qualified {
t.Fatal(stored, err)
}
list, err := listQualifiedCandidates(db, "googlevideo", 1)
if err != nil || len(list) != 0 {
t.Fatal(list, err)
}
})
}
}
@@ -0,0 +1,65 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
)
func TestManagedGooglevideoFreshPrivateSamples(t *testing.T) {
ids := []string{"dQw4w9WgXcQ", "aqz-KE-bpKQ", "jNQXAC9IVRw"}
hosts := []string{"rr1---sn-alpha.googlevideo.com", "rr2---sn-beta.googlevideo.com", "rr5---sn-gamma.googlevideo.com"}
for _, scenario := range []string{"diverse", "duplicate", "expired", "sabr"} {
t.Run(scenario, func(t *testing.T) {
calls := 0
collect := managedGooglevideoSamples(ids, func(_ context.Context, id string) ([]byte, error) {
i := calls % len(ids)
calls++
if id != ids[i] {
t.Fatal("configured ID changed", id)
}
if scenario == "sabr" {
return []byte(`{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"private-secret"}}`), nil
}
host := hosts[i]
if scenario == "duplicate" {
host = hosts[0]
}
expires := time.Now().Add(time.Hour).Unix()
if scenario == "expired" {
expires = time.Now().Unix() + 30
}
url, _ := json.Marshal(fmt.Sprintf("https://%s/videoplayback?expire=%d&sig=private-secret", host, expires))
return []byte(fmt.Sprintf(`{"playabilityStatus":{"status":"OK"},"streamingData":{"adaptiveFormats":[{"mimeType":"video/mp4","url":%s}]}}`, url)), nil
})
for n := 0; n < 2; n++ {
resources, err := collect(context.Background())
if scenario != "diverse" {
var insufficient *InsufficientAdaptiveSamples
if !errors.As(err, &insufficient) || strings.Contains(err.Error(), "private-secret") || len(resources) != 0 {
t.Fatal("unresolved samples accepted or leaked", resources, err)
}
continue
}
var actual []string
for _, r := range resources {
actual = append(actual, r.host)
if !strings.Contains(r.url, "sig=private-secret") {
t.Fatal("signed URL changed")
}
}
if err != nil || !reflect.DeepEqual(actual, hosts) {
t.Fatal(actual, err)
}
}
if calls != 2*len(ids) {
t.Fatal("samples were reused", calls)
}
})
}
}
+22 -1
View File
@@ -111,13 +111,27 @@ func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr
now: time.Now, wait: waitManagedMonitor, now: time.Now, wait: waitManagedMonitor,
reportRecoveryError: func(error) { fmt.Fprintln(stderr, "managed recovery failed (details suppressed)") }, reportRecoveryError: func(error) { fmt.Fprintln(stderr, "managed recovery failed (details suppressed)") },
} }
if recovery.service {
monitor.reportProbeError = func(err error) {
var failure *managedResourceFailure
if errors.As(err, &failure) {
fmt.Fprintln(stderr, failure.PublicError())
} else {
fmt.Fprintln(stderr, "DEGRADED: service scope unresolved")
}
}
}
if err := wireManagedRecovery(&monitor, recovery, func(error) { if err := wireManagedRecovery(&monitor, recovery, func(error) {
fmt.Fprintln(stderr, "managed GGC unresolved; monitoring continues (not playback health)") fmt.Fprintln(stderr, "managed GGC unresolved; monitoring continues (not playback health)")
}); err != nil { }); err != nil {
fmt.Fprintln(stderr, "error: managed monitor wiring refused") fmt.Fprintln(stderr, "error: managed monitor wiring refused")
return 78 return 78
} }
if recovery.service {
fmt.Fprintln(stdout, "managed monitor: service resource checks; SABR unknown; owned temporary asset verifier")
} else {
fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health") fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health")
}
err = supervise(ctx, monitor) err = supervise(ctx, monitor)
if err != nil { if err != nil {
var fatal *managedRecoveryFatal var fatal *managedRecoveryFatal
@@ -174,7 +188,14 @@ func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand
defer cancel() defer cancel()
return command(bounded, path, args...) return command(bounded, path, args...)
} }
return managedRecoveryDependencies(target, db, scan, check, run) d, err := managedRecoveryDependencies(target, db, scan, check, run)
if err == nil && d.service {
d.verifyAssets = newManagedAssetVerifier(target)
if d.adaptive {
d.verifyGooglevideo = newManagedGooglevideoVerifier(target)
}
}
return d, err
} }
// Joined cancellation plus failed cleanup/DB/lock close is not a graceful exit. // Joined cancellation plus failed cleanup/DB/lock close is not a graceful exit.
@@ -26,10 +26,18 @@ func TestManagedPrestartHelper(t *testing.T) {
for len(args) > 0 && args[0] != "--" { for len(args) > 0 && args[0] != "--" {
args = args[1:] args = args[1:]
} }
if len(args) < 3 { if len(args) < 2 {
os.Exit(1) os.Exit(1)
} }
args = args[2:] // -- managed COMMAND args = args[1:] // -- COMMAND ...
if len(args) == 1 && args[0] == "target-config-check" {
// Use the production validator, not a fixture-only success response.
os.Exit(run(args, os.Stdout, os.Stderr))
}
if len(args) < 2 || args[0] != "managed" {
os.Exit(1)
}
args = args[1:] // managed COMMAND
if args[0] == "monitor" { if args[0] == "monitor" {
os.Exit(78) // Stop after initial activation; no scans or network. os.Exit(78) // Stop after initial activation; no scans or network.
} }
@@ -41,7 +49,7 @@ func TestManagedPrestartHelper(t *testing.T) {
} }
func TestManagedEntrypointPrestartReconciliation(t *testing.T) { func TestManagedEntrypointPrestartReconciliation(t *testing.T) {
for _, mode := range []string{"fresh", "baseline", "committed", "prepared", "drift", "unknown", "legacy", "corrupt-backup", "unqualified", "reload-failure"} { for _, mode := range []string{"fresh", "baseline", "committed", "prepared", "drift", "unknown", "legacy", "corrupt-backup", "unqualified", "reload-failure", "invalid-dns", "invalid-service"} {
t.Run(mode, func(t *testing.T) { t.Run(mode, func(t *testing.T) {
d, env, _ := managedCLIFixture(t) d, env, _ := managedCLIFixture(t)
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db" env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
@@ -125,7 +133,7 @@ func TestManagedEntrypointPrestartReconciliation(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
write("awk", "#!/bin/sh\nexit 0\n") write("awk", "#!/bin/sh\nexit 0\n")
child := "#!/bin/sh\nsleeper=\ntrap '[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; exit 0' TERM INT\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n" child := "#!/bin/sh\n[ -z \"${SS_PASSWORD-}\" ] || exit 1\n[ \"$#\" -eq 2 ] || { [ \"$#\" -eq 4 ] && [ \"$3\" = -d ] && [ \"$4\" = \"$SS_DNS_SERVER\" ]; } || exit 1\n: > \"$FIXTURE_CHILD_STARTED\"\nsleeper=\ntrap '[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; exit 0' TERM INT\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n"
write("ss-server", child) write("ss-server", child)
write("ss-local", child) write("ss-local", child)
log := filepath.Join(root, "log") log := filepath.Join(root, "log")
@@ -156,11 +164,34 @@ func TestManagedEntrypointPrestartReconciliation(t *testing.T) {
// The CLI guard sees fixed production bindings; only fake daemon reads // The CLI guard sees fixed production bindings; only fake daemon reads
// use the fixture config, via its own script rather than CLI path flags. // use the fixture config, via its own script rather than CLI path flags.
cmd.Env = append(cmd.Env, "FIXTURE_CONFIG="+filepath.Join(d.directory, "config")) cmd.Env = append(cmd.Env, "FIXTURE_CONFIG="+filepath.Join(d.directory, "config"))
childStarted := filepath.Join(root, "child-started")
cmd.Env = append(cmd.Env, "FIXTURE_CHILD_STARTED="+childStarted, "SS_DNS_SERVER=8.8.8.8")
wantExit := 78
if mode == "invalid-dns" {
cmd.Env = append(cmd.Env, "SS_DNS_SERVER=8.8.8.8\n--invalid")
wantExit = 64
}
if mode == "invalid-service" {
cmd.Env = append(cmd.Env, "ZAPRET_AUTOTUNE_YOUTUBE_SERVICE=invalid")
wantExit = 64
}
output, err := cmd.CombinedOutput() output, err := cmd.CombinedOutput()
var exit *exec.ExitError var exit *exec.ExitError
if ctx.Err() != nil || !errors.As(err, &exit) || exit.ExitCode() != 78 { if ctx.Err() != nil || !errors.As(err, &exit) || exit.ExitCode() != wantExit {
t.Fatal(err, ctx.Err(), string(output)) t.Fatal(err, ctx.Err(), string(output))
} }
if bytes.Contains(output, []byte(strings.Repeat("a", 32))) {
t.Fatal("startup exposed password")
}
if wantExit == 64 {
if _, err := os.Stat(childStarted); !errors.Is(err, os.ErrNotExist) {
t.Fatal("invalid configuration launched a service", err)
}
files, err := os.ReadDir(ss)
if err != nil || len(files) != 0 || !strings.Contains(string(output), "configerror:") {
t.Fatal("invalid configuration did not fail before config creation", err, string(output))
}
}
b, _ := os.ReadFile(log) b, _ := os.ReadFile(log)
allowed := mode == "fresh" || mode == "baseline" || mode == "committed" || mode == "prepared" allowed := mode == "fresh" || mode == "baseline" || mode == "committed" || mode == "prepared"
if !allowed && mode != "reload-failure" && len(b) != 0 { if !allowed && mode != "reload-failure" && len(b) != 0 {
+84 -5
View File
@@ -30,6 +30,15 @@ type managedRecoveryDeps struct {
health func(context.Context) (string, error) health func(context.Context) (string, error)
run func(context.Context, string, ...string) error run func(context.Context, string, ...string) error
jobTimeout time.Duration // Zero uses the production budget; tests may shorten it. jobTimeout time.Duration // Zero uses the production budget; tests may shorten it.
service bool
frontHealth func(context.Context) (string, error)
resources func(context.Context) (managedServiceInventory, error)
// Verify owns temporary raw-config activation and must join/restore before
// returning. It must not touch permanent config or the transaction journal.
verifyAssets func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error)
adaptive bool
mediaSamples func(context.Context) ([]mediaResource, error)
verifyGooglevideo func(context.Context, storedCandidate, []mediaResource) (GooglevideoQualificationProof, error)
} }
// Target-local adapter: reuses bounded Player requests without invoking a CLI. // Target-local adapter: reuses bounded Player requests without invoking a CLI.
@@ -47,7 +56,7 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
return d, err return d, err
} }
d.transaction = &managedTransaction{Dir: target.directory, Name: "config"} d.transaction = &managedTransaction{Dir: target.directory, Name: "config"}
request := func(ctx context.Context) ([]byte, error) { requestVideo := func(ctx context.Context, videoID string) ([]byte, error) {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return nil, err return nil, err
} }
@@ -56,7 +65,7 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
} }
bounded, cancel := context.WithTimeout(ctx, 35*time.Second) bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
defer cancel() defer cancel()
body, err := requestPlayer(cfg.VideoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error { body, err := requestPlayer(videoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error {
return target.player(bounded, args, out, diagnostics) return target.player(bounded, args, out, diagnostics)
}) })
if bounded.Err() != nil { if bounded.Err() != nil {
@@ -64,6 +73,15 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
} }
return body, err return body, err
} }
request := func(ctx context.Context) ([]byte, error) { return requestVideo(ctx, cfg.VideoID) }
mode, videoIDs, err := managedGooglevideoConfig(target.lookup, cfg.VideoID)
if err != nil {
return d, err
}
d.adaptive = mode == QualificationAdaptive
if d.adaptive {
d.mediaSamples = managedGooglevideoSamples(videoIDs, requestVideo)
}
d.health = func(ctx context.Context) (string, error) { d.health = func(ctx context.Context) (string, error) {
body, err := request(ctx) body, err := request(ctx)
if err != nil { if err != nil {
@@ -95,6 +113,16 @@ func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan,
} }
return playerHostname(body) return playerHostname(body)
} }
d.service, err = managedServiceEnabled(target.lookup)
if err != nil {
return d, err
}
if d.service {
d.frontHealth = d.health
d.resources = managedServiceResources(cfg, request, target.player)
d.health = managedServiceHealth(cfg.SOCKSProxy, d.frontHealth, d.resources, runResourceCurl)
d.verifyAssets = unwiredManagedAssetVerifier
}
return d, nil return d, nil
} }
@@ -199,6 +227,17 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return result, err return result, err
} }
if d.service {
_, healthErr := d.health(ctx)
var fatal *managedRecoveryFatal
if ctx.Err() != nil || errors.As(healthErr, &fatal) || errors.Is(healthErr, context.Canceled) || errors.Is(healthErr, context.DeadlineExceeded) {
return result, managedRecoveryContextError(ctx, healthErr)
}
var failure *managedResourceFailure
if errors.As(healthErr, &failure) && assetEndpoint(failure.Host) {
return result, repairManagedAssets(ctx, d, failure.Host)
}
}
m := *d.transaction m := *d.transaction
// Rollback must restart even after job cancellation; independently bounded. // Rollback must restart even after job cancellation; independently bounded.
var cleanupCtx context.Context var cleanupCtx context.Context
@@ -221,7 +260,13 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
return nil return nil
} }
m.Health = func() (string, error) { m.Health = func() (string, error) {
status, err := d.health(ctx) health := d.health
if d.service && d.frontHealth != nil {
health = d.frontHealth
}
// Commit frontend/GGC progress independently; the monitor's overall
// service probe still requires media and both images before READY.
status, err := health(ctx)
if ctx.Err() != nil { if ctx.Err() != nil {
return "", managedRecoveryContextError(ctx, err) return "", managedRecoveryContextError(ctx, err)
} }
@@ -262,6 +307,16 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return err return err
} }
if d.service {
current, e := m.ActiveCandidates(d.db)
if e != nil {
return &managedRecoveryFatal{Err: e}
}
ids, e = managedFullDesiredSet(d.db, current, ids)
if e != nil {
return &managedRecoveryFatal{Err: e}
}
}
cleanupReload := m.Reload cleanupReload := m.Reload
first := true first := true
m.Reload = func() error { m.Reload = func() error {
@@ -301,7 +356,11 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
} }
healthy := false healthy := false
if result.FrontendID != 0 { if result.FrontendID != 0 {
status, e := d.health(ctx) frontHealth := d.health
if d.service && d.frontHealth != nil {
frontHealth = d.frontHealth
}
status, e := frontHealth(ctx)
e = managedRecoveryContextError(ctx, e) e = managedRecoveryContextError(ctx, e)
var fatal *managedRecoveryFatal var fatal *managedRecoveryFatal
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) { if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
@@ -372,11 +431,19 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
if e != nil { if e != nil {
return result, e return result, e
} }
if len(current) != 1 || current[0] != result.FrontendID { if !d.service && (len(current) != 1 || current[0] != result.FrontendID) {
if e := apply([]int64{result.FrontendID}); e != nil { if e := apply([]int64{result.FrontendID}); e != nil {
return result, e return result, e
} }
} }
if d.adaptive {
result.GooglevideoID, result.GGCError = recoverManagedGooglevideo(ctx, d)
var fatal *managedRecoveryFatal
if errors.As(result.GGCError, &fatal) || ctx.Err() != nil {
return result, result.GGCError
}
return result, nil
}
host, e := d.discover(ctx) host, e := d.discover(ctx)
e = managedRecoveryContextError(ctx, e) e = managedRecoveryContextError(ctx, e)
var discoveryFatal *managedRecoveryFatal var discoveryFatal *managedRecoveryFatal
@@ -578,6 +645,9 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
result, err := recoverManagedTarget(ctx, recovery) result, err := recoverManagedTarget(ctx, recovery)
if result.GGCError != nil { if result.GGCError != nil {
reportGGC(result.GGCError) reportGGC(result.GGCError)
if recovery.adaptive && err == nil {
return result.GGCError
}
} }
// The supervisor reports ordinary recovery failures and retries only // The supervisor reports ordinary recovery failures and retries only
// after its durable cooldown. Safety failures (rollback/cleanup/drift) // after its durable cooldown. Safety failures (rollback/cleanup/drift)
@@ -667,6 +737,15 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
if err != nil { if err != nil {
return &managedRecoveryFatal{Err: err} return &managedRecoveryFatal{Err: err}
} }
if recovery.adaptive && candidate.Group == "googlevideo" && candidate.Mode == QualificationAdaptive && candidate.Qualified {
journal, e := transaction.load()
if e != nil {
return &managedRecoveryFatal{Err: e}
}
if journal.Scope == GooglevideoScopeSuffix {
return nil
}
}
if candidate.Group == "googlevideo" && len(candidate.Endpoints) == 1 && candidate.Endpoints[0] == host { if candidate.Group == "googlevideo" && len(candidate.Endpoints) == 1 && candidate.Endpoints[0] == host {
return nil return nil
} }
@@ -0,0 +1,156 @@
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"testing"
"time"
)
func TestManagedExpiredAdaptiveJournalRecovery(t *testing.T) {
for _, scenario := range []string{"committed", "prepared", "rollback"} {
t.Run(scenario, func(t *testing.T) {
d, front, _ := recoveryFixture(t, true)
r, c, resources, _ := googlevideoRuntimeFixture(t)
proof, err := r.verifyGooglevideo(context.Background(), c, resources)
if err != nil {
t.Fatal(err)
}
tx, err := d.db.Begin()
if err != nil {
t.Fatal(err)
}
id, err := insertStrategyCandidate(tx, c.Group, c.Endpoints, c.Args, nil, c.Mode)
if err == nil {
err = saveAdaptiveGooglevideoEvidence(tx, id, proof)
}
if err != nil {
_ = tx.Rollback()
t.Fatal(err)
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
ids := []int64{front, unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true), unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true), id}
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); err != nil {
t.Fatal(err)
}
before, err := d.transaction.read("config")
if err != nil {
t.Fatal(err)
}
if scenario == "prepared" {
crash := errors.New("process died after replacement")
d.transaction.fault = func(point string) error {
if point == "replaced" {
return crash
}
return nil
}
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); !errors.Is(err, crash) {
t.Fatal(err)
}
d.transaction.fault = nil
}
proof.Attestation.CheckedAt = time.Now().Add(-25 * time.Hour)
raw, err := json.Marshal(proof)
if err != nil {
t.Fatal(err)
}
if _, err := d.db.Exec("UPDATE googlevideo_scope_evidence SET proof=? WHERE candidate_id=?", string(raw), id); err != nil {
t.Fatal(err)
}
if err := d.transaction.Reconcile(d.db); err != nil {
t.Fatal("aged committed/pending refs prevented reconciliation", err)
}
assertActiveCandidates(t, d.transaction, d.db, ids)
stored, err := readStrategyCandidate(d.db, id)
if err != nil || stored.Qualified {
t.Fatal("aged proof authorized fresh qualification", stored, err)
}
if err := validateGooglevideoSuffixEvidence(d.db, stored); err == nil {
t.Fatal("aged proof authorized new suffix")
}
if err := d.transaction.Apply(d.db, ids, GooglevideoScopeSuffix); err == nil {
t.Fatal("aged proof authorized new apply")
}
if err := d.transaction.applyRenderedDB(before, d.db, ids...); err == nil {
t.Fatal("pre-rendered bytes bypassed freshness")
}
d.service, d.adaptive = true, true
d.frontHealth = d.health
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/sample.jpg", "yt3.ggpht.com": "https://yt3.ggpht.com/sample.jpg"}, proxy: r.proxy, runner: adaptiveImageRunner}, nil
}
samples, verifies := 0, 0
d.mediaSamples = func(context.Context) ([]mediaResource, error) {
samples++
return resources, nil
}
d.verifyGooglevideo = func(ctx context.Context, c storedCandidate, resources []mediaResource) (GooglevideoQualificationProof, error) {
verifies++
return r.verifyGooglevideo(ctx, c, resources)
}
if scenario == "rollback" {
d.health = func(context.Context) (string, error) {
return "DEGRADED", &managedResourceFailure{"media", resources[0].host, nil}
}
}
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || samples != 1 || verifies != 1 {
t.Fatal("aged active state blocked fresh owned verification", result, samples, verifies, err)
}
if scenario == "rollback" {
if result.GGCError == nil {
t.Fatal("unhealthy apply accepted")
}
assertActiveCandidates(t, d.transaction, d.db, ids)
after, err := d.transaction.read("config")
if err != nil || !reflect.DeepEqual(before, after) {
t.Fatal("aged rollback changed committed bytes", err)
}
} else {
if result.GGCError != nil || result.GooglevideoID <= id {
t.Fatal("fresh revalidation did not replace expired proof", result)
}
ids[len(ids)-1] = result.GooglevideoID
assertActiveCandidates(t, d.transaction, d.db, ids)
}
// Expiry alone is safe; malformed/invalid historical proof is not.
activeID := ids[len(ids)-1]
if _, err := d.db.Exec("UPDATE googlevideo_scope_evidence SET proof='{}' WHERE candidate_id=?", activeID); err != nil {
t.Fatal(err)
}
if _, err := d.transaction.ActiveCandidates(d.db); err == nil {
t.Fatal("invalid committed proof accepted")
}
if err := d.transaction.Reconcile(d.db); err == nil {
t.Fatal("invalid committed proof reconciled")
}
})
}
}
func TestManagedServiceDispatchSafetyPrecedesImageRepair(t *testing.T) {
for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} {
for _, failure := range []error{&managedRecoveryFatal{Err: errors.New("guard failed")}, context.Canceled, context.DeadlineExceeded} {
t.Run(host+"/"+failure.Error(), func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
d.service = true
mixed := errors.Join(&managedResourceFailure{managedImageProbeKind(host), host, nil}, failure)
d.health = func(context.Context) (string, error) { return "DEGRADED", mixed }
d.resources = func(context.Context) (managedServiceInventory, error) {
t.Fatal("mixed safety failure dispatched image repair")
return managedServiceInventory{}, nil
}
_, err := recoverManagedTarget(context.Background(), d)
if !errors.Is(err, failure) || len(*events) != 0 {
t.Fatal("joined safety failure swallowed/retried", err, *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
})
}
}
}
+350
View File
@@ -0,0 +1,350 @@
package main
import (
"context"
"errors"
"io"
"strings"
"testing"
)
func TestManagedServiceOptIn(t *testing.T) {
for _, value := range []string{"", "0", "1", "true", "secret"} {
enabled, err := managedServiceEnabled(func(key string) (string, bool) { return value, key == "ZAPRET_AUTOTUNE_YOUTUBE_SERVICE" && value != "" })
valid := value == "" || value == "0" || value == "1"
if (err == nil) != valid || (valid && enabled != (value == "1")) {
t.Fatalf("value %q: %v %v", value, enabled, err)
}
}
}
func TestManagedServiceMissingResourcesNeverGreen(t *testing.T) {
front := func(context.Context) (string, error) { return "OK", nil }
for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} {
i := managedServiceInventory{images: map[string]string{}}
if host == "yt3.ggpht.com" {
i.images["i.ytimg.com"] = "https://i.ytimg.com/test"
}
// The first missing image must fail before any resource process is spawned.
if host == "yt3.ggpht.com" {
continue
}
health := managedServiceHealth("socks5://127.0.0.1:11080", front, func(context.Context) (managedServiceInventory, error) { return i, nil }, func(context.Context, []string, io.Reader, io.Writer, io.Writer) error {
t.Fatal("unexpected transfer")
return nil
})
status, err := health(context.Background())
var failure *managedResourceFailure
if status != "DEGRADED" || !errors.As(err, &failure) || failure.Host != host {
t.Fatalf("%s %v", status, err)
}
}
if public := (&managedResourceFailure{"https://secret/?token=x", "secret/?token=x", errors.New("private")}).PublicError(); strings.Contains(public, "secret") || strings.Contains(public, "private") {
t.Fatal(public)
}
}
func TestManagedServiceFullDesiredSet(t *testing.T) {
d, front, _ := recoveryFixture(t, true)
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
thumb := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
newFront := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=multisplit", true)
ids, err := managedFullDesiredSet(d.db, []int64{front, ggc, thumb, avatar}, []int64{newFront})
if err != nil {
t.Fatal(err)
}
if len(ids) != 4 || ids[0] != ggc || ids[1] != thumb || ids[2] != avatar || ids[3] != newFront {
t.Fatal(ids)
}
}
func TestManagedServiceSavedAssetRepairPreservesFullSet(t *testing.T) {
d, front, events := recoveryFixture(t, true)
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
old := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
saved := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=multisplit", true)
if err := d.transaction.Apply(d.db, []int64{front, ggc, old, avatar}); err != nil {
t.Fatal(err)
}
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/fresh"}}, nil
}
joined := false
d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) {
if c.ID != saved || len(*events) == 0 || (*events)[len(*events)-1] != "stop" {
t.Fatal("not saved-first under downtime", c.ID, *events)
}
joined = true
return []assetMeasurement{unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args)}, nil
}
d.health = func(context.Context) (string, error) {
if !joined {
t.Fatal("apply before verifier joined")
}
return "OK", nil
}
if err := repairManagedAssets(context.Background(), d, "i.ytimg.com"); err != nil {
t.Fatal(err)
}
ids, err := d.transaction.ActiveCandidates(d.db)
if err != nil || len(ids) != 4 {
t.Fatal(ids, err)
}
for _, required := range []int64{front, ggc, avatar} {
found := false
for _, id := range ids {
found = found || id == required
}
if !found {
t.Fatal("lost sibling", required, ids)
}
}
}
func TestManagedServiceAssetRollbackAndCancellation(t *testing.T) {
for _, cancelJob := range []bool{false, true} {
t.Run(map[bool]string{false: "false-media-rollback", true: "cancel-joins"}[cancelJob], func(t *testing.T) {
d, front, events := recoveryFixture(t, true)
old := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
if err := d.transaction.Apply(d.db, []int64{front, old, avatar}); err != nil {
t.Fatal(err)
}
unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=multisplit", true)
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/fresh"}}, nil
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
joined := false
d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) {
joined = true
if cancelJob {
cancel()
return nil, context.Canceled
}
return []assetMeasurement{unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args)}, nil
}
d.health = func(context.Context) (string, error) {
return "DEGRADED", &managedResourceFailure{"media", "fresh.googlevideo.com", nil}
}
err := repairManagedAssets(ctx, d, "i.ytimg.com")
if err == nil || !joined {
t.Fatal("unverified success or unjoined verifier", err)
}
if cancelJob && (!errors.Is(err, context.Canceled) || (*events)[len(*events)-1] != "start") {
t.Fatal("missing bounded restoration", err, *events)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front, old, avatar})
})
}
}
func TestManagedServiceFreshPrivateInventory(t *testing.T) {
cfg := healthConfig{SOCKSProxy: "socks5://127.0.0.1:11080"}
resources := managedServiceResources(cfg, func(context.Context) ([]byte, error) {
return []byte(`{"playabilityStatus":{"status":"OK"},"videoDetails":{"thumbnail":{"thumbnails":[{"url":"https://i.ytimg.com/fresh?private=thumbnail"}]}},"streamingData":{}}`), nil
}, func(_ context.Context, args []string, out, _ io.Writer) error {
if args[len(args)-1] != "https://www.youtube.com/" || strings.Contains(strings.Join(args, " "), "private=") {
t.Fatal(args)
}
_, err := io.WriteString(out, `var ytInitialData = {"avatar":{"url":"https://yt3.ggpht.com/fresh?private=avatar"}};`)
return err
})
i, err := resources(context.Background())
if err != nil || len(i.images) != 2 || i.media.url != "" {
t.Fatal("fresh inventory or SABR unknown contract", err)
}
}
func TestManagedServicePartialInventoryChecksAndRepairsAvailableImage(t *testing.T) {
for _, host := range []string{"i.ytimg.com", "yt3.ggpht.com"} {
t.Run(host, func(t *testing.T) {
d, front, _ := recoveryFixture(t, true)
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
thumb := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
if err := d.transaction.Apply(d.db, []int64{front, ggc, thumb, avatar}); err != nil {
t.Fatal(err)
}
r, _, _ := assetRuntimeFixture(t)
d.service = true
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
missingHost := "yt3.ggpht.com"
if host == missingHost {
missingHost = "i.ytimg.com"
}
missing := &managedResourceFailure{managedImageProbeKind(missingHost), missingHost, errors.New("private metadata unavailable")}
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{host: "https://" + host + "/image?token=private"}, proxy: r.proxy, runner: r.curl, resourceErr: &managedResourceFailure{"media", "unknown", nil}}, missing
}
repaired, checked := false, false
d.health = managedServiceHealth(r.proxy, d.frontHealth, d.resources, func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
checked = true
if !repaired {
resourceWire("https://"+host+"/image?token=private", "cn", out, diag)
return nil
}
return r.curl(ctx, args, in, out, diag)
})
status, err := d.health(context.Background())
var failure *managedResourceFailure
if status != "DEGRADED" || !checked || !errors.As(err, &failure) || failure.Host != host || failure.ProbeKind != managedImageProbeKind(host) || strings.Contains(err.Error(), "private") {
t.Fatal(status, checked, err)
}
d.verifyAssets = func(ctx context.Context, c storedCandidate, i managedServiceInventory) ([]assetMeasurement, error) {
m, err := r.verify(ctx, c, i)
repaired = err == nil
return m, err
}
if _, err := recoverManagedTarget(context.Background(), d); err != nil || !repaired {
t.Fatal("partial metadata blocked independent asset repair", err)
}
ids, err := d.transaction.ActiveCandidates(d.db)
if err != nil || len(ids) != 4 {
t.Fatal("lost full desired set", ids, err)
}
status, err = d.health(context.Background())
if status != "DEGRADED" || !errors.As(err, &failure) || failure.Host != missingHost {
t.Fatal("missing resources became green", status, err)
}
})
}
}
func TestManagedServiceBootstrapCommitsFrontendBeforeAssetRepair(t *testing.T) {
d, _, _ := recoveryFixture(t, false)
d.service = true
r, _, _ := assetRuntimeFixture(t)
frontChecks := 0
d.frontHealth = func(context.Context) (string, error) {
frontChecks++
if frontChecks == 1 {
return "DEGRADED", nil
}
return "OK", nil
}
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}, proxy: r.proxy, runner: r.curl, resourceErr: &managedResourceFailure{"media", "unknown", nil}}, nil
}
repaired := false
d.health = managedServiceHealth(r.proxy, d.frontHealth, d.resources, func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
if !repaired {
resourceWire("https://i.ytimg.com/image", "cn", out, diag)
return nil
}
return r.curl(ctx, args, in, out, diag)
})
result, err := recoverManagedTarget(context.Background(), d)
if err != nil || result.FrontendID == 0 || result.GooglevideoID == 0 {
t.Fatal("broken thumbnail vetoed frontend commit", result, err)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
d.verifyAssets = func(ctx context.Context, c storedCandidate, i managedServiceInventory) ([]assetMeasurement, error) {
m, err := r.verify(ctx, c, i)
repaired = err == nil
return m, err
}
if _, err := recoverManagedTarget(context.Background(), d); err != nil || !repaired {
t.Fatal("thumbnail repair blocked by missing media", err)
}
ids, err := d.transaction.ActiveCandidates(d.db)
if err != nil || len(ids) != 3 || ids[0] != result.FrontendID || ids[1] != result.GooglevideoID {
t.Fatal("asset commit lost frontend/GGC", ids, err)
}
status, err := d.health(context.Background())
var failure *managedResourceFailure
if status != "DEGRADED" || !errors.As(err, &failure) || failure.ProbeKind != "media" {
t.Fatal("missing media became service-ready", status, err)
}
}
func TestManagedServiceAssetLocalHealthFailureRollsBack(t *testing.T) {
d, front, _ := recoveryFixture(t, true)
old := unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=fake", true)
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
prior := []int64{front, ggc, old, avatar}
if err := d.transaction.Apply(d.db, prior); err != nil {
t.Fatal(err)
}
before, err := d.transaction.read("config")
if err != nil {
t.Fatal(err)
}
r, _, _ := assetRuntimeFixture(t)
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}, proxy: r.proxy, runner: func(ctx context.Context, args []string, in io.Reader, out, diag io.Writer) error {
input, err := io.ReadAll(in)
if err != nil {
return err
}
if strings.Contains(string(input), "i.ytimg.com") {
resourceWire("https://i.ytimg.com/image", "truncated", out, diag)
return nil
}
return r.curl(ctx, args, strings.NewReader(string(input)), out, diag)
}}, nil
}
i, _ := d.resources(context.Background())
if status, err := managedAssetInventoryHealth(context.Background(), i, []string{"yt3.ggpht.com"}); status != "OK" || err != nil {
t.Fatal("old sibling must remain healthy", status, err)
}
d.verifyAssets = r.verify
if err := repairManagedAssets(context.Background(), d, "i.ytimg.com"); !errors.Is(err, errNoQualifiedStrategy) {
t.Fatal(err)
}
assertActiveCandidates(t, d.transaction, d.db, prior)
after, err := d.transaction.read("config")
if err != nil || string(before) != string(after) {
t.Fatal("rejected asset changed prior config", err)
}
}
func TestManagedAssetHealthNonpassPreservesMixedSafetyFailures(t *testing.T) {
badBody := &managedRecoveryFatal{Err: errors.New("invalid image body")}
for _, tc := range []struct {
err error
nonpass bool
}{
{badBody, true},
{errors.Join(badBody, &managedRecoveryFatal{Err: errors.New("guard failed")}), false},
{errors.Join(badBody, context.Canceled), false},
{&managedAssetCleanupFailure{Err: errors.New("cleanup failed")}, false},
} {
if managedAssetHealthNonpass(tc.err) != tc.nonpass {
t.Fatal(tc.err)
}
}
}
func TestManagedAssetRepairMixedFatalNeverRetries(t *testing.T) {
for _, stage := range []string{"verify", "health"} {
t.Run(stage, func(t *testing.T) {
d, front, _ := recoveryFixture(t, true)
r, _, _ := assetRuntimeFixture(t)
d.resources = func(context.Context) (managedServiceInventory, error) {
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}, proxy: r.proxy, runner: r.curl}, nil
}
mixed := errors.Join(&managedRecoveryFatal{Err: errors.New("guard failed")}, errCandidateNonpass, &managedResourceFailure{"thumbnail", "i.ytimg.com", nil})
calls := 0
d.verifyAssets = func(ctx context.Context, c storedCandidate, i managedServiceInventory) ([]assetMeasurement, error) {
calls++
if stage == "verify" {
return nil, mixed
}
return r.verify(ctx, c, i)
}
d.frontHealth = func(context.Context) (string, error) { return "DEGRADED", mixed }
err := repairManagedAssets(context.Background(), d, "i.ytimg.com")
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) || calls != 1 {
t.Fatal("mixed safety failure retried as rejection", calls, err)
}
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
})
}
}
+59 -14
View File
@@ -38,6 +38,8 @@ type managedJournal struct {
Next string `json:"pending_hash,omitempty"` Next string `json:"pending_hash,omitempty"`
Candidates []int64 `json:"committed_candidate_ids,omitempty"` Candidates []int64 `json:"committed_candidate_ids,omitempty"`
PendingCandidates []int64 `json:"pending_candidate_ids,omitempty"` PendingCandidates []int64 `json:"pending_candidate_ids,omitempty"`
Scope GooglevideoScope `json:"committed_googlevideo_scope,omitempty"`
PendingScope GooglevideoScope `json:"pending_googlevideo_scope,omitempty"`
} }
// Hooks take no commands or arguments. The target adapter binds Reload // Hooks take no commands or arguments. The target adapter binds Reload
@@ -189,7 +191,9 @@ func (m *managedTransaction) load() (managedJournal, error) {
return err == nil && len(b) == 32 && strings.ToLower(s) == s return err == nil && len(b) == 32 && strings.ToLower(s) == s
} }
validIDs := func(ids []int64) bool { validIDs := func(ids []int64) bool {
if len(ids) > 2 { // Two legacy groups plus at most two disjoint inventory-bound assets.
// Group and scope policy remains in the DB-qualified renderer.
if len(ids) > 4 {
return false return false
} }
seen := map[int64]bool{} seen := map[int64]bool{}
@@ -201,7 +205,7 @@ func (m *managedTransaction) load() (managedJournal, error) {
} }
return true return true
} }
if (j.Version != 1 && j.Version != 2) || j.ID == "" || !validHash(j.Hash) || (j.State != "committed" && j.State != "prepared") || (j.State == "prepared" && !validHash(j.Next)) || (j.State == "committed" && (j.Next != "" || len(j.PendingCandidates) != 0)) || !validIDs(j.Candidates) || !validIDs(j.PendingCandidates) || (j.Version == 1 && (len(j.Candidates) != 0 || len(j.PendingCandidates) != 0)) { if (j.Version != 1 && j.Version != 2 && j.Version != 3) || !j.Scope.valid() || !j.PendingScope.valid() || (j.Version < 3 && (j.Scope != "" || j.PendingScope != "")) || (j.Version == 3 && (j.Scope == "" || (j.State == "prepared" && j.PendingScope == ""))) || j.ID == "" || !validHash(j.Hash) || (j.State != "committed" && j.State != "prepared") || (j.State == "prepared" && !validHash(j.Next)) || (j.State == "committed" && (j.Next != "" || len(j.PendingCandidates) != 0 || j.PendingScope != "")) || !validIDs(j.Candidates) || !validIDs(j.PendingCandidates) || (j.Version == 1 && (len(j.Candidates) != 0 || len(j.PendingCandidates) != 0)) {
return j, fmt.Errorf("invalid managed journal") return j, fmt.Errorf("invalid managed journal")
} }
return j, nil return j, nil
@@ -232,13 +236,16 @@ func (m *managedTransaction) checkpoint(point string) error {
return nil return nil
} }
func (m *managedTransaction) Apply(db *sql.DB, ids []int64) error { func (m *managedTransaction) Apply(db *sql.DB, ids []int64, scopes ...GooglevideoScope) error {
ids = append([]int64(nil), ids...) ids = append([]int64(nil), ids...)
b, err := renderManagedConfig(db, ids) if db == nil || len(scopes) > 1 || (len(scopes) == 1 && (!scopes[0].valid() || scopes[0] == "")) {
if err != nil { return fmt.Errorf("candidate database and optional explicit scope required")
return err
} }
return m.applyRenderedDB(b, db, ids...) var scope *GooglevideoScope
if len(scopes) == 1 {
scope = &scopes[0]
}
return m.applyRenderedPolicy(nil, db, scope, ids...)
} }
// ActiveCandidates uses the journal as active-state authority. The DB supplies // ActiveCandidates uses the journal as active-state authority. The DB supplies
@@ -269,7 +276,7 @@ func (m *managedTransaction) ActiveCandidates(db *sql.DB) ([]int64, error) {
if db == nil { if db == nil {
return fmt.Errorf("candidate database required") return fmt.Errorf("candidate database required")
} }
rendered, err := renderManagedConfig(db, j.Candidates) rendered, err := renderStoredManagedConfigScope(db, j.Candidates, j.Scope)
if err != nil { if err != nil {
return err return err
} }
@@ -286,6 +293,9 @@ func (m *managedTransaction) ActiveCandidates(db *sql.DB) ([]int64, error) {
func committedManagedJournal(j managedJournal) managedJournal { func committedManagedJournal(j managedJournal) managedJournal {
j.State, j.Hash, j.Next = "committed", j.Next, "" j.State, j.Hash, j.Next = "committed", j.Next, ""
j.Candidates, j.PendingCandidates = j.PendingCandidates, nil j.Candidates, j.PendingCandidates = j.PendingCandidates, nil
if j.Version == 3 {
j.Scope, j.PendingScope = j.PendingScope, ""
}
return j return j
} }
@@ -306,14 +316,18 @@ func (m *managedTransaction) applyRendered(b []byte, ids ...int64) error {
return m.applyRenderedDB(b, nil, ids...) return m.applyRenderedDB(b, nil, ids...)
} }
func validateManagedRefs(db *sql.DB, ids []int64, hash string) error { func validateManagedRefs(db *sql.DB, ids []int64, hash string, scopes ...GooglevideoScope) error {
if len(ids) == 0 { if len(ids) == 0 {
return nil return nil
} }
if db == nil { if db == nil {
return fmt.Errorf("candidate database required") return fmt.Errorf("candidate database required")
} }
b, err := renderManagedConfig(db, ids) var scope GooglevideoScope
if len(scopes) != 0 {
scope = scopes[0]
}
b, err := renderStoredManagedConfigScope(db, ids, scope)
if err != nil || managedHash(b) != hash { if err != nil || managedHash(b) != hash {
return fmt.Errorf("managed candidate state unresolved") return fmt.Errorf("managed candidate state unresolved")
} }
@@ -321,6 +335,10 @@ func validateManagedRefs(db *sql.DB, ids []int64, hash string) error {
} }
func (m *managedTransaction) applyRenderedDB(b []byte, db *sql.DB, ids ...int64) error { func (m *managedTransaction) applyRenderedDB(b []byte, db *sql.DB, ids ...int64) error {
return m.applyRenderedPolicy(b, db, nil, ids...)
}
func (m *managedTransaction) applyRenderedPolicy(b []byte, db *sql.DB, requested *GooglevideoScope, ids ...int64) error {
return m.locked(func() error { return m.locked(func() error {
j, err := m.load() j, err := m.load()
if err != nil { if err != nil {
@@ -329,10 +347,28 @@ func (m *managedTransaction) applyRenderedDB(b []byte, db *sql.DB, ids ...int64)
if j.State != "committed" { if j.State != "committed" {
return fmt.Errorf("incomplete transaction: reconcile first") return fmt.Errorf("incomplete transaction: reconcile first")
} }
if err := validateManagedRefs(db, j.Candidates, j.Hash); err != nil { if err := validateManagedRefs(db, j.Candidates, j.Hash, j.Scope); err != nil {
return err return err
} }
if err := validateManagedRefs(db, ids, managedHash(b)); err != nil { scope := j.Scope
if scope == "" {
scope = GooglevideoScopeExact
}
if requested != nil {
scope = *requested
}
if b == nil {
b, err = renderManagedConfigScope(db, ids, scope)
if err != nil {
return err
}
} else if len(ids) != 0 {
fresh, err := renderManagedConfigScope(db, ids, scope)
if err != nil || managedHash(fresh) != managedHash(b) {
return fmt.Errorf("fresh qualified candidate state required")
}
}
if err := validateManagedRefs(db, ids, managedHash(b), scope); err != nil {
return err return err
} }
prior, err := m.read(m.Name) prior, err := m.read(m.Name)
@@ -374,7 +410,15 @@ func (m *managedTransaction) applyRenderedDB(b []byte, db *sql.DB, ids ...int64)
return err return err
} }
j.Generation++ j.Generation++
if requested != nil || j.Version == 3 {
j.Version = 3
if j.Scope == "" {
j.Scope = GooglevideoScopeExact
}
j.PendingScope = scope
} else {
j.Version = 2 j.Version = 2
}
j.ID, j.State, j.Next = hex.EncodeToString(id), "prepared", managedHash(b) j.ID, j.State, j.Next = hex.EncodeToString(id), "prepared", managedHash(b)
j.PendingCandidates = append([]int64(nil), ids...) j.PendingCandidates = append([]int64(nil), ids...)
if err := m.journal(j); err != nil { if err := m.journal(j); err != nil {
@@ -487,6 +531,7 @@ func (m *managedTransaction) restore(j managedJournal) error {
} }
j.State, j.Next = "committed", "" j.State, j.Next = "committed", ""
j.PendingCandidates = nil j.PendingCandidates = nil
j.PendingScope = ""
return m.journal(j) return m.journal(j)
} }
@@ -502,10 +547,10 @@ func (m *managedTransaction) Reconcile(databases ...*sql.DB) error {
if len(databases) != 0 { if len(databases) != 0 {
db = databases[0] db = databases[0]
} }
if err := validateManagedRefs(db, j.Candidates, j.Hash); err != nil { if err := validateManagedRefs(db, j.Candidates, j.Hash, j.Scope); err != nil {
return err return err
} }
if err := validateManagedRefs(db, j.PendingCandidates, j.Next); err != nil { if err := validateManagedRefs(db, j.PendingCandidates, j.Next, j.PendingScope); err != nil {
return err return err
} }
if j.State == "prepared" { if j.State == "prepared" {
+4
View File
@@ -39,6 +39,7 @@ type managedMonitorDeps struct {
now func() time.Time now func() time.Time
wait func(context.Context, time.Duration) error wait func(context.Context, time.Duration) error
reportRecoveryError func(error) reportRecoveryError func(error)
reportProbeError func(error)
} }
// A startup reconciliation can be consumed once, never during live recovery. // A startup reconciliation can be consumed once, never during live recovery.
@@ -155,6 +156,9 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
return err return err
} }
probeErr := d.probe(ctx) probeErr := d.probe(ctx)
if probeErr != nil && d.reportProbeError != nil {
d.reportProbeError(probeErr)
}
var fatalProbe *managedRecoveryFatal var fatalProbe *managedRecoveryFatal
if errors.As(probeErr, &fatalProbe) { if errors.As(probeErr, &fatalProbe) {
return probeErr return probeErr
+8 -4
View File
@@ -287,6 +287,7 @@ func monitorMigrationSnapshot(t *testing.T, db *sql.DB) string {
func TestMonitorV3MigrationPreservesHistory(t *testing.T) { func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
db := testStore(t) db := testStore(t)
saveProposal(t, db, "--lua-desync=fake") saveProposal(t, db, "--lua-desync=fake")
restoreV5CandidateSchema(t, db)
if _, err := db.Exec(`INSERT INTO googlevideo_tune_runs (started_at, finished_at, state, error) if _, err := db.Exec(`INSERT INTO googlevideo_tune_runs (started_at, finished_at, state, error)
VALUES ('start', 'finish', 'FAILED', 'original failure'); VALUES ('start', 'finish', 'FAILED', 'original failure');
DROP TABLE monitor_state; PRAGMA user_version = 3;`); err != nil { DROP TABLE monitor_state; PRAGMA user_version = 3;`); err != nil {
@@ -299,14 +300,16 @@ func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
} }
} }
var version, count int var version, count int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 7 {
t.Fatalf("version=%d err=%v", version, err) t.Fatalf("version=%d err=%v", version, err)
} }
if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 { if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 {
t.Fatalf("monitor count=%d err=%v", count, err) t.Fatalf("monitor count=%d err=%v", count, err)
} }
if after := monitorMigrationSnapshot(t, db); before != after { // The additive v7 candidate column contributes one trailing cold default;
t.Fatal("v3 history/candidate data changed") // compare the legacy row projection, not the new field's presence.
if after := strings.ReplaceAll(monitorMigrationSnapshot(t, db), `, "cold"}`, `}`); before != after {
t.Fatalf("v3 history/candidate data changed:\nbefore=%s\nafter=%s", before, after)
} }
} }
@@ -391,6 +394,7 @@ func TestMonitorCrashReconciliation(t *testing.T) {
func TestMonitorV4MigrationPreservesUnverifiableLease(t *testing.T) { func TestMonitorV4MigrationPreservesUnverifiableLease(t *testing.T) {
db := testStore(t) db := testStore(t)
restoreV5CandidateSchema(t, db)
if _, err := db.Exec("DROP TABLE monitor_state;" + monitorSchema); err != nil { if _, err := db.Exec("DROP TABLE monitor_state;" + monitorSchema); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -408,7 +412,7 @@ func TestMonitorV4MigrationPreservesUnverifiableLease(t *testing.T) {
} }
} }
var version int var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 7 {
t.Fatalf("version=%d err=%v", version, err) t.Fatalf("version=%d err=%v", version, err)
} }
clear, err := reconcileMonitorRecovery(db, "service", time.Minute, func(int) (int64, error) { clear, err := reconcileMonitorRecovery(db, "service", time.Minute, func(int) (int64, error) {
+8 -1
View File
@@ -79,7 +79,7 @@ func (c *candidate) validate() error {
} }
func (c *candidate) validateMeasurements() error { func (c *candidate) validateMeasurements() error {
if c.Group != "frontend" && c.Group != "googlevideo" { if c.Group != "frontend" && c.Group != "googlevideo" && c.Group != "assets" {
return fmt.Errorf("unknown group") return fmt.Errorf("unknown group")
} }
if len(c.Domains) == 0 { if len(c.Domains) == 0 {
@@ -90,6 +90,9 @@ func (c *candidate) validateMeasurements() error {
return fmt.Errorf("invalid DNS hostname") return fmt.Errorf("invalid DNS hostname")
} }
c.Domains[i] = strings.ToLower(domain) c.Domains[i] = strings.ToLower(domain)
if c.Group == "assets" && !assetEndpoint(c.Domains[i]) {
return fmt.Errorf("asset endpoint outside service inventory")
}
} }
sort.Strings(c.Domains) sort.Strings(c.Domains)
for _, metric := range []float64{c.SuccessRate, c.Coverage, c.MedianLatency, c.MedianSpeed} { for _, metric := range []float64{c.SuccessRate, c.Coverage, c.MedianLatency, c.MedianSpeed} {
@@ -106,6 +109,10 @@ func (c *candidate) validateMeasurements() error {
return nil return nil
} }
func assetEndpoint(host string) bool {
return host == "i.ytimg.com" || host == "yt3.ggpht.com"
}
func (c *candidate) validateArgs() error { func (c *candidate) validateArgs() error {
// Only shell-inert, unquoted option values can be rendered verbatim. // Only shell-inert, unquoted option values can be rendered verbatim.
for _, ch := range c.Args { for _, ch := range c.Args {
+373
View File
@@ -0,0 +1,373 @@
package main
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"mime"
"net/http"
"net/textproto"
"net/url"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
)
// stdin is mandatory: ephemeral signed URLs must never enter argv or diagnostics.
type subprocessRunner func(context.Context, []string, io.Reader, io.Writer, io.Writer) error
func runResourceCurl(ctx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
cmd := exec.CommandContext(ctx, "curl", args...)
cmd.Stdin, cmd.Stdout, cmd.Stderr = stdin, out, diagnostic
return cmd.Run()
}
type resourceError struct{ cause error }
func (e *resourceError) PublicError() string { return "resource qualification failed" }
func (e *resourceError) Error() string { return e.PublicError() }
func (e *resourceError) Unwrap() error { return e.cause }
func resourceFailure(ctx context.Context, err error) error {
if ctx.Err() != nil {
err = errors.Join(err, ctx.Err())
}
if !managedRecoveryTimeoutSafe(err) {
err = &managedRecoveryFatal{Err: err}
}
return &resourceError{cause: err}
}
func privateResourceURL(raw, host string) (*url.URL, error) {
if len(raw) == 0 || len(raw) > 16<<10 {
return nil, errors.New("invalid resource URL")
}
for _, c := range raw {
if c <= 32 || c == 127 {
return nil, errors.New("invalid resource URL")
}
}
u, err := url.Parse(raw)
if err != nil || u.Scheme != "https" || u.User != nil || u.Opaque != "" || u.Fragment != "" || u.Hostname() != host || (u.Port() != "" && u.Port() != "443") || strings.HasSuffix(u.Host, ":") || !discoveryHostname(host) {
return nil, errors.New("invalid resource origin")
}
return u, nil
}
func thumbnailResource(videoID string) (string, error) {
if !videoIDPattern.MatchString(videoID) {
return "", errors.New("invalid video ID")
}
return "https://i.ytimg.com/vi/" + videoID + "/hqdefault.jpg", nil
}
// Parse a fresh structural player/ytInitialData object, not HTML URL regexes.
// Callers extracting ytInitialData from a WWW page pass that JSON object here.
func avatarResource(metadata []byte) (string, error) {
if len(metadata) > playerResponseLimit {
return "", errors.New("metadata exceeds limit")
}
var root interface{}
if json.Unmarshal(metadata, &root) != nil {
return "", errors.New("invalid metadata")
}
var visit func(interface{}, int) string
visit = func(v interface{}, depth int) string {
if depth > 64 {
return ""
}
switch x := v.(type) {
case map[string]interface{}:
if raw, ok := x["url"].(string); ok {
if _, err := privateResourceURL(raw, "yt3.ggpht.com"); err == nil {
return raw
}
}
for _, child := range x {
if raw := visit(child, depth+1); raw != "" {
return raw
}
}
case []interface{}:
for _, child := range x {
if raw := visit(child, depth+1); raw != "" {
return raw
}
}
}
return ""
}
if raw := visit(root, 0); raw != "" {
return raw, nil
}
return "", errors.New("avatar unavailable")
}
// The WWW page is transient; decode the embedded JSON structurally and never
// scan HTML for arbitrary URLs to fetch.
func avatarResourceFromPage(page []byte) (string, error) {
if len(page) > 5<<20 {
return "", errors.New("page exceeds limit")
}
for _, marker := range []string{"var ytInitialData = ", "ytInitialData = ", "window[\"ytInitialData\"] = "} {
if i := bytes.Index(page, []byte(marker)); i >= 0 {
var object json.RawMessage
if json.NewDecoder(bytes.NewReader(page[i+len(marker):])).Decode(&object) == nil {
return avatarResource(object)
}
}
}
return "", errors.New("structural avatar metadata unavailable")
}
var resourceContentRange = regexp.MustCompile(`^bytes 0-([0-9]+)/([0-9]+)$`)
type mediaResource struct{ url, host, kind string }
type mediaCapability struct {
OrdinaryURL bool
SABRSupported bool
}
// SABR is deliberately unknown/not supported, never fabricated as readiness.
func playerMediaResource(body []byte) (mediaResource, mediaCapability, error) {
var p struct {
PlayabilityStatus struct {
Status string `json:"status"`
} `json:"playabilityStatus"`
StreamingData struct {
Formats []struct {
URL string `json:"url"`
MimeType string `json:"mimeType"`
} `json:"formats"`
AdaptiveFormats []struct {
URL string `json:"url"`
MimeType string `json:"mimeType"`
} `json:"adaptiveFormats"`
} `json:"streamingData"`
}
if len(body) > playerResponseLimit || json.Unmarshal(body, &p) != nil || p.PlayabilityStatus.Status != "OK" {
return mediaResource{}, mediaCapability{}, errors.New("invalid playable metadata")
}
for _, f := range append(p.StreamingData.Formats, p.StreamingData.AdaptiveFormats...) {
u, err := url.Parse(f.URL)
if err != nil {
continue
}
host := u.Hostname()
if !strings.HasSuffix(host, ".googlevideo.com") {
continue
}
if _, err := privateResourceURL(f.URL, host); err != nil {
continue
}
kind, _, err := mime.ParseMediaType(f.MimeType)
if err == nil && (kind == "video/mp4" || kind == "audio/mp4") {
return mediaResource{f.URL, host, kind}, mediaCapability{OrdinaryURL: true}, nil
}
}
return mediaResource{}, mediaCapability{}, nil
}
const resourceTrailer = "\nAUTOTUNE_RESOURCE_METRICS:"
func collectResourceTransfer(ctx context.Context, proxy, raw, host, protocol string, media bool, runner subprocessRunner) (assetTransfer, http.Header, error) {
fail := func(err error) (assetTransfer, http.Header, error) {
return assetTransfer{}, nil, resourceFailure(ctx, err)
}
if !validDiscoveryProxy(proxy) || runner == nil {
return fail(errors.New("invalid owned proxy or runner"))
}
if _, err := privateResourceURL(raw, host); err != nil {
return fail(err)
}
version, tlsFlag, maxTLS := uint16(0x0303), "--tlsv1.2", "1.2"
if protocol == "tls13" {
version, tlsFlag, maxTLS = 0x0304, "--tlsv1.3", "1.3"
} else if protocol != "tls12" {
return fail(errors.New("invalid TLS protocol"))
}
limit := 5 << 20
if media {
limit = 65536
}
// Qualification uses HTTP/1.1, not browser HTTP/2. --raw preserves native
// transfer framing so net/http, rather than curl and Go twice, decodes it.
args := []string{"--disable", "--silent", "--show-error", "--verbose", "--http1.1", "--raw", "--noproxy", "", "--proxy", proxy, "--proto", "=https", tlsFlag, "--tls-max", maxTLS, "--max-time", "30", "--max-filesize", strconv.Itoa(limit), "--suppress-connect-headers", "--include", "--write-out", resourceTrailer + "%{json}", "--config", "-"}
if media {
args = append(args, "--range", "0-65535")
}
// curl config quotes are not JSON quotes; reject controls above, escape only these.
config := "url = \"" + strings.NewReplacer("\\", "\\\\", "\"", "\\\"").Replace(raw) + "\"\n"
childCtx, cancel := context.WithTimeout(ctx, 35*time.Second)
defer cancel()
var output, diagnostic bytes.Buffer
out := &playerBoundedWriter{dst: &output, limit: limit + (96 << 10)}
diag := &playerBoundedWriter{dst: &diagnostic, limit: 64 << 10}
err := runner(childCtx, args, strings.NewReader(config), out, diag)
if out.exceeded || diag.exceeded {
err = errors.Join(err, errors.New("resource output exceeds limit"))
}
if err != nil || childCtx.Err() != nil {
return fail(errors.Join(err, childCtx.Err()))
}
i := bytes.LastIndex(output.Bytes(), []byte(resourceTrailer))
if i < 0 {
return fail(errors.New("missing transfer metrics"))
}
var metrics struct {
URL string `json:"url_effective"`
Verify *int `json:"ssl_verify_result"`
Redirects *int `json:"num_redirects"`
Status int `json:"http_code"`
Time float64 `json:"time_total"`
Speed float64 `json:"speed_download"`
Size float64 `json:"size_download"`
}
if json.Unmarshal(output.Bytes()[i+len(resourceTrailer):], &metrics) != nil {
return fail(errors.New("invalid transfer metrics"))
}
if _, err := privateResourceURL(metrics.URL, host); err != nil || metrics.URL != raw || metrics.Verify == nil || *metrics.Verify != 0 || metrics.Redirects == nil || *metrics.Redirects != 0 {
return fail(errors.New("resource origin verification failed"))
}
actualTLS := "SSL connection using TLSv1.2"
if version == 0x0304 {
actualTLS = "SSL connection using TLSv1.3"
}
otherTLS := "SSL connection using TLSv1.3"
if version == 0x0304 {
otherTLS = "SSL connection using TLSv1.2"
}
if !strings.Contains(diagnostic.String(), actualTLS) || strings.Contains(diagnostic.String(), otherTLS) {
return fail(errors.New("actual TLS version mismatch"))
}
wire := output.Bytes()[:i]
headerEnd := bytes.Index(wire, []byte("\r\n\r\n"))
statusEnd := bytes.Index(wire, []byte("\r\n"))
if headerEnd < 0 || headerEnd > 64<<10 || statusEnd < 0 {
return fail(errors.New("missing or oversized response headers"))
}
h, err := textproto.NewReader(bufio.NewReader(bytes.NewReader(wire[statusEnd+2 : headerEnd+4]))).ReadMIMEHeader()
if err != nil {
return fail(errors.New("invalid response headers"))
}
for _, key := range []string{"Content-Type", "Content-Length", "Transfer-Encoding", "Content-Range"} {
if len(h.Values(key)) > 1 {
return fail(errors.New("duplicate response framing headers"))
}
}
if h.Get("Transfer-Encoding") != "" && (h.Get("Transfer-Encoding") != "chunked" || h.Get("Content-Length") != "") {
return fail(errors.New("ambiguous response framing"))
}
r := bufio.NewReader(bytes.NewReader(wire))
response, err := http.ReadResponse(r, nil)
if err != nil {
return fail(errors.New("invalid response headers"))
}
defer response.Body.Close()
// CONNECT headers are suppressed by curl. Interim/duplicate responses and
// unrequested versions are outside this contract, never heuristically skipped.
if response.ProtoMajor != 1 || response.ProtoMinor != 1 || response.StatusCode < 200 || response.Header.Get("Content-Type") == "" {
return fail(errors.New("unsupported response contract"))
}
body, err := io.ReadAll(io.LimitReader(response.Body, int64(limit+1)))
// With --raw curl counts the wire body, including chunk framing; the image
// limit and durable byte identity apply to the fully decoded payload instead.
wireSize := len(wire) - headerEnd - 4
_, remaining := r.Peek(1)
if err != nil || remaining != io.EOF || len(body) == 0 || len(body) > limit || metrics.Size != float64(wireSize) || response.StatusCode != metrics.Status {
return fail(errors.New("incomplete or oversized resource body"))
}
for _, n := range []float64{metrics.Time, metrics.Speed} {
if n <= 0 || math.IsNaN(n) || math.IsInf(n, 0) {
return fail(errors.New("invalid resource metrics"))
}
}
contentType, _, err := mime.ParseMediaType(response.Header.Get("Content-Type"))
if err != nil {
return fail(errors.New("invalid content type"))
}
t := assetTransfer{EffectiveHost: host, SSLVerifyResult: metrics.Verify, Redirects: *metrics.Redirects, TLSVersion: version, Status: metrics.Status, ContentType: contentType, Body: body, LatencyMS: metrics.Time * 1000, SpeedKBPS: metrics.Speed / 1024}
return t, response.Header, nil
}
// The ordinary 3/3 sampler is shared by image and media qualification.
func sampleResource(ctx context.Context, transfer func(context.Context) (assetTransfer, error)) ([]assetTransfer, error) {
var samples []assetTransfer
for i := 0; i < 3; i++ {
t, err := transfer(ctx)
if err != nil {
return nil, err
}
samples = append(samples, t)
}
return samples, nil
}
func collectAssetMeasurement(ctx context.Context, proxy, raw, host, protocol, args string, runner subprocessRunner) (assetMeasurement, error) {
m := assetMeasurement{Source: "image-transfer", Endpoint: host, Protocol: protocol, Args: args}
if !assetEndpoint(host) {
return m, resourceFailure(ctx, errors.New("invalid image endpoint"))
}
samples, err := sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) {
t, _, err := collectResourceTransfer(ctx, proxy, raw, host, protocol, false, runner)
if err == nil {
if t.Status != 200 {
err = errors.New("image HTTP status")
} else {
err = normalizeAssetBody(&t)
}
}
if err != nil {
return assetTransfer{}, resourceFailure(ctx, err)
}
t.Body = nil
return t, nil
})
m.Transfers = samples
return m, err
}
func collectMediaSamples(ctx context.Context, proxy string, resource mediaResource, protocol string, runner subprocessRunner) ([]assetTransfer, error) {
if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") {
return nil, resourceFailure(ctx, errors.New("ordinary media unavailable"))
}
return sampleResource(ctx, func(ctx context.Context) (assetTransfer, error) {
t, _, err := collectMediaTransfer(ctx, proxy, resource, protocol, runner)
return t, err
})
}
// Only normalized range facts leave this collector; URLs, headers and payloads
// remain transient. Used by both cold sampling and post-warmup sampling.
func collectMediaTransfer(ctx context.Context, proxy string, resource mediaResource, protocol string, runner subprocessRunner) (assetTransfer, GooglevideoPass, error) {
if !strings.HasSuffix(resource.host, ".googlevideo.com") || (resource.kind != "video/mp4" && resource.kind != "audio/mp4") {
return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("ordinary media unavailable"))
}
t, h, err := collectResourceTransfer(ctx, proxy, resource.url, resource.host, protocol, true, runner)
if err != nil {
return assetTransfer{}, GooglevideoPass{}, err
}
parts := resourceContentRange.FindStringSubmatch(h.Get("Content-Range"))
var end, total int64
if len(parts) == 3 {
end, _ = strconv.ParseInt(parts[1], 10, 64)
total, _ = strconv.ParseInt(parts[2], 10, 64)
}
if t.Status != 206 || t.ContentType != resource.kind || len(parts) != 3 || end < 15 || end > 65535 || end+1 != int64(len(t.Body)) || total <= end || len(t.Body) < 16 || string(t.Body[4:8]) != "ftyp" || binary.BigEndian.Uint32(t.Body[:4]) < 16 || int(binary.BigEndian.Uint32(t.Body[:4])) > len(t.Body) {
return assetTransfer{}, GooglevideoPass{}, resourceFailure(ctx, errors.New("invalid media range or container"))
}
// Retain only normalized byte identity; no signed URL or payload.
t.Size, t.Format, t.BodyHash = len(t.Body), "mp4", fmt.Sprintf("%x", sha256.Sum256(t.Body))
t.Body = nil
p := GooglevideoPass{Protocol: protocol, VerifiedTLS: true, Status: t.Status, RangeStart: 0, RangeEnd: end, ResourceBytes: total, Bytes: int64(t.Size), FTYP: true, LatencyMS: t.LatencyMS}
return t, p, nil
}
@@ -0,0 +1,310 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"image"
"image/png"
"io"
"os"
"os/exec"
"strconv"
"strings"
"testing"
)
func resourceFixture() []byte {
var b bytes.Buffer
if err := png.Encode(&b, image.NewRGBA(image.Rect(0, 0, 1, 1))); err != nil {
panic(err)
}
return b.Bytes()
}
func resourceWire(raw, mode string, out, diagnostic io.Writer) {
body, contentType, status := resourceFixture(), "image/png", 200
header := ""
if mode == "media" || mode == "bad-range" {
body = []byte{0, 0, 0, 24, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm', 0, 0, 0, 0, 'i', 's', 'o', 'm', 'm', 'p', '4', '2'}
contentType, status, header = "video/mp4", 206, "Content-Range: bytes 0-23/100\r\n"
if mode == "bad-range" {
header = "Content-Range: bytes 1-24/100\r\n"
}
}
verify, redirects, effective := 0, 0, raw
switch mode {
case "root":
body, contentType, status = []byte("not found"), "text/html", 404
case "html":
body, contentType, status = []byte("bad request"), "text/html", 400
case "cn":
verify = 60
case "redirect":
status, redirects = 302, 1
case "host":
effective = "https://other.ytimg.com/resource"
case "truncated":
body = body[:len(body)/2]
case "type":
contentType = "image/jpeg"
}
fmt.Fprintf(out, "HTTP/1.1 %d result\r\nContent-Type: %s\r\nContent-Length: %d\r\n%s\r\n", status, contentType, len(body), header)
out.Write(body)
metrics, _ := json.Marshal(map[string]interface{}{"url_effective": effective, "ssl_verify_result": verify, "num_redirects": redirects, "http_code": status, "time_total": 0.1, "speed_download": 10000, "size_download": len(body)})
fmt.Fprint(out, resourceTrailer, string(metrics))
fmt.Fprintln(diagnostic, "* SSL connection using TLSv1.2 / test")
}
// A real child exercises the stdin/argv protocol, without any socket or curl.
func TestResourceChild(t *testing.T) {
if os.Getenv("AUTOTUNE_RESOURCE_CHILD") != "1" {
return
}
b, err := io.ReadAll(io.LimitReader(os.Stdin, 32<<10))
if err != nil {
os.Exit(3)
}
raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(b), "url = "), "\n"))
if err != nil || strings.ContainsAny(raw, "\r\n") {
os.Exit(4)
}
resourceWire(raw, "", os.Stdout, os.Stderr)
os.Exit(0)
}
func TestResourcePrivateChildAndEvidence(t *testing.T) {
raw := "https://yt3.ggpht.com/avatar?signature=private-secret&quote=\"&slash=\\"
calls := 0
runner := func(ctx context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
calls++
joined := strings.Join(args, " ")
if strings.Contains(joined, "private-secret") || strings.Contains(joined, "https://") || strings.Contains(joined, "-k") || args[0] != "--disable" {
t.Fatalf("unsafe argv: %v", args)
}
if !strings.Contains(joined, "--noproxy --proxy socks5://127.0.0.1:1080") || !strings.Contains(joined, "--tlsv1.2 --tls-max 1.2") {
t.Fatal("missing explicit routing/TLS")
}
if !strings.Contains(joined, "--http1.1 --raw") || !strings.Contains(joined, "--suppress-connect-headers") {
t.Fatal("missing native HTTP/1.1 raw contract")
}
cmd := exec.CommandContext(ctx, os.Args[0], append([]string{"-test.run=^TestResourceChild$", "--"}, args...)...)
cmd.Env = append(os.Environ(), "AUTOTUNE_RESOURCE_CHILD=1")
cmd.Stdin, cmd.Stdout, cmd.Stderr = stdin, out, diagnostic
return cmd.Run()
}
m, err := collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", raw, "yt3.ggpht.com", "tls12", "--lua-desync=fake", runner)
if err != nil || calls != 3 {
t.Fatalf("calls=%d err=%v", calls, err)
}
if _, _, err := validateAssetTransfers(m); err != nil {
t.Fatal(err)
}
b, _ := json.Marshal(m)
if strings.Contains(string(b), "private-secret") || strings.Contains(string(b), "body\"") || len(m.Transfers[0].Body) != 0 || m.Transfers[0].Size != len(resourceFixture()) {
t.Fatal("durable evidence leaks payload or misses byte identity")
}
}
func TestResourceNativeFraming(t *testing.T) {
raw := "https://yt3.ggpht.com/avatar?signature=private-secret"
for _, mode := range []string{"length", "chunked", "close", "normalized-chunked", "h2", "empty", "truncated-header", "truncated-length", "truncated-chunk", "duplicate-length", "duplicate-type", "duplicate-transfer", "mixed-framing", "interim", "duplicate-response", "extra-bytes"} {
t.Run(mode, func(t *testing.T) {
payload := resourceFixture()
// A delimiter inside a body must not steal the final private footer.
payload = append(payload, []byte(resourceTrailer+`{"url_effective":"private-body-marker"}`)...)
header := fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n", len(payload))
body := payload
switch mode {
case "chunked", "normalized-chunked", "truncated-chunk":
header = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nTransfer-Encoding: chunked\r\n\r\n"
if mode != "normalized-chunked" {
var wire bytes.Buffer
fmt.Fprintf(&wire, "%x\r\n", len(payload[:20]))
wire.Write(payload[:20])
fmt.Fprint(&wire, "\r\n")
fmt.Fprintf(&wire, "%x\r\n", len(payload[20:]))
wire.Write(payload[20:])
fmt.Fprint(&wire, "\r\n0\r\n\r\n")
body = wire.Bytes()
if mode == "truncated-chunk" {
body = body[:len(body)-5]
}
}
case "close":
header = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nConnection: close\r\n\r\n"
case "h2":
header = strings.Replace(header, "HTTP/1.1 200 OK", "HTTP/2 200", 1)
case "empty":
header, body = "", nil
case "truncated-header":
header = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\n"
case "truncated-length":
body = payload[:len(payload)-1]
case "duplicate-length":
header = strings.Replace(header, "Content-Length:", fmt.Sprintf("Content-Length: %d\r\nContent-Length:", len(payload)), 1)
case "duplicate-type":
header = strings.Replace(header, "Content-Type:", "Content-Type: image/png\r\nContent-Type:", 1)
case "duplicate-transfer":
header = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nTransfer-Encoding: chunked\r\nTransfer-Encoding: chunked\r\n\r\n"
case "mixed-framing":
header = strings.Replace(header, "Content-Length:", "Transfer-Encoding: chunked\r\nContent-Length:", 1)
case "interim":
header = "HTTP/1.1 100 Continue\r\n\r\n" + header
case "duplicate-response":
header = "HTTP/1.1 200 Connection established\r\n\r\n" + header
case "extra-bytes":
body = append(append([]byte(nil), payload...), []byte("unframed bytes")...)
}
runner := func(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
if !strings.Contains(strings.Join(args, " "), "--http1.1 --raw") {
t.Fatal("HTTP version/framing not pinned")
}
fmt.Fprint(out, header)
out.Write(body)
metrics, _ := json.Marshal(map[string]interface{}{"url_effective": raw, "ssl_verify_result": 0, "num_redirects": 0, "http_code": 200, "time_total": 0.1, "speed_download": 10000, "size_download": len(body)})
fmt.Fprint(out, resourceTrailer, string(metrics))
fmt.Fprintln(diagnostic, "SSL connection using TLSv1.2")
return nil
}
m, err := collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", raw, "yt3.ggpht.com", "tls12", "", runner)
valid := mode == "length" || mode == "chunked" || mode == "close"
if (err == nil) != valid {
t.Fatalf("valid=%v error=%v", valid, err)
}
if err != nil && strings.Contains(err.Error(), "private-secret") {
t.Fatal("private footer leaked")
}
if valid && (len(m.Transfers) != 3 || m.Transfers[0].Size != len(payload)) {
t.Fatal("wrong decoded body identity")
}
})
}
}
func TestResourceRejectionsAndCancellation(t *testing.T) {
for _, mode := range []string{"root", "html", "cn", "redirect", "host", "truncated", "type", "tls", "overflow", "unknown", "mixed", "cancel"} {
t.Run(mode, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
runner := func(_ context.Context, _ []string, stdin io.Reader, out, diagnostic io.Writer) error {
if mode == "cancel" {
cancel()
return context.Canceled
}
if mode == "unknown" {
return errors.New("private-secret")
}
if mode == "mixed" {
cancel()
return errors.Join(context.Canceled, errors.New("private-secret"))
}
if mode == "overflow" {
_, err := out.Write(make([]byte, 6<<20))
return err
}
resourceWire("https://yt3.ggpht.com/avatar?signature=private-secret", mode, out, diagnostic)
if mode == "tls" {
fmt.Fprintln(diagnostic, "SSL connection using TLSv1.3")
}
return nil
}
_, err := collectAssetMeasurement(ctx, "socks5://127.0.0.1:1080", "https://yt3.ggpht.com/avatar?signature=private-secret", "yt3.ggpht.com", "tls12", "", runner)
if err == nil || strings.Contains(err.Error(), "private-secret") {
t.Fatal("accepted failure or exposed secret", err)
}
var fatal *managedRecoveryFatal
if errors.As(err, &fatal) != (mode != "cancel") {
t.Fatal("unsafe cancellation classification", err)
}
if (mode == "cancel" || mode == "mixed") && !errors.Is(err, context.Canceled) {
t.Fatal("lost cancellation")
}
})
}
}
func TestResourceDiscoveryAndMedia(t *testing.T) {
if raw, err := avatarResourceFromPage([]byte(`<html><script>var ytInitialData = {"owner":{"thumbnail":{"thumbnails":[{"url":"https://yt3.ggpht.com/fresh"}]}}};</script></html>`)); err != nil || raw != "https://yt3.ggpht.com/fresh" {
t.Fatal("WWW structural parsing", err)
}
for _, raw := range []string{"http://yt3.ggpht.com/a", "https://yt3.ggpht.com.evil/a", "https://user@yt3.ggpht.com/a", "https://yt3.ggpht.com:444/a", "https://yt3.ggpht.com/a\nurl=bad"} {
if _, err := privateResourceURL(raw, "yt3.ggpht.com"); err == nil {
t.Fatal("accepted unsafe URL", raw)
}
}
if _, err := thumbnailResource("invalid"); err == nil {
t.Fatal("invalid thumbnail ID")
}
if raw, err := avatarResource([]byte(`{"owner":{"thumbnail":{"thumbnails":[{"url":"https://yt3.ggpht.com/fresh"}]}}}`)); err != nil || raw != "https://yt3.ggpht.com/fresh" {
t.Fatal("avatar parsing", err)
}
resource, capability, err := playerMediaResource([]byte(`{"playabilityStatus":{"status":"OK"},"streamingData":{"adaptiveFormats":[{"url":"https://rr1.unit.googlevideo.com/videoplayback?sig=private-secret","mimeType":"video/mp4; codecs=\"avc1\""}]}}`))
if err != nil || !capability.OrdinaryURL || capability.SABRSupported {
t.Fatal("media capability", err)
}
for _, mode := range []string{"media", "bad-range", "root"} {
runner := func(_ context.Context, args []string, _ io.Reader, out, diagnostic io.Writer) error {
if !strings.Contains(strings.Join(args, " "), "--range 0-65535") {
t.Fatal("missing bounded range")
}
resourceWire(resource.url, mode, out, diagnostic)
return nil
}
samples, err := collectMediaSamples(context.Background(), "socks5://127.0.0.1:1080", resource, "tls12", runner)
if (err == nil) != (mode == "media") {
t.Fatal(mode, err)
}
if err == nil && (len(samples) != 3 || samples[0].Body != nil || samples[0].Format != "mp4") {
t.Fatal("invalid normalized media samples")
}
}
_, capability, err = playerMediaResource([]byte(`{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://rr1.unit.googlevideo.com/sabr?sig=private"}}`))
if err != nil || capability.OrdinaryURL || capability.SABRSupported {
t.Fatal("SABR falsely green")
}
}
func TestResourceTLS13AndStrictDTO(t *testing.T) {
raw := "https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg"
runner := func(_ context.Context, args []string, _ io.Reader, out, diagnostic io.Writer) error {
if !strings.Contains(strings.Join(args, " "), "--tlsv1.3 --tls-max 1.3") {
t.Fatal("TLS13 not pinned")
}
var private bytes.Buffer
resourceWire(raw, "", out, &private)
fmt.Fprintln(diagnostic, "* SSL connection using TLSv1.3 / test")
return nil
}
m, err := collectAssetMeasurement(context.Background(), "socks5://127.0.0.1:1080", raw, "i.ytimg.com", "tls13", "", runner)
if err != nil {
t.Fatal(err)
}
if _, _, err := validateAssetTransfers(m); err != nil {
t.Fatal(err)
}
encoded, _ := json.Marshal(m.Transfers[0])
var fields map[string]json.RawMessage
json.Unmarshal(encoded, &fields)
for key := range fields {
copy := make(map[string]json.RawMessage)
for k, v := range fields {
if k != key {
copy[k] = v
}
}
b, _ := json.Marshal(copy)
var transfer assetTransfer
if json.Unmarshal(b, &transfer) == nil {
t.Fatal("missing required field accepted", key)
}
}
fields["url"] = json.RawMessage(`"https://signed.invalid/?signature=private"`)
b, _ := json.Marshal(fields)
var transfer assetTransfer
if json.Unmarshal(b, &transfer) == nil {
t.Fatal("unknown private field accepted")
}
}
+11 -1
View File
@@ -114,7 +114,7 @@ func migrate(db *sql.DB) error {
if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil { if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil {
return err return err
} }
if version > 5 { if version > 7 {
return fmt.Errorf("unsupported database schema version %d", version) return fmt.Errorf("unsupported database schema version %d", version)
} }
if version == 0 { if version == 0 {
@@ -241,6 +241,16 @@ PRAGMA user_version = 1;`)
return err return err
} }
} }
if version < 6 {
if err := migrateAssetCandidates(ctx, conn); err != nil {
return err
}
}
if version < 7 {
if _, err := conn.ExecContext(ctx, googlevideoQualificationSchema); err != nil {
return err
}
}
_, err = conn.ExecContext(ctx, "COMMIT") _, err = conn.ExecContext(ctx, "COMMIT")
return err return err
} }
+2 -2
View File
@@ -224,7 +224,7 @@ func TestConcurrentFirstOpens(t *testing.T) {
} }
for _, db := range stores { for _, db := range stores {
var version, objects int var version, objects int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 { if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 7 {
t.Fatalf("version=%d err=%v", version, err) t.Fatalf("version=%d err=%v", version, err)
} }
if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE
@@ -377,7 +377,7 @@ func TestSchemaConstraintsAndIncompleteEvidence(t *testing.T) {
func TestUnsupportedSchemaVersion(t *testing.T) { func TestUnsupportedSchemaVersion(t *testing.T) {
db := testStore(t) db := testStore(t)
if _, err := db.Exec("PRAGMA user_version = 6"); err != nil { if _, err := db.Exec("PRAGMA user_version = 8"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") { if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") {
+40
View File
@@ -0,0 +1,40 @@
package main
import (
"bytes"
"strings"
"testing"
)
func TestTargetConfigCheck(t *testing.T) {
t.Setenv("ZAPRET_AUTOTUNE_YOUTUBE_SERVICE", "0")
t.Setenv("ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE", "cold")
t.Setenv("ZAPRET_AUTOTUNE_VIDEO_ID", "dQw4w9WgXcQ")
t.Setenv("ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS", `["dQw4w9WgXcQ"]`)
t.Setenv("ZAPRET_AUTOTUNE_SOCKS_PROXY", "socks5://127.0.0.1:1080")
for _, dns := range []string{"", "8.8.8.8", "2001:4860:4860::8888", "::1", "resolver.example", "8.8.8.8:53", "[::1]", "fe80::1%eth0", "-d 8.8.8.8", "8.8.8.8\n", "\x00", "1.2.3.999"} {
t.Run(strings.ReplaceAll(dns, "\x00", "NUL"), func(t *testing.T) {
t.Setenv("SS_DNS_SERVER", strings.ReplaceAll(dns, "\x00", "\r")) // Env cannot contain NUL.
var out, errOut bytes.Buffer
code := run([]string{"target-config-check"}, &out, &errOut)
valid := dns == "" || dns == "8.8.8.8" || dns == "2001:4860:4860::8888" || dns == "::1"
if (code == 0) != valid || out.Len() != 0 || (!valid && !strings.HasPrefix(errOut.String(), "configerror:")) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, out.String(), errOut.String())
}
})
}
t.Setenv("SS_DNS_SERVER", "")
for key, value := range map[string]string{
"ZAPRET_AUTOTUNE_YOUTUBE_SERVICE": "true",
"ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE": "adaptive",
"ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS": `[]`,
} {
t.Run(key, func(t *testing.T) {
t.Setenv(key, value)
var out, errOut bytes.Buffer
if run([]string{"target-config-check"}, &out, &errOut) != 64 || out.Len() != 0 || !strings.HasPrefix(errOut.String(), "configerror:") {
t.Fatal(out.String(), errOut.String())
}
})
}
}
+8
View File
@@ -1 +1,9 @@
ZAPRET_AUTOTUNE_HOST_SOCKS_PORT=11080 ZAPRET_AUTOTUNE_HOST_SOCKS_PORT=11080
# Legacy Player API/exact-host behavior remains the default.
ZAPRET_AUTOTUNE_YOUTUBE_SERVICE=0
ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE=cold
ZAPRET_AUTOTUNE_VIDEO_ID=dQw4w9WgXcQ
# Optional JSON array: 1..5 unique 11-character VideoIDs (not signed URLs).
# ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS='["dQw4w9WgXcQ","aqz-KE-bpKQ","jNQXAC9IVRw"]'
# Optional target ss-server native DNS; empty retains the base resolver behavior.
SS_DNS_SERVER=
+34
View File
@@ -40,11 +40,45 @@ with a tag or local default.
Offline static guard (from repository root): Offline static guard (from repository root):
`python3 deploy/managed-target/check_static.py`. `python3 deploy/managed-target/check_static.py`.
Offline real pre-launch validation and private Compose forwarding fixtures:
`python3 deploy/managed-target/check_config.py` (requires local Go and Compose).
Isolated, accelerated shell supervision fixtures (no network): Isolated, accelerated shell supervision fixtures (no network):
`python3 deploy/managed-target/check_supervision.py`. `python3 deploy/managed-target/check_supervision.py`.
## In-container supervision ## In-container supervision
### Opt-in service/adaptive and target-native DNS
Compose forwards `ZAPRET_AUTOTUNE_YOUTUBE_SERVICE` (`0` or `1`, default `0`),
`ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE` (`cold` or `adaptive`, default `cold`),
`ZAPRET_AUTOTUNE_VIDEO_ID` (default `dQw4w9WgXcQ`) and
`ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS` (JSON array of 1..5 unique VideoIDs;
default only the primary ID). Explicit empty/invalid settings are rejected,
not silently replaced. Adaptive requires service `1`. See `.env.example`.
Service `0` preserves legacy Player API/exact-host behavior.
Adaptive qualification is **post-warmup**, distinct from cold qualification;
initial connections may fail and native learning must repeat after restart.
It requires at least **three real unique signed ordinary media host samples**.
Three sample videos may all resolve to the same CDN host: that means
`DEGRADED`, no authorized googlevideo suffix, not fabricated coverage. Service
asset checks cover sampled thumbnail/avatar routes, not the whole thumbnail pool.
The current service/adaptive hardware acceptance gate is **pending**; the old
clean-run PASS below does not qualify these new modes.
Optional `SS_DNS_SERVER` is an unbracketed literal IPv4/IPv6 address only (for
example `8.8.8.8` or `2001:4860:4860::8888`); empty preserves existing resolver
behavior. No domain, port, zone, control characters or option strings are
accepted. Validation runs before service launch and reports opaque `configerror`
without echoing input. DNS applies only to this target's native `ss-server`.
The image build checks installed `ss-server -h` for native `-d <addr>` support;
the validated nonsecret address is passed as a separate `-d` argv value, not
eval or an assumed JSON key. Passwords still go only to private JSON configs.
The reported `8.8.8.8` diagnostic corrected a tampered avatar response, but does
not fix/prove all thumbnail routes. Native DNS correction is **not full asset
availability proof**. No DoH bridge, permanent Google IP pins or global host DNS
changes are supplied. PID 1 remains the sole startup/recovery owner.
Compose has no automatic restart. PID 1 restarts the entire owned runtime only Compose has no automatic restart. PID 1 restarts the entire owned runtime only
for technical failures, with three starts per rolling six hours (including initial for technical failures, with three starts per rolling six hours (including initial
startup) and cancellable 15/30-second cooldowns. Clean typed recovery timeouts, startup) and cancellable 15/30-second cooldowns. Clean typed recovery timeouts,
+65
View File
@@ -0,0 +1,65 @@
"""Offline entrypoint/config fixtures: python3 deploy/managed-target/check_config.py."""
import json
import os
from pathlib import Path
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[2]
env = dict(os.environ, GOPROXY='off', GOTOOLCHAIN='local')
with tempfile.TemporaryDirectory(prefix='target-config-') as tmp:
root = Path(tmp)
binary = root / 'autotune'
subprocess.run(['go', 'build', '-o', str(binary), './cmd/zapret-autotune'],
cwd=ROOT, env=env, check=True)
private = root / 'private'
private.mkdir()
# Exercise only the real entrypoint's pre-launch region; no services/network.
script = (ROOT / 'managed-target-entrypoint.sh').read_text().split('\nserver_pid=')[0]
script = script.replace('/usr/local/bin/zapret-autotune', str(binary))
script = script.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={private}')
start = script.index("awk '$2 ==")
end = script.index('chmod 0700 "$config_dir"', start)
script = script[:start] + script[end:]
entry = root / 'entry.sh'
entry.write_text(script)
base = dict(env, SS_PASSWORD='ab' * 32,
ZAPRET_AUTOTUNE_YOUTUBE_SERVICE='0', ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE='cold',
ZAPRET_AUTOTUNE_VIDEO_ID='dQw4w9WgXcQ',
ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS='["dQw4w9WgXcQ"]',
ZAPRET_AUTOTUNE_SOCKS_PROXY='socks5://127.0.0.1:1080')
for dns, valid in (('', True), ('8.8.8.8', True), ('2001:4860:4860::8888', True),
('example.org', False), ('8.8.8.8\n', False), ('-d 8.8.8.8', False)):
for path in private.iterdir():
path.unlink()
result = subprocess.run(['sh', str(entry)], env=dict(base, SS_DNS_SERVER=dns),
capture_output=True, timeout=5)
assert result.returncode == (0 if valid else 64), result.stderr
assert not result.stdout and base['SS_PASSWORD'].encode() not in result.stderr
if not valid:
assert b'configerror:' in result.stderr and not list(private.iterdir())
else:
for name in ('server', 'local'):
path = private / (name + '.json')
assert path.stat().st_mode & 0o777 == 0o600
assert json.loads(path.read_text())['password'] == base['SS_PASSWORD']
# Render privately, never print the config/password; config does not deploy.
compose_env = {k: v for k, v in base.items() if not k.startswith('ZAPRET_AUTOTUNE_')}
for overrides, service, mode, ids in (
({}, '0', 'cold', ['dQw4w9WgXcQ']),
({'ZAPRET_AUTOTUNE_VIDEO_ID': 'aqz-KE-bpKQ'}, '0', 'cold', ['aqz-KE-bpKQ']),
({'ZAPRET_AUTOTUNE_YOUTUBE_SERVICE': '1', 'ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE': 'adaptive',
'ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS': '["dQw4w9WgXcQ","aqz-KE-bpKQ","jNQXAC9IVRw"]',
'SS_DNS_SERVER': '8.8.8.8'}, '1', 'adaptive', ['dQw4w9WgXcQ', 'aqz-KE-bpKQ', 'jNQXAC9IVRw']),
):
result = subprocess.run(['docker', 'compose', '--env-file', '/dev/null', '-f',
str(ROOT / 'deploy/managed-target/compose.yaml'),
'config', '--format', 'json'], env=dict(compose_env, **overrides),
capture_output=True, timeout=10)
assert result.returncode == 0, 'Compose config failed (output suppressed)'
settings = json.loads(result.stdout)['services']['target']['environment']
assert settings['ZAPRET_AUTOTUNE_YOUTUBE_SERVICE'] == service
assert settings['ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE'] == mode
assert json.loads(settings['ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS']) == ids
assert settings['SS_DNS_SERVER'] == overrides.get('SS_DNS_SERVER', '')
print('target config fixtures passed')
+11
View File
@@ -19,6 +19,17 @@ assert " source: ./target-state\n" in compose
assert " target: /var/lib/zapret-autotune/target\n" in compose assert " target: /var/lib/zapret-autotune/target\n" in compose
assert " create_host_path: false\n" in compose assert " create_host_path: false\n" in compose
assert "SS_PASSWORD: ${SS_PASSWORD:?Set SS_PASSWORD in the deployment environment}" in compose assert "SS_PASSWORD: ${SS_PASSWORD:?Set SS_PASSWORD in the deployment environment}" in compose
for setting in (
'SS_DNS_SERVER: ${SS_DNS_SERVER-}',
'ZAPRET_AUTOTUNE_YOUTUBE_SERVICE: ${ZAPRET_AUTOTUNE_YOUTUBE_SERVICE-0}',
'ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE: ${ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE-cold}',
'ZAPRET_AUTOTUNE_VIDEO_ID: ${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}',
"ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS: '${ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS-[\"${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}\"]}'",
):
assert setting in compose, setting
assert entrypoint.index('target-config-check || exit 64') < entrypoint.index('cat > "$config_dir/server.json"')
assert 'set -- "$@" -d "$SS_DNS_SERVER"' in entrypoint
assert 'eval ' not in entrypoint
assert "secrets:" not in compose assert "secrets:" not in compose
assert "ss_password=${SS_PASSWORD-}" in entrypoint assert "ss_password=${SS_PASSWORD-}" in entrypoint
assert "unset SS_PASSWORD" in entrypoint assert "unset SS_PASSWORD" in entrypoint
+11 -3
View File
@@ -73,7 +73,9 @@ if name in ('ss-server', 'ss-local'):
with (root / 'children').open('a') as f: with (root / 'children').open('a') as f:
f.write(str(os.getpid()) + '\n') f.write(str(os.getpid()) + '\n')
assert 'SS_PASSWORD' not in os.environ assert 'SS_PASSWORD' not in os.environ
assert sys.argv[1] == '-c' and len(sys.argv) == 3 assert sys.argv[1] == '-c'
dns = os.environ.get('SS_DNS_SERVER', '') if name == 'ss-server' else ''
assert sys.argv[3:] == (['-d', dns] if dns else [])
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600 assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
log(name) log(name)
if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail') and increment('ss_runs') == 1: if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail') and increment('ss_runs') == 1:
@@ -92,6 +94,8 @@ if name == 'zapret2':
sys.exit(1) sys.exit(1)
sys.exit(0) sys.exit(0)
assert name == 'autotune' assert name == 'autotune'
if sys.argv[1:] == ['target-config-check']:
sys.exit(0)
command = sys.argv[2] command = sys.argv[2]
if command == 'init': if command == 'init':
log('init') log('init')
@@ -140,7 +144,7 @@ elif command == 'monitor':
''' '''
def run(scenario, expected): def run(scenario, expected, dns=''):
with tempfile.TemporaryDirectory(prefix='supervision-') as tmp: with tempfile.TemporaryDirectory(prefix='supervision-') as tmp:
root = Path(tmp) root = Path(tmp)
binary = root / 'bin' binary = root / 'bin'
@@ -165,7 +169,8 @@ def run(scenario, expected):
entry = root / 'entry.sh' entry = root / 'entry.sh'
entry.write_text(text) entry.write_text(text)
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp, env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config')) SCENARIO=scenario, SS_PASSWORD='ab' * 32, SS_DNS_SERVER=dns,
ZAPRET_CONFIG=str(root / 'config'))
peer = None peer = None
if scenario == 'orphans': if scenario == 'orphans':
# Local fixture is not PID 1. Adopt orphan workers here so the test # Local fixture is not PID 1. Adopt orphan workers here so the test
@@ -224,6 +229,9 @@ events = run('cancel', 0)
assert events.count('start') == 1 assert events.count('start') == 1
events = run('prestartfail', 78) events = run('prestartfail', 78)
assert 'start' not in events and 'stop' not in events assert 'start' not in events and 'stop' not in events
for dns in ('8.8.8.8', '2001:4860:4860::8888'):
events = run('fatal64', 64, dns)
assert events.count('ss-server') == 1 and events.count('ss-local') == 1
def check_watchdog_traps(shell): def check_watchdog_traps(shell):
+5
View File
@@ -11,6 +11,11 @@ services:
cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID, KILL] cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID, KILL]
environment: environment:
SS_PASSWORD: ${SS_PASSWORD:?Set SS_PASSWORD in the deployment environment} SS_PASSWORD: ${SS_PASSWORD:?Set SS_PASSWORD in the deployment environment}
SS_DNS_SERVER: ${SS_DNS_SERVER-}
ZAPRET_AUTOTUNE_YOUTUBE_SERVICE: ${ZAPRET_AUTOTUNE_YOUTUBE_SERVICE-0}
ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE: ${ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE-cold}
ZAPRET_AUTOTUNE_VIDEO_ID: ${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}
ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS: '${ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS-["${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}"]}'
SOCKS_PORT: "1080" SOCKS_PORT: "1080"
ZAPRET_AUTOTUNE_TARGET_ROLE: managed-test ZAPRET_AUTOTUNE_TARGET_ROLE: managed-test
ZAPRET_AUTOTUNE_TARGET_ID: zapret-autotune-managed-v1 ZAPRET_AUTOTUNE_TARGET_ID: zapret-autotune-managed-v1
+6 -1
View File
@@ -12,6 +12,8 @@ case "$ss_password" in
echo 'error: target secret must contain only hexadecimal characters' >&2; exit 64 ;; echo 'error: target secret must contain only hexadecimal characters' >&2; exit 64 ;;
esac esac
[ "${#ss_password}" -ge 32 ] || { echo 'error: target secret must be at least 32 hex characters' >&2; exit 64; } [ "${#ss_password}" -ge 32 ] || { echo 'error: target secret must be at least 32 hex characters' >&2; exit 64; }
# Validate before creating configs or launching any service. Errors never echo input.
/usr/local/bin/zapret-autotune target-config-check || exit 64
config_dir=/run/zapret-autotune-ss config_dir=/run/zapret-autotune-ss
awk '$2 == "/run/zapret-autotune-ss" && $3 == "tmpfs" { found=1 } END { exit !found }' /proc/mounts || { awk '$2 == "/run/zapret-autotune-ss" && $3 == "tmpfs" { found=1 } END { exit !found }' /proc/mounts || {
echo 'error: target Shadowsocks config directory must be a dedicated tmpfs' >&2; exit 78; echo 'error: target Shadowsocks config directory must be a dedicated tmpfs' >&2; exit 78;
@@ -163,7 +165,10 @@ trap 'exit 0' INT TERM
# Never invoke /opt/entrypoint.sh: zapret must start AFTER reconciliation. # Never invoke /opt/entrypoint.sh: zapret must start AFTER reconciliation.
run_cycle() { run_cycle() {
ss-server -c "$config_dir/server.json" & set -- ss-server -c "$config_dir/server.json"
# DNS is not a secret. Native -d avoids assuming support for a JSON DNS key.
[ -z "${SS_DNS_SERVER-}" ] || set -- "$@" -d "$SS_DNS_SERVER"
"$@" &
server_pid=$! server_pid=$!
ss-local -c "$config_dir/local.json" & ss-local -c "$config_dir/local.json" &
local_pid=$! local_pid=$!