Files
zapret-autotune/cmd/zapret-autotune/assets.go
T

239 lines
8.7 KiB
Go

package main
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/json"
"fmt"
"image"
"image/gif"
_ "image/jpeg"
_ "image/png"
"io"
"math"
"sort"
"strings"
"time"
)
// Rebuild only the two bounded candidate tables, inside migrate's transaction.
// Refuse unexpected objects rather than silently discarding custom constraints.
func migrateAssetCandidates(ctx context.Context, conn *sql.Conn) error {
var actual []string
rows, err := conn.QueryContext(ctx, `SELECT sql FROM sqlite_master WHERE tbl_name IN ('strategy_candidates', 'candidate_evidence') AND sql IS NOT NULL ORDER BY sql`)
if err != nil {
return err
}
for rows.Next() {
var s string
if err := rows.Scan(&s); err != nil {
rows.Close()
return err
}
actual = append(actual, strings.TrimSpace(s))
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
var expected []string
for _, s := range strings.Split(strings.TrimSpace(candidateSchema), ";\n") {
if !strings.HasPrefix(s, "PRAGMA") {
expected = append(expected, strings.TrimSpace(s))
}
}
sort.Strings(expected)
if strings.Join(actual, "\n") != strings.Join(expected, "\n") {
return fmt.Errorf("unsupported candidate schema; asset migration refused")
}
if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE asset_candidate_backup AS SELECT * FROM strategy_candidates;
CREATE TEMP TABLE asset_evidence_backup AS SELECT * FROM candidate_evidence;
DROP TABLE candidate_evidence;
DROP TABLE strategy_candidates;`); err != nil {
return err
}
schema := strings.Replace(candidateSchema, "('frontend', 'googlevideo')", "('frontend', 'googlevideo', 'assets')", 1)
if _, err := conn.ExecContext(ctx, schema); err != nil {
return err
}
if _, err := conn.ExecContext(ctx, `CREATE TRIGGER asset_candidate_scope BEFORE INSERT ON strategy_candidates
WHEN NEW.strategy_group = 'assets' AND (json_array_length(NEW.endpoints) > 2
OR EXISTS (SELECT 1 FROM json_each(NEW.endpoints) WHERE type != 'text' OR value NOT IN ('i.ytimg.com', 'yt3.ggpht.com'))
OR (SELECT count(DISTINCT value) FROM json_each(NEW.endpoints)) != json_array_length(NEW.endpoints))
BEGIN SELECT RAISE(ABORT, 'invalid asset service inventory'); END;`); err != nil {
return err
}
_, err = conn.ExecContext(ctx, `INSERT INTO strategy_candidates SELECT * FROM asset_candidate_backup;
INSERT INTO candidate_evidence SELECT * FROM asset_evidence_backup;
DROP TABLE asset_candidate_backup;
DROP TABLE asset_evidence_backup;
PRAGMA user_version = 6;`)
return err
}
// No URLs: the caller supplies actual response bodies, never a blockcheck report.
// Three transfers per exact TLS version are required; no probing is wired here.
type assetTransfer struct {
EffectiveHost string `json:"effective_host"`
SSLVerifyResult *int `json:"ssl_verify_result"`
Redirects int `json:"redirects"`
ContentType string `json:"content_type"`
BodyHash string `json:"body_sha256"`
Format string `json:"format"`
Size int `json:"size"`
TLSVersion uint16 `json:"tls_version"`
Status int `json:"status"`
Body []byte `json:"-"`
LatencyMS float64 `json:"latency_ms"`
SpeedKBPS float64 `json:"speed_kbps"`
}
type assetMeasurement struct {
Source string `json:"source"`
Endpoint string `json:"endpoint"`
Protocol string `json:"protocol"`
Args string `json:"args"`
Transfers []assetTransfer `json:"transfers"`
}
func (t *assetTransfer) UnmarshalJSON(raw []byte) error {
type fields assetTransfer
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
return err
}
for _, key := range []string{"effective_host", "ssl_verify_result", "redirects", "content_type", "body_sha256", "format", "size", "tls_version", "status", "latency_ms", "speed_kbps"} {
if value, ok := keys[key]; !ok || bytes.Equal(bytes.TrimSpace(value), []byte("null")) {
return fmt.Errorf("missing image transfer field")
}
}
var value fields
d := json.NewDecoder(bytes.NewReader(raw))
d.DisallowUnknownFields()
if err := d.Decode(&value); err != nil {
return err
}
*t = assetTransfer(value)
return nil
}
func saveAssetEvidence(tx *sql.Tx, id int64, m assetMeasurement, checked time.Time) error {
if m.Source != "image-transfer" || !assetEndpoint(m.Endpoint) || (m.Protocol != "tls12" && m.Protocol != "tls13") || checked.IsZero() || len(m.Transfers) != 3 {
return fmt.Errorf("invalid image-transfer evidence")
}
var group, args string
if err := tx.QueryRow(`SELECT strategy_group, args FROM strategy_candidates WHERE id = ?`, id).Scan(&group, &args); err != nil {
return err
}
if group != "assets" || m.Args != args {
return fmt.Errorf("asset evidence definition mismatch")
}
latency, speed, err := validateAssetTransfers(m)
if err != nil {
return err
}
for i := range m.Transfers {
if err := normalizeAssetBody(&m.Transfers[i]); err != nil {
return err
}
}
raw, err := json.Marshal(m)
if err != nil {
return err
}
_, err = tx.Exec(`INSERT INTO candidate_evidence (candidate_id, endpoint, protocol, passes_ok, passes_total, success_rate, coverage, median_latency_ms, median_speed_kbps, checked_at, raw_measurement) VALUES (?, ?, ?, 3, 3, 1, 1, ?, ?, ?, ?)`, id, m.Endpoint, m.Protocol, latency, speed, checked.UTC().Format(time.RFC3339Nano), string(raw))
return err
}
func validateAssetTransfers(m assetMeasurement) (float64, float64, error) {
if m.Source != "image-transfer" || !assetEndpoint(m.Endpoint) || (m.Protocol != "tls12" && m.Protocol != "tls13") || len(m.Transfers) != 3 {
return 0, 0, fmt.Errorf("invalid image-transfer evidence")
}
tlsVersion := uint16(0x0303)
if m.Protocol == "tls13" {
tlsVersion = 0x0304
}
var latency, speed []float64
for _, transfer := range m.Transfers {
if transfer.TLSVersion != tlsVersion || transfer.Status != 200 || transfer.EffectiveHost != m.Endpoint || transfer.SSLVerifyResult == nil || *transfer.SSLVerifyResult != 0 || transfer.Redirects != 0 {
return 0, 0, fmt.Errorf("image transfer must match TLS and HTTP 200 with a bounded image body")
}
if err := normalizeAssetBody(&transfer); err != nil {
return 0, 0, err
}
for _, n := range []float64{transfer.LatencyMS, transfer.SpeedKBPS} {
if n < 0 || math.IsNaN(n) || math.IsInf(n, 0) {
return 0, 0, fmt.Errorf("invalid image transfer metric")
}
}
latency = append(latency, transfer.LatencyMS)
speed = append(speed, transfer.SpeedKBPS)
}
sort.Float64s(latency)
sort.Float64s(speed)
return latency[1], speed[1], nil
}
// Bodies are transient. Stored evidence carries only verified normalized facts.
func normalizeAssetBody(t *assetTransfer) error {
if len(t.Body) != 0 {
if len(t.Body) > 5<<20 {
return fmt.Errorf("image body exceeds limit")
}
cfg, format, err := image.DecodeConfig(bytes.NewReader(t.Body))
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 || cfg.Width > (16<<20)/cfg.Height {
return fmt.Errorf("invalid image dimensions")
}
if format == "gif" {
if _, err := gif.DecodeAll(bytes.NewReader(t.Body)); err != nil {
return fmt.Errorf("invalid GIF body")
}
} else if _, _, err := image.Decode(bytes.NewReader(t.Body)); err != nil {
return fmt.Errorf("invalid image body")
}
t.Format, t.Size, t.BodyHash = format, len(t.Body), fmt.Sprintf("%x", sha256.Sum256(t.Body))
}
want := map[string]string{"png": "image/png", "jpeg": "image/jpeg", "gif": "image/gif"}[t.Format]
if want == "" || t.ContentType != want || t.Size <= 0 || t.Size > 5<<20 || len(t.BodyHash) != 64 || strings.Trim(t.BodyHash, "0123456789abcdef") != "" {
return fmt.Errorf("invalid normalized image evidence")
}
return nil
}
// Revalidate asset bodies on reads too: corrupted raw evidence cannot qualify.
func validateStoredAssetEvidence(db *sql.DB, c storedCandidate) (bool, error) {
rows, err := db.Query(`SELECT endpoint, protocol, median_latency_ms, median_speed_kbps, checked_at, raw_measurement FROM candidate_evidence WHERE candidate_id = ?`, c.ID)
if err != nil {
return false, err
}
defer rows.Close()
for rows.Next() {
var endpoint, protocol, checkedAt, raw string
var latency, speed float64
if err := rows.Scan(&endpoint, &protocol, &latency, &speed, &checkedAt, &raw); err != nil {
return false, err
}
checked, err := time.Parse(time.RFC3339Nano, checkedAt)
if err != nil || checked.IsZero() {
return false, nil
}
var m assetMeasurement
d := json.NewDecoder(strings.NewReader(raw))
d.DisallowUnknownFields()
if err := d.Decode(&m); err != nil {
return false, nil
}
if err := d.Decode(new(interface{})); err != io.EOF {
return false, nil
}
l, s, err := validateAssetTransfers(m)
if err != nil || m.Endpoint != endpoint || m.Protocol != protocol || m.Args != c.Args || l != latency || s != speed {
return false, nil
}
}
return true, rows.Err()
}