feat(monitor): wire isolated target recovery
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
2d4e4878ae
commit
5f3a6953ee
16 files changed
+1919
-73
No files matched your search
@@ -121,17 +121,65 @@ type storedCandidate struct {
|
||||
Qualified bool
|
||||
}
|
||||
|
||||
// Qualification precedes LIMIT: at most ten immutable definitions per trial.
|
||||
func qualifiedCandidateIDs(db *sql.DB, group string) ([]int64, error) {
|
||||
candidates, err := listQualifiedCandidates(db, group, 10)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []int64
|
||||
for _, c := range candidates {
|
||||
ids = append(ids, c.ID)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
const candidateQualifiedSQL = `NOT EXISTS (SELECT 1 FROM json_each(c.endpoints) e CROSS JOIN
|
||||
(SELECT 'tls12' AS protocol UNION ALL SELECT 'tls13') p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM candidate_evidence r WHERE r.candidate_id = c.id
|
||||
AND r.endpoint = e.value AND r.protocol = p.protocol AND r.passes_ok = 3
|
||||
AND r.passes_total = 3 AND r.success_rate = 1 AND r.coverage = 1))`
|
||||
|
||||
// Returns detached snapshots of immutable definitions, not active state.
|
||||
func listQualifiedCandidates(db *sql.DB, group string, limit int) ([]storedCandidate, error) {
|
||||
if group != "frontend" && group != "googlevideo" {
|
||||
return nil, fmt.Errorf("unknown candidate group")
|
||||
}
|
||||
if limit < 1 || limit > 10 {
|
||||
return nil, fmt.Errorf("candidate limit must be between 1 and 10")
|
||||
}
|
||||
rows, err := db.Query(`SELECT id, strategy_group, args, fingerprint, endpoints FROM strategy_candidates c
|
||||
WHERE strategy_group = ? AND `+candidateQualifiedSQL+` ORDER BY id DESC LIMIT ?`, group, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var result []storedCandidate
|
||||
for rows.Next() {
|
||||
var c storedCandidate
|
||||
var scope string
|
||||
if err := rows.Scan(&c.ID, &c.Group, &c.Args, &c.Fingerprint, &scope); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(scope), &c.Endpoints); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Qualified = true
|
||||
result = append(result, c)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, rows.Close()
|
||||
}
|
||||
|
||||
// One shared qualification rule for the complete endpoint × TLS matrix.
|
||||
// Missing or failed evidence is never a qualification; this is not active state.
|
||||
func readStrategyCandidate(db *sql.DB, id int64) (storedCandidate, error) {
|
||||
var c storedCandidate
|
||||
var scope string
|
||||
err := db.QueryRow(`SELECT id, strategy_group, args, fingerprint, endpoints,
|
||||
NOT EXISTS (SELECT 1 FROM json_each(c.endpoints) e CROSS JOIN
|
||||
(SELECT 'tls12' AS protocol UNION ALL SELECT 'tls13') p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM candidate_evidence r WHERE r.candidate_id = c.id
|
||||
AND r.endpoint = e.value AND r.protocol = p.protocol AND r.passes_ok = 3
|
||||
AND r.passes_total = 3 AND r.success_rate = 1 AND r.coverage = 1))
|
||||
`+candidateQualifiedSQL+`
|
||||
FROM strategy_candidates c WHERE id = ?`, id).Scan(&c.ID, &c.Group, &c.Args, &c.Fingerprint, &scope, &c.Qualified)
|
||||
if err != nil {
|
||||
return c, err
|
||||
|
||||
@@ -15,8 +15,8 @@ Commands:
|
||||
discover youtube --video-id ID --proxy socks5://HOST:PORT
|
||||
Discover cache hostname only (no media-transfer check)
|
||||
health youtube One-shot Player API health through configured loopback SOCKS
|
||||
managed init|status|apply
|
||||
Manual activation inside the designated managed-test target only
|
||||
managed init|status|apply|monitor
|
||||
Guarded activation/monitoring in the dedicated managed-test target only
|
||||
check youtube Refuse checks until probe URLs/video IDs are configured and agreed
|
||||
tune youtube Test youtube.com frontend only; print options without applying
|
||||
tune googlevideo --video-id ID --proxy socks5://HOST:PORT
|
||||
|
||||
@@ -305,6 +305,9 @@ func managedRuntimeGuard(d managedDeps) (int, error) {
|
||||
}
|
||||
|
||||
func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
|
||||
if len(args) > 0 && args[0] == "monitor" {
|
||||
return managedMonitorCLI(args[1:], stdout, stderr, d)
|
||||
}
|
||||
command, ids, err := managedArguments(args)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error:", err)
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Provisioning owns executable, database, proxy and target paths; no path flags.
|
||||
func managedMonitorConfig(args []string, lookup func(string) (string, bool)) (healthConfig, error) {
|
||||
cfg, err := parseHealthConfig(lookup)
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
f := flag.NewFlagSet("managed monitor", flag.ContinueOnError)
|
||||
f.SetOutput(io.Discard)
|
||||
f.DurationVar(&cfg.HealthInterval, "interval", cfg.HealthInterval, "")
|
||||
f.IntVar(&cfg.FailureThreshold, "failure-threshold", cfg.FailureThreshold, "")
|
||||
f.DurationVar(&cfg.RecoveryCooldown, "cooldown", cfg.RecoveryCooldown, "")
|
||||
if f.Parse(args) != nil || f.NArg() != 0 || cfg.HealthInterval <= 0 || cfg.HealthInterval > 24*time.Hour || cfg.RecoveryCooldown <= 0 || cfg.RecoveryCooldown > 7*24*time.Hour || cfg.FailureThreshold < 1 || cfg.FailureThreshold > 1000 {
|
||||
return healthConfig{}, fmt.Errorf("monitor requires --interval in (0,24h], --failure-threshold in [1,1000], --cooldown in (0,168h]; no path/command flags")
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func managedMonitorDatabase(d managedDeps) (string, error) {
|
||||
path, _ := d.lookup("ZAPRET_AUTOTUNE_DB")
|
||||
if path != managedTargetDirectory+"/candidates.db" {
|
||||
return "", fmt.Errorf("monitor database must be target-owned candidates.db")
|
||||
}
|
||||
for _, suffix := range []string{"", "-wal", "-shm", "-journal"} {
|
||||
p := filepath.Join(d.directory, "candidates.db"+suffix)
|
||||
if _, err := os.Lstat(p); errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := managedPrivateFile(p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return filepath.Join(d.directory, "candidates.db"), nil
|
||||
}
|
||||
|
||||
func managedMonitorCLI(args []string, stdout, stderr io.Writer, target managedDeps) int {
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
return managedMonitorCLIContext(ctx, args, stdout, stderr, target, runManagedMonitor)
|
||||
}
|
||||
|
||||
func managedMonitorCLIContext(ctx context.Context, args []string, stdout, stderr io.Writer, target managedDeps, supervise func(context.Context, managedMonitorDeps) error) int {
|
||||
cfg, err := managedMonitorConfig(args, target.lookup)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error: invalid managed monitor configuration")
|
||||
return 2
|
||||
}
|
||||
if code, err := managedRuntimeGuard(target); err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor target guard refused")
|
||||
return code
|
||||
}
|
||||
m := &managedTransaction{Dir: target.directory, Name: "config"}
|
||||
if err := managedDirectory(target.directory); err != nil {
|
||||
fmt.Fprintln(stderr, "error: unsafe managed monitor directory")
|
||||
return 1
|
||||
}
|
||||
if err := managedCheckIdentity(m); err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor identity refused")
|
||||
return 1
|
||||
}
|
||||
path, err := managedMonitorDatabase(target)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error: unsafe managed monitor database")
|
||||
return 1
|
||||
}
|
||||
db, err := openStore(path)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor database unavailable")
|
||||
return 1
|
||||
}
|
||||
defer db.Close()
|
||||
recovery, err := managedMonitorRecoveryAdapters(target, db, runBlockcheck, runCombinedBlockcheckForHost, func(ctx context.Context, path string, args ...string) error {
|
||||
return runBlockcheckCommand(ctx, path, "", args, io.Discard, io.Discard)
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor adapters refused")
|
||||
return 1
|
||||
}
|
||||
monitor := managedMonitorDeps{
|
||||
target: target, db: db, serviceID: managedTargetID + "/youtube",
|
||||
interval: cfg.HealthInterval, threshold: cfg.FailureThreshold, cooldown: cfg.RecoveryCooldown,
|
||||
now: time.Now, wait: waitManagedMonitor,
|
||||
reportRecoveryError: func(error) { fmt.Fprintln(stderr, "managed recovery failed (details suppressed)") },
|
||||
}
|
||||
if err := wireManagedRecovery(&monitor, recovery, func(error) {
|
||||
fmt.Fprintln(stderr, "managed GGC unresolved; monitoring continues (not playback health)")
|
||||
}); err != nil {
|
||||
fmt.Fprintln(stderr, "error: managed monitor wiring refused")
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintln(stdout, "managed monitor: Player API via owned loopback SOCKS; not playback health")
|
||||
err = supervise(ctx, monitor)
|
||||
if err != nil {
|
||||
var fatal *managedRecoveryFatal
|
||||
if ctx.Err() != nil && managedCancellationOnly(err, ctx.Err()) && !errors.As(err, &fatal) {
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintln(stderr, "error: managed monitor stopped; safety/database/runtime failure (details suppressed)")
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand cacheScan, checkCommand cacheCheck, command func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
|
||||
if scanCommand == nil || checkCommand == nil || command == nil {
|
||||
return managedRecoveryDeps{}, fmt.Errorf("complete managed process adapters required")
|
||||
}
|
||||
// The blockcheck process-group runner cancels and waits descendants.
|
||||
scan := func(ctx context.Context, _ string, host string, _, _ io.Writer) (string, error) {
|
||||
return scanCommand(ctx, "/usr/bin/blockcheckw", host, io.Discard, io.Discard)
|
||||
}
|
||||
check := func(ctx context.Context, _ string, path, args, host string, _, _ io.Writer) error {
|
||||
return checkCommand(ctx, "/usr/bin/blockcheckw", path, args, host, io.Discard, io.Discard)
|
||||
}
|
||||
run := func(ctx context.Context, path string, args ...string) error {
|
||||
if path != managedZapretScript || len(args) != 1 || (args[0] != "start" && args[0] != "stop" && args[0] != "restart") {
|
||||
return fmt.Errorf("invalid target-local daemon operation")
|
||||
}
|
||||
bounded, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
return command(bounded, path, args...)
|
||||
}
|
||||
return managedRecoveryDependencies(target, db, scan, check, run)
|
||||
}
|
||||
|
||||
// Joined cancellation plus failed cleanup/DB/lock close is not a graceful exit.
|
||||
func managedCancellationOnly(err, cancellation error) bool {
|
||||
if err == cancellation {
|
||||
return true
|
||||
}
|
||||
if joined, ok := err.(interface{ Unwrap() []error }); ok {
|
||||
for _, e := range joined.Unwrap() {
|
||||
if !managedCancellationOnly(e, cancellation) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(joined.Unwrap()) != 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestManagedMonitorArguments(t *testing.T) {
|
||||
lookup := func(string) (string, bool) { return "", false }
|
||||
cfg, err := managedMonitorConfig(nil, lookup)
|
||||
if err != nil || cfg.HealthInterval != 5*time.Minute || cfg.FailureThreshold != 3 || cfg.RecoveryCooldown != 30*time.Minute {
|
||||
t.Fatal(cfg, err)
|
||||
}
|
||||
cfg, err = managedMonitorConfig([]string{"--interval", "2s", "--failure-threshold", "2", "--cooldown", "1m"}, lookup)
|
||||
if err != nil || cfg.HealthInterval != 2*time.Second || cfg.FailureThreshold != 2 || cfg.RecoveryCooldown != time.Minute {
|
||||
t.Fatal(cfg, err)
|
||||
}
|
||||
for _, args := range [][]string{{"--interval", "0"}, {"--interval", "25h"}, {"--cooldown", "169h"}, {"--cooldown", "-1s"}, {"--failure-threshold", "1001"}, {"--failure-threshold", "0"}, {"--command", "secret"}, {"--db", "/etc/db"}, {"extra"}} {
|
||||
if _, err := managedMonitorConfig(args, lookup); err == nil {
|
||||
t.Fatal("accepted", args)
|
||||
}
|
||||
}
|
||||
if _, err := managedMonitorConfig(nil, func(key string) (string, bool) { return "999h", key == "ZAPRET_AUTOTUNE_HEALTH_INTERVAL" }); err == nil {
|
||||
t.Fatal("accepted oversized environment policy")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorCancellationSafety(t *testing.T) {
|
||||
if !managedCancellationOnly(errors.Join(context.Canceled), context.Canceled) || managedCancellationOnly(errors.Join(context.Canceled, errors.New("DB failure")), context.Canceled) || managedCancellationOnly(&managedRecoveryFatal{Err: context.Canceled}, context.Canceled) {
|
||||
t.Fatal("cancellation hid a fatal error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorProductionAdapterBinding(t *testing.T) {
|
||||
recovery, _, _ := recoveryFixture(t, false)
|
||||
target, _, _ := managedCLIFixture(t)
|
||||
target.directory = recovery.transaction.Dir
|
||||
target.player = func(ctx context.Context, args []string, out, _ io.Writer) error {
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
if !strings.Contains(strings.Join(args, "|"), "--noproxy||--proxy|socks5://127.0.0.1:11080|") {
|
||||
t.Fatal(args)
|
||||
}
|
||||
_, err := io.WriteString(out, `{"playabilityStatus":{"status":"OK"},"streamingData":{"serverAbrStreamingUrl":"https://fresh.googlevideo.com/video?secret=never-print"}}`)
|
||||
return err
|
||||
}
|
||||
scans, checks, commands := 0, 0, 0
|
||||
adapted, err := managedMonitorRecoveryAdapters(target, recovery.db,
|
||||
func(ctx context.Context, binary, host string, out, diagnostic io.Writer) (string, error) {
|
||||
if binary != "/usr/bin/blockcheckw" {
|
||||
t.Fatal(binary)
|
||||
}
|
||||
scans++
|
||||
return recovery.scan(ctx, "blockcheckw", host, out, diagnostic)
|
||||
},
|
||||
func(ctx context.Context, binary, path, args, host string, out, diagnostic io.Writer) error {
|
||||
if binary != "/usr/bin/blockcheckw" {
|
||||
t.Fatal(binary)
|
||||
}
|
||||
checks++
|
||||
return recovery.check(ctx, "blockcheckw", path, args, host, out, diagnostic)
|
||||
},
|
||||
func(ctx context.Context, path string, args ...string) error {
|
||||
if _, ok := ctx.Deadline(); !ok {
|
||||
t.Fatal("unbounded daemon operation")
|
||||
}
|
||||
commands++
|
||||
return recovery.run(ctx, path, args...)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := recoverManagedTarget(context.Background(), adapted)
|
||||
if err != nil || result.GGCError != nil || result.FrontendID == 0 || result.GooglevideoID == 0 || scans != 2 || checks != 2 || commands == 0 {
|
||||
t.Fatal(result, err, scans, checks, commands)
|
||||
}
|
||||
if err := adapted.run(context.Background(), "/bin/sh", "-c", "password"); err == nil {
|
||||
t.Fatal("accepted shell command")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) {
|
||||
recovery, _, _ := recoveryFixture(t, false)
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
||||
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recovery.discover = func(context.Context) (string, error) { return "", errors.New("secret") }
|
||||
reports := 0
|
||||
if err := wireManagedRecovery(&monitor, recovery, func(error) { reports++ }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := monitor.bootstrap(context.Background()); err != nil || reports != 1 {
|
||||
t.Fatal(err, reports)
|
||||
}
|
||||
if err := monitor.probe(context.Background()); err != nil {
|
||||
t.Fatal("unresolved GGC stopped API probe", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorCLIGuardsBeforeLock(t *testing.T) {
|
||||
for _, mode := range []string{"role", "identity", "db", "alias", "config"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, env, _ := managedCLIFixture(t)
|
||||
initManagedCLI(t, d)
|
||||
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
|
||||
switch mode {
|
||||
case "role":
|
||||
env["ZAPRET_AUTOTUNE_TARGET_ROLE"] = "old-target"
|
||||
case "identity":
|
||||
os.Remove(filepath.Join(d.directory, managedMarkerName))
|
||||
case "db":
|
||||
env["ZAPRET_AUTOTUNE_DB"] = "/other/db"
|
||||
case "alias":
|
||||
if err := os.Symlink(filepath.Join(t.TempDir(), "db"), filepath.Join(d.directory, "candidates.db")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "config":
|
||||
env["ZAPRET_AUTOTUNE_VIDEO_ID"] = "password-do-not-print"
|
||||
}
|
||||
var out bytes.Buffer
|
||||
code := managedMonitorCLIContext(context.Background(), nil, &out, &out, d, func(context.Context, managedMonitorDeps) error { t.Fatal("unsafe supervisor start"); return nil })
|
||||
if code == 0 || strings.Contains(out.String(), "password-do-not-print") {
|
||||
t.Fatal(code, out.String())
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(d.directory, managedMonitorLockName)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("guard acquired lock", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorCLIBindingAndCancellation(t *testing.T) {
|
||||
d, env, _ := managedCLIFixture(t)
|
||||
initManagedCLI(t, d)
|
||||
env["ZAPRET_AUTOTUNE_DB"] = managedTargetDirectory + "/candidates.db"
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
requests := 0
|
||||
d.player = func(ctx context.Context, args []string, out, _ io.Writer) error {
|
||||
requests++
|
||||
if !strings.Contains(strings.Join(args, "|"), "--noproxy||--proxy|socks5://127.0.0.1:11080|") {
|
||||
t.Fatal(args)
|
||||
}
|
||||
if requests == 1 {
|
||||
_, err := io.WriteString(out, `{"playabilityStatus":{"status":"OK"}}`)
|
||||
return err
|
||||
}
|
||||
cancel()
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
var out bytes.Buffer
|
||||
code := managedMonitorCLIContext(ctx, nil, &out, &out, d, func(ctx context.Context, m managedMonitorDeps) error {
|
||||
if m.bootstrap == nil || m.recover == nil || m.interval != 5*time.Minute || m.threshold != 3 || m.cooldown != 30*time.Minute {
|
||||
t.Fatal("missing wiring/defaults")
|
||||
}
|
||||
if err := m.probe(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = m.probe(ctx)
|
||||
return ctx.Err()
|
||||
})
|
||||
if code != 0 || requests != 2 {
|
||||
t.Fatal(code, requests, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedMonitorBootstrapExistingFrontend(t *testing.T) {
|
||||
recovery, _, events := recoveryFixture(t, true)
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
||||
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := wireManagedRecovery(&monitor, recovery, func(error) { t.Fatal("unexpected GGC recovery") }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := monitor.bootstrap(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*events) != 0 {
|
||||
t.Fatal("startup rescanned qualified active frontend", *events)
|
||||
}
|
||||
}
|
||||
|
||||
// Execute only rewritten target-local contracts and fake programs; no network,
|
||||
// Docker, real credentials, zapret or host firewall. Exercise the actual traps.
|
||||
func TestManagedEntrypointSupervision(t *testing.T) {
|
||||
for _, mode := range []string{"term", "fatal"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
config := filepath.Join(root, "ss")
|
||||
if err := os.Mkdir(config, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write := func(name, text string) string {
|
||||
path := filepath.Join(root, name)
|
||||
if err := os.WriteFile(path, []byte(text), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
log := filepath.Join(root, "log")
|
||||
write("secret", strings.Repeat("a", 32))
|
||||
write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n")
|
||||
child := "#!/bin/sh\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do sleep 0.1 & wait $!; done\n"
|
||||
write("ss-server", child)
|
||||
write("ss-local", child)
|
||||
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\n")
|
||||
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then echo managed-$2 >> \"$LOG\"; exit 0; fi\ntrap 'echo monitor-stopped >> \"$LOG\"; exit 0' TERM INT\necho managed-monitor >> \"$LOG\"\nif [ \"$MODE\" = fatal ]; then exit 7; fi\nwhile :; do sleep 0.1 & wait $!; done\n")
|
||||
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := strings.NewReplacer("/run/secrets/ss_password", filepath.Join(root, "secret"), "/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
|
||||
entry := write("entrypoint", text)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "/bin/sh", entry)
|
||||
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "ZAPRET_CONFIG="+filepath.Join(root, "config"))
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
var output bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &output, &output
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
exited := make(chan struct{})
|
||||
go func() { done <- cmd.Wait(); close(exited) }()
|
||||
t.Cleanup(func() { syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); <-exited })
|
||||
if mode == "term" {
|
||||
for {
|
||||
b, _ := os.ReadFile(log)
|
||||
if strings.Contains(string(b), "managed-monitor") {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatal("premature exit", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
err = <-done
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("entrypoint did not join children", ctx.Err())
|
||||
}
|
||||
if mode == "term" && err != nil {
|
||||
t.Fatal(err, output.String())
|
||||
}
|
||||
if mode == "fatal" {
|
||||
var exit *exec.ExitError
|
||||
if !errors.As(err, &exit) || exit.ExitCode() != 7 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
b, err := os.ReadFile(log)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text = string(b)
|
||||
if !strings.Contains(text, "managed-init\nzapret-start\nmanaged-monitor\n") || !strings.Contains(text, "zapret-stop") || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-server")) || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-local")) {
|
||||
t.Fatal(text)
|
||||
}
|
||||
if mode == "term" && (!strings.Contains(text, "monitor-stopped") || strings.Index(text, "monitor-stopped") > strings.Index(text, "zapret-stop")) {
|
||||
t.Fatal("stop before monitor joined", text)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const managedZapretScript = "/opt/zapret2/init.d/sysv/zapret2"
|
||||
|
||||
// Caller owns startup reconciliation and the monitor singleton. Discover and
|
||||
// Health must bind to the guarded target SOCKS. No CLI parsing or remote work.
|
||||
type managedRecoveryDeps struct {
|
||||
db *sql.DB
|
||||
transaction *managedTransaction
|
||||
scan cacheScan
|
||||
check cacheCheck
|
||||
discover func(context.Context) (string, error)
|
||||
health func(context.Context) (string, error)
|
||||
run func(context.Context, string, ...string) error
|
||||
}
|
||||
|
||||
// Target-local adapter: reuses bounded Player requests without invoking a CLI.
|
||||
// The injected command runner receives only fixed script/argv from orchestration.
|
||||
func managedRecoveryDependencies(target managedDeps, db *sql.DB, scan cacheScan, check cacheCheck, run func(context.Context, string, ...string) error) (managedRecoveryDeps, error) {
|
||||
d := managedRecoveryDeps{db: db, scan: scan, check: check, run: run}
|
||||
if target.lookup == nil || target.player == nil || db == nil || scan == nil || check == nil || run == nil {
|
||||
return d, fmt.Errorf("complete target-local recovery adapter dependencies required")
|
||||
}
|
||||
if _, err := managedRuntimeGuard(target); err != nil {
|
||||
return d, err
|
||||
}
|
||||
cfg, err := parseHealthConfig(target.lookup)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
d.transaction = &managedTransaction{Dir: target.directory, Name: "config"}
|
||||
request := func(ctx context.Context) ([]byte, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := managedRuntimeGuard(target); err != nil {
|
||||
return nil, &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
bounded, cancel := context.WithTimeout(ctx, 35*time.Second)
|
||||
defer cancel()
|
||||
body, err := requestPlayer(cfg.VideoID, cfg.SOCKSProxy, func(_ context.Context, args []string, out, diagnostics io.Writer) error {
|
||||
return target.player(bounded, args, out, diagnostics)
|
||||
})
|
||||
if bounded.Err() != nil {
|
||||
return nil, bounded.Err()
|
||||
}
|
||||
return body, err
|
||||
}
|
||||
d.health = func(ctx context.Context) (string, error) {
|
||||
body, err := request(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var response struct {
|
||||
Playability struct {
|
||||
Status string `json:"status"`
|
||||
} `json:"playabilityStatus"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &response); err != nil {
|
||||
return "", fmt.Errorf("malformed Player API health response")
|
||||
}
|
||||
if response.Playability.Status == "" {
|
||||
return "", fmt.Errorf("missing Player API health status")
|
||||
}
|
||||
if response.Playability.Status != "OK" {
|
||||
return response.Playability.Status, nil
|
||||
}
|
||||
if err := playerHealth(body); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "OK", nil
|
||||
}
|
||||
d.discover = func(ctx context.Context) (string, error) {
|
||||
body, err := request(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return playerHostname(body)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
type managedRecoveryFatal struct{ Err error }
|
||||
|
||||
func (e *managedRecoveryFatal) Error() string {
|
||||
return "managed recovery safety failure: " + e.Err.Error()
|
||||
}
|
||||
func (e *managedRecoveryFatal) Unwrap() error { return e.Err }
|
||||
|
||||
type managedRecoveryResult struct {
|
||||
FrontendID, GooglevideoID int64
|
||||
GGCError error // Unresolved; never a playback/Player API health claim.
|
||||
}
|
||||
|
||||
func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result managedRecoveryResult, err error) {
|
||||
if d.db == nil || d.transaction == nil || d.scan == nil || d.check == nil || d.discover == nil || d.health == nil || d.run == nil {
|
||||
return result, fmt.Errorf("complete managed recovery dependencies required")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
m := *d.transaction
|
||||
// Rollback must restart even after job cancellation; independently bounded.
|
||||
m.Reload = func() error {
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := d.run(cleanup, managedZapretScript, "restart"); err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
m.Health = func() (string, error) { return d.health(ctx) }
|
||||
ids, err := m.ActiveCandidates(d.db)
|
||||
if err != nil {
|
||||
return result, &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
for _, id := range ids {
|
||||
c, e := readStrategyCandidate(d.db, id)
|
||||
if e != nil {
|
||||
return result, &managedRecoveryFatal{Err: e}
|
||||
}
|
||||
if c.Group == "frontend" {
|
||||
result.FrontendID = id
|
||||
}
|
||||
}
|
||||
stopped := false
|
||||
defer func() {
|
||||
if stopped {
|
||||
// Never start bytes left pending by a failed rollback. Reconcile restores
|
||||
// the last committed set or fails closed without activating unknown bytes.
|
||||
if e := m.Reconcile(d.db); e != nil {
|
||||
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
||||
return
|
||||
}
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cancel()
|
||||
if e := d.run(cleanup, managedZapretScript, "start"); e != nil {
|
||||
err = &managedRecoveryFatal{Err: errors.Join(err, e)}
|
||||
}
|
||||
}
|
||||
}()
|
||||
stop := func() error {
|
||||
stopped = true // A partially failed stop also needs restoration.
|
||||
return d.run(ctx, managedZapretScript, "stop")
|
||||
}
|
||||
apply := func(ids []int64) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
cleanupReload := m.Reload
|
||||
first := true
|
||||
m.Reload = func() error {
|
||||
if !first {
|
||||
return cleanupReload()
|
||||
}
|
||||
first = false
|
||||
activation, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if err := d.run(activation, managedZapretScript, "restart"); err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
defer func() { m.Reload = cleanupReload }()
|
||||
e := m.Apply(d.db, ids)
|
||||
if e == nil {
|
||||
stopped = false
|
||||
}
|
||||
var rollback *managedRollbackError
|
||||
if errors.As(e, &rollback) {
|
||||
return &managedRecoveryFatal{Err: e}
|
||||
}
|
||||
var rejection *managedHealthRejection
|
||||
if e != nil && !errors.As(e, &rejection) && !errors.Is(e, context.Canceled) && !errors.Is(e, context.DeadlineExceeded) {
|
||||
return &managedRecoveryFatal{Err: e}
|
||||
}
|
||||
return e
|
||||
}
|
||||
healthy := false
|
||||
if result.FrontendID != 0 {
|
||||
status, e := d.health(ctx)
|
||||
var rejection *managedHealthRejection
|
||||
if e != nil && !errors.As(e, &rejection) {
|
||||
return result, e
|
||||
}
|
||||
healthy = status == "OK"
|
||||
if e != nil {
|
||||
healthy = false
|
||||
}
|
||||
}
|
||||
if !healthy {
|
||||
if e := stop(); e != nil {
|
||||
return result, e
|
||||
}
|
||||
// Both bootstrap and live recovery check saved definitions before scanning.
|
||||
saved, e := qualifiedCandidateIDs(d.db, "frontend")
|
||||
if e != nil {
|
||||
return result, &managedRecoveryFatal{Err: e}
|
||||
}
|
||||
for _, id := range saved {
|
||||
c, e := readStrategyCandidate(d.db, id)
|
||||
if e != nil {
|
||||
return result, &managedRecoveryFatal{Err: e}
|
||||
}
|
||||
if len(c.Endpoints) != 1 || c.Endpoints[0] != "youtube.com" {
|
||||
return result, fmt.Errorf("saved frontend scope is not youtube.com")
|
||||
}
|
||||
_, e = managedRecheck(ctx, d.check, c.Args, "frontend", "youtube.com")
|
||||
if errors.Is(e, errCandidateNonpass) {
|
||||
continue
|
||||
}
|
||||
if e != nil {
|
||||
return result, e
|
||||
}
|
||||
e = apply([]int64{id})
|
||||
if e == nil {
|
||||
result.FrontendID, healthy = id, true
|
||||
break
|
||||
}
|
||||
var fatal *managedRecoveryFatal
|
||||
var rejection *managedHealthRejection
|
||||
if errors.As(e, &fatal) || !errors.As(e, &rejection) {
|
||||
return result, e
|
||||
}
|
||||
// Health rollback restarted the prior config: stop before rechecking.
|
||||
if e := stop(); e != nil {
|
||||
return result, e
|
||||
}
|
||||
}
|
||||
if !healthy {
|
||||
id, e := managedScanCandidate(ctx, d, "frontend", "youtube.com")
|
||||
if e != nil {
|
||||
return result, e
|
||||
}
|
||||
if e := apply([]int64{id}); e != nil {
|
||||
return result, e
|
||||
}
|
||||
result.FrontendID = id
|
||||
}
|
||||
}
|
||||
// A frontend-only rollback boundary prevents cache failures undoing bootstrap.
|
||||
current, e := m.ActiveCandidates(d.db)
|
||||
if e != nil {
|
||||
return result, e
|
||||
}
|
||||
if len(current) != 1 || current[0] != result.FrontendID {
|
||||
if e := apply([]int64{result.FrontendID}); e != nil {
|
||||
return result, e
|
||||
}
|
||||
}
|
||||
host, e := d.discover(ctx)
|
||||
if ctx.Err() != nil {
|
||||
return result, ctx.Err()
|
||||
}
|
||||
if e != nil || !discoveryHostname(host) || host != strings.ToLower(host) || !strings.HasSuffix(host, ".googlevideo.com") {
|
||||
result.GGCError = fmt.Errorf("exact GGC discovery failed")
|
||||
return result, nil
|
||||
}
|
||||
if e := stop(); e != nil {
|
||||
return result, e
|
||||
}
|
||||
args, report, e := revalidateGooglevideo(ctx, d.db, host, io.Discard, d.check)
|
||||
var cacheID int64
|
||||
if e == nil && args != "" {
|
||||
e = saveGooglevideoRun(d.db, time.Now(), host, args, report, nil)
|
||||
if e != nil {
|
||||
return result, &managedRecoveryFatal{Err: e}
|
||||
}
|
||||
if e == nil {
|
||||
cacheID, e = newestManagedCandidateForScope(d.db, "googlevideo", host, args)
|
||||
if e != nil {
|
||||
return result, &managedRecoveryFatal{Err: e}
|
||||
}
|
||||
}
|
||||
} else if e == nil {
|
||||
cacheID, e = managedScanCandidate(ctx, d, "googlevideo", host)
|
||||
}
|
||||
if e == nil {
|
||||
e = apply([]int64{result.FrontendID, cacheID})
|
||||
}
|
||||
if e != nil {
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(e, &fatal) {
|
||||
return result, e
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return result, ctx.Err()
|
||||
}
|
||||
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
|
||||
return result, nil
|
||||
}
|
||||
result.GooglevideoID = cacheID
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func newestManagedCandidate(db *sql.DB, group string) (int64, error) {
|
||||
ids, err := qualifiedCandidateIDs(db, group)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return 0, fmt.Errorf("no qualified candidate")
|
||||
}
|
||||
return ids[0], nil
|
||||
}
|
||||
|
||||
func newestManagedCandidateForScope(db *sql.DB, group, host, args string) (int64, error) {
|
||||
id, err := newestManagedCandidate(db, group)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
c, err := readStrategyCandidate(db, id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if c.Args != args || len(c.Endpoints) != 1 || c.Endpoints[0] != host {
|
||||
return 0, fmt.Errorf("latest saved candidate scope changed")
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func managedRecheck(ctx context.Context, check cacheCheck, args, group, host string) ([]byte, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := os.CreateTemp("", "blockcheckw-managed-*.txt")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
path := f.Name()
|
||||
if err := f.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, check)
|
||||
}
|
||||
|
||||
// Reuses proposal selection and transactional persistence with caller context.
|
||||
func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, host string) (int64, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
started := time.Now()
|
||||
path, err := d.scan(ctx, "blockcheckw", host, io.Discard, io.Discard)
|
||||
var report []byte
|
||||
var args, output string
|
||||
if err == nil {
|
||||
var info os.FileInfo
|
||||
info, err = os.Stat(path + ".check.json")
|
||||
// ponytail: whole-second mtime permits same-second artifacts from injected
|
||||
// scans; production uses unique paths. Require that contract if precision grows.
|
||||
if err == nil && (!info.Mode().IsRegular() || info.ModTime().Before(started.Truncate(time.Second))) {
|
||||
err = fmt.Errorf("missing fresh scan shortlist")
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
report, err = os.ReadFile(path + ".check.json")
|
||||
}
|
||||
if err == nil {
|
||||
args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, host)
|
||||
}
|
||||
if err == nil {
|
||||
report, err = revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, d.check)
|
||||
}
|
||||
if err == nil {
|
||||
err = ctx.Err()
|
||||
}
|
||||
if group == "frontend" {
|
||||
if err == nil {
|
||||
output, err = combinedBlockcheckOutput(bytes.NewReader(report), args)
|
||||
}
|
||||
if e := saveRun(d.db, started, args, output, report, err); e != nil {
|
||||
return 0, &managedRecoveryFatal{Err: e}
|
||||
}
|
||||
} else {
|
||||
var safe error
|
||||
if err != nil {
|
||||
safe = fmt.Errorf("cache scan or verification failed")
|
||||
}
|
||||
if e := saveGooglevideoRun(d.db, started, host, args, report, safe); e != nil {
|
||||
return 0, &managedRecoveryFatal{Err: e}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
id, err := newestManagedCandidateForScope(d.db, group, host, args)
|
||||
if err != nil {
|
||||
return 0, &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// Installs callbacks. runManagedMonitor performs startup
|
||||
// under its singleton. Target guards are repeated before every recovery/probe.
|
||||
func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDeps, reportGGC func(error)) error {
|
||||
if monitor == nil || monitor.db == nil || monitor.db != recovery.db || recovery.transaction == nil || reportGGC == nil || recovery.health == nil || recovery.run == nil || recovery.scan == nil || recovery.check == nil || recovery.discover == nil {
|
||||
return fmt.Errorf("matching monitor/recovery database and hooks required")
|
||||
}
|
||||
if recovery.transaction.Dir != monitor.target.directory || recovery.transaction.Name != "config" {
|
||||
return fmt.Errorf("recovery transaction must own the monitor target config")
|
||||
}
|
||||
guard := func() error {
|
||||
if _, err := managedRuntimeGuard(monitor.target); err != nil {
|
||||
return err
|
||||
}
|
||||
return managedCheckIdentity(recovery.transaction)
|
||||
}
|
||||
callback := func(ctx context.Context) error {
|
||||
if err := guard(); err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
result, err := recoverManagedTarget(ctx, recovery)
|
||||
if result.GGCError != nil {
|
||||
reportGGC(result.GGCError)
|
||||
}
|
||||
// The supervisor reports ordinary recovery failures and retries only
|
||||
// after its durable cooldown. Safety failures (rollback/cleanup/drift)
|
||||
// are already typed managedRecoveryFatal by the transaction boundary.
|
||||
return err
|
||||
}
|
||||
monitor.bootstrap = func(ctx context.Context) error {
|
||||
if err := guard(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Reconcile once at startup, never adopt a pending live deployment.
|
||||
transaction := *recovery.transaction
|
||||
transaction.Health = func() (string, error) { return recovery.health(ctx) }
|
||||
transaction.Reload = func() error {
|
||||
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cancel()
|
||||
return recovery.run(cleanup, managedZapretScript, "restart")
|
||||
}
|
||||
if err := transaction.Reconcile(recovery.db); err != nil {
|
||||
return err
|
||||
}
|
||||
ids, err := transaction.ActiveCandidates(recovery.db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range ids {
|
||||
candidate, err := readStrategyCandidate(recovery.db, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if candidate.Group == "frontend" && candidate.Qualified {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return callback(ctx)
|
||||
}
|
||||
monitor.recover = callback
|
||||
monitor.probe = func(ctx context.Context) error {
|
||||
if err := guard(); err != nil {
|
||||
return &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
status, err := recovery.health(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if status != "OK" {
|
||||
return &managedHealthRejection{Status: status}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,514 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func recoveryFixture(t *testing.T, active bool) (managedRecoveryDeps, int64, *[]string) {
|
||||
t.Helper()
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
db := testStore(t)
|
||||
m, _ := managedFixture(t)
|
||||
if err := m.applyRendered([]byte(managedBaseline)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var front int64
|
||||
if active {
|
||||
front = managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
|
||||
if err := m.Apply(db, []int64{front}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
events := new([]string)
|
||||
stopped := false
|
||||
d := managedRecoveryDeps{
|
||||
db: db, transaction: m,
|
||||
health: func(context.Context) (string, error) { return "OK", nil },
|
||||
discover: func(context.Context) (string, error) { return "fresh.googlevideo.com", nil },
|
||||
run: func(ctx context.Context, path string, args ...string) error {
|
||||
if path != managedZapretScript || len(args) != 1 {
|
||||
t.Fatalf("unsafe command %s %v", path, args)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
t.Fatal("cleanup received cancelled context", err)
|
||||
}
|
||||
*events = append(*events, args[0])
|
||||
stopped = args[0] == "stop"
|
||||
return nil
|
||||
},
|
||||
scan: func(ctx context.Context, binary, host string, _, _ io.Writer) (string, error) {
|
||||
if !stopped || binary != "blockcheckw" {
|
||||
t.Fatal("scan without dedicated downtime")
|
||||
}
|
||||
*events = append(*events, "scan:"+host)
|
||||
path := filepath.Join(t.TempDir(), "scan")
|
||||
return path, os.WriteFile(path+".check.json", []byte(combinedReportFixture("--lua-desync=fake")), 0600)
|
||||
},
|
||||
check: func(ctx context.Context, binary, path, args, host string, _, _ io.Writer) error {
|
||||
if !stopped || binary != "blockcheckw" {
|
||||
t.Fatal("check without dedicated downtime")
|
||||
}
|
||||
*events = append(*events, "check:"+host+":"+args)
|
||||
return os.WriteFile(path+".combined.json", []byte(combinedReportFixture(args)), 0600)
|
||||
},
|
||||
}
|
||||
return d, front, events
|
||||
}
|
||||
|
||||
func TestManagedRecoveryBootstrap(t *testing.T) {
|
||||
d, _, events := recoveryFixture(t, false)
|
||||
result, err := recoverManagedTarget(context.Background(), d)
|
||||
if err != nil || result.GGCError != nil || result.FrontendID == 0 || result.GooglevideoID == 0 {
|
||||
t.Fatalf("%+v %v", result, err)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
|
||||
config, err := d.transaction.read("config")
|
||||
if err != nil || !strings.Contains(string(config), "--hostlist-domains=^fresh.googlevideo.com ") || strings.Contains(string(config), "--hostlist-domains=googlevideo.com") {
|
||||
t.Fatalf("%s %v", config, err)
|
||||
}
|
||||
if (*events)[0] != "stop" || strings.Count(strings.Join(*events, ","), "restart") != 2 {
|
||||
t.Fatal(*events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
|
||||
d, _, events := recoveryFixture(t, false)
|
||||
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
|
||||
result, err := recoverManagedTarget(context.Background(), d)
|
||||
if err != nil || result.GGCError != nil || result.FrontendID != savedID {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
for _, event := range *events {
|
||||
if event == "scan:youtube.com" {
|
||||
t.Fatalf("frontend scan ran despite a passing saved candidate: %v", *events)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.Join(*events, ","), "check:youtube.com:--lua-desync=saved") {
|
||||
t.Fatalf("saved candidate was not revalidated: %v", *events)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
|
||||
}
|
||||
|
||||
func TestManagedRecoverySavedFirstAndFailClosed(t *testing.T) {
|
||||
for _, mode := range []string{"pass", "nonpass", "malformed", "runtime", "checker-nonpass", "missing-report", "incomplete", "health-reject", "apply-infrastructure", "cancel", "cleanup-failure"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, old, events := recoveryFixture(t, true)
|
||||
newID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=multisplit", true)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
probes := 0
|
||||
d.health = func(context.Context) (string, error) {
|
||||
probes++
|
||||
if mode == "apply-infrastructure" && probes == 2 {
|
||||
return "", errors.New("probe infrastructure failed")
|
||||
}
|
||||
if probes == 1 || mode == "health-reject" && probes == 2 {
|
||||
return "DEGRADED", nil
|
||||
}
|
||||
return "OK", nil
|
||||
}
|
||||
d.discover = func(context.Context) (string, error) { return "", errors.New("unresolved") }
|
||||
checked := []string{}
|
||||
baseCheck := d.check
|
||||
d.check = func(ctx context.Context, binary, path, args, host string, out, stderr io.Writer) error {
|
||||
checked = append(checked, args)
|
||||
if mode == "checker-nonpass" {
|
||||
return errCandidateNonpass
|
||||
}
|
||||
if mode == "runtime" || mode == "cleanup-failure" {
|
||||
return errors.New("runner failed")
|
||||
}
|
||||
if mode == "cancel" {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
if mode == "missing-report" {
|
||||
return nil
|
||||
}
|
||||
if err := baseCheck(ctx, binary, path, args, host, out, stderr); err != nil {
|
||||
return err
|
||||
}
|
||||
report := combinedReportFixture(args)
|
||||
switch mode {
|
||||
case "malformed":
|
||||
report = `{}`
|
||||
case "incomplete":
|
||||
report = strings.ReplaceAll(report, "HTTPS/TLS1.3", "HTTPS/TLS1.2")
|
||||
case "nonpass":
|
||||
if !strings.Contains(path, "/scan") {
|
||||
report = strings.Replace(report, `"passes_ok":3`, `"passes_ok":2`, 1)
|
||||
}
|
||||
}
|
||||
return os.WriteFile(path+".combined.json", []byte(report), 0600)
|
||||
}
|
||||
if mode == "cleanup-failure" {
|
||||
run := d.run
|
||||
d.run = func(ctx context.Context, path string, args ...string) error {
|
||||
if err := run(ctx, path, args...); err != nil {
|
||||
return err
|
||||
}
|
||||
if args[0] == "start" {
|
||||
return errors.New("cleanup failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
result, err := recoverManagedTarget(ctx, d)
|
||||
success := mode == "pass" || mode == "nonpass" || mode == "health-reject"
|
||||
if (err == nil) != success {
|
||||
t.Fatalf("result=%+v err=%v events=%v", result, err, *events)
|
||||
}
|
||||
if len(checked) == 0 || checked[0] != "--lua-desync=multisplit" {
|
||||
t.Fatal(checked)
|
||||
}
|
||||
scanned := strings.Contains(strings.Join(*events, ","), "scan:")
|
||||
if scanned != (mode == "nonpass") {
|
||||
t.Fatal(*events)
|
||||
}
|
||||
want := old
|
||||
if mode == "pass" {
|
||||
want = newID
|
||||
}
|
||||
if mode == "nonpass" {
|
||||
want = result.FrontendID
|
||||
if want == old || want == newID {
|
||||
t.Fatal("scan did not save fresh candidate")
|
||||
}
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{want})
|
||||
if !success && (*events)[len(*events)-1] != "start" {
|
||||
t.Fatal("no cleanup", *events)
|
||||
}
|
||||
if mode == "health-reject" && (len(checked) != 2 || strings.Count(strings.Join(*events, ","), "restart") != 3) {
|
||||
t.Fatal("health rejection did not rollback and try older", *events, checked)
|
||||
}
|
||||
if mode == "cleanup-failure" {
|
||||
var fatal *managedRecoveryFatal
|
||||
if !errors.As(err, &fatal) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryScanArtifactsFailClosed(t *testing.T) {
|
||||
for _, mode := range []string{"stale-combined", "remove-error", "missing-shortlist", "stale-shortlist", "malformed-combined", "cancel"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, _, events := recoveryFixture(t, false)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
path := filepath.Join(t.TempDir(), "scan")
|
||||
d.scan = func(context.Context, string, string, io.Writer, io.Writer) (string, error) {
|
||||
if mode != "missing-shortlist" {
|
||||
if err := os.WriteFile(path+".check.json", []byte(combinedReportFixture("--lua-desync=fake")), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if mode == "stale-shortlist" {
|
||||
old := time.Now().Add(-time.Hour)
|
||||
if err := os.Chtimes(path+".check.json", old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if mode == "remove-error" {
|
||||
if err := os.Mkdir(path+".combined.json", 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(path+".combined.json", "keep"), nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else if err := os.WriteFile(path+".combined.json", []byte(combinedReportFixture("--lua-desync=fake")), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mode == "cancel" {
|
||||
cancel()
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
checked := false
|
||||
d.check = func(context.Context, string, string, string, string, io.Writer, io.Writer) error {
|
||||
checked = true
|
||||
if _, err := os.Stat(path + ".combined.json"); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("old combined artifact survived: %v", err)
|
||||
}
|
||||
if mode == "malformed-combined" {
|
||||
return os.WriteFile(path+".combined.json", []byte(`{}`), 0600)
|
||||
}
|
||||
return nil // No fresh report: old passing evidence must not qualify.
|
||||
}
|
||||
result, err := recoverManagedTarget(ctx, d)
|
||||
if err == nil || result.FrontendID != 0 || result.GooglevideoID != 0 {
|
||||
t.Fatalf("artifact qualified: %+v %v", result, err)
|
||||
}
|
||||
wantCheck := mode == "stale-combined" || mode == "malformed-combined"
|
||||
if checked != wantCheck {
|
||||
t.Fatalf("checked=%v want=%v", checked, wantCheck)
|
||||
}
|
||||
if mode == "cancel" && !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(*events, []string{"stop", "start"}) {
|
||||
t.Fatalf("target not restored: %v", *events)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, nil)
|
||||
assertManagedContent(t, d.transaction, managedBaseline)
|
||||
ids, e := qualifiedCandidateIDs(d.db, "frontend")
|
||||
if e != nil || len(ids) != 0 {
|
||||
t.Fatalf("failed scan saved candidate: %v %v", ids, e)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryDiscoveryDropsOnlyOldGGC(t *testing.T) {
|
||||
d, front, events := recoveryFixture(t, true)
|
||||
cache := managedCandidateFixture(t, d.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
|
||||
if err := d.transaction.Apply(d.db, []int64{front, cache}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.discover = func(context.Context) (string, error) { return "", errors.New("discovery failed") }
|
||||
result, err := recoverManagedTarget(context.Background(), d)
|
||||
if err != nil || result.GGCError == nil || result.FrontendID != front || result.GooglevideoID != 0 {
|
||||
t.Fatalf("%+v %v", result, err)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
||||
if !reflect.DeepEqual(*events, []string{"restart"}) {
|
||||
t.Fatal(*events)
|
||||
}
|
||||
old, err := readStrategyCandidate(d.db, cache)
|
||||
if err != nil || !old.Qualified || !reflect.DeepEqual(old.Endpoints, []string{"old.googlevideo.com"}) {
|
||||
t.Fatal("immutable old GGC changed", old, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryRollbackFailureNeverTriesNext(t *testing.T) {
|
||||
d, front, events := recoveryFixture(t, true)
|
||||
managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=multisplit", true)
|
||||
d.health = func(context.Context) (string, error) { return "DEGRADED", nil }
|
||||
run := d.run
|
||||
restarts := 0
|
||||
d.run = func(ctx context.Context, path string, args ...string) error {
|
||||
if err := run(ctx, path, args...); err != nil {
|
||||
return err
|
||||
}
|
||||
if args[0] == "restart" {
|
||||
restarts++
|
||||
if restarts == 2 {
|
||||
return errors.New("rollback restart failed")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, err := recoverManagedTarget(context.Background(), d)
|
||||
var fatal *managedRecoveryFatal
|
||||
if !errors.As(err, &fatal) {
|
||||
t.Fatalf("rollback failure not fatal: %v", err)
|
||||
}
|
||||
joined := strings.Join(*events, ",")
|
||||
if strings.Contains(joined, "scan:") || strings.Count(joined, "check:") != 1 {
|
||||
t.Fatalf("rollback failure authorized another candidate: %v", *events)
|
||||
}
|
||||
if (*events)[len(*events)-1] != "start" {
|
||||
t.Fatalf("reconciliation did not restore target: %v", *events)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
||||
}
|
||||
|
||||
func TestManagedRecoveryFreshGGCAndPartialFailure(t *testing.T) {
|
||||
for _, mode := range []string{"pass", "check-error", "health-reject", "discovery-error", "wildcard", "cancel"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, front, events := recoveryFixture(t, true)
|
||||
oldCache := managedCandidateFixture(t, d.db, "googlevideo", "old.googlevideo.com", "--lua-desync=multisplit", true)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
if mode == "discovery-error" {
|
||||
d.discover = func(context.Context) (string, error) { return "", errors.New("secret URL") }
|
||||
}
|
||||
if mode == "wildcard" {
|
||||
d.discover = func(context.Context) (string, error) { return "googlevideo.com", nil }
|
||||
}
|
||||
check := d.check
|
||||
d.check = func(ctx context.Context, binary, path, args, host string, out, stderr io.Writer) error {
|
||||
if host != "fresh.googlevideo.com" {
|
||||
t.Fatal("old or wildcard scope", host)
|
||||
}
|
||||
if mode == "check-error" {
|
||||
return errors.New("infrastructure")
|
||||
}
|
||||
if mode == "cancel" {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
}
|
||||
return check(ctx, binary, path, args, host, out, stderr)
|
||||
}
|
||||
probes := 0
|
||||
d.health = func(context.Context) (string, error) {
|
||||
probes++
|
||||
if mode == "health-reject" && probes > 1 {
|
||||
return "DEGRADED", nil
|
||||
}
|
||||
return "OK", nil
|
||||
}
|
||||
result, err := recoverManagedTarget(ctx, d)
|
||||
if mode == "cancel" {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.FrontendID != front {
|
||||
t.Fatalf("%+v %v", result, err)
|
||||
}
|
||||
if strings.Contains(strings.Join(*events, ","), "scan:") {
|
||||
t.Fatal("passing frontend/cache caused scan", *events)
|
||||
}
|
||||
want := []int64{front}
|
||||
if mode == "pass" {
|
||||
if result.GGCError != nil || result.GooglevideoID <= oldCache {
|
||||
t.Fatalf("%+v", result)
|
||||
}
|
||||
want = append(want, result.GooglevideoID)
|
||||
fresh, e := readStrategyCandidate(d.db, result.GooglevideoID)
|
||||
old, oldErr := readStrategyCandidate(d.db, oldCache)
|
||||
if e != nil || oldErr != nil || !reflect.DeepEqual(fresh.Endpoints, []string{"fresh.googlevideo.com"}) || !reflect.DeepEqual(old.Endpoints, []string{"old.googlevideo.com"}) {
|
||||
t.Fatal(fresh, old, e, oldErr)
|
||||
}
|
||||
} else if mode != "cancel" && result.GGCError == nil {
|
||||
t.Fatal("cache failure claimed success")
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedQualifiedCandidateBound(t *testing.T) {
|
||||
db := testStore(t)
|
||||
var last int64
|
||||
for i := 0; i < 12; i++ {
|
||||
last = managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", true)
|
||||
}
|
||||
managedCandidateFixture(t, db, "frontend", "youtube.com", "--lua-desync=fake", false)
|
||||
ids, err := qualifiedCandidateIDs(db, "frontend")
|
||||
if err != nil || len(ids) != 10 || ids[0] != last || ids[9] != last-9 {
|
||||
t.Fatal(ids, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryMonitorCallbacks(t *testing.T) {
|
||||
recovery, _, _ := recoveryFixture(t, false)
|
||||
monitor := monitorRuntimeFixture(t)
|
||||
monitor.db = recovery.db
|
||||
monitor.target.directory = recovery.transaction.Dir
|
||||
identity, err := json.Marshal(managedIdentity{Role: "managed-test", Directory: recovery.transaction.Dir, ID: strings.Repeat("01", 16)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recovery.transaction.replace(managedMarkerName, identity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reports []error
|
||||
if err := wireManagedRecovery(&monitor, recovery, func(err error) { reports = append(reports, err) }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := monitor.bootstrap(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := monitor.probe(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(reports) != 0 {
|
||||
t.Fatal(reports)
|
||||
}
|
||||
ids, err := recovery.transaction.ActiveCandidates(recovery.db)
|
||||
if err != nil || len(ids) != 2 {
|
||||
t.Fatal(ids, err)
|
||||
}
|
||||
// A drifted deployment is never adopted by startup/recovery callbacks.
|
||||
if err := os.WriteFile(filepath.Join(recovery.transaction.Dir, "config"), []byte("external"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := monitor.recover(context.Background()); !errors.Is(err, errManagedDrift) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryRuntimeFailures(t *testing.T) {
|
||||
for _, mode := range []string{"stop", "restart", "health-infrastructure"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, front, events := recoveryFixture(t, true)
|
||||
d.health = func(context.Context) (string, error) {
|
||||
if mode == "health-infrastructure" {
|
||||
return "", errors.New("request infrastructure failed")
|
||||
}
|
||||
return "DEGRADED", nil
|
||||
}
|
||||
run := d.run
|
||||
d.run = func(ctx context.Context, path string, args ...string) error {
|
||||
if err := run(ctx, path, args...); err != nil {
|
||||
return err
|
||||
}
|
||||
if args[0] == mode {
|
||||
return errors.New("runtime failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, err := recoverManagedTarget(context.Background(), d)
|
||||
if err == nil {
|
||||
t.Fatal("runtime failure accepted")
|
||||
}
|
||||
if strings.Contains(strings.Join(*events, ","), "scan:") {
|
||||
t.Fatal("infrastructure triggered scan", *events)
|
||||
}
|
||||
if mode == "restart" {
|
||||
var fatal *managedRecoveryFatal
|
||||
if !errors.As(err, &fatal) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Failed rollback restart keeps a pending journal; do not start it.
|
||||
if ids, err := d.transaction.ActiveCandidates(d.db); err == nil || len(ids) != 0 {
|
||||
t.Fatal(ids, err)
|
||||
}
|
||||
} else {
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
||||
if mode == "stop" && (*events)[len(*events)-1] != "start" {
|
||||
t.Fatal(*events)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryPlayerAdapter(t *testing.T) {
|
||||
target, _, _ := managedCLIFixture(t)
|
||||
recovery, _, _ := recoveryFixture(t, false)
|
||||
d, err := managedRecoveryDependencies(target, recovery.db, recovery.scan, recovery.check, recovery.run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, err := d.health(context.Background())
|
||||
if status != "OK" || err != nil {
|
||||
t.Fatal(status, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := d.health(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.discover(context.Background()); err == nil {
|
||||
t.Fatal("health-only response claimed GGC discovery")
|
||||
}
|
||||
if d.transaction.Dir != target.directory || d.transaction.Name != "config" {
|
||||
t.Fatal("wrong target binding")
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,10 @@ import (
|
||||
|
||||
var errManagedDrift = errors.New("managed config drift: refusing overwrite")
|
||||
|
||||
type managedHealthRejection struct{ Status string }
|
||||
|
||||
func (e *managedHealthRejection) Error() string { return fmt.Sprintf("unhealthy status: %q", e.Status) }
|
||||
|
||||
type managedRollbackError struct{ Cause, Rollback error }
|
||||
|
||||
func (e *managedRollbackError) Error() string {
|
||||
@@ -410,7 +414,7 @@ func (m *managedTransaction) applyRenderedDB(b []byte, db *sql.DB, ids ...int64)
|
||||
var status string
|
||||
status, err = m.Health()
|
||||
if err == nil && status != "OK" {
|
||||
err = fmt.Errorf("unhealthy status: %q", status)
|
||||
err = &managedHealthRejection{Status: status}
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
|
||||
const managedMonitorLockName = ".autotune-monitor-lock"
|
||||
|
||||
// No production adapter or entrypoint yet. Callbacks must honor ctx and join
|
||||
// Callbacks must honor ctx and join
|
||||
// their children before returning; the supervisor never detaches callbacks.
|
||||
type managedMonitorDeps struct {
|
||||
target managedDeps
|
||||
@@ -25,6 +25,7 @@ type managedMonitorDeps struct {
|
||||
cooldown time.Duration
|
||||
probe func(context.Context) error
|
||||
recover func(context.Context) error
|
||||
bootstrap func(context.Context) error
|
||||
now func() time.Time
|
||||
wait func(context.Context, time.Duration) error
|
||||
reportRecoveryError func(error)
|
||||
@@ -120,11 +121,20 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
||||
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, singleton, d.cooldown, readMonitorProcessStarttime, d.now); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.bootstrap != nil {
|
||||
if err := d.bootstrap(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
probeErr := d.probe(ctx)
|
||||
var fatalProbe *managedRecoveryFatal
|
||||
if errors.As(probeErr, &fatalProbe) {
|
||||
return probeErr
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -144,6 +154,10 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
||||
}
|
||||
if recoveryErr != nil {
|
||||
d.reportRecoveryError(recoveryErr)
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(recoveryErr, &fatal) {
|
||||
return recoveryErr
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,32 +8,78 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Only complete, well-formed non-passing measurements permit scan fallback.
|
||||
var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols")
|
||||
|
||||
func revalidateGooglevideo(ctx context.Context, db *sql.DB, host string, stderr io.Writer, check cacheCheck) (string, []byte, error) {
|
||||
rows, err := db.Query(`SELECT id FROM strategy_candidates WHERE strategy_group = 'googlevideo' ORDER BY id DESC`)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
// Rechecks exact saved args using runCombinedBlockcheckForHost's cacheCheck
|
||||
// contract. The caller supplies a fresh temporary artifact path; artifacts are
|
||||
// retained, but no stored definition or evidence is updated. A fresh GGC host
|
||||
// is allowed only as an exact endpoint, never as wildcard qualification.
|
||||
func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, path string, stderr io.Writer, check cacheCheck) ([]byte, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return "", nil, err
|
||||
if (group != "frontend" && group != "googlevideo") || c.Group != group || !c.Qualified || path == "" || check == nil {
|
||||
return nil, fmt.Errorf("invalid candidate revalidation request")
|
||||
}
|
||||
definition := candidate{Group: c.Group, Domains: append([]string(nil), c.Endpoints...), Args: c.Args}
|
||||
if err := definition.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !validDomain(host) || host != strings.ToLower(host) {
|
||||
return nil, fmt.Errorf("invalid revalidation endpoint")
|
||||
}
|
||||
if group == "googlevideo" {
|
||||
if !discoveryHostname(host) || !strings.HasSuffix(host, ".googlevideo.com") {
|
||||
return nil, fmt.Errorf("invalid exact cache scope")
|
||||
}
|
||||
} else {
|
||||
inScope := false
|
||||
for _, endpoint := range c.Endpoints {
|
||||
inScope = inScope || endpoint == host
|
||||
}
|
||||
if !inScope {
|
||||
return nil, fmt.Errorf("frontend endpoint outside candidate scope")
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
err = rows.Err()
|
||||
closeErr := rows.Close() // Release the store's single connection before candidate reads.
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
fmt.Fprintln(stderr, "Rechecking saved candidate:", c.ID, "exact endpoint:", host, "Blockcheck artifacts:", path)
|
||||
// A checker that writes nothing must not reuse an earlier measurement.
|
||||
if err := os.Remove(path + ".combined.json"); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, err
|
||||
}
|
||||
var raw bytes.Buffer
|
||||
err := check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw)
|
||||
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
// Only our report parser may issue the nonpass authorization.
|
||||
if errors.Is(err, errCandidateNonpass) {
|
||||
return nil, fmt.Errorf("candidate checker failed: %v", err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
report, err := os.ReadFile(path + ".combined.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, err = selectBlockcheckEvidence(bytes.NewReader(report), c.Args, group, host, true)
|
||||
if contextErr := ctx.Err(); contextErr != nil {
|
||||
return nil, contextErr
|
||||
}
|
||||
return report, err
|
||||
}
|
||||
|
||||
func revalidateGooglevideo(ctx context.Context, db *sql.DB, host string, stderr io.Writer, check cacheCheck) (string, []byte, error) {
|
||||
ids, err := qualifiedCandidateIDs(db, "googlevideo")
|
||||
if err != nil {
|
||||
return "", nil, &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
for _, id := range ids {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -41,7 +87,7 @@ func revalidateGooglevideo(ctx context.Context, db *sql.DB, host string, stderr
|
||||
}
|
||||
c, err := readStrategyCandidate(db, id)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
return "", nil, &managedRecoveryFatal{Err: err}
|
||||
}
|
||||
if !c.Qualified {
|
||||
continue
|
||||
@@ -54,23 +100,7 @@ func revalidateGooglevideo(ctx context.Context, db *sql.DB, host string, stderr
|
||||
if err := artifact.Close(); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
fmt.Fprintln(stderr, "Rechecking saved candidate:", id, "exact endpoint:", host, "Blockcheck artifacts:", path)
|
||||
var raw bytes.Buffer
|
||||
err = check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw)
|
||||
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
|
||||
err = e
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
report, err := os.ReadFile(path + ".combined.json")
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
_, err = selectBlockcheckEvidence(bytes.NewReader(report), c.Args, "googlevideo", host, true)
|
||||
report, err := revalidateCandidate(ctx, c, "googlevideo", host, path, stderr, check)
|
||||
if errors.Is(err, errCandidateNonpass) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func evidenceSnapshot(t *testing.T, db *sql.DB) string {
|
||||
t.Helper()
|
||||
var snapshot string
|
||||
if err := db.QueryRow(`SELECT group_concat(row, ';') FROM (SELECT
|
||||
quote(candidate_id)||','||quote(endpoint)||','||quote(protocol)||','||quote(passes_ok)||','||quote(passes_total)||','||quote(success_rate)||','||quote(coverage)||','||quote(median_latency_ms)||','||quote(median_speed_kbps)||','||quote(checked_at)||','||quote(raw_measurement) AS row
|
||||
FROM candidate_evidence ORDER BY candidate_id, endpoint, protocol)`).Scan(&snapshot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func TestListQualifiedCandidates(t *testing.T) {
|
||||
db := testStore(t)
|
||||
args := "--lua-desync=multisplit"
|
||||
for i := 0; i < 12; i++ {
|
||||
saveProposal(t, db, args)
|
||||
}
|
||||
if err := saveGooglevideoRun(db, time.Now(), cacheTestHost, args, []byte(combinedReportFixture(args)), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := insertStrategyCandidate(tx, "frontend", []string{"youtube.com"}, args, nil); err != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := evidenceSnapshot(t, db)
|
||||
for _, limit := range []int{1, 3, 10} {
|
||||
candidates, err := listQualifiedCandidates(db, "frontend", limit)
|
||||
if err != nil || len(candidates) != limit {
|
||||
t.Fatalf("limit=%d candidates=%+v err=%v", limit, candidates, err)
|
||||
}
|
||||
for i, c := range candidates {
|
||||
if c.ID != int64(12-i) || !c.Qualified || c.Group != "frontend" {
|
||||
t.Fatalf("unexpected candidate: %+v", c)
|
||||
}
|
||||
}
|
||||
// Returned slices cannot mutate definitions or later snapshots.
|
||||
candidates[0].Endpoints[0] = "changed.example"
|
||||
stored, err := readStrategyCandidate(db, candidates[0].ID)
|
||||
if err != nil || stored.Endpoints[0] != "youtube.com" {
|
||||
t.Fatalf("stored snapshot changed: %+v %v", stored, err)
|
||||
}
|
||||
}
|
||||
for _, limit := range []int{-1, 0, 11} {
|
||||
if _, err := listQualifiedCandidates(db, "frontend", limit); err == nil {
|
||||
t.Fatalf("accepted limit %d", limit)
|
||||
}
|
||||
}
|
||||
if _, err := listQualifiedCandidates(db, "unknown", 1); err == nil {
|
||||
t.Fatal("accepted unknown group")
|
||||
}
|
||||
ggc, err := listQualifiedCandidates(db, "googlevideo", 10)
|
||||
if err != nil || len(ggc) != 1 || ggc[0].Group != "googlevideo" {
|
||||
t.Fatalf("googlevideo=%+v err=%v", ggc, err)
|
||||
}
|
||||
if before != evidenceSnapshot(t, db) {
|
||||
t.Fatal("listing mutated evidence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRevalidateCandidate(t *testing.T) {
|
||||
db := testStore(t)
|
||||
args := "--lua-desync=multisplit"
|
||||
saveProposal(t, db, args)
|
||||
c, err := readStrategyCandidate(db, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := c
|
||||
original.Endpoints = append([]string(nil), c.Endpoints...)
|
||||
before := evidenceSnapshot(t, db)
|
||||
passing := combinedReportFixture(args)
|
||||
var parsed struct {
|
||||
Strategies []json.RawMessage `json:"strategies"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(passing), &parsed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mode := range []string{"pass", "one-nonpass", "both-nonpass", "nonpass-malformed", "malformed", "incomplete", "duplicate", "wrong-args", "wrong-host", "wrong-group", "missing-metric", "invalid-metric", "incomplete-passes", "trailing-json", "runner", "runner-sentinel", "missing-file", "stale-file", "filesystem", "cancelled"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "check")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
report := passing
|
||||
switch mode {
|
||||
case "one-nonpass":
|
||||
report = strings.Replace(report, `"passes_ok":3`, `"passes_ok":2`, 1)
|
||||
case "both-nonpass":
|
||||
report = strings.ReplaceAll(report, `"passes_ok":3`, `"passes_ok":0`)
|
||||
case "nonpass-malformed":
|
||||
report = strings.Replace(report, `"passes_ok":3`, `"passes_ok":2`, 1)
|
||||
i := strings.LastIndex(report, `"coverage":1`)
|
||||
report = report[:i] + strings.Replace(report[i:], `"coverage":1`, `"coverage":null`, 1)
|
||||
case "malformed":
|
||||
report = `{`
|
||||
case "incomplete":
|
||||
report = `{"strategies":[` + string(parsed.Strategies[0]) + `]}`
|
||||
case "duplicate":
|
||||
report = `{"strategies":[` + string(parsed.Strategies[0]) + `,` + string(parsed.Strategies[0]) + `,` + string(parsed.Strategies[1]) + `]}`
|
||||
case "wrong-args":
|
||||
report = combinedReportFixture("--lua-desync=fake")
|
||||
case "wrong-host":
|
||||
report = strings.ReplaceAll(report, `"protocol":`, `"domains":["other.example"],"protocol":`)
|
||||
case "wrong-group":
|
||||
report = strings.ReplaceAll(report, `"protocol":`, `"group":"googlevideo","protocol":`)
|
||||
case "missing-metric":
|
||||
report = strings.Replace(report, `"coverage":1`, `"coverage":null`, 1)
|
||||
case "invalid-metric":
|
||||
report = strings.Replace(report, `"median_speed_kbps":100`, `"median_speed_kbps":-1`, 1)
|
||||
case "incomplete-passes":
|
||||
report = strings.Replace(report, `"passes_total":3`, `"passes_total":4`, 1)
|
||||
case "trailing-json":
|
||||
report += `{}`
|
||||
case "filesystem":
|
||||
path = filepath.Join(path, "missing", "check")
|
||||
case "stale-file":
|
||||
if err := os.WriteFile(path+".combined.json", []byte(passing), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
check := func(_ context.Context, binary, gotPath, gotArgs, host string, _, _ io.Writer) error {
|
||||
if binary != "blockcheckw" || gotPath != path || gotArgs != args || host != "youtube.com" {
|
||||
t.Fatalf("wrong combined check: %s %s %s %s", binary, gotPath, gotArgs, host)
|
||||
}
|
||||
switch mode {
|
||||
case "runner":
|
||||
return fmt.Errorf("runner failed")
|
||||
case "runner-sentinel":
|
||||
return errCandidateNonpass
|
||||
case "missing-file", "stale-file", "filesystem":
|
||||
return nil
|
||||
case "cancelled":
|
||||
cancel()
|
||||
}
|
||||
return os.WriteFile(path+".combined.json", []byte(report), 0600)
|
||||
}
|
||||
got, err := revalidateCandidate(ctx, c, "frontend", "youtube.com", path, io.Discard, check)
|
||||
if mode == "pass" {
|
||||
if err != nil || string(got) != passing {
|
||||
t.Fatalf("passing report=%s err=%v", got, err)
|
||||
}
|
||||
} else if mode == "one-nonpass" || mode == "both-nonpass" {
|
||||
if !errors.Is(err, errCandidateNonpass) {
|
||||
t.Fatalf("expected nonpass, got %v", err)
|
||||
}
|
||||
} else if err == nil || errors.Is(err, errCandidateNonpass) {
|
||||
t.Fatalf("expected infrastructure error, got %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(c, original) || before != evidenceSnapshot(t, db) {
|
||||
t.Fatal("revalidation mutated candidate or evidence")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRevalidateCandidateScope(t *testing.T) {
|
||||
db := testStore(t)
|
||||
args := "--lua-desync=multisplit"
|
||||
if err := saveGooglevideoRun(db, time.Now(), "old.googlevideo.com", args, []byte(combinedReportFixture(args)), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, err := readStrategyCandidate(db, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, target := range []struct{ group, host string }{
|
||||
{"googlevideo", cacheTestHost}, {"googlevideo", "googlevideo.com"},
|
||||
{"googlevideo", "^" + cacheTestHost}, {"googlevideo", "*.googlevideo.com"},
|
||||
{"googlevideo", "youtube.com"}, {"frontend", "youtube.com"}, {"unknown", cacheTestHost},
|
||||
} {
|
||||
calls := 0
|
||||
check := func(_ context.Context, _, path, gotArgs, host string, _, _ io.Writer) error {
|
||||
calls++
|
||||
if gotArgs != args || host != cacheTestHost {
|
||||
t.Fatal("widened scope or changed args")
|
||||
}
|
||||
return os.WriteFile(path+".combined.json", []byte(combinedReportFixture(args)), 0600)
|
||||
}
|
||||
_, err := revalidateCandidate(context.Background(), c, target.group, target.host, filepath.Join(t.TempDir(), "check"), io.Discard, check)
|
||||
valid := target.group == "googlevideo" && target.host == cacheTestHost
|
||||
if valid && (err != nil || calls != 1) || !valid && (err == nil || calls != 0 || errors.Is(err, errCandidateNonpass)) {
|
||||
t.Fatalf("target=%+v calls=%d err=%v", target, calls, err)
|
||||
}
|
||||
}
|
||||
stored, err := readStrategyCandidate(db, 1)
|
||||
if err != nil || !reflect.DeepEqual(c, stored) || c.Endpoints[0] != "old.googlevideo.com" {
|
||||
t.Fatalf("changed stored scope: %+v %v", stored, err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user