fix(monitor): keep supervision after empty scans

This commit is contained in:
Zeusina committed 2026-10-10 11:36:57 +03:00
1 parent c19c4ffbd9
commit b2a41270a9
8 files changed
+89 -21

No files matched your search

+2 -2
View File
@@ -4,8 +4,8 @@
## Готово ## Готово
- **2026-10-10 update:** switched the new target password to Compose environment input (`SS_PASSWORD`), removed `DAC_OVERRIDE` and secret-file mount, and clear the variable before launching child processes. Commit: `558b215`; local `go test ./...`, build, Compose config, static deploy guard and entrypoint syntax passed. Remote read-only checks found Docker 29.8.2, the old target still running untouched, and host SOCKS port 11080 free. No target/state was created. Remote deployment is currently blocked because SSH MCP `sftp-upload-file` has no configured `defaults.transferRoot`; direct SCP is unavailable. Configure its transfer root as `/tmp/opencode` (or provide another upload path) to continue the full fresh-target acceptance. - **2026-10-10 remote acceptance:** target image built from pinned base and started in a fresh directory/state volume; host SOCKS port `11080` was free, old container stayed untouched. Initial managed status confirmed generation `0`, `groups=none`, no active candidate IDs, and `NFQWS2_ENABLE=0`. Automated frontend scan of `youtube.com` completed with `total=4323, working=0`; candidate check produced an empty `strategies` array, recorded one failed run, and bootstrap error caused monitor/entrypoint exit. No strategy was applied. New target remains stopped with its fresh, strategy-free state preserved. This exposed that managed bootstrap should scan the actual Player API host (`www.youtube.com`) and ordinary no-candidate errors must not terminate ongoing monitoring. Updated code/tests are in progress; rerun required. Old container and Xray were not changed.
- Added `managed monitor` and supervision only in the dedicated target entrypoint: guarded target-owned DB/identity before singleton, policy 5m/3/30m with bounded CLI/env validation, owned-loopback Player API probe/discovery without direct fallback, fixed blockcheck/zapret argv with context cancellation/join. Startup bootstraps only without a qualified active frontend, tries saved scenarios first, then scans; unresolved GGC keeps monitoring; safety/DB errors stop it. SIGTERM waits for monitor restoration before stopping target-local zapret/SS. Local fake CLI/entrypoint tests are present. Live remote behavior remains unverified; the existing target and Xray are unchanged. - Added `managed monitor` and supervision only in the dedicated target entrypoint: guarded target-owned DB/identity before singleton, policy 5m/3/30m with bounded CLI/env validation, owned-loopback Player API probe/discovery without direct fallback, fixed blockcheck/zapret argv with context cancellation/join. Startup bootstraps only without a qualified active frontend, tries saved scenarios first, then scans; unresolved GGC keeps monitoring; safety/DB errors stop it. SIGTERM waits for monitor restoration before stopping target-local zapret/SS. Local fake CLI/entrypoint tests are present.
- Hardened managed recovery artifacts: общий `revalidateCandidate` удаляет старый combined JSON перед каждым check, remove errors/missing/malformed report fail closed; scan shortlist проверяется на regular file и mtime не раньше whole-second scan start (production использует unique temp path). Добавлены fake-only regression tests stale/missing shortlist, stale combined, remove failure и infrastructure classification/restoration. Tests/build/network не запускались; verification deferred orchestrator, CLI/entrypoint не добавлены. - Hardened managed recovery artifacts: общий `revalidateCandidate` удаляет старый combined JSON перед каждым check, remove errors/missing/malformed report fail closed; scan shortlist проверяется на regular file и mtime не раньше whole-second scan start (production использует unique temp path). Добавлены fake-only regression tests stale/missing shortlist, stale combined, remove failure и infrastructure classification/restoration. Tests/build/network не запускались; verification deferred orchestrator, CLI/entrypoint не добавлены.
+14 -10
View File
@@ -215,10 +215,14 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
if e != nil { if e != nil {
return result, &managedRecoveryFatal{Err: e} return result, &managedRecoveryFatal{Err: e}
} }
if len(c.Endpoints) != 1 || c.Endpoints[0] != "youtube.com" { inScope := false
return result, fmt.Errorf("saved frontend scope is not youtube.com") for _, endpoint := range c.Endpoints {
inScope = inScope || endpoint == managedFrontendHost || strings.HasSuffix(managedFrontendHost, "."+endpoint)
} }
_, e = managedRecheck(ctx, d.check, c.Args, "frontend", "youtube.com") if !inScope {
return result, &managedRecoveryFatal{Err: fmt.Errorf("saved frontend scope excludes Player API host")}
}
_, e = managedRecheck(ctx, d.check, c.Args, "frontend", managedFrontendHost)
if errors.Is(e, errCandidateNonpass) { if errors.Is(e, errCandidateNonpass) {
continue continue
} }
@@ -241,7 +245,7 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
} }
} }
if !healthy { if !healthy {
id, e := managedScanCandidate(ctx, d, "frontend", "youtube.com") id, e := managedScanCandidate(ctx, d, "frontend", managedFrontendHost)
if e != nil { if e != nil {
return result, e return result, e
} }
@@ -379,9 +383,9 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos
} }
if group == "frontend" { if group == "frontend" {
if err == nil { if err == nil {
output, err = combinedBlockcheckOutput(bytes.NewReader(report), args) output, err = combinedBlockcheckOutputForHost(bytes.NewReader(report), args, host)
} }
if e := saveRun(d.db, started, args, output, report, err); e != nil { if e := saveFrontendRunForHost(d.db, started, args, output, report, err, host); e != nil {
return 0, &managedRecoveryFatal{Err: e} return 0, &managedRecoveryFatal{Err: e}
} }
} else { } else {
@@ -433,7 +437,7 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
} }
monitor.bootstrap = func(ctx context.Context) error { monitor.bootstrap = func(ctx context.Context) error {
if err := guard(); err != nil { if err := guard(); err != nil {
return err return &managedRecoveryFatal{Err: err}
} }
// Reconcile once at startup, never adopt a pending live deployment. // Reconcile once at startup, never adopt a pending live deployment.
transaction := *recovery.transaction transaction := *recovery.transaction
@@ -444,16 +448,16 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
return recovery.run(cleanup, managedZapretScript, "restart") return recovery.run(cleanup, managedZapretScript, "restart")
} }
if err := transaction.Reconcile(recovery.db); err != nil { if err := transaction.Reconcile(recovery.db); err != nil {
return err return &managedRecoveryFatal{Err: err}
} }
ids, err := transaction.ActiveCandidates(recovery.db) ids, err := transaction.ActiveCandidates(recovery.db)
if err != nil { if err != nil {
return err return &managedRecoveryFatal{Err: err}
} }
for _, id := range ids { for _, id := range ids {
candidate, err := readStrategyCandidate(recovery.db, id) candidate, err := readStrategyCandidate(recovery.db, id)
if err != nil { if err != nil {
return err return &managedRecoveryFatal{Err: err}
} }
if candidate.Group == "frontend" && candidate.Qualified { if candidate.Group == "frontend" && candidate.Qualified {
return nil return nil
+7 -3
View File
@@ -71,8 +71,12 @@ func TestManagedRecoveryBootstrap(t *testing.T) {
t.Fatalf("%+v %v", result, err) t.Fatalf("%+v %v", result, err)
} }
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID}) assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
frontend, err := readStrategyCandidate(d.db, result.FrontendID)
if err != nil || len(frontend.Endpoints) != 1 || frontend.Endpoints[0] != managedFrontendHost {
t.Fatalf("frontend scope=%v err=%v", frontend.Endpoints, err)
}
config, err := d.transaction.read("config") config, err := d.transaction.read("config")
if err != nil || !strings.Contains(string(config), "--hostlist-domains=^fresh.googlevideo.com ") || strings.Contains(string(config), "--hostlist-domains=googlevideo.com") { if err != nil || !strings.Contains(string(config), "--hostlist-domains=www.youtube.com ") || !strings.Contains(string(config), "--hostlist-domains=^fresh.googlevideo.com ") || strings.Contains(string(config), "--hostlist-domains=googlevideo.com") {
t.Fatalf("%s %v", config, err) t.Fatalf("%s %v", config, err)
} }
if (*events)[0] != "stop" || strings.Count(strings.Join(*events, ","), "restart") != 2 { if (*events)[0] != "stop" || strings.Count(strings.Join(*events, ","), "restart") != 2 {
@@ -88,11 +92,11 @@ func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
t.Fatalf("result=%+v err=%v", result, err) t.Fatalf("result=%+v err=%v", result, err)
} }
for _, event := range *events { for _, event := range *events {
if event == "scan:youtube.com" { if event == "scan:"+managedFrontendHost {
t.Fatalf("frontend scan ran despite a passing saved candidate: %v", *events) t.Fatalf("frontend scan ran despite a passing saved candidate: %v", *events)
} }
} }
if !strings.Contains(strings.Join(*events, ","), "check:youtube.com:--lua-desync=saved") { if !strings.Contains(strings.Join(*events, ","), "check:"+managedFrontendHost+":--lua-desync=saved") {
t.Fatalf("saved candidate was not revalidated: %v", *events) t.Fatalf("saved candidate was not revalidated: %v", *events)
} }
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID}) assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
+9
View File
@@ -123,8 +123,17 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
} }
if d.bootstrap != nil { if d.bootstrap != nil {
if err := d.bootstrap(ctx); err != nil { if err := d.bootstrap(ctx); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
var fatal *managedRecoveryFatal
if errors.As(err, &fatal) {
return err return err
} }
// Bootstrap scans can validly find no qualified strategy. Keep the
// supervisor alive so durable probes/cooldown govern the next attempt.
d.reportRecoveryError(err)
}
} }
for { for {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
@@ -188,6 +188,42 @@ func TestManagedMonitorImmediatePeriodicAndReset(t *testing.T) {
assertMonitorUnlocked(t, d) assertMonitorUnlocked(t, d)
} }
func TestManagedMonitorContinuesAfterOrdinaryBootstrapFailure(t *testing.T) {
d := monitorRuntimeFixture(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probes, reports := 0, 0
d.bootstrap = func(context.Context) error { return errors.New("no qualified bootstrap candidate") }
d.probe = func(context.Context) error { probes++; return nil }
d.reportRecoveryError = func(err error) {
if err == nil || err.Error() != "no qualified bootstrap candidate" {
t.Fatalf("bootstrap report: %v", err)
}
reports++
}
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if probes != 1 || reports != 1 {
t.Fatalf("probes=%d reports=%d", probes, reports)
}
assertMonitorUnlocked(t, d)
}
func TestManagedMonitorFailsClosedOnBootstrapSafetyError(t *testing.T) {
d := monitorRuntimeFixture(t)
d.bootstrap = func(context.Context) error { return &managedRecoveryFatal{Err: errors.New("state drift")} }
probes := 0
d.probe = func(context.Context) error { probes++; return nil }
err := runManagedMonitor(context.Background(), d)
var fatal *managedRecoveryFatal
if !errors.As(err, &fatal) || probes != 0 {
t.Fatalf("err=%v probes=%d", err, probes)
}
assertMonitorUnlocked(t, d)
}
func TestManagedMonitorCooldownFromCompletionEitherOutcome(t *testing.T) { func TestManagedMonitorCooldownFromCompletionEitherOutcome(t *testing.T) {
for _, failed := range []bool{false, true} { for _, failed := range []bool{false, true} {
t.Run(map[bool]string{false: "success", true: "failure"}[failed], func(t *testing.T) { t.Run(map[bool]string{false: "success", true: "failure"}[failed], func(t *testing.T) {
+3 -1
View File
@@ -14,6 +14,8 @@ import (
// Only complete, well-formed non-passing measurements permit scan fallback. // Only complete, well-formed non-passing measurements permit scan fallback.
var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols") var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols")
const managedFrontendHost = "www.youtube.com"
// Rechecks exact saved args using runCombinedBlockcheckForHost's cacheCheck // Rechecks exact saved args using runCombinedBlockcheckForHost's cacheCheck
// contract. The caller supplies a fresh temporary artifact path; artifacts are // contract. The caller supplies a fresh temporary artifact path; artifacts are
// retained, but no stored definition or evidence is updated. A fresh GGC host // retained, but no stored definition or evidence is updated. A fresh GGC host
@@ -39,7 +41,7 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
} else { } else {
inScope := false inScope := false
for _, endpoint := range c.Endpoints { for _, endpoint := range c.Endpoints {
inScope = inScope || endpoint == host inScope = inScope || endpoint == host || strings.HasSuffix(host, "."+endpoint)
} }
if !inScope { if !inScope {
return nil, fmt.Errorf("frontend endpoint outside candidate scope") return nil, fmt.Errorf("frontend endpoint outside candidate scope")
+12 -3
View File
@@ -268,13 +268,22 @@ func strategyFingerprint(args string) (string, error) {
// Validation is shared with tune's output path; only combined 3/3 evidence is stored. // Validation is shared with tune's output path; only combined 3/3 evidence is stored.
func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, runErr error) error { func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, runErr error) error {
return saveFrontendRunForHost(db, started, args, output, report, runErr, "youtube.com")
}
// saveFrontendRunForHost keeps the service-level run target stable while
// recording the actual tested frontend endpoint and candidate scope.
func saveFrontendRunForHost(db *sql.DB, started time.Time, args, output string, report []byte, runErr error, host string) error {
if !validDomain(host) || host != strings.ToLower(host) {
return fmt.Errorf("invalid frontend evidence endpoint")
}
var evidence struct { var evidence struct {
Strategies []json.RawMessage `json:"strategies"` Strategies []json.RawMessage `json:"strategies"`
} }
state := "FAILED" state := "FAILED"
var savedArgs, savedOutput, savedError, fingerprint interface{} var savedArgs, savedOutput, savedError, fingerprint interface{}
if runErr == nil { if runErr == nil {
verifiedOutput, err := combinedBlockcheckOutput(bytes.NewReader(report), args) verifiedOutput, err := combinedBlockcheckOutputForHost(bytes.NewReader(report), args, host)
if err != nil { if err != nil {
return err return err
} }
@@ -309,7 +318,7 @@ func saveRun(db *sql.DB, started time.Time, args, output string, report []byte,
var candidateID int64 var candidateID int64
checked := time.Now().UTC() checked := time.Now().UTC()
if runErr == nil { if runErr == nil {
candidateID, err = insertStrategyCandidate(tx, "frontend", []string{"youtube.com"}, args, id) candidateID, err = insertStrategyCandidate(tx, "frontend", []string{host}, args, id)
if err != nil { if err != nil {
return err return err
} }
@@ -325,7 +334,7 @@ func saveRun(db *sql.DB, started time.Time, args, output string, report []byte,
if _, err := tx.Exec(`INSERT INTO protocol_results (run_id, protocol, passes_ok, passes_total) VALUES (?, ?, ?, ?)`, id, normalizeProtocol(row.Protocol), row.PassesOK, row.PassesTotal); err != nil { if _, err := tx.Exec(`INSERT INTO protocol_results (run_id, protocol, passes_ok, passes_total) VALUES (?, ?, ?, ?)`, id, normalizeProtocol(row.Protocol), row.PassesOK, row.PassesTotal); err != nil {
return err return err
} }
if err := saveCandidateEvidence(tx, candidateID, "youtube.com", normalizeProtocol(row.Protocol), raw, checked); err != nil { if err := saveCandidateEvidence(tx, candidateID, host, normalizeProtocol(row.Protocol), raw, checked); err != nil {
return err return err
} }
} }
+6 -2
View File
@@ -96,13 +96,17 @@ func selectBlockcheckReport(r io.Reader) (string, error) {
} }
func combinedBlockcheckOutput(r io.Reader, args string) (string, error) { func combinedBlockcheckOutput(r io.Reader, args string) (string, error) {
return combinedBlockcheckOutputForHost(r, args, "youtube.com")
}
func combinedBlockcheckOutputForHost(r io.Reader, args, host string) (string, error) {
if args == "" { if args == "" {
return "", fmt.Errorf("empty combined args") return "", fmt.Errorf("empty combined args")
} }
if _, err := selectBlockcheckArgs(r, args); err != nil { if _, err := selectBlockcheckArgsForTarget(r, args, "frontend", host); err != nil {
return "", err return "", err
} }
return "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=youtube.com " + args + "'", nil return "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=" + host + " " + args + "'", nil
} }
func selectBlockcheckArgs(r io.Reader, expected string) (string, error) { func selectBlockcheckArgs(r io.Reader, expected string) (string, error) {