fix(monitor): keep supervision after empty scans
This commit is contained in:
1 parent
c19c4ffbd9
commit
b2a41270a9
8 files changed
+89
-21
No files matched your search
+2
-2
@@ -4,8 +4,8 @@
|
|||||||
|
|
||||||
## Готово
|
## Готово
|
||||||
|
|
||||||
- **2026-10-10 update:** switched the new target password to Compose environment input (`SS_PASSWORD`), removed `DAC_OVERRIDE` and secret-file mount, and clear the variable before launching child processes. Commit: `558b215`; local `go test ./...`, build, Compose config, static deploy guard and entrypoint syntax passed. Remote read-only checks found Docker 29.8.2, the old target still running untouched, and host SOCKS port 11080 free. No target/state was created. Remote deployment is currently blocked because SSH MCP `sftp-upload-file` has no configured `defaults.transferRoot`; direct SCP is unavailable. Configure its transfer root as `/tmp/opencode` (or provide another upload path) to continue the full fresh-target acceptance.
|
- **2026-10-10 remote acceptance:** target image built from pinned base and started in a fresh directory/state volume; host SOCKS port `11080` was free, old container stayed untouched. Initial managed status confirmed generation `0`, `groups=none`, no active candidate IDs, and `NFQWS2_ENABLE=0`. Automated frontend scan of `youtube.com` completed with `total=4323, working=0`; candidate check produced an empty `strategies` array, recorded one failed run, and bootstrap error caused monitor/entrypoint exit. No strategy was applied. New target remains stopped with its fresh, strategy-free state preserved. This exposed that managed bootstrap should scan the actual Player API host (`www.youtube.com`) and ordinary no-candidate errors must not terminate ongoing monitoring. Updated code/tests are in progress; rerun required. Old container and Xray were not changed.
|
||||||
- Added `managed monitor` and supervision only in the dedicated target entrypoint: guarded target-owned DB/identity before singleton, policy 5m/3/30m with bounded CLI/env validation, owned-loopback Player API probe/discovery without direct fallback, fixed blockcheck/zapret argv with context cancellation/join. Startup bootstraps only without a qualified active frontend, tries saved scenarios first, then scans; unresolved GGC keeps monitoring; safety/DB errors stop it. SIGTERM waits for monitor restoration before stopping target-local zapret/SS. Local fake CLI/entrypoint tests are present. Live remote behavior remains unverified; the existing target and Xray are unchanged.
|
- Added `managed monitor` and supervision only in the dedicated target entrypoint: guarded target-owned DB/identity before singleton, policy 5m/3/30m with bounded CLI/env validation, owned-loopback Player API probe/discovery without direct fallback, fixed blockcheck/zapret argv with context cancellation/join. Startup bootstraps only without a qualified active frontend, tries saved scenarios first, then scans; unresolved GGC keeps monitoring; safety/DB errors stop it. SIGTERM waits for monitor restoration before stopping target-local zapret/SS. Local fake CLI/entrypoint tests are present.
|
||||||
|
|
||||||
- Hardened managed recovery artifacts: общий `revalidateCandidate` удаляет старый combined JSON перед каждым check, remove errors/missing/malformed report fail closed; scan shortlist проверяется на regular file и mtime не раньше whole-second scan start (production использует unique temp path). Добавлены fake-only regression tests stale/missing shortlist, stale combined, remove failure и infrastructure classification/restoration. Tests/build/network не запускались; verification deferred orchestrator, CLI/entrypoint не добавлены.
|
- Hardened managed recovery artifacts: общий `revalidateCandidate` удаляет старый combined JSON перед каждым check, remove errors/missing/malformed report fail closed; scan shortlist проверяется на regular file и mtime не раньше whole-second scan start (production использует unique temp path). Добавлены fake-only regression tests stale/missing shortlist, stale combined, remove failure и infrastructure classification/restoration. Tests/build/network не запускались; verification deferred orchestrator, CLI/entrypoint не добавлены.
|
||||||
|
|
||||||
|
|||||||
@@ -215,10 +215,14 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
|||||||
if e != nil {
|
if e != nil {
|
||||||
return result, &managedRecoveryFatal{Err: e}
|
return result, &managedRecoveryFatal{Err: e}
|
||||||
}
|
}
|
||||||
if len(c.Endpoints) != 1 || c.Endpoints[0] != "youtube.com" {
|
inScope := false
|
||||||
return result, fmt.Errorf("saved frontend scope is not youtube.com")
|
for _, endpoint := range c.Endpoints {
|
||||||
|
inScope = inScope || endpoint == managedFrontendHost || strings.HasSuffix(managedFrontendHost, "."+endpoint)
|
||||||
}
|
}
|
||||||
_, e = managedRecheck(ctx, d.check, c.Args, "frontend", "youtube.com")
|
if !inScope {
|
||||||
|
return result, &managedRecoveryFatal{Err: fmt.Errorf("saved frontend scope excludes Player API host")}
|
||||||
|
}
|
||||||
|
_, e = managedRecheck(ctx, d.check, c.Args, "frontend", managedFrontendHost)
|
||||||
if errors.Is(e, errCandidateNonpass) {
|
if errors.Is(e, errCandidateNonpass) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -241,7 +245,7 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !healthy {
|
if !healthy {
|
||||||
id, e := managedScanCandidate(ctx, d, "frontend", "youtube.com")
|
id, e := managedScanCandidate(ctx, d, "frontend", managedFrontendHost)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return result, e
|
return result, e
|
||||||
}
|
}
|
||||||
@@ -379,9 +383,9 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos
|
|||||||
}
|
}
|
||||||
if group == "frontend" {
|
if group == "frontend" {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
output, err = combinedBlockcheckOutput(bytes.NewReader(report), args)
|
output, err = combinedBlockcheckOutputForHost(bytes.NewReader(report), args, host)
|
||||||
}
|
}
|
||||||
if e := saveRun(d.db, started, args, output, report, err); e != nil {
|
if e := saveFrontendRunForHost(d.db, started, args, output, report, err, host); e != nil {
|
||||||
return 0, &managedRecoveryFatal{Err: e}
|
return 0, &managedRecoveryFatal{Err: e}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -433,7 +437,7 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
|||||||
}
|
}
|
||||||
monitor.bootstrap = func(ctx context.Context) error {
|
monitor.bootstrap = func(ctx context.Context) error {
|
||||||
if err := guard(); err != nil {
|
if err := guard(); err != nil {
|
||||||
return err
|
return &managedRecoveryFatal{Err: err}
|
||||||
}
|
}
|
||||||
// Reconcile once at startup, never adopt a pending live deployment.
|
// Reconcile once at startup, never adopt a pending live deployment.
|
||||||
transaction := *recovery.transaction
|
transaction := *recovery.transaction
|
||||||
@@ -444,16 +448,16 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
|||||||
return recovery.run(cleanup, managedZapretScript, "restart")
|
return recovery.run(cleanup, managedZapretScript, "restart")
|
||||||
}
|
}
|
||||||
if err := transaction.Reconcile(recovery.db); err != nil {
|
if err := transaction.Reconcile(recovery.db); err != nil {
|
||||||
return err
|
return &managedRecoveryFatal{Err: err}
|
||||||
}
|
}
|
||||||
ids, err := transaction.ActiveCandidates(recovery.db)
|
ids, err := transaction.ActiveCandidates(recovery.db)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return &managedRecoveryFatal{Err: err}
|
||||||
}
|
}
|
||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
candidate, err := readStrategyCandidate(recovery.db, id)
|
candidate, err := readStrategyCandidate(recovery.db, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return &managedRecoveryFatal{Err: err}
|
||||||
}
|
}
|
||||||
if candidate.Group == "frontend" && candidate.Qualified {
|
if candidate.Group == "frontend" && candidate.Qualified {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -71,8 +71,12 @@ func TestManagedRecoveryBootstrap(t *testing.T) {
|
|||||||
t.Fatalf("%+v %v", result, err)
|
t.Fatalf("%+v %v", result, err)
|
||||||
}
|
}
|
||||||
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
|
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
|
||||||
|
frontend, err := readStrategyCandidate(d.db, result.FrontendID)
|
||||||
|
if err != nil || len(frontend.Endpoints) != 1 || frontend.Endpoints[0] != managedFrontendHost {
|
||||||
|
t.Fatalf("frontend scope=%v err=%v", frontend.Endpoints, err)
|
||||||
|
}
|
||||||
config, err := d.transaction.read("config")
|
config, err := d.transaction.read("config")
|
||||||
if err != nil || !strings.Contains(string(config), "--hostlist-domains=^fresh.googlevideo.com ") || strings.Contains(string(config), "--hostlist-domains=googlevideo.com") {
|
if err != nil || !strings.Contains(string(config), "--hostlist-domains=www.youtube.com ") || !strings.Contains(string(config), "--hostlist-domains=^fresh.googlevideo.com ") || strings.Contains(string(config), "--hostlist-domains=googlevideo.com") {
|
||||||
t.Fatalf("%s %v", config, err)
|
t.Fatalf("%s %v", config, err)
|
||||||
}
|
}
|
||||||
if (*events)[0] != "stop" || strings.Count(strings.Join(*events, ","), "restart") != 2 {
|
if (*events)[0] != "stop" || strings.Count(strings.Join(*events, ","), "restart") != 2 {
|
||||||
@@ -88,11 +92,11 @@ func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
|
|||||||
t.Fatalf("result=%+v err=%v", result, err)
|
t.Fatalf("result=%+v err=%v", result, err)
|
||||||
}
|
}
|
||||||
for _, event := range *events {
|
for _, event := range *events {
|
||||||
if event == "scan:youtube.com" {
|
if event == "scan:"+managedFrontendHost {
|
||||||
t.Fatalf("frontend scan ran despite a passing saved candidate: %v", *events)
|
t.Fatalf("frontend scan ran despite a passing saved candidate: %v", *events)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !strings.Contains(strings.Join(*events, ","), "check:youtube.com:--lua-desync=saved") {
|
if !strings.Contains(strings.Join(*events, ","), "check:"+managedFrontendHost+":--lua-desync=saved") {
|
||||||
t.Fatalf("saved candidate was not revalidated: %v", *events)
|
t.Fatalf("saved candidate was not revalidated: %v", *events)
|
||||||
}
|
}
|
||||||
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
|
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
|
||||||
|
|||||||
@@ -123,8 +123,17 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
|||||||
}
|
}
|
||||||
if d.bootstrap != nil {
|
if d.bootstrap != nil {
|
||||||
if err := d.bootstrap(ctx); err != nil {
|
if err := d.bootstrap(ctx); err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if errors.As(err, &fatal) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// Bootstrap scans can validly find no qualified strategy. Keep the
|
||||||
|
// supervisor alive so durable probes/cooldown govern the next attempt.
|
||||||
|
d.reportRecoveryError(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for {
|
for {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
|
|||||||
@@ -188,6 +188,42 @@ func TestManagedMonitorImmediatePeriodicAndReset(t *testing.T) {
|
|||||||
assertMonitorUnlocked(t, d)
|
assertMonitorUnlocked(t, d)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorContinuesAfterOrdinaryBootstrapFailure(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
probes, reports := 0, 0
|
||||||
|
d.bootstrap = func(context.Context) error { return errors.New("no qualified bootstrap candidate") }
|
||||||
|
d.probe = func(context.Context) error { probes++; return nil }
|
||||||
|
d.reportRecoveryError = func(err error) {
|
||||||
|
if err == nil || err.Error() != "no qualified bootstrap candidate" {
|
||||||
|
t.Fatalf("bootstrap report: %v", err)
|
||||||
|
}
|
||||||
|
reports++
|
||||||
|
}
|
||||||
|
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||||
|
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if probes != 1 || reports != 1 {
|
||||||
|
t.Fatalf("probes=%d reports=%d", probes, reports)
|
||||||
|
}
|
||||||
|
assertMonitorUnlocked(t, d)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorFailsClosedOnBootstrapSafetyError(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
d.bootstrap = func(context.Context) error { return &managedRecoveryFatal{Err: errors.New("state drift")} }
|
||||||
|
probes := 0
|
||||||
|
d.probe = func(context.Context) error { probes++; return nil }
|
||||||
|
err := runManagedMonitor(context.Background(), d)
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if !errors.As(err, &fatal) || probes != 0 {
|
||||||
|
t.Fatalf("err=%v probes=%d", err, probes)
|
||||||
|
}
|
||||||
|
assertMonitorUnlocked(t, d)
|
||||||
|
}
|
||||||
|
|
||||||
func TestManagedMonitorCooldownFromCompletionEitherOutcome(t *testing.T) {
|
func TestManagedMonitorCooldownFromCompletionEitherOutcome(t *testing.T) {
|
||||||
for _, failed := range []bool{false, true} {
|
for _, failed := range []bool{false, true} {
|
||||||
t.Run(map[bool]string{false: "success", true: "failure"}[failed], func(t *testing.T) {
|
t.Run(map[bool]string{false: "success", true: "failure"}[failed], func(t *testing.T) {
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ import (
|
|||||||
// Only complete, well-formed non-passing measurements permit scan fallback.
|
// Only complete, well-formed non-passing measurements permit scan fallback.
|
||||||
var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols")
|
var errCandidateNonpass = errors.New("saved scenario did not pass both TLS protocols")
|
||||||
|
|
||||||
|
const managedFrontendHost = "www.youtube.com"
|
||||||
|
|
||||||
// Rechecks exact saved args using runCombinedBlockcheckForHost's cacheCheck
|
// Rechecks exact saved args using runCombinedBlockcheckForHost's cacheCheck
|
||||||
// contract. The caller supplies a fresh temporary artifact path; artifacts are
|
// contract. The caller supplies a fresh temporary artifact path; artifacts are
|
||||||
// retained, but no stored definition or evidence is updated. A fresh GGC host
|
// retained, but no stored definition or evidence is updated. A fresh GGC host
|
||||||
@@ -39,7 +41,7 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
|
|||||||
} else {
|
} else {
|
||||||
inScope := false
|
inScope := false
|
||||||
for _, endpoint := range c.Endpoints {
|
for _, endpoint := range c.Endpoints {
|
||||||
inScope = inScope || endpoint == host
|
inScope = inScope || endpoint == host || strings.HasSuffix(host, "."+endpoint)
|
||||||
}
|
}
|
||||||
if !inScope {
|
if !inScope {
|
||||||
return nil, fmt.Errorf("frontend endpoint outside candidate scope")
|
return nil, fmt.Errorf("frontend endpoint outside candidate scope")
|
||||||
|
|||||||
@@ -268,13 +268,22 @@ func strategyFingerprint(args string) (string, error) {
|
|||||||
|
|
||||||
// Validation is shared with tune's output path; only combined 3/3 evidence is stored.
|
// Validation is shared with tune's output path; only combined 3/3 evidence is stored.
|
||||||
func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, runErr error) error {
|
func saveRun(db *sql.DB, started time.Time, args, output string, report []byte, runErr error) error {
|
||||||
|
return saveFrontendRunForHost(db, started, args, output, report, runErr, "youtube.com")
|
||||||
|
}
|
||||||
|
|
||||||
|
// saveFrontendRunForHost keeps the service-level run target stable while
|
||||||
|
// recording the actual tested frontend endpoint and candidate scope.
|
||||||
|
func saveFrontendRunForHost(db *sql.DB, started time.Time, args, output string, report []byte, runErr error, host string) error {
|
||||||
|
if !validDomain(host) || host != strings.ToLower(host) {
|
||||||
|
return fmt.Errorf("invalid frontend evidence endpoint")
|
||||||
|
}
|
||||||
var evidence struct {
|
var evidence struct {
|
||||||
Strategies []json.RawMessage `json:"strategies"`
|
Strategies []json.RawMessage `json:"strategies"`
|
||||||
}
|
}
|
||||||
state := "FAILED"
|
state := "FAILED"
|
||||||
var savedArgs, savedOutput, savedError, fingerprint interface{}
|
var savedArgs, savedOutput, savedError, fingerprint interface{}
|
||||||
if runErr == nil {
|
if runErr == nil {
|
||||||
verifiedOutput, err := combinedBlockcheckOutput(bytes.NewReader(report), args)
|
verifiedOutput, err := combinedBlockcheckOutputForHost(bytes.NewReader(report), args, host)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -309,7 +318,7 @@ func saveRun(db *sql.DB, started time.Time, args, output string, report []byte,
|
|||||||
var candidateID int64
|
var candidateID int64
|
||||||
checked := time.Now().UTC()
|
checked := time.Now().UTC()
|
||||||
if runErr == nil {
|
if runErr == nil {
|
||||||
candidateID, err = insertStrategyCandidate(tx, "frontend", []string{"youtube.com"}, args, id)
|
candidateID, err = insertStrategyCandidate(tx, "frontend", []string{host}, args, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -325,7 +334,7 @@ func saveRun(db *sql.DB, started time.Time, args, output string, report []byte,
|
|||||||
if _, err := tx.Exec(`INSERT INTO protocol_results (run_id, protocol, passes_ok, passes_total) VALUES (?, ?, ?, ?)`, id, normalizeProtocol(row.Protocol), row.PassesOK, row.PassesTotal); err != nil {
|
if _, err := tx.Exec(`INSERT INTO protocol_results (run_id, protocol, passes_ok, passes_total) VALUES (?, ?, ?, ?)`, id, normalizeProtocol(row.Protocol), row.PassesOK, row.PassesTotal); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := saveCandidateEvidence(tx, candidateID, "youtube.com", normalizeProtocol(row.Protocol), raw, checked); err != nil {
|
if err := saveCandidateEvidence(tx, candidateID, host, normalizeProtocol(row.Protocol), raw, checked); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -96,13 +96,17 @@ func selectBlockcheckReport(r io.Reader) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func combinedBlockcheckOutput(r io.Reader, args string) (string, error) {
|
func combinedBlockcheckOutput(r io.Reader, args string) (string, error) {
|
||||||
|
return combinedBlockcheckOutputForHost(r, args, "youtube.com")
|
||||||
|
}
|
||||||
|
|
||||||
|
func combinedBlockcheckOutputForHost(r io.Reader, args, host string) (string, error) {
|
||||||
if args == "" {
|
if args == "" {
|
||||||
return "", fmt.Errorf("empty combined args")
|
return "", fmt.Errorf("empty combined args")
|
||||||
}
|
}
|
||||||
if _, err := selectBlockcheckArgs(r, args); err != nil {
|
if _, err := selectBlockcheckArgsForTarget(r, args, "frontend", host); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=youtube.com " + args + "'", nil
|
return "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=" + host + " " + args + "'", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func selectBlockcheckArgs(r io.Reader, expected string) (string, error) {
|
func selectBlockcheckArgs(r io.Reader, expected string) (string, error) {
|
||||||
|
|||||||
Reference in new issue
Block a user