feat(monitor): add crash-safe supervisor loop

Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
ZeusinaandGPT-6 Bot committed 2026-10-10 09:16:58 +03:00
1 parent 54a0fb79f8
commit 468532dc8f
12 files changed
+971 -44

No files matched your search

+6 -4
View File
@@ -4,18 +4,20 @@
## Готово ## Готово
- Добавлен internal `runManagedMonitor(ctx, deps)`: immediate/periodic probe, durable streak/reset, threshold/cooldown recovery через injected callbacks, cooldown от completion при любом результате, fatal DB/lock/finalization errors и cancellable wait. Shared runtime guards выполняются до lifetime nonblocking `flock` в target-owned directory. Одноразовая startup reconciliation под singleton использует schema-v5 PID/starttime API: exact lease снимается только при доказанном dead/PID-reused owner и запускает crash cooldown; live owner остаётся busy, legacy/unverifiable owner — fail closed. Fake-time unit-тесты scheduler core, reset/cooldown, reopen, concurrent start, cancellation и DB/finalization failures добавлены; tests/build/network не запускались в этом increment — verification отложена orchestrator. Actual recovery adapter/monitor command/entrypoint wiring и развёрнутый live target пока отсутствуют; deploy, контейнер и Xray не затронуты.
- Добавлены `Dockerfile.target`, ordered target entrypoint и `deploy/managed-target/compose.yaml`: pinned runtime digest, отдельная bridge-сеть, `cap_drop: ALL` и только `NET_ADMIN`, `NET_RAW`, `SETUID`, `SETGID` (nft/NFQUEUE, raw send и смена UID/GID nfqws2), новый private bind `./target-state`, отдельный gitignored `.env`, SOCKS TCP/UDP только на host loopback (default 11080), внутренний health proxy 1080; SS port не опубликован. Startup: ss-server/ss-local → listener → managed init (status при recreation) → zapret2 start; shutdown ограничен target-local процессами/firewall. Только deployment definition и process supervision, не autonomous monitor/recovery. Compose config/tests/build/pull/run/remote producer не выполнял; проверка отложена orchestrator, runtime gates открыты. - Добавлены `Dockerfile.target`, ordered target entrypoint и `deploy/managed-target/compose.yaml`: pinned runtime digest, отдельная bridge-сеть, `cap_drop: ALL` и только `NET_ADMIN`, `NET_RAW`, `SETUID`, `SETGID` (nft/NFQUEUE, raw send и смена UID/GID nfqws2), новый private bind `./target-state`, отдельный gitignored `.env`, SOCKS TCP/UDP только на host loopback (default 11080), внутренний health proxy 1080; SS port не опубликован. Startup: ss-server/ss-local → listener → managed init (status при recreation) → zapret2 start; shutdown ограничен target-local процессами/firewall. Только deployment definition и process supervision, не autonomous monitor/recovery. Compose config/tests/build/pull/run/remote producer не выполнял; проверка отложена orchestrator, runtime gates открыты.
- Усилен guarded manual CLI `managed init|status|apply`: exact role `managed-test`, fixed ID `zapret-autotune-managed-v1`, provisioned runtime marker/base executables и private dedicated mountpoint `/var/lib/zapret-autotune/target` обязательны для всех команд. `ZAPRET_CONFIG` допускает только `/var/lib/zapret-autotune/target/config`; standalone full config разрешён только в новом target-owned volume, не поверх существующего service config (include fragments не реализованы). Proxy обязан совпадать с `socks5://127.0.0.1:$SOCKS_PORT`; проверяется ownership listening socket процессом `ss-local`. Старый контейнер/произвольный каталог/loopback proxy не проходят guards. Добавлены injected unit-тесты отказов и fresh-target success; transaction engine и shell config safety не изменены. Tests/build/network этого increment отложены producer; контейнер, deploy, remote и Xray не затронуты, image/config-loading compatibility ещё не доказана. - Усилен guarded manual CLI `managed init|status|apply`: exact role `managed-test`, fixed ID `zapret-autotune-managed-v1`, provisioned runtime marker/base executables и private dedicated mountpoint `/var/lib/zapret-autotune/target` обязательны для всех команд. `ZAPRET_CONFIG` допускает только `/var/lib/zapret-autotune/target/config`; standalone full config разрешён только в новом target-owned volume, не поверх существующего service config (include fragments не реализованы). Proxy обязан совпадать с `socks5://127.0.0.1:$SOCKS_PORT`; проверяется ownership listening socket процессом `ss-local`. Старый контейнер/произвольный каталог/loopback proxy не проходят guards. Добавлены injected unit-тесты отказов и fresh-target success; transaction engine и shell config safety не изменены. Tests/build/network этого increment отложены producer; контейнер, deploy, remote и Xray не затронуты, image/config-loading compatibility ещё не доказана.
- Добавлена foundation schema v4: durable policy state по stable service ID, last probe/streak, recovery lock и cooldown. Операции с injected time атомарно записывают probe и захватывают recovery только при threshold/истёкшем cooldown/свободном lock; завершение требует точный ownership token и запускает cooldown от completion time. Stale/duplicate finish не меняет новую lease; crash сохраняет lock fail-closed (reconciliation пока не реализован). Unit-тесты streak/reset, threshold, competing acquisition, reload/cooldown и v3 migration прошли. - Добавлена foundation schema v4 и owner identity schema v5: durable policy state по stable service ID, last probe/streak, recovery lock и cooldown. Операции с injected time атомарно записывают probe и захватывают recovery только при threshold/истёкшем cooldown/свободном lock; завершение требует точный ownership token и запускает cooldown от completion time. Stale/duplicate finish не меняет новую lease; crash сохраняет lease до PID/starttime-safe reconciliation под exclusive singleton, с новым crash cooldown. Проверка текущей v5 интеграции отложена orchestrator.
- Добавлен одноразовый `health youtube`: bounded Player API-запрос только через loopback SOCKS (`ZAPRET_AUTOTUNE_SOCKS_PROXY`, default `socks5://127.0.0.1:11080`), без ambient proxy/direct fallback. Health требует `playabilityStatus.status=OK`, не streaming URL и не playback. Config валидирует video ID (default `dQw4w9WgXcQ`), interval `5m`, failure threshold `3`, cooldown `30m`. `go test ./...`, `go build` и `git diff --check` прошли для обоих последних increments. - Добавлен одноразовый `health youtube`: bounded Player API-запрос только через loopback SOCKS (`ZAPRET_AUTOTUNE_SOCKS_PROXY`, default `socks5://127.0.0.1:11080`), без ambient proxy/direct fallback. Health требует `playabilityStatus.status=OK`, не streaming URL и не playback. Config валидирует video ID (default `dQw4w9WgXcQ`), interval `5m`, failure threshold `3`, cooldown `30m`. `go test ./...`, `go build` и `git diff --check` прошли для обоих последних increments.
- Добавлена тестируемая transaction foundation для target-owned config: только квалифицированные stored candidates, separate frontend/exact-GGC profiles, versioned durable journal/backup, atomic replace, injected daemon reload + Player API post-check, rollback и crash reconciliation; external config drift fail-closed. Renderer не сохраняет wildcard GGC scope. На момент foundation CLI ещё не вызывал apply; теперь manual adapter есть, target config/runtime ещё не проверены. `go build` и повторный `go test ./...` прошли для foundation; первый test run поймал прежний flaky `TestRunBlockcheckCancellation/kill` process-state race. - Добавлена тестируемая transaction foundation для target-owned config: только квалифицированные stored candidates, separate frontend/exact-GGC profiles, versioned durable journal/backup, atomic replace, injected daemon reload + Player API post-check, rollback и crash reconciliation; external config drift fail-closed. Renderer не сохраняет wildcard GGC scope. На момент foundation CLI ещё не вызывал apply; теперь manual adapter есть, target config/runtime ещё не проверены. `go build` и повторный `go test ./...` прошли для foundation; первый test run поймал прежний flaky `TestRunBlockcheckCancellation/kill` process-state race.
- Завершён удалённый ручной GGC scan для `rr16---sn-n8v7znsz.googlevideo.com`: TLS 1.2 — 1 525 рабочих стратегий, TLS 1.3 — 800. Shortlist check нашёл 20 проходящих вариантов; точная объединённая стратегия `--lua-init=fake_default_tls=tls_mod(fake_default_tls,'rnd') --payload=tls_client_hello --lua-desync=multidisorder:pos=2,midsld:seqovl=1:seqovl_pattern=fake_default_tls --in-range=-s1 --lua-desync=oob:urp=midsld` прошла по 3/3 для TLS 1.2 и TLS 1.3. Результат относится только к указанному hostname; не означает playback/SABR и не применялся. - Завершён удалённый ручной GGC scan для `rr16---sn-n8v7znsz.googlevideo.com`: TLS 1.2 — 1 525 рабочих стратегий, TLS 1.3 — 800. Shortlist check нашёл 20 проходящих вариантов; точная объединённая стратегия `--lua-init=fake_default_tls=tls_mod(fake_default_tls,'rnd') --payload=tls_client_hello --lua-desync=multidisorder:pos=2,midsld:seqovl=1:seqovl_pattern=fake_default_tls --in-range=-s1 --lua-desync=oob:urp=midsld` прошла по 3/3 для TLS 1.2 и TLS 1.3. Результат относится только к указанному hostname; не означает playback/SABR и не применялся.
- Добавлена proposal-only команда `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`: discovery одного свежего exact host → qualified schema-v3 сценарии newest-first → combined TLS1.2/1.3 3/3 точных сохранённых args. Первый успешный сценарий сохраняется новым immutable candidate/run и выдаётся как NOT ACTIVE без scan. Только отсутствие eligible candidates или валидные non-pass результаты разрешают обычный scan на уже найденном host; infrastructure/malformed-report ошибки fail closed. Apply/monitor и удалённый scan не затронуты. - Добавлена proposal-only команда `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`: discovery одного свежего exact host → qualified schema-v3 сценарии newest-first → combined TLS1.2/1.3 3/3 точных сохранённых args. Первый успешный сценарий сохраняется новым immutable candidate/run и выдаётся как NOT ACTIVE без scan. Только отсутствие eligible candidates или валидные non-pass результаты разрешают обычный scan на уже найденном host; infrastructure/malformed-report ошибки fail closed. Apply/monitor и удалённый scan не затронуты.
- Unit-тесты recovery порядка, fresh-host scope, short-circuit, fallback и fail-closed ошибок прошли; локальные `go test ./...`, `go build` и `git diff --check` успешны. - Unit-тесты recovery порядка, fresh-host scope, short-circuit, fallback и fail-closed ошибок прошли; локальные `go test ./...`, `go build` и `git diff --check` успешны.
- CLI на Go: `tune youtube`, `tune googlevideo --video-id ID --proxy socks5://HOST:PORT`, `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`, `discover youtube`, `health youtube`, `status`, `history`, `strategies youtube`, `render`, `managed init|status|apply`. `check youtube` пока намеренно отказывает; manual managed apply только внутри designated target, rollback автоматический; мониторинг не реализован. - CLI на Go: `tune youtube`, `tune googlevideo --video-id ID --proxy socks5://HOST:PORT`, `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`, `discover youtube`, `health youtube`, `status`, `history`, `strategies youtube`, `render`, `managed init|status|apply`. `check youtube` пока намеренно отказывает; manual managed apply только внутри designated target, rollback автоматический; monitor scheduling core есть, но command/recovery wiring отсутствует.
- Proposal-only GGC pipeline: Android 21.02.35 discovery через явный proxy → scan TLS1.2/1.3 только одного найденного hostname → data-transfer shortlist (`--take 10 --passes 3`) → проверка точной комбинации 3/3 на обоих TLS → один инертный профиль с exact-host фильтром. Ранжирование по blockcheckw, не Selectel. SQLite v3 атомарно сохраняет googlevideo candidate и оба endpoint×TLS evidence; отдельный additive journal сохраняет failed runs. `history` и `strategies youtube` показывают точный endpoint/group; `status` пока относится только к frontend journal. - Proposal-only GGC pipeline: Android 21.02.35 discovery через явный proxy → scan TLS1.2/1.3 только одного найденного hostname → data-transfer shortlist (`--take 10 --passes 3`) → проверка точной комбинации 3/3 на обоих TLS → один инертный профиль с exact-host фильтром. Ранжирование по blockcheckw, не Selectel. SQLite v3 атомарно сохраняет googlevideo candidate и оба endpoint×TLS evidence; отдельный additive journal сохраняет failed runs. `history` и `strategies youtube` показывают точный endpoint/group; `status` пока относится только к frontend journal.
- Добавлены injectable unit-тесты GGC success/order/scope/filter/persistence, обязательных флагов, discovery failure, неполной/неуспешной combined проверки и fail-closed DB. После исправления exact-host фильтра `go test ./...`, `go build` и `git diff --check` прошли локально. Полный новый `tune googlevideo` ещё не прогонялся удалённо. - Добавлены injectable unit-тесты GGC success/order/scope/filter/persistence, обязательных флагов, discovery failure, неполной/неуспешной combined проверки и fail-closed DB. После исправления exact-host фильтра `go test ./...`, `go build` и `git diff --check` прошли локально. Полный новый `tune googlevideo` ещё не прогонялся удалённо.
- `tune youtube` сканирует frontend `youtube.com`, проверяет shortlist через `blockcheckw` и выдаёт один TLS-профиль только после совместной проверки TLS 1.2 и 1.3 по 3/3. Это предложение конфигурации, оно ничего не применяет. - `tune youtube` сканирует frontend `youtube.com`, проверяет shortlist через `blockcheckw` и выдаёт один TLS-профиль только после совместной проверки TLS 1.2 и 1.3 по 3/3. Это предложение конфигурации, оно ничего не применяет.
@@ -40,7 +42,7 @@
## Не готово ## Не готово
- Проверка реального YouTube video-transfer/SABR. Прямой GET `serverAbrStreamingUrl` возвращает `sabr.malformed_config`; hostname discovery и `blockcheckw` data-transfer checks не равны проверке воспроизведения видео. - Проверка реального YouTube video-transfer/SABR. Прямой GET `serverAbrStreamingUrl` возвращает `sabr.malformed_config`; hostname discovery и `blockcheckw` data-transfer checks не равны проверке воспроизведения видео.
- Применение подключено только к guarded manual CLI внутри designated target: transaction core, adapter и deployment definition есть, но нет proven image/runtime compatibility, выполненного container provisioning, monitor/recovery loop и реально запущенного managed target. Одноразовый Player API health и persisted policy пока не запускают automation. Цель: новый отдельный prod-like zapret2-ss контейнер только в тестовой среде; существующий контейнер не трогать. Стабильный настраиваемый loopback SOCKS для Xray; Xray config вне scope. Перед runtime пройти route/rollback gates. - Применение подключено только к guarded manual CLI внутри designated target: transaction core, adapter и deployment definition есть, но нет proven image/runtime compatibility, выполненного container provisioning, actual recovery callback/monitor wiring и реально запущенного managed target. Injected supervisor scheduling core добавлен, но одноразовый Player API health и policy сами не запускают live automation. Цель: новый отдельный prod-like zapret2-ss контейнер только в тестовой среде; существующий контейнер не трогать. Стабильный настраиваемый loopback SOCKS для Xray; Xray config вне scope. Перед runtime пройти route/rollback gates.
- Полный сервис из требований: YAML-конфигурация сервисов, endpoint-история/покрытие, daemon/периодический мониторинг, API/метрики и multi-service режим. - Полный сервис из требований: YAML-конфигурация сервисов, endpoint-история/покрытие, daemon/периодический мониторинг, API/метрики и multi-service режим.
## Открытые ограничения и следующие шаги ## Открытые ограничения и следующие шаги
@@ -48,6 +50,6 @@
1. Проверить новый `tune googlevideo` и `recover googlevideo` end-to-end на удалённом сервере; отдельно решить, как импортировать или переиспользовать уже полученные scan результаты без повторного перебора. 1. Проверить новый `tune googlevideo` и `recover googlevideo` end-to-end на удалённом сервере; отдельно решить, как импортировать или переиспользовать уже полученные scan результаты без повторного перебора.
2. Воспроизводимое описание отдельного prod-like target добавлено в `deploy/managed-target`; orchestrator должен проверить Compose syntax. Build/run отложены. До запуска проверить baseline-конфиг и NFQUEUE внутри container network namespace; текущие mounts, сеть, порты и конфиг не переиспользовать. 2. Воспроизводимое описание отдельного prod-like target добавлено в `deploy/managed-target`; orchestrator должен проверить Compose syntax. Build/run отложены. До запуска проверить baseline-конфиг и NFQUEUE внутри container network namespace; текущие mounts, сеть, порты и конфиг не переиспользовать.
3. Проверить готовый manual managed CLI на новом изолированном target; подтвердить shell config rendering, config binding, full firewall/daemon restart и pre/post health. 3. Проверить готовый manual managed CLI на новом изолированном target; подтвердить shell config rendering, config binding, full firewall/daemon restart и pre/post health.
4. Подключить persistent monitor policy к supervisor, recovery через сохранённые сценарии перед новым scan и crash reconciliation. 4. Подключить готовый injected monitor supervisor к functional managed recovery (сохранённые сценарии перед новым scan); только после этого добавлять command/entrypoint wiring. Startup singleton reconciliation уже есть; live runtime ещё не проверен.
5. До autonomous mode доказать safety gates из `Architecture.md`: изоляцию target, эквивалентность managed SOCKS пути, версионирование, atomic activation/rollback и crash recovery. 5. До autonomous mode доказать safety gates из `Architecture.md`: изоляцию target, эквивалентность managed SOCKS пути, версионирование, atomic activation/rollback и crash recovery.
6. Отдельно определить реальную video/SABR-проверку и traffic budget; API health не равен playback и не обнаруживает все GGC playback failures. 6. Отдельно определить реальную video/SABR-проверку и traffic budget; API health не равен playback и не обнаруживает все GGC playback failures.
+25 -11
View File
@@ -52,7 +52,8 @@ with a loopback IP or `localhost`, port 1–65535, and no credentials/path/query
Video IDs must be 11 characters; durations and the failure threshold must be Video IDs must be 11 characters; durations and the failure threshold must be
positive. Explicit empty values are invalid. Interval, consecutive-failure positive. Explicit empty values are invalid. Interval, consecutive-failure
threshold and cooldown are validated future recovery policy settings only: threshold and cooldown are validated future recovery policy settings only:
**there is no scheduler, autonomous monitoring or recovery yet**. **An injected autonomous scheduling core exists, but no monitor command, actual
recovery callback or live managed target is wired yet**.
The future managed target is a **brand-new prod-like zapret2-ss container in the The future managed target is a **brand-new prod-like zapret2-ss container in the
test environment only**, not the existing container. Its stable configurable test environment only**, not the existing container. Its stable configurable
@@ -96,7 +97,8 @@ Signed URLs are not printed. Scanner diagnostics are retained in restricted
artifact files rather than relayed to the terminal. No configuration is applied, artifact files rather than relayed to the terminal. No configuration is applied,
no profile is marked active, and no service is restarted. The staged pipeline is no profile is marked active, and no service is restarted. The staged pipeline is
scan/propose → guarded manual managed-test activation → future monitoring; scan/propose → guarded manual managed-test activation → future monitoring;
the deployment definition exists but is not deployed; monitoring is **not implemented**. Manual the deployment definition exists but is not deployed; the monitor scheduling core
is internal and **not wired to a command or actual recovery**. Manual
proposal-only recovery is available below. proposal-only recovery is available below.
No-argument `tune youtube` remains frontend-only and unchanged. No-argument `tune youtube` remains frontend-only and unchanged.
@@ -165,7 +167,7 @@ it must be writable. In Docker, mount that directory to retain history:
docker run --rm -v autotune-data:/var/lib/zapret-autotune zapret-autotune status docker run --rm -v autotune-data:/var/lib/zapret-autotune zapret-autotune status
``` ```
Schema version 4 is initialized transactionally on `tune` and `status`; subsequent Schema version 5 is initialized transactionally on `tune` and `status`; subsequent
opens are idempotent and newer, unsupported schema versions are rejected. opens are idempotent and newer, unsupported schema versions are rejected.
Schema v4 adds a policy-state foundation keyed by stable service ID (not a cache Schema v4 adds a policy-state foundation keyed by stable service ID (not a cache
@@ -176,13 +178,24 @@ configured threshold, an elapsed cooldown and no running recovery. Completion,
successful or failed, requires the exact per-acquisition ownership token returned successful or failed, requires the exact per-acquisition ownership token returned
by acquisition, clears the lock and starts cooldown from completion time. Stale by acquisition, clears the lock and starts cooldown from completion time. Stale
or duplicate completion is rejected without changing the lock, token or cooldown, or duplicate completion is rejected without changing the lock, token or cooldown,
even after a newer acquisition starts. A crash still leaves the durable lock held; even after a newer acquisition starts. State survives reopening the database.
the token does not introduce automatic expiry or crash reconciliation. The internal `runManagedMonitor` supervisor probes immediately and periodically,
State survives reopening the database; a crash during recovery leaves the lock persists observations, and invokes injected recovery only at threshold outside
held, pending future explicit crash reconciliation. Existing history/candidates cooldown. Recovery errors are reported without stopping monitoring; DB, lock and
are unchanged. These operations are not wired to `health youtube` or recovery: finalization errors are fatal. Cooldown starts when recovery returns, including
**no daemon, autonomous probe or container/Xray changes are added**. Manual failure or cancellation. Callbacks must honor cancellation and join their children.
managed activation is separate from this policy foundation. Existing target guards run before a lifetime nonblocking `flock` in the private
target state directory. With that exclusive singleton held, one-shot startup
reconciliation uses schema-v5 PID/starttime identity: only a proven dead or
PID-reused owner permits clearing its exact lease and starting crash cooldown.
A live owner remains busy; legacy or unverifiable owners fail closed. Probe
history is preserved. The lock file is never unlinked.
The scheduler core has fake-time tests for scheduling, reset, cooldown,
cancellation, parallel starts, fatal DB errors, reopen and safe startup
reconciliation. **There is no actual recovery adapter,
monitor CLI/entrypoint wiring or live target yet**; container/Xray are untouched.
Manual managed activation remains separate. Verification of this increment is
deferred to the orchestrator.
Schema v3 adds immutable strategy candidate definitions (group, declared exact Schema v3 adds immutable strategy candidate definitions (group, declared exact
endpoints, args and normalized-args fingerprint) and endpoint/protocol evidence. endpoints, args and normalized-args fingerprint) and endpoint/protocol evidence.
@@ -192,7 +205,8 @@ in the same transaction as the existing run. Qualification requires every declar
endpoint to pass exactly 3/3 for both TLS1.2 and TLS1.3, with full success/coverage. endpoint to pass exactly 3/3 for both TLS1.2 and TLS1.3, with full success/coverage.
Existing v2 history is unchanged and is not backfilled into independent candidates. Existing v2 history is unchanged and is not backfilled into independent candidates.
Candidates have no DB active/installed state; managed activation uses its own Candidates have no DB active/installed state; managed activation uses its own
durable config journal. Autonomous monitoring remains unimplemented. durable config journal. Autonomous scheduling is internal only; actual recovery
and live monitoring remain unwired.
The explicit `tune googlevideo` path adds exact-cache proposals only. The explicit `tune googlevideo` path adds exact-cache proposals only.
Proposed runs store a SHA-256 fingerprint of validated arguments with separator Proposed runs store a SHA-256 fingerprint of validated arguments with separator
whitespace normalized and argument order preserved; repeated proposals retain whitespace normalized and argument order preserved; repeated proposals retain
+1 -1
View File
@@ -99,7 +99,7 @@ func TestPopulatedV2CandidateMigration(t *testing.T) {
} }
} }
var version int var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 { if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d err=%v", version, err) t.Fatalf("version=%d err=%v", version, err)
} }
if before != legacySnapshot(t, db) { if before != legacySnapshot(t, db) {
+1 -1
View File
@@ -117,7 +117,7 @@ func TestPopulatedV1FingerprintMigration(t *testing.T) {
} }
assertRowCounts(t, db, 3, 4) assertRowCounts(t, db, 3, 4)
var version int var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 { if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d err=%v", version, err) t.Fatalf("version=%d err=%v", version, err)
} }
for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} { for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} {
+1 -1
View File
@@ -154,7 +154,7 @@ fi
if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil { if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if evidence != 2 || legacy != 0 || version != 4 { if evidence != 2 || legacy != 0 || version != 5 {
t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version) t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version)
} }
for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} { for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} {
+19 -17
View File
@@ -218,16 +218,10 @@ func managedStatus(m *managedTransaction, out io.Writer) error {
}) })
} }
func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int { func managedRuntimeGuard(d managedDeps) (int, error) {
command, ids, err := managedArguments(args)
if err != nil {
fmt.Fprintln(stderr, "error:", err)
return 2
}
role, _ := d.lookup("ZAPRET_AUTOTUNE_TARGET_ROLE") role, _ := d.lookup("ZAPRET_AUTOTUNE_TARGET_ROLE")
if role != "managed-test" { if role != "managed-test" {
fmt.Fprintln(stderr, "error: ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test") return 2, fmt.Errorf("ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test")
return 2
} }
for key, expected := range map[string]string{ for key, expected := range map[string]string{
"ZAPRET_AUTOTUNE_TARGET_ID": managedTargetID, "ZAPRET_AUTOTUNE_TARGET_ID": managedTargetID,
@@ -235,28 +229,36 @@ func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
"ZAPRET_CONFIG": managedTargetConfig, "ZAPRET_CONFIG": managedTargetConfig,
} { } {
if value, _ := d.lookup(key); value != expected { if value, _ := d.lookup(key); value != expected {
fmt.Fprintln(stderr, "error:", key, "does not match the fresh managed target contract") return 2, fmt.Errorf("%s does not match the fresh managed target contract", key)
return 2
} }
} }
portText, _ := d.lookup("SOCKS_PORT") portText, _ := d.lookup("SOCKS_PORT")
port, portErr := strconv.Atoi(portText) port, portErr := strconv.Atoi(portText)
proxy, _ := d.lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY") proxy, _ := d.lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
if portErr != nil || port < 1 || port > 65535 || strconv.Itoa(port) != portText || proxy != "socks5://127.0.0.1:"+portText { if portErr != nil || port < 1 || port > 65535 || strconv.Itoa(port) != portText || proxy != "socks5://127.0.0.1:"+portText {
fmt.Fprintln(stderr, "error: managed SOCKS proxy must match the target SOCKS_PORT listener") return 2, fmt.Errorf("managed SOCKS proxy must match the target SOCKS_PORT listener")
return 2
} }
if d.verifyRuntime == nil || d.verifySOCKS == nil { if d.verifyRuntime == nil || d.verifySOCKS == nil {
fmt.Fprintln(stderr, "error: managed runtime validators unavailable") return 1, fmt.Errorf("managed runtime validators unavailable")
return 1
} }
if err := d.verifyRuntime(managedTargetDirectory); err != nil { if err := d.verifyRuntime(managedTargetDirectory); err != nil {
fmt.Fprintln(stderr, "error:", err) return 1, err
return 1
} }
if err := d.verifySOCKS(port); err != nil { if err := d.verifySOCKS(port); err != nil {
return 1, err
}
return 0, nil
}
func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
command, ids, err := managedArguments(args)
if err != nil {
fmt.Fprintln(stderr, "error:", err) fmt.Fprintln(stderr, "error:", err)
return 1 return 2
}
if code, err := managedRuntimeGuard(d); err != nil {
fmt.Fprintln(stderr, "error:", err)
return code
} }
dir := d.directory dir := d.directory
if err := managedDirectory(dir); err != nil { if err := managedDirectory(dir); err != nil {
+125 -5
View File
@@ -4,7 +4,10 @@ import (
"crypto/rand" "crypto/rand"
"database/sql" "database/sql"
"encoding/hex" "encoding/hex"
"errors"
"fmt" "fmt"
"os"
"strconv"
"strings" "strings"
"time" "time"
) )
@@ -24,6 +27,59 @@ CREATE TABLE monitor_state (
); );
PRAGMA user_version = 4;` PRAGMA user_version = 4;`
// Legacy v4 leases have no provable owner: preserve them and fail closed.
const monitorOwnerSchema = `
ALTER TABLE monitor_state ADD COLUMN recovery_owner_pid INTEGER
CHECK (recovery_owner_pid IS NULL OR (typeof(recovery_owner_pid) = 'integer' AND recovery_owner_pid > 0 AND recovery_running = 1));
ALTER TABLE monitor_state ADD COLUMN recovery_owner_starttime INTEGER
CHECK ((recovery_owner_pid IS NULL AND recovery_owner_starttime IS NULL) OR
(recovery_owner_pid IS NOT NULL AND typeof(recovery_owner_starttime) = 'integer' AND recovery_owner_starttime > 0));
PRAGMA user_version = 5;`
// Only os.ErrNotExist proves absence; all other errors are indeterminate.
type monitorProcessReader func(pid int) (int64, error)
func readMonitorProcessStarttime(pid int) (int64, error) {
if pid <= 0 {
return 0, fmt.Errorf("invalid recovery owner PID")
}
data, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
if err != nil {
if errors.Is(err, os.ErrNotExist) {
// A missing stat file alone does not prove a missing process (or
// even a mounted procfs). Confirm absence and a readable procfs.
_, dirErr := os.Stat(fmt.Sprintf("/proc/%d", pid))
self, selfErr := os.ReadFile("/proc/self/stat")
if selfErr == nil {
_, selfErr = parseMonitorProcessStat(os.Getpid(), string(self))
}
if !errors.Is(dirErr, os.ErrNotExist) || selfErr != nil {
return 0, fmt.Errorf("cannot establish process absence for PID %d", pid)
}
}
return 0, err
}
return parseMonitorProcessStat(pid, string(data))
}
func parseMonitorProcessStat(pid int, stat string) (int64, error) {
// comm can contain spaces and parentheses. Field 22 is starttime;
// fields following the final ')' begin with field 3 (state).
open, close := strings.IndexByte(stat, '('), strings.LastIndexByte(stat, ')')
if open < 0 || close <= open || strings.TrimSpace(stat[:open]) != strconv.Itoa(pid) {
return 0, fmt.Errorf("invalid process stat identity")
}
fields := strings.Fields(stat[close+1:])
if len(fields) < 20 || len(fields[0]) != 1 || !strings.Contains("RSDZTtXxKWPI", fields[0]) {
return 0, fmt.Errorf("invalid process stat fields")
}
start, err := strconv.ParseInt(fields[19], 10, 64)
if err != nil || start <= 0 {
return 0, fmt.Errorf("invalid process starttime")
}
return start, nil
}
type monitorState struct { type monitorState struct {
ConsecutiveFailures int ConsecutiveFailures int
LastProbeAt time.Time LastProbeAt time.Time
@@ -88,21 +144,35 @@ func readMonitorState(db *sql.DB, serviceID string) (monitorState, error) {
// A nonempty token identifies this acquisition only; an empty token means busy // A nonempty token identifies this acquisition only; an empty token means busy
// or ineligible. The caller must retain it for finalization, including after reopen. // or ineligible. The caller must retain it for finalization, including after reopen.
func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.Time) (string, error) { func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.Time) (string, error) {
return beginMonitorRecoveryWithProcess(db, serviceID, threshold, now, os.Getpid(), readMonitorProcessStarttime)
}
func beginMonitorRecoveryWithProcess(db *sql.DB, serviceID string, threshold int, now time.Time, pid int, read monitorProcessReader) (string, error) {
if err := validateMonitorTime(serviceID, now); err != nil { if err := validateMonitorTime(serviceID, now); err != nil {
return "", err return "", err
} }
if threshold <= 0 { if threshold <= 0 {
return "", fmt.Errorf("failure threshold must be positive") return "", fmt.Errorf("failure threshold must be positive")
} }
if pid <= 0 || read == nil {
return "", fmt.Errorf("invalid recovery process reader/owner")
}
start, err := read(pid)
if err != nil {
return "", fmt.Errorf("read recovery owner: %w", err)
}
if start <= 0 {
return "", fmt.Errorf("invalid recovery owner starttime")
}
var ownership [32]byte var ownership [32]byte
if _, err := rand.Read(ownership[:]); err != nil { if _, err := rand.Read(ownership[:]); err != nil {
return "", err return "", err
} }
token := hex.EncodeToString(ownership[:]) token := hex.EncodeToString(ownership[:])
// Compare-and-set in one SQLite write, including across independent processes. // Compare-and-set in one SQLite write, including across independent processes.
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 1, recovery_token = ? result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 1, recovery_token = ?, recovery_owner_pid = ?, recovery_owner_starttime = ?
WHERE service_id = ? AND consecutive_failures >= ? AND recovery_running = 0 WHERE service_id = ? AND consecutive_failures >= ? AND recovery_running = 0
AND (cooldown_until IS NULL OR cooldown_until <= ?)`, token, serviceID, threshold, now.UnixNano()) AND (cooldown_until IS NULL OR cooldown_until <= ?)`, token, pid, start, serviceID, threshold, now.UnixNano())
if err != nil { if err != nil {
return "", err return "", err
} }
@@ -113,8 +183,7 @@ func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.
return token, nil return token, nil
} }
// Call on either success or failure. A crash leaves the durable lock held; // Call on either success or failure. Only this acquisition's token can finish.
// automatic crash reconciliation is deliberately not part of this foundation.
func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.Time, cooldown time.Duration) error { func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.Time, cooldown time.Duration) error {
if err := validateMonitorTime(serviceID, completed); err != nil { if err := validateMonitorTime(serviceID, completed); err != nil {
return err return err
@@ -129,7 +198,7 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
if err := validateMonitorTime(serviceID, expires); err != nil { if err := validateMonitorTime(serviceID, expires); err != nil {
return err return err
} }
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, cooldown_until = ? result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), serviceID, token) WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), serviceID, token)
if err != nil { if err != nil {
return err return err
@@ -140,3 +209,54 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
} }
return err return err
} }
// Startup policy only: call for each configured service before monitoring.
// A legacy/indeterminate owner is an error, never permission to unlock.
// The conditional write also protects against a finish/reacquire during the read.
func reconcileMonitorRecovery(db *sql.DB, serviceID string, cooldown time.Duration, read monitorProcessReader, clock func() time.Time) (bool, error) {
if read == nil || clock == nil || cooldown <= 0 {
return false, fmt.Errorf("invalid reconciliation dependencies/cooldown")
}
now := clock()
if err := validateMonitorTime(serviceID, now); err != nil {
return false, err
}
expires := now.Add(cooldown)
if err := validateMonitorTime(serviceID, expires); err != nil {
return false, err
}
var token string
var pid, start sql.NullInt64
err := db.QueryRow(`SELECT recovery_token, recovery_owner_pid, recovery_owner_starttime
FROM monitor_state WHERE service_id = ? AND recovery_running = 1`, serviceID).Scan(&token, &pid, &start)
if err == sql.ErrNoRows {
return false, nil
}
if err != nil {
return false, err
}
if !pid.Valid || !start.Valid || pid.Int64 <= 0 || int64(int(pid.Int64)) != pid.Int64 || start.Int64 <= 0 || len(token) != 64 {
return false, fmt.Errorf("unverifiable recovery owner for service %q", serviceID)
}
current, err := read(int(pid.Int64))
if err != nil && !errors.Is(err, os.ErrNotExist) {
return false, fmt.Errorf("read recovery owner: %w", err)
}
if err == nil {
if current <= 0 {
return false, fmt.Errorf("invalid recovery owner starttime")
}
if current == start.Int64 {
return false, nil
}
}
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL,
recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?
AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), serviceID, token, pid.Int64, start.Int64)
if err != nil {
return false, err
}
count, err := result.RowsAffected()
return count == 1, err
}
+157
View File
@@ -0,0 +1,157 @@
package main
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"syscall"
"time"
)
const managedMonitorLockName = ".autotune-monitor-lock"
// No production adapter or entrypoint yet. Callbacks must honor ctx and join
// their children before returning; the supervisor never detaches callbacks.
type managedMonitorDeps struct {
target managedDeps
db *sql.DB
serviceID string
interval time.Duration
threshold int
cooldown time.Duration
probe func(context.Context) error
recover func(context.Context) error
now func() time.Time
wait func(context.Context, time.Duration) error
reportRecoveryError func(error)
}
// A startup reconciliation can be consumed once, never during live recovery.
type managedMonitorSingleton struct {
mu sync.Mutex
file *os.File
reconciled bool
}
func acquireManagedMonitorSingleton(d managedDeps) (*managedMonitorSingleton, error) {
if d.lookup == nil {
return nil, fmt.Errorf("managed runtime lookup unavailable")
}
if _, err := managedRuntimeGuard(d); err != nil {
return nil, err
}
if err := managedDirectory(d.directory); err != nil {
return nil, err
}
path := filepath.Join(d.directory, managedMonitorLockName)
fd, err := syscall.Open(path, syscall.O_CREAT|syscall.O_RDWR|syscall.O_NOFOLLOW|syscall.O_NONBLOCK|syscall.O_CLOEXEC, 0600)
if err != nil {
return nil, err
}
f := os.NewFile(uintptr(fd), "managed monitor singleton")
if err := managedPrivateFile(path); err != nil {
f.Close()
return nil, err
}
if err := syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
f.Close()
return nil, fmt.Errorf("managed monitor singleton: %w", err)
}
return &managedMonitorSingleton{file: f}, nil
}
func (s *managedMonitorSingleton) close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.file == nil {
return nil
}
err := s.file.Close() // Releases flock; never unlink the lock inode.
s.file = nil
return err
}
// Singleton serialization does not prove a recovery owner is dead. Delegate
// that decision to the v5 PID/starttime policy, including its crash cooldown.
func reconcileManagedMonitorRecovery(db *sql.DB, serviceID string, s *managedMonitorSingleton, cooldown time.Duration, read monitorProcessReader, now func() time.Time) error {
if s == nil {
return fmt.Errorf("monitor singleton required")
}
s.mu.Lock()
defer s.mu.Unlock()
if s.file == nil || s.reconciled {
return fmt.Errorf("startup singleton context unavailable")
}
s.reconciled = true
_, err := reconcileMonitorRecovery(db, serviceID, cooldown, read, now)
return err
}
func waitManagedMonitor(ctx context.Context, d time.Duration) error {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
if d.db == nil || d.interval <= 0 || d.threshold <= 0 || d.cooldown <= 0 || d.probe == nil || d.recover == nil || d.now == nil || d.wait == nil || d.reportRecoveryError == nil {
return fmt.Errorf("complete managed monitor dependencies and positive policy required")
}
if err := validateMonitorTime(d.serviceID, d.now()); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
singleton, err := acquireManagedMonitorSingleton(d.target)
if err != nil {
return err
}
defer func() { err = errors.Join(err, singleton.close()) }()
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, singleton, d.cooldown, readMonitorProcessStarttime, d.now); err != nil {
return err
}
for {
if err := ctx.Err(); err != nil {
return err
}
probeErr := d.probe(ctx)
if err := ctx.Err(); err != nil {
return err
}
if err := recordMonitorProbe(d.db, d.serviceID, probeErr == nil, d.now()); err != nil {
return err
}
if probeErr != nil {
token, err := beginMonitorRecovery(d.db, d.serviceID, d.threshold, d.now())
if err != nil {
return err
}
if token != "" {
recoveryErr := d.recover(ctx)
// Finalize even on cancellation, using the non-context DB operation.
if err := finishMonitorRecovery(d.db, d.serviceID, token, d.now(), d.cooldown); err != nil {
return err
}
if recoveryErr != nil {
d.reportRecoveryError(recoveryErr)
}
}
}
if err := ctx.Err(); err != nil {
return err
}
if err := d.wait(ctx, d.interval); err != nil {
return err
}
}
}
+471
View File
@@ -0,0 +1,471 @@
package main
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
)
var errMonitorTestUnhealthy = errors.New("unhealthy")
// Exercise the production process reader under the real singleton. PID reuse
// proves the old owner is gone and clears only that lease; a live owner stays
// busy, while an owner without v5 identity makes startup fail closed.
func TestManagedMonitorStartupOwnerPolicy(t *testing.T) {
for _, owner := range []string{"live", "reused", "legacy"} {
t.Run(owner, func(t *testing.T) {
d := monitorRuntimeFixture(t)
for i := 0; i < d.threshold; i++ {
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
t.Fatal(err)
}
}
token := requireMonitorBegin(t, d.db, d.serviceID, monitorTestTime, true)
if owner == "reused" {
if _, err := d.db.Exec(`UPDATE monitor_state SET recovery_owner_starttime = recovery_owner_starttime + 1 WHERE service_id = ?`, d.serviceID); err != nil {
t.Fatal(err)
}
}
if owner == "legacy" {
if _, err := d.db.Exec(`UPDATE monitor_state SET recovery_owner_pid = NULL, recovery_owner_starttime = NULL WHERE service_id = ?`, d.serviceID); err != nil {
t.Fatal(err)
}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probes := 0
d.probe = func(context.Context) error { probes++; return errMonitorTestUnhealthy }
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
err := runManagedMonitor(ctx, d)
if owner == "legacy" {
if err == nil || errors.Is(err, context.Canceled) || probes != 0 {
t.Fatalf("legacy startup: probes=%d err=%v", probes, err)
}
} else if !errors.Is(err, context.Canceled) || probes != 1 {
t.Fatalf("startup: probes=%d err=%v", probes, err)
}
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.RecoveryRunning != (owner != "reused") {
t.Fatalf("owner=%s state=%+v err=%v", owner, state, err)
}
if owner == "reused" {
if !state.CooldownUntil.Equal(monitorTestTime.Add(d.cooldown)) {
t.Fatalf("crash cooldown=%v", state.CooldownUntil)
}
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err == nil {
t.Fatal("old owner token accepted")
}
} else {
var stored string
if err := d.db.QueryRow(`SELECT recovery_token FROM monitor_state WHERE service_id = ?`, d.serviceID).Scan(&stored); err != nil || stored != token {
t.Fatalf("lease changed: %q %v", stored, err)
}
if !state.CooldownUntil.IsZero() {
t.Fatal("retained owner cooldown changed")
}
}
assertMonitorUnlocked(t, d)
})
}
}
func TestManagedMonitorUnverifiableOwnerUnderSingleton(t *testing.T) {
d := monitorRuntimeFixture(t)
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
t.Fatal(err)
}
token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime)
if err != nil || token == "" {
t.Fatalf("token=%q err=%v", token, err)
}
s, err := acquireManagedMonitorSingleton(d.target)
if err != nil {
t.Fatal(err)
}
defer s.close()
reader := func(int) (int64, error) { return 0, os.ErrPermission }
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, reader, d.now); !errors.Is(err, os.ErrPermission) {
t.Fatalf("unverifiable owner: %v", err)
}
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || !state.RecoveryRunning || !state.CooldownUntil.IsZero() {
t.Fatalf("state=%+v err=%v", state, err)
}
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err != nil {
t.Fatal(err)
}
if err := s.close(); err != nil {
t.Fatal(err)
}
assertMonitorUnlocked(t, d)
}
func monitorRuntimeFixture(t *testing.T) managedMonitorDeps {
t.Helper()
target, _, _ := managedCLIFixture(t)
if err := os.Chmod(target.directory, 0700); err != nil {
t.Fatal(err)
}
db, err := openStore(filepath.Join(t.TempDir(), "monitor.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
return managedMonitorDeps{
target: target, db: db, serviceID: "managed-youtube",
interval: 5 * time.Minute, threshold: 3, cooldown: 30 * time.Minute,
now: func() time.Time { return monitorTestTime },
probe: func(context.Context) error { return nil },
recover: func(context.Context) error { t.Fatal("unexpected recovery"); return nil },
wait: waitManagedMonitor,
reportRecoveryError: func(error) { t.Fatal("unexpected recovery error") },
}
}
func assertMonitorUnlocked(t *testing.T, d managedMonitorDeps) {
t.Helper()
s, err := acquireManagedMonitorSingleton(d.target)
if err != nil {
t.Fatalf("singleton leaked: %v", err)
}
if err := s.close(); err != nil {
t.Fatal(err)
}
}
func TestManagedMonitorImmediatePeriodicAndReset(t *testing.T) {
d := monitorRuntimeFixture(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
now := monitorTestTime
d.now = func() time.Time { return now }
observations := []bool{false, false, true, false, false, false}
wantStreak := []int{1, 2, 0, 1, 2, 3}
probes, waits, recoveries := 0, 0, 0
d.probe = func(context.Context) error {
if probes != waits || !now.Equal(monitorTestTime.Add(time.Duration(probes)*d.interval)) {
t.Fatal("probe not immediate/periodic")
}
healthy := observations[probes]
probes++
if healthy {
return nil
}
return errMonitorTestUnhealthy
}
d.recover = func(context.Context) error {
recoveries++
if probes != 6 {
t.Fatalf("recovery at probe %d", probes)
}
return nil
}
d.wait = func(ctx context.Context, interval time.Duration) error {
if interval != d.interval {
t.Fatal("wrong interval")
}
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.ConsecutiveFailures != wantStreak[probes-1] || !state.LastProbeAt.Equal(now) {
t.Fatalf("state=%+v err=%v", state, err)
}
waits++
if probes == len(observations) {
cancel()
return ctx.Err()
}
now = now.Add(interval)
return nil
}
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if probes != 6 || recoveries != 1 {
t.Fatalf("probes=%d recoveries=%d", probes, recoveries)
}
assertMonitorUnlocked(t, d)
}
func TestManagedMonitorCooldownFromCompletionEitherOutcome(t *testing.T) {
for _, failed := range []bool{false, true} {
t.Run(map[bool]string{false: "success", true: "failure"}[failed], func(t *testing.T) {
d := monitorRuntimeFixture(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
now := monitorTestTime
d.now = func() time.Time { return now }
probes, recoveries, reports := 0, 0, 0
d.probe = func(context.Context) error { probes++; return errMonitorTestUnhealthy }
d.recover = func(context.Context) error {
recoveries++
want := monitorTestTime.Add(10 * time.Minute)
if recoveries == 2 {
want = monitorTestTime.Add(50 * time.Minute)
}
if recoveries > 2 || !now.Equal(want) {
t.Fatalf("recovery %d at %v", recoveries, now)
}
if token := requireMonitorBegin(t, d.db, d.serviceID, now, false); token != "" {
t.Fatal("double start")
}
now = now.Add(10 * time.Minute)
if failed {
return errMonitorTestUnhealthy
}
return nil
}
d.reportRecoveryError = func(err error) {
if !errors.Is(err, errMonitorTestUnhealthy) {
t.Fatal(err)
}
reports++
}
d.wait = func(ctx context.Context, interval time.Duration) error {
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.RecoveryRunning {
t.Fatalf("state=%+v err=%v", state, err)
}
if recoveries > 0 {
want := monitorTestTime.Add(50 * time.Minute)
if recoveries == 2 {
want = monitorTestTime.Add(90 * time.Minute)
}
if !state.CooldownUntil.Equal(want) {
t.Fatalf("cooldown=%v want=%v", state.CooldownUntil, want)
}
}
if probes == 9 {
cancel()
return ctx.Err()
}
now = now.Add(interval)
return nil
}
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if recoveries != 2 || (failed && reports != 2) || (!failed && reports != 0) {
t.Fatalf("recoveries=%d reports=%d", recoveries, reports)
}
assertMonitorUnlocked(t, d)
})
}
}
func TestManagedMonitorStartupLeaseAfterReopenStartsCrashCooldown(t *testing.T) {
d := monitorRuntimeFixture(t)
path := filepath.Join(t.TempDir(), "monitor.db")
db, err := openStore(path)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 3; i++ {
if err := recordMonitorProbe(db, d.serviceID, false, monitorTestTime); err != nil {
t.Fatal(err)
}
}
// An impossible Linux PID proves a dead owner without spawning a child.
oldToken, err := beginMonitorRecoveryWithProcess(db, d.serviceID, 3, monitorTestTime, 1<<30, func(int) (int64, error) { return 100, nil })
if err != nil || oldToken == "" {
t.Fatalf("token=%q err=%v", oldToken, err)
}
expires := monitorTestTime.Add(d.cooldown)
if _, err := db.Exec(`UPDATE monitor_state SET cooldown_until = ? WHERE service_id = ?`, monitorTestTime.Add(time.Hour).UnixNano(), d.serviceID); err != nil {
t.Fatal(err)
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
d.db, err = openStore(path)
if err != nil {
t.Fatal(err)
}
defer d.db.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
d.probe = func(context.Context) error {
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(expires) || state.ConsecutiveFailures != 3 {
t.Fatalf("startup=%+v err=%v", state, err)
}
if err := finishMonitorRecovery(d.db, d.serviceID, oldToken, monitorTestTime, d.cooldown); err == nil {
t.Fatal("stale finish accepted")
}
return errMonitorTestUnhealthy
}
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.ConsecutiveFailures != 4 || !state.CooldownUntil.Equal(expires) {
t.Fatalf("state=%+v err=%v", state, err)
}
assertMonitorUnlocked(t, d)
}
func TestManagedMonitorConcurrentStartAndCancellation(t *testing.T) {
for _, phase := range []string{"probe", "recovery", "wait"} {
t.Run(phase, func(t *testing.T) {
d := monitorRuntimeFixture(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
entered := make(chan struct{})
childExited := make(chan struct{})
block := func(ctx context.Context) error {
go func() { <-ctx.Done(); close(childExited) }()
close(entered)
<-childExited // Callback joins its child before the supervisor releases the lock.
return ctx.Err()
}
switch phase {
case "probe":
d.probe = block
case "recovery":
d.threshold = 1
d.probe = func(context.Context) error { return errMonitorTestUnhealthy }
d.recover = block
d.reportRecoveryError = func(error) {}
case "wait":
d.wait = func(ctx context.Context, _ time.Duration) error { return block(ctx) }
}
done := make(chan error, 1)
finished := false
t.Cleanup(func() {
cancel()
if !finished {
select {
case <-done:
case <-time.After(5 * time.Second):
t.Error("supervisor leaked during cleanup")
}
}
})
go func() { done <- runManagedMonitor(ctx, d) }()
select {
case <-entered:
case err := <-done:
finished = true
t.Fatalf("early exit: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("monitor did not start")
}
if err := runManagedMonitor(context.Background(), d); err == nil {
t.Fatal("parallel monitor accepted")
}
cancel()
select {
case err := <-done:
finished = true
if !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("cancellation blocked")
}
select {
case <-childExited:
default:
t.Fatal("callback child leaked")
}
state, err := readMonitorState(d.db, d.serviceID)
if err != nil || state.RecoveryRunning {
t.Fatalf("state=%+v err=%v", state, err)
}
if phase == "recovery" && !state.CooldownUntil.Equal(monitorTestTime.Add(d.cooldown)) {
t.Fatal("cancelled recovery not finalized")
}
assertMonitorUnlocked(t, d)
})
}
}
func TestManagedMonitorReconciliationCapabilityAndGuards(t *testing.T) {
d := monitorRuntimeFixture(t)
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, nil, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
t.Fatal("missing singleton accepted")
}
s, err := acquireManagedMonitorSingleton(d.target)
if err != nil {
t.Fatal(err)
}
defer s.close()
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err != nil {
t.Fatal(err)
}
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
t.Fatal(err)
}
token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime)
if err != nil || token == "" {
t.Fatalf("token=%q err=%v", token, err)
}
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
t.Fatal("live lease cleared by repeated startup")
}
if err := s.close(); err != nil {
t.Fatal(err)
}
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
t.Fatal("closed capability accepted")
}
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err != nil {
t.Fatal(err)
}
d.target.directory = t.TempDir()
d.target.lookup = func(string) (string, bool) { return "", false }
if _, err := acquireManagedMonitorSingleton(d.target); err == nil {
t.Fatal("unguarded singleton accepted")
}
if _, err := os.Lstat(filepath.Join(d.target.directory, managedMonitorLockName)); !os.IsNotExist(err) {
t.Fatalf("guard touched lock: %v", err)
}
}
func TestManagedMonitorFatalStoreAndFinalizationErrors(t *testing.T) {
for _, phase := range []string{"startup", "record", "begin", "finish"} {
t.Run(phase, func(t *testing.T) {
d := monitorRuntimeFixture(t)
d.threshold = 1
if phase == "startup" {
if err := d.db.Close(); err != nil {
t.Fatal(err)
}
}
d.probe = func(context.Context) error {
if phase == "startup" {
t.Fatal("probe ran after startup DB error")
}
if phase == "begin" {
_, err := d.db.Exec(`CREATE TRIGGER reject_monitor_begin BEFORE UPDATE OF recovery_running ON monitor_state WHEN NEW.recovery_running = 1 BEGIN SELECT RAISE(ABORT, 'begin rejected'); END`)
if err != nil {
t.Fatal(err)
}
}
if phase == "record" {
if err := d.db.Close(); err != nil {
t.Fatal(err)
}
}
return errMonitorTestUnhealthy
}
d.recover = func(context.Context) error {
if phase != "finish" {
t.Fatal("recovery ran after fatal DB error")
}
_, err := d.db.Exec(`CREATE TRIGGER reject_monitor_finish BEFORE UPDATE OF recovery_running ON monitor_state WHEN NEW.recovery_running = 0 BEGIN SELECT RAISE(ABORT, 'finish rejected'); END`)
if err != nil {
t.Fatal(err)
}
return nil
}
d.wait = func(context.Context, time.Duration) error { t.Fatal("continued after fatal DB error"); return nil }
if err := runManagedMonitor(context.Background(), d); err == nil {
t.Fatal("DB error ignored")
}
assertMonitorUnlocked(t, d)
})
}
}
+157 -1
View File
@@ -3,7 +3,9 @@ package main
import ( import (
"database/sql" "database/sql"
"fmt" "fmt"
"os"
"path/filepath" "path/filepath"
"strings"
"sync" "sync"
"testing" "testing"
"time" "time"
@@ -297,7 +299,7 @@ func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
} }
} }
var version, count int var version, count int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 { if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d err=%v", version, err) t.Fatalf("version=%d err=%v", version, err)
} }
if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 { if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 {
@@ -308,6 +310,160 @@ func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
} }
} }
func TestMonitorCrashReconciliation(t *testing.T) {
for _, tc := range []struct {
name string
start int64
err error
clear, wantErr bool
}{
{"live", 100, nil, false, false},
{"missing", 0, os.ErrNotExist, true, false},
{"reused", 101, nil, true, false},
{"unreadable", 0, os.ErrPermission, false, true},
{"invalid", 0, nil, false, true},
} {
t.Run(tc.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "crash.db")
db, err := openStore(path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
for i := 0; i < 3; i++ {
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
t.Fatal(err)
}
}
token, err := beginMonitorRecoveryWithProcess(db, "service", 3, monitorTestTime, 42, func(pid int) (int64, error) {
if pid != 42 {
t.Fatalf("pid=%d", pid)
}
return 100, nil
})
if err != nil || token == "" {
t.Fatalf("acquire: %q %v", token, err)
}
var pid, start int64
if err := db.QueryRow(`SELECT recovery_owner_pid, recovery_owner_starttime FROM monitor_state WHERE service_id = 'service'`).Scan(&pid, &start); err != nil || pid != 42 || start != 100 {
t.Fatalf("owner=%d/%d err=%v", pid, start, err)
}
now := monitorTestTime.Add(time.Hour)
clear, err := reconcileMonitorRecovery(db, "service", 30*time.Minute, func(pid int) (int64, error) {
if pid != 42 {
t.Fatalf("pid=%d", pid)
}
return tc.start, tc.err
}, func() time.Time { return now })
if clear != tc.clear || (err != nil) != tc.wantErr {
t.Fatalf("clear=%v err=%v", clear, err)
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
db, err = openStore(path)
if err != nil {
t.Fatal(err)
}
state, err := readMonitorState(db, "service")
if err != nil || state.RecoveryRunning == tc.clear {
t.Fatalf("state=%+v err=%v", state, err)
}
if tc.clear {
expires := now.Add(30 * time.Minute)
if !state.CooldownUntil.Equal(expires) {
t.Fatalf("cooldown=%v", state.CooldownUntil)
}
requireMonitorBegin(t, db, "service", expires.Add(-time.Nanosecond), false)
newToken := requireMonitorBegin(t, db, "service", expires, true)
if err := finishMonitorRecovery(db, "service", token, expires, time.Hour); err == nil {
t.Fatal("stale crash token accepted")
}
if err := finishMonitorRecovery(db, "service", newToken, expires, time.Minute); err != nil {
t.Fatal(err)
}
} else if !state.CooldownUntil.IsZero() {
t.Fatal("retained lease changed cooldown")
}
})
}
}
func TestMonitorV4MigrationPreservesUnverifiableLease(t *testing.T) {
db := testStore(t)
if _, err := db.Exec("DROP TABLE monitor_state;" + monitorSchema); err != nil {
t.Fatal(err)
}
token := strings.Repeat("a", 64)
if _, err := db.Exec(`INSERT INTO monitor_state(service_id, consecutive_failures, last_probe_at, recovery_running, recovery_token, cooldown_until) VALUES ('service', 3, ?, 1, ?, ?)`, monitorTestTime.UnixNano(), token, monitorTestTime.Add(time.Minute).UnixNano()); err != nil {
t.Fatal(err)
}
before, err := readMonitorState(db, "service")
if err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
if err := migrate(db); err != nil {
t.Fatal(err)
}
}
var version int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d err=%v", version, err)
}
clear, err := reconcileMonitorRecovery(db, "service", time.Minute, func(int) (int64, error) {
t.Fatal("legacy owner must not be guessed")
return 0, os.ErrNotExist
}, func() time.Time { return monitorTestTime.Add(time.Hour) })
if clear || err == nil {
t.Fatalf("legacy clear=%v err=%v", clear, err)
}
after, err := readMonitorState(db, "service")
if err != nil || after != before {
t.Fatalf("migration changed state: %+v -> %+v err=%v", before, after, err)
}
var storedToken string
if err := db.QueryRow("SELECT recovery_token FROM monitor_state WHERE service_id = 'service'").Scan(&storedToken); err != nil || storedToken != token {
t.Fatalf("token=%q err=%v", storedToken, err)
}
}
func TestMonitorReconciliationCannotClearNewAcquisition(t *testing.T) {
db := testStore(t)
for i := 0; i < 3; i++ {
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
t.Fatal(err)
}
}
token := requireMonitorBegin(t, db, "service", monitorTestTime, true)
var newToken string
clear, err := reconcileMonitorRecovery(db, "service", time.Hour, func(int) (int64, error) {
if err := finishMonitorRecovery(db, "service", token, monitorTestTime, time.Minute); err != nil {
t.Fatal(err)
}
newToken = requireMonitorBegin(t, db, "service", monitorTestTime.Add(time.Minute), true)
return 0, os.ErrNotExist
}, func() time.Time { return monitorTestTime.Add(time.Minute) })
if err != nil || clear {
t.Fatalf("clear=%v err=%v", clear, err)
}
if err := finishMonitorRecovery(db, "service", newToken, monitorTestTime.Add(time.Minute), time.Minute); err != nil {
t.Fatal(err)
}
}
func TestMonitorProcessStatParser(t *testing.T) {
stat := "42 (name with ) parentheses) S" + strings.Repeat(" 0", 18) + " 123 0"
if start, err := parseMonitorProcessStat(42, stat); err != nil || start != 123 {
t.Fatalf("start=%d err=%v", start, err)
}
for _, bad := range []string{"", "41 (name) S" + strings.Repeat(" 0", 18) + " 123", "42 (name) S", strings.Replace(stat, "123", "invalid", 1), strings.Replace(stat, "123", "0", 1)} {
if _, err := parseMonitorProcessStat(42, bad); err == nil {
t.Fatalf("accepted %q", bad)
}
}
}
func TestMonitorV3MigrationRollback(t *testing.T) { func TestMonitorV3MigrationRollback(t *testing.T) {
db := testStore(t) db := testStore(t)
saveProposal(t, db, "--lua-desync=fake") saveProposal(t, db, "--lua-desync=fake")
+6 -1
View File
@@ -114,7 +114,7 @@ func migrate(db *sql.DB) error {
if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil { if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil {
return err return err
} }
if version > 4 { if version > 5 {
return fmt.Errorf("unsupported database schema version %d", version) return fmt.Errorf("unsupported database schema version %d", version)
} }
if version == 0 { if version == 0 {
@@ -236,6 +236,11 @@ PRAGMA user_version = 1;`)
return err return err
} }
} }
if version < 5 {
if _, err := conn.ExecContext(ctx, monitorOwnerSchema); err != nil {
return err
}
}
_, err = conn.ExecContext(ctx, "COMMIT") _, err = conn.ExecContext(ctx, "COMMIT")
return err return err
} }
+2 -2
View File
@@ -224,7 +224,7 @@ func TestConcurrentFirstOpens(t *testing.T) {
} }
for _, db := range stores { for _, db := range stores {
var version, objects int var version, objects int
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 { if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
t.Fatalf("version=%d err=%v", version, err) t.Fatalf("version=%d err=%v", version, err)
} }
if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE
@@ -377,7 +377,7 @@ func TestSchemaConstraintsAndIncompleteEvidence(t *testing.T) {
func TestUnsupportedSchemaVersion(t *testing.T) { func TestUnsupportedSchemaVersion(t *testing.T) {
db := testStore(t) db := testStore(t)
if _, err := db.Exec("PRAGMA user_version = 5"); err != nil { if _, err := db.Exec("PRAGMA user_version = 6"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") { if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") {