feat(monitor): add crash-safe supervisor loop
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
54a0fb79f8
commit
468532dc8f
12 files changed
+971
-44
No files matched your search
+6
-4
@@ -4,18 +4,20 @@
|
|||||||
|
|
||||||
## Готово
|
## Готово
|
||||||
|
|
||||||
|
- Добавлен internal `runManagedMonitor(ctx, deps)`: immediate/periodic probe, durable streak/reset, threshold/cooldown recovery через injected callbacks, cooldown от completion при любом результате, fatal DB/lock/finalization errors и cancellable wait. Shared runtime guards выполняются до lifetime nonblocking `flock` в target-owned directory. Одноразовая startup reconciliation под singleton использует schema-v5 PID/starttime API: exact lease снимается только при доказанном dead/PID-reused owner и запускает crash cooldown; live owner остаётся busy, legacy/unverifiable owner — fail closed. Fake-time unit-тесты scheduler core, reset/cooldown, reopen, concurrent start, cancellation и DB/finalization failures добавлены; tests/build/network не запускались в этом increment — verification отложена orchestrator. Actual recovery adapter/monitor command/entrypoint wiring и развёрнутый live target пока отсутствуют; deploy, контейнер и Xray не затронуты.
|
||||||
|
|
||||||
- Добавлены `Dockerfile.target`, ordered target entrypoint и `deploy/managed-target/compose.yaml`: pinned runtime digest, отдельная bridge-сеть, `cap_drop: ALL` и только `NET_ADMIN`, `NET_RAW`, `SETUID`, `SETGID` (nft/NFQUEUE, raw send и смена UID/GID nfqws2), новый private bind `./target-state`, отдельный gitignored `.env`, SOCKS TCP/UDP только на host loopback (default 11080), внутренний health proxy 1080; SS port не опубликован. Startup: ss-server/ss-local → listener → managed init (status при recreation) → zapret2 start; shutdown ограничен target-local процессами/firewall. Только deployment definition и process supervision, не autonomous monitor/recovery. Compose config/tests/build/pull/run/remote producer не выполнял; проверка отложена orchestrator, runtime gates открыты.
|
- Добавлены `Dockerfile.target`, ordered target entrypoint и `deploy/managed-target/compose.yaml`: pinned runtime digest, отдельная bridge-сеть, `cap_drop: ALL` и только `NET_ADMIN`, `NET_RAW`, `SETUID`, `SETGID` (nft/NFQUEUE, raw send и смена UID/GID nfqws2), новый private bind `./target-state`, отдельный gitignored `.env`, SOCKS TCP/UDP только на host loopback (default 11080), внутренний health proxy 1080; SS port не опубликован. Startup: ss-server/ss-local → listener → managed init (status при recreation) → zapret2 start; shutdown ограничен target-local процессами/firewall. Только deployment definition и process supervision, не autonomous monitor/recovery. Compose config/tests/build/pull/run/remote producer не выполнял; проверка отложена orchestrator, runtime gates открыты.
|
||||||
|
|
||||||
- Усилен guarded manual CLI `managed init|status|apply`: exact role `managed-test`, fixed ID `zapret-autotune-managed-v1`, provisioned runtime marker/base executables и private dedicated mountpoint `/var/lib/zapret-autotune/target` обязательны для всех команд. `ZAPRET_CONFIG` допускает только `/var/lib/zapret-autotune/target/config`; standalone full config разрешён только в новом target-owned volume, не поверх существующего service config (include fragments не реализованы). Proxy обязан совпадать с `socks5://127.0.0.1:$SOCKS_PORT`; проверяется ownership listening socket процессом `ss-local`. Старый контейнер/произвольный каталог/loopback proxy не проходят guards. Добавлены injected unit-тесты отказов и fresh-target success; transaction engine и shell config safety не изменены. Tests/build/network этого increment отложены producer; контейнер, deploy, remote и Xray не затронуты, image/config-loading compatibility ещё не доказана.
|
- Усилен guarded manual CLI `managed init|status|apply`: exact role `managed-test`, fixed ID `zapret-autotune-managed-v1`, provisioned runtime marker/base executables и private dedicated mountpoint `/var/lib/zapret-autotune/target` обязательны для всех команд. `ZAPRET_CONFIG` допускает только `/var/lib/zapret-autotune/target/config`; standalone full config разрешён только в новом target-owned volume, не поверх существующего service config (include fragments не реализованы). Proxy обязан совпадать с `socks5://127.0.0.1:$SOCKS_PORT`; проверяется ownership listening socket процессом `ss-local`. Старый контейнер/произвольный каталог/loopback proxy не проходят guards. Добавлены injected unit-тесты отказов и fresh-target success; transaction engine и shell config safety не изменены. Tests/build/network этого increment отложены producer; контейнер, deploy, remote и Xray не затронуты, image/config-loading compatibility ещё не доказана.
|
||||||
|
|
||||||
- Добавлена foundation schema v4: durable policy state по stable service ID, last probe/streak, recovery lock и cooldown. Операции с injected time атомарно записывают probe и захватывают recovery только при threshold/истёкшем cooldown/свободном lock; завершение требует точный ownership token и запускает cooldown от completion time. Stale/duplicate finish не меняет новую lease; crash сохраняет lock fail-closed (reconciliation пока не реализован). Unit-тесты streak/reset, threshold, competing acquisition, reload/cooldown и v3 migration прошли.
|
- Добавлена foundation schema v4 и owner identity schema v5: durable policy state по stable service ID, last probe/streak, recovery lock и cooldown. Операции с injected time атомарно записывают probe и захватывают recovery только при threshold/истёкшем cooldown/свободном lock; завершение требует точный ownership token и запускает cooldown от completion time. Stale/duplicate finish не меняет новую lease; crash сохраняет lease до PID/starttime-safe reconciliation под exclusive singleton, с новым crash cooldown. Проверка текущей v5 интеграции отложена orchestrator.
|
||||||
- Добавлен одноразовый `health youtube`: bounded Player API-запрос только через loopback SOCKS (`ZAPRET_AUTOTUNE_SOCKS_PROXY`, default `socks5://127.0.0.1:11080`), без ambient proxy/direct fallback. Health требует `playabilityStatus.status=OK`, не streaming URL и не playback. Config валидирует video ID (default `dQw4w9WgXcQ`), interval `5m`, failure threshold `3`, cooldown `30m`. `go test ./...`, `go build` и `git diff --check` прошли для обоих последних increments.
|
- Добавлен одноразовый `health youtube`: bounded Player API-запрос только через loopback SOCKS (`ZAPRET_AUTOTUNE_SOCKS_PROXY`, default `socks5://127.0.0.1:11080`), без ambient proxy/direct fallback. Health требует `playabilityStatus.status=OK`, не streaming URL и не playback. Config валидирует video ID (default `dQw4w9WgXcQ`), interval `5m`, failure threshold `3`, cooldown `30m`. `go test ./...`, `go build` и `git diff --check` прошли для обоих последних increments.
|
||||||
- Добавлена тестируемая transaction foundation для target-owned config: только квалифицированные stored candidates, separate frontend/exact-GGC profiles, versioned durable journal/backup, atomic replace, injected daemon reload + Player API post-check, rollback и crash reconciliation; external config drift fail-closed. Renderer не сохраняет wildcard GGC scope. На момент foundation CLI ещё не вызывал apply; теперь manual adapter есть, target config/runtime ещё не проверены. `go build` и повторный `go test ./...` прошли для foundation; первый test run поймал прежний flaky `TestRunBlockcheckCancellation/kill` process-state race.
|
- Добавлена тестируемая transaction foundation для target-owned config: только квалифицированные stored candidates, separate frontend/exact-GGC profiles, versioned durable journal/backup, atomic replace, injected daemon reload + Player API post-check, rollback и crash reconciliation; external config drift fail-closed. Renderer не сохраняет wildcard GGC scope. На момент foundation CLI ещё не вызывал apply; теперь manual adapter есть, target config/runtime ещё не проверены. `go build` и повторный `go test ./...` прошли для foundation; первый test run поймал прежний flaky `TestRunBlockcheckCancellation/kill` process-state race.
|
||||||
- Завершён удалённый ручной GGC scan для `rr16---sn-n8v7znsz.googlevideo.com`: TLS 1.2 — 1 525 рабочих стратегий, TLS 1.3 — 800. Shortlist check нашёл 20 проходящих вариантов; точная объединённая стратегия `--lua-init=fake_default_tls=tls_mod(fake_default_tls,'rnd') --payload=tls_client_hello --lua-desync=multidisorder:pos=2,midsld:seqovl=1:seqovl_pattern=fake_default_tls --in-range=-s1 --lua-desync=oob:urp=midsld` прошла по 3/3 для TLS 1.2 и TLS 1.3. Результат относится только к указанному hostname; не означает playback/SABR и не применялся.
|
- Завершён удалённый ручной GGC scan для `rr16---sn-n8v7znsz.googlevideo.com`: TLS 1.2 — 1 525 рабочих стратегий, TLS 1.3 — 800. Shortlist check нашёл 20 проходящих вариантов; точная объединённая стратегия `--lua-init=fake_default_tls=tls_mod(fake_default_tls,'rnd') --payload=tls_client_hello --lua-desync=multidisorder:pos=2,midsld:seqovl=1:seqovl_pattern=fake_default_tls --in-range=-s1 --lua-desync=oob:urp=midsld` прошла по 3/3 для TLS 1.2 и TLS 1.3. Результат относится только к указанному hostname; не означает playback/SABR и не применялся.
|
||||||
- Добавлена proposal-only команда `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`: discovery одного свежего exact host → qualified schema-v3 сценарии newest-first → combined TLS1.2/1.3 3/3 точных сохранённых args. Первый успешный сценарий сохраняется новым immutable candidate/run и выдаётся как NOT ACTIVE без scan. Только отсутствие eligible candidates или валидные non-pass результаты разрешают обычный scan на уже найденном host; infrastructure/malformed-report ошибки fail closed. Apply/monitor и удалённый scan не затронуты.
|
- Добавлена proposal-only команда `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`: discovery одного свежего exact host → qualified schema-v3 сценарии newest-first → combined TLS1.2/1.3 3/3 точных сохранённых args. Первый успешный сценарий сохраняется новым immutable candidate/run и выдаётся как NOT ACTIVE без scan. Только отсутствие eligible candidates или валидные non-pass результаты разрешают обычный scan на уже найденном host; infrastructure/malformed-report ошибки fail closed. Apply/monitor и удалённый scan не затронуты.
|
||||||
- Unit-тесты recovery порядка, fresh-host scope, short-circuit, fallback и fail-closed ошибок прошли; локальные `go test ./...`, `go build` и `git diff --check` успешны.
|
- Unit-тесты recovery порядка, fresh-host scope, short-circuit, fallback и fail-closed ошибок прошли; локальные `go test ./...`, `go build` и `git diff --check` успешны.
|
||||||
|
|
||||||
- CLI на Go: `tune youtube`, `tune googlevideo --video-id ID --proxy socks5://HOST:PORT`, `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`, `discover youtube`, `health youtube`, `status`, `history`, `strategies youtube`, `render`, `managed init|status|apply`. `check youtube` пока намеренно отказывает; manual managed apply только внутри designated target, rollback автоматический; мониторинг не реализован.
|
- CLI на Go: `tune youtube`, `tune googlevideo --video-id ID --proxy socks5://HOST:PORT`, `recover googlevideo --video-id ID --proxy socks5://HOST:PORT`, `discover youtube`, `health youtube`, `status`, `history`, `strategies youtube`, `render`, `managed init|status|apply`. `check youtube` пока намеренно отказывает; manual managed apply только внутри designated target, rollback автоматический; monitor scheduling core есть, но command/recovery wiring отсутствует.
|
||||||
- Proposal-only GGC pipeline: Android 21.02.35 discovery через явный proxy → scan TLS1.2/1.3 только одного найденного hostname → data-transfer shortlist (`--take 10 --passes 3`) → проверка точной комбинации 3/3 на обоих TLS → один инертный профиль с exact-host фильтром. Ранжирование по blockcheckw, не Selectel. SQLite v3 атомарно сохраняет googlevideo candidate и оба endpoint×TLS evidence; отдельный additive journal сохраняет failed runs. `history` и `strategies youtube` показывают точный endpoint/group; `status` пока относится только к frontend journal.
|
- Proposal-only GGC pipeline: Android 21.02.35 discovery через явный proxy → scan TLS1.2/1.3 только одного найденного hostname → data-transfer shortlist (`--take 10 --passes 3`) → проверка точной комбинации 3/3 на обоих TLS → один инертный профиль с exact-host фильтром. Ранжирование по blockcheckw, не Selectel. SQLite v3 атомарно сохраняет googlevideo candidate и оба endpoint×TLS evidence; отдельный additive journal сохраняет failed runs. `history` и `strategies youtube` показывают точный endpoint/group; `status` пока относится только к frontend journal.
|
||||||
- Добавлены injectable unit-тесты GGC success/order/scope/filter/persistence, обязательных флагов, discovery failure, неполной/неуспешной combined проверки и fail-closed DB. После исправления exact-host фильтра `go test ./...`, `go build` и `git diff --check` прошли локально. Полный новый `tune googlevideo` ещё не прогонялся удалённо.
|
- Добавлены injectable unit-тесты GGC success/order/scope/filter/persistence, обязательных флагов, discovery failure, неполной/неуспешной combined проверки и fail-closed DB. После исправления exact-host фильтра `go test ./...`, `go build` и `git diff --check` прошли локально. Полный новый `tune googlevideo` ещё не прогонялся удалённо.
|
||||||
- `tune youtube` сканирует frontend `youtube.com`, проверяет shortlist через `blockcheckw` и выдаёт один TLS-профиль только после совместной проверки TLS 1.2 и 1.3 по 3/3. Это предложение конфигурации, оно ничего не применяет.
|
- `tune youtube` сканирует frontend `youtube.com`, проверяет shortlist через `blockcheckw` и выдаёт один TLS-профиль только после совместной проверки TLS 1.2 и 1.3 по 3/3. Это предложение конфигурации, оно ничего не применяет.
|
||||||
@@ -40,7 +42,7 @@
|
|||||||
## Не готово
|
## Не готово
|
||||||
|
|
||||||
- Проверка реального YouTube video-transfer/SABR. Прямой GET `serverAbrStreamingUrl` возвращает `sabr.malformed_config`; hostname discovery и `blockcheckw` data-transfer checks не равны проверке воспроизведения видео.
|
- Проверка реального YouTube video-transfer/SABR. Прямой GET `serverAbrStreamingUrl` возвращает `sabr.malformed_config`; hostname discovery и `blockcheckw` data-transfer checks не равны проверке воспроизведения видео.
|
||||||
- Применение подключено только к guarded manual CLI внутри designated target: transaction core, adapter и deployment definition есть, но нет proven image/runtime compatibility, выполненного container provisioning, monitor/recovery loop и реально запущенного managed target. Одноразовый Player API health и persisted policy пока не запускают automation. Цель: новый отдельный prod-like zapret2-ss контейнер только в тестовой среде; существующий контейнер не трогать. Стабильный настраиваемый loopback SOCKS для Xray; Xray config вне scope. Перед runtime пройти route/rollback gates.
|
- Применение подключено только к guarded manual CLI внутри designated target: transaction core, adapter и deployment definition есть, но нет proven image/runtime compatibility, выполненного container provisioning, actual recovery callback/monitor wiring и реально запущенного managed target. Injected supervisor scheduling core добавлен, но одноразовый Player API health и policy сами не запускают live automation. Цель: новый отдельный prod-like zapret2-ss контейнер только в тестовой среде; существующий контейнер не трогать. Стабильный настраиваемый loopback SOCKS для Xray; Xray config вне scope. Перед runtime пройти route/rollback gates.
|
||||||
- Полный сервис из требований: YAML-конфигурация сервисов, endpoint-история/покрытие, daemon/периодический мониторинг, API/метрики и multi-service режим.
|
- Полный сервис из требований: YAML-конфигурация сервисов, endpoint-история/покрытие, daemon/периодический мониторинг, API/метрики и multi-service режим.
|
||||||
|
|
||||||
## Открытые ограничения и следующие шаги
|
## Открытые ограничения и следующие шаги
|
||||||
@@ -48,6 +50,6 @@
|
|||||||
1. Проверить новый `tune googlevideo` и `recover googlevideo` end-to-end на удалённом сервере; отдельно решить, как импортировать или переиспользовать уже полученные scan результаты без повторного перебора.
|
1. Проверить новый `tune googlevideo` и `recover googlevideo` end-to-end на удалённом сервере; отдельно решить, как импортировать или переиспользовать уже полученные scan результаты без повторного перебора.
|
||||||
2. Воспроизводимое описание отдельного prod-like target добавлено в `deploy/managed-target`; orchestrator должен проверить Compose syntax. Build/run отложены. До запуска проверить baseline-конфиг и NFQUEUE внутри container network namespace; текущие mounts, сеть, порты и конфиг не переиспользовать.
|
2. Воспроизводимое описание отдельного prod-like target добавлено в `deploy/managed-target`; orchestrator должен проверить Compose syntax. Build/run отложены. До запуска проверить baseline-конфиг и NFQUEUE внутри container network namespace; текущие mounts, сеть, порты и конфиг не переиспользовать.
|
||||||
3. Проверить готовый manual managed CLI на новом изолированном target; подтвердить shell config rendering, config binding, full firewall/daemon restart и pre/post health.
|
3. Проверить готовый manual managed CLI на новом изолированном target; подтвердить shell config rendering, config binding, full firewall/daemon restart и pre/post health.
|
||||||
4. Подключить persistent monitor policy к supervisor, recovery через сохранённые сценарии перед новым scan и crash reconciliation.
|
4. Подключить готовый injected monitor supervisor к functional managed recovery (сохранённые сценарии перед новым scan); только после этого добавлять command/entrypoint wiring. Startup singleton reconciliation уже есть; live runtime ещё не проверен.
|
||||||
5. До autonomous mode доказать safety gates из `Architecture.md`: изоляцию target, эквивалентность managed SOCKS пути, версионирование, atomic activation/rollback и crash recovery.
|
5. До autonomous mode доказать safety gates из `Architecture.md`: изоляцию target, эквивалентность managed SOCKS пути, версионирование, atomic activation/rollback и crash recovery.
|
||||||
6. Отдельно определить реальную video/SABR-проверку и traffic budget; API health не равен playback и не обнаруживает все GGC playback failures.
|
6. Отдельно определить реальную video/SABR-проверку и traffic budget; API health не равен playback и не обнаруживает все GGC playback failures.
|
||||||
@@ -52,7 +52,8 @@ with a loopback IP or `localhost`, port 1–65535, and no credentials/path/query
|
|||||||
Video IDs must be 11 characters; durations and the failure threshold must be
|
Video IDs must be 11 characters; durations and the failure threshold must be
|
||||||
positive. Explicit empty values are invalid. Interval, consecutive-failure
|
positive. Explicit empty values are invalid. Interval, consecutive-failure
|
||||||
threshold and cooldown are validated future recovery policy settings only:
|
threshold and cooldown are validated future recovery policy settings only:
|
||||||
**there is no scheduler, autonomous monitoring or recovery yet**.
|
**An injected autonomous scheduling core exists, but no monitor command, actual
|
||||||
|
recovery callback or live managed target is wired yet**.
|
||||||
|
|
||||||
The future managed target is a **brand-new prod-like zapret2-ss container in the
|
The future managed target is a **brand-new prod-like zapret2-ss container in the
|
||||||
test environment only**, not the existing container. Its stable configurable
|
test environment only**, not the existing container. Its stable configurable
|
||||||
@@ -96,7 +97,8 @@ Signed URLs are not printed. Scanner diagnostics are retained in restricted
|
|||||||
artifact files rather than relayed to the terminal. No configuration is applied,
|
artifact files rather than relayed to the terminal. No configuration is applied,
|
||||||
no profile is marked active, and no service is restarted. The staged pipeline is
|
no profile is marked active, and no service is restarted. The staged pipeline is
|
||||||
scan/propose → guarded manual managed-test activation → future monitoring;
|
scan/propose → guarded manual managed-test activation → future monitoring;
|
||||||
the deployment definition exists but is not deployed; monitoring is **not implemented**. Manual
|
the deployment definition exists but is not deployed; the monitor scheduling core
|
||||||
|
is internal and **not wired to a command or actual recovery**. Manual
|
||||||
proposal-only recovery is available below.
|
proposal-only recovery is available below.
|
||||||
No-argument `tune youtube` remains frontend-only and unchanged.
|
No-argument `tune youtube` remains frontend-only and unchanged.
|
||||||
|
|
||||||
@@ -165,7 +167,7 @@ it must be writable. In Docker, mount that directory to retain history:
|
|||||||
docker run --rm -v autotune-data:/var/lib/zapret-autotune zapret-autotune status
|
docker run --rm -v autotune-data:/var/lib/zapret-autotune zapret-autotune status
|
||||||
```
|
```
|
||||||
|
|
||||||
Schema version 4 is initialized transactionally on `tune` and `status`; subsequent
|
Schema version 5 is initialized transactionally on `tune` and `status`; subsequent
|
||||||
opens are idempotent and newer, unsupported schema versions are rejected.
|
opens are idempotent and newer, unsupported schema versions are rejected.
|
||||||
|
|
||||||
Schema v4 adds a policy-state foundation keyed by stable service ID (not a cache
|
Schema v4 adds a policy-state foundation keyed by stable service ID (not a cache
|
||||||
@@ -176,13 +178,24 @@ configured threshold, an elapsed cooldown and no running recovery. Completion,
|
|||||||
successful or failed, requires the exact per-acquisition ownership token returned
|
successful or failed, requires the exact per-acquisition ownership token returned
|
||||||
by acquisition, clears the lock and starts cooldown from completion time. Stale
|
by acquisition, clears the lock and starts cooldown from completion time. Stale
|
||||||
or duplicate completion is rejected without changing the lock, token or cooldown,
|
or duplicate completion is rejected without changing the lock, token or cooldown,
|
||||||
even after a newer acquisition starts. A crash still leaves the durable lock held;
|
even after a newer acquisition starts. State survives reopening the database.
|
||||||
the token does not introduce automatic expiry or crash reconciliation.
|
The internal `runManagedMonitor` supervisor probes immediately and periodically,
|
||||||
State survives reopening the database; a crash during recovery leaves the lock
|
persists observations, and invokes injected recovery only at threshold outside
|
||||||
held, pending future explicit crash reconciliation. Existing history/candidates
|
cooldown. Recovery errors are reported without stopping monitoring; DB, lock and
|
||||||
are unchanged. These operations are not wired to `health youtube` or recovery:
|
finalization errors are fatal. Cooldown starts when recovery returns, including
|
||||||
**no daemon, autonomous probe or container/Xray changes are added**. Manual
|
failure or cancellation. Callbacks must honor cancellation and join their children.
|
||||||
managed activation is separate from this policy foundation.
|
Existing target guards run before a lifetime nonblocking `flock` in the private
|
||||||
|
target state directory. With that exclusive singleton held, one-shot startup
|
||||||
|
reconciliation uses schema-v5 PID/starttime identity: only a proven dead or
|
||||||
|
PID-reused owner permits clearing its exact lease and starting crash cooldown.
|
||||||
|
A live owner remains busy; legacy or unverifiable owners fail closed. Probe
|
||||||
|
history is preserved. The lock file is never unlinked.
|
||||||
|
The scheduler core has fake-time tests for scheduling, reset, cooldown,
|
||||||
|
cancellation, parallel starts, fatal DB errors, reopen and safe startup
|
||||||
|
reconciliation. **There is no actual recovery adapter,
|
||||||
|
monitor CLI/entrypoint wiring or live target yet**; container/Xray are untouched.
|
||||||
|
Manual managed activation remains separate. Verification of this increment is
|
||||||
|
deferred to the orchestrator.
|
||||||
|
|
||||||
Schema v3 adds immutable strategy candidate definitions (group, declared exact
|
Schema v3 adds immutable strategy candidate definitions (group, declared exact
|
||||||
endpoints, args and normalized-args fingerprint) and endpoint/protocol evidence.
|
endpoints, args and normalized-args fingerprint) and endpoint/protocol evidence.
|
||||||
@@ -192,7 +205,8 @@ in the same transaction as the existing run. Qualification requires every declar
|
|||||||
endpoint to pass exactly 3/3 for both TLS1.2 and TLS1.3, with full success/coverage.
|
endpoint to pass exactly 3/3 for both TLS1.2 and TLS1.3, with full success/coverage.
|
||||||
Existing v2 history is unchanged and is not backfilled into independent candidates.
|
Existing v2 history is unchanged and is not backfilled into independent candidates.
|
||||||
Candidates have no DB active/installed state; managed activation uses its own
|
Candidates have no DB active/installed state; managed activation uses its own
|
||||||
durable config journal. Autonomous monitoring remains unimplemented.
|
durable config journal. Autonomous scheduling is internal only; actual recovery
|
||||||
|
and live monitoring remain unwired.
|
||||||
The explicit `tune googlevideo` path adds exact-cache proposals only.
|
The explicit `tune googlevideo` path adds exact-cache proposals only.
|
||||||
Proposed runs store a SHA-256 fingerprint of validated arguments with separator
|
Proposed runs store a SHA-256 fingerprint of validated arguments with separator
|
||||||
whitespace normalized and argument order preserved; repeated proposals retain
|
whitespace normalized and argument order preserved; repeated proposals retain
|
||||||
|
|||||||
@@ -99,7 +99,7 @@ func TestPopulatedV2CandidateMigration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
var version int
|
var version int
|
||||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
||||||
t.Fatalf("version=%d err=%v", version, err)
|
t.Fatalf("version=%d err=%v", version, err)
|
||||||
}
|
}
|
||||||
if before != legacySnapshot(t, db) {
|
if before != legacySnapshot(t, db) {
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ func TestPopulatedV1FingerprintMigration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
assertRowCounts(t, db, 3, 4)
|
assertRowCounts(t, db, 3, 4)
|
||||||
var version int
|
var version int
|
||||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
||||||
t.Fatalf("version=%d err=%v", version, err)
|
t.Fatalf("version=%d err=%v", version, err)
|
||||||
}
|
}
|
||||||
for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} {
|
for id, input := range map[int]string{7: "--lua-desync=fake", 9: args} {
|
||||||
|
|||||||
@@ -154,7 +154,7 @@ fi
|
|||||||
if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
|
if err := db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if evidence != 2 || legacy != 0 || version != 4 {
|
if evidence != 2 || legacy != 0 || version != 5 {
|
||||||
t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version)
|
t.Fatalf("evidence=%d legacy=%d version=%d", evidence, legacy, version)
|
||||||
}
|
}
|
||||||
for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} {
|
for _, command := range [][]string{{"history"}, {"strategies", "youtube"}} {
|
||||||
|
|||||||
@@ -218,16 +218,10 @@ func managedStatus(m *managedTransaction, out io.Writer) error {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
|
func managedRuntimeGuard(d managedDeps) (int, error) {
|
||||||
command, ids, err := managedArguments(args)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintln(stderr, "error:", err)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
role, _ := d.lookup("ZAPRET_AUTOTUNE_TARGET_ROLE")
|
role, _ := d.lookup("ZAPRET_AUTOTUNE_TARGET_ROLE")
|
||||||
if role != "managed-test" {
|
if role != "managed-test" {
|
||||||
fmt.Fprintln(stderr, "error: ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test")
|
return 2, fmt.Errorf("ZAPRET_AUTOTUNE_TARGET_ROLE must be managed-test")
|
||||||
return 2
|
|
||||||
}
|
}
|
||||||
for key, expected := range map[string]string{
|
for key, expected := range map[string]string{
|
||||||
"ZAPRET_AUTOTUNE_TARGET_ID": managedTargetID,
|
"ZAPRET_AUTOTUNE_TARGET_ID": managedTargetID,
|
||||||
@@ -235,28 +229,36 @@ func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
|
|||||||
"ZAPRET_CONFIG": managedTargetConfig,
|
"ZAPRET_CONFIG": managedTargetConfig,
|
||||||
} {
|
} {
|
||||||
if value, _ := d.lookup(key); value != expected {
|
if value, _ := d.lookup(key); value != expected {
|
||||||
fmt.Fprintln(stderr, "error:", key, "does not match the fresh managed target contract")
|
return 2, fmt.Errorf("%s does not match the fresh managed target contract", key)
|
||||||
return 2
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
portText, _ := d.lookup("SOCKS_PORT")
|
portText, _ := d.lookup("SOCKS_PORT")
|
||||||
port, portErr := strconv.Atoi(portText)
|
port, portErr := strconv.Atoi(portText)
|
||||||
proxy, _ := d.lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
|
proxy, _ := d.lookup("ZAPRET_AUTOTUNE_SOCKS_PROXY")
|
||||||
if portErr != nil || port < 1 || port > 65535 || strconv.Itoa(port) != portText || proxy != "socks5://127.0.0.1:"+portText {
|
if portErr != nil || port < 1 || port > 65535 || strconv.Itoa(port) != portText || proxy != "socks5://127.0.0.1:"+portText {
|
||||||
fmt.Fprintln(stderr, "error: managed SOCKS proxy must match the target SOCKS_PORT listener")
|
return 2, fmt.Errorf("managed SOCKS proxy must match the target SOCKS_PORT listener")
|
||||||
return 2
|
|
||||||
}
|
}
|
||||||
if d.verifyRuntime == nil || d.verifySOCKS == nil {
|
if d.verifyRuntime == nil || d.verifySOCKS == nil {
|
||||||
fmt.Fprintln(stderr, "error: managed runtime validators unavailable")
|
return 1, fmt.Errorf("managed runtime validators unavailable")
|
||||||
return 1
|
|
||||||
}
|
}
|
||||||
if err := d.verifyRuntime(managedTargetDirectory); err != nil {
|
if err := d.verifyRuntime(managedTargetDirectory); err != nil {
|
||||||
fmt.Fprintln(stderr, "error:", err)
|
return 1, err
|
||||||
return 1
|
|
||||||
}
|
}
|
||||||
if err := d.verifySOCKS(port); err != nil {
|
if err := d.verifySOCKS(port); err != nil {
|
||||||
|
return 1, err
|
||||||
|
}
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func managedCLI(args []string, stdout, stderr io.Writer, d managedDeps) int {
|
||||||
|
command, ids, err := managedArguments(args)
|
||||||
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, "error:", err)
|
fmt.Fprintln(stderr, "error:", err)
|
||||||
return 1
|
return 2
|
||||||
|
}
|
||||||
|
if code, err := managedRuntimeGuard(d); err != nil {
|
||||||
|
fmt.Fprintln(stderr, "error:", err)
|
||||||
|
return code
|
||||||
}
|
}
|
||||||
dir := d.directory
|
dir := d.directory
|
||||||
if err := managedDirectory(dir); err != nil {
|
if err := managedDirectory(dir); err != nil {
|
||||||
|
|||||||
@@ -4,7 +4,10 @@ import (
|
|||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -24,6 +27,59 @@ CREATE TABLE monitor_state (
|
|||||||
);
|
);
|
||||||
PRAGMA user_version = 4;`
|
PRAGMA user_version = 4;`
|
||||||
|
|
||||||
|
// Legacy v4 leases have no provable owner: preserve them and fail closed.
|
||||||
|
const monitorOwnerSchema = `
|
||||||
|
ALTER TABLE monitor_state ADD COLUMN recovery_owner_pid INTEGER
|
||||||
|
CHECK (recovery_owner_pid IS NULL OR (typeof(recovery_owner_pid) = 'integer' AND recovery_owner_pid > 0 AND recovery_running = 1));
|
||||||
|
ALTER TABLE monitor_state ADD COLUMN recovery_owner_starttime INTEGER
|
||||||
|
CHECK ((recovery_owner_pid IS NULL AND recovery_owner_starttime IS NULL) OR
|
||||||
|
(recovery_owner_pid IS NOT NULL AND typeof(recovery_owner_starttime) = 'integer' AND recovery_owner_starttime > 0));
|
||||||
|
PRAGMA user_version = 5;`
|
||||||
|
|
||||||
|
// Only os.ErrNotExist proves absence; all other errors are indeterminate.
|
||||||
|
type monitorProcessReader func(pid int) (int64, error)
|
||||||
|
|
||||||
|
func readMonitorProcessStarttime(pid int) (int64, error) {
|
||||||
|
if pid <= 0 {
|
||||||
|
return 0, fmt.Errorf("invalid recovery owner PID")
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
// A missing stat file alone does not prove a missing process (or
|
||||||
|
// even a mounted procfs). Confirm absence and a readable procfs.
|
||||||
|
_, dirErr := os.Stat(fmt.Sprintf("/proc/%d", pid))
|
||||||
|
self, selfErr := os.ReadFile("/proc/self/stat")
|
||||||
|
if selfErr == nil {
|
||||||
|
_, selfErr = parseMonitorProcessStat(os.Getpid(), string(self))
|
||||||
|
}
|
||||||
|
if !errors.Is(dirErr, os.ErrNotExist) || selfErr != nil {
|
||||||
|
return 0, fmt.Errorf("cannot establish process absence for PID %d", pid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return parseMonitorProcessStat(pid, string(data))
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseMonitorProcessStat(pid int, stat string) (int64, error) {
|
||||||
|
// comm can contain spaces and parentheses. Field 22 is starttime;
|
||||||
|
// fields following the final ')' begin with field 3 (state).
|
||||||
|
open, close := strings.IndexByte(stat, '('), strings.LastIndexByte(stat, ')')
|
||||||
|
if open < 0 || close <= open || strings.TrimSpace(stat[:open]) != strconv.Itoa(pid) {
|
||||||
|
return 0, fmt.Errorf("invalid process stat identity")
|
||||||
|
}
|
||||||
|
fields := strings.Fields(stat[close+1:])
|
||||||
|
if len(fields) < 20 || len(fields[0]) != 1 || !strings.Contains("RSDZTtXxKWPI", fields[0]) {
|
||||||
|
return 0, fmt.Errorf("invalid process stat fields")
|
||||||
|
}
|
||||||
|
start, err := strconv.ParseInt(fields[19], 10, 64)
|
||||||
|
if err != nil || start <= 0 {
|
||||||
|
return 0, fmt.Errorf("invalid process starttime")
|
||||||
|
}
|
||||||
|
return start, nil
|
||||||
|
}
|
||||||
|
|
||||||
type monitorState struct {
|
type monitorState struct {
|
||||||
ConsecutiveFailures int
|
ConsecutiveFailures int
|
||||||
LastProbeAt time.Time
|
LastProbeAt time.Time
|
||||||
@@ -88,21 +144,35 @@ func readMonitorState(db *sql.DB, serviceID string) (monitorState, error) {
|
|||||||
// A nonempty token identifies this acquisition only; an empty token means busy
|
// A nonempty token identifies this acquisition only; an empty token means busy
|
||||||
// or ineligible. The caller must retain it for finalization, including after reopen.
|
// or ineligible. The caller must retain it for finalization, including after reopen.
|
||||||
func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.Time) (string, error) {
|
func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.Time) (string, error) {
|
||||||
|
return beginMonitorRecoveryWithProcess(db, serviceID, threshold, now, os.Getpid(), readMonitorProcessStarttime)
|
||||||
|
}
|
||||||
|
|
||||||
|
func beginMonitorRecoveryWithProcess(db *sql.DB, serviceID string, threshold int, now time.Time, pid int, read monitorProcessReader) (string, error) {
|
||||||
if err := validateMonitorTime(serviceID, now); err != nil {
|
if err := validateMonitorTime(serviceID, now); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if threshold <= 0 {
|
if threshold <= 0 {
|
||||||
return "", fmt.Errorf("failure threshold must be positive")
|
return "", fmt.Errorf("failure threshold must be positive")
|
||||||
}
|
}
|
||||||
|
if pid <= 0 || read == nil {
|
||||||
|
return "", fmt.Errorf("invalid recovery process reader/owner")
|
||||||
|
}
|
||||||
|
start, err := read(pid)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("read recovery owner: %w", err)
|
||||||
|
}
|
||||||
|
if start <= 0 {
|
||||||
|
return "", fmt.Errorf("invalid recovery owner starttime")
|
||||||
|
}
|
||||||
var ownership [32]byte
|
var ownership [32]byte
|
||||||
if _, err := rand.Read(ownership[:]); err != nil {
|
if _, err := rand.Read(ownership[:]); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
token := hex.EncodeToString(ownership[:])
|
token := hex.EncodeToString(ownership[:])
|
||||||
// Compare-and-set in one SQLite write, including across independent processes.
|
// Compare-and-set in one SQLite write, including across independent processes.
|
||||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 1, recovery_token = ?
|
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 1, recovery_token = ?, recovery_owner_pid = ?, recovery_owner_starttime = ?
|
||||||
WHERE service_id = ? AND consecutive_failures >= ? AND recovery_running = 0
|
WHERE service_id = ? AND consecutive_failures >= ? AND recovery_running = 0
|
||||||
AND (cooldown_until IS NULL OR cooldown_until <= ?)`, token, serviceID, threshold, now.UnixNano())
|
AND (cooldown_until IS NULL OR cooldown_until <= ?)`, token, pid, start, serviceID, threshold, now.UnixNano())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -113,8 +183,7 @@ func beginMonitorRecovery(db *sql.DB, serviceID string, threshold int, now time.
|
|||||||
return token, nil
|
return token, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Call on either success or failure. A crash leaves the durable lock held;
|
// Call on either success or failure. Only this acquisition's token can finish.
|
||||||
// automatic crash reconciliation is deliberately not part of this foundation.
|
|
||||||
func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.Time, cooldown time.Duration) error {
|
func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.Time, cooldown time.Duration) error {
|
||||||
if err := validateMonitorTime(serviceID, completed); err != nil {
|
if err := validateMonitorTime(serviceID, completed); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -129,7 +198,7 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
|
|||||||
if err := validateMonitorTime(serviceID, expires); err != nil {
|
if err := validateMonitorTime(serviceID, expires); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, cooldown_until = ?
|
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
|
||||||
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), serviceID, token)
|
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), serviceID, token)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -140,3 +209,54 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
|
|||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Startup policy only: call for each configured service before monitoring.
|
||||||
|
// A legacy/indeterminate owner is an error, never permission to unlock.
|
||||||
|
// The conditional write also protects against a finish/reacquire during the read.
|
||||||
|
func reconcileMonitorRecovery(db *sql.DB, serviceID string, cooldown time.Duration, read monitorProcessReader, clock func() time.Time) (bool, error) {
|
||||||
|
if read == nil || clock == nil || cooldown <= 0 {
|
||||||
|
return false, fmt.Errorf("invalid reconciliation dependencies/cooldown")
|
||||||
|
}
|
||||||
|
now := clock()
|
||||||
|
if err := validateMonitorTime(serviceID, now); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
expires := now.Add(cooldown)
|
||||||
|
if err := validateMonitorTime(serviceID, expires); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
var token string
|
||||||
|
var pid, start sql.NullInt64
|
||||||
|
err := db.QueryRow(`SELECT recovery_token, recovery_owner_pid, recovery_owner_starttime
|
||||||
|
FROM monitor_state WHERE service_id = ? AND recovery_running = 1`, serviceID).Scan(&token, &pid, &start)
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if !pid.Valid || !start.Valid || pid.Int64 <= 0 || int64(int(pid.Int64)) != pid.Int64 || start.Int64 <= 0 || len(token) != 64 {
|
||||||
|
return false, fmt.Errorf("unverifiable recovery owner for service %q", serviceID)
|
||||||
|
}
|
||||||
|
current, err := read(int(pid.Int64))
|
||||||
|
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return false, fmt.Errorf("read recovery owner: %w", err)
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
if current <= 0 {
|
||||||
|
return false, fmt.Errorf("invalid recovery owner starttime")
|
||||||
|
}
|
||||||
|
if current == start.Int64 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL,
|
||||||
|
recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
|
||||||
|
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?
|
||||||
|
AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), serviceID, token, pid.Int64, start.Int64)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
count, err := result.RowsAffected()
|
||||||
|
return count == 1, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const managedMonitorLockName = ".autotune-monitor-lock"
|
||||||
|
|
||||||
|
// No production adapter or entrypoint yet. Callbacks must honor ctx and join
|
||||||
|
// their children before returning; the supervisor never detaches callbacks.
|
||||||
|
type managedMonitorDeps struct {
|
||||||
|
target managedDeps
|
||||||
|
db *sql.DB
|
||||||
|
serviceID string
|
||||||
|
interval time.Duration
|
||||||
|
threshold int
|
||||||
|
cooldown time.Duration
|
||||||
|
probe func(context.Context) error
|
||||||
|
recover func(context.Context) error
|
||||||
|
now func() time.Time
|
||||||
|
wait func(context.Context, time.Duration) error
|
||||||
|
reportRecoveryError func(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A startup reconciliation can be consumed once, never during live recovery.
|
||||||
|
type managedMonitorSingleton struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
file *os.File
|
||||||
|
reconciled bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func acquireManagedMonitorSingleton(d managedDeps) (*managedMonitorSingleton, error) {
|
||||||
|
if d.lookup == nil {
|
||||||
|
return nil, fmt.Errorf("managed runtime lookup unavailable")
|
||||||
|
}
|
||||||
|
if _, err := managedRuntimeGuard(d); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := managedDirectory(d.directory); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
path := filepath.Join(d.directory, managedMonitorLockName)
|
||||||
|
fd, err := syscall.Open(path, syscall.O_CREAT|syscall.O_RDWR|syscall.O_NOFOLLOW|syscall.O_NONBLOCK|syscall.O_CLOEXEC, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
f := os.NewFile(uintptr(fd), "managed monitor singleton")
|
||||||
|
if err := managedPrivateFile(path); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return nil, fmt.Errorf("managed monitor singleton: %w", err)
|
||||||
|
}
|
||||||
|
return &managedMonitorSingleton{file: f}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *managedMonitorSingleton) close() error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.file == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
err := s.file.Close() // Releases flock; never unlink the lock inode.
|
||||||
|
s.file = nil
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Singleton serialization does not prove a recovery owner is dead. Delegate
|
||||||
|
// that decision to the v5 PID/starttime policy, including its crash cooldown.
|
||||||
|
func reconcileManagedMonitorRecovery(db *sql.DB, serviceID string, s *managedMonitorSingleton, cooldown time.Duration, read monitorProcessReader, now func() time.Time) error {
|
||||||
|
if s == nil {
|
||||||
|
return fmt.Errorf("monitor singleton required")
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.file == nil || s.reconciled {
|
||||||
|
return fmt.Errorf("startup singleton context unavailable")
|
||||||
|
}
|
||||||
|
s.reconciled = true
|
||||||
|
_, err := reconcileMonitorRecovery(db, serviceID, cooldown, read, now)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitManagedMonitor(ctx context.Context, d time.Duration) error {
|
||||||
|
timer := time.NewTimer(d)
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-timer.C:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
||||||
|
if d.db == nil || d.interval <= 0 || d.threshold <= 0 || d.cooldown <= 0 || d.probe == nil || d.recover == nil || d.now == nil || d.wait == nil || d.reportRecoveryError == nil {
|
||||||
|
return fmt.Errorf("complete managed monitor dependencies and positive policy required")
|
||||||
|
}
|
||||||
|
if err := validateMonitorTime(d.serviceID, d.now()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
singleton, err := acquireManagedMonitorSingleton(d.target)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { err = errors.Join(err, singleton.close()) }()
|
||||||
|
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, singleton, d.cooldown, readMonitorProcessStarttime, d.now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
probeErr := d.probe(ctx)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := recordMonitorProbe(d.db, d.serviceID, probeErr == nil, d.now()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if probeErr != nil {
|
||||||
|
token, err := beginMonitorRecovery(d.db, d.serviceID, d.threshold, d.now())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if token != "" {
|
||||||
|
recoveryErr := d.recover(ctx)
|
||||||
|
// Finalize even on cancellation, using the non-context DB operation.
|
||||||
|
if err := finishMonitorRecovery(d.db, d.serviceID, token, d.now(), d.cooldown); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if recoveryErr != nil {
|
||||||
|
d.reportRecoveryError(recoveryErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := d.wait(ctx, d.interval); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,471 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var errMonitorTestUnhealthy = errors.New("unhealthy")
|
||||||
|
|
||||||
|
// Exercise the production process reader under the real singleton. PID reuse
|
||||||
|
// proves the old owner is gone and clears only that lease; a live owner stays
|
||||||
|
// busy, while an owner without v5 identity makes startup fail closed.
|
||||||
|
func TestManagedMonitorStartupOwnerPolicy(t *testing.T) {
|
||||||
|
for _, owner := range []string{"live", "reused", "legacy"} {
|
||||||
|
t.Run(owner, func(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
for i := 0; i < d.threshold; i++ {
|
||||||
|
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
token := requireMonitorBegin(t, d.db, d.serviceID, monitorTestTime, true)
|
||||||
|
if owner == "reused" {
|
||||||
|
if _, err := d.db.Exec(`UPDATE monitor_state SET recovery_owner_starttime = recovery_owner_starttime + 1 WHERE service_id = ?`, d.serviceID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if owner == "legacy" {
|
||||||
|
if _, err := d.db.Exec(`UPDATE monitor_state SET recovery_owner_pid = NULL, recovery_owner_starttime = NULL WHERE service_id = ?`, d.serviceID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
probes := 0
|
||||||
|
d.probe = func(context.Context) error { probes++; return errMonitorTestUnhealthy }
|
||||||
|
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||||
|
err := runManagedMonitor(ctx, d)
|
||||||
|
if owner == "legacy" {
|
||||||
|
if err == nil || errors.Is(err, context.Canceled) || probes != 0 {
|
||||||
|
t.Fatalf("legacy startup: probes=%d err=%v", probes, err)
|
||||||
|
}
|
||||||
|
} else if !errors.Is(err, context.Canceled) || probes != 1 {
|
||||||
|
t.Fatalf("startup: probes=%d err=%v", probes, err)
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(d.db, d.serviceID)
|
||||||
|
if err != nil || state.RecoveryRunning != (owner != "reused") {
|
||||||
|
t.Fatalf("owner=%s state=%+v err=%v", owner, state, err)
|
||||||
|
}
|
||||||
|
if owner == "reused" {
|
||||||
|
if !state.CooldownUntil.Equal(monitorTestTime.Add(d.cooldown)) {
|
||||||
|
t.Fatalf("crash cooldown=%v", state.CooldownUntil)
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err == nil {
|
||||||
|
t.Fatal("old owner token accepted")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
var stored string
|
||||||
|
if err := d.db.QueryRow(`SELECT recovery_token FROM monitor_state WHERE service_id = ?`, d.serviceID).Scan(&stored); err != nil || stored != token {
|
||||||
|
t.Fatalf("lease changed: %q %v", stored, err)
|
||||||
|
}
|
||||||
|
if !state.CooldownUntil.IsZero() {
|
||||||
|
t.Fatal("retained owner cooldown changed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertMonitorUnlocked(t, d)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorUnverifiableOwnerUnderSingleton(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime)
|
||||||
|
if err != nil || token == "" {
|
||||||
|
t.Fatalf("token=%q err=%v", token, err)
|
||||||
|
}
|
||||||
|
s, err := acquireManagedMonitorSingleton(d.target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer s.close()
|
||||||
|
reader := func(int) (int64, error) { return 0, os.ErrPermission }
|
||||||
|
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, reader, d.now); !errors.Is(err, os.ErrPermission) {
|
||||||
|
t.Fatalf("unverifiable owner: %v", err)
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(d.db, d.serviceID)
|
||||||
|
if err != nil || !state.RecoveryRunning || !state.CooldownUntil.IsZero() {
|
||||||
|
t.Fatalf("state=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := s.close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assertMonitorUnlocked(t, d)
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitorRuntimeFixture(t *testing.T) managedMonitorDeps {
|
||||||
|
t.Helper()
|
||||||
|
target, _, _ := managedCLIFixture(t)
|
||||||
|
if err := os.Chmod(target.directory, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
db, err := openStore(filepath.Join(t.TempDir(), "monitor.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { db.Close() })
|
||||||
|
return managedMonitorDeps{
|
||||||
|
target: target, db: db, serviceID: "managed-youtube",
|
||||||
|
interval: 5 * time.Minute, threshold: 3, cooldown: 30 * time.Minute,
|
||||||
|
now: func() time.Time { return monitorTestTime },
|
||||||
|
probe: func(context.Context) error { return nil },
|
||||||
|
recover: func(context.Context) error { t.Fatal("unexpected recovery"); return nil },
|
||||||
|
wait: waitManagedMonitor,
|
||||||
|
reportRecoveryError: func(error) { t.Fatal("unexpected recovery error") },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertMonitorUnlocked(t *testing.T, d managedMonitorDeps) {
|
||||||
|
t.Helper()
|
||||||
|
s, err := acquireManagedMonitorSingleton(d.target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("singleton leaked: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorImmediatePeriodicAndReset(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
now := monitorTestTime
|
||||||
|
d.now = func() time.Time { return now }
|
||||||
|
observations := []bool{false, false, true, false, false, false}
|
||||||
|
wantStreak := []int{1, 2, 0, 1, 2, 3}
|
||||||
|
probes, waits, recoveries := 0, 0, 0
|
||||||
|
d.probe = func(context.Context) error {
|
||||||
|
if probes != waits || !now.Equal(monitorTestTime.Add(time.Duration(probes)*d.interval)) {
|
||||||
|
t.Fatal("probe not immediate/periodic")
|
||||||
|
}
|
||||||
|
healthy := observations[probes]
|
||||||
|
probes++
|
||||||
|
if healthy {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return errMonitorTestUnhealthy
|
||||||
|
}
|
||||||
|
d.recover = func(context.Context) error {
|
||||||
|
recoveries++
|
||||||
|
if probes != 6 {
|
||||||
|
t.Fatalf("recovery at probe %d", probes)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
d.wait = func(ctx context.Context, interval time.Duration) error {
|
||||||
|
if interval != d.interval {
|
||||||
|
t.Fatal("wrong interval")
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(d.db, d.serviceID)
|
||||||
|
if err != nil || state.ConsecutiveFailures != wantStreak[probes-1] || !state.LastProbeAt.Equal(now) {
|
||||||
|
t.Fatalf("state=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
waits++
|
||||||
|
if probes == len(observations) {
|
||||||
|
cancel()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
now = now.Add(interval)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if probes != 6 || recoveries != 1 {
|
||||||
|
t.Fatalf("probes=%d recoveries=%d", probes, recoveries)
|
||||||
|
}
|
||||||
|
assertMonitorUnlocked(t, d)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorCooldownFromCompletionEitherOutcome(t *testing.T) {
|
||||||
|
for _, failed := range []bool{false, true} {
|
||||||
|
t.Run(map[bool]string{false: "success", true: "failure"}[failed], func(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
now := monitorTestTime
|
||||||
|
d.now = func() time.Time { return now }
|
||||||
|
probes, recoveries, reports := 0, 0, 0
|
||||||
|
d.probe = func(context.Context) error { probes++; return errMonitorTestUnhealthy }
|
||||||
|
d.recover = func(context.Context) error {
|
||||||
|
recoveries++
|
||||||
|
want := monitorTestTime.Add(10 * time.Minute)
|
||||||
|
if recoveries == 2 {
|
||||||
|
want = monitorTestTime.Add(50 * time.Minute)
|
||||||
|
}
|
||||||
|
if recoveries > 2 || !now.Equal(want) {
|
||||||
|
t.Fatalf("recovery %d at %v", recoveries, now)
|
||||||
|
}
|
||||||
|
if token := requireMonitorBegin(t, d.db, d.serviceID, now, false); token != "" {
|
||||||
|
t.Fatal("double start")
|
||||||
|
}
|
||||||
|
now = now.Add(10 * time.Minute)
|
||||||
|
if failed {
|
||||||
|
return errMonitorTestUnhealthy
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
d.reportRecoveryError = func(err error) {
|
||||||
|
if !errors.Is(err, errMonitorTestUnhealthy) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reports++
|
||||||
|
}
|
||||||
|
d.wait = func(ctx context.Context, interval time.Duration) error {
|
||||||
|
state, err := readMonitorState(d.db, d.serviceID)
|
||||||
|
if err != nil || state.RecoveryRunning {
|
||||||
|
t.Fatalf("state=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
if recoveries > 0 {
|
||||||
|
want := monitorTestTime.Add(50 * time.Minute)
|
||||||
|
if recoveries == 2 {
|
||||||
|
want = monitorTestTime.Add(90 * time.Minute)
|
||||||
|
}
|
||||||
|
if !state.CooldownUntil.Equal(want) {
|
||||||
|
t.Fatalf("cooldown=%v want=%v", state.CooldownUntil, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if probes == 9 {
|
||||||
|
cancel()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
now = now.Add(interval)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if recoveries != 2 || (failed && reports != 2) || (!failed && reports != 0) {
|
||||||
|
t.Fatalf("recoveries=%d reports=%d", recoveries, reports)
|
||||||
|
}
|
||||||
|
assertMonitorUnlocked(t, d)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorStartupLeaseAfterReopenStartsCrashCooldown(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
path := filepath.Join(t.TempDir(), "monitor.db")
|
||||||
|
db, err := openStore(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if err := recordMonitorProbe(db, d.serviceID, false, monitorTestTime); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// An impossible Linux PID proves a dead owner without spawning a child.
|
||||||
|
oldToken, err := beginMonitorRecoveryWithProcess(db, d.serviceID, 3, monitorTestTime, 1<<30, func(int) (int64, error) { return 100, nil })
|
||||||
|
if err != nil || oldToken == "" {
|
||||||
|
t.Fatalf("token=%q err=%v", oldToken, err)
|
||||||
|
}
|
||||||
|
expires := monitorTestTime.Add(d.cooldown)
|
||||||
|
if _, err := db.Exec(`UPDATE monitor_state SET cooldown_until = ? WHERE service_id = ?`, monitorTestTime.Add(time.Hour).UnixNano(), d.serviceID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := db.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d.db, err = openStore(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer d.db.Close()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
d.probe = func(context.Context) error {
|
||||||
|
state, err := readMonitorState(d.db, d.serviceID)
|
||||||
|
if err != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(expires) || state.ConsecutiveFailures != 3 {
|
||||||
|
t.Fatalf("startup=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(d.db, d.serviceID, oldToken, monitorTestTime, d.cooldown); err == nil {
|
||||||
|
t.Fatal("stale finish accepted")
|
||||||
|
}
|
||||||
|
return errMonitorTestUnhealthy
|
||||||
|
}
|
||||||
|
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||||
|
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(d.db, d.serviceID)
|
||||||
|
if err != nil || state.ConsecutiveFailures != 4 || !state.CooldownUntil.Equal(expires) {
|
||||||
|
t.Fatalf("state=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
assertMonitorUnlocked(t, d)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorConcurrentStartAndCancellation(t *testing.T) {
|
||||||
|
for _, phase := range []string{"probe", "recovery", "wait"} {
|
||||||
|
t.Run(phase, func(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
entered := make(chan struct{})
|
||||||
|
childExited := make(chan struct{})
|
||||||
|
block := func(ctx context.Context) error {
|
||||||
|
go func() { <-ctx.Done(); close(childExited) }()
|
||||||
|
close(entered)
|
||||||
|
<-childExited // Callback joins its child before the supervisor releases the lock.
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
switch phase {
|
||||||
|
case "probe":
|
||||||
|
d.probe = block
|
||||||
|
case "recovery":
|
||||||
|
d.threshold = 1
|
||||||
|
d.probe = func(context.Context) error { return errMonitorTestUnhealthy }
|
||||||
|
d.recover = block
|
||||||
|
d.reportRecoveryError = func(error) {}
|
||||||
|
case "wait":
|
||||||
|
d.wait = func(ctx context.Context, _ time.Duration) error { return block(ctx) }
|
||||||
|
}
|
||||||
|
done := make(chan error, 1)
|
||||||
|
finished := false
|
||||||
|
t.Cleanup(func() {
|
||||||
|
cancel()
|
||||||
|
if !finished {
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Error("supervisor leaked during cleanup")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
go func() { done <- runManagedMonitor(ctx, d) }()
|
||||||
|
select {
|
||||||
|
case <-entered:
|
||||||
|
case err := <-done:
|
||||||
|
finished = true
|
||||||
|
t.Fatalf("early exit: %v", err)
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("monitor did not start")
|
||||||
|
}
|
||||||
|
if err := runManagedMonitor(context.Background(), d); err == nil {
|
||||||
|
t.Fatal("parallel monitor accepted")
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
finished = true
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("cancellation blocked")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-childExited:
|
||||||
|
default:
|
||||||
|
t.Fatal("callback child leaked")
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(d.db, d.serviceID)
|
||||||
|
if err != nil || state.RecoveryRunning {
|
||||||
|
t.Fatalf("state=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
if phase == "recovery" && !state.CooldownUntil.Equal(monitorTestTime.Add(d.cooldown)) {
|
||||||
|
t.Fatal("cancelled recovery not finalized")
|
||||||
|
}
|
||||||
|
assertMonitorUnlocked(t, d)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorReconciliationCapabilityAndGuards(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, nil, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
|
||||||
|
t.Fatal("missing singleton accepted")
|
||||||
|
}
|
||||||
|
s, err := acquireManagedMonitorSingleton(d.target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer s.close()
|
||||||
|
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime)
|
||||||
|
if err != nil || token == "" {
|
||||||
|
t.Fatalf("token=%q err=%v", token, err)
|
||||||
|
}
|
||||||
|
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
|
||||||
|
t.Fatal("live lease cleared by repeated startup")
|
||||||
|
}
|
||||||
|
if err := s.close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := reconcileManagedMonitorRecovery(d.db, d.serviceID, s, d.cooldown, readMonitorProcessStarttime, d.now); err == nil {
|
||||||
|
t.Fatal("closed capability accepted")
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, d.cooldown); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d.target.directory = t.TempDir()
|
||||||
|
d.target.lookup = func(string) (string, bool) { return "", false }
|
||||||
|
if _, err := acquireManagedMonitorSingleton(d.target); err == nil {
|
||||||
|
t.Fatal("unguarded singleton accepted")
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(filepath.Join(d.target.directory, managedMonitorLockName)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("guard touched lock: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorFatalStoreAndFinalizationErrors(t *testing.T) {
|
||||||
|
for _, phase := range []string{"startup", "record", "begin", "finish"} {
|
||||||
|
t.Run(phase, func(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
d.threshold = 1
|
||||||
|
if phase == "startup" {
|
||||||
|
if err := d.db.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d.probe = func(context.Context) error {
|
||||||
|
if phase == "startup" {
|
||||||
|
t.Fatal("probe ran after startup DB error")
|
||||||
|
}
|
||||||
|
if phase == "begin" {
|
||||||
|
_, err := d.db.Exec(`CREATE TRIGGER reject_monitor_begin BEFORE UPDATE OF recovery_running ON monitor_state WHEN NEW.recovery_running = 1 BEGIN SELECT RAISE(ABORT, 'begin rejected'); END`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if phase == "record" {
|
||||||
|
if err := d.db.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errMonitorTestUnhealthy
|
||||||
|
}
|
||||||
|
d.recover = func(context.Context) error {
|
||||||
|
if phase != "finish" {
|
||||||
|
t.Fatal("recovery ran after fatal DB error")
|
||||||
|
}
|
||||||
|
_, err := d.db.Exec(`CREATE TRIGGER reject_monitor_finish BEFORE UPDATE OF recovery_running ON monitor_state WHEN NEW.recovery_running = 0 BEGIN SELECT RAISE(ABORT, 'finish rejected'); END`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
d.wait = func(context.Context, time.Duration) error { t.Fatal("continued after fatal DB error"); return nil }
|
||||||
|
if err := runManagedMonitor(context.Background(), d); err == nil {
|
||||||
|
t.Fatal("DB error ignored")
|
||||||
|
}
|
||||||
|
assertMonitorUnlocked(t, d)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,7 +3,9 @@ package main
|
|||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -297,7 +299,7 @@ func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
var version, count int
|
var version, count int
|
||||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
||||||
t.Fatalf("version=%d err=%v", version, err)
|
t.Fatalf("version=%d err=%v", version, err)
|
||||||
}
|
}
|
||||||
if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 {
|
if err := db.QueryRow("SELECT count(*) FROM monitor_state").Scan(&count); err != nil || count != 0 {
|
||||||
@@ -308,6 +310,160 @@ func TestMonitorV3MigrationPreservesHistory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMonitorCrashReconciliation(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
start int64
|
||||||
|
err error
|
||||||
|
clear, wantErr bool
|
||||||
|
}{
|
||||||
|
{"live", 100, nil, false, false},
|
||||||
|
{"missing", 0, os.ErrNotExist, true, false},
|
||||||
|
{"reused", 101, nil, true, false},
|
||||||
|
{"unreadable", 0, os.ErrPermission, false, true},
|
||||||
|
{"invalid", 0, nil, false, true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "crash.db")
|
||||||
|
db, err := openStore(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { db.Close() })
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
token, err := beginMonitorRecoveryWithProcess(db, "service", 3, monitorTestTime, 42, func(pid int) (int64, error) {
|
||||||
|
if pid != 42 {
|
||||||
|
t.Fatalf("pid=%d", pid)
|
||||||
|
}
|
||||||
|
return 100, nil
|
||||||
|
})
|
||||||
|
if err != nil || token == "" {
|
||||||
|
t.Fatalf("acquire: %q %v", token, err)
|
||||||
|
}
|
||||||
|
var pid, start int64
|
||||||
|
if err := db.QueryRow(`SELECT recovery_owner_pid, recovery_owner_starttime FROM monitor_state WHERE service_id = 'service'`).Scan(&pid, &start); err != nil || pid != 42 || start != 100 {
|
||||||
|
t.Fatalf("owner=%d/%d err=%v", pid, start, err)
|
||||||
|
}
|
||||||
|
now := monitorTestTime.Add(time.Hour)
|
||||||
|
clear, err := reconcileMonitorRecovery(db, "service", 30*time.Minute, func(pid int) (int64, error) {
|
||||||
|
if pid != 42 {
|
||||||
|
t.Fatalf("pid=%d", pid)
|
||||||
|
}
|
||||||
|
return tc.start, tc.err
|
||||||
|
}, func() time.Time { return now })
|
||||||
|
if clear != tc.clear || (err != nil) != tc.wantErr {
|
||||||
|
t.Fatalf("clear=%v err=%v", clear, err)
|
||||||
|
}
|
||||||
|
if err := db.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
db, err = openStore(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(db, "service")
|
||||||
|
if err != nil || state.RecoveryRunning == tc.clear {
|
||||||
|
t.Fatalf("state=%+v err=%v", state, err)
|
||||||
|
}
|
||||||
|
if tc.clear {
|
||||||
|
expires := now.Add(30 * time.Minute)
|
||||||
|
if !state.CooldownUntil.Equal(expires) {
|
||||||
|
t.Fatalf("cooldown=%v", state.CooldownUntil)
|
||||||
|
}
|
||||||
|
requireMonitorBegin(t, db, "service", expires.Add(-time.Nanosecond), false)
|
||||||
|
newToken := requireMonitorBegin(t, db, "service", expires, true)
|
||||||
|
if err := finishMonitorRecovery(db, "service", token, expires, time.Hour); err == nil {
|
||||||
|
t.Fatal("stale crash token accepted")
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(db, "service", newToken, expires, time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
} else if !state.CooldownUntil.IsZero() {
|
||||||
|
t.Fatal("retained lease changed cooldown")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorV4MigrationPreservesUnverifiableLease(t *testing.T) {
|
||||||
|
db := testStore(t)
|
||||||
|
if _, err := db.Exec("DROP TABLE monitor_state;" + monitorSchema); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
token := strings.Repeat("a", 64)
|
||||||
|
if _, err := db.Exec(`INSERT INTO monitor_state(service_id, consecutive_failures, last_probe_at, recovery_running, recovery_token, cooldown_until) VALUES ('service', 3, ?, 1, ?, ?)`, monitorTestTime.UnixNano(), token, monitorTestTime.Add(time.Minute).UnixNano()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before, err := readMonitorState(db, "service")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
if err := migrate(db); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var version int
|
||||||
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
||||||
|
t.Fatalf("version=%d err=%v", version, err)
|
||||||
|
}
|
||||||
|
clear, err := reconcileMonitorRecovery(db, "service", time.Minute, func(int) (int64, error) {
|
||||||
|
t.Fatal("legacy owner must not be guessed")
|
||||||
|
return 0, os.ErrNotExist
|
||||||
|
}, func() time.Time { return monitorTestTime.Add(time.Hour) })
|
||||||
|
if clear || err == nil {
|
||||||
|
t.Fatalf("legacy clear=%v err=%v", clear, err)
|
||||||
|
}
|
||||||
|
after, err := readMonitorState(db, "service")
|
||||||
|
if err != nil || after != before {
|
||||||
|
t.Fatalf("migration changed state: %+v -> %+v err=%v", before, after, err)
|
||||||
|
}
|
||||||
|
var storedToken string
|
||||||
|
if err := db.QueryRow("SELECT recovery_token FROM monitor_state WHERE service_id = 'service'").Scan(&storedToken); err != nil || storedToken != token {
|
||||||
|
t.Fatalf("token=%q err=%v", storedToken, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorReconciliationCannotClearNewAcquisition(t *testing.T) {
|
||||||
|
db := testStore(t)
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if err := recordMonitorProbe(db, "service", false, monitorTestTime); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
token := requireMonitorBegin(t, db, "service", monitorTestTime, true)
|
||||||
|
var newToken string
|
||||||
|
clear, err := reconcileMonitorRecovery(db, "service", time.Hour, func(int) (int64, error) {
|
||||||
|
if err := finishMonitorRecovery(db, "service", token, monitorTestTime, time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
newToken = requireMonitorBegin(t, db, "service", monitorTestTime.Add(time.Minute), true)
|
||||||
|
return 0, os.ErrNotExist
|
||||||
|
}, func() time.Time { return monitorTestTime.Add(time.Minute) })
|
||||||
|
if err != nil || clear {
|
||||||
|
t.Fatalf("clear=%v err=%v", clear, err)
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(db, "service", newToken, monitorTestTime.Add(time.Minute), time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorProcessStatParser(t *testing.T) {
|
||||||
|
stat := "42 (name with ) parentheses) S" + strings.Repeat(" 0", 18) + " 123 0"
|
||||||
|
if start, err := parseMonitorProcessStat(42, stat); err != nil || start != 123 {
|
||||||
|
t.Fatalf("start=%d err=%v", start, err)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"", "41 (name) S" + strings.Repeat(" 0", 18) + " 123", "42 (name) S", strings.Replace(stat, "123", "invalid", 1), strings.Replace(stat, "123", "0", 1)} {
|
||||||
|
if _, err := parseMonitorProcessStat(42, bad); err == nil {
|
||||||
|
t.Fatalf("accepted %q", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMonitorV3MigrationRollback(t *testing.T) {
|
func TestMonitorV3MigrationRollback(t *testing.T) {
|
||||||
db := testStore(t)
|
db := testStore(t)
|
||||||
saveProposal(t, db, "--lua-desync=fake")
|
saveProposal(t, db, "--lua-desync=fake")
|
||||||
|
|||||||
@@ -114,7 +114,7 @@ func migrate(db *sql.DB) error {
|
|||||||
if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil {
|
if err = conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if version > 4 {
|
if version > 5 {
|
||||||
return fmt.Errorf("unsupported database schema version %d", version)
|
return fmt.Errorf("unsupported database schema version %d", version)
|
||||||
}
|
}
|
||||||
if version == 0 {
|
if version == 0 {
|
||||||
@@ -236,6 +236,11 @@ PRAGMA user_version = 1;`)
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if version < 5 {
|
||||||
|
if _, err := conn.ExecContext(ctx, monitorOwnerSchema); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
_, err = conn.ExecContext(ctx, "COMMIT")
|
_, err = conn.ExecContext(ctx, "COMMIT")
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -224,7 +224,7 @@ func TestConcurrentFirstOpens(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, db := range stores {
|
for _, db := range stores {
|
||||||
var version, objects int
|
var version, objects int
|
||||||
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 4 {
|
if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil || version != 5 {
|
||||||
t.Fatalf("version=%d err=%v", version, err)
|
t.Fatalf("version=%d err=%v", version, err)
|
||||||
}
|
}
|
||||||
if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE
|
if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE
|
||||||
@@ -377,7 +377,7 @@ func TestSchemaConstraintsAndIncompleteEvidence(t *testing.T) {
|
|||||||
|
|
||||||
func TestUnsupportedSchemaVersion(t *testing.T) {
|
func TestUnsupportedSchemaVersion(t *testing.T) {
|
||||||
db := testStore(t)
|
db := testStore(t)
|
||||||
if _, err := db.Exec("PRAGMA user_version = 5"); err != nil {
|
if _, err := db.Exec("PRAGMA user_version = 6"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") {
|
if err := migrate(db); err == nil || !strings.Contains(err.Error(), "unsupported") {
|
||||||
|
|||||||
Reference in new issue
Block a user