fix(runtime): keep supervision and absence retries in container
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
226d93e2aa
commit
3f9fd2f36f
21 files changed
+1056
-137
No files matched your search
@@ -7,3 +7,4 @@
|
|||||||
!autotune-entrypoint.sh
|
!autotune-entrypoint.sh
|
||||||
!Dockerfile.target
|
!Dockerfile.target
|
||||||
!managed-target-entrypoint.sh
|
!managed-target-entrypoint.sh
|
||||||
|
!managed-target-supervision.sh
|
||||||
@@ -10,6 +10,7 @@ FROM vernette/ss-zapret2@sha256:ba158180f27cd71a291ee521c291490374fd3fcf2f81eb14
|
|||||||
USER root
|
USER root
|
||||||
COPY --from=build /out/zapret-autotune /usr/local/bin/zapret-autotune
|
COPY --from=build /out/zapret-autotune /usr/local/bin/zapret-autotune
|
||||||
COPY managed-target-entrypoint.sh /opt/managed-target-entrypoint.sh
|
COPY managed-target-entrypoint.sh /opt/managed-target-entrypoint.sh
|
||||||
|
COPY managed-target-supervision.sh /opt/managed-target-supervision.sh
|
||||||
# Retain the pinned image's fakes and Lua; never mount host replacements.
|
# Retain the pinned image's fakes and Lua; never mount host replacements.
|
||||||
RUN if [ ! -e /opt/zapret2/lua ] && [ ! -L /opt/zapret2/lua ]; then \
|
RUN if [ ! -e /opt/zapret2/lua ] && [ ! -L /opt/zapret2/lua ]; then \
|
||||||
cp -a /opt/zapret2/lua.dist /opt/zapret2/lua; \
|
cp -a /opt/zapret2/lua.dist /opt/zapret2/lua; \
|
||||||
|
|||||||
+47
-23
@@ -162,10 +162,10 @@ not rescanned merely because the process restarted.
|
|||||||
|
|
||||||
Entrypoint ordering is SS children → listener → fresh managed init if needed → managed reconcile → zapret start
|
Entrypoint ordering is SS children → listener → fresh managed init if needed → managed reconcile → zapret start
|
||||||
→ one installed `/usr/local/bin/zapret-autotune managed monitor` child. It supervises
|
→ one installed `/usr/local/bin/zapret-autotune managed monitor` child. It supervises
|
||||||
all three children; monitor exit is fatal even if it exits zero unexpectedly.
|
all three children; unexpected monitor exit is not a successful shutdown.
|
||||||
SIGTERM/SIGINT cancels and waits the monitor before target-local zapret stop and
|
SIGTERM/SIGINT cancels and waits the monitor before target-local zapret stop and
|
||||||
SS child termination/wait. Existing targets and Xray remain unchanged. Deployment
|
SS child termination/wait. Existing targets and Xray remain unchanged. Deployment
|
||||||
allows 120s Compose shutdown grace and 180s systemd stop timeout for bounded
|
allows 240s Compose shutdown grace for bounded
|
||||||
monitor cancellation/restoration and child joins. Code integration is not
|
monitor cancellation/restoration and child joins. Code integration is not
|
||||||
image/routing/signal live verification.
|
image/routing/signal live verification.
|
||||||
|
|
||||||
@@ -185,7 +185,7 @@ This is a standalone **new isolated target** candidate, not a production-approve
|
|||||||
deployment and not an integration into existing zapret/Xray. Keep the exact
|
deployment and not an integration into existing zapret/Xray. Keep the exact
|
||||||
`managed-test` role, ID, marker, mount, config, DB and owned-listener guards above.
|
`managed-test` role, ID, marker, mount, config, DB and owned-listener guards above.
|
||||||
Do not change the role to `production`, adopt old state, share an old network
|
Do not change the role to `production`, adopt old state, share an old network
|
||||||
namespace, or relax a refusal. Provisioning and optional service installation
|
namespace, or relax a refusal. Container provisioning instructions
|
||||||
are described in [deploy/managed-target/README.md](deploy/managed-target/README.md).
|
are described in [deploy/managed-target/README.md](deploy/managed-target/README.md).
|
||||||
|
|
||||||
### Build once, promote the same image
|
### Build once, promote the same image
|
||||||
@@ -227,8 +227,8 @@ docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/co
|
|||||||
docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml pull target
|
docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml pull target
|
||||||
```
|
```
|
||||||
|
|
||||||
The supplied systemd unit uses this same `.env` and starts with `--no-build`;
|
Start the selected container image with Compose `up --no-build`;
|
||||||
no image-selection override or unit edit is needed. Keep the selected digest and
|
no host service installation is required. Keep the selected digest and
|
||||||
private configuration together for upgrades and rollback. Never run a build
|
private configuration together for upgrades and rollback. Never run a build
|
||||||
against the production digest selector; build/tag separately as shown above.
|
against the production digest selector; build/tag separately as shown above.
|
||||||
|
|
||||||
@@ -268,31 +268,55 @@ existing Xray deployment as part of bootstrap.
|
|||||||
|
|
||||||
### Supervision and failure handling
|
### Supervision and failure handling
|
||||||
|
|
||||||
Compose restart is disabled. Optional native systemd runs **foreground** Compose
|
The container entrypoint is PID 1 and owns all technical child restart control.
|
||||||
with `--no-build --abort-on-container-exit --exit-code-from target`; do not also
|
After a retryable monitor/SS failure it joins children, reconciles trusted state
|
||||||
run detached Compose for that project. The unit requests restart only for exit
|
and stops target-local zapret before starting a new runtime cycle in the **same
|
||||||
75 via `Restart=no` plus `RestartForceExitStatus=75`, bounded by
|
container**. Retries use cancellable backoff (15s, 30s, capped at 60s) and at most
|
||||||
`StartLimitBurst=3` in six hours (starts, including the initial
|
three runtime starts per rolling six hours, including the initial start. Exhaustion
|
||||||
start, not three additional retries). Compose stop grace is 120s, systemd 180s.
|
stops fail-closed with 78; configuration, safety, DB, journal, rollback or cleanup
|
||||||
|
errors never authorize retry. No new dependency, host service installation or
|
||||||
|
external process manager is required. SIGTERM/SIGINT cancels backoff or the running
|
||||||
|
monitor, waits for bounded restoration and joins children before exit.
|
||||||
|
|
||||||
|
Compose restart remains disabled: a blanket container restart could loop over a
|
||||||
|
safety refusal or reset the in-container retry budget. Compose stop grace is 240s:
|
||||||
|
the sequential cleanup budget is 140s (45s monitor join, 45s reconciliation plus
|
||||||
|
5s kill grace, 30s zapret stop plus 5s kill grace, and two 5s SS joins), with room
|
||||||
|
for a deferred 45s startup command plus kill grace and scheduling overhead.
|
||||||
|
The join watchdog runs in a fresh shell and is cancelled when its child finishes;
|
||||||
|
graceful shutdown does not wait out the watchdog's full deadline.
|
||||||
|
Container-platform lifecycle, creation/recreation and host-boot policy belong to
|
||||||
|
the deployment operator, not an external manager supplied by this project.
|
||||||
|
|
||||||
| Target exit | Operator meaning |
|
| Target exit | Operator meaning |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `64` | Invalid configuration; fix settings before starting |
|
| `64` | Invalid configuration; fix settings before starting |
|
||||||
| `78` | Safety/DB/journal/rollback/cleanup failure; investigate, no automatic restart |
|
| `78` | Safety/DB/journal/rollback/cleanup failure; investigate, no automatic restart |
|
||||||
| `75` | Technical failure eligible for bounded restart only after safe cleanup |
|
| `75` | Internal technical-cycle result; PID 1 retries only after safe cleanup, within its budget |
|
||||||
| `0` | Graceful signal shutdown; no automatic restart |
|
| `0` | Graceful signal shutdown; no automatic restart |
|
||||||
|
|
||||||
Bootstrap/recovery has a fixed **90-minute whole-job deadline**, plus an
|
Bootstrap/recovery has a fixed **90-minute whole-job deadline**, plus an
|
||||||
independent **30-second restoration reserve**. This is a catastrophic technical
|
independent **30-second restoration reserve**. This is a catastrophic technical
|
||||||
ceiling, not a normal candidate-check timeout and not an operator tunable. Valid
|
ceiling, not a normal candidate-check timeout and not an operator tunable. Valid
|
||||||
nonpassing candidates/unresolved GGC are normal recovery outcomes governed by
|
nonpassing candidates/unresolved GGC are normal recovery outcomes, not a reason
|
||||||
the persisted threshold/cooldown, not a reason to restart the container. Failed
|
to restart the container. Genuine **no qualified strategy** from validated scan/check
|
||||||
|
evidence persists a recovery wait of at least **two hours** in SQLite, including
|
||||||
|
empty bootstrap and frontend/GGC exhaustion. The same container stays alive and
|
||||||
|
continues light periodic Player API probes; it does not scan or apply during that
|
||||||
|
wait. Startup respects the durable expiry, and recovery resumes after expiry under
|
||||||
|
the normal failure-threshold policy. In both fresh bootstrap and periodic recovery,
|
||||||
|
the delay is the greater of two hours and the configured recovery cooldown;
|
||||||
|
a longer configured cooldown is not shortened.
|
||||||
|
Other ordinary recovery outcomes retain the configured cooldown (default **30m**).
|
||||||
|
Malformed/incomplete reports, runner errors and unsafe state are not strategy
|
||||||
|
absence and must not enter this wait. The two-hour minimum absence policy is not a
|
||||||
|
new operator setting, technical retry backoff, or the **90-minute watchdog**. Failed
|
||||||
restoration or unsafe state vetoes retry: timeout exit 75 requires the job's own
|
restoration or unsafe state vetoes retry: timeout exit 75 requires the job's own
|
||||||
deadline and only timeout/cancellation-related errors, with no joined cleanup,
|
deadline and only timeout/cancellation-related errors, with no joined cleanup,
|
||||||
DB or safety failure. Unexpected monitor/SS failures are separately technical
|
DB or safety failure. Unexpected monitor/SS failures are separately technical
|
||||||
only if shutdown cleanup succeeds. Shutdown joins the monitor for up to 60s,
|
only if shutdown cleanup succeeds. Shutdown uses bounded monitor join,
|
||||||
target-local zapret stop gets 30s then forced kill after five more, and SS children
|
guarded reconciliation, target-local zapret stop and SS child joins;
|
||||||
get 5s each; failure to join/restore is fail-closed. The reserve is not extra
|
failure to join/restore is fail-closed. The reserve is not extra
|
||||||
scanning time. The 90-minute budget is shared by the whole bootstrap/recovery
|
scanning time. The 90-minute budget is shared by the whole bootstrap/recovery
|
||||||
job, not renewed per candidate or phase.
|
job, not renewed per candidate or phase.
|
||||||
|
|
||||||
@@ -304,7 +328,7 @@ be reported as complete playback health.
|
|||||||
### Upgrade, rollback and operator recovery
|
### Upgrade, rollback and operator recovery
|
||||||
|
|
||||||
1. Record the current image digest and private deployment configuration. Stop
|
1. Record the current image digest and private deployment configuration. Stop
|
||||||
the service (or foreground Compose), wait for complete cleanup, and verify
|
the target with Compose, wait for complete cleanup, and verify
|
||||||
the target is stopped. If cleanup fails, investigate before upgrading.
|
the target is stopped. If cleanup fails, investigate before upgrading.
|
||||||
2. With the target stopped, make a root-only backup of the **entire** state
|
2. With the target stopped, make a root-only backup of the **entire** state
|
||||||
directory (including DB sidecars, journal, config, identity and lock file),
|
directory (including DB sidecars, journal, config, identity and lock file),
|
||||||
@@ -329,16 +353,16 @@ is rolled back to qualified committed bytes before any reload/start; fresh
|
|||||||
baseline and known committed state are idempotent. Unknown/legacy state, drift,
|
baseline and known committed state are idempotent. Unknown/legacy state, drift,
|
||||||
missing qualification or failed rollback blocks startup with a non-retry exit.
|
missing qualification or failed rollback blocks startup with a non-retry exit.
|
||||||
|
|
||||||
On drift, unreconcilable pending transaction, rollback failure, or exhausted systemd start
|
On drift, unreconcilable pending transaction, rollback failure, or exhausted PID 1 retry
|
||||||
limit, leave state intact and service stopped. Diagnose using sanitized status
|
budget, leave state intact and the container stopped. Diagnose using sanitized status
|
||||||
and private artifacts; repair the actual provisioning/configuration issue or
|
and private artifacts; repair the actual provisioning/configuration issue or
|
||||||
restore a known consistent backup. Only then manually start (and reset the
|
restore a known consistent backup. Only then manually start the target with Compose.
|
||||||
systemd failed/start-limit state if needed). There is no automatic wipe or
|
There is no automatic wipe or
|
||||||
automatic adoption path.
|
automatic adoption path.
|
||||||
|
|
||||||
Rollout acceptance remains pending real isolated-target restart/deadline tests:
|
Rollout acceptance remains pending real isolated-target restart/deadline tests:
|
||||||
empty bootstrap, saved-first recovery, exact-host GGC rotation, image binding,
|
empty bootstrap, saved-first recovery, exact-host GGC rotation, image binding,
|
||||||
NFQUEUE activation/rollback, SIGTERM during downtime, deadline restoration,
|
NFQUEUE activation/rollback, SIGTERM during downtime, deadline restoration,
|
||||||
retry limits/exit propagation through foreground Compose/systemd, persistent
|
in-container retry limits/backoff, durable two-hour absence wait and resumption, persistent
|
||||||
recreation and backup restore. Perform these on the new target only; local unit
|
recreation and backup restore. Perform these on the new target only; local unit
|
||||||
or static checks cannot establish production readiness.
|
or static checks cannot establish production readiness.
|
||||||
+18
-5
@@ -1,8 +1,21 @@
|
|||||||
# Прогресс реализации
|
# Прогресс реализации
|
||||||
|
|
||||||
Срез на **2026-10-10**. Этот файл описывает фактическое состояние кода, а не весь объём требований из `Requirements.md` и `Architecture.md`.
|
Срез на **2026-10-11**. Этот файл описывает фактическое состояние кода, а не весь объём требований из `Requirements.md` и `Architecture.md`.
|
||||||
|
|
||||||
## Готово
|
## Текущий статус
|
||||||
|
|
||||||
|
- **2026-10-11 container-owned control (pending working-tree changes):** all technical child restart control belongs to the entrypoint/PID 1 inside the same container, not host systemd. Safe cleanup precedes each retry; three runtime starts per rolling six hours include the initial start, with cancellable 15s/30s backoff capped at 60s. Configuration/safety/DB/journal/rollback/cleanup failures and exhausted retry budget stop fail-closed. SIGTERM cancels and joins the monitor, restores safely, then stops target-local zapret and SS children. No additional dependency or host service installation is required. Compose keeps `restart: "no"` to avoid safety loops and now allows 240s stop grace: 140s sequential cleanup including 45s monitor join, plus room for a deferred 45s startup command/kill grace and overhead. A fresh-shell join watchdog is cancelled promptly on child completion; graceful shutdown never needs its full timeout. Container-platform lifecycle and host-boot policy remain deployment-operator responsibilities.
|
||||||
|
- **2026-10-11 strategy absence (pending working-tree changes):** genuine validated no-qualified-strategy outcomes in bootstrap/recovery persist a wait of at least two hours in SQLite. Both fresh bootstrap and periodic recovery use the greater of two hours and the configured recovery cooldown, preserving longer settings. The same container continues light probes without scans/apply during the wait; startup respects the persisted expiry, and normal threshold-driven recovery resumes after expiry. Other ordinary outcomes keep the default 30m cooldown. Malformed reports/runner/safety errors are not absence. The separate 90-minute whole-job watchdog plus 30-second restoration reserve remains an abnormal technical ceiling, not a normal no-strategy result or the absence wait.
|
||||||
|
- Root operator docs now describe this contract, preserving immutable tagged/digest promotion, BuildKit/pinned builds, private `.env`, whole stopped-state backup/restore and exact legacy `managed-test` guards for a new isolated target only. No old target adoption or Xray changes. New runtime changes still need their owners' focused checks and isolated live acceptance (retry/backoff, SIGTERM, durable absence expiry/resume); this documentation update does not claim those checks were run.
|
||||||
|
|
||||||
|
## История реализации (не текущие инструкции)
|
||||||
|
|
||||||
|
The dated facts below are historical snapshots. In particular, host-systemd unit
|
||||||
|
installation/restart/start-limit guidance and its acceptance tests are **superseded**
|
||||||
|
by container-owned control above; they impose no current deployment requirement.
|
||||||
|
Earlier “not deployed/not verified” snapshots do not override later recorded
|
||||||
|
isolated-target acceptance. Production acceptance of the new supervision/absence
|
||||||
|
policy remains pending.
|
||||||
|
|
||||||
- **2026-10-11 production-candidate update (remote checks dated 2026-10-10 UTC):** `ed4298d` adds one 90-minute budget per managed bootstrap/recovery job, shared across phases. Watchdog expiration is abnormal/fatal, never ordinary nonpass/cooldown continuation; clean timeout permits technical exit 75 only after worker join and bounded 30-second restoration. Any guard/DB/rollback/cleanup or mixed unknown error vetoes restart (78); configuration errors use 64, graceful shutdown 0. Independent review reproduced and closed mixed-timeout, fatal-discovery and fatal-health/deadline races. Preferred frontend seed is now the layout-safe alternative verified in prior 60/60 requests; cached definitions are not rewritten.
|
- **2026-10-11 production-candidate update (remote checks dated 2026-10-10 UTC):** `ed4298d` adds one 90-minute budget per managed bootstrap/recovery job, shared across phases. Watchdog expiration is abnormal/fatal, never ordinary nonpass/cooldown continuation; clean timeout permits technical exit 75 only after worker join and bounded 30-second restoration. Any guard/DB/rollback/cleanup or mixed unknown error vetoes restart (78); configuration errors use 64, graceful shutdown 0. Independent review reproduced and closed mixed-timeout, fatal-discovery and fatal-health/deadline races. Preferred frontend seed is now the layout-safe alternative verified in prior 60/60 requests; cached definitions are not rewritten.
|
||||||
- **2026-10-10 restart/startup hardening:** optional `deploy/managed-target/zapret-autotune.service` uses foreground Compose, no blanket Docker restart, systemd allowlisted 75 with bounded starts (3 per 6h); shutdown grace 120s / unit stop budget 180s. `61e6e00` adds guarded fixed-DB `managed reconcile` before initial activation; init/reconcile/start are bounded (45/45/30s plus 5s kill grace), all startup refusals map to 78. Unknown state is not sourced even by cleanup. Actual systemd tests: technical 75 retries bounded, 78 does not retry; killing monitor in final image caused one whole-target restart, preserving both groups and Player API OK. Permanent production service was not installed/enabled; transient acceptance units were stopped.
|
- **2026-10-10 restart/startup hardening:** optional `deploy/managed-target/zapret-autotune.service` uses foreground Compose, no blanket Docker restart, systemd allowlisted 75 with bounded starts (3 per 6h); shutdown grace 120s / unit stop budget 180s. `61e6e00` adds guarded fixed-DB `managed reconcile` before initial activation; init/reconcile/start are bounded (45/45/30s plus 5s kill grace), all startup refusals map to 78. Unknown state is not sourced even by cleanup. Actual systemd tests: technical 75 retries bounded, 78 does not retry; killing monitor in final image caused one whole-target restart, preserving both groups and Player API OK. Permanent production service was not installed/enabled; transient acceptance units were stopped.
|
||||||
@@ -65,18 +78,18 @@
|
|||||||
- На одном найденном GGC hostname текущая статическая стратегия `fakeddisorder:pos=midsld:tcp_flags_set=SYN` прошла data-transfer check 3/3 для TLS 1.2 и TLS 1.3. Frontend-комбинация на этом GGC hostname не прошла; frontend и GGC требуют отдельных профилей.
|
- На одном найденном GGC hostname текущая статическая стратегия `fakeddisorder:pos=midsld:tcp_flags_set=SYN` прошла data-transfer check 3/3 для TLS 1.2 и TLS 1.3. Frontend-комбинация на этом GGC hostname не прошла; frontend и GGC требуют отдельных профилей.
|
||||||
- Отдельный Selectel speedtest с SNI `googlevideo` был проверен экспериментально, но **не используется** для ранжирования: принято решение доверять TLS/data-transfer результатам `blockcheckw`.
|
- Отдельный Selectel speedtest с SNI `googlevideo` был проверен экспериментально, но **не используется** для ранжирования: принято решение доверять TLS/data-transfer результатам `blockcheckw`.
|
||||||
|
|
||||||
## Выполняется сейчас
|
## Исторические незавершённые работы (на момент записи)
|
||||||
|
|
||||||
- Scan и shortlist check завершены, контейнер `zapret-autotune-googlevideo-scan` остановился с кодом 0. Проверка exact combined args прошла; `tune googlevideo` не запускался end-to-end и результаты вручную в SQLite не импортировались.
|
- Scan и shortlist check завершены, контейнер `zapret-autotune-googlevideo-scan` остановился с кодом 0. Проверка exact combined args прошла; `tune googlevideo` не запускался end-to-end и результаты вручную в SQLite не импортировались.
|
||||||
- Новый отдельный `tune googlevideo` proposal-only и не меняет frontend-only `tune youtube`; удалённый end-to-end запуск команды ещё не выполнен.
|
- Новый отдельный `tune googlevideo` proposal-only и не меняет frontend-only `tune youtube`; удалённый end-to-end запуск команды ещё не выполнен.
|
||||||
|
|
||||||
## Не готово
|
## Исторические ограничения (на момент записи)
|
||||||
|
|
||||||
- Проверка реального YouTube video-transfer/SABR. Прямой GET `serverAbrStreamingUrl` возвращает `sabr.malformed_config`; hostname discovery и `blockcheckw` data-transfer checks не равны проверке воспроизведения видео.
|
- Проверка реального YouTube video-transfer/SABR. Прямой GET `serverAbrStreamingUrl` возвращает `sabr.malformed_config`; hostname discovery и `blockcheckw` data-transfer checks не равны проверке воспроизведения видео.
|
||||||
- Guarded managed CLI/monitor интегрированы внутри designated target: transaction core, bootstrap/recovery callback и deployment definition есть, но нет proven image/runtime compatibility, выполненного container provisioning и реально запущенного managed target. Одноразовый Player API health сам не запускает live automation. Цель: новый отдельный prod-like zapret2-ss контейнер только в тестовой среде; существующий контейнер не трогать. Стабильный настраиваемый loopback SOCKS для Xray; Xray config вне scope. Перед runtime пройти route/rollback gates.
|
- Guarded managed CLI/monitor интегрированы внутри designated target: transaction core, bootstrap/recovery callback и deployment definition есть, но нет proven image/runtime compatibility, выполненного container provisioning и реально запущенного managed target. Одноразовый Player API health сам не запускает live automation. Цель: новый отдельный prod-like zapret2-ss контейнер только в тестовой среде; существующий контейнер не трогать. Стабильный настраиваемый loopback SOCKS для Xray; Xray config вне scope. Перед runtime пройти route/rollback gates.
|
||||||
- Полный сервис из требований: YAML-конфигурация сервисов, endpoint-история/покрытие, daemon/периодический мониторинг, API/метрики и multi-service режим.
|
- Полный сервис из требований: YAML-конфигурация сервисов, endpoint-история/покрытие, daemon/периодический мониторинг, API/метрики и multi-service режим.
|
||||||
|
|
||||||
## Открытые ограничения и следующие шаги
|
## Исторические следующие шаги (текущие gates указаны выше)
|
||||||
|
|
||||||
1. Проверить новый `tune googlevideo` и `recover googlevideo` end-to-end на удалённом сервере; отдельно решить, как импортировать или переиспользовать уже полученные scan результаты без повторного перебора.
|
1. Проверить новый `tune googlevideo` и `recover googlevideo` end-to-end на удалённом сервере; отдельно решить, как импортировать или переиспользовать уже полученные scan результаты без повторного перебора.
|
||||||
2. Воспроизводимое описание отдельного prod-like target добавлено в `deploy/managed-target`; orchestrator должен проверить Compose syntax. Build/run отложены. До запуска проверить baseline-конфиг и NFQUEUE внутри container network namespace; текущие mounts, сеть, порты и конфиг не переиспользовать.
|
2. Воспроизводимое описание отдельного prod-like target добавлено в `deploy/managed-target`; orchestrator должен проверить Compose syntax. Build/run отложены. До запуска проверить baseline-конфиг и NFQUEUE внутри container network namespace; текущие mounts, сеть, порты и конфиг не переиспользовать.
|
||||||
|
|||||||
@@ -8,17 +8,27 @@ The standalone deployment candidate is a **new isolated target**, never an
|
|||||||
upgrade/adoption of an existing zapret or Xray service. The legacy `managed-test`
|
upgrade/adoption of an existing zapret or Xray service. The legacy `managed-test`
|
||||||
role and fixed identity guards remain intentional, including for prod-like runs.
|
role and fixed identity guards remain intentional, including for prod-like runs.
|
||||||
See [release, configuration, upgrade and rollback guidance](MANAGED_TARGET.md#production-preparation)
|
See [release, configuration, upgrade and rollback guidance](MANAGED_TARGET.md#production-preparation)
|
||||||
and the [deployment/service instructions](deploy/managed-target/README.md).
|
and the [container deployment instructions](deploy/managed-target/README.md).
|
||||||
Pinned builds and supervision are preparation, not production acceptance: real
|
Pinned builds and supervision are preparation, not production acceptance: real
|
||||||
restart/deadline tests and end-to-end target validation remain rollout gates.
|
restart/deadline tests and end-to-end target validation remain rollout gates.
|
||||||
Select an immutable managed-target digest with `ZAPRET_AUTOTUNE_IMAGE` in the
|
Select an immutable managed-target digest with `ZAPRET_AUTOTUNE_IMAGE` in the
|
||||||
private deployment `.env`; tags/local defaults are only build/test candidates.
|
private deployment `.env`; tags/local defaults are only build/test candidates.
|
||||||
Bootstrap/recovery shares a 90-minute whole-job deadline plus a separate 30-second
|
Bootstrap/recovery shares a 90-minute whole-job deadline plus a separate 30-second
|
||||||
restoration reserve, an abnormal ceiling rather than normal candidate failure.
|
restoration reserve, an abnormal ceiling rather than normal candidate failure.
|
||||||
Timeout exit 75 requires clean timeout/cancellation only; cleanup/safety errors
|
PID 1 handles technical child failures inside the same container, with safe
|
||||||
veto retry. Optional foreground systemd uses `Restart=no` plus forced exit-75
|
cleanup, bounded retries (three runtime starts per rolling six hours, including
|
||||||
restart, limited to three starts per six hours; Compose stop grace is 120s and
|
the initial start) and backoff. Cleanup/safety/configuration errors veto retry;
|
||||||
systemd stop timeout 180s.
|
SIGTERM gracefully cancels and joins children. Compose restart remains disabled
|
||||||
|
to prevent safety-failure loops; no host service installation or external process
|
||||||
|
manager is required. Container-platform lifecycle and host-boot policy belong to
|
||||||
|
the deployment operator. Compose stop grace is 240s; monitor join is bounded at
|
||||||
|
45s. Graceful shutdown cancels its watchdog rather than waiting out that deadline.
|
||||||
|
Genuine no-qualified-strategy outcomes instead persist a recovery wait of at least two hours
|
||||||
|
in SQLite while the same container continues light probes, then resume recovery
|
||||||
|
after expiry under the normal threshold policy. Both fresh bootstrap and periodic
|
||||||
|
recovery preserve a configured recovery cooldown longer than two hours.
|
||||||
|
This is not a watchdog timeout:
|
||||||
|
ordinary recovery cooldown stays 30m, and the abnormal whole-job ceiling stays 90m.
|
||||||
|
|
||||||
## Manual managed-test activation (not deployed)
|
## Manual managed-test activation (not deployed)
|
||||||
|
|
||||||
@@ -221,8 +231,12 @@ even after a newer acquisition starts. State survives reopening the database.
|
|||||||
The internal `runManagedMonitor` supervisor probes immediately and periodically,
|
The internal `runManagedMonitor` supervisor probes immediately and periodically,
|
||||||
persists observations, and invokes injected recovery only at threshold outside
|
persists observations, and invokes injected recovery only at threshold outside
|
||||||
cooldown. Unresolved GGC is reported without stopping monitoring; DB, safety,
|
cooldown. Unresolved GGC is reported without stopping monitoring; DB, safety,
|
||||||
runner, lock and finalization errors are fatal. Cooldown starts when recovery returns, including
|
runner, lock and finalization errors are fatal. Ordinary cooldown starts when recovery returns,
|
||||||
failure or cancellation. Callbacks must honor cancellation and join their children.
|
including failure or cancellation; validated strategy absence extends the durable
|
||||||
|
wait to at least two hours (or the longer configured recovery cooldown), in both
|
||||||
|
fresh bootstrap and periodic recovery, without restarting the container. Startup respects
|
||||||
|
the persisted expiry, and light probes continue without scans during the wait.
|
||||||
|
Callbacks must honor cancellation and join their children.
|
||||||
Existing target guards run before a lifetime nonblocking `flock` in the private
|
Existing target guards run before a lifetime nonblocking `flock` in the private
|
||||||
target state directory. With that exclusive singleton held, one-shot startup
|
target state directory. With that exclusive singleton held, one-shot startup
|
||||||
reconciliation uses schema-v5 PID/starttime identity: only a proven dead or
|
reconciliation uses schema-v5 PID/starttime identity: only a proven dead or
|
||||||
|
|||||||
@@ -160,11 +160,11 @@ func managedMonitorRecoveryAdapters(target managedDeps, db *sql.DB, scanCommand
|
|||||||
return managedRecoveryDeps{}, fmt.Errorf("complete managed process adapters required")
|
return managedRecoveryDeps{}, fmt.Errorf("complete managed process adapters required")
|
||||||
}
|
}
|
||||||
// The blockcheck process-group runner cancels and waits descendants.
|
// The blockcheck process-group runner cancels and waits descendants.
|
||||||
scan := func(ctx context.Context, _ string, host string, _, _ io.Writer) (string, error) {
|
scan := func(ctx context.Context, _ string, host string, _ io.Writer, diagnostics io.Writer) (string, error) {
|
||||||
return scanCommand(ctx, "/usr/bin/blockcheckw", host, io.Discard, io.Discard)
|
return scanCommand(ctx, "/usr/bin/blockcheckw", host, io.Discard, diagnostics)
|
||||||
}
|
}
|
||||||
check := func(ctx context.Context, _ string, path, args, host string, _, _ io.Writer) error {
|
check := func(ctx context.Context, _ string, path, args, host string, _ io.Writer, diagnostics io.Writer) error {
|
||||||
return checkCommand(ctx, "/usr/bin/blockcheckw", path, args, host, io.Discard, io.Discard)
|
return checkCommand(ctx, "/usr/bin/blockcheckw", path, args, host, io.Discard, diagnostics)
|
||||||
}
|
}
|
||||||
run := func(ctx context.Context, path string, args ...string) error {
|
run := func(ctx context.Context, path string, args ...string) error {
|
||||||
if path != managedZapretScript || len(args) != 1 || (args[0] != "start" && args[0] != "stop" && args[0] != "restart") {
|
if path != managedZapretScript || len(args) != 1 || (args[0] != "start" && args[0] != "stop" && args[0] != "restart") {
|
||||||
|
|||||||
@@ -3,11 +3,13 @@ package main
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -123,7 +125,7 @@ func TestManagedEntrypointPrestartReconciliation(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
write("awk", "#!/bin/sh\nexit 0\n")
|
write("awk", "#!/bin/sh\nexit 0\n")
|
||||||
child := "#!/bin/sh\ntrap 'exit 0' TERM INT\nwhile :; do sleep 0.1 & wait $!; done\n"
|
child := "#!/bin/sh\nsleeper=\ntrap '[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; exit 0' TERM INT\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n"
|
||||||
write("ss-server", child)
|
write("ss-server", child)
|
||||||
write("ss-local", child)
|
write("ss-local", child)
|
||||||
log := filepath.Join(root, "log")
|
log := filepath.Join(root, "log")
|
||||||
@@ -137,10 +139,16 @@ func TestManagedEntrypointPrestartReconciliation(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
entry := write("entrypoint", strings.NewReplacer("/run/zapret-autotune-ss", ss, "/usr/local/bin/zapret-autotune", autotune, managedZapretScript, zapret, `"$ZAPRET_CONFIG"`, `"$FIXTURE_CONFIG"`).Replace(string(data)))
|
helper, err := filepath.Abs("../../managed-target-supervision.sh")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entry := write("entrypoint", strings.NewReplacer("/opt/managed-target-supervision.sh", helper, "/run/zapret-autotune-ss", ss, "/usr/local/bin/zapret-autotune", autotune, managedZapretScript, zapret, `"$ZAPRET_CONFIG"`, `"$FIXTURE_CONFIG"`).Replace(string(data)))
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
cmd := exec.CommandContext(ctx, "/bin/sh", entry)
|
cmd := exec.CommandContext(ctx, "/bin/sh", entry)
|
||||||
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||||
|
cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
|
||||||
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "SS_PASSWORD="+strings.Repeat("a", 32), "MANAGED_PRESTART_HELPER=1", "FIXTURE_BINARY="+binary, "FIXTURE_TARGET="+d.directory, "FIXTURE_SCRIPT="+zapret)
|
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "SS_PASSWORD="+strings.Repeat("a", 32), "MANAGED_PRESTART_HELPER=1", "FIXTURE_BINARY="+binary, "FIXTURE_TARGET="+d.directory, "FIXTURE_SCRIPT="+zapret)
|
||||||
for key, value := range env {
|
for key, value := range env {
|
||||||
cmd.Env = append(cmd.Env, key+"="+value)
|
cmd.Env = append(cmd.Env, key+"="+value)
|
||||||
@@ -354,6 +362,125 @@ func TestManagedMonitorProductionAdapterBinding(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorProductionAbsenceDiagnostics(t *testing.T) {
|
||||||
|
for _, mode := range []string{"empty-nfq", "partial-nfq", "combined-nfq", "combined-capture-nfq", "success-scan-nfq", "success-combined-nfq", "artifact-nfq", "empty-timeout", "artifact-timeout", "malformed"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
recovery, _, _ := recoveryFixture(t, false)
|
||||||
|
monitor := monitorRuntimeFixture(t)
|
||||||
|
monitor.db, monitor.target.directory = recovery.db, recovery.transaction.Dir
|
||||||
|
if err := os.Chmod(recovery.transaction.Dir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
identity := `{"Role":"managed-test","Directory":"` + recovery.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||||
|
if err := recovery.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const infra = "FAIL 0/1 nfqws2 exited immediately (code 1) secret-never-print"
|
||||||
|
adapted, err := managedMonitorRecoveryAdapters(monitor.target, recovery.db,
|
||||||
|
func(_ context.Context, _, host string, out, diagnostics io.Writer) (string, error) {
|
||||||
|
if out != io.Discard || diagnostics == io.Discard {
|
||||||
|
t.Fatal("streams not privately isolated")
|
||||||
|
}
|
||||||
|
// Model exec's independent stdout/stderr writers without sharing a buffer.
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { io.WriteString(out, "secret-never-print"); close(done) }()
|
||||||
|
message := "FAIL 0/1 timeout"
|
||||||
|
if strings.Contains(mode, "nfq") || mode == "artifact-timeout" {
|
||||||
|
message = infra
|
||||||
|
}
|
||||||
|
if strings.Contains(mode, "combined") {
|
||||||
|
message = "FAIL 0/1 timeout"
|
||||||
|
}
|
||||||
|
io.WriteString(diagnostics, message)
|
||||||
|
<-done
|
||||||
|
data := []byte(`{"domain":"` + host + `","total":2,"working":0,"strategies":[]}`)
|
||||||
|
if mode == "partial-nfq" {
|
||||||
|
var report struct {
|
||||||
|
Strategies []json.RawMessage `json:"strategies"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(combinedReportFixture("--lua-desync=fake")), &report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report.Strategies = report.Strategies[:1]
|
||||||
|
partial, marshalErr := json.Marshal(report)
|
||||||
|
if marshalErr != nil {
|
||||||
|
t.Fatal(marshalErr)
|
||||||
|
}
|
||||||
|
data = partial
|
||||||
|
}
|
||||||
|
if strings.Contains(mode, "combined") || mode == "success-scan-nfq" {
|
||||||
|
data = []byte(combinedReportFixture("--lua-desync=fake"))
|
||||||
|
}
|
||||||
|
if mode == "malformed" {
|
||||||
|
data = []byte(`{"strategies":[]}`)
|
||||||
|
}
|
||||||
|
path := filepath.Join(t.TempDir(), "scan")
|
||||||
|
if mode == "artifact-nfq" || mode == "artifact-timeout" {
|
||||||
|
artifact := infra
|
||||||
|
if mode == "artifact-timeout" {
|
||||||
|
artifact = "FAIL 0/1 timeout"
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path+".check.json.stderr", []byte(artifact), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return path, os.WriteFile(path+".check.json", data, 0600)
|
||||||
|
},
|
||||||
|
func(_ context.Context, _, path, _, _ string, out, diagnostics io.Writer) error {
|
||||||
|
if out != io.Discard || diagnostics == io.Discard {
|
||||||
|
t.Fatal("check streams not privately isolated")
|
||||||
|
}
|
||||||
|
io.WriteString(diagnostics, infra)
|
||||||
|
if mode != "combined-capture-nfq" {
|
||||||
|
if err := os.WriteFile(path+".combined.json.stderr", []byte(infra), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
data := nonpassingManagedReport(t)
|
||||||
|
if mode == "success-combined-nfq" {
|
||||||
|
data = []byte(combinedReportFixture("--lua-desync=fake"))
|
||||||
|
}
|
||||||
|
return os.WriteFile(path+".combined.json", data, 0600)
|
||||||
|
}, recovery.run)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := wireManagedRecovery(&monitor, adapted, func(error) { t.Fatal("unexpected GGC report") }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
monitor.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||||
|
var reported error
|
||||||
|
monitor.reportRecoveryError = func(err error) { reported = err }
|
||||||
|
err = runManagedMonitor(ctx, monitor)
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
wantFatal := strings.Contains(mode, "nfq")
|
||||||
|
if errors.As(err, &fatal) != wantFatal || (!wantFatal && !errors.Is(err, context.Canceled)) {
|
||||||
|
t.Fatalf("mode=%s err=%v", mode, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(errors.Join(err, reported).Error(), "secret-never-print") {
|
||||||
|
t.Fatal("diagnostics leaked")
|
||||||
|
}
|
||||||
|
state, stateErr := readMonitorState(monitor.db, monitor.serviceID)
|
||||||
|
if stateErr != nil {
|
||||||
|
t.Fatal(stateErr)
|
||||||
|
}
|
||||||
|
wantAbsence := mode == "empty-timeout" || mode == "artifact-timeout"
|
||||||
|
if errors.Is(reported, errNoQualifiedStrategy) != wantAbsence || errors.Is(err, errNoQualifiedStrategy) {
|
||||||
|
t.Fatalf("err=%v reported=%v", err, reported)
|
||||||
|
}
|
||||||
|
if wantAbsence {
|
||||||
|
if !state.CooldownUntil.Equal(monitorTestTime.Add(2 * time.Hour)) {
|
||||||
|
t.Fatalf("pause=%v", state.CooldownUntil)
|
||||||
|
}
|
||||||
|
} else if !state.CooldownUntil.IsZero() {
|
||||||
|
t.Fatalf("unexpected pause=%v", state.CooldownUntil)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) {
|
func TestManagedMonitorBootstrapUnresolvedGGC(t *testing.T) {
|
||||||
recovery, _, _ := recoveryFixture(t, false)
|
recovery, _, _ := recoveryFixture(t, false)
|
||||||
monitor := monitorRuntimeFixture(t)
|
monitor := monitorRuntimeFixture(t)
|
||||||
@@ -468,7 +595,7 @@ func TestManagedMonitorBootstrapExistingFrontend(t *testing.T) {
|
|||||||
// Execute only rewritten target-local contracts and fake programs; no network,
|
// Execute only rewritten target-local contracts and fake programs; no network,
|
||||||
// Docker, real credentials, zapret or host firewall. Exercise the actual traps.
|
// Docker, real credentials, zapret or host firewall. Exercise the actual traps.
|
||||||
func TestManagedEntrypointSupervision(t *testing.T) {
|
func TestManagedEntrypointSupervision(t *testing.T) {
|
||||||
modes := []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"}
|
modes := []string{"term", "fatal", "technical", "panic", "killed", "segv", "zero", "unknown", "ss", "ss-early", "ss-unsafe", "ss-unknown", "ss-technical", "term-unsafe", "term-unknown", "term-hang", "stop-failure", "stop-hang"}
|
||||||
for _, stage := range []string{"init", "reconcile", "start"} {
|
for _, stage := range []string{"init", "reconcile", "start"} {
|
||||||
for _, failure := range []string{"hang", "75", "124", "137", "7"} {
|
for _, failure := range []string{"hang", "75", "124", "137", "7"} {
|
||||||
modes = append(modes, "startup-"+stage+"-"+failure)
|
modes = append(modes, "startup-"+stage+"-"+failure)
|
||||||
@@ -499,21 +626,37 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
|||||||
write("timeout", "#!/bin/sh\nif [ \"$3\" = 45 ] || { [ \"$3\" = 30 ] && [ \"$5\" = start ]; }; then\nshift 3\nexec \"$REAL_TIMEOUT\" -k 1 1 \"$@\"\nfi\nexec \"$REAL_TIMEOUT\" \"$@\"\n")
|
write("timeout", "#!/bin/sh\nif [ \"$3\" = 45 ] || { [ \"$3\" = 30 ] && [ \"$5\" = start ]; }; then\nshift 3\nexec \"$REAL_TIMEOUT\" -k 1 1 \"$@\"\nfi\nexec \"$REAL_TIMEOUT\" \"$@\"\n")
|
||||||
t.Setenv("REAL_TIMEOUT", timeout)
|
t.Setenv("REAL_TIMEOUT", timeout)
|
||||||
}
|
}
|
||||||
startupFailure := "case \"$MODE\" in startup-$stage-hang) trap '' TERM INT; while :; do sleep 0.1; done;; startup-$stage-*) exit \"${MODE##*-}\";; esac\n"
|
// Accelerate only restart cooldowns; watchdog deadlines remain real.
|
||||||
|
realSleep, err := exec.LookPath("sleep")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
write("sleep", "#!/bin/sh\ncase \"$1\" in 15|30|60) exec \"$REAL_SLEEP\" 0.02;; *) exec \"$REAL_SLEEP\" \"$@\";; esac\n")
|
||||||
|
t.Setenv("REAL_SLEEP", realSleep)
|
||||||
|
startupFailure := "case \"$MODE\" in startup-$stage-hang) trap '' TERM INT; exec sleep 300;; startup-$stage-*) exit \"${MODE##*-}\";; esac\n"
|
||||||
write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n")
|
write("awk", "#!/bin/sh\ncase \"$*\" in */proc/net/tcp*) until grep -q ready-ss-server \"$LOG\" && grep -q ready-ss-local \"$LOG\"; do sleep 0.02; done;; esac\nexit 0\n")
|
||||||
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\ntrap 'echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & wait $!; done\n"
|
joinSleeper := "[ -z \"$sleeper\" ] || { kill \"$sleeper\" 2>/dev/null || true; wait \"$sleeper\" 2>/dev/null || true; }; "
|
||||||
|
child := "#!/bin/sh\n[ -z \"${SS_PASSWORD+x}\" ] || exit 88\nsleeper=\ntrap '" + joinSleeper + "echo stopped-$0 >> \"$LOG\"; exit 0' TERM INT\necho ready-$(basename \"$0\") >> \"$LOG\"\nwhile :; do\ncase \"$MODE:$0\" in ss*:*/ss-local) if grep -q managed-monitor \"$LOG\"; then echo stopped-$0 >> \"$LOG\"; exit 7; fi;; esac\nsleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n"
|
||||||
write("ss-server", child)
|
write("ss-server", child)
|
||||||
write("ss-local", child)
|
write("ss-local", child)
|
||||||
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nstage=$1\n"+startupFailure+"if [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; while :; do sleep 0.1; done; fi\n")
|
zapret := write("zapret", "#!/bin/sh\necho zapret-$1 >> \"$LOG\"\nstage=$1\n"+startupFailure+"if [ \"$MODE:$1\" = ss-early:start ] && [ \"$(grep -c zapret-start \"$LOG\")\" -gt 1 ]; then\nuntil [ \"$(grep -c 'stopped-.*/ss-local' \"$LOG\")\" -ge \"$(grep -c zapret-start \"$LOG\")\" ]; do sleep 0.02; done\nsleep 0.02\nfi\nif [ \"$MODE:$1\" = stop-failure:stop ]; then exit 1; fi\nif [ \"$MODE:$1\" = stop-hang:stop ]; then trap '' TERM INT; exec sleep 300; fi\n")
|
||||||
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then\necho managed-$2 >> \"$LOG\"\nstage=$2\n"+startupFailure+"exit 0\nfi\ntrap 'echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; fi\necho managed-monitor >> \"$LOG\"\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & wait $!; done\n")
|
monitor := write("autotune", "#!/bin/sh\nif [ \"$2\" != monitor ]; then\necho managed-$2 >> \"$LOG\"\nstage=$2\n"+startupFailure+"exit 0\nfi\nsleeper=\ntrap '"+joinSleeper+"echo monitor-stopped >> \"$LOG\"; case \"$MODE\" in *unsafe) exit 78;; *unknown) exit 7;; ss-technical) exit 75;; esac; exit 0' TERM INT\necho managed-monitor >> \"$LOG\"\nif [ \"$MODE\" = term-hang ]; then trap '' TERM INT; exec sleep 300; fi\ncase \"$MODE\" in fatal) exit 78;; technical|stop-failure|stop-hang) exit 75;; panic) exit 2;; killed) kill -KILL $$;; segv) exit 139;; zero) exit 0;; unknown) exit 7;; esac\nwhile :; do sleep 0.1 & sleeper=$!; wait \"$sleeper\"; done\n")
|
||||||
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
|
data, err := os.ReadFile("../../managed-target-entrypoint.sh")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
text := strings.NewReplacer("/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
|
helper, err := filepath.Abs("../../managed-target-supervision.sh")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
text := strings.NewReplacer("/opt/managed-target-supervision.sh", helper, "/run/zapret-autotune-ss", config, "/usr/local/bin/zapret-autotune", monitor, managedZapretScript, zapret).Replace(string(data))
|
||||||
if mode == "term-hang" {
|
if mode == "term-hang" {
|
||||||
// Shorten only the fixture's watchdog, not the production contract.
|
// Shorten only the fixture's watchdog, not the production contract.
|
||||||
text = strings.Replace(text, `bounded_wait "$monitor_pid" 60`, `bounded_wait "$monitor_pid" 1`, 1)
|
shortened := strings.Replace(text, `bounded_wait "$monitor_pid" 45`, `bounded_wait "$monitor_pid" 1`, 1)
|
||||||
|
if shortened == text {
|
||||||
|
t.Fatal("fixture did not shorten monitor watchdog")
|
||||||
|
}
|
||||||
|
text = shortened
|
||||||
}
|
}
|
||||||
if mode == "stop-hang" {
|
if mode == "stop-hang" {
|
||||||
text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1)
|
text = strings.Replace(text, "timeout -k 5 30", "timeout -k 1 1", 1)
|
||||||
@@ -524,6 +667,21 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
|||||||
cmd := exec.CommandContext(ctx, "/bin/sh", entry)
|
cmd := exec.CommandContext(ctx, "/bin/sh", entry)
|
||||||
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "ZAPRET_CONFIG="+filepath.Join(root, "config"), "SS_PASSWORD="+strings.Repeat("a", 32))
|
cmd.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "LOG="+log, "MODE="+mode, "ZAPRET_CONFIG="+filepath.Join(root, "config"), "SS_PASSWORD="+strings.Repeat("a", 32))
|
||||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||||
|
var shutdownSnapshot string
|
||||||
|
cmd.Cancel = func() error {
|
||||||
|
// Snapshot only this fixture's process group before deadline cleanup;
|
||||||
|
// no environment or private credential/config contents are collected.
|
||||||
|
processes, _ := exec.Command("ps", "-eo", "pid,ppid,pgid,stat,wchan:24,args").Output()
|
||||||
|
for _, line := range strings.Split(string(processes), "\n") {
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) >= 3 && fields[2] == strconv.Itoa(cmd.Process.Pid) {
|
||||||
|
shutdownSnapshot += line + "\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fixtureLog, _ := os.ReadFile(log)
|
||||||
|
shutdownSnapshot += "fixture log:\n" + string(fixtureLog)
|
||||||
|
return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
||||||
|
}
|
||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
cmd.Stdout, cmd.Stderr = &output, &output
|
cmd.Stdout, cmd.Stderr = &output, &output
|
||||||
if err := cmd.Start(); err != nil {
|
if err := cmd.Start(); err != nil {
|
||||||
@@ -553,9 +711,11 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
|||||||
}
|
}
|
||||||
err = <-done
|
err = <-done
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
t.Fatal("entrypoint did not join children", ctx.Err())
|
t.Fatal("entrypoint did not join children", ctx.Err(), shutdownSnapshot, output.String())
|
||||||
}
|
}
|
||||||
want := 75
|
// Repeated technical failures are supervised inside PID 1; after three
|
||||||
|
// starts the budget fails closed, rather than exporting exit 75.
|
||||||
|
want := 78
|
||||||
if strings.HasPrefix(mode, "startup-") {
|
if strings.HasPrefix(mode, "startup-") {
|
||||||
want = 78
|
want = 78
|
||||||
}
|
}
|
||||||
@@ -579,6 +739,14 @@ func TestManagedEntrypointSupervision(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
text = string(b)
|
text = string(b)
|
||||||
|
retrying := mode == "technical" || mode == "panic" || mode == "killed" || mode == "segv" || mode == "zero" || mode == "ss" || mode == "ss-early"
|
||||||
|
starts := strings.Count(text, "zapret-start\n")
|
||||||
|
if retrying && starts != 3 || !retrying && !strings.HasPrefix(mode, "startup-") && starts != 1 {
|
||||||
|
t.Fatal("unexpected internal restart count", starts, text)
|
||||||
|
}
|
||||||
|
if mode == "ss-early" && strings.Count(text, "managed-monitor\n") != 1 {
|
||||||
|
t.Fatal("forked monitor after replacement SS child already exited", text)
|
||||||
|
}
|
||||||
if strings.HasPrefix(mode, "startup-") {
|
if strings.HasPrefix(mode, "startup-") {
|
||||||
if strings.Contains(text, "managed-monitor") || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-server")) || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-local")) {
|
if strings.Contains(text, "managed-monitor") || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-server")) || !strings.Contains(text, "stopped-"+filepath.Join(root, "ss-local")) {
|
||||||
t.Fatal("startup failure escaped cleanup", text)
|
t.Fatal("startup failure escaped cleanup", text)
|
||||||
|
|||||||
@@ -107,6 +107,7 @@ func (e *managedRecoveryTimeout) Error() string { return e.Err.Error() }
|
|||||||
func (e *managedRecoveryTimeout) Unwrap() error { return e.Err }
|
func (e *managedRecoveryTimeout) Unwrap() error { return e.Err }
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
errNoQualifiedStrategy = errors.New("no fully passing check strategy for both tls12 and tls13")
|
||||||
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
|
errManagedGGCUnresolved = errors.New("managed GGC scope unresolved")
|
||||||
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
|
errManagedGGCScopeChanged = errors.New("managed GGC scope changed or missing")
|
||||||
errManagedRecoveryDeadline = errors.New("managed recovery total deadline exceeded")
|
errManagedRecoveryDeadline = errors.New("managed recovery total deadline exceeded")
|
||||||
@@ -420,6 +421,9 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
|||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return result, e
|
return result, e
|
||||||
}
|
}
|
||||||
|
if errors.Is(e, errNoQualifiedStrategy) {
|
||||||
|
return result, e
|
||||||
|
}
|
||||||
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
|
result.GGCError = fmt.Errorf("GGC validation or activation unresolved")
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
@@ -474,7 +478,13 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos
|
|||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
started := time.Now()
|
started := time.Now()
|
||||||
path, err := d.scan(ctx, "blockcheckw", host, io.Discard, io.Discard)
|
var diagnostics bytes.Buffer
|
||||||
|
path, err := d.scan(ctx, "blockcheckw", host, io.Discard, &diagnostics)
|
||||||
|
// Only validated report evidence, never a runner error, authorizes absence.
|
||||||
|
if errors.Is(err, errNoQualifiedStrategy) || errors.Is(err, errCandidateNonpass) {
|
||||||
|
err = fmt.Errorf("candidate scanner failed: %v", err)
|
||||||
|
}
|
||||||
|
err = managedCheckerError(err, path+".check.json.stderr", diagnostics.Bytes())
|
||||||
var report []byte
|
var report []byte
|
||||||
var args, output string
|
var args, output string
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -491,9 +501,16 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos
|
|||||||
}
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, host)
|
args, err = selectBlockcheckArgsForTarget(bytes.NewReader(report), "", group, host)
|
||||||
|
if err != nil && emptyManagedScanReport(report, host) {
|
||||||
|
err = errNoQualifiedStrategy
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
report, err = revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, d.check)
|
report, err = revalidateCandidate(ctx, storedCandidate{Group: group, Endpoints: []string{host}, Args: args, Qualified: true}, group, host, path, io.Discard, d.check)
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if !errors.As(err, &fatal) && errors.Is(err, errCandidateNonpass) {
|
||||||
|
err = errNoQualifiedStrategy
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = ctx.Err()
|
err = ctx.Err()
|
||||||
@@ -524,6 +541,21 @@ func managedScanCandidate(ctx context.Context, d managedRecoveryDeps, group, hos
|
|||||||
return id, nil
|
return id, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A successful owned scan may omit rejected rows. Missing/null metadata is not
|
||||||
|
// absence evidence; qualification still requires the usual full TLS checks.
|
||||||
|
func emptyManagedScanReport(raw []byte, host string) bool {
|
||||||
|
var report struct {
|
||||||
|
Domain string `json:"domain"`
|
||||||
|
Total *int `json:"total"`
|
||||||
|
Working *int `json:"working"`
|
||||||
|
Strategies *[]json.RawMessage `json:"strategies"`
|
||||||
|
}
|
||||||
|
d := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
return d.Decode(&report) == nil && d.Decode(new(json.RawMessage)) == io.EOF &&
|
||||||
|
report.Domain == host && report.Total != nil && *report.Total > 0 &&
|
||||||
|
report.Working != nil && *report.Working == 0 && report.Strategies != nil && len(*report.Strategies) == 0
|
||||||
|
}
|
||||||
|
|
||||||
// Installs callbacks. runManagedMonitor performs startup
|
// Installs callbacks. runManagedMonitor performs startup
|
||||||
// under its singleton. Target guards are repeated before every recovery/probe.
|
// under its singleton. Target guards are repeated before every recovery/probe.
|
||||||
func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDeps, reportGGC func(error)) error {
|
func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDeps, reportGGC func(error)) error {
|
||||||
@@ -567,6 +599,14 @@ func wireManagedRecovery(monitor *managedMonitorDeps, recovery managedRecoveryDe
|
|||||||
if err := transaction.Reconcile(recovery.db); err != nil {
|
if err := transaction.Reconcile(recovery.db); err != nil {
|
||||||
return &managedRecoveryFatal{Err: err}
|
return &managedRecoveryFatal{Err: err}
|
||||||
}
|
}
|
||||||
|
state, err := readMonitorState(monitor.db, monitor.serviceID)
|
||||||
|
if err != nil {
|
||||||
|
return &managedRecoveryFatal{Err: err}
|
||||||
|
}
|
||||||
|
// Reconciliation and identity guards precede even a paused bootstrap.
|
||||||
|
if state.RecoveryRunning || monitor.now().Before(state.CooldownUntil) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
ids, err := transaction.ActiveCandidates(recovery.db)
|
ids, err := transaction.ActiveCandidates(recovery.db)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &managedRecoveryFatal{Err: err}
|
return &managedRecoveryFatal{Err: err}
|
||||||
|
|||||||
@@ -86,6 +86,202 @@ func TestManagedRecoveryBootstrap(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestManagedSavedGGCInfrastructureCannotAuthorizeFallback(t *testing.T) {
|
||||||
|
for _, artifact := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprint("artifact=", artifact), func(t *testing.T) {
|
||||||
|
d, front, _ := recoveryFixture(t, true)
|
||||||
|
managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
|
||||||
|
monitor := monitorRuntimeFixture(t)
|
||||||
|
monitor.db, monitor.target.directory = d.db, d.transaction.Dir
|
||||||
|
monitor.threshold = 1
|
||||||
|
if err := os.Chmod(d.transaction.Dir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||||
|
if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d.scan = func(context.Context, string, string, io.Writer, io.Writer) (string, error) {
|
||||||
|
t.Fatal("infrastructure failure authorized fallback scan")
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
checks := 0
|
||||||
|
d.check = func(_ context.Context, _, path, _, host string, out, diagnostics io.Writer) error {
|
||||||
|
checks++
|
||||||
|
if host != "fresh.googlevideo.com" || out != io.Discard {
|
||||||
|
t.Fatal("unexpected saved check")
|
||||||
|
}
|
||||||
|
const infra = "FAIL 0/1 timeout: nfqws2 exited immediately (code 1) private-secret"
|
||||||
|
if artifact {
|
||||||
|
if err := os.WriteFile(path+".combined.json.stderr", []byte(infra), 0600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
io.WriteString(diagnostics, infra)
|
||||||
|
}
|
||||||
|
return os.WriteFile(path+".combined.json", nonpassingManagedReport(t), 0600)
|
||||||
|
}
|
||||||
|
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("infrastructure became unresolved GGC") }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
monitor.probe = func(context.Context) error { return errMonitorTestUnhealthy }
|
||||||
|
monitor.reportRecoveryError = func(err error) {
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if !errors.As(err, &fatal) || errors.Is(err, errNoQualifiedStrategy) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err := runManagedMonitor(context.Background(), monitor)
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if !errors.As(err, &fatal) || errors.Is(err, errCandidateNonpass) || strings.Contains(err.Error(), "private-secret") || checks != 1 {
|
||||||
|
t.Fatalf("checks=%d err=%v", checks, err)
|
||||||
|
}
|
||||||
|
state, stateErr := readMonitorState(d.db, monitor.serviceID)
|
||||||
|
if stateErr != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(monitor.cooldown)) {
|
||||||
|
t.Fatalf("%+v %v", state, stateErr)
|
||||||
|
}
|
||||||
|
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func nonpassingManagedReport(t *testing.T) []byte {
|
||||||
|
t.Helper()
|
||||||
|
var report struct {
|
||||||
|
Strategies []map[string]any `json:"strategies"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(combinedReportFixture("--lua-desync=fake")), &report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, row := range report.Strategies {
|
||||||
|
row["passes_ok"], row["success_rate"] = 0, 0
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(report)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedAbsenceClassification(t *testing.T) {
|
||||||
|
for _, mode := range []string{"shortlist", "combined", "malformed", "empty", "empty-valid", "empty-nfq", "empty-wrong-target", "runner"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
d, _, _ := recoveryFixture(t, false)
|
||||||
|
raw := nonpassingManagedReport(t)
|
||||||
|
d.scan = func(_ context.Context, _, host string, _, diagnostics io.Writer) (string, error) {
|
||||||
|
if mode == "runner" {
|
||||||
|
return "", errCandidateNonpass
|
||||||
|
}
|
||||||
|
data := raw
|
||||||
|
switch mode {
|
||||||
|
case "combined":
|
||||||
|
data = []byte(combinedReportFixture("--lua-desync=fake"))
|
||||||
|
case "malformed":
|
||||||
|
data = []byte(`{"strategies":[{"protocol":"tls12"}]}`)
|
||||||
|
case "empty":
|
||||||
|
data = []byte(`{"strategies":[]}`)
|
||||||
|
case "empty-valid", "empty-nfq", "empty-wrong-target":
|
||||||
|
if mode == "empty-nfq" {
|
||||||
|
fmt.Fprintln(diagnostics, "nfqws2 exited with status 1")
|
||||||
|
}
|
||||||
|
if mode == "empty-wrong-target" {
|
||||||
|
host = "untrusted.example"
|
||||||
|
}
|
||||||
|
data = []byte(fmt.Sprintf(`{"domain":%q,"total":10,"working":0,"strategies":[]}`, host))
|
||||||
|
}
|
||||||
|
path := filepath.Join(t.TempDir(), "scan")
|
||||||
|
return path, os.WriteFile(path+".check.json", data, 0600)
|
||||||
|
}
|
||||||
|
d.check = func(_ context.Context, _, path, _, _ string, _, _ io.Writer) error {
|
||||||
|
return os.WriteFile(path+".combined.json", raw, 0600)
|
||||||
|
}
|
||||||
|
_, err := recoverManagedTarget(context.Background(), d)
|
||||||
|
want := mode == "shortlist" || mode == "combined" || mode == "empty-valid"
|
||||||
|
if err == nil || errors.Is(err, errNoQualifiedStrategy) != want {
|
||||||
|
t.Fatalf("classification: %v", err)
|
||||||
|
}
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if errors.As(err, &fatal) && mode != "empty-nfq" {
|
||||||
|
t.Fatalf("unexpected fatal: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedBootstrapAbsencePauseRestartAndExpiry(t *testing.T) {
|
||||||
|
d, _, _ := recoveryFixture(t, false)
|
||||||
|
monitor := monitorRuntimeFixture(t)
|
||||||
|
monitor.db, monitor.target.directory = d.db, d.transaction.Dir
|
||||||
|
if err := os.Chmod(d.transaction.Dir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
identity := `{"Role":"managed-test","Directory":"` + d.transaction.Dir + `","ID":"` + strings.Repeat("01", 16) + `"}`
|
||||||
|
if err := d.transaction.replace(managedMarkerName, []byte(identity)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
now := monitorTestTime
|
||||||
|
monitor.now = func() time.Time { return now }
|
||||||
|
scans := 0
|
||||||
|
d.scan = func(_ context.Context, _, _ string, _, _ io.Writer) (string, error) {
|
||||||
|
scans++
|
||||||
|
path := filepath.Join(t.TempDir(), "scan")
|
||||||
|
return path, os.WriteFile(path+".check.json", nonpassingManagedReport(t), 0600)
|
||||||
|
}
|
||||||
|
d.health = func(context.Context) (string, error) { return "UNHEALTHY", nil }
|
||||||
|
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("unexpected GGC error") }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
monitor.reportRecoveryError = func(err error) {
|
||||||
|
if !errors.Is(err, errNoQualifiedStrategy) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for phase, elapsed := range []time.Duration{0, time.Hour, 2 * time.Hour} {
|
||||||
|
now = monitorTestTime.Add(elapsed)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
probes := 0
|
||||||
|
monitor.wait = func(ctx context.Context, _ time.Duration) error {
|
||||||
|
probes++
|
||||||
|
if probes == 4 {
|
||||||
|
cancel()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := runManagedMonitor(ctx, monitor); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
wantScans := 1
|
||||||
|
if phase == 2 {
|
||||||
|
wantScans = 2
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(monitor.db, monitor.serviceID)
|
||||||
|
if err != nil || scans != wantScans || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(time.Duration(wantScans)*2*time.Hour)) {
|
||||||
|
t.Fatalf("phase=%d scans=%d state=%+v err=%v", phase, scans, state, err)
|
||||||
|
}
|
||||||
|
// Reopen the same durable database before the next monitor instance.
|
||||||
|
var dbPath string
|
||||||
|
var seq int
|
||||||
|
var name string
|
||||||
|
if err := monitor.db.QueryRow(`PRAGMA database_list`).Scan(&seq, &name, &dbPath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := monitor.db.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reopened, err := openStore(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { reopened.Close() })
|
||||||
|
monitor.db, d.db = reopened, reopened
|
||||||
|
if err := wireManagedRecovery(&monitor, d, func(error) { t.Fatal("unexpected GGC error") }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestManagedRecoveryJobDeadline(t *testing.T) {
|
func TestManagedRecoveryJobDeadline(t *testing.T) {
|
||||||
for _, mode := range []string{"bootstrap-scan", "bootstrap-saved", "periodic-saved", "periodic-apply", "periodic-ggc", "parent-cancel", "parent-deadline"} {
|
for _, mode := range []string{"bootstrap-scan", "bootstrap-saved", "periodic-saved", "periodic-apply", "periodic-ggc", "parent-cancel", "parent-deadline"} {
|
||||||
t.Run(mode, func(t *testing.T) {
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
|||||||
@@ -198,8 +198,8 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
|
|||||||
if err := validateMonitorTime(serviceID, expires); err != nil {
|
if err := validateMonitorTime(serviceID, expires); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
|
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL, recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = MAX(COALESCE(cooldown_until, ?), ?)
|
||||||
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), serviceID, token)
|
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?`, expires.UnixNano(), expires.UnixNano(), serviceID, token)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -210,6 +210,23 @@ func finishMonitorRecovery(db *sql.DB, serviceID, token string, completed time.T
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Extend bootstrap backoff without changing probes or any recovery ownership.
|
||||||
|
func pauseMonitorRecovery(db *sql.DB, serviceID string, now time.Time, cooldown time.Duration) error {
|
||||||
|
if err := validateMonitorTime(serviceID, now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if cooldown <= 0 {
|
||||||
|
return fmt.Errorf("recovery cooldown must be positive")
|
||||||
|
}
|
||||||
|
expires := now.Add(cooldown)
|
||||||
|
if err := validateMonitorTime(serviceID, expires); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := db.Exec(`INSERT INTO monitor_state (service_id, cooldown_until) VALUES (?, ?)
|
||||||
|
ON CONFLICT(service_id) DO UPDATE SET cooldown_until = MAX(COALESCE(monitor_state.cooldown_until, excluded.cooldown_until), excluded.cooldown_until)`, serviceID, expires.UnixNano())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// Startup policy only: call for each configured service before monitoring.
|
// Startup policy only: call for each configured service before monitoring.
|
||||||
// A legacy/indeterminate owner is an error, never permission to unlock.
|
// A legacy/indeterminate owner is an error, never permission to unlock.
|
||||||
// The conditional write also protects against a finish/reacquire during the read.
|
// The conditional write also protects against a finish/reacquire during the read.
|
||||||
@@ -251,9 +268,9 @@ func reconcileMonitorRecovery(db *sql.DB, serviceID string, cooldown time.Durati
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL,
|
result, err := db.Exec(`UPDATE monitor_state SET recovery_running = 0, recovery_token = NULL,
|
||||||
recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = ?
|
recovery_owner_pid = NULL, recovery_owner_starttime = NULL, cooldown_until = MAX(COALESCE(cooldown_until, ?), ?)
|
||||||
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?
|
WHERE service_id = ? AND recovery_running = 1 AND recovery_token = ?
|
||||||
AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), serviceID, token, pid.Int64, start.Int64)
|
AND recovery_owner_pid = ? AND recovery_owner_starttime = ?`, expires.UnixNano(), expires.UnixNano(), serviceID, token, pid.Int64, start.Int64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,16 @@ import (
|
|||||||
|
|
||||||
const managedMonitorLockName = ".autotune-monitor-lock"
|
const managedMonitorLockName = ".autotune-monitor-lock"
|
||||||
|
|
||||||
|
const noQualifiedStrategyCooldown = 2 * time.Hour
|
||||||
|
|
||||||
|
func managedRecoveryCooldown(err error, configured time.Duration) time.Duration {
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if !errors.As(err, &fatal) && errors.Is(err, errNoQualifiedStrategy) && configured < noQualifiedStrategyCooldown {
|
||||||
|
return noQualifiedStrategyCooldown
|
||||||
|
}
|
||||||
|
return configured
|
||||||
|
}
|
||||||
|
|
||||||
// Callbacks must honor ctx and join
|
// Callbacks must honor ctx and join
|
||||||
// their children before returning; the supervisor never detaches callbacks.
|
// their children before returning; the supervisor never detaches callbacks.
|
||||||
type managedMonitorDeps struct {
|
type managedMonitorDeps struct {
|
||||||
@@ -130,6 +140,11 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
|||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
|
if errors.Is(err, errNoQualifiedStrategy) {
|
||||||
|
if err := pauseMonitorRecovery(d.db, d.serviceID, d.now(), managedRecoveryCooldown(err, d.cooldown)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
// Bootstrap scans can validly find no qualified strategy. Keep the
|
// Bootstrap scans can validly find no qualified strategy. Keep the
|
||||||
// supervisor alive so durable probes/cooldown govern the next attempt.
|
// supervisor alive so durable probes/cooldown govern the next attempt.
|
||||||
d.reportRecoveryError(err)
|
d.reportRecoveryError(err)
|
||||||
@@ -157,8 +172,9 @@ func runManagedMonitor(ctx context.Context, d managedMonitorDeps) (err error) {
|
|||||||
}
|
}
|
||||||
if token != "" {
|
if token != "" {
|
||||||
recoveryErr := d.recover(ctx)
|
recoveryErr := d.recover(ctx)
|
||||||
|
cooldown := managedRecoveryCooldown(recoveryErr, d.cooldown)
|
||||||
// Finalize even on cancellation, using the non-context DB operation.
|
// Finalize even on cancellation, using the non-context DB operation.
|
||||||
if err := finishMonitorRecovery(d.db, d.serviceID, token, d.now(), d.cooldown); err != nil {
|
if err := finishMonitorRecovery(d.db, d.serviceID, token, d.now(), cooldown); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if recoveryErr != nil {
|
if recoveryErr != nil {
|
||||||
|
|||||||
@@ -11,6 +11,95 @@ import (
|
|||||||
|
|
||||||
var errMonitorTestUnhealthy = errors.New("unhealthy")
|
var errMonitorTestUnhealthy = errors.New("unhealthy")
|
||||||
|
|
||||||
|
func TestManagedMonitorBootstrapConfiguredLongerPause(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
d.cooldown = 4 * time.Hour
|
||||||
|
d.bootstrap = func(context.Context) error { return errNoQualifiedStrategy }
|
||||||
|
d.reportRecoveryError = func(err error) {
|
||||||
|
if !errors.Is(err, errNoQualifiedStrategy) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
d.wait = func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() }
|
||||||
|
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(d.db, d.serviceID)
|
||||||
|
if err != nil || !state.CooldownUntil.Equal(monitorTestTime.Add(4*time.Hour)) {
|
||||||
|
t.Fatalf("%+v %v", state, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedMonitorPeriodicAbsenceCooldown(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
d.threshold = 1
|
||||||
|
now := monitorTestTime
|
||||||
|
d.now = func() time.Time { return now }
|
||||||
|
d.probe = func(context.Context) error { return errMonitorTestUnhealthy }
|
||||||
|
recoveries := 0
|
||||||
|
d.recover = func(context.Context) error { recoveries++; return errNoQualifiedStrategy }
|
||||||
|
d.reportRecoveryError = func(err error) {
|
||||||
|
if !errors.Is(err, errNoQualifiedStrategy) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
waits := 0
|
||||||
|
d.wait = func(ctx context.Context, _ time.Duration) error {
|
||||||
|
waits++
|
||||||
|
state, err := readMonitorState(d.db, d.serviceID)
|
||||||
|
want := monitorTestTime.Add(2 * time.Hour)
|
||||||
|
if waits == 3 {
|
||||||
|
want = monitorTestTime.Add(4 * time.Hour)
|
||||||
|
}
|
||||||
|
if err != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(want) {
|
||||||
|
t.Fatalf("%+v %v", state, err)
|
||||||
|
}
|
||||||
|
if waits == 3 {
|
||||||
|
cancel()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
now = now.Add(time.Hour)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := runManagedMonitor(ctx, d); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if recoveries != 2 {
|
||||||
|
t.Fatalf("recoveries=%d", recoveries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorPausePreservesLongerCooldownAndLease(t *testing.T) {
|
||||||
|
d := monitorRuntimeFixture(t)
|
||||||
|
if err := recordMonitorProbe(d.db, d.serviceID, false, monitorTestTime); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
token, err := beginMonitorRecovery(d.db, d.serviceID, 1, monitorTestTime)
|
||||||
|
if err != nil || token == "" {
|
||||||
|
t.Fatalf("token=%s err=%v", token, err)
|
||||||
|
}
|
||||||
|
for _, duration := range []time.Duration{4 * time.Hour, 2 * time.Hour} {
|
||||||
|
if err := pauseMonitorRecovery(d.db, d.serviceID, monitorTestTime, duration); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
state, err := readMonitorState(d.db, d.serviceID)
|
||||||
|
if err != nil || !state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(4*time.Hour)) {
|
||||||
|
t.Fatalf("%+v %v", state, err)
|
||||||
|
}
|
||||||
|
if err := finishMonitorRecovery(d.db, d.serviceID, token, monitorTestTime, 30*time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err = readMonitorState(d.db, d.serviceID)
|
||||||
|
if err != nil || state.RecoveryRunning || !state.CooldownUntil.Equal(monitorTestTime.Add(4*time.Hour)) {
|
||||||
|
t.Fatalf("%+v %v", state, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Exercise the production process reader under the real singleton. PID reuse
|
// Exercise the production process reader under the real singleton. PID reuse
|
||||||
// proves the old owner is gone and clears only that lease; a live owner stays
|
// proves the old owner is gone and clears only that lease; a live owner stays
|
||||||
// busy, while an owner without v5 identity makes startup fail closed.
|
// busy, while an owner without v5 identity makes startup fail closed.
|
||||||
@@ -307,7 +396,7 @@ func TestManagedMonitorStartupLeaseAfterReopenStartsCrashCooldown(t *testing.T)
|
|||||||
if err != nil || oldToken == "" {
|
if err != nil || oldToken == "" {
|
||||||
t.Fatalf("token=%q err=%v", oldToken, err)
|
t.Fatalf("token=%q err=%v", oldToken, err)
|
||||||
}
|
}
|
||||||
expires := monitorTestTime.Add(d.cooldown)
|
expires := monitorTestTime.Add(time.Hour)
|
||||||
if _, err := db.Exec(`UPDATE monitor_state SET cooldown_until = ? WHERE service_id = ?`, monitorTestTime.Add(time.Hour).UnixNano(), d.serviceID); err != nil {
|
if _, err := db.Exec(`UPDATE monitor_state SET cooldown_until = ? WHERE service_id = ?`, monitorTestTime.Add(time.Hour).UnixNano(), d.serviceID); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -61,10 +61,11 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
|
|||||||
return nil, fmt.Errorf("candidate diagnostic artifact already exists or cannot be inspected: %v", err)
|
return nil, fmt.Errorf("candidate diagnostic artifact already exists or cannot be inspected: %v", err)
|
||||||
}
|
}
|
||||||
var raw bytes.Buffer
|
var raw bytes.Buffer
|
||||||
err := check(ctx, "blockcheckw", path, c.Args, host, &raw, &raw)
|
err := check(ctx, "blockcheckw", path, c.Args, host, io.Discard, &raw)
|
||||||
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
|
if e := os.WriteFile(path+".stdout", raw.Bytes(), 0600); e != nil {
|
||||||
return nil, e
|
return nil, e
|
||||||
}
|
}
|
||||||
|
err = managedCheckerError(err, path+".combined.json.stderr", raw.Bytes())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Only our report parser may issue the nonpass authorization.
|
// Only our report parser may issue the nonpass authorization.
|
||||||
if errors.Is(err, errCandidateNonpass) {
|
if errors.Is(err, errCandidateNonpass) {
|
||||||
@@ -91,6 +92,27 @@ func revalidateCandidate(ctx context.Context, c storedCandidate, group, host, pa
|
|||||||
return report, err
|
return report, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check infrastructure before interpreting any positive or negative evidence.
|
||||||
|
// Prefer the owned invocation's artifact over earlier scan/seed diagnostics;
|
||||||
|
// captured stderr is private, and stdout never shares its buffer.
|
||||||
|
func managedCheckerError(err error, artifact string, captured []byte) error {
|
||||||
|
var fatal *managedRecoveryFatal
|
||||||
|
if errors.As(err, &fatal) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
diagnostics, readErr := os.ReadFile(artifact)
|
||||||
|
if readErr != nil {
|
||||||
|
if !errors.Is(readErr, os.ErrNotExist) {
|
||||||
|
return &managedRecoveryFatal{Err: fmt.Errorf("cannot inspect owned checker diagnostics")}
|
||||||
|
}
|
||||||
|
diagnostics = captured
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.ToLower(string(diagnostics)), "nfqws2 exited") {
|
||||||
|
return &managedRecoveryFatal{Err: fmt.Errorf("checker nfqws2 worker exited unexpectedly")}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
var earlyTimeoutResult = regexp.MustCompile(`^ FAIL ([0-2])/([1-3]) timeout$`)
|
var earlyTimeoutResult = regexp.MustCompile(`^ FAIL ([0-2])/([1-3]) timeout$`)
|
||||||
var earlySuccessResult = regexp.MustCompile(`^ OK median [0-9]+ms, [0-9]+\.[0-9] KB/s$`)
|
var earlySuccessResult = regexp.MustCompile(`^ OK median [0-9]+ms, [0-9]+\.[0-9] KB/s$`)
|
||||||
var earlyCheckPreamble = regexp.MustCompile("^(! --workers has no effect on `check` \\(it verifies strategies sequentially by design\\)\n)?" +
|
var earlyCheckPreamble = regexp.MustCompile("^(! --workers has no effect on `check` \\(it verifies strategies sequentially by design\\)\n)?" +
|
||||||
|
|||||||
@@ -211,7 +211,7 @@ func selectBlockcheckEvidence(r io.Reader, expected, group, host string, recover
|
|||||||
for _, protocol := range []string{"tls12", "tls13"} {
|
for _, protocol := range []string{"tls12", "tls13"} {
|
||||||
c, ok := best[protocol]
|
c, ok := best[protocol]
|
||||||
if !ok {
|
if !ok {
|
||||||
return "", fmt.Errorf("no fully passing check strategy for both tls12 and tls13")
|
return "", errNoQualifiedStrategy
|
||||||
}
|
}
|
||||||
if protocol == "tls13" && c.Args == best["tls12"].Args {
|
if protocol == "tls13" && c.Args == best["tls12"].Args {
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -34,36 +34,34 @@ in logs or shared terminals.
|
|||||||
By default Compose uses the locally built `zapret-autotune-managed-target:local`.
|
By default Compose uses the locally built `zapret-autotune-managed-target:local`.
|
||||||
Tags are only local build/test candidates, not production release references.
|
Tags are only local build/test candidates, not production release references.
|
||||||
For production promotion, set `ZAPRET_AUTOTUNE_IMAGE=<registry>/<managed-target>@sha256:<digest>`
|
For production promotion, set `ZAPRET_AUTOTUNE_IMAGE=<registry>/<managed-target>@sha256:<digest>`
|
||||||
in the private `.env` and pull that immutable image before starting the unit.
|
in the private `.env` and pull that immutable image before starting the target.
|
||||||
Use a managed target image, not the generic image; the systemd command never
|
Use a managed target image, not the generic image. Do not deploy production
|
||||||
builds images. Do not enable the production service with a tag or local default.
|
with a tag or local default.
|
||||||
|
|
||||||
Offline static guard (from repository root):
|
Offline static guard (from repository root):
|
||||||
`python3 deploy/managed-target/check_static.py`.
|
`python3 deploy/managed-target/check_static.py`.
|
||||||
|
Isolated, accelerated shell supervision fixtures (no network):
|
||||||
|
`python3 deploy/managed-target/check_supervision.py`.
|
||||||
|
|
||||||
## Optional native systemd supervision
|
## In-container supervision
|
||||||
|
|
||||||
For unattended boot, place this checkout at `/opt/zapret2-autotune`, provision
|
Compose has no automatic restart. PID 1 restarts the entire owned runtime only
|
||||||
`deploy/managed-target/.env` and state as above, and build the target image first.
|
for technical failures, with three starts per rolling six hours (including initial
|
||||||
For production, select and pull the immutable managed-target digest as above
|
startup) and cancellable 15/30-second cooldowns. Clean typed recovery timeouts,
|
||||||
instead of using a local build tag.
|
|
||||||
Install `deploy/managed-target/zapret-autotune.service` in `/etc/systemd/system/`,
|
|
||||||
then run `systemctl daemon-reload` and `systemctl enable --now zapret-autotune`.
|
|
||||||
Do not simultaneously start the same project with detached Compose.
|
|
||||||
Enabling the unit starts this isolated target on subsequent boots. `ExecStop`
|
|
||||||
only stops `target`; it does not replace the foreground process's failure code.
|
|
||||||
An explicit service stop suppresses restart, including during a shutdown race.
|
|
||||||
|
|
||||||
Compose has no automatic restart. The foreground systemd unit restarts **only
|
|
||||||
exit 75**, with at most three starts per six hours. Clean typed recovery timeouts,
|
|
||||||
unexpected monitor exits/crashes, and Shadowsocks failures qualify only if
|
unexpected monitor exits/crashes, and Shadowsocks failures qualify only if
|
||||||
shutdown cleanup succeeds. Configuration (64), safety/database/rollback/cleanup
|
shutdown cleanup succeeds. Configuration (64), safety/database/rollback/cleanup
|
||||||
failures (78), and graceful SIGTERM (0) never request restart. Investigate and
|
failures (78), and graceful SIGTERM (0) never request restart. Exhausting the
|
||||||
|
restart budget exits 78. Investigate and
|
||||||
repair nonretry failures before manually starting again; do not clear state.
|
repair nonretry failures before manually starting again; do not clear state.
|
||||||
Monitor cancellation/join is bounded to 60 seconds, final target-local zapret
|
Monitor cancellation/join is bounded to 45 seconds, reconciliation gets 45 seconds
|
||||||
|
plus five to force kill, and final target-local zapret
|
||||||
stop gets 30 seconds then forced kill after five more, and each Shadowsocks join
|
stop gets 30 seconds then forced kill after five more, and each Shadowsocks join
|
||||||
gets five seconds. Exceeding a join bound fails closed. Compose grants 120
|
gets five seconds. Exceeding a join bound fails closed. Compose grants 240
|
||||||
seconds; systemd grants 180 seconds.
|
seconds. Reconcile after joining the monitor and before even stopping zapret:
|
||||||
|
stop also sources config, so unknown state is never sourced. The sequential
|
||||||
|
cleanup budget is 140 seconds; 240 seconds also covers an in-flight startup
|
||||||
|
command and scheduling grace. Watchdogs run in fresh shell interpreters and
|
||||||
|
cancel/join their own sleepers, including during PID 1's EXIT trap.
|
||||||
|
|
||||||
## Isolation and runtime contract
|
## Isolation and runtime contract
|
||||||
|
|
||||||
@@ -97,8 +95,9 @@ Any startup command failure/timeout exits 78, never the retry code 75. These
|
|||||||
bounds do not shorten the monitor's 90-minute bootstrap/recovery scan budget.
|
bounds do not shorten the monitor's 90-minute bootstrap/recovery scan budget.
|
||||||
|
|
||||||
SIGTERM/SIGINT cancels and waits for the monitor before stopping target-local
|
SIGTERM/SIGINT cancels and waits for the monitor before stopping target-local
|
||||||
zapret and Shadowsocks children. Monitor fatal exits stop the
|
zapret and Shadowsocks children. Nonretry monitor fatal exits stop the
|
||||||
entrypoint. Monitor health is YouTube Player API status through the owned SOCKS,
|
entrypoint; technical exits and Shadowsocks failures restart from private configs
|
||||||
|
only after safe cleanup. Monitor health is YouTube Player API status through the owned SOCKS,
|
||||||
not video playback/SABR; GGC candidates are scoped to the exact discovered
|
not video playback/SABR; GGC candidates are scoped to the exact discovered
|
||||||
hostname.
|
hostname.
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ directory = Path(__file__).resolve().parent
|
|||||||
compose = (directory / "compose.yaml").read_text()
|
compose = (directory / "compose.yaml").read_text()
|
||||||
entrypoint = (directory.parents[1] / "managed-target-entrypoint.sh").read_text()
|
entrypoint = (directory.parents[1] / "managed-target-entrypoint.sh").read_text()
|
||||||
readme = (directory / "README.md").read_text()
|
readme = (directory / "README.md").read_text()
|
||||||
unit = (directory / "zapret-autotune.service").read_text()
|
|
||||||
|
|
||||||
# ponytail: checks the committed Compose shape, not arbitrary YAML; use a YAML
|
# ponytail: checks the committed Compose shape, not arbitrary YAML; use a YAML
|
||||||
# parser if this definition grows beyond its single target service.
|
# parser if this definition grows beyond its single target service.
|
||||||
@@ -29,19 +28,15 @@ assert "Docker administrators/root can still read container" in readme
|
|||||||
assert "removed\nfrom the entrypoint environment" in readme
|
assert "removed\nfrom the entrypoint environment" in readme
|
||||||
assert "No host networking" in readme
|
assert "No host networking" in readme
|
||||||
assert ' restart: "no"\n' in compose
|
assert ' restart: "no"\n' in compose
|
||||||
assert " stop_grace_period: 120s\n" in compose
|
assert " stop_grace_period: 240s\n" in compose
|
||||||
directives = dict(line.split("=", 1) for line in unit.splitlines() if "=" in line)
|
assert 'bounded_wait "$monitor_pid" 45' in entrypoint
|
||||||
assert directives["Restart"] == "no"
|
assert '/bin/sh /opt/managed-target-supervision.sh --watchdog' in entrypoint
|
||||||
assert directives["RestartForceExitStatus"] == "75"
|
assert 'COPY managed-target-supervision.sh /opt/managed-target-supervision.sh' in (directory.parents[1] / 'Dockerfile.target').read_text()
|
||||||
assert "RestartPreventExitStatus" not in directives
|
assert '!managed-target-supervision.sh' in (directory.parents[1] / '.dockerignore').read_text()
|
||||||
assert directives["StartLimitIntervalSec"] == "6h"
|
assert '"$count" -lt 3' in entrypoint
|
||||||
assert directives["StartLimitBurst"] == "3"
|
assert '"$((now - started))" -lt 21600' in entrypoint
|
||||||
assert directives["TimeoutStopSec"] == "180s"
|
assert "backoff=15" in entrypoint
|
||||||
assert directives["WorkingDirectory"] == "/opt/zapret2-autotune"
|
assert "systemctl" not in entrypoint and "docker " not in entrypoint
|
||||||
assert directives["WantedBy"] == "multi-user.target"
|
|
||||||
compose_command = "/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml"
|
|
||||||
assert directives["ExecStart"] == compose_command + " up --no-build --abort-on-container-exit --exit-code-from target target"
|
|
||||||
assert directives["ExecStop"] == compose_command + " stop target"
|
|
||||||
assert "timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop" in entrypoint
|
assert "timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop" in entrypoint
|
||||||
for command, seconds in (("/usr/local/bin/zapret-autotune managed init", 45),
|
for command, seconds in (("/usr/local/bin/zapret-autotune managed init", 45),
|
||||||
("/usr/local/bin/zapret-autotune managed reconcile", 45),
|
("/usr/local/bin/zapret-autotune managed reconcile", 45),
|
||||||
|
|||||||
@@ -0,0 +1,223 @@
|
|||||||
|
"""Isolated shell fixtures: python3 deploy/managed-target/check_supervision.py.
|
||||||
|
|
||||||
|
Only a temporary COPY of the entrypoint is redirected to fake executables;
|
||||||
|
production has no test switches, real network probes, or accelerated timers.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
FAKE = r'''#!/usr/bin/env python3
|
||||||
|
import os, pathlib, signal, sys, time
|
||||||
|
root = pathlib.Path(os.environ['FIXTURE'])
|
||||||
|
name = pathlib.Path(sys.argv[0]).name
|
||||||
|
scenario = os.environ['SCENARIO']
|
||||||
|
def log(message):
|
||||||
|
with (root / 'events').open('a') as f:
|
||||||
|
f.write(message + '\n')
|
||||||
|
def increment(key):
|
||||||
|
path = root / key
|
||||||
|
n = int(path.read_text()) + 1 if path.exists() else 1
|
||||||
|
path.write_text(str(n))
|
||||||
|
return n
|
||||||
|
def linger(code=0):
|
||||||
|
signal.signal(signal.SIGTERM, lambda *_: sys.exit(code))
|
||||||
|
while True:
|
||||||
|
time.sleep(.01)
|
||||||
|
if name == 'awk':
|
||||||
|
sys.exit(0)
|
||||||
|
if name == 'date':
|
||||||
|
n = increment('clock')
|
||||||
|
print(100000 + (n * 21601 if scenario == 'window' else n))
|
||||||
|
sys.exit(0)
|
||||||
|
if name == 'sleep':
|
||||||
|
seconds = int(sys.argv[1])
|
||||||
|
if seconds in (15, 30):
|
||||||
|
log('cooldown ' + str(seconds))
|
||||||
|
if scenario == 'cancel':
|
||||||
|
os.kill(os.getppid(), signal.SIGTERM)
|
||||||
|
time.sleep(1)
|
||||||
|
else:
|
||||||
|
time.sleep(.01)
|
||||||
|
else:
|
||||||
|
time.sleep(.01 if seconds == 1 else 2)
|
||||||
|
sys.exit(0)
|
||||||
|
if name == 'timeout':
|
||||||
|
os.execv(sys.argv[4], sys.argv[4:])
|
||||||
|
if name in ('ss-server', 'ss-local'):
|
||||||
|
with (root / 'children').open('a') as f:
|
||||||
|
f.write(str(os.getpid()) + '\n')
|
||||||
|
assert 'SS_PASSWORD' not in os.environ
|
||||||
|
assert sys.argv[1] == '-c' and len(sys.argv) == 3
|
||||||
|
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
|
||||||
|
log(name)
|
||||||
|
if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail') and increment('ss_runs') == 1:
|
||||||
|
# Let startup/monitor begin before simulating a child crash.
|
||||||
|
while not (root / 'monitor_runs').exists():
|
||||||
|
time.sleep(.01)
|
||||||
|
time.sleep(.03)
|
||||||
|
sys.exit(137)
|
||||||
|
linger()
|
||||||
|
if name == 'zapret2':
|
||||||
|
assert not (root / 'prepared').exists(), 'sourced prepared config'
|
||||||
|
log(sys.argv[1])
|
||||||
|
if scenario == 'stopfail' and sys.argv[1] == 'stop':
|
||||||
|
sys.exit(1)
|
||||||
|
sys.exit(0)
|
||||||
|
assert name == 'autotune'
|
||||||
|
command = sys.argv[2]
|
||||||
|
if command == 'init':
|
||||||
|
log('init')
|
||||||
|
(root / 'config').touch()
|
||||||
|
elif command == 'reconcile':
|
||||||
|
n = increment('reconciles')
|
||||||
|
log('reconcile')
|
||||||
|
if scenario == 'prestartfail':
|
||||||
|
sys.exit(78)
|
||||||
|
if scenario in ('unknown', 'unknown75') and n == 2:
|
||||||
|
sys.exit(78)
|
||||||
|
(root / 'prepared').unlink(missing_ok=True)
|
||||||
|
elif command == 'monitor':
|
||||||
|
with (root / 'children').open('a') as f:
|
||||||
|
f.write(str(os.getpid()) + '\n')
|
||||||
|
n = increment('monitor_runs')
|
||||||
|
log('monitor ' + str(n))
|
||||||
|
if scenario in ('budget', 'cancel', 'stopfail', 'unknown75', 'ackfail'):
|
||||||
|
sys.exit(75)
|
||||||
|
if scenario.startswith('fatal'):
|
||||||
|
sys.exit(int(scenario[5:]))
|
||||||
|
if n == 1 and scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'unknown', 'window'):
|
||||||
|
if scenario.startswith('crash') or scenario == 'unknown':
|
||||||
|
(root / 'prepared').touch()
|
||||||
|
code = int(scenario[5:]) if scenario.startswith('crash') and scenario != 'crash' else 2
|
||||||
|
sys.exit(code if scenario.startswith('crash') or scenario == 'unknown' else 75)
|
||||||
|
if scenario == 'joinfail':
|
||||||
|
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||||
|
while True:
|
||||||
|
time.sleep(.01)
|
||||||
|
if n == 1 and scenario in ('ss', 'mixed'):
|
||||||
|
linger(78 if scenario == 'mixed' else 0)
|
||||||
|
# Successful second runtime: external SIGTERM must exit 0, not restart.
|
||||||
|
time.sleep(.03)
|
||||||
|
os.kill(os.getppid(), signal.SIGTERM)
|
||||||
|
linger()
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def run(scenario, expected):
|
||||||
|
with tempfile.TemporaryDirectory(prefix='supervision-') as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
binary = root / 'bin'
|
||||||
|
binary.mkdir()
|
||||||
|
fake = binary / 'fake'
|
||||||
|
fake.write_text(FAKE)
|
||||||
|
fake.chmod(0o755)
|
||||||
|
for name in ('awk', 'date', 'sleep', 'timeout', 'ss-server', 'ss-local', 'autotune', 'zapret2'):
|
||||||
|
(binary / name).symlink_to(fake)
|
||||||
|
configs = root / 'private'
|
||||||
|
configs.mkdir()
|
||||||
|
text = (ROOT / 'managed-target-entrypoint.sh').read_text()
|
||||||
|
helper = ROOT / 'managed-target-supervision.sh'
|
||||||
|
if scenario == 'ackfail':
|
||||||
|
broken_helper = root / 'broken-helper.sh'
|
||||||
|
broken_helper.write_text(helper.read_text().replace('sleep "$seconds" & sleeper=$!', 'exit 78'))
|
||||||
|
helper = broken_helper
|
||||||
|
text = text.replace('/opt/managed-target-supervision.sh', str(helper))
|
||||||
|
text = text.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={configs}')
|
||||||
|
text = text.replace('/usr/local/bin/zapret-autotune', str(binary / 'autotune'))
|
||||||
|
text = text.replace('/opt/zapret2/init.d/sysv/zapret2', str(binary / 'zapret2'))
|
||||||
|
entry = root / 'entry.sh'
|
||||||
|
entry.write_text(text)
|
||||||
|
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
|
||||||
|
SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config'))
|
||||||
|
result = subprocess.run(['sh', str(entry)], env=env, capture_output=True, timeout=15)
|
||||||
|
events = (root / 'events').read_text().splitlines()
|
||||||
|
assert result.returncode == expected, (scenario, result.returncode, result.stderr, events)
|
||||||
|
assert b'ab' * 32 not in result.stdout + result.stderr
|
||||||
|
for pid in (root / 'children').read_text().splitlines():
|
||||||
|
assert not Path(f'/proc/{pid}').exists(), (scenario, 'child not joined', pid)
|
||||||
|
# Every activation and stop must immediately follow successful reconcile.
|
||||||
|
for i, event in enumerate(events):
|
||||||
|
if event in ('start', 'stop'):
|
||||||
|
assert events[i - 1] == 'reconcile', (scenario, events)
|
||||||
|
return events
|
||||||
|
|
||||||
|
|
||||||
|
for scenario in ('technical', 'crash', 'crash0', 'crash137', 'crash139', 'ss', 'window'):
|
||||||
|
events = run(scenario, 0)
|
||||||
|
assert events.count('start') == 2, (scenario, events)
|
||||||
|
assert events.count('ss-server') == 2 and events.count('ss-local') == 2
|
||||||
|
assert events.count('stop') == 2
|
||||||
|
assert events.count('cooldown 15') == 1
|
||||||
|
for code in (64, 78, 1):
|
||||||
|
events = run(f'fatal{code}', code if code in (64, 78) else 78)
|
||||||
|
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
|
||||||
|
for scenario in ('mixed', 'unknown', 'unknown75', 'stopfail', 'joinfail', 'ackfail'):
|
||||||
|
events = run(scenario, 78)
|
||||||
|
assert events.count('start') == 1 and not any(e.startswith('cooldown') for e in events)
|
||||||
|
if scenario.startswith('unknown'):
|
||||||
|
assert 'stop' not in events
|
||||||
|
events = run('budget', 78)
|
||||||
|
assert events.count('start') == 3
|
||||||
|
assert [e for e in events if e.startswith('cooldown')] == ['cooldown 15', 'cooldown 30']
|
||||||
|
events = run('cancel', 0)
|
||||||
|
assert events.count('start') == 1
|
||||||
|
events = run('prestartfail', 78)
|
||||||
|
assert 'start' not in events and 'stop' not in events
|
||||||
|
|
||||||
|
|
||||||
|
def check_watchdog_traps(shell):
|
||||||
|
"""Same bounded_wait from normal flow, plain EXIT, and TERM -> EXIT traps."""
|
||||||
|
entrypoint = (ROOT / 'managed-target-entrypoint.sh').read_text()
|
||||||
|
bounded_wait = entrypoint[entrypoint.index('bounded_wait() {'):entrypoint.index('cleanup_cycle() {')]
|
||||||
|
bounded_wait = bounded_wait.replace('/opt/managed-target-supervision.sh', str(ROOT / 'managed-target-supervision.sh'))
|
||||||
|
for mode in ('normal', 'exit', 'term-exit'):
|
||||||
|
with tempfile.TemporaryDirectory(prefix='watchdog-traps-') as tmp:
|
||||||
|
directory = Path(tmp)
|
||||||
|
sleeper = directory / 'sleep'
|
||||||
|
sleeper.write_text('#!/bin/sh\necho $$ >> "$PIDS"\nexec "' + shutil.which('sleep') + '" "$@"\n')
|
||||||
|
sleeper.chmod(0o755)
|
||||||
|
script = directory / 'check.sh'
|
||||||
|
script.write_text(f'''set -eu
|
||||||
|
mode=$1
|
||||||
|
shift
|
||||||
|
config_dir={directory}
|
||||||
|
. {ROOT / 'managed-target-supervision.sh'}
|
||||||
|
{bounded_wait}
|
||||||
|
cleanup() {{
|
||||||
|
trap - EXIT
|
||||||
|
trap ':' INT TERM
|
||||||
|
kill "$owned" 2>/dev/null || true
|
||||||
|
bounded_wait "$owned" 5
|
||||||
|
[ "$join_timed_out" -eq 0 ]
|
||||||
|
}}
|
||||||
|
sleep 300 & owned=$!
|
||||||
|
case "$mode" in
|
||||||
|
normal) cleanup ;;
|
||||||
|
exit) trap cleanup EXIT; exit 0 ;;
|
||||||
|
term-exit) trap cleanup EXIT; trap 'exit 0' TERM; kill -TERM $$ ;;
|
||||||
|
esac
|
||||||
|
''')
|
||||||
|
env = dict(os.environ, PATH=f'{directory}:{os.environ["PATH"]}', PIDS=str(directory / 'pids'))
|
||||||
|
started = time.monotonic()
|
||||||
|
result = subprocess.run([*shell, str(script), mode], env=env, capture_output=True, timeout=10)
|
||||||
|
assert result.returncode == 0, (shell, mode, result.stderr)
|
||||||
|
assert time.monotonic() - started < 2, (shell, mode, 'watchdog cancellation delayed')
|
||||||
|
if (directory / 'pids').exists():
|
||||||
|
for pid in (directory / 'pids').read_text().splitlines():
|
||||||
|
assert not Path(f'/proc/{pid}').exists(), (shell, mode, 'orphan sleeper', pid)
|
||||||
|
|
||||||
|
|
||||||
|
check_watchdog_traps(['/bin/sh'])
|
||||||
|
busybox = shutil.which('busybox')
|
||||||
|
if busybox:
|
||||||
|
check_watchdog_traps([busybox, 'ash'])
|
||||||
|
for pid, seconds in (('1', '5'), ('0', '5'), ('-1', '5'), ('42', '0'), ('42', 'bad')):
|
||||||
|
result = subprocess.run(['/bin/sh', str(ROOT / 'managed-target-supervision.sh'), '--watchdog', pid, seconds, '/'], capture_output=True, timeout=2)
|
||||||
|
assert result.returncode == 64
|
||||||
|
print('entrypoint supervision fixtures passed')
|
||||||
@@ -31,7 +31,7 @@ services:
|
|||||||
- /run/zapret-autotune-ss:rw,noexec,nosuid,nodev,size=1m,mode=0700,uid=0,gid=0
|
- /run/zapret-autotune-ss:rw,noexec,nosuid,nodev,size=1m,mode=0700,uid=0,gid=0
|
||||||
networks: [target]
|
networks: [target]
|
||||||
restart: "no"
|
restart: "no"
|
||||||
stop_grace_period: 120s
|
stop_grace_period: 240s
|
||||||
networks:
|
networks:
|
||||||
target:
|
target:
|
||||||
driver: bridge
|
driver: bridge
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Isolated zapret-autotune managed target
|
|
||||||
Requires=docker.service
|
|
||||||
After=docker.service
|
|
||||||
StartLimitIntervalSec=6h
|
|
||||||
StartLimitBurst=3
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
WorkingDirectory=/opt/zapret2-autotune
|
|
||||||
ExecStart=/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml up --no-build --abort-on-container-exit --exit-code-from target target
|
|
||||||
ExecStop=/usr/bin/docker compose --env-file deploy/managed-target/.env -f deploy/managed-target/compose.yaml stop target
|
|
||||||
Restart=no
|
|
||||||
RestartForceExitStatus=75
|
|
||||||
RestartSec=30s
|
|
||||||
TimeoutStopSec=180s
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
+100
-21
@@ -30,54 +30,97 @@ server_pid=
|
|||||||
local_pid=
|
local_pid=
|
||||||
monitor_pid=
|
monitor_pid=
|
||||||
zapret_started=0
|
zapret_started=0
|
||||||
|
cooldown_pid=
|
||||||
|
# Signal only descendants of an owned child, never the container/host namespace.
|
||||||
|
# The monitor gets TERM itself first so it can join scans and restore bytes;
|
||||||
|
# descendant signalling is reserved for SS shutdown and forced watchdog cleanup.
|
||||||
|
. /opt/managed-target-supervision.sh
|
||||||
|
pause() {
|
||||||
|
sleep "$1" </dev/null >/dev/null 2>&1 & cooldown_pid=$!
|
||||||
|
wait "$cooldown_pid"
|
||||||
|
cooldown_pid=
|
||||||
|
}
|
||||||
# Wait with a watchdog: never let an unresponsive child exceed Docker's grace.
|
# Wait with a watchdog: never let an unresponsive child exceed Docker's grace.
|
||||||
bounded_wait() {
|
bounded_wait() {
|
||||||
child=$1
|
child=$1
|
||||||
seconds=$2
|
seconds=$2
|
||||||
(
|
rm -f "$config_dir/join-timeout" "$config_dir/join-ready"
|
||||||
sleep "$seconds" & sleeper=$!
|
# Fresh interpreter: a forked shell can inherit a deferred EXIT/TERM trap
|
||||||
trap 'kill "$sleeper" 2>/dev/null || true; exit 0' TERM INT
|
# context and never run its cancellation handler during PID 1 shutdown.
|
||||||
wait "$sleeper"
|
/bin/sh /opt/managed-target-supervision.sh --watchdog "$child" "$seconds" "$config_dir" </dev/null >/dev/null 2>&1 &
|
||||||
kill -KILL "$child" 2>/dev/null || true
|
|
||||||
) </dev/null >/dev/null 2>&1 &
|
|
||||||
watchdog=$!
|
watchdog=$!
|
||||||
|
# A fast-exiting child must not cancel the watchdog before its trap exists.
|
||||||
|
# Fail closed if exec/setup fails before acknowledgement; never wait forever.
|
||||||
|
ack_deadline=$(($(date +%s) + 5))
|
||||||
|
while [ ! -e "$config_dir/join-ready" ]; do
|
||||||
|
if ! kill -0 "$watchdog" 2>/dev/null || [ "$(date +%s)" -ge "$ack_deadline" ]; then
|
||||||
|
: > "$config_dir/join-timeout"
|
||||||
|
signal_tree "$watchdog" KILL
|
||||||
|
signal_tree "$child" KILL
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
waited_code=0
|
waited_code=0
|
||||||
wait "$child" 2>/dev/null || waited_code=$?
|
wait "$child" 2>/dev/null || waited_code=$?
|
||||||
kill -TERM "$watchdog" 2>/dev/null || true
|
kill -TERM "$watchdog" 2>/dev/null || true
|
||||||
wait "$watchdog" 2>/dev/null || true
|
wait "$watchdog" 2>/dev/null || true
|
||||||
|
join_timed_out=0
|
||||||
|
[ ! -e "$config_dir/join-timeout" ] || join_timed_out=1
|
||||||
}
|
}
|
||||||
cleanup() {
|
cleanup_cycle() {
|
||||||
final_code=$?
|
cleanup_code=0
|
||||||
case "$final_code" in 0|64|75|78) ;; *) final_code=78 ;; esac
|
|
||||||
trap - EXIT INT TERM
|
|
||||||
# Let the monitor cancel/join scans and restore committed bytes BEFORE stop.
|
# Let the monitor cancel/join scans and restore committed bytes BEFORE stop.
|
||||||
if [ -n "$monitor_pid" ]; then
|
if [ -n "$monitor_pid" ]; then
|
||||||
kill -TERM "$monitor_pid" 2>/dev/null || true
|
kill -TERM "$monitor_pid" 2>/dev/null || true
|
||||||
bounded_wait "$monitor_pid" 60
|
bounded_wait "$monitor_pid" 45
|
||||||
# Cancellation must restore safely even when SS failure requested retry.
|
# Cancellation must restore safely even when SS failure requested retry.
|
||||||
case "$waited_code" in 0|143) ;; *) final_code=78 ;; esac
|
case "$waited_code" in 0|143) ;; *) echo "error: monitor cleanup status $waited_code" >&2; cleanup_code=78 ;; esac
|
||||||
|
[ "$join_timed_out" -eq 0 ] || { echo 'error: monitor join timeout' >&2; cleanup_code=78; }
|
||||||
|
monitor_pid=
|
||||||
fi
|
fi
|
||||||
# Dedicated PID/network namespace: no host/other-target stop.
|
# Dedicated PID/network namespace: no host/other-target stop.
|
||||||
if [ "$zapret_started" -eq 1 ]; then
|
if [ "$zapret_started" -eq 1 ]; then
|
||||||
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop || final_code=78
|
# Stop also sources config: clear trust before reconciliation, never
|
||||||
|
# source unknown bytes left by a crashed monitor, even for shutdown.
|
||||||
|
zapret_started=0
|
||||||
|
if timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile; then
|
||||||
|
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 stop || cleanup_code=78
|
||||||
|
else
|
||||||
|
cleanup_code=78
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
for pid in "$local_pid" "$server_pid"; do
|
for pid in "$local_pid" "$server_pid"; do
|
||||||
[ -z "$pid" ] || kill "$pid" 2>/dev/null || true
|
[ -z "$pid" ] || signal_tree "$pid" TERM
|
||||||
done
|
done
|
||||||
for pid in "$local_pid" "$server_pid"; do
|
for pid in "$local_pid" "$server_pid"; do
|
||||||
if [ -n "$pid" ]; then
|
if [ -n "$pid" ]; then
|
||||||
bounded_wait "$pid" 5
|
bounded_wait "$pid" 5
|
||||||
# Its original failure triggered supervision; only forced shutdown
|
# Its original failure triggered supervision; only forced shutdown
|
||||||
# is a cleanup failure, not the already-dead Shadowsocks status.
|
# is a cleanup failure, not the already-dead Shadowsocks status.
|
||||||
[ "$waited_code" -ne 137 ] || final_code=78
|
[ "$join_timed_out" -eq 0 ] || { echo 'error: Shadowsocks join timeout' >&2; cleanup_code=78; }
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
server_pid=
|
||||||
|
local_pid=
|
||||||
|
return "$cleanup_code"
|
||||||
|
}
|
||||||
|
cleanup() {
|
||||||
|
final_code=$?
|
||||||
|
trap - EXIT
|
||||||
|
# A caught (not ignored) signal lets watchdog children install TERM traps.
|
||||||
|
trap ':' INT TERM
|
||||||
|
if [ -n "$cooldown_pid" ]; then
|
||||||
|
kill "$cooldown_pid" 2>/dev/null || true
|
||||||
|
wait "$cooldown_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
cleanup_cycle || final_code=78
|
||||||
exit "$final_code"
|
exit "$final_code"
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
trap 'exit 0' INT TERM
|
trap 'exit 0' INT TERM
|
||||||
|
|
||||||
# Never invoke /opt/entrypoint.sh: zapret must start AFTER reconciliation.
|
# Never invoke /opt/entrypoint.sh: zapret must start AFTER reconciliation.
|
||||||
|
run_cycle() {
|
||||||
ss-server -c "$config_dir/server.json" &
|
ss-server -c "$config_dir/server.json" &
|
||||||
server_pid=$!
|
server_pid=$!
|
||||||
ss-local -c "$config_dir/local.json" &
|
ss-local -c "$config_dir/local.json" &
|
||||||
@@ -86,10 +129,10 @@ local_pid=$!
|
|||||||
# Bounded readiness, no network probe. Managed commands verify socket ownership.
|
# Bounded readiness, no network probe. Managed commands verify socket ownership.
|
||||||
attempt=0
|
attempt=0
|
||||||
until awk '$2 == "00000000:0438" && $4 == "0A" { found=1 } END { exit !found }' /proc/net/tcp; do
|
until awk '$2 == "00000000:0438" && $4 == "0A" { found=1 } END { exit !found }' /proc/net/tcp; do
|
||||||
{ kill -0 "$server_pid" && kill -0 "$local_pid"; } || exit 75
|
{ kill -0 "$server_pid" && kill -0 "$local_pid"; } || return 75
|
||||||
attempt=$((attempt + 1))
|
attempt=$((attempt + 1))
|
||||||
[ "$attempt" -lt 30 ] || { echo 'error: target SOCKS listener timeout' >&2; exit 75; }
|
[ "$attempt" -lt 30 ] || { echo 'error: target SOCKS listener timeout' >&2; return 75; }
|
||||||
sleep 1
|
pause 1
|
||||||
done
|
done
|
||||||
if [ ! -e "$ZAPRET_CONFIG" ]; then
|
if [ ! -e "$ZAPRET_CONFIG" ]; then
|
||||||
timeout -k 5 45 /usr/local/bin/zapret-autotune managed init || exit 78
|
timeout -k 5 45 /usr/local/bin/zapret-autotune managed init || exit 78
|
||||||
@@ -100,10 +143,14 @@ timeout -k 5 45 /usr/local/bin/zapret-autotune managed reconcile || exit 78
|
|||||||
zapret_started=1
|
zapret_started=1
|
||||||
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || exit 78
|
timeout -k 5 30 /opt/zapret2/init.d/sysv/zapret2 start || exit 78
|
||||||
|
|
||||||
|
# A replacement SS child can fail during reconciliation/activation. Do not
|
||||||
|
# fork a monitor we already need to cancel: an early TERM can race interpreter
|
||||||
|
# startup before its cancellation handler exists. Clean this cycle directly.
|
||||||
|
{ kill -0 "$server_pid" 2>/dev/null && kill -0 "$local_pid" 2>/dev/null; } || return 75
|
||||||
/usr/local/bin/zapret-autotune managed monitor &
|
/usr/local/bin/zapret-autotune managed monitor &
|
||||||
monitor_pid=$!
|
monitor_pid=$!
|
||||||
while kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do
|
while kill -0 "$server_pid" && kill -0 "$local_pid" && kill -0 "$monitor_pid"; do
|
||||||
sleep 1
|
pause 1
|
||||||
done
|
done
|
||||||
if ! kill -0 "$monitor_pid" 2>/dev/null; then
|
if ! kill -0 "$monitor_pid" 2>/dev/null; then
|
||||||
code=0
|
code=0
|
||||||
@@ -115,7 +162,39 @@ if ! kill -0 "$monitor_pid" 2>/dev/null; then
|
|||||||
64|75|78) ;;
|
64|75|78) ;;
|
||||||
*) code=78 ;;
|
*) code=78 ;;
|
||||||
esac
|
esac
|
||||||
exit "$code"
|
return "$code"
|
||||||
fi
|
fi
|
||||||
echo 'error: target child process exited' >&2
|
echo 'error: target child process exited' >&2
|
||||||
exit 75
|
return 75
|
||||||
|
}
|
||||||
|
|
||||||
|
# Three starts per rolling six hours, including initial startup. PID 1 owns
|
||||||
|
# the entire runtime cycle; cooldown is cancellable without any external manager.
|
||||||
|
starts=
|
||||||
|
backoff=15
|
||||||
|
while :; do
|
||||||
|
now=$(date +%s)
|
||||||
|
recent=
|
||||||
|
count=0
|
||||||
|
for started in $starts; do
|
||||||
|
if [ "$((now - started))" -lt 21600 ]; then
|
||||||
|
recent="$recent $started"
|
||||||
|
count=$((count + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
[ "$count" -lt 3 ] || { echo 'error: target restart budget exhausted' >&2; exit 78; }
|
||||||
|
starts="$recent $now"
|
||||||
|
code=0
|
||||||
|
run_cycle || code=$?
|
||||||
|
cleanup_cycle || code=78
|
||||||
|
[ "$code" -eq 75 ] || exit "$code"
|
||||||
|
# A long-running monitor may outlive the window; recheck at failure time.
|
||||||
|
now=$(date +%s)
|
||||||
|
retry_count=0
|
||||||
|
for started in $starts; do
|
||||||
|
[ "$((now - started))" -ge 21600 ] || retry_count=$((retry_count + 1))
|
||||||
|
done
|
||||||
|
[ "$retry_count" -lt 3 ] || { echo 'error: target restart budget exhausted' >&2; exit 78; }
|
||||||
|
pause "$backoff"
|
||||||
|
[ "$backoff" -ge 60 ] || backoff=$((backoff * 2))
|
||||||
|
done
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Shared owned-tree signalling; sourced by PID 1, executed afresh for watchdogs.
|
||||||
|
signal_tree() {
|
||||||
|
if [ -r "/proc/$1/task/$1/children" ]; then
|
||||||
|
descendants=
|
||||||
|
read -r descendants < "/proc/$1/task/$1/children" || true
|
||||||
|
for descendant in $descendants; do
|
||||||
|
signal_tree "$descendant" "$2"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
kill "-$2" "$1" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ "${1-}" = --watchdog ]; then
|
||||||
|
set -eu
|
||||||
|
[ "$#" -eq 4 ] || exit 64
|
||||||
|
case "$2" in ''|*[!0-9]*) exit 64 ;; esac
|
||||||
|
case "$3" in ''|*[!0-9]*) exit 64 ;; esac
|
||||||
|
[ "$2" -gt 1 ] && [ "$3" -gt 0 ] || exit 64
|
||||||
|
child=$2
|
||||||
|
seconds=$3
|
||||||
|
config_dir=$4
|
||||||
|
sleeper=
|
||||||
|
watchdog_cleanup() {
|
||||||
|
trap - EXIT
|
||||||
|
trap ':' INT TERM
|
||||||
|
if [ -n "$sleeper" ]; then
|
||||||
|
kill "$sleeper" 2>/dev/null || true
|
||||||
|
wait "$sleeper" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap watchdog_cleanup EXIT
|
||||||
|
trap 'exit 0' INT TERM
|
||||||
|
sleep "$seconds" & sleeper=$!
|
||||||
|
: > "$config_dir/join-ready"
|
||||||
|
wait "$sleeper"
|
||||||
|
: > "$config_dir/join-timeout"
|
||||||
|
signal_tree "$child" KILL
|
||||||
|
elif [ "$#" -ne 0 ]; then
|
||||||
|
exit 64
|
||||||
|
fi
|
||||||
Reference in new issue
Block a user