feat: add autonomous YouTube resource recovery and adaptive GGC
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
8826c591dc
commit
b4e1e1e9ca
43 files changed
+5163
-93
No files matched your search
@@ -1 +1,9 @@
|
||||
ZAPRET_AUTOTUNE_HOST_SOCKS_PORT=11080
|
||||
# Legacy Player API/exact-host behavior remains the default.
|
||||
ZAPRET_AUTOTUNE_YOUTUBE_SERVICE=0
|
||||
ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE=cold
|
||||
ZAPRET_AUTOTUNE_VIDEO_ID=dQw4w9WgXcQ
|
||||
# Optional JSON array: 1..5 unique 11-character VideoIDs (not signed URLs).
|
||||
# ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS='["dQw4w9WgXcQ","aqz-KE-bpKQ","jNQXAC9IVRw"]'
|
||||
# Optional target ss-server native DNS; empty retains the base resolver behavior.
|
||||
SS_DNS_SERVER=
|
||||
@@ -40,11 +40,45 @@ with a tag or local default.
|
||||
|
||||
Offline static guard (from repository root):
|
||||
`python3 deploy/managed-target/check_static.py`.
|
||||
Offline real pre-launch validation and private Compose forwarding fixtures:
|
||||
`python3 deploy/managed-target/check_config.py` (requires local Go and Compose).
|
||||
Isolated, accelerated shell supervision fixtures (no network):
|
||||
`python3 deploy/managed-target/check_supervision.py`.
|
||||
|
||||
## In-container supervision
|
||||
|
||||
### Opt-in service/adaptive and target-native DNS
|
||||
|
||||
Compose forwards `ZAPRET_AUTOTUNE_YOUTUBE_SERVICE` (`0` or `1`, default `0`),
|
||||
`ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE` (`cold` or `adaptive`, default `cold`),
|
||||
`ZAPRET_AUTOTUNE_VIDEO_ID` (default `dQw4w9WgXcQ`) and
|
||||
`ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS` (JSON array of 1..5 unique VideoIDs;
|
||||
default only the primary ID). Explicit empty/invalid settings are rejected,
|
||||
not silently replaced. Adaptive requires service `1`. See `.env.example`.
|
||||
Service `0` preserves legacy Player API/exact-host behavior.
|
||||
|
||||
Adaptive qualification is **post-warmup**, distinct from cold qualification;
|
||||
initial connections may fail and native learning must repeat after restart.
|
||||
It requires at least **three real unique signed ordinary media host samples**.
|
||||
Three sample videos may all resolve to the same CDN host: that means
|
||||
`DEGRADED`, no authorized googlevideo suffix, not fabricated coverage. Service
|
||||
asset checks cover sampled thumbnail/avatar routes, not the whole thumbnail pool.
|
||||
The current service/adaptive hardware acceptance gate is **pending**; the old
|
||||
clean-run PASS below does not qualify these new modes.
|
||||
|
||||
Optional `SS_DNS_SERVER` is an unbracketed literal IPv4/IPv6 address only (for
|
||||
example `8.8.8.8` or `2001:4860:4860::8888`); empty preserves existing resolver
|
||||
behavior. No domain, port, zone, control characters or option strings are
|
||||
accepted. Validation runs before service launch and reports opaque `configerror`
|
||||
without echoing input. DNS applies only to this target's native `ss-server`.
|
||||
The image build checks installed `ss-server -h` for native `-d <addr>` support;
|
||||
the validated nonsecret address is passed as a separate `-d` argv value, not
|
||||
eval or an assumed JSON key. Passwords still go only to private JSON configs.
|
||||
The reported `8.8.8.8` diagnostic corrected a tampered avatar response, but does
|
||||
not fix/prove all thumbnail routes. Native DNS correction is **not full asset
|
||||
availability proof**. No DoH bridge, permanent Google IP pins or global host DNS
|
||||
changes are supplied. PID 1 remains the sole startup/recovery owner.
|
||||
|
||||
Compose has no automatic restart. PID 1 restarts the entire owned runtime only
|
||||
for technical failures, with three starts per rolling six hours (including initial
|
||||
startup) and cancellable 15/30-second cooldowns. Clean typed recovery timeouts,
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Offline entrypoint/config fixtures: python3 deploy/managed-target/check_config.py."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
env = dict(os.environ, GOPROXY='off', GOTOOLCHAIN='local')
|
||||
with tempfile.TemporaryDirectory(prefix='target-config-') as tmp:
|
||||
root = Path(tmp)
|
||||
binary = root / 'autotune'
|
||||
subprocess.run(['go', 'build', '-o', str(binary), './cmd/zapret-autotune'],
|
||||
cwd=ROOT, env=env, check=True)
|
||||
private = root / 'private'
|
||||
private.mkdir()
|
||||
# Exercise only the real entrypoint's pre-launch region; no services/network.
|
||||
script = (ROOT / 'managed-target-entrypoint.sh').read_text().split('\nserver_pid=')[0]
|
||||
script = script.replace('/usr/local/bin/zapret-autotune', str(binary))
|
||||
script = script.replace('config_dir=/run/zapret-autotune-ss', f'config_dir={private}')
|
||||
start = script.index("awk '$2 ==")
|
||||
end = script.index('chmod 0700 "$config_dir"', start)
|
||||
script = script[:start] + script[end:]
|
||||
entry = root / 'entry.sh'
|
||||
entry.write_text(script)
|
||||
base = dict(env, SS_PASSWORD='ab' * 32,
|
||||
ZAPRET_AUTOTUNE_YOUTUBE_SERVICE='0', ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE='cold',
|
||||
ZAPRET_AUTOTUNE_VIDEO_ID='dQw4w9WgXcQ',
|
||||
ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS='["dQw4w9WgXcQ"]',
|
||||
ZAPRET_AUTOTUNE_SOCKS_PROXY='socks5://127.0.0.1:1080')
|
||||
for dns, valid in (('', True), ('8.8.8.8', True), ('2001:4860:4860::8888', True),
|
||||
('example.org', False), ('8.8.8.8\n', False), ('-d 8.8.8.8', False)):
|
||||
for path in private.iterdir():
|
||||
path.unlink()
|
||||
result = subprocess.run(['sh', str(entry)], env=dict(base, SS_DNS_SERVER=dns),
|
||||
capture_output=True, timeout=5)
|
||||
assert result.returncode == (0 if valid else 64), result.stderr
|
||||
assert not result.stdout and base['SS_PASSWORD'].encode() not in result.stderr
|
||||
if not valid:
|
||||
assert b'configerror:' in result.stderr and not list(private.iterdir())
|
||||
else:
|
||||
for name in ('server', 'local'):
|
||||
path = private / (name + '.json')
|
||||
assert path.stat().st_mode & 0o777 == 0o600
|
||||
assert json.loads(path.read_text())['password'] == base['SS_PASSWORD']
|
||||
# Render privately, never print the config/password; config does not deploy.
|
||||
compose_env = {k: v for k, v in base.items() if not k.startswith('ZAPRET_AUTOTUNE_')}
|
||||
for overrides, service, mode, ids in (
|
||||
({}, '0', 'cold', ['dQw4w9WgXcQ']),
|
||||
({'ZAPRET_AUTOTUNE_VIDEO_ID': 'aqz-KE-bpKQ'}, '0', 'cold', ['aqz-KE-bpKQ']),
|
||||
({'ZAPRET_AUTOTUNE_YOUTUBE_SERVICE': '1', 'ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE': 'adaptive',
|
||||
'ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS': '["dQw4w9WgXcQ","aqz-KE-bpKQ","jNQXAC9IVRw"]',
|
||||
'SS_DNS_SERVER': '8.8.8.8'}, '1', 'adaptive', ['dQw4w9WgXcQ', 'aqz-KE-bpKQ', 'jNQXAC9IVRw']),
|
||||
):
|
||||
result = subprocess.run(['docker', 'compose', '--env-file', '/dev/null', '-f',
|
||||
str(ROOT / 'deploy/managed-target/compose.yaml'),
|
||||
'config', '--format', 'json'], env=dict(compose_env, **overrides),
|
||||
capture_output=True, timeout=10)
|
||||
assert result.returncode == 0, 'Compose config failed (output suppressed)'
|
||||
settings = json.loads(result.stdout)['services']['target']['environment']
|
||||
assert settings['ZAPRET_AUTOTUNE_YOUTUBE_SERVICE'] == service
|
||||
assert settings['ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE'] == mode
|
||||
assert json.loads(settings['ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS']) == ids
|
||||
assert settings['SS_DNS_SERVER'] == overrides.get('SS_DNS_SERVER', '')
|
||||
print('target config fixtures passed')
|
||||
@@ -19,6 +19,17 @@ assert " source: ./target-state\n" in compose
|
||||
assert " target: /var/lib/zapret-autotune/target\n" in compose
|
||||
assert " create_host_path: false\n" in compose
|
||||
assert "SS_PASSWORD: ${SS_PASSWORD:?Set SS_PASSWORD in the deployment environment}" in compose
|
||||
for setting in (
|
||||
'SS_DNS_SERVER: ${SS_DNS_SERVER-}',
|
||||
'ZAPRET_AUTOTUNE_YOUTUBE_SERVICE: ${ZAPRET_AUTOTUNE_YOUTUBE_SERVICE-0}',
|
||||
'ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE: ${ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE-cold}',
|
||||
'ZAPRET_AUTOTUNE_VIDEO_ID: ${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}',
|
||||
"ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS: '${ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS-[\"${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}\"]}'",
|
||||
):
|
||||
assert setting in compose, setting
|
||||
assert entrypoint.index('target-config-check || exit 64') < entrypoint.index('cat > "$config_dir/server.json"')
|
||||
assert 'set -- "$@" -d "$SS_DNS_SERVER"' in entrypoint
|
||||
assert 'eval ' not in entrypoint
|
||||
assert "secrets:" not in compose
|
||||
assert "ss_password=${SS_PASSWORD-}" in entrypoint
|
||||
assert "unset SS_PASSWORD" in entrypoint
|
||||
|
||||
@@ -73,7 +73,9 @@ if name in ('ss-server', 'ss-local'):
|
||||
with (root / 'children').open('a') as f:
|
||||
f.write(str(os.getpid()) + '\n')
|
||||
assert 'SS_PASSWORD' not in os.environ
|
||||
assert sys.argv[1] == '-c' and len(sys.argv) == 3
|
||||
assert sys.argv[1] == '-c'
|
||||
dns = os.environ.get('SS_DNS_SERVER', '') if name == 'ss-server' else ''
|
||||
assert sys.argv[3:] == (['-d', dns] if dns else [])
|
||||
assert (pathlib.Path(sys.argv[2]).stat().st_mode & 0o777) == 0o600
|
||||
log(name)
|
||||
if name == 'ss-local' and scenario in ('ss', 'mixed', 'joinfail') and increment('ss_runs') == 1:
|
||||
@@ -92,6 +94,8 @@ if name == 'zapret2':
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
assert name == 'autotune'
|
||||
if sys.argv[1:] == ['target-config-check']:
|
||||
sys.exit(0)
|
||||
command = sys.argv[2]
|
||||
if command == 'init':
|
||||
log('init')
|
||||
@@ -140,7 +144,7 @@ elif command == 'monitor':
|
||||
'''
|
||||
|
||||
|
||||
def run(scenario, expected):
|
||||
def run(scenario, expected, dns=''):
|
||||
with tempfile.TemporaryDirectory(prefix='supervision-') as tmp:
|
||||
root = Path(tmp)
|
||||
binary = root / 'bin'
|
||||
@@ -165,7 +169,8 @@ def run(scenario, expected):
|
||||
entry = root / 'entry.sh'
|
||||
entry.write_text(text)
|
||||
env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}', FIXTURE=tmp,
|
||||
SCENARIO=scenario, SS_PASSWORD='ab' * 32, ZAPRET_CONFIG=str(root / 'config'))
|
||||
SCENARIO=scenario, SS_PASSWORD='ab' * 32, SS_DNS_SERVER=dns,
|
||||
ZAPRET_CONFIG=str(root / 'config'))
|
||||
peer = None
|
||||
if scenario == 'orphans':
|
||||
# Local fixture is not PID 1. Adopt orphan workers here so the test
|
||||
@@ -224,6 +229,9 @@ events = run('cancel', 0)
|
||||
assert events.count('start') == 1
|
||||
events = run('prestartfail', 78)
|
||||
assert 'start' not in events and 'stop' not in events
|
||||
for dns in ('8.8.8.8', '2001:4860:4860::8888'):
|
||||
events = run('fatal64', 64, dns)
|
||||
assert events.count('ss-server') == 1 and events.count('ss-local') == 1
|
||||
|
||||
|
||||
def check_watchdog_traps(shell):
|
||||
|
||||
@@ -11,6 +11,11 @@ services:
|
||||
cap_add: [NET_ADMIN, NET_RAW, SETUID, SETGID, KILL]
|
||||
environment:
|
||||
SS_PASSWORD: ${SS_PASSWORD:?Set SS_PASSWORD in the deployment environment}
|
||||
SS_DNS_SERVER: ${SS_DNS_SERVER-}
|
||||
ZAPRET_AUTOTUNE_YOUTUBE_SERVICE: ${ZAPRET_AUTOTUNE_YOUTUBE_SERVICE-0}
|
||||
ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE: ${ZAPRET_AUTOTUNE_GOOGLEVIDEO_MODE-cold}
|
||||
ZAPRET_AUTOTUNE_VIDEO_ID: ${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}
|
||||
ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS: '${ZAPRET_AUTOTUNE_GOOGLEVIDEO_VIDEO_IDS-["${ZAPRET_AUTOTUNE_VIDEO_ID-dQw4w9WgXcQ}"]}'
|
||||
SOCKS_PORT: "1080"
|
||||
ZAPRET_AUTOTUNE_TARGET_ROLE: managed-test
|
||||
ZAPRET_AUTOTUNE_TARGET_ID: zapret-autotune-managed-v1
|
||||
|
||||
Reference in new issue
Block a user