release: bump version to 0.74.6
This commit is contained in:
237
scripts/docker_proxy.py
Normal file
237
scripts/docker_proxy.py
Normal file
@@ -0,0 +1,237 @@
|
||||
"""Select a reachable build route and manage only Planet-owned daemon proxy settings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
CONFIG = Path("/etc/docker/daemon.json")
|
||||
STATE = Path("/etc/docker/planet-proxy-state.json")
|
||||
PROXY_KEYS = ("http-proxy", "https-proxy", "no-proxy")
|
||||
PROXY_ENV = ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy")
|
||||
PROXY_SCHEMES = ("http", "https")
|
||||
DEFAULT_NO_PROXY = "localhost,127.0.0.1,::1"
|
||||
CONFIG_ERROR = "PLANET_PROXY_CONFIG_FAILED"
|
||||
COMMAND_TIMEOUT_SECONDS = 60
|
||||
CONNECT_TIMEOUT_SECONDS = 3
|
||||
REQUEST_TIMEOUT_SECONDS = 8
|
||||
PROBE_PROCESS_TIMEOUT_SECONDS = 10
|
||||
|
||||
|
||||
class ProxyError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def run(args: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
args, text=True, capture_output=True, check=True, timeout=COMMAND_TIMEOUT_SECONDS, **kwargs
|
||||
)
|
||||
|
||||
|
||||
def daemon_proxies() -> dict[str, str]:
|
||||
info = json.loads(run(["docker", "info", "--format", "{{json .}}"]).stdout)
|
||||
values = dict(
|
||||
zip(PROXY_KEYS, (info.get("HttpProxy"), info.get("HttpsProxy"), info.get("NoProxy")))
|
||||
)
|
||||
return {key: value for key, value in values.items() if value}
|
||||
|
||||
|
||||
def registry_urls(environment: dict[str, str]) -> list[str]:
|
||||
dockerfile = Path(__file__).resolve().parents[1] / "aiprovider/Dockerfile"
|
||||
defaults = dict(re.findall(r"^ARG (PYTHON_IMAGE|UV_IMAGE)=(.+)$", dockerfile.read_text(), re.M))
|
||||
registries = set()
|
||||
for key in ("PYTHON_IMAGE", "UV_IMAGE"):
|
||||
image = environment.get(key) or defaults[key]
|
||||
prefix = image.split("/")[0]
|
||||
registry = (
|
||||
prefix
|
||||
if "/" in image and ("." in prefix or ":" in prefix or prefix == "localhost")
|
||||
else "docker.io"
|
||||
)
|
||||
registries.add("registry-1.docker.io" if registry == "docker.io" else registry)
|
||||
return [f"https://{registry}/v2/" for registry in sorted(registries)]
|
||||
|
||||
|
||||
def probe_url(url: str, proxy: str, no_proxy: str = "") -> bool:
|
||||
environment = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key.lower() not in ("http_proxy", "https_proxy", "all_proxy", "no_proxy")
|
||||
}
|
||||
if proxy:
|
||||
environment.update(http_proxy=proxy, https_proxy=proxy)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"curl",
|
||||
"-q",
|
||||
"--silent",
|
||||
"--output",
|
||||
"/dev/null",
|
||||
"--write-out",
|
||||
"%{http_code}",
|
||||
"--noproxy",
|
||||
no_proxy if proxy else "",
|
||||
"--connect-timeout",
|
||||
str(CONNECT_TIMEOUT_SECONDS),
|
||||
"--max-time",
|
||||
str(REQUEST_TIMEOUT_SECONDS),
|
||||
url,
|
||||
],
|
||||
env=environment,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=PROBE_PROCESS_TIMEOUT_SECONDS,
|
||||
)
|
||||
return result.returncode == 0 and (
|
||||
result.stdout.startswith("2") or result.stdout in ("401", "403", "429")
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return False
|
||||
|
||||
|
||||
def reachable(urls: list[str], proxy: str, no_proxy: str = "") -> bool:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(urls)) as pool:
|
||||
return all(pool.map(lambda url: probe_url(url, proxy, no_proxy), urls))
|
||||
|
||||
|
||||
def select_route(environment: dict[str, str], urls: list[str]) -> dict[str, object]:
|
||||
candidates = list(dict.fromkeys(environment[key] for key in PROXY_ENV if environment.get(key)))
|
||||
no_proxy = environment.get("NO_PROXY") or environment.get("no_proxy") or DEFAULT_NO_PROXY
|
||||
for proxy in candidates:
|
||||
if urlsplit(proxy).scheme not in PROXY_SCHEMES:
|
||||
continue
|
||||
if reachable(urls, proxy, no_proxy):
|
||||
return {
|
||||
"desired": dict(zip(PROXY_KEYS, (proxy, proxy, no_proxy))),
|
||||
"mode": "proxy",
|
||||
"reachable": True,
|
||||
}
|
||||
return {"desired": {}, "mode": "direct", "reachable": reachable(urls, "")}
|
||||
|
||||
|
||||
def atomic_write(path: Path, contents: bytes) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as output:
|
||||
output.write(contents)
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def encode(value: dict[str, object]) -> bytes:
|
||||
return (json.dumps(value, indent=2) + "\n").encode()
|
||||
|
||||
|
||||
def validate_plan(plan: dict[str, object]) -> None:
|
||||
for field in ("current", "desired"):
|
||||
values = plan.get(field)
|
||||
if not isinstance(values, dict) or set(values) - set(PROXY_KEYS):
|
||||
raise ProxyError(CONFIG_ERROR)
|
||||
if any(not isinstance(value, str) or "\n" in value for value in values.values()):
|
||||
raise ProxyError(CONFIG_ERROR)
|
||||
for key in ("http-proxy", "https-proxy"):
|
||||
value = plan["desired"].get(key, "")
|
||||
if value and urlsplit(value).scheme not in PROXY_SCHEMES:
|
||||
raise ProxyError(CONFIG_ERROR)
|
||||
|
||||
|
||||
def restore_containers(containers: list[str]) -> None:
|
||||
if containers:
|
||||
run(["docker", "start", *containers])
|
||||
|
||||
|
||||
def apply_plan(plan: dict[str, object], config: Path = CONFIG, state: Path = STATE) -> None:
|
||||
validate_plan(plan)
|
||||
current = daemon_proxies()
|
||||
if current != plan["current"]:
|
||||
raise ProxyError("PLANET_PROXY_CONFIG_CHANGED")
|
||||
if current == plan["desired"]:
|
||||
return
|
||||
original = config.read_bytes() if config.exists() else None
|
||||
data = json.loads(original) if original else {}
|
||||
owned = json.loads(state.read_text()) if state.exists() else None
|
||||
configured = {key: value for key, value in data.get("proxies", {}).items() if value}
|
||||
if (configured or current) and (
|
||||
not owned or owned.get("managed") != configured or current != configured
|
||||
):
|
||||
raise ProxyError("PLANET_PROXY_EXTERNAL")
|
||||
if plan["desired"]:
|
||||
data["proxies"] = plan["desired"]
|
||||
else:
|
||||
data.pop("proxies", None)
|
||||
containers = run(["docker", "ps", "-q"]).stdout.split()
|
||||
if original is not None:
|
||||
atomic_write(config.with_suffix(".json.planet-proxy.bak"), original)
|
||||
atomic_write(config, encode(data))
|
||||
restart_requested = False
|
||||
try:
|
||||
run(["dockerd", "--validate", "--config-file", str(config)])
|
||||
restart_requested = True
|
||||
run(["systemctl", "restart", "docker"])
|
||||
restore_containers(containers)
|
||||
if daemon_proxies() != plan["desired"]:
|
||||
raise ProxyError(CONFIG_ERROR)
|
||||
atomic_write(state, encode({"managed": plan["desired"]}))
|
||||
except Exception as error:
|
||||
rollback(config, original, containers, restart_requested)
|
||||
raise ProxyError(CONFIG_ERROR) from error
|
||||
|
||||
|
||||
def rollback(
|
||||
config: Path, original: bytes | None, containers: list[str], restart_requested: bool
|
||||
) -> None:
|
||||
if original is None:
|
||||
config.unlink(missing_ok=True)
|
||||
else:
|
||||
atomic_write(config, original)
|
||||
if not restart_requested:
|
||||
return
|
||||
try:
|
||||
run(["systemctl", "restart", "docker"])
|
||||
restore_containers(containers)
|
||||
except Exception as error:
|
||||
raise ProxyError("PLANET_PROXY_ROLLBACK_FAILED") from error
|
||||
|
||||
|
||||
def main() -> None:
|
||||
action = sys.argv[1]
|
||||
if action == "plan":
|
||||
plan = select_route(dict(os.environ), registry_urls(dict(os.environ)))
|
||||
plan["current"] = daemon_proxies()
|
||||
json.dump(plan, sys.stdout)
|
||||
elif action == "status":
|
||||
plan = json.load(sys.stdin)
|
||||
changed = int(plan["current"] != plan["desired"])
|
||||
print(plan["mode"], changed, int(plan["reachable"]))
|
||||
elif action == "apply":
|
||||
if os.geteuid() != 0:
|
||||
raise ProxyError(CONFIG_ERROR)
|
||||
CONFIG.parent.mkdir(parents=True, exist_ok=True)
|
||||
with (CONFIG.parent / "planet-proxy.lock").open("a") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
apply_plan(json.load(sys.stdin))
|
||||
else:
|
||||
raise ProxyError(CONFIG_ERROR)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except ProxyError as error:
|
||||
sys.stderr.write(str(error) + "\n")
|
||||
sys.exit(1)
|
||||
except Exception:
|
||||
# Subprocess output and proxy URLs may contain credentials.
|
||||
sys.stderr.write(CONFIG_ERROR + "\n")
|
||||
sys.exit(1)
|
||||
@@ -14,8 +14,12 @@ main() {
|
||||
harness_run git diff --check
|
||||
harness_run zsh -n planet.sh
|
||||
harness_run zsh -n scripts/lib/docker-bootstrap.zsh
|
||||
harness_run zsh -n scripts/lib/docker-proxy.zsh
|
||||
harness_run zsh -n scripts/lib/error-diagnostics.zsh
|
||||
harness_run "$uv_bin" run --frozen --project "$ROOT_DIR" python scripts/harness/test_docker_bootstrap.py
|
||||
harness_run "$uv_bin" run --frozen --project "$ROOT_DIR" python scripts/harness/test_docker_proxy.py
|
||||
harness_run "$uv_bin" run --frozen --project "$ROOT_DIR" python scripts/harness/test_database_startup.py
|
||||
harness_run "$uv_bin" run --frozen --project "$ROOT_DIR" python scripts/harness/test_error_diagnostics.py
|
||||
harness_run bash -n scripts/bootstrap-dev.sh
|
||||
harness_run bash -n scripts/harness/lib.sh
|
||||
harness_run bash -n scripts/harness/doctor.sh
|
||||
|
||||
@@ -43,6 +43,74 @@ def run_shell(functions: list[str], setup: str, action: str) -> subprocess.Compl
|
||||
)
|
||||
|
||||
|
||||
class ComposeSelectionTests(unittest.TestCase):
|
||||
def test_operations_use_v1_only_when_v2_is_unavailable(self) -> None:
|
||||
operations = (
|
||||
("compose_up", "up -d postgres"),
|
||||
("compose_supports_build", "build --help"),
|
||||
("build_ai_provider_image", "build aiprovider"),
|
||||
)
|
||||
for function, arguments in operations:
|
||||
for v2_available, v1_available in (
|
||||
(True, True),
|
||||
(True, False),
|
||||
(False, True),
|
||||
(False, False),
|
||||
):
|
||||
for command_status in (0, 1):
|
||||
with self.subTest(
|
||||
function=function,
|
||||
v2=v2_available,
|
||||
v1=v1_available,
|
||||
command_status=command_status,
|
||||
):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
calls_file = Path(directory) / "calls"
|
||||
action = function
|
||||
if function == "compose_up":
|
||||
action += " " + arguments
|
||||
result = run_shell(
|
||||
[function],
|
||||
f"""
|
||||
compose_available() {{ return {int(not v2_available)}; }}
|
||||
compose_v1_available() {{ return {int(not v1_available)}; }}
|
||||
docker() {{
|
||||
echo "docker $*" >> '{calls_file}'
|
||||
echo ORIGINAL_ERROR >&2
|
||||
return {command_status}
|
||||
}}
|
||||
docker-compose() {{
|
||||
echo "docker-compose $*" >> '{calls_file}'
|
||||
return {command_status}
|
||||
}}
|
||||
run_ai_provider_build_command() {{
|
||||
echo "$1 build aiprovider" >> '{calls_file}'
|
||||
echo ORIGINAL_ERROR >&2
|
||||
return {command_status}
|
||||
}}
|
||||
set_wait_detail() {{ :; }}
|
||||
clear_wait_spinner() {{ :; }}
|
||||
log_error() {{ echo "$*"; }}
|
||||
log_note() {{ echo "$*"; }}
|
||||
log_warn() {{ echo "$*"; }}
|
||||
report_missing_compose() {{ echo MISSING_COMPOSE; return 1; }}
|
||||
""",
|
||||
f"if {action}; then exit 0; else exit $?; fi",
|
||||
)
|
||||
calls = (
|
||||
calls_file.read_text().splitlines() if calls_file.exists() else []
|
||||
)
|
||||
command = "docker compose" if v2_available else "docker-compose"
|
||||
available = v2_available or v1_available
|
||||
self.assertEqual(calls, [f"{command} {arguments}"] if available else [])
|
||||
self.assertEqual(result.returncode, command_status if available else 1)
|
||||
self.assertNotIn("回退", result.stdout)
|
||||
if available:
|
||||
self.assertNotIn("MISSING_COMPOSE", result.stdout)
|
||||
if v2_available and function != "compose_supports_build":
|
||||
self.assertIn("ORIGINAL_ERROR", result.stderr)
|
||||
|
||||
|
||||
class DatabaseLifecycleTests(unittest.TestCase):
|
||||
def test_ai_start_uses_host_readiness_without_waiting_for_docker_probe_schedule(self) -> None:
|
||||
for recreate in (0, 1):
|
||||
|
||||
236
scripts/harness/test_docker_proxy.py
Normal file
236
scripts/harness/test_docker_proxy.py
Normal file
@@ -0,0 +1,236 @@
|
||||
"""Automatic proxy routing and rollback tests; never restart the host Docker daemon."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from test_database_startup import run_shell
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
import docker_proxy as proxy # noqa: E402
|
||||
|
||||
URLS = ["https://registry.example/v2/"]
|
||||
PROXIES = {
|
||||
"http-proxy": "http://localhost:1234",
|
||||
"https-proxy": "http://localhost:1234",
|
||||
"no-proxy": "localhost,127.0.0.1,::1",
|
||||
}
|
||||
|
||||
|
||||
class DockerProxyTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary = tempfile.TemporaryDirectory(prefix="planet-proxy-test-")
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.config = Path(self.temporary.name) / "daemon.json"
|
||||
self.state = Path(self.temporary.name) / "proxy-state.json"
|
||||
|
||||
def test_no_host_proxy_uses_direct_without_adding_configuration(self) -> None:
|
||||
with patch.object(proxy, "reachable", return_value=True) as probe:
|
||||
plan = proxy.select_route({}, URLS)
|
||||
self.assertEqual(plan, {"desired": {}, "mode": "direct", "reachable": True})
|
||||
probe.assert_called_once_with(URLS, "")
|
||||
plan["current"] = {}
|
||||
with (
|
||||
patch.object(proxy, "daemon_proxies", return_value={}),
|
||||
patch.object(proxy, "run") as run,
|
||||
):
|
||||
proxy.apply_plan(plan, self.config, self.state)
|
||||
run.assert_not_called()
|
||||
self.assertFalse(self.config.exists())
|
||||
|
||||
def test_proxy_is_selected_only_after_registry_probe(self) -> None:
|
||||
with patch.object(proxy, "reachable", return_value=True) as probe:
|
||||
plan = proxy.select_route({"https_proxy": PROXIES["https-proxy"]}, URLS)
|
||||
self.assertEqual(plan["desired"], PROXIES)
|
||||
probe.assert_called_once_with(URLS, PROXIES["https-proxy"], PROXIES["no-proxy"])
|
||||
|
||||
def test_dead_proxy_falls_back_to_direct(self) -> None:
|
||||
with patch.object(proxy, "reachable", side_effect=[False, True]) as probe:
|
||||
plan = proxy.select_route({"HTTPS_PROXY": PROXIES["https-proxy"]}, URLS)
|
||||
self.assertEqual(plan["desired"], {})
|
||||
self.assertTrue(plan["reachable"])
|
||||
self.assertEqual(probe.call_count, 2)
|
||||
|
||||
def test_both_routes_failing_is_not_reported_as_connected(self) -> None:
|
||||
with patch.object(proxy, "reachable", return_value=False):
|
||||
plan = proxy.select_route({"HTTP_PROXY": PROXIES["http-proxy"]}, URLS)
|
||||
self.assertEqual(plan["mode"], "direct")
|
||||
self.assertFalse(plan["reachable"])
|
||||
|
||||
def test_no_proxy_rules_are_preserved_in_probe_and_config(self) -> None:
|
||||
with patch.object(proxy, "reachable", return_value=True) as probe:
|
||||
plan = proxy.select_route(
|
||||
{"HTTPS_PROXY": PROXIES["https-proxy"], "NO_PROXY": "*"}, URLS
|
||||
)
|
||||
self.assertEqual(plan["desired"]["no-proxy"], "*")
|
||||
probe.assert_called_once_with(URLS, PROXIES["https-proxy"], "*")
|
||||
|
||||
def test_actual_image_registry_overrides_are_probed(self) -> None:
|
||||
urls = proxy.registry_urls(
|
||||
{
|
||||
"PYTHON_IMAGE": "internal.example:5000/python:3",
|
||||
"UV_IMAGE": "internal.example:5000/uv:latest",
|
||||
}
|
||||
)
|
||||
self.assertEqual(urls, ["https://internal.example:5000/v2/"])
|
||||
|
||||
def test_only_actual_builds_prepare_proxy_and_failed_preparation_stops_build(self) -> None:
|
||||
for no_build, fingerprint, proxy_status, expected_calls in (
|
||||
(1, "old", 0, []),
|
||||
(0, "new", 0, []),
|
||||
(0, "old", 0, ["PROXY_CHECK", "BUILD"]),
|
||||
(0, "old", 1, ["PROXY_CHECK"]),
|
||||
):
|
||||
with self.subTest(no_build=no_build, fingerprint=fingerprint, status=proxy_status):
|
||||
result = run_shell(
|
||||
["ensure_ai_provider_image_current"],
|
||||
f"""
|
||||
AI_PROVIDER_NO_BUILD={no_build}
|
||||
AI_PROVIDER_BUILD_LOG_FILE=/dev/null
|
||||
prepare_uv_build_config() {{ :; }}
|
||||
compute_ai_provider_build_fingerprint() {{ echo new; }}
|
||||
read_ai_provider_build_stamp() {{ echo old; }}
|
||||
ai_provider_image_exists() {{ return 0; }}
|
||||
read_ai_provider_image_fingerprint() {{ echo {fingerprint}; }}
|
||||
write_ai_provider_build_stamp() {{ :; }}
|
||||
set_wait_detail() {{ :; }}
|
||||
log_note() {{ :; }}
|
||||
log_success() {{ :; }}
|
||||
compose_supports_build() {{ return 0; }}
|
||||
prepare_docker_build_proxy() {{ echo PROXY_CHECK; return {proxy_status}; }}
|
||||
build_ai_provider_image() {{ echo BUILD; }}
|
||||
""",
|
||||
"ensure_ai_provider_image_current",
|
||||
)
|
||||
self.assertEqual(result.stdout.splitlines(), expected_calls)
|
||||
self.assertEqual(result.returncode, proxy_status)
|
||||
|
||||
def fake_run(self, args: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
|
||||
output = "postgres-id\nredis-id\n" if args == ["docker", "ps", "-q"] else ""
|
||||
return subprocess.CompletedProcess(args, 0, output, "")
|
||||
|
||||
def test_add_proxy_preserves_other_settings_and_restores_containers(self) -> None:
|
||||
original = b'{"log-driver": "local"}\n'
|
||||
self.config.write_bytes(original)
|
||||
with (
|
||||
patch.object(proxy, "daemon_proxies", side_effect=[{}, PROXIES]),
|
||||
patch.object(proxy, "run", side_effect=self.fake_run) as run,
|
||||
):
|
||||
proxy.apply_plan({"current": {}, "desired": PROXIES}, self.config, self.state)
|
||||
self.assertEqual(
|
||||
json.loads(self.config.read_text()), {"log-driver": "local", "proxies": PROXIES}
|
||||
)
|
||||
self.assertEqual(json.loads(self.state.read_text())["managed"], PROXIES)
|
||||
self.assertEqual(self.config.with_suffix(".json.planet-proxy.bak").read_bytes(), original)
|
||||
self.assertEqual(self.state.stat().st_mode & 0o777, 0o600)
|
||||
run.assert_any_call(["docker", "start", "postgres-id", "redis-id"])
|
||||
|
||||
def test_remove_owned_proxy_when_host_proxy_disappears(self) -> None:
|
||||
self.config.write_text(json.dumps({"proxies": PROXIES, "log-driver": "local"}))
|
||||
self.state.write_text(json.dumps({"managed": PROXIES}))
|
||||
with (
|
||||
patch.object(proxy, "daemon_proxies", side_effect=[PROXIES, {}]),
|
||||
patch.object(proxy, "run", side_effect=self.fake_run),
|
||||
):
|
||||
proxy.apply_plan({"current": PROXIES, "desired": {}}, self.config, self.state)
|
||||
self.assertEqual(json.loads(self.config.read_text()), {"log-driver": "local"})
|
||||
|
||||
def test_unchanged_proxy_does_not_restart_docker(self) -> None:
|
||||
with (
|
||||
patch.object(proxy, "daemon_proxies", return_value=PROXIES),
|
||||
patch.object(proxy, "run") as run,
|
||||
):
|
||||
proxy.apply_plan({"current": PROXIES, "desired": PROXIES}, self.config, self.state)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_user_managed_proxy_is_not_overwritten(self) -> None:
|
||||
original = json.dumps({"proxies": PROXIES})
|
||||
self.config.write_text(original)
|
||||
with (
|
||||
patch.object(proxy, "daemon_proxies", return_value=PROXIES),
|
||||
patch.object(proxy, "run") as run,
|
||||
self.assertRaisesRegex(proxy.ProxyError, "PLANET_PROXY_EXTERNAL"),
|
||||
):
|
||||
proxy.apply_plan({"current": PROXIES, "desired": {}}, self.config, self.state)
|
||||
run.assert_not_called()
|
||||
self.assertEqual(self.config.read_text(), original)
|
||||
|
||||
def test_external_edit_of_managed_config_is_not_overwritten(self) -> None:
|
||||
changed = {**PROXIES, "https-proxy": "http://different.example:8080"}
|
||||
self.config.write_text(json.dumps({"proxies": changed}))
|
||||
self.state.write_text(json.dumps({"managed": PROXIES}))
|
||||
with (
|
||||
patch.object(proxy, "daemon_proxies", return_value=changed),
|
||||
self.assertRaisesRegex(proxy.ProxyError, "PLANET_PROXY_EXTERNAL"),
|
||||
):
|
||||
proxy.apply_plan({"current": changed, "desired": {}}, self.config, self.state)
|
||||
|
||||
def test_restart_failure_restores_original_config_and_containers(self) -> None:
|
||||
original = b'{"log-driver": "local"}\n'
|
||||
self.config.write_bytes(original)
|
||||
restarts = 0
|
||||
|
||||
def fail_first_restart(args: list[str]) -> subprocess.CompletedProcess[str]:
|
||||
nonlocal restarts
|
||||
if args == ["systemctl", "restart", "docker"]:
|
||||
restarts += 1
|
||||
if restarts == 1:
|
||||
raise subprocess.CalledProcessError(1, args)
|
||||
return self.fake_run(args)
|
||||
|
||||
with (
|
||||
patch.object(proxy, "daemon_proxies", return_value={}),
|
||||
patch.object(proxy, "run", side_effect=fail_first_restart) as run,
|
||||
self.assertRaisesRegex(proxy.ProxyError, "PLANET_PROXY_CONFIG_FAILED"),
|
||||
):
|
||||
proxy.apply_plan({"current": {}, "desired": PROXIES}, self.config, self.state)
|
||||
self.assertEqual(self.config.read_bytes(), original)
|
||||
self.assertFalse(self.state.exists())
|
||||
self.assertEqual(restarts, 2)
|
||||
run.assert_any_call(["docker", "start", "postgres-id", "redis-id"])
|
||||
|
||||
def test_probe_does_not_put_proxy_credentials_in_command_arguments(self) -> None:
|
||||
with patch.object(
|
||||
proxy.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, "401")
|
||||
) as run:
|
||||
self.assertTrue(proxy.probe_url(URLS[0], "http://user:secret@proxy.example:1234"))
|
||||
self.assertNotIn("secret", " ".join(run.call_args.args[0]))
|
||||
self.assertIn("secret", run.call_args.kwargs["env"]["https_proxy"])
|
||||
|
||||
def test_registry_rejection_does_not_mark_the_proxy_unreachable(self) -> None:
|
||||
for status in ("401", "403", "429"):
|
||||
with (
|
||||
self.subTest(status=status),
|
||||
patch.object(
|
||||
proxy.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, status)
|
||||
),
|
||||
):
|
||||
self.assertTrue(proxy.probe_url(URLS[0], PROXIES["https-proxy"]))
|
||||
|
||||
def test_validation_failure_restores_config_without_restarting_docker(self) -> None:
|
||||
original = b'{"log-driver": "local"}\n'
|
||||
self.config.write_bytes(original)
|
||||
|
||||
def fail_validation(args: list[str]) -> subprocess.CompletedProcess[str]:
|
||||
if args[0] == "dockerd":
|
||||
raise subprocess.CalledProcessError(1, args)
|
||||
return self.fake_run(args)
|
||||
|
||||
with (
|
||||
patch.object(proxy, "daemon_proxies", return_value={}),
|
||||
patch.object(proxy, "run", side_effect=fail_validation) as run,
|
||||
self.assertRaisesRegex(proxy.ProxyError, "PLANET_PROXY_CONFIG_FAILED"),
|
||||
):
|
||||
proxy.apply_plan({"current": {}, "desired": PROXIES}, self.config, self.state)
|
||||
self.assertEqual(self.config.read_bytes(), original)
|
||||
self.assertFalse(any(call.args[0][0] == "systemctl" for call in run.call_args_list))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
152
scripts/harness/test_error_diagnostics.py
Normal file
152
scripts/harness/test_error_diagnostics.py
Normal file
@@ -0,0 +1,152 @@
|
||||
"""Verify catalog-driven diagnostics without touching Docker or live services."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from test_database_startup import run_shell, shell_function
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MODULE = ROOT / "scripts/lib/error-diagnostics.zsh"
|
||||
|
||||
|
||||
def catalog(language: str) -> dict[str, list[str]]:
|
||||
text = (ROOT / f"docs/technical/{language}/ops-runbook.md").read_text()
|
||||
entries = {}
|
||||
for line in text.splitlines():
|
||||
if line.startswith("| P_"):
|
||||
fields = [field.strip() for field in line.split("|")[1:-1]]
|
||||
assert len(fields) == 4, line
|
||||
assert fields[0] not in entries, fields[0]
|
||||
entries[fields[0]] = fields[1:]
|
||||
return entries
|
||||
|
||||
|
||||
class ErrorDiagnosticsTests(unittest.TestCase):
|
||||
def diagnose(self, message: str, evidence: str = "") -> list[str]:
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
log = Path(folder) / "build.log"
|
||||
log.write_text(evidence)
|
||||
result = subprocess.run(
|
||||
[
|
||||
"zsh",
|
||||
"-f",
|
||||
"-c",
|
||||
f"""
|
||||
SCRIPT_DIR={shlex.quote(str(ROOT))}
|
||||
source {shlex.quote(str(MODULE))}
|
||||
planet_error_record "$1" "$2"
|
||||
""",
|
||||
"test-diagnostics",
|
||||
message,
|
||||
str(log),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=True,
|
||||
)
|
||||
return result.stdout.strip().split("\t")
|
||||
|
||||
def test_bilingual_catalogs_have_identical_codes_and_matching_fragments(self) -> None:
|
||||
zh, en = catalog("zh"), catalog("en")
|
||||
self.assertGreater(len(zh), 0)
|
||||
self.assertEqual(list(zh), list(en))
|
||||
self.assertEqual(list(zh)[-1], "P_UNKNOWN")
|
||||
for code, (signals, reason, action) in zh.items():
|
||||
with self.subTest(code=code):
|
||||
self.assertEqual(signals, en[code][0])
|
||||
self.assertTrue(reason and action)
|
||||
self.assertNotEqual(reason, en[code][1])
|
||||
|
||||
def test_every_recorded_fragment_returns_the_exact_catalog_wording(self) -> None:
|
||||
for code, (signals, reason, action) in catalog("zh").items():
|
||||
for signal in signals.split(";"):
|
||||
with self.subTest(code=code, signal=signal):
|
||||
self.assertEqual(self.diagnose(signal), [code, reason, action])
|
||||
|
||||
def test_specific_build_evidence_precedes_generic_summary(self) -> None:
|
||||
for evidence, code in (
|
||||
(
|
||||
'Head "https://registry-1.docker.io/v2/test": dial tcp [::1]:443: i/o timeout',
|
||||
"P_NETWORK_TIMEOUT",
|
||||
),
|
||||
(
|
||||
'Head "https://ghcr.io/v2/test": net/http: TLS handshake timeout',
|
||||
"P_NETWORK_TIMEOUT",
|
||||
),
|
||||
("failed to solve: lookup registry.example: no such host", "P_DNS"),
|
||||
("X509: certificate signed by unknown authority", "P_TLS_CERT"),
|
||||
(
|
||||
"failed to solve: unexpected status from HEAD request: 429 Too Many Requests",
|
||||
"P_REGISTRY_RATE_LIMIT",
|
||||
),
|
||||
("new unexpected build error", "P_BUILD_FAILED"),
|
||||
):
|
||||
with self.subTest(code=code):
|
||||
result = self.diagnose("AI Provider 镜像构建失败", evidence + " TOKEN_SENTINEL")
|
||||
self.assertEqual(result[0], code)
|
||||
self.assertNotIn("TOKEN_SENTINEL", " ".join(result))
|
||||
|
||||
def test_unknown_error_does_not_claim_a_network_or_proxy_cause(self) -> None:
|
||||
self.assertEqual(self.diagnose("an unrecognized failure")[0], "P_UNKNOWN")
|
||||
|
||||
def test_existing_literal_shell_errors_have_catalog_entries(self) -> None:
|
||||
for path in (ROOT / "planet.sh", ROOT / "scripts/lib/docker-bootstrap.zsh"):
|
||||
for message in re.findall(r'^\s*log_error "([^"\n]+)"', path.read_text(), re.M):
|
||||
message = re.sub(r"\$\{[^}]+\}|\$[0-9]+", "", message)
|
||||
if not message.strip():
|
||||
continue # Dynamic retry failures are classified using their runtime message.
|
||||
with self.subTest(path=path.name, message=message):
|
||||
self.assertNotEqual(self.diagnose(message)[0], "P_UNKNOWN")
|
||||
|
||||
def test_log_error_reads_cause_and_remedy_from_the_table(self) -> None:
|
||||
result = subprocess.run(
|
||||
["zsh", "-f"],
|
||||
input=f"""
|
||||
SCRIPT_DIR={shlex.quote(str(ROOT))}
|
||||
source {shlex.quote(str(MODULE))}
|
||||
log_line() {{ echo "$3"; }}
|
||||
log_note() {{ echo "$1"; }}
|
||||
{shell_function('log_error')}
|
||||
log_error 'new failure'
|
||||
""",
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=True,
|
||||
)
|
||||
_, reason, action = catalog("zh")["P_UNKNOWN"]
|
||||
self.assertIn(f"原因 [P_UNKNOWN]: {reason}", result.stdout)
|
||||
self.assertIn(f"处理: {action}", result.stdout)
|
||||
|
||||
def test_verbose_build_preserves_failure_and_literal_log_path(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
binary = Path(folder) / "docker"
|
||||
binary.write_text("#!/bin/sh\nprintf 'TLS handshake timeout\\n'\nexit 17\n")
|
||||
binary.chmod(0o755)
|
||||
for verbose in (0, 1):
|
||||
with self.subTest(verbose=verbose):
|
||||
log = Path(folder) / "build ' quoted $(not-a-command).log"
|
||||
result = run_shell(
|
||||
["run_ai_provider_build_command"],
|
||||
f"""
|
||||
PATH={shlex.quote(folder + ':' + os.environ['PATH'])}
|
||||
VERBOSE={verbose}
|
||||
AI_PROVIDER_BUILD_LOG_FILE={shlex.quote(str(log))}
|
||||
run_command_with_spinner() {{ shift; "$@"; }}
|
||||
""",
|
||||
'if run_ai_provider_build_command "docker compose"; '
|
||||
"then exit 0; else exit $?; fi",
|
||||
)
|
||||
self.assertEqual(result.returncode, 17, result.stderr)
|
||||
self.assertEqual(log.read_text(), "TLS handshake timeout\n")
|
||||
self.assertNotIn("not-a-command", result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
50
scripts/lib/docker-proxy.zsh
Normal file
50
scripts/lib/docker-proxy.zsh
Normal file
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env zsh
|
||||
|
||||
prepare_docker_build_proxy() {
|
||||
# Desktop and remote/rootless daemons are managed in their own environment.
|
||||
docker_uses_local_engine || return 0
|
||||
docker_desktop_present && return 0
|
||||
|
||||
local python_bin="$(command -v python3)"
|
||||
local helper="$SCRIPT_DIR/scripts/docker_proxy.py"
|
||||
local plan_file error_file
|
||||
local mode changed connected status_text
|
||||
if [ -z "$python_bin" ] || ! plan_file="$(mktemp "$PLANET_STATE_DIR/docker-proxy.XXXXXX")"; then
|
||||
log_error "Docker 构建代理检测失败"
|
||||
return 1
|
||||
fi
|
||||
error_file="${plan_file}.error"
|
||||
chmod 600 "$plan_file"
|
||||
: > "$error_file"
|
||||
chmod 600 "$error_file"
|
||||
{
|
||||
if ! "$python_bin" "$helper" plan > "$plan_file" 2> "$error_file"; then
|
||||
log_error "Docker 构建代理检测失败" "$error_file"
|
||||
return 1
|
||||
fi
|
||||
if ! status_text="$("$python_bin" "$helper" status < "$plan_file" 2> "$error_file")"; then
|
||||
log_error "Docker 构建代理检测失败" "$error_file"
|
||||
return 1
|
||||
fi
|
||||
read -r mode changed connected <<< "$status_text"
|
||||
if [ "$changed" -eq 1 ]; then
|
||||
docker_require_sudo || return 1
|
||||
log_note "更新 Docker 构建代理配置,重启后恢复原先运行的容器"
|
||||
if ! docker_as_root "$python_bin" "$helper" apply < "$plan_file" 2> "$error_file"; then
|
||||
log_error "Docker 构建代理更新失败" "$error_file"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
if [ "$connected" -ne 1 ]; then
|
||||
log_error "PLANET_PROXY_NO_ROUTE"
|
||||
return 1
|
||||
fi
|
||||
if [ "$mode" = proxy ]; then
|
||||
log_note "Docker 构建使用已验证可用的主机代理"
|
||||
else
|
||||
log_note "未发现可用主机代理,Docker 构建使用直连"
|
||||
fi
|
||||
} always {
|
||||
rm -f -- "$plan_file" "$error_file"
|
||||
}
|
||||
}
|
||||
61
scripts/lib/error-diagnostics.zsh
Normal file
61
scripts/lib/error-diagnostics.zsh
Normal file
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env zsh
|
||||
|
||||
# The operator-facing table is also the runtime source of diagnostic wording.
|
||||
planet_error_record() {
|
||||
local message="$1"
|
||||
local evidence_file="${2:-/dev/null}"
|
||||
local catalog="$SCRIPT_DIR/docs/technical/zh/ops-runbook.md"
|
||||
[ -r "$catalog" ] || return 1
|
||||
[ -r "$evidence_file" ] || evidence_file=/dev/null
|
||||
|
||||
awk -F '|' -v message="$message" '
|
||||
function trim(value) {
|
||||
sub(/^[[:space:]]+/, "", value)
|
||||
sub(/[[:space:]]+$/, "", value)
|
||||
return value
|
||||
}
|
||||
NR == FNR {
|
||||
if ($0 == "<!-- planet-error-catalog:start -->") active = 1
|
||||
if ($0 == "<!-- planet-error-catalog:end -->") active = 0
|
||||
if (active && trim($2) ~ /^P_[A-Z_]+$/) {
|
||||
count++
|
||||
codes[count] = trim($2)
|
||||
signals[count] = tolower(trim($3))
|
||||
reasons[count] = trim($4)
|
||||
actions[count] = trim($5)
|
||||
if (codes[count] == "P_UNKNOWN") fallback = count
|
||||
}
|
||||
next
|
||||
}
|
||||
{ evidence = evidence "\n" tolower($0) }
|
||||
END {
|
||||
evidence = tolower(message) "\n" evidence
|
||||
selected = fallback
|
||||
for (i = 1; i <= count; i++) {
|
||||
if (i == fallback) continue
|
||||
size = split(signals[i], patterns, ";")
|
||||
for (j = 1; j <= size; j++) {
|
||||
pattern = trim(patterns[j])
|
||||
if (length(pattern) && index(evidence, pattern)) {
|
||||
selected = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if (selected != fallback) break
|
||||
}
|
||||
if (!selected) exit 1
|
||||
printf "%s\t%s\t%s\n", codes[selected], reasons[selected], actions[selected]
|
||||
}
|
||||
' "$catalog" "$evidence_file"
|
||||
}
|
||||
|
||||
report_error_reason() {
|
||||
local record code reason action
|
||||
if ! record="$(planet_error_record "$1" "${2:-/dev/null}")"; then
|
||||
log_note "无法读取运维错误对照表,请检查 docs/technical/zh/ops-runbook.md"
|
||||
return 0
|
||||
fi
|
||||
IFS=$'\t' read -r code reason action <<< "$record"
|
||||
log_note "原因 [${code}]: ${reason}"
|
||||
log_note "处理: ${action}"
|
||||
}
|
||||
Reference in New Issue
Block a user