From 7abf391c74fbebed7be0e9e0bc2286c1a00c72d0 Mon Sep 17 00:00:00 2001 From: rayd1o Date: Tue, 7 Apr 2026 23:25:13 +0800 Subject: [PATCH] Harden startup retry and recovery flow --- README.md | 26 +++++ docs/CHANGELOG.md | 2 + planet.sh | 281 +++++++++++++++++++++++++++++++++++++--------- 3 files changed, 254 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 25665855..1df787fb 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,32 @@ 启动服务后访问: `http://localhost:8000/docs` +## 启动容错参数 + +`planet.sh` 现在为依赖安装、数据库、AI Provider 启动加入了有限次重试，并会在数据库与 `aiprovider` 启动后额外等待 Docker healthcheck。 + +可通过环境变量临时调整: + +```bash +# 例: 放宽 AI Provider 与数据库在网络抖动下的自愈次数 +AI_PROVIDER_START_MAX_RETRIES=5 \ +AI_PROVIDER_RETRY_INTERVAL=10 \ +DATABASE_START_MAX_RETRIES=5 \ +DATABASE_RETRY_INTERVAL=10 \ +./planet.sh restart +``` + +常用参数: + +- `DEPENDENCY_INSTALL_MAX_RETRIES` / `DEPENDENCY_INSTALL_RETRY_INTERVAL`: 控制 `uv sync`、`bun install` 的重试次数与间隔，默认 `3` 次、`5` 秒 +- `DATABASE_START_MAX_RETRIES` / `DATABASE_RETRY_INTERVAL`: 控制 `postgres`、`redis` 的启动/重启与健康检查自愈，默认 `3` 次、`5` 秒 +- `AI_PROVIDER_START_MAX_RETRIES` / `AI_PROVIDER_RETRY_INTERVAL`: 控制 `aiprovider` 的构建/启动与容器重启自愈，默认 `3` 次、`5` 秒 +- `BACKEND_MAX_RETRIES`: 控制后端进程启动重试次数，默认 `3` +- `FRONTEND_MAX_RETRIES`: 控制前端 dev server 启动重试次数，默认 `3` +- `BACKEND_HEALTH_CHECK_ATTEMPTS` / `BACKEND_HEALTH_CHECK_INTERVAL`: 控制后端 HTTP 健康检查等待次数与间隔，默认 `10` 次、`2` 秒 +- `FRONTEND_HEALTH_CHECK_ATTEMPTS` / `FRONTEND_HEALTH_CHECK_INTERVAL`: 控制前端 HTTP 可访问检查等待次数与间隔，默认 `10` 次、`2` 秒 +- `AI_PROVIDER_HEALTH_CHECK_ATTEMPTS` / `AI_PROVIDER_HEALTH_CHECK_INTERVAL`: 控制 `aiprovider` HTTP 健康检查等待次数与间隔，默认 `10` 次、`2` 秒 + ## AI 接口预留 项目现在采用“两层”设计: diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0407a8de..301e72bb 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -31,6 +31,8 @@ Released: 2026-04-07 - Improved backend-to-provider tracing by propagating `X-Request-ID` through the AI call chain and returning the same header from both backend and `aiprovider`. - Improved resilience by adding lightweight retry handling to both `backend -> aiprovider` and `aiprovider -> model provider` HTTP calls. - Improved operator workflow by folding `aiprovider` startup, health checks, restart support, and log viewing into [planet.sh](/home/ray/dev/linkong/planet/planet.sh). +- Improved [planet.sh](/home/ray/dev/linkong/planet/planet.sh) by adding bounded retry and container-health self-recovery for dependency installs, database startup, `aiprovider` startup, and interactive PostgreSQL boot paths. +- Improved [README.md](/home/ray/dev/linkong/planet/README.md) by documenting the new `planet.sh` retry and health-check tuning environment variables with concrete override examples. - Improved container consistency by switching [backend/Dockerfile](/home/ray/dev/linkong/planet/backend/Dockerfile) and [aiprovider/Dockerfile](/home/ray/dev/linkong/planet/aiprovider/Dockerfile) to `uv sync` / `uv run`. ### Changed diff --git a/planet.sh b/planet.sh index b487a830..2e2ae5bc 100755 --- a/planet.sh +++ b/planet.sh @@ -14,8 +14,14 @@ NC='\033[0m' BACKEND_MAX_RETRIES="${BACKEND_MAX_RETRIES:-3}" BACKEND_HEALTH_CHECK_ATTEMPTS="${BACKEND_HEALTH_CHECK_ATTEMPTS:-10}" BACKEND_HEALTH_CHECK_INTERVAL="${BACKEND_HEALTH_CHECK_INTERVAL:-2}" +DEPENDENCY_INSTALL_MAX_RETRIES="${DEPENDENCY_INSTALL_MAX_RETRIES:-3}" +DEPENDENCY_INSTALL_RETRY_INTERVAL="${DEPENDENCY_INSTALL_RETRY_INTERVAL:-5}" AI_PROVIDER_HEALTH_CHECK_ATTEMPTS="${AI_PROVIDER_HEALTH_CHECK_ATTEMPTS:-10}" AI_PROVIDER_HEALTH_CHECK_INTERVAL="${AI_PROVIDER_HEALTH_CHECK_INTERVAL:-2}" +AI_PROVIDER_START_MAX_RETRIES="${AI_PROVIDER_START_MAX_RETRIES:-3}" +AI_PROVIDER_RETRY_INTERVAL="${AI_PROVIDER_RETRY_INTERVAL:-5}" +DATABASE_START_MAX_RETRIES="${DATABASE_START_MAX_RETRIES:-3}" +DATABASE_RETRY_INTERVAL="${DATABASE_RETRY_INTERVAL:-5}" FRONTEND_MAX_RETRIES="${FRONTEND_MAX_RETRIES:-3}" FRONTEND_HEALTH_CHECK_ATTEMPTS="${FRONTEND_HEALTH_CHECK_ATTEMPTS:-10}" FRONTEND_HEALTH_CHECK_INTERVAL="${FRONTEND_HEALTH_CHECK_INTERVAL:-2}" @@ -31,6 +37,33 @@ compose_up() { fi } +run_with_retry() { + local max_retries="$1" + local interval="$2" + local failure_message="$3" + local retry_label="$4" + shift 4 + + local attempt=1 + + while [ "$attempt" -le "$max_retries" ]; do + if "$@"; then + return 0 + fi + + if [ "$attempt" -eq "$max_retries" ]; then + echo -e "${RED}❌ ${failure_message}${NC}" + return 1 + fi + + echo -e "${YELLOW}⚠️ ${retry_label} 第 ${attempt}/${max_retries} 次失败，${interval} 秒后重试...${NC}" + sleep "$interval" + attempt=$((attempt + 1)) + done + + return 1 +} + ensure_uv_backend_deps() { echo -e "${BLUE}📦 检查后端 uv 环境...${NC}" @@ -43,7 +76,14 @@ ensure_uv_backend_deps() { if [ ! -x "$SCRIPT_DIR/.venv/bin/python" ]; then echo -e "${YELLOW}⚠️ 未检测到 .venv，正在执行 uv sync...${NC}" - uv sync --group dev + if ! run_with_retry \ + "$DEPENDENCY_INSTALL_MAX_RETRIES" \ + "$DEPENDENCY_INSTALL_RETRY_INTERVAL" \ + "uv 环境初始化失败，已重试 ${DEPENDENCY_INSTALL_MAX_RETRIES} 次" \ + "uv sync" \ + uv sync --group dev; then + exit 1 + fi fi if [ ! -x "$SCRIPT_DIR/.venv/bin/python" ]; then @@ -64,7 +104,14 @@ ensure_frontend_deps() { if [ ! -x "$SCRIPT_DIR/frontend/node_modules/.bin/vite" ]; then echo -e "${YELLOW}⚠️ 前端依赖缺失，正在执行 bun install...${NC}" - bun install + if ! run_with_retry \ + "$DEPENDENCY_INSTALL_MAX_RETRIES" \ + "$DEPENDENCY_INSTALL_RETRY_INTERVAL" \ + "前端依赖安装失败，已重试 ${DEPENDENCY_INSTALL_MAX_RETRIES} 次" \ + "bun install" \ + bun install; then + exit 1 + fi fi if [ ! -x "$SCRIPT_DIR/frontend/node_modules/.bin/vite" ]; then @@ -93,6 +140,50 @@ wait_for_http() { return 1 } +wait_for_container_health() { + local container_name="$1" + local attempts="$2" + local interval="$3" + local service_name="$4" + local attempt=1 + local status + + while [ "$attempt" -le "$attempts" ]; do + status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_name" 2>/dev/null || true)" + + if [ "$status" = "healthy" ] || [ "$status" = "running" ]; then + return 0 + fi + + echo -e "${YELLOW}⏳ 等待${service_name}容器健康检查通过...${NC}" + sleep "$interval" + attempt=$((attempt + 1)) + done + + return 1 +} + +wait_for_database_health() { + wait_for_container_health "planet_postgres" "$AI_PROVIDER_HEALTH_CHECK_ATTEMPTS" "$DATABASE_RETRY_INTERVAL" "PostgreSQL" && + wait_for_container_health "planet_redis" "$AI_PROVIDER_HEALTH_CHECK_ATTEMPTS" "$DATABASE_RETRY_INTERVAL" "Redis" +} + +wait_for_postgres_health() { + wait_for_container_health "planet_postgres" "$AI_PROVIDER_HEALTH_CHECK_ATTEMPTS" "$DATABASE_RETRY_INTERVAL" "PostgreSQL" +} + +start_database_services() { + docker start planet_postgres planet_redis 2>/dev/null || compose_up up -d postgres redis +} + +restart_database_services() { + docker restart planet_postgres planet_redis 2>/dev/null || compose_up up -d postgres redis +} + +start_postgres_service() { + docker start planet_postgres 2>/dev/null || compose_up up -d postgres +} + start_backend_with_retry() { local backend_port="$1" local retry=1 @@ -117,14 +208,33 @@ start_backend_with_retry() { start_ai_provider_service() { local ai_provider_port="${1:-$DEFAULT_AI_PROVIDER_PORT}" + local retry=1 echo -e "${BLUE}🧠 启动 AI Provider...${NC}" - docker start planet_aiprovider 2>/dev/null || compose_up up -d aiprovider - if ! wait_for_http "http://localhost:${ai_provider_port}/health" "$AI_PROVIDER_HEALTH_CHECK_ATTEMPTS" "$AI_PROVIDER_HEALTH_CHECK_INTERVAL" "AI Provider"; then - echo -e "${RED}❌ AI Provider 启动失败${NC}" - exit 1 - fi + while [ "$retry" -le "$AI_PROVIDER_START_MAX_RETRIES" ]; do + if docker start planet_aiprovider 2>/dev/null || compose_up up -d aiprovider; then + if wait_for_container_health "planet_aiprovider" "$AI_PROVIDER_HEALTH_CHECK_ATTEMPTS" "$AI_PROVIDER_HEALTH_CHECK_INTERVAL" "AI Provider" && + wait_for_http "http://localhost:${ai_provider_port}/health" "$AI_PROVIDER_HEALTH_CHECK_ATTEMPTS" "$AI_PROVIDER_HEALTH_CHECK_INTERVAL" "AI Provider"; then + return 0 + fi + fi + + if [ "$retry" -eq "$AI_PROVIDER_START_MAX_RETRIES" ]; then + echo -e "${RED}❌ AI Provider 启动失败，已重试 ${AI_PROVIDER_START_MAX_RETRIES} 次${NC}" + docker logs --tail 20 planet_aiprovider 2>/dev/null || true + exit 1 + fi + + echo -e "${YELLOW}⚠️ AI Provider 第 ${retry}/${AI_PROVIDER_START_MAX_RETRIES} 次启动后仍未健康，正在重启容器...${NC}" + docker restart planet_aiprovider 2>/dev/null || true + sleep "$AI_PROVIDER_RETRY_INTERVAL" + retry=$((retry + 1)) + done + + echo -e "${RED}❌ AI Provider 启动失败${NC}" + docker logs --tail 20 planet_aiprovider 2>/dev/null || true + exit 1 } start_frontend_with_retry() { @@ -150,6 +260,114 @@ start_frontend_with_retry() { return 1 } +ensure_database_services_healthy() { + local retry=1 + + while [ "$retry" -le "$DATABASE_START_MAX_RETRIES" ]; do + if start_database_services && wait_for_database_health; then + return 0 + fi + + if [ "$retry" -eq "$DATABASE_START_MAX_RETRIES" ]; then + echo -e "${RED}❌ 数据库启动失败，已重试 ${DATABASE_START_MAX_RETRIES} 次${NC}" + docker logs --tail 20 planet_postgres 2>/dev/null || true + docker logs --tail 20 planet_redis 2>/dev/null || true + exit 1 + fi + + echo -e "${YELLOW}⚠️ 数据库第 ${retry}/${DATABASE_START_MAX_RETRIES} 次启动后仍未健康，正在重启容器...${NC}" + docker restart planet_postgres planet_redis 2>/dev/null || true + sleep "$DATABASE_RETRY_INTERVAL" + retry=$((retry + 1)) + done +} + +ensure_postgres_service_healthy() { + local retry=1 + + while [ "$retry" -le "$DATABASE_START_MAX_RETRIES" ]; do + if start_postgres_service && wait_for_postgres_health; then + return 0 + fi + + if [ "$retry" -eq "$DATABASE_START_MAX_RETRIES" ]; then + echo -e "${RED}❌ PostgreSQL 启动失败，已重试 ${DATABASE_START_MAX_RETRIES} 次${NC}" + docker logs --tail 20 planet_postgres 2>/dev/null || true + exit 1 + fi + + echo -e "${YELLOW}⚠️ PostgreSQL 第 ${retry}/${DATABASE_START_MAX_RETRIES} 次启动后仍未健康，正在重启容器...${NC}" + docker restart planet_postgres 2>/dev/null || true + sleep "$DATABASE_RETRY_INTERVAL" + retry=$((retry + 1)) + done +} + +restart_database_service() { + local retry=1 + + echo -e "${BLUE}🗄️ 重启数据库...${NC}" + + while [ "$retry" -le "$DATABASE_START_MAX_RETRIES" ]; do + if restart_database_services && wait_for_database_health; then + sleep 3 + return 0 + fi + + if [ "$retry" -eq "$DATABASE_START_MAX_RETRIES" ]; then + echo -e "${RED}❌ 数据库重启失败，已重试 ${DATABASE_START_MAX_RETRIES} 次${NC}" + docker logs --tail 20 planet_postgres 2>/dev/null || true + docker logs --tail 20 planet_redis 2>/dev/null || true + exit 1 + fi + + echo -e "${YELLOW}⚠️ 数据库第 ${retry}/${DATABASE_START_MAX_RETRIES} 次重启后仍未健康，继续重试...${NC}" + sleep "$DATABASE_RETRY_INTERVAL" + retry=$((retry + 1)) + done +} + +start_backend_service() { + local backend_port="$1" + local backend_port_requested="$2" + local ai_provider_port="${3:-$DEFAULT_AI_PROVIDER_PORT}" + + echo -e "${BLUE}🗄️ 启动数据库...${NC}" + ensure_database_services_healthy + sleep 3 + + start_ai_provider_service "$ai_provider_port" + + if [ "$backend_port_requested" -eq 1 ]; then + kill_port_if_requested "$backend_port" "后端" + fi + + echo -e "${BLUE}🔧 启动后端...${NC}" + ensure_uv_backend_deps + if ! start_backend_with_retry "$backend_port"; then + echo -e "${RED}❌ 后端启动失败，已重试 ${BACKEND_MAX_RETRIES} 次${NC}" + tail -10 /tmp/planet_backend.log + exit 1 + fi +} + +start_frontend_service() { + local frontend_port="$1" + local frontend_port_requested="$2" + + if [ "$frontend_port_requested" -eq 1 ]; then + kill_port_if_requested "$frontend_port" "前端" + fi + + echo -e "${BLUE}🌐 启动前端...${NC}" + ensure_frontend_deps + if ! start_frontend_with_retry "$frontend_port"; then + echo -e "${RED}❌ 前端启动失败，已重试 ${FRONTEND_MAX_RETRIES} 次${NC}" + tail -10 /tmp/planet_frontend.log + exit 1 + fi +} + validate_port() { local port="$1" @@ -263,53 +481,6 @@ stop_frontend_service() { pkill -f "bun run dev" 2>/dev/null || true } -restart_database_service() { - echo -e "${BLUE}🗄️ 重启数据库...${NC}" - docker restart planet_postgres planet_redis 2>/dev/null || compose_up up -d postgres redis - sleep 3 -} - -start_backend_service() { - local backend_port="$1" - local backend_port_requested="$2" - local ai_provider_port="${3:-$DEFAULT_AI_PROVIDER_PORT}" - - echo -e "${BLUE}🗄️ 启动数据库...${NC}" - docker start planet_postgres planet_redis 2>/dev/null || compose_up up -d postgres redis - sleep 3 - - start_ai_provider_service "$ai_provider_port" - - if [ "$backend_port_requested" -eq 1 ]; then - kill_port_if_requested "$backend_port" "后端" - fi - - echo -e "${BLUE}🔧 启动后端...${NC}" - ensure_uv_backend_deps - if ! start_backend_with_retry "$backend_port"; then - echo -e "${RED}❌ 后端启动失败，已重试 ${BACKEND_MAX_RETRIES} 次${NC}" - tail -10 /tmp/planet_backend.log - exit 1 - fi -} - -start_frontend_service() { - local frontend_port="$1" - local frontend_port_requested="$2" - - if [ "$frontend_port_requested" -eq 1 ]; then - kill_port_if_requested "$frontend_port" "前端" - fi - - echo -e "${BLUE}🌐 启动前端...${NC}" - ensure_frontend_deps - if ! start_frontend_with_retry "$frontend_port"; then - echo -e "${RED}❌ 前端启动失败，已重试 ${FRONTEND_MAX_RETRIES} 次${NC}" - tail -10 /tmp/planet_frontend.log - exit 1 - fi -} - create_user() { local username local password @@ -322,7 +493,7 @@ create_user() { ensure_uv_backend_deps echo -e "${BLUE}🗄️ 启动数据库...${NC}" - docker start planet_postgres 2>/dev/null || compose_up up -d postgres + ensure_postgres_service_healthy sleep 2 echo -e "${BLUE}👤 创建用户${NC}"