fix: add controlled service restart console

backend/app/api/main.py

@@ -12,6 +12,7 @@ from app.api.v1 import (
     collected_data,
     visualization,
     bgp,
+    system_control,
 )
 
 api_router = APIRouter()
@@ -27,5 +28,6 @@ api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
 api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboard"])
 api_router.include_router(alerts.router, prefix="/alerts", tags=["alerts"])
 api_router.include_router(settings.router, prefix="/settings", tags=["settings"])
+api_router.include_router(system_control.router, prefix="/system", tags=["system"])
 api_router.include_router(visualization.router, prefix="/visualization", tags=["visualization"])
 api_router.include_router(bgp.router, prefix="/bgp", tags=["bgp"])

backend/app/api/v1/system_control.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import BaseModel
+
+from app.core.config import ROOT_DIR
+from app.core.security import get_current_user
+from app.models.user import User
+from app.services.system_control import (
+    build_task_id,
+    clear_active_task_id,
+    get_active_task_id,
+    get_allowed_command,
+    get_runner_script_path,
+    is_task_stale,
+    get_task_logs,
+    require_super_admin,
+    serialize_task,
+    set_active_task_id,
+    upsert_task_state,
+)
+
+router = APIRouter()
+
+
+class RestartTaskCreate(BaseModel):
+    action: str
+
+
+class RestartTaskResponse(BaseModel):
+    task_id: str
+    action: str
+    status: str
+    stage: str
+    message: str
+    created_at: str
+    updated_at: str
+    requested_by: dict[str, object] | None = None
+
+
+class RestartTaskLogsResponse(BaseModel):
+    task_id: str
+    lines: list[str]
+
+
+def ensure_super_admin(current_user: User) -> None:
+    if not require_super_admin(current_user.role):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Only super_admin can restart services",
+        )
+
+
+@router.post("/restart-tasks", response_model=RestartTaskResponse)
+async def create_restart_task(
+    payload: RestartTaskCreate,
+    current_user: User = Depends(get_current_user),
+):
+    ensure_super_admin(current_user)
+
+    command = get_allowed_command(payload.action)
+    if command is None:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Unsupported system action",
+        )
+
+    active_task_id = get_active_task_id()
+    if active_task_id:
+        active_task = serialize_task(active_task_id)
+        if active_task and is_task_stale(active_task):
+            upsert_task_state(
+                active_task_id,
+                status="failed",
+                stage="failed",
+                message="Previous restart task became stale and was released",
+            )
+            clear_active_task_id(active_task_id)
+        elif active_task and active_task.get("status") in {"queued", "running"}:
+            raise HTTPException(
+                status_code=status.HTTP_409_CONFLICT,
+                detail="Another restart task is already in progress",
+            )
+
+    task_id = build_task_id()
+    requested_by = {"id": current_user.id, "username": current_user.username}
+    task_state = upsert_task_state(
+        task_id,
+        action=payload.action,
+        status="queued",
+        stage="accepted",
+        message="Restart task accepted",
+        requested_by=requested_by,
+    )
+    set_active_task_id(task_id)
+
+    env = os.environ.copy()
+    backend_path = str(ROOT_DIR / "backend")
+    existing_pythonpath = env.get("PYTHONPATH", "")
+    env["PYTHONPATH"] = (
+        f"{backend_path}{os.pathsep}{existing_pythonpath}"
+        if existing_pythonpath
+        else backend_path
+    )
+
+    try:
+        subprocess.Popen(
+            [
+                sys.executable,
+                str(get_runner_script_path()),
+                "--task-id",
+                task_id,
+                "--action",
+                payload.action,
+            ],
+            cwd=str(ROOT_DIR),
+            env=env,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            start_new_session=True,
+        )
+    except OSError as exc:
+        task_state = upsert_task_state(
+            task_id,
+            action=payload.action,
+            status="failed",
+            stage="failed",
+            message=f"Unable to start restart runner: {exc}",
+            requested_by=requested_by,
+        )
+        clear_active_task_id(task_id)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=task_state["message"],
+        ) from exc
+
+    return task_state
+
+
+@router.get("/restart-tasks/{task_id}", response_model=RestartTaskResponse)
+async def get_restart_task(
+    task_id: str,
+    current_user: User = Depends(get_current_user),
+):
+    ensure_super_admin(current_user)
+
+    task = serialize_task(task_id)
+    if task is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Restart task not found")
+    return task
+
+
+@router.get("/restart-tasks/{task_id}/logs", response_model=RestartTaskLogsResponse)
+async def get_restart_task_logs(
+    task_id: str,
+    current_user: User = Depends(get_current_user),
+):
+    ensure_super_admin(current_user)
+
+    task = serialize_task(task_id)
+    if task is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Restart task not found")
+    return {"task_id": task_id, "lines": get_task_logs(task_id)}

backend/app/services/system_control.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import json
+import secrets
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from app.core.config import ROOT_DIR
+from app.core.security import redis_client
+
+SYSTEM_TASK_TTL_SECONDS = 24 * 60 * 60
+SYSTEM_TASK_LOG_LIMIT = 100
+SYSTEM_TASK_ACTIVE_KEY = "system:restart_task:active"
+SYSTEM_TASK_STALE_SECONDS = 5 * 60
+
+ALLOWED_ACTIONS: dict[str, dict[str, Any]] = {
+    "restart-backend": {
+        "command": ["./planet.sh", "restart", "-b"],
+        "recovery_mode": "backend",
+    },
+    "restart-database": {
+        "command": ["./planet.sh", "restart", "-d"],
+        "recovery_mode": "database",
+    },
+    "restart-system": {
+        "command": ["./planet.sh", "restart"],
+        "recovery_mode": "system",
+    },
+}
+
+
+def utc_now_iso() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+def normalize_user_role(role: Any) -> str:
+    return role.value if hasattr(role, "value") else str(role)
+
+
+def require_super_admin(user_role: Any) -> bool:
+    return normalize_user_role(user_role) == "super_admin"
+
+
+def build_task_id(prefix: str = "restart") -> str:
+    timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+    return f"{prefix}_{timestamp}_{secrets.token_hex(3)}"
+
+
+def get_task_key(task_id: str) -> str:
+    return f"system:restart_task:{task_id}"
+
+
+def get_task_logs_key(task_id: str) -> str:
+    return f"{get_task_key(task_id)}:logs"
+
+
+def get_allowed_command(action: str) -> list[str] | None:
+    config = ALLOWED_ACTIONS.get(action)
+    if config is None:
+        return None
+    return list(config["command"])
+
+
+def get_action_recovery_mode(action: str) -> str | None:
+    config = ALLOWED_ACTIONS.get(action)
+    if config is None:
+        return None
+    return str(config["recovery_mode"])
+
+
+def serialize_task(task_id: str) -> dict[str, Any] | None:
+    payload = redis_client.hgetall(get_task_key(task_id))
+    if not payload:
+        return None
+
+    if payload.get("requested_by"):
+        try:
+            payload["requested_by"] = json.loads(payload["requested_by"])
+        except json.JSONDecodeError:
+            payload["requested_by"] = {"username": payload["requested_by"]}
+
+    return payload
+
+
+def append_task_log(task_id: str, line: str) -> None:
+    logs_key = get_task_logs_key(task_id)
+    redis_client.rpush(logs_key, line)
+    redis_client.ltrim(logs_key, -SYSTEM_TASK_LOG_LIMIT, -1)
+    redis_client.expire(logs_key, SYSTEM_TASK_TTL_SECONDS)
+
+
+def upsert_task_state(
+    task_id: str,
+    *,
+    action: str | None = None,
+    status: str,
+    stage: str,
+    message: str,
+    requested_by: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    existing = serialize_task(task_id) or {}
+    now = utc_now_iso()
+
+    payload: dict[str, Any] = {
+        "task_id": task_id,
+        "action": action or existing.get("action") or "",
+        "status": status,
+        "stage": stage,
+        "message": message,
+        "created_at": existing.get("created_at") or now,
+        "updated_at": now,
+    }
+
+    if requested_by is not None:
+        payload["requested_by"] = requested_by
+    elif existing.get("requested_by") is not None:
+        payload["requested_by"] = existing["requested_by"]
+
+    redis_payload = {
+        key: json.dumps(value, ensure_ascii=False) if key == "requested_by" else str(value)
+        for key, value in payload.items()
+        if value is not None
+    }
+    task_key = get_task_key(task_id)
+    redis_client.hset(task_key, mapping=redis_payload)
+    redis_client.expire(task_key, SYSTEM_TASK_TTL_SECONDS)
+    return payload
+
+
+def get_task_logs(task_id: str) -> list[str]:
+    return [str(item) for item in redis_client.lrange(get_task_logs_key(task_id), 0, -1)]
+
+
+def get_active_task_id() -> str | None:
+    value = redis_client.get(SYSTEM_TASK_ACTIVE_KEY)
+    return str(value) if value else None
+
+
+def set_active_task_id(task_id: str) -> None:
+    redis_client.set(SYSTEM_TASK_ACTIVE_KEY, task_id, ex=SYSTEM_TASK_TTL_SECONDS)
+
+
+def clear_active_task_id(task_id: str) -> None:
+    current = get_active_task_id()
+    if current == task_id:
+        redis_client.delete(SYSTEM_TASK_ACTIVE_KEY)
+
+
+def parse_task_timestamp(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        return datetime.fromisoformat(value)
+    except ValueError:
+        return None
+
+
+def is_task_stale(task: dict[str, Any], *, max_age_seconds: int = SYSTEM_TASK_STALE_SECONDS) -> bool:
+    if task.get("status") not in {"queued", "running"}:
+        return False
+
+    updated_at = parse_task_timestamp(str(task.get("updated_at") or ""))
+    if updated_at is None:
+        return False
+
+    if updated_at.tzinfo is None:
+        updated_at = updated_at.replace(tzinfo=UTC)
+
+    return datetime.now(UTC) - updated_at > timedelta(seconds=max_age_seconds)
+
+
+def get_runner_script_path() -> Path:
+    return ROOT_DIR / "backend" / "scripts" / "system_restart_runner.py"

backend/scripts/system_restart_runner.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+from urllib.error import URLError
+from urllib.request import urlopen
+
+
+ROOT_DIR = Path(__file__).resolve().parents[2]
+BACKEND_DIR = ROOT_DIR / "backend"
+
+if str(BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(BACKEND_DIR))
+
+from app.services.system_control import (  # noqa: E402
+    append_task_log,
+    clear_active_task_id,
+    get_allowed_command,
+    get_action_recovery_mode,
+    upsert_task_state,
+)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--task-id", required=True)
+    parser.add_argument("--action", required=True)
+    return parser.parse_args()
+
+
+def wait_for_http(url: str, timeout_seconds: int = 90, interval_seconds: float = 2.0) -> bool:
+    deadline = time.time() + timeout_seconds
+    success_streak = 0
+
+    while time.time() < deadline:
+        try:
+            with urlopen(url, timeout=2) as response:
+                if response.status == 200:
+                    success_streak += 1
+                    if success_streak >= 2:
+                        return True
+                else:
+                    success_streak = 0
+        except URLError:
+            success_streak = 0
+        except Exception:
+            success_streak = 0
+
+        time.sleep(interval_seconds)
+
+    return False
+
+
+def wait_for_recovery(action: str) -> tuple[bool, str]:
+    recovery_mode = get_action_recovery_mode(action)
+    if recovery_mode == "backend":
+        return wait_for_http("http://localhost:8000/health"), "backend health recovery"
+    if recovery_mode == "database":
+        return True, "database container restart completion"
+    if recovery_mode == "system":
+        backend_ok = wait_for_http("http://localhost:8000/health")
+        frontend_ok = wait_for_http("http://localhost:3000")
+        return backend_ok and frontend_ok, "system service recovery"
+    return False, "unsupported recovery mode"
+
+
+def main() -> int:
+    args = parse_args()
+    command = get_allowed_command(args.action)
+    recovery_mode = get_action_recovery_mode(args.action)
+    if command is None or recovery_mode is None:
+        upsert_task_state(
+            args.task_id,
+            action=args.action,
+            status="failed",
+            stage="failed",
+            message="Unsupported system action",
+        )
+        clear_active_task_id(args.task_id)
+        return 1
+
+    upsert_task_state(
+        args.task_id,
+        action=args.action,
+        status="running",
+        stage="spawning",
+        message="Spawning restart command",
+    )
+    append_task_log(args.task_id, f"accepted {args.action} request")
+    append_task_log(args.task_id, f"resolved command: {' '.join(command)}")
+
+    env = os.environ.copy()
+    env["PATH"] = f"{Path.home() / '.bun' / 'bin'}:{Path.home() / '.local' / 'bin'}:{env.get('PATH', '')}"
+
+    try:
+        upsert_task_state(
+            args.task_id,
+            action=args.action,
+            status="running",
+            stage="stopping",
+            message="Restart command is running",
+        )
+        append_task_log(args.task_id, "restart command started")
+
+        completed = subprocess.run(
+            command,
+            cwd=str(ROOT_DIR),
+            env=env,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+        if completed.stdout.strip():
+            for line in completed.stdout.strip().splitlines()[-20:]:
+                append_task_log(args.task_id, line)
+        if completed.stderr.strip():
+            for line in completed.stderr.strip().splitlines()[-20:]:
+                append_task_log(args.task_id, line)
+
+        if completed.returncode != 0:
+            upsert_task_state(
+                args.task_id,
+                action=args.action,
+                status="failed",
+                stage="failed",
+                message=f"Restart command failed with exit code {completed.returncode}",
+            )
+            clear_active_task_id(args.task_id)
+            return completed.returncode
+
+        upsert_task_state(
+            args.task_id,
+            action=args.action,
+            status="running",
+            stage="waiting_for_health",
+            message=f"Waiting for {recovery_mode} recovery",
+        )
+        append_task_log(args.task_id, f"waiting for {recovery_mode} recovery")
+
+        recovered, recovery_label = wait_for_recovery(args.action)
+        if recovered:
+            upsert_task_state(
+                args.task_id,
+                action=args.action,
+                status="succeeded",
+                stage="healthy",
+                message="Restart completed successfully",
+            )
+            append_task_log(args.task_id, f"{recovery_label} completed")
+            clear_active_task_id(args.task_id)
+            return 0
+
+        upsert_task_state(
+            args.task_id,
+            action=args.action,
+            status="timeout",
+            stage="failed",
+            message="Restart timed out waiting for recovery",
+        )
+        append_task_log(args.task_id, f"{recovery_label} timed out")
+        clear_active_task_id(args.task_id)
+        return 2
+    except Exception as exc:
+        append_task_log(args.task_id, f"runner exception: {exc}")
+        upsert_task_state(
+            args.task_id,
+            action=args.action,
+            status="failed",
+            stage="failed",
+            message=f"Restart runner failed: {exc}",
+        )
+        clear_active_task_id(args.task_id)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())