Files
planet/backend/app/api/v1/system_control.py
rayd1o b18ffa0b0a
Some checks failed
ci / backend (push) Has been cancelled
ci / frontend (push) Has been cancelled
ci / delivery (push) Has been cancelled
release / images (push) Has been cancelled
release: bump version to 0.67.0
2026-05-27 13:50:16 +08:00

469 lines
14 KiB
Python

from __future__ import annotations
import os
import subprocess
import sys
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import ROOT_DIR
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.user import User
from app.services.persistent_logs import record_audit_log, record_system_log
from app.services.system_control import (
build_task_id,
clear_active_task_id,
get_active_task_id,
get_allowed_command,
get_runner_script_path,
is_task_stale,
get_task_logs,
require_super_admin,
serialize_task,
set_active_task_id,
upsert_task_state,
)
from app.services.system_logs import (
DEFAULT_LOG_LINE_LIMIT,
MAX_LOG_LINE_LIMIT,
SUPPORTED_LOG_LEVELS,
append_buffer_log,
list_log_sources,
normalize_log_level,
read_database_log_snapshot,
read_log_snapshot,
)
from app.services.earth_layer_cache import earth_layer_cache
router = APIRouter()
class RestartTaskCreate(BaseModel):
action: str
class RestartTaskResponse(BaseModel):
task_id: str
action: str
status: str
stage: str
message: str
created_at: str
updated_at: str
requested_by: dict[str, object] | None = None
class RestartTaskLogsResponse(BaseModel):
task_id: str
lines: list[str]
class SystemLogSourceSummary(BaseModel):
source_id: str
name: str
kind: str
location: str
description: str
category: str
status: str
class SystemLogSourcesResponse(BaseModel):
items: list[SystemLogSourceSummary]
class SystemLogDailyMarker(BaseModel):
date_token: str
total: int
dominant_level: str
class SystemLogSnapshotResponse(BaseModel):
source_id: str
name: str
kind: str
location: str
description: str
category: str
status: str
level: str
selected_levels: list[str] = []
search_query: str = ""
available_levels: list[str]
daily_markers: list[SystemLogDailyMarker] = []
line_limit: int
line_count: int
lines: list[str]
class EarthClientLogEventCreate(BaseModel):
level: str = "error"
message: str
category: str | None = None
url: str | None = None
module: str | None = None
detail: str | None = None
class EarthClientLogEventResponse(BaseModel):
accepted: bool
source_id: str
level: str
async def ingest_client_log_event(
source_id: str,
*,
service: str,
event: str,
default_module: str,
default_category: str,
payload: EarthClientLogEventCreate,
request: Request,
) -> EarthClientLogEventResponse:
normalized_level = normalize_log_level(payload.level)
append_buffer_log(
source_id,
level=normalized_level,
message=payload.message,
context={
"category": payload.category or "",
"url": payload.url or "",
"module": payload.module or "",
"detail": payload.detail or "",
},
)
await record_system_log(
source=source_id,
service=service,
module=payload.module or default_module,
event=event,
level=normalized_level,
message=payload.message,
category=payload.category or default_category,
context={
"url": payload.url or "",
"detail": payload.detail or "",
"module": payload.module or "",
"client_ip": request.client.host if request.client else "",
},
)
return EarthClientLogEventResponse(accepted=True, source_id=source_id, level=normalized_level)
class EarthLayerCacheStatusResponse(BaseModel):
prefix: str
key_count: int
memory_bytes: int
layers: dict[str, dict[str, int]]
class EarthLayerCacheClearResponse(BaseModel):
deleted: int
def ensure_super_admin(current_user: User) -> None:
if not require_super_admin(current_user.role):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can restart services",
)
def validate_log_date(raw_value: str | None, field_name: str) -> str | None:
if raw_value in {None, ""}:
return None
try:
return datetime.strptime(raw_value, "%Y-%m-%d").date().isoformat()
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"{field_name} must be in YYYY-MM-DD format",
) from exc
@router.get("/cache/earth-layers", response_model=EarthLayerCacheStatusResponse)
async def get_earth_layer_cache_status(
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
try:
return earth_layer_cache.status()
except Exception as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=f"Unable to read Earth layer cache status: {exc}",
) from exc
@router.delete("/cache/earth-layers", response_model=EarthLayerCacheClearResponse)
async def clear_earth_layer_cache(
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
try:
return {"deleted": earth_layer_cache.delete_pattern()}
except Exception as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=f"Unable to clear Earth layer cache: {exc}",
) from exc
@router.post("/restart-tasks", response_model=RestartTaskResponse)
async def create_restart_task(
payload: RestartTaskCreate,
request: Request,
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
command = get_allowed_command(payload.action)
if command is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Unsupported system action",
)
active_task_id = get_active_task_id()
if active_task_id:
active_task = serialize_task(active_task_id)
if active_task and is_task_stale(active_task):
upsert_task_state(
active_task_id,
status="failed",
stage="failed",
message="Previous restart task became stale and was released",
)
clear_active_task_id(active_task_id)
elif active_task and active_task.get("status") in {"queued", "running"}:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="Another restart task is already in progress",
)
task_id = build_task_id()
requested_by = {"id": current_user.id, "username": current_user.username}
task_state = upsert_task_state(
task_id,
action=payload.action,
status="queued",
stage="accepted",
message="Restart task accepted",
requested_by=requested_by,
)
set_active_task_id(task_id)
env = os.environ.copy()
backend_path = str(ROOT_DIR / "backend")
existing_pythonpath = env.get("PYTHONPATH", "")
env["PYTHONPATH"] = (
f"{backend_path}{os.pathsep}{existing_pythonpath}"
if existing_pythonpath
else backend_path
)
try:
subprocess.Popen(
[
sys.executable,
str(get_runner_script_path()),
"--task-id",
task_id,
"--action",
payload.action,
],
cwd=str(ROOT_DIR),
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True,
)
except OSError as exc:
task_state = upsert_task_state(
task_id,
action=payload.action,
status="failed",
stage="failed",
message=f"Unable to start restart runner: {exc}",
requested_by=requested_by,
)
clear_active_task_id(task_id)
await record_audit_log(
action="system.restart_task.requested",
actor_id=current_user.id,
actor_name=current_user.username,
target_type="restart_task",
target_id=task_id,
result="failed",
ip=request.client.host if request.client else None,
details={"action": payload.action, "message": task_state["message"]},
)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=task_state["message"],
) from exc
await record_audit_log(
action="system.restart_task.requested",
actor_id=current_user.id,
actor_name=current_user.username,
target_type="restart_task",
target_id=task_id,
result="accepted",
ip=request.client.host if request.client else None,
details={"action": payload.action},
)
return task_state
@router.get("/restart-tasks/{task_id}", response_model=RestartTaskResponse)
async def get_restart_task(
task_id: str,
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
task = serialize_task(task_id)
if task is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Restart task not found")
return task
@router.get("/restart-tasks/{task_id}/logs", response_model=RestartTaskLogsResponse)
async def get_restart_task_logs(
task_id: str,
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
task = serialize_task(task_id)
if task is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Restart task not found")
return {"task_id": task_id, "lines": get_task_logs(task_id)}
@router.get("/logs/sources", response_model=SystemLogSourcesResponse)
async def get_system_log_sources(
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
return {
"items": [
*list_log_sources(),
{
"source_id": "system-db",
"name": "系统事件",
"kind": "database",
"location": "table://system_logs",
"description": "后端持久化系统事件、AI 和采集器操作日志。",
"category": "database",
"status": "ok",
},
{
"source_id": "audit-db",
"name": "审计事件",
"kind": "database",
"location": "table://audit_logs",
"description": "管理员敏感操作和密钥 reveal 审计记录。",
"category": "audit",
"status": "ok",
},
]
}
@router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse)
async def get_system_log_snapshot(
source_id: str,
limit: int = DEFAULT_LOG_LINE_LIMIT,
level: str = "all",
levels: str | None = Query(None, description="Comma-separated log levels"),
start_date: str | None = Query(None, description="Filter logs from this date (YYYY-MM-DD)"),
end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"),
search: str | None = Query(None, description="Case-insensitive substring search"),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
ensure_super_admin(current_user)
if limit < 1 or limit > MAX_LOG_LINE_LIMIT:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"limit must be between 1 and {MAX_LOG_LINE_LIMIT}",
)
if str(level).strip().lower() not in SUPPORTED_LOG_LEVELS and normalize_log_level(level) == "all" and str(level).strip().lower() not in {"", "all"}:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unsupported log level")
if levels:
for raw_level in str(levels).split(","):
normalized_level = str(raw_level).strip().lower()
if not normalized_level:
continue
if normalized_level not in SUPPORTED_LOG_LEVELS and normalize_log_level(normalized_level) == "all":
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unsupported log level")
normalized_start_date = validate_log_date(start_date, "start_date")
normalized_end_date = validate_log_date(end_date, "end_date")
if normalized_start_date and normalized_end_date and normalized_start_date > normalized_end_date:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="start_date must be earlier than or equal to end_date")
snapshot = await read_database_log_snapshot(
source_id,
limit=limit,
level=level,
levels=levels,
start_date=normalized_start_date,
end_date=normalized_end_date,
search=search,
db=db,
)
if snapshot is None:
snapshot = read_log_snapshot(
source_id,
limit,
level=level,
levels=levels,
start_date=normalized_start_date,
end_date=normalized_end_date,
search=search,
)
if snapshot is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Log source not found")
return snapshot
@router.post("/logs/earth-client", response_model=EarthClientLogEventResponse)
async def ingest_earth_client_log(
payload: EarthClientLogEventCreate,
request: Request,
):
return await ingest_client_log_event(
"earth-client",
service="earth",
event="earth.client.runtime_log",
default_module="earth-client",
default_category="client-runtime",
payload=payload,
request=request,
)
@router.post("/logs/admin-client", response_model=EarthClientLogEventResponse)
async def ingest_admin_client_log(
payload: EarthClientLogEventCreate,
request: Request,
):
return await ingest_client_log_event(
"admin-client",
service="admin",
event="admin.client.runtime_log",
default_module="admin-client",
default_category="client-runtime",
payload=payload,
request=request,
)