355 lines
11 KiB
Python
355 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
from datetime import datetime
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
|
from pydantic import BaseModel
|
|
|
|
from app.core.config import ROOT_DIR
|
|
from app.core.security import get_current_user
|
|
from app.models.user import User
|
|
from app.services.persistent_logs import record_audit_log, record_system_log
|
|
from app.services.system_control import (
|
|
build_task_id,
|
|
clear_active_task_id,
|
|
get_active_task_id,
|
|
get_allowed_command,
|
|
get_runner_script_path,
|
|
is_task_stale,
|
|
get_task_logs,
|
|
require_super_admin,
|
|
serialize_task,
|
|
set_active_task_id,
|
|
upsert_task_state,
|
|
)
|
|
from app.services.system_logs import (
|
|
DEFAULT_LOG_LINE_LIMIT,
|
|
MAX_LOG_LINE_LIMIT,
|
|
SUPPORTED_LOG_LEVELS,
|
|
append_buffer_log,
|
|
list_log_sources,
|
|
normalize_log_level,
|
|
read_log_snapshot,
|
|
)
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
class RestartTaskCreate(BaseModel):
|
|
action: str
|
|
|
|
|
|
class RestartTaskResponse(BaseModel):
|
|
task_id: str
|
|
action: str
|
|
status: str
|
|
stage: str
|
|
message: str
|
|
created_at: str
|
|
updated_at: str
|
|
requested_by: dict[str, object] | None = None
|
|
|
|
|
|
class RestartTaskLogsResponse(BaseModel):
|
|
task_id: str
|
|
lines: list[str]
|
|
|
|
|
|
class SystemLogSourceSummary(BaseModel):
|
|
source_id: str
|
|
name: str
|
|
kind: str
|
|
location: str
|
|
description: str
|
|
category: str
|
|
status: str
|
|
|
|
|
|
class SystemLogSourcesResponse(BaseModel):
|
|
items: list[SystemLogSourceSummary]
|
|
|
|
|
|
class SystemLogDailyMarker(BaseModel):
|
|
date_token: str
|
|
total: int
|
|
dominant_level: str
|
|
|
|
|
|
class SystemLogSnapshotResponse(BaseModel):
|
|
source_id: str
|
|
name: str
|
|
kind: str
|
|
location: str
|
|
description: str
|
|
category: str
|
|
status: str
|
|
level: str
|
|
selected_levels: list[str] = []
|
|
search_query: str = ""
|
|
available_levels: list[str]
|
|
daily_markers: list[SystemLogDailyMarker] = []
|
|
line_limit: int
|
|
line_count: int
|
|
lines: list[str]
|
|
|
|
|
|
class EarthClientLogEventCreate(BaseModel):
|
|
level: str = "error"
|
|
message: str
|
|
category: str | None = None
|
|
url: str | None = None
|
|
module: str | None = None
|
|
detail: str | None = None
|
|
|
|
|
|
class EarthClientLogEventResponse(BaseModel):
|
|
accepted: bool
|
|
source_id: str
|
|
level: str
|
|
|
|
|
|
def ensure_super_admin(current_user: User) -> None:
|
|
if not require_super_admin(current_user.role):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail="Only super_admin can restart services",
|
|
)
|
|
|
|
|
|
def validate_log_date(raw_value: str | None, field_name: str) -> str | None:
|
|
if raw_value in {None, ""}:
|
|
return None
|
|
try:
|
|
return datetime.strptime(raw_value, "%Y-%m-%d").date().isoformat()
|
|
except ValueError as exc:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=f"{field_name} must be in YYYY-MM-DD format",
|
|
) from exc
|
|
|
|
|
|
@router.post("/restart-tasks", response_model=RestartTaskResponse)
|
|
async def create_restart_task(
|
|
payload: RestartTaskCreate,
|
|
request: Request,
|
|
current_user: User = Depends(get_current_user),
|
|
):
|
|
ensure_super_admin(current_user)
|
|
|
|
command = get_allowed_command(payload.action)
|
|
if command is None:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail="Unsupported system action",
|
|
)
|
|
|
|
active_task_id = get_active_task_id()
|
|
if active_task_id:
|
|
active_task = serialize_task(active_task_id)
|
|
if active_task and is_task_stale(active_task):
|
|
upsert_task_state(
|
|
active_task_id,
|
|
status="failed",
|
|
stage="failed",
|
|
message="Previous restart task became stale and was released",
|
|
)
|
|
clear_active_task_id(active_task_id)
|
|
elif active_task and active_task.get("status") in {"queued", "running"}:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_409_CONFLICT,
|
|
detail="Another restart task is already in progress",
|
|
)
|
|
|
|
task_id = build_task_id()
|
|
requested_by = {"id": current_user.id, "username": current_user.username}
|
|
task_state = upsert_task_state(
|
|
task_id,
|
|
action=payload.action,
|
|
status="queued",
|
|
stage="accepted",
|
|
message="Restart task accepted",
|
|
requested_by=requested_by,
|
|
)
|
|
set_active_task_id(task_id)
|
|
|
|
env = os.environ.copy()
|
|
backend_path = str(ROOT_DIR / "backend")
|
|
existing_pythonpath = env.get("PYTHONPATH", "")
|
|
env["PYTHONPATH"] = (
|
|
f"{backend_path}{os.pathsep}{existing_pythonpath}"
|
|
if existing_pythonpath
|
|
else backend_path
|
|
)
|
|
|
|
try:
|
|
subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
str(get_runner_script_path()),
|
|
"--task-id",
|
|
task_id,
|
|
"--action",
|
|
payload.action,
|
|
],
|
|
cwd=str(ROOT_DIR),
|
|
env=env,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
start_new_session=True,
|
|
)
|
|
except OSError as exc:
|
|
task_state = upsert_task_state(
|
|
task_id,
|
|
action=payload.action,
|
|
status="failed",
|
|
stage="failed",
|
|
message=f"Unable to start restart runner: {exc}",
|
|
requested_by=requested_by,
|
|
)
|
|
clear_active_task_id(task_id)
|
|
await record_audit_log(
|
|
action="system.restart_task.requested",
|
|
actor_id=current_user.id,
|
|
actor_name=current_user.username,
|
|
target_type="restart_task",
|
|
target_id=task_id,
|
|
result="failed",
|
|
ip=request.client.host if request.client else None,
|
|
details={"action": payload.action, "message": task_state["message"]},
|
|
)
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=task_state["message"],
|
|
) from exc
|
|
|
|
await record_audit_log(
|
|
action="system.restart_task.requested",
|
|
actor_id=current_user.id,
|
|
actor_name=current_user.username,
|
|
target_type="restart_task",
|
|
target_id=task_id,
|
|
result="accepted",
|
|
ip=request.client.host if request.client else None,
|
|
details={"action": payload.action},
|
|
)
|
|
return task_state
|
|
|
|
|
|
@router.get("/restart-tasks/{task_id}", response_model=RestartTaskResponse)
|
|
async def get_restart_task(
|
|
task_id: str,
|
|
current_user: User = Depends(get_current_user),
|
|
):
|
|
ensure_super_admin(current_user)
|
|
|
|
task = serialize_task(task_id)
|
|
if task is None:
|
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Restart task not found")
|
|
return task
|
|
|
|
|
|
@router.get("/restart-tasks/{task_id}/logs", response_model=RestartTaskLogsResponse)
|
|
async def get_restart_task_logs(
|
|
task_id: str,
|
|
current_user: User = Depends(get_current_user),
|
|
):
|
|
ensure_super_admin(current_user)
|
|
|
|
task = serialize_task(task_id)
|
|
if task is None:
|
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Restart task not found")
|
|
return {"task_id": task_id, "lines": get_task_logs(task_id)}
|
|
|
|
|
|
@router.get("/logs/sources", response_model=SystemLogSourcesResponse)
|
|
async def get_system_log_sources(
|
|
current_user: User = Depends(get_current_user),
|
|
):
|
|
ensure_super_admin(current_user)
|
|
return {"items": list_log_sources()}
|
|
|
|
|
|
@router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse)
|
|
async def get_system_log_snapshot(
|
|
source_id: str,
|
|
limit: int = DEFAULT_LOG_LINE_LIMIT,
|
|
level: str = "all",
|
|
levels: str | None = Query(None, description="Comma-separated log levels"),
|
|
start_date: str | None = Query(None, description="Filter logs from this date (YYYY-MM-DD)"),
|
|
end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"),
|
|
search: str | None = Query(None, description="Case-insensitive substring search"),
|
|
current_user: User = Depends(get_current_user),
|
|
):
|
|
ensure_super_admin(current_user)
|
|
|
|
if limit < 1 or limit > MAX_LOG_LINE_LIMIT:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=f"limit must be between 1 and {MAX_LOG_LINE_LIMIT}",
|
|
)
|
|
if str(level).strip().lower() not in SUPPORTED_LOG_LEVELS and normalize_log_level(level) == "all" and str(level).strip().lower() not in {"", "all"}:
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unsupported log level")
|
|
if levels:
|
|
for raw_level in str(levels).split(","):
|
|
normalized_level = str(raw_level).strip().lower()
|
|
if not normalized_level:
|
|
continue
|
|
if normalized_level not in SUPPORTED_LOG_LEVELS and normalize_log_level(normalized_level) == "all":
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unsupported log level")
|
|
normalized_start_date = validate_log_date(start_date, "start_date")
|
|
normalized_end_date = validate_log_date(end_date, "end_date")
|
|
if normalized_start_date and normalized_end_date and normalized_start_date > normalized_end_date:
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="start_date must be earlier than or equal to end_date")
|
|
|
|
snapshot = read_log_snapshot(
|
|
source_id,
|
|
limit,
|
|
level=level,
|
|
levels=levels,
|
|
start_date=normalized_start_date,
|
|
end_date=normalized_end_date,
|
|
search=search,
|
|
)
|
|
if snapshot is None:
|
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Log source not found")
|
|
return snapshot
|
|
|
|
|
|
@router.post("/logs/earth-client", response_model=EarthClientLogEventResponse)
|
|
async def ingest_earth_client_log(
|
|
payload: EarthClientLogEventCreate,
|
|
request: Request,
|
|
):
|
|
normalized_level = normalize_log_level(payload.level)
|
|
append_buffer_log(
|
|
"earth-client",
|
|
level=normalized_level,
|
|
message=payload.message,
|
|
context={
|
|
"category": payload.category or "",
|
|
"url": payload.url or "",
|
|
"module": payload.module or "",
|
|
"detail": payload.detail or "",
|
|
},
|
|
)
|
|
await record_system_log(
|
|
source="earth-client",
|
|
service="earth",
|
|
module=payload.module or "earth-client",
|
|
event="earth.client.runtime_log",
|
|
level=normalized_level,
|
|
message=payload.message,
|
|
category=payload.category or "client-runtime",
|
|
context={
|
|
"url": payload.url or "",
|
|
"detail": payload.detail or "",
|
|
"module": payload.module or "",
|
|
"client_ip": request.client.host if request.client else "",
|
|
},
|
|
)
|
|
return {"accepted": True, "source_id": "earth-client", "level": normalized_level}
|