release: bump version to 0.67.0
Some checks failed
ci / backend (push) Has been cancelled
ci / frontend (push) Has been cancelled
ci / delivery (push) Has been cancelled
release / images (push) Has been cancelled

This commit is contained in:
rayd1o
2026-05-27 13:50:16 +08:00
parent d15a9d488a
commit b18ffa0b0a
46 changed files with 2116 additions and 648 deletions

View File

@@ -1,7 +1,6 @@
from __future__ import annotations
import os
import json
import subprocess
import sys
@@ -9,13 +8,11 @@ from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import ROOT_DIR
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.system_log import AuditLog, SystemLog
from app.models.user import User
from app.services.persistent_logs import record_audit_log, record_system_log
from app.services.system_control import (
@@ -38,6 +35,7 @@ from app.services.system_logs import (
append_buffer_log,
list_log_sources,
normalize_log_level,
read_database_log_snapshot,
read_log_snapshot,
)
from app.services.earth_layer_cache import earth_layer_cache
@@ -45,42 +43,6 @@ from app.services.earth_layer_cache import earth_layer_cache
router = APIRouter()
def _compact_log_context(context: dict | None) -> str:
if not context:
return ""
allowed = {
key: value
for key, value in (context or {}).items()
if key
in {
"status",
"duration_ms",
"provider",
"model",
"result_provider",
"result_model",
"collector_name",
"datasource_id",
"task_id",
"snapshot_id",
"raw_count",
"transformed_count",
"saved_count",
"created",
"updated",
"unchanged",
"deleted",
"result_count",
"status_code",
"error_type",
"error",
}
}
if not allowed:
return ""
return json.dumps(allowed, ensure_ascii=False, sort_keys=True)
class RestartTaskCreate(BaseModel):
action: str
@@ -154,6 +116,46 @@ class EarthClientLogEventResponse(BaseModel):
level: str
async def ingest_client_log_event(
source_id: str,
*,
service: str,
event: str,
default_module: str,
default_category: str,
payload: EarthClientLogEventCreate,
request: Request,
) -> EarthClientLogEventResponse:
normalized_level = normalize_log_level(payload.level)
append_buffer_log(
source_id,
level=normalized_level,
message=payload.message,
context={
"category": payload.category or "",
"url": payload.url or "",
"module": payload.module or "",
"detail": payload.detail or "",
},
)
await record_system_log(
source=source_id,
service=service,
module=payload.module or default_module,
event=event,
level=normalized_level,
message=payload.message,
category=payload.category or default_category,
context={
"url": payload.url or "",
"detail": payload.detail or "",
"module": payload.module or "",
"client_ip": request.client.host if request.client else "",
},
)
return EarthClientLogEventResponse(accepted=True, source_id=source_id, level=normalized_level)
class EarthLayerCacheStatusResponse(BaseModel):
prefix: str
key_count: int
@@ -376,100 +378,6 @@ async def get_system_log_sources(
}
async def read_database_log_snapshot(
source_id: str,
*,
limit: int,
level: str,
levels: str | None,
start_date: str | None,
end_date: str | None,
search: str | None,
db: AsyncSession,
) -> dict | None:
selected_levels = set(normalize_log_level(item) for item in (levels or level).split(",") if item.strip())
selected_levels.discard("all")
search_query = (search or "").strip().lower()
lines: list[str] = []
if source_id == "system-db":
query = select(SystemLog).order_by(SystemLog.occurred_at.desc().nullslast(), SystemLog.id.desc()).limit(limit * 5)
result = await db.execute(query)
records = result.scalars().all()
for record in records:
record_level = normalize_log_level(record.level)
if selected_levels and record_level not in selected_levels:
continue
occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else ""
if start_date and occurred_at and occurred_at < start_date:
continue
if end_date and occurred_at and occurred_at > end_date:
continue
line = " ".join(
part
for part in [
record.occurred_at.isoformat() if record.occurred_at else "",
record_level.upper(),
record.source,
record.category or "",
record.event or "",
f"request_id={record.request_id}" if record.request_id else "",
record.message,
_compact_log_context(record.context),
]
if part
)
if search_query and search_query not in line.lower():
continue
lines.append(line)
elif source_id == "audit-db":
query = select(AuditLog).order_by(AuditLog.occurred_at.desc().nullslast(), AuditLog.id.desc()).limit(limit * 5)
result = await db.execute(query)
records = result.scalars().all()
for record in records:
occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else ""
if start_date and occurred_at and occurred_at < start_date:
continue
if end_date and occurred_at and occurred_at > end_date:
continue
line = " ".join(
part
for part in [
record.occurred_at.isoformat() if record.occurred_at else "",
"INFO",
record.action,
record.target_type or "",
record.target_id or "",
record.result or "",
]
if part
)
if search_query and search_query not in line.lower():
continue
lines.append(line)
else:
return None
lines = list(reversed(lines[:limit]))
return {
"source_id": source_id,
"name": "系统事件" if source_id == "system-db" else "审计事件",
"kind": "database",
"location": "table://system_logs" if source_id == "system-db" else "table://audit_logs",
"description": "数据库持久化日志",
"category": "database" if source_id == "system-db" else "audit",
"status": "ok" if lines else "empty",
"level": level,
"selected_levels": sorted(selected_levels),
"search_query": search or "",
"available_levels": ["all", "error", "warning", "info", "debug"],
"daily_markers": [],
"line_limit": limit,
"line_count": len(lines),
"lines": lines,
}
@router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse)
async def get_system_log_snapshot(
source_id: str,
@@ -533,31 +441,28 @@ async def ingest_earth_client_log(
payload: EarthClientLogEventCreate,
request: Request,
):
normalized_level = normalize_log_level(payload.level)
append_buffer_log(
return await ingest_client_log_event(
"earth-client",
level=normalized_level,
message=payload.message,
context={
"category": payload.category or "",
"url": payload.url or "",
"module": payload.module or "",
"detail": payload.detail or "",
},
)
await record_system_log(
source="earth-client",
service="earth",
module=payload.module or "earth-client",
event="earth.client.runtime_log",
level=normalized_level,
message=payload.message,
category=payload.category or "client-runtime",
context={
"url": payload.url or "",
"detail": payload.detail or "",
"module": payload.module or "",
"client_ip": request.client.host if request.client else "",
},
default_module="earth-client",
default_category="client-runtime",
payload=payload,
request=request,
)
@router.post("/logs/admin-client", response_model=EarthClientLogEventResponse)
async def ingest_admin_client_log(
payload: EarthClientLogEventCreate,
request: Request,
):
return await ingest_client_log_event(
"admin-client",
service="admin",
event="admin.client.runtime_log",
default_module="admin-client",
default_category="client-runtime",
payload=payload,
request=request,
)
return {"accepted": True, "source_id": "earth-client", "level": normalized_level}

View File

@@ -6,11 +6,14 @@ from typing import Optional
from fastapi import APIRouter, WebSocket, WebSocketDisconnect, Query
from jose import jwt, JWTError
from sqlalchemy import text
from app.core.config import settings
from app.core.logging import get_logger
from app.core.time import to_iso8601_utc
from app.core.websocket.manager import manager
from app.db.session import async_session_factory
from app.services.log_tail import LOG_TAIL_CHANNEL, log_tail_manager
logger = get_logger(__name__, service="api")
router = APIRouter()
@@ -37,6 +40,28 @@ async def authenticate_token(token: str) -> Optional[dict]:
return None
async def load_websocket_user_role(user_id: str | None) -> str | None:
if not user_id:
return None
try:
async with async_session_factory() as db:
result = await db.execute(
text("SELECT role, is_active FROM users WHERE id = :id"),
{"id": int(user_id)},
)
row = result.fetchone()
except Exception as exc:
logger.warning_event(
"WebSocket user role lookup failed",
event="auth.websocket.role_lookup_failed",
context={"user_id": user_id, "error": str(exc)},
)
return None
if row is None or not row[1]:
return None
return str(row[0] or "")
@router.websocket("/ws")
async def websocket_endpoint(
websocket: WebSocket,
@@ -59,6 +84,7 @@ async def websocket_endpoint(
is_anonymous = payload is None
user_id = str(payload.get("sub")) if payload else f"anonymous:{id(websocket)}"
user_role = await load_websocket_user_role(user_id) if payload else None
supported_channels = ["vessels", "earth_news", EARTH_UPDATES_CHANNEL] if is_anonymous else [
"gpu_clusters",
"submarine_cables",
@@ -70,6 +96,8 @@ async def websocket_endpoint(
"earth_news",
EARTH_UPDATES_CHANNEL,
]
if user_role == "super_admin":
supported_channels = [*supported_channels, LOG_TAIL_CHANNEL]
await manager.connect(websocket, user_id)
try:
@@ -100,6 +128,7 @@ async def websocket_endpoint(
payload_data = data.get("data", {})
if not isinstance(payload_data, dict):
payload_data = {}
log_tail_config = None
channels = payload_data.get("channels", [])
if isinstance(channels, str):
channels = [channels]
@@ -108,6 +137,26 @@ async def websocket_endpoint(
channel = payload_data.get("channel")
if channel and channel not in channels:
channels = [*channels, channel]
if LOG_TAIL_CHANNEL in channels:
if user_role != "super_admin":
await websocket.send_json(
{
"type": "subscription_error",
"data": {"channel": LOG_TAIL_CHANNEL, "detail": "Only super_admin can subscribe logs"},
}
)
channels = [item for item in channels if item != LOG_TAIL_CHANNEL]
else:
try:
log_tail_config = await log_tail_manager.subscribe(websocket, payload_data)
except ValueError as exc:
await websocket.send_json(
{
"type": "subscription_error",
"data": {"channel": LOG_TAIL_CHANNEL, "detail": str(exc)},
}
)
channels = [item for item in channels if item != LOG_TAIL_CHANNEL]
if is_anonymous:
channels = [channel for channel in channels if channel in supported_channels]
vessel_subscription = None
@@ -131,14 +180,20 @@ async def websocket_endpoint(
"action": "subscribe",
"channels": [
*channels,
*([LOG_TAIL_CHANNEL] if log_tail_config else []),
*(["vessels"] if vessel_subscription else []),
],
"vessels": vessel_subscription,
"logs_tail": log_tail_config.__dict__ if log_tail_config else None,
},
}
)
elif data.get("type") == "unsubscribe":
channels = data.get("data", {}).get("channels", [])
if isinstance(channels, str):
channels = [channels]
if LOG_TAIL_CHANNEL in channels:
await log_tail_manager.unsubscribe(websocket)
manager.unsubscribe(websocket, channels)
await websocket.send_json(
{
@@ -159,4 +214,5 @@ async def websocket_endpoint(
except WebSocketDisconnect:
pass
finally:
await log_tail_manager.disconnect(websocket)
manager.disconnect(websocket, user_id)

View File

@@ -0,0 +1,161 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
from fastapi import WebSocket
from app.db.session import async_session_factory
from app.services.system_logs import (
DEFAULT_LOG_LINE_LIMIT,
LOG_SOURCES,
MAX_LOG_LINE_LIMIT,
read_database_log_events,
read_log_events,
)
DATABASE_LOG_SOURCE_IDS = {"system-db", "audit-db"}
LOG_TAIL_CHANNEL = "logs_tail"
LOG_TAIL_INTERVAL_SECONDS = 1.5
LOG_TAIL_SCAN_MULTIPLIER = 5
@dataclass(frozen=True)
class LogTailConfig:
source_id: str
limit: int = DEFAULT_LOG_LINE_LIMIT
level: str = "all"
levels: str | None = None
start_date: str | None = None
end_date: str | None = None
search: str | None = None
@dataclass
class LogTailSubscription:
config: LogTailConfig
emitted_cursors: set[str] = field(default_factory=set)
task: asyncio.Task | None = None
class LogTailManager:
def __init__(self) -> None:
self._subscriptions: dict[WebSocket, LogTailSubscription] = {}
def normalize_config(self, payload: dict[str, Any]) -> LogTailConfig:
source_id = str(payload.get("source_id") or payload.get("source") or "").strip()
if not source_id:
raise ValueError("source_id is required")
if source_id not in LOG_SOURCES and source_id not in DATABASE_LOG_SOURCE_IDS:
raise ValueError("Log source not found")
try:
limit = int(payload.get("limit") or DEFAULT_LOG_LINE_LIMIT)
except (TypeError, ValueError) as exc:
raise ValueError("limit must be a number") from exc
if limit < 1 or limit > MAX_LOG_LINE_LIMIT:
raise ValueError(f"limit must be between 1 and {MAX_LOG_LINE_LIMIT}")
return LogTailConfig(
source_id=source_id,
limit=limit,
level=str(payload.get("level") or "all"),
levels=str(payload.get("levels")).strip() if payload.get("levels") else None,
start_date=str(payload.get("start_date")).strip() if payload.get("start_date") else None,
end_date=str(payload.get("end_date")).strip() if payload.get("end_date") else None,
search=str(payload.get("search")).strip() if payload.get("search") else None,
)
async def subscribe(self, websocket: WebSocket, payload: dict[str, Any]) -> LogTailConfig:
config = self.normalize_config(payload)
await self.unsubscribe(websocket)
subscription = LogTailSubscription(config=config)
subscription.task = asyncio.create_task(self._run_tail(websocket, subscription))
self._subscriptions[websocket] = subscription
return config
async def unsubscribe(self, websocket: WebSocket) -> None:
subscription = self._subscriptions.pop(websocket, None)
if subscription and subscription.task:
subscription.task.cancel()
try:
await subscription.task
except asyncio.CancelledError:
pass
async def disconnect(self, websocket: WebSocket) -> None:
await self.unsubscribe(websocket)
async def _run_tail(self, websocket: WebSocket, subscription: LogTailSubscription) -> None:
first_frame = True
while True:
events = await self._read_events(subscription.config)
if first_frame:
visible_events = events[-subscription.config.limit :]
subscription.emitted_cursors.update(event.cursor for event in visible_events)
await self._send_frame(websocket, subscription.config, "snapshot", visible_events)
first_frame = False
else:
new_events = [
event
for event in events
if event.cursor not in subscription.emitted_cursors
]
if new_events:
visible_events = new_events[-subscription.config.limit :]
subscription.emitted_cursors.update(event.cursor for event in visible_events)
await self._send_frame(websocket, subscription.config, "append", visible_events)
await asyncio.sleep(LOG_TAIL_INTERVAL_SECONDS)
async def _read_events(self, config: LogTailConfig):
scan_limit = max(config.limit * LOG_TAIL_SCAN_MULTIPLIER, config.limit)
if config.source_id in DATABASE_LOG_SOURCE_IDS:
async with async_session_factory() as db:
events = await read_database_log_events(
config.source_id,
scan_limit=scan_limit,
level=config.level,
levels=config.levels,
start_date=config.start_date,
end_date=config.end_date,
search=config.search,
db=db,
)
return events or []
events = read_log_events(
config.source_id,
scan_limit=scan_limit,
level=config.level,
levels=config.levels,
start_date=config.start_date,
end_date=config.end_date,
search=config.search,
)
return events or []
async def _send_frame(self, websocket: WebSocket, config: LogTailConfig, mode: str, events) -> None:
await websocket.send_json(
{
"type": "data_frame",
"channel": LOG_TAIL_CHANNEL,
"timestamp": datetime.now(UTC).isoformat(),
"payload": {
"mode": mode,
"source_id": config.source_id,
"line_count": len(events),
"lines": [event.line for event in events],
"filters": {
"limit": config.limit,
"level": config.level,
"levels": config.levels,
"start_date": config.start_date,
"end_date": config.end_date,
"search": config.search,
},
"status": "ok",
},
}
)
log_tail_manager = LogTailManager()

View File

@@ -5,6 +5,7 @@ import os
import re
import shutil
import subprocess
import hashlib
from collections import Counter, deque
from dataclasses import dataclass
@@ -13,6 +14,9 @@ from pathlib import Path
from typing import Any
from app.core.security import redis_client
from app.models.system_log import AuditLog, SystemLog
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
DEFAULT_LOG_LINE_LIMIT = 200
MAX_LOG_LINE_LIMIT = 1000
@@ -99,6 +103,16 @@ class StructuredLogEntry:
search_text: str
@dataclass(frozen=True)
class LogEvent:
source_id: str
cursor: str
timestamp: datetime | None
level: str | None
line: str
search_text: str
@dataclass
class DailyLogMarker:
date_token: str
@@ -135,7 +149,7 @@ LOG_SOURCES: dict[str, LogSource] = {
name="前端开发服务",
kind="file",
location=_state_log_path("frontend.log"),
description="控制台与 Earth 前端开发服务输出。",
description="控制台与智能星球前端开发服务输出。",
category="service",
fallback_locations=("/tmp/planet_frontend.log",),
),
@@ -150,13 +164,22 @@ LOG_SOURCES: dict[str, LogSource] = {
),
"earth-client": LogSource(
source_id="earth-client",
name="Earth 浏览器端",
name="智能星球浏览器端",
kind="buffer",
location="redis://planet:system_logs:earth-client",
description="Earth 浏览器端上报的运行时错误与关键业务日志。",
description="智能星球浏览器端上报的运行时错误与关键业务日志。",
category="client",
buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:earth-client",
),
"admin-client": LogSource(
source_id="admin-client",
name="控制台浏览器端",
kind="buffer",
location="redis://planet:system_logs:admin-client",
description="控制台浏览器端上报的运行时错误。",
category="client",
buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:admin-client",
),
}
@@ -365,6 +388,44 @@ def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry:
)
def compact_log_context(context: dict | None) -> str:
if not context:
return ""
allowed = {
key: value
for key, value in (context or {}).items()
if key
in {
"status",
"duration_ms",
"provider",
"model",
"result_provider",
"result_model",
"collector_name",
"datasource_id",
"task_id",
"snapshot_id",
"raw_count",
"transformed_count",
"saved_count",
"created",
"updated",
"unchanged",
"deleted",
"result_count",
"status_code",
"error_type",
"error",
"route",
"module",
}
}
if not allowed:
return ""
return json.dumps(allowed, ensure_ascii=False, sort_keys=True)
def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
path = resolve_file_log_path(source)
if not path.exists():
@@ -437,6 +498,176 @@ def read_source_entries(source: LogSource, scan_limit: int) -> list[StructuredLo
return []
def _database_event_from_system_record(record: SystemLog) -> LogEvent:
record_level = normalize_log_level(record.level)
line = " ".join(
part
for part in [
record.occurred_at.isoformat() if record.occurred_at else "",
record_level.upper(),
record.source,
record.category or "",
record.event or "",
f"request_id={record.request_id}" if record.request_id else "",
record.message,
compact_log_context(record.context),
]
if part
)
search_text = " ".join(
[
line,
f"id={record.id}",
f"user_id={record.user_id}" if record.user_id else "",
json.dumps(record.context or {}, ensure_ascii=False, sort_keys=True),
]
).lower()
return LogEvent(
source_id="system-db",
cursor=f"system-db:{record.id}",
timestamp=record.occurred_at,
level=None if record_level == LOG_LEVEL_ALL else record_level,
line=line,
search_text=search_text,
)
def _database_event_from_audit_record(record: AuditLog) -> LogEvent:
line = " ".join(
part
for part in [
record.occurred_at.isoformat() if record.occurred_at else "",
"INFO",
record.action,
record.target_type or "",
record.target_id or "",
record.result or "",
f"request_id={record.request_id}" if record.request_id else "",
]
if part
)
search_text = " ".join(
[
line,
f"id={record.id}",
f"actor_id={record.actor_id}" if record.actor_id else "",
record.actor_name or "",
json.dumps(record.details or {}, ensure_ascii=False, sort_keys=True),
]
).lower()
return LogEvent(
source_id="audit-db",
cursor=f"audit-db:{record.id}",
timestamp=record.occurred_at,
level=LOG_LEVEL_INFO,
line=line,
search_text=search_text,
)
async def read_database_log_events(
source_id: str,
*,
scan_limit: int,
level: str = LOG_LEVEL_ALL,
levels: str | None = None,
start_date: str | None = None,
end_date: str | None = None,
search: str | None = None,
db: AsyncSession,
) -> list[LogEvent] | None:
selected_levels = normalize_log_levels(level, levels)
search_query = (search or "").strip()
if source_id == "system-db":
query = select(SystemLog).order_by(SystemLog.occurred_at.desc().nullslast(), SystemLog.id.desc()).limit(scan_limit)
result = await db.execute(query)
events = [_database_event_from_system_record(record) for record in result.scalars().all()]
elif source_id == "audit-db":
query = select(AuditLog).order_by(AuditLog.occurred_at.desc().nullslast(), AuditLog.id.desc()).limit(scan_limit)
result = await db.execute(query)
events = [_database_event_from_audit_record(record) for record in result.scalars().all()]
else:
return None
events = list(reversed(events))
return [
event
for event in events
if event_matches_levels(event, selected_levels)
and event_matches_search(event, search_query)
and event_matches_date_range(event, start_date, end_date)
]
async def read_database_log_snapshot(
source_id: str,
*,
limit: int,
level: str,
levels: str | None,
start_date: str | None,
end_date: str | None,
search: str | None,
db: AsyncSession,
) -> dict[str, Any] | None:
events = await read_database_log_events(
source_id,
scan_limit=limit * 5,
level=level,
levels=levels,
start_date=start_date,
end_date=end_date,
search=search,
db=db,
)
if events is None:
return None
visible_events = events[-limit:]
selected_levels = normalize_log_levels(level, levels)
return {
"source_id": source_id,
"name": "系统事件" if source_id == "system-db" else "审计事件",
"kind": "database",
"location": "table://system_logs" if source_id == "system-db" else "table://audit_logs",
"description": "数据库持久化日志",
"category": "database" if source_id == "system-db" else "audit",
"status": "ok" if visible_events else "empty",
"level": level,
"selected_levels": list(selected_levels),
"search_query": search or "",
"available_levels": ["all", "error", "warning", "info", "debug"],
"daily_markers": build_daily_log_markers_from_events(events),
"line_limit": limit,
"line_count": len(visible_events),
"lines": [event.line for event in visible_events],
}
def _stable_hash(value: str) -> str:
return hashlib.sha1(value.encode("utf-8", errors="replace")).hexdigest()[:16]
def build_log_events(source_id: str, entries: list[StructuredLogEntry]) -> list[LogEvent]:
events: list[LogEvent] = []
seen: dict[str, int] = {}
for entry in entries:
stable_value = entry.raw_line or entry.display_line
digest = _stable_hash(stable_value)
occurrence = seen.get(digest, 0) + 1
seen[digest] = occurrence
events.append(
LogEvent(
source_id=source_id,
cursor=f"{source_id}:{digest}:{occurrence}",
timestamp=entry.timestamp,
level=entry.level,
line=entry.display_line,
search_text=entry.search_text,
)
)
return events
def matches_levels(entry: StructuredLogEntry, selected_levels: tuple[str, ...]) -> bool:
if not selected_levels:
return True
@@ -469,6 +700,34 @@ def matches_search(entry: StructuredLogEntry, search: str | None) -> bool:
return query in entry.search_text
def event_matches_levels(event: LogEvent, selected_levels: tuple[str, ...]) -> bool:
if not selected_levels:
return True
return event.level in selected_levels
def event_matches_date_range(event: LogEvent, start_date: str | None, end_date: str | None) -> bool:
if not start_date and not end_date:
return True
if event.timestamp is None:
return False
date_token = event.timestamp.astimezone(UTC).date().isoformat()
if start_date and date_token < start_date:
return False
if end_date and date_token > end_date:
return False
return True
def event_matches_search(event: LogEvent, search: str | None) -> bool:
if search is None:
return True
query = search.strip().lower()
if not query:
return True
return query in event.search_text
def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str, Any]]:
grouped: dict[str, list[StructuredLogEntry]] = {}
for entry in entries:
@@ -503,6 +762,69 @@ def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str,
return [marker.__dict__ for marker in markers]
def build_daily_log_markers_from_events(events: list[LogEvent]) -> list[dict[str, Any]]:
grouped: dict[str, list[LogEvent]] = {}
for event in events:
if event.timestamp is None:
continue
date_token = event.timestamp.astimezone(UTC).date().isoformat()
grouped.setdefault(date_token, []).append(event)
markers: list[DailyLogMarker] = []
for date_token, group in sorted(grouped.items()):
level_counts = Counter(
event.level
for event in group
if event.level in SUPPORTED_LOG_LEVELS and event.level != LOG_LEVEL_ALL
)
dominant_level = LOG_LEVEL_INFO
if level_counts:
dominant_level = sorted(
level_counts.items(),
key=lambda item: (
-item[1],
("error", "warning", "info", "debug").index(item[0]),
),
)[0][0]
markers.append(
DailyLogMarker(
date_token=date_token,
total=len(group),
dominant_level=dominant_level,
)
)
return [marker.__dict__ for marker in markers]
def read_log_events(
source_id: str,
scan_limit: int,
*,
level: str = LOG_LEVEL_ALL,
levels: str | None = None,
start_date: str | None = None,
end_date: str | None = None,
search: str | None = None,
) -> list[LogEvent] | None:
source = LOG_SOURCES.get(source_id)
if source is None:
return None
selected_levels = normalize_log_levels(level, levels)
search_query = (search or "").strip()
events = build_log_events(source_id, read_source_entries(source, scan_limit))
marker_events = [
event
for event in events
if event_matches_levels(event, selected_levels) and event_matches_search(event, search_query)
]
return [
event
for event in marker_events
if event_matches_date_range(event, start_date, end_date)
]
def read_log_snapshot(
source_id: str,
limit: int,
@@ -520,18 +842,18 @@ def read_log_snapshot(
selected_levels = normalize_log_levels(level, levels)
search_query = (search or "").strip()
scan_limit = max(min(MAX_LOG_LINE_LIMIT * 5, 5000), limit * 5, BUFFER_LOG_LIMIT if source.kind == "buffer" else 1000)
all_entries = read_source_entries(source, scan_limit)
marker_entries = [
entry
for entry in all_entries
if matches_levels(entry, selected_levels) and matches_search(entry, search_query)
all_events = build_log_events(source_id, read_source_entries(source, scan_limit))
marker_events = [
event
for event in all_events
if event_matches_levels(event, selected_levels) and event_matches_search(event, search_query)
]
filtered_entries = [
entry
for entry in marker_entries
if matches_date_range(entry, start_date, end_date)
filtered_events = [
event
for event in marker_events
if event_matches_date_range(event, start_date, end_date)
]
visible_entries = filtered_entries[-limit:]
visible_events = filtered_events[-limit:]
compatibility_level = selected_levels[0] if len(selected_levels) == 1 else LOG_LEVEL_ALL
return {
@@ -552,8 +874,8 @@ def read_log_snapshot(
LOG_LEVEL_INFO,
LOG_LEVEL_DEBUG,
],
"daily_markers": build_daily_log_markers(marker_entries),
"daily_markers": build_daily_log_markers_from_events(marker_events),
"line_limit": limit,
"line_count": len(visible_entries),
"lines": [entry.display_line for entry in visible_entries],
"line_count": len(visible_events),
"lines": [event.line for event in visible_events],
}

View File

@@ -672,6 +672,39 @@ async def test_ingest_earth_client_log_accepts_public_events():
app.dependency_overrides.clear()
@pytest.mark.asyncio
async def test_ingest_admin_client_log_accepts_public_events():
transport = ASGITransport(app=app)
try:
with patch("app.api.v1.system_control.append_buffer_log") as mock_append_buffer_log:
with patch("app.api.v1.system_control.record_system_log", new_callable=AsyncMock) as mock_record_system_log:
async with AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.post(
"/api/v1/system/logs/admin-client",
json={
"level": "error",
"message": "控制台发生未处理 Promise 错误",
"category": "unhandledrejection",
"module": "admin",
"url": "http://test/logs",
"detail": "stack preview",
},
)
assert response.status_code == 200
data = response.json()
assert data["accepted"] is True
assert data["source_id"] == "admin-client"
mock_append_buffer_log.assert_called_once()
mock_record_system_log.assert_awaited_once()
persisted_kwargs = mock_record_system_log.await_args.kwargs
assert persisted_kwargs["source"] == "admin-client"
assert persisted_kwargs["event"] == "admin.client.runtime_log"
assert persisted_kwargs["category"] == "unhandledrejection"
assert persisted_kwargs["context"]["url"] == "http://test/logs"
finally:
app.dependency_overrides.clear()
@pytest.mark.asyncio
async def test_earth_layer_cache_status_requires_super_admin(auth_headers, monkeypatch):
def override_get_current_user():

View File

@@ -40,10 +40,10 @@ def test_read_log_snapshot_uses_structured_buffer_timestamp_level_and_search(mon
{
"earth-client": system_logs.LogSource(
source_id="earth-client",
name="Earth 浏览器端",
name="智能星球浏览器端",
kind="buffer",
location="redis://planet:system_logs:earth-client",
description="Earth 浏览器端上报日志",
description="智能星球浏览器端上报日志",
category="client",
buffer_key=system_logs.get_buffer_log_key("earth-client"),
)
@@ -156,6 +156,52 @@ def test_append_buffer_log_persists_normalized_level(monkeypatch):
assert payload["message"] == "feed delayed"
def test_list_log_sources_includes_admin_client(monkeypatch):
fake_redis = FakeRedis()
monkeypatch.setattr(system_logs, "redis_client", fake_redis)
sources = system_logs.list_log_sources()
admin_source = next(item for item in sources if item["source_id"] == "admin-client")
assert admin_source["kind"] == "buffer"
assert admin_source["category"] == "client"
def test_read_log_events_returns_stable_cursors(tmp_path: Path, monkeypatch):
log_path = tmp_path / "backend.log"
log_path.write_text(
"\n".join(
[
"2026-04-22 08:00:00 INFO service booted",
"2026-04-22 08:01:00 ERROR service failed",
]
),
encoding="utf-8",
)
monkeypatch.setattr(
system_logs,
"LOG_SOURCES",
{
"backend": system_logs.LogSource(
source_id="backend",
name="后端服务",
kind="file",
location=str(log_path),
description="测试文件日志",
category="service",
)
},
)
events = system_logs.read_log_events("backend", 50, level="error")
assert events is not None
assert len(events) == 1
assert events[0].source_id == "backend"
assert events[0].cursor.startswith("backend:")
assert events[0].line.endswith("ERROR service failed")
def test_infer_log_level_prefers_leading_prefix_over_query_string():
line = 'INFO: 127.0.0.1 - "GET /api/v1/system/logs/backend?limit=200&level=error&levels=error HTTP/1.1" 200 OK'