release: bump version to 0.67.0

backend/app/services/log_tail.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any
+
+from fastapi import WebSocket
+
+from app.db.session import async_session_factory
+from app.services.system_logs import (
+    DEFAULT_LOG_LINE_LIMIT,
+    LOG_SOURCES,
+    MAX_LOG_LINE_LIMIT,
+    read_database_log_events,
+    read_log_events,
+)
+
+DATABASE_LOG_SOURCE_IDS = {"system-db", "audit-db"}
+LOG_TAIL_CHANNEL = "logs_tail"
+LOG_TAIL_INTERVAL_SECONDS = 1.5
+LOG_TAIL_SCAN_MULTIPLIER = 5
+
+
+@dataclass(frozen=True)
+class LogTailConfig:
+    source_id: str
+    limit: int = DEFAULT_LOG_LINE_LIMIT
+    level: str = "all"
+    levels: str | None = None
+    start_date: str | None = None
+    end_date: str | None = None
+    search: str | None = None
+
+
+@dataclass
+class LogTailSubscription:
+    config: LogTailConfig
+    emitted_cursors: set[str] = field(default_factory=set)
+    task: asyncio.Task | None = None
+
+
+class LogTailManager:
+    def __init__(self) -> None:
+        self._subscriptions: dict[WebSocket, LogTailSubscription] = {}
+
+    def normalize_config(self, payload: dict[str, Any]) -> LogTailConfig:
+        source_id = str(payload.get("source_id") or payload.get("source") or "").strip()
+        if not source_id:
+            raise ValueError("source_id is required")
+        if source_id not in LOG_SOURCES and source_id not in DATABASE_LOG_SOURCE_IDS:
+            raise ValueError("Log source not found")
+        try:
+            limit = int(payload.get("limit") or DEFAULT_LOG_LINE_LIMIT)
+        except (TypeError, ValueError) as exc:
+            raise ValueError("limit must be a number") from exc
+        if limit < 1 or limit > MAX_LOG_LINE_LIMIT:
+            raise ValueError(f"limit must be between 1 and {MAX_LOG_LINE_LIMIT}")
+        return LogTailConfig(
+            source_id=source_id,
+            limit=limit,
+            level=str(payload.get("level") or "all"),
+            levels=str(payload.get("levels")).strip() if payload.get("levels") else None,
+            start_date=str(payload.get("start_date")).strip() if payload.get("start_date") else None,
+            end_date=str(payload.get("end_date")).strip() if payload.get("end_date") else None,
+            search=str(payload.get("search")).strip() if payload.get("search") else None,
+        )
+
+    async def subscribe(self, websocket: WebSocket, payload: dict[str, Any]) -> LogTailConfig:
+        config = self.normalize_config(payload)
+        await self.unsubscribe(websocket)
+        subscription = LogTailSubscription(config=config)
+        subscription.task = asyncio.create_task(self._run_tail(websocket, subscription))
+        self._subscriptions[websocket] = subscription
+        return config
+
+    async def unsubscribe(self, websocket: WebSocket) -> None:
+        subscription = self._subscriptions.pop(websocket, None)
+        if subscription and subscription.task:
+            subscription.task.cancel()
+            try:
+                await subscription.task
+            except asyncio.CancelledError:
+                pass
+
+    async def disconnect(self, websocket: WebSocket) -> None:
+        await self.unsubscribe(websocket)
+
+    async def _run_tail(self, websocket: WebSocket, subscription: LogTailSubscription) -> None:
+        first_frame = True
+        while True:
+            events = await self._read_events(subscription.config)
+            if first_frame:
+                visible_events = events[-subscription.config.limit :]
+                subscription.emitted_cursors.update(event.cursor for event in visible_events)
+                await self._send_frame(websocket, subscription.config, "snapshot", visible_events)
+                first_frame = False
+            else:
+                new_events = [
+                    event
+                    for event in events
+                    if event.cursor not in subscription.emitted_cursors
+                ]
+                if new_events:
+                    visible_events = new_events[-subscription.config.limit :]
+                    subscription.emitted_cursors.update(event.cursor for event in visible_events)
+                    await self._send_frame(websocket, subscription.config, "append", visible_events)
+            await asyncio.sleep(LOG_TAIL_INTERVAL_SECONDS)
+
+    async def _read_events(self, config: LogTailConfig):
+        scan_limit = max(config.limit * LOG_TAIL_SCAN_MULTIPLIER, config.limit)
+        if config.source_id in DATABASE_LOG_SOURCE_IDS:
+            async with async_session_factory() as db:
+                events = await read_database_log_events(
+                    config.source_id,
+                    scan_limit=scan_limit,
+                    level=config.level,
+                    levels=config.levels,
+                    start_date=config.start_date,
+                    end_date=config.end_date,
+                    search=config.search,
+                    db=db,
+                )
+                return events or []
+        events = read_log_events(
+            config.source_id,
+            scan_limit=scan_limit,
+            level=config.level,
+            levels=config.levels,
+            start_date=config.start_date,
+            end_date=config.end_date,
+            search=config.search,
+        )
+        return events or []
+
+    async def _send_frame(self, websocket: WebSocket, config: LogTailConfig, mode: str, events) -> None:
+        await websocket.send_json(
+            {
+                "type": "data_frame",
+                "channel": LOG_TAIL_CHANNEL,
+                "timestamp": datetime.now(UTC).isoformat(),
+                "payload": {
+                    "mode": mode,
+                    "source_id": config.source_id,
+                    "line_count": len(events),
+                    "lines": [event.line for event in events],
+                    "filters": {
+                        "limit": config.limit,
+                        "level": config.level,
+                        "levels": config.levels,
+                        "start_date": config.start_date,
+                        "end_date": config.end_date,
+                        "search": config.search,
+                    },
+                    "status": "ok",
+                },
+            }
+        )
+
+
+log_tail_manager = LogTailManager()

backend/app/services/system_logs.py

@@ -5,6 +5,7 @@ import os
 import re
 import shutil
 import subprocess
+import hashlib
 
 from collections import Counter, deque
 from dataclasses import dataclass
@@ -13,6 +14,9 @@ from pathlib import Path
 from typing import Any
 
 from app.core.security import redis_client
+from app.models.system_log import AuditLog, SystemLog
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
 
 DEFAULT_LOG_LINE_LIMIT = 200
 MAX_LOG_LINE_LIMIT = 1000
@@ -99,6 +103,16 @@ class StructuredLogEntry:
     search_text: str
 
 
+@dataclass(frozen=True)
+class LogEvent:
+    source_id: str
+    cursor: str
+    timestamp: datetime | None
+    level: str | None
+    line: str
+    search_text: str
+
+
 @dataclass
 class DailyLogMarker:
     date_token: str
@@ -135,7 +149,7 @@ LOG_SOURCES: dict[str, LogSource] = {
         name="前端开发服务",
         kind="file",
         location=_state_log_path("frontend.log"),
-        description="控制台与 Earth 前端开发服务输出。",
+        description="控制台与智能星球前端开发服务输出。",
         category="service",
         fallback_locations=("/tmp/planet_frontend.log",),
     ),
@@ -150,13 +164,22 @@ LOG_SOURCES: dict[str, LogSource] = {
     ),
     "earth-client": LogSource(
         source_id="earth-client",
-        name="Earth 浏览器端",
+        name="智能星球浏览器端",
         kind="buffer",
         location="redis://planet:system_logs:earth-client",
-        description="Earth 浏览器端上报的运行时错误与关键业务日志。",
+        description="智能星球浏览器端上报的运行时错误与关键业务日志。",
         category="client",
         buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:earth-client",
     ),
+    "admin-client": LogSource(
+        source_id="admin-client",
+        name="控制台浏览器端",
+        kind="buffer",
+        location="redis://planet:system_logs:admin-client",
+        description="控制台浏览器端上报的运行时错误。",
+        category="client",
+        buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:admin-client",
+    ),
 }
 
 
@@ -365,6 +388,44 @@ def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry:
     )
 
 
+def compact_log_context(context: dict | None) -> str:
+    if not context:
+        return ""
+    allowed = {
+        key: value
+        for key, value in (context or {}).items()
+        if key
+        in {
+            "status",
+            "duration_ms",
+            "provider",
+            "model",
+            "result_provider",
+            "result_model",
+            "collector_name",
+            "datasource_id",
+            "task_id",
+            "snapshot_id",
+            "raw_count",
+            "transformed_count",
+            "saved_count",
+            "created",
+            "updated",
+            "unchanged",
+            "deleted",
+            "result_count",
+            "status_code",
+            "error_type",
+            "error",
+            "route",
+            "module",
+        }
+    }
+    if not allowed:
+        return ""
+    return json.dumps(allowed, ensure_ascii=False, sort_keys=True)
+
+
 def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
     path = resolve_file_log_path(source)
     if not path.exists():
@@ -437,6 +498,176 @@ def read_source_entries(source: LogSource, scan_limit: int) -> list[StructuredLo
     return []
 
 
+def _database_event_from_system_record(record: SystemLog) -> LogEvent:
+    record_level = normalize_log_level(record.level)
+    line = " ".join(
+        part
+        for part in [
+            record.occurred_at.isoformat() if record.occurred_at else "",
+            record_level.upper(),
+            record.source,
+            record.category or "",
+            record.event or "",
+            f"request_id={record.request_id}" if record.request_id else "",
+            record.message,
+            compact_log_context(record.context),
+        ]
+        if part
+    )
+    search_text = " ".join(
+        [
+            line,
+            f"id={record.id}",
+            f"user_id={record.user_id}" if record.user_id else "",
+            json.dumps(record.context or {}, ensure_ascii=False, sort_keys=True),
+        ]
+    ).lower()
+    return LogEvent(
+        source_id="system-db",
+        cursor=f"system-db:{record.id}",
+        timestamp=record.occurred_at,
+        level=None if record_level == LOG_LEVEL_ALL else record_level,
+        line=line,
+        search_text=search_text,
+    )
+
+
+def _database_event_from_audit_record(record: AuditLog) -> LogEvent:
+    line = " ".join(
+        part
+        for part in [
+            record.occurred_at.isoformat() if record.occurred_at else "",
+            "INFO",
+            record.action,
+            record.target_type or "",
+            record.target_id or "",
+            record.result or "",
+            f"request_id={record.request_id}" if record.request_id else "",
+        ]
+        if part
+    )
+    search_text = " ".join(
+        [
+            line,
+            f"id={record.id}",
+            f"actor_id={record.actor_id}" if record.actor_id else "",
+            record.actor_name or "",
+            json.dumps(record.details or {}, ensure_ascii=False, sort_keys=True),
+        ]
+    ).lower()
+    return LogEvent(
+        source_id="audit-db",
+        cursor=f"audit-db:{record.id}",
+        timestamp=record.occurred_at,
+        level=LOG_LEVEL_INFO,
+        line=line,
+        search_text=search_text,
+    )
+
+
+async def read_database_log_events(
+    source_id: str,
+    *,
+    scan_limit: int,
+    level: str = LOG_LEVEL_ALL,
+    levels: str | None = None,
+    start_date: str | None = None,
+    end_date: str | None = None,
+    search: str | None = None,
+    db: AsyncSession,
+) -> list[LogEvent] | None:
+    selected_levels = normalize_log_levels(level, levels)
+    search_query = (search or "").strip()
+    if source_id == "system-db":
+        query = select(SystemLog).order_by(SystemLog.occurred_at.desc().nullslast(), SystemLog.id.desc()).limit(scan_limit)
+        result = await db.execute(query)
+        events = [_database_event_from_system_record(record) for record in result.scalars().all()]
+    elif source_id == "audit-db":
+        query = select(AuditLog).order_by(AuditLog.occurred_at.desc().nullslast(), AuditLog.id.desc()).limit(scan_limit)
+        result = await db.execute(query)
+        events = [_database_event_from_audit_record(record) for record in result.scalars().all()]
+    else:
+        return None
+
+    events = list(reversed(events))
+    return [
+        event
+        for event in events
+        if event_matches_levels(event, selected_levels)
+        and event_matches_search(event, search_query)
+        and event_matches_date_range(event, start_date, end_date)
+    ]
+
+
+async def read_database_log_snapshot(
+    source_id: str,
+    *,
+    limit: int,
+    level: str,
+    levels: str | None,
+    start_date: str | None,
+    end_date: str | None,
+    search: str | None,
+    db: AsyncSession,
+) -> dict[str, Any] | None:
+    events = await read_database_log_events(
+        source_id,
+        scan_limit=limit * 5,
+        level=level,
+        levels=levels,
+        start_date=start_date,
+        end_date=end_date,
+        search=search,
+        db=db,
+    )
+    if events is None:
+        return None
+    visible_events = events[-limit:]
+    selected_levels = normalize_log_levels(level, levels)
+    return {
+        "source_id": source_id,
+        "name": "系统事件" if source_id == "system-db" else "审计事件",
+        "kind": "database",
+        "location": "table://system_logs" if source_id == "system-db" else "table://audit_logs",
+        "description": "数据库持久化日志",
+        "category": "database" if source_id == "system-db" else "audit",
+        "status": "ok" if visible_events else "empty",
+        "level": level,
+        "selected_levels": list(selected_levels),
+        "search_query": search or "",
+        "available_levels": ["all", "error", "warning", "info", "debug"],
+        "daily_markers": build_daily_log_markers_from_events(events),
+        "line_limit": limit,
+        "line_count": len(visible_events),
+        "lines": [event.line for event in visible_events],
+    }
+
+
+def _stable_hash(value: str) -> str:
+    return hashlib.sha1(value.encode("utf-8", errors="replace")).hexdigest()[:16]
+
+
+def build_log_events(source_id: str, entries: list[StructuredLogEntry]) -> list[LogEvent]:
+    events: list[LogEvent] = []
+    seen: dict[str, int] = {}
+    for entry in entries:
+        stable_value = entry.raw_line or entry.display_line
+        digest = _stable_hash(stable_value)
+        occurrence = seen.get(digest, 0) + 1
+        seen[digest] = occurrence
+        events.append(
+            LogEvent(
+                source_id=source_id,
+                cursor=f"{source_id}:{digest}:{occurrence}",
+                timestamp=entry.timestamp,
+                level=entry.level,
+                line=entry.display_line,
+                search_text=entry.search_text,
+            )
+        )
+    return events
+
+
 def matches_levels(entry: StructuredLogEntry, selected_levels: tuple[str, ...]) -> bool:
     if not selected_levels:
         return True
@@ -469,6 +700,34 @@ def matches_search(entry: StructuredLogEntry, search: str | None) -> bool:
     return query in entry.search_text
 
 
+def event_matches_levels(event: LogEvent, selected_levels: tuple[str, ...]) -> bool:
+    if not selected_levels:
+        return True
+    return event.level in selected_levels
+
+
+def event_matches_date_range(event: LogEvent, start_date: str | None, end_date: str | None) -> bool:
+    if not start_date and not end_date:
+        return True
+    if event.timestamp is None:
+        return False
+    date_token = event.timestamp.astimezone(UTC).date().isoformat()
+    if start_date and date_token < start_date:
+        return False
+    if end_date and date_token > end_date:
+        return False
+    return True
+
+
+def event_matches_search(event: LogEvent, search: str | None) -> bool:
+    if search is None:
+        return True
+    query = search.strip().lower()
+    if not query:
+        return True
+    return query in event.search_text
+
+
 def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str, Any]]:
     grouped: dict[str, list[StructuredLogEntry]] = {}
     for entry in entries:
@@ -503,6 +762,69 @@ def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str,
     return [marker.__dict__ for marker in markers]
 
 
+def build_daily_log_markers_from_events(events: list[LogEvent]) -> list[dict[str, Any]]:
+    grouped: dict[str, list[LogEvent]] = {}
+    for event in events:
+        if event.timestamp is None:
+            continue
+        date_token = event.timestamp.astimezone(UTC).date().isoformat()
+        grouped.setdefault(date_token, []).append(event)
+
+    markers: list[DailyLogMarker] = []
+    for date_token, group in sorted(grouped.items()):
+        level_counts = Counter(
+            event.level
+            for event in group
+            if event.level in SUPPORTED_LOG_LEVELS and event.level != LOG_LEVEL_ALL
+        )
+        dominant_level = LOG_LEVEL_INFO
+        if level_counts:
+            dominant_level = sorted(
+                level_counts.items(),
+                key=lambda item: (
+                    -item[1],
+                    ("error", "warning", "info", "debug").index(item[0]),
+                ),
+            )[0][0]
+        markers.append(
+            DailyLogMarker(
+                date_token=date_token,
+                total=len(group),
+                dominant_level=dominant_level,
+            )
+        )
+    return [marker.__dict__ for marker in markers]
+
+
+def read_log_events(
+    source_id: str,
+    scan_limit: int,
+    *,
+    level: str = LOG_LEVEL_ALL,
+    levels: str | None = None,
+    start_date: str | None = None,
+    end_date: str | None = None,
+    search: str | None = None,
+) -> list[LogEvent] | None:
+    source = LOG_SOURCES.get(source_id)
+    if source is None:
+        return None
+
+    selected_levels = normalize_log_levels(level, levels)
+    search_query = (search or "").strip()
+    events = build_log_events(source_id, read_source_entries(source, scan_limit))
+    marker_events = [
+        event
+        for event in events
+        if event_matches_levels(event, selected_levels) and event_matches_search(event, search_query)
+    ]
+    return [
+        event
+        for event in marker_events
+        if event_matches_date_range(event, start_date, end_date)
+    ]
+
+
 def read_log_snapshot(
     source_id: str,
     limit: int,
@@ -520,18 +842,18 @@ def read_log_snapshot(
     selected_levels = normalize_log_levels(level, levels)
     search_query = (search or "").strip()
     scan_limit = max(min(MAX_LOG_LINE_LIMIT * 5, 5000), limit * 5, BUFFER_LOG_LIMIT if source.kind == "buffer" else 1000)
-    all_entries = read_source_entries(source, scan_limit)
-    marker_entries = [
-        entry
-        for entry in all_entries
-        if matches_levels(entry, selected_levels) and matches_search(entry, search_query)
+    all_events = build_log_events(source_id, read_source_entries(source, scan_limit))
+    marker_events = [
+        event
+        for event in all_events
+        if event_matches_levels(event, selected_levels) and event_matches_search(event, search_query)
     ]
-    filtered_entries = [
-        entry
-        for entry in marker_entries
-        if matches_date_range(entry, start_date, end_date)
+    filtered_events = [
+        event
+        for event in marker_events
+        if event_matches_date_range(event, start_date, end_date)
     ]
-    visible_entries = filtered_entries[-limit:]
+    visible_events = filtered_events[-limit:]
 
     compatibility_level = selected_levels[0] if len(selected_levels) == 1 else LOG_LEVEL_ALL
     return {
@@ -552,8 +874,8 @@ def read_log_snapshot(
             LOG_LEVEL_INFO,
             LOG_LEVEL_DEBUG,
         ],
-        "daily_markers": build_daily_log_markers(marker_entries),
+        "daily_markers": build_daily_log_markers_from_events(marker_events),
         "line_limit": limit,
-        "line_count": len(visible_entries),
-        "lines": [entry.display_line for entry in visible_entries],
+        "line_count": len(visible_events),
+        "lines": [event.line for event in visible_events],
     }