release: bump version to 0.67.0

backend/app/services/system_logs.py

@@ -5,6 +5,7 @@ import os
 import re
 import shutil
 import subprocess
+import hashlib
 
 from collections import Counter, deque
 from dataclasses import dataclass
@@ -13,6 +14,9 @@ from pathlib import Path
 from typing import Any
 
 from app.core.security import redis_client
+from app.models.system_log import AuditLog, SystemLog
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
 
 DEFAULT_LOG_LINE_LIMIT = 200
 MAX_LOG_LINE_LIMIT = 1000
@@ -99,6 +103,16 @@ class StructuredLogEntry:
     search_text: str
 
 
+@dataclass(frozen=True)
+class LogEvent:
+    source_id: str
+    cursor: str
+    timestamp: datetime | None
+    level: str | None
+    line: str
+    search_text: str
+
+
 @dataclass
 class DailyLogMarker:
     date_token: str
@@ -135,7 +149,7 @@ LOG_SOURCES: dict[str, LogSource] = {
         name="前端开发服务",
         kind="file",
         location=_state_log_path("frontend.log"),
-        description="控制台与 Earth 前端开发服务输出。",
+        description="控制台与智能星球前端开发服务输出。",
         category="service",
         fallback_locations=("/tmp/planet_frontend.log",),
     ),
@@ -150,13 +164,22 @@ LOG_SOURCES: dict[str, LogSource] = {
     ),
     "earth-client": LogSource(
         source_id="earth-client",
-        name="Earth 浏览器端",
+        name="智能星球浏览器端",
         kind="buffer",
         location="redis://planet:system_logs:earth-client",
-        description="Earth 浏览器端上报的运行时错误与关键业务日志。",
+        description="智能星球浏览器端上报的运行时错误与关键业务日志。",
         category="client",
         buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:earth-client",
     ),
+    "admin-client": LogSource(
+        source_id="admin-client",
+        name="控制台浏览器端",
+        kind="buffer",
+        location="redis://planet:system_logs:admin-client",
+        description="控制台浏览器端上报的运行时错误。",
+        category="client",
+        buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:admin-client",
+    ),
 }
 
 
@@ -365,6 +388,44 @@ def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry:
     )
 
 
+def compact_log_context(context: dict | None) -> str:
+    if not context:
+        return ""
+    allowed = {
+        key: value
+        for key, value in (context or {}).items()
+        if key
+        in {
+            "status",
+            "duration_ms",
+            "provider",
+            "model",
+            "result_provider",
+            "result_model",
+            "collector_name",
+            "datasource_id",
+            "task_id",
+            "snapshot_id",
+            "raw_count",
+            "transformed_count",
+            "saved_count",
+            "created",
+            "updated",
+            "unchanged",
+            "deleted",
+            "result_count",
+            "status_code",
+            "error_type",
+            "error",
+            "route",
+            "module",
+        }
+    }
+    if not allowed:
+        return ""
+    return json.dumps(allowed, ensure_ascii=False, sort_keys=True)
+
+
 def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
     path = resolve_file_log_path(source)
     if not path.exists():
@@ -437,6 +498,176 @@ def read_source_entries(source: LogSource, scan_limit: int) -> list[StructuredLo
     return []
 
 
+def _database_event_from_system_record(record: SystemLog) -> LogEvent:
+    record_level = normalize_log_level(record.level)
+    line = " ".join(
+        part
+        for part in [
+            record.occurred_at.isoformat() if record.occurred_at else "",
+            record_level.upper(),
+            record.source,
+            record.category or "",
+            record.event or "",
+            f"request_id={record.request_id}" if record.request_id else "",
+            record.message,
+            compact_log_context(record.context),
+        ]
+        if part
+    )
+    search_text = " ".join(
+        [
+            line,
+            f"id={record.id}",
+            f"user_id={record.user_id}" if record.user_id else "",
+            json.dumps(record.context or {}, ensure_ascii=False, sort_keys=True),
+        ]
+    ).lower()
+    return LogEvent(
+        source_id="system-db",
+        cursor=f"system-db:{record.id}",
+        timestamp=record.occurred_at,
+        level=None if record_level == LOG_LEVEL_ALL else record_level,
+        line=line,
+        search_text=search_text,
+    )
+
+
+def _database_event_from_audit_record(record: AuditLog) -> LogEvent:
+    line = " ".join(
+        part
+        for part in [
+            record.occurred_at.isoformat() if record.occurred_at else "",
+            "INFO",
+            record.action,
+            record.target_type or "",
+            record.target_id or "",
+            record.result or "",
+            f"request_id={record.request_id}" if record.request_id else "",
+        ]
+        if part
+    )
+    search_text = " ".join(
+        [
+            line,
+            f"id={record.id}",
+            f"actor_id={record.actor_id}" if record.actor_id else "",
+            record.actor_name or "",
+            json.dumps(record.details or {}, ensure_ascii=False, sort_keys=True),
+        ]
+    ).lower()
+    return LogEvent(
+        source_id="audit-db",
+        cursor=f"audit-db:{record.id}",
+        timestamp=record.occurred_at,
+        level=LOG_LEVEL_INFO,
+        line=line,
+        search_text=search_text,
+    )
+
+
+async def read_database_log_events(
+    source_id: str,
+    *,
+    scan_limit: int,
+    level: str = LOG_LEVEL_ALL,
+    levels: str | None = None,
+    start_date: str | None = None,
+    end_date: str | None = None,
+    search: str | None = None,
+    db: AsyncSession,
+) -> list[LogEvent] | None:
+    selected_levels = normalize_log_levels(level, levels)
+    search_query = (search or "").strip()
+    if source_id == "system-db":
+        query = select(SystemLog).order_by(SystemLog.occurred_at.desc().nullslast(), SystemLog.id.desc()).limit(scan_limit)
+        result = await db.execute(query)
+        events = [_database_event_from_system_record(record) for record in result.scalars().all()]
+    elif source_id == "audit-db":
+        query = select(AuditLog).order_by(AuditLog.occurred_at.desc().nullslast(), AuditLog.id.desc()).limit(scan_limit)
+        result = await db.execute(query)
+        events = [_database_event_from_audit_record(record) for record in result.scalars().all()]
+    else:
+        return None
+
+    events = list(reversed(events))
+    return [
+        event
+        for event in events
+        if event_matches_levels(event, selected_levels)
+        and event_matches_search(event, search_query)
+        and event_matches_date_range(event, start_date, end_date)
+    ]
+
+
+async def read_database_log_snapshot(
+    source_id: str,
+    *,
+    limit: int,
+    level: str,
+    levels: str | None,
+    start_date: str | None,
+    end_date: str | None,
+    search: str | None,
+    db: AsyncSession,
+) -> dict[str, Any] | None:
+    events = await read_database_log_events(
+        source_id,
+        scan_limit=limit * 5,
+        level=level,
+        levels=levels,
+        start_date=start_date,
+        end_date=end_date,
+        search=search,
+        db=db,
+    )
+    if events is None:
+        return None
+    visible_events = events[-limit:]
+    selected_levels = normalize_log_levels(level, levels)
+    return {
+        "source_id": source_id,
+        "name": "系统事件" if source_id == "system-db" else "审计事件",
+        "kind": "database",
+        "location": "table://system_logs" if source_id == "system-db" else "table://audit_logs",
+        "description": "数据库持久化日志",
+        "category": "database" if source_id == "system-db" else "audit",
+        "status": "ok" if visible_events else "empty",
+        "level": level,
+        "selected_levels": list(selected_levels),
+        "search_query": search or "",
+        "available_levels": ["all", "error", "warning", "info", "debug"],
+        "daily_markers": build_daily_log_markers_from_events(events),
+        "line_limit": limit,
+        "line_count": len(visible_events),
+        "lines": [event.line for event in visible_events],
+    }
+
+
+def _stable_hash(value: str) -> str:
+    return hashlib.sha1(value.encode("utf-8", errors="replace")).hexdigest()[:16]
+
+
+def build_log_events(source_id: str, entries: list[StructuredLogEntry]) -> list[LogEvent]:
+    events: list[LogEvent] = []
+    seen: dict[str, int] = {}
+    for entry in entries:
+        stable_value = entry.raw_line or entry.display_line
+        digest = _stable_hash(stable_value)
+        occurrence = seen.get(digest, 0) + 1
+        seen[digest] = occurrence
+        events.append(
+            LogEvent(
+                source_id=source_id,
+                cursor=f"{source_id}:{digest}:{occurrence}",
+                timestamp=entry.timestamp,
+                level=entry.level,
+                line=entry.display_line,
+                search_text=entry.search_text,
+            )
+        )
+    return events
+
+
 def matches_levels(entry: StructuredLogEntry, selected_levels: tuple[str, ...]) -> bool:
     if not selected_levels:
         return True
@@ -469,6 +700,34 @@ def matches_search(entry: StructuredLogEntry, search: str | None) -> bool:
     return query in entry.search_text
 
 
+def event_matches_levels(event: LogEvent, selected_levels: tuple[str, ...]) -> bool:
+    if not selected_levels:
+        return True
+    return event.level in selected_levels
+
+
+def event_matches_date_range(event: LogEvent, start_date: str | None, end_date: str | None) -> bool:
+    if not start_date and not end_date:
+        return True
+    if event.timestamp is None:
+        return False
+    date_token = event.timestamp.astimezone(UTC).date().isoformat()
+    if start_date and date_token < start_date:
+        return False
+    if end_date and date_token > end_date:
+        return False
+    return True
+
+
+def event_matches_search(event: LogEvent, search: str | None) -> bool:
+    if search is None:
+        return True
+    query = search.strip().lower()
+    if not query:
+        return True
+    return query in event.search_text
+
+
 def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str, Any]]:
     grouped: dict[str, list[StructuredLogEntry]] = {}
     for entry in entries:
@@ -503,6 +762,69 @@ def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str,
     return [marker.__dict__ for marker in markers]
 
 
+def build_daily_log_markers_from_events(events: list[LogEvent]) -> list[dict[str, Any]]:
+    grouped: dict[str, list[LogEvent]] = {}
+    for event in events:
+        if event.timestamp is None:
+            continue
+        date_token = event.timestamp.astimezone(UTC).date().isoformat()
+        grouped.setdefault(date_token, []).append(event)
+
+    markers: list[DailyLogMarker] = []
+    for date_token, group in sorted(grouped.items()):
+        level_counts = Counter(
+            event.level
+            for event in group
+            if event.level in SUPPORTED_LOG_LEVELS and event.level != LOG_LEVEL_ALL
+        )
+        dominant_level = LOG_LEVEL_INFO
+        if level_counts:
+            dominant_level = sorted(
+                level_counts.items(),
+                key=lambda item: (
+                    -item[1],
+                    ("error", "warning", "info", "debug").index(item[0]),
+                ),
+            )[0][0]
+        markers.append(
+            DailyLogMarker(
+                date_token=date_token,
+                total=len(group),
+                dominant_level=dominant_level,
+            )
+        )
+    return [marker.__dict__ for marker in markers]
+
+
+def read_log_events(
+    source_id: str,
+    scan_limit: int,
+    *,
+    level: str = LOG_LEVEL_ALL,
+    levels: str | None = None,
+    start_date: str | None = None,
+    end_date: str | None = None,
+    search: str | None = None,
+) -> list[LogEvent] | None:
+    source = LOG_SOURCES.get(source_id)
+    if source is None:
+        return None
+
+    selected_levels = normalize_log_levels(level, levels)
+    search_query = (search or "").strip()
+    events = build_log_events(source_id, read_source_entries(source, scan_limit))
+    marker_events = [
+        event
+        for event in events
+        if event_matches_levels(event, selected_levels) and event_matches_search(event, search_query)
+    ]
+    return [
+        event
+        for event in marker_events
+        if event_matches_date_range(event, start_date, end_date)
+    ]
+
+
 def read_log_snapshot(
     source_id: str,
     limit: int,
@@ -520,18 +842,18 @@ def read_log_snapshot(
     selected_levels = normalize_log_levels(level, levels)
     search_query = (search or "").strip()
     scan_limit = max(min(MAX_LOG_LINE_LIMIT * 5, 5000), limit * 5, BUFFER_LOG_LIMIT if source.kind == "buffer" else 1000)
-    all_entries = read_source_entries(source, scan_limit)
-    marker_entries = [
-        entry
-        for entry in all_entries
-        if matches_levels(entry, selected_levels) and matches_search(entry, search_query)
+    all_events = build_log_events(source_id, read_source_entries(source, scan_limit))
+    marker_events = [
+        event
+        for event in all_events
+        if event_matches_levels(event, selected_levels) and event_matches_search(event, search_query)
     ]
-    filtered_entries = [
-        entry
-        for entry in marker_entries
-        if matches_date_range(entry, start_date, end_date)
+    filtered_events = [
+        event
+        for event in marker_events
+        if event_matches_date_range(event, start_date, end_date)
     ]
-    visible_entries = filtered_entries[-limit:]
+    visible_events = filtered_events[-limit:]
 
     compatibility_level = selected_levels[0] if len(selected_levels) == 1 else LOG_LEVEL_ALL
     return {
@@ -552,8 +874,8 @@ def read_log_snapshot(
             LOG_LEVEL_INFO,
             LOG_LEVEL_DEBUG,
         ],
-        "daily_markers": build_daily_log_markers(marker_entries),
+        "daily_markers": build_daily_log_markers_from_events(marker_events),
         "line_limit": limit,
-        "line_count": len(visible_entries),
-        "lines": [entry.display_line for entry in visible_entries],
+        "line_count": len(visible_events),
+        "lines": [event.line for event in visible_events],
     }