from __future__ import annotations import json import os import re import shutil import subprocess import hashlib from collections import Counter, deque from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any from app.core.security import redis_client from app.models.system_log import AuditLog, SystemLog from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession DEFAULT_LOG_LINE_LIMIT = 200 MAX_LOG_LINE_LIMIT = 1000 BUFFER_LOG_LIMIT = 1000 BUFFER_LOG_TTL_SECONDS = 7 * 24 * 60 * 60 LOG_BUFFER_KEY_PREFIX = "planet:system_logs" LOG_LEVEL_ERROR = "error" LOG_LEVEL_WARNING = "warning" LOG_LEVEL_INFO = "info" LOG_LEVEL_DEBUG = "debug" LOG_LEVEL_ALL = "all" SUPPORTED_LOG_LEVELS = { LOG_LEVEL_ALL, LOG_LEVEL_ERROR, LOG_LEVEL_WARNING, LOG_LEVEL_INFO, LOG_LEVEL_DEBUG, } LOG_LEVEL_ALIASES = { "warn": LOG_LEVEL_WARNING, "warning": LOG_LEVEL_WARNING, "err": LOG_LEVEL_ERROR, "error": LOG_LEVEL_ERROR, "info": LOG_LEVEL_INFO, "information": LOG_LEVEL_INFO, "debug": LOG_LEVEL_DEBUG, "trace": LOG_LEVEL_DEBUG, "critical": LOG_LEVEL_ERROR, "fatal": LOG_LEVEL_ERROR, } TIMESTAMP_FORMATS = ( "%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S", ) LEVEL_PATTERNS = ( ("CRITICAL", LOG_LEVEL_ERROR), ("FATAL", LOG_LEVEL_ERROR), ("ERROR", LOG_LEVEL_ERROR), ("WARNING", LOG_LEVEL_WARNING), ("WARN", LOG_LEVEL_WARNING), ("INFO", LOG_LEVEL_INFO), ("DEBUG", LOG_LEVEL_DEBUG), ("TRACE", LOG_LEVEL_DEBUG), ) LEADING_LEVEL_PATTERN = re.compile( r"^\s*(?:\[[^\]]+\]\s*)?(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG|TRACE)\b[:\s-]*", re.IGNORECASE, ) EMBEDDED_LEVEL_PATTERN = re.compile( r"\b(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG|TRACE)\b", re.IGNORECASE, ) CONTROL_CHAR_PATTERN = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") @dataclass(frozen=True) class LogSource: source_id: str name: str kind: str location: str description: str category: str status: str = "ok" buffer_key: str | None = None container_name: str | None = None fallback_locations: tuple[str, ...] = () @dataclass class StructuredLogEntry: timestamp: datetime | None level: str | None display_line: str raw_line: str search_text: str @dataclass(frozen=True) class LogEvent: source_id: str cursor: str timestamp: datetime | None level: str | None line: str search_text: str @dataclass class DailyLogMarker: date_token: str total: int dominant_level: str def _planet_state_dir() -> Path: configured = os.getenv("PLANET_STATE_DIR") if configured: return Path(configured).expanduser() xdg_state = os.getenv("XDG_STATE_HOME") if xdg_state: return Path(xdg_state).expanduser() / "planet" return Path.home() / ".local" / "state" / "planet" def _state_log_path(filename: str) -> str: return str(_planet_state_dir() / filename) LOG_SOURCES: dict[str, LogSource] = { "backend": LogSource( source_id="backend", name="后端服务", kind="file", location=_state_log_path("backend.log"), description="FastAPI 后端、调度器和采集任务共享日志。", category="service", fallback_locations=("/tmp/planet_backend.log",), ), "frontend": LogSource( source_id="frontend", name="前端开发服务", kind="file", location=_state_log_path("frontend.log"), description="控制台与智能星球前端开发服务输出。", category="service", fallback_locations=("/tmp/planet_frontend.log",), ), "ai-provider": LogSource( source_id="ai-provider", name="AI Provider", kind="docker", location="docker://planet_aiprovider", description="AI Provider 容器实时输出日志。", category="service", container_name="planet_aiprovider", ), "earth-client": LogSource( source_id="earth-client", name="智能星球浏览器端", kind="buffer", location="redis://planet:system_logs:earth-client", description="智能星球浏览器端上报的运行时错误与关键业务日志。", category="client", buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:earth-client", ), "admin-client": LogSource( source_id="admin-client", name="控制台浏览器端", kind="buffer", location="redis://planet:system_logs:admin-client", description="控制台浏览器端上报的运行时错误。", category="client", buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:admin-client", ), } def normalize_log_level(level: str | None) -> str: if level is None: return LOG_LEVEL_ALL normalized = str(level).strip().lower() if normalized in {"", LOG_LEVEL_ALL}: return LOG_LEVEL_ALL return LOG_LEVEL_ALIASES.get(normalized, LOG_LEVEL_ALL) def normalize_log_levels(level: str | None = None, levels: str | None = None) -> tuple[str, ...]: normalized_levels: list[str] = [] if levels: for item in str(levels).split(","): normalized = normalize_log_level(item) if normalized != LOG_LEVEL_ALL and normalized not in normalized_levels: normalized_levels.append(normalized) normalized_level = normalize_log_level(level) if normalized_level != LOG_LEVEL_ALL and normalized_level not in normalized_levels: normalized_levels.append(normalized_level) return tuple(normalized_levels) def resolve_file_log_path(source: LogSource) -> Path: primary = Path(source.location).expanduser() candidates = (primary, *(Path(item).expanduser() for item in source.fallback_locations)) for candidate in candidates: if candidate.exists(): return candidate return primary def get_source_status(source: LogSource) -> str: if source.kind == "file": path = resolve_file_log_path(source) if not path.exists(): return "missing" return "ok" if path.stat().st_size > 0 else "empty" if source.kind == "docker": return "ok" if shutil.which("docker") else "docker_unavailable" if source.kind == "buffer": if not source.buffer_key: return "source_unavailable" try: return "ok" if redis_client.llen(source.buffer_key) > 0 else "empty" except Exception: return "source_unavailable" return "source_unavailable" def list_log_sources() -> list[dict[str, str]]: items: list[dict[str, str]] = [] for source in LOG_SOURCES.values(): items.append( { "source_id": source.source_id, "name": source.name, "kind": source.kind, "location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location, "description": source.description, "category": source.category, "status": get_source_status(source), } ) return items def get_buffer_log_key(source_id: str) -> str: return f"{LOG_BUFFER_KEY_PREFIX}:{source_id}" def append_buffer_log( source_id: str, *, level: str, message: str, context: dict[str, Any] | None = None, ) -> None: payload = { "timestamp": datetime.now(tz=UTC).isoformat(), "level": normalize_log_level(level), "message": message, "context": context or {}, } buffer_key = get_buffer_log_key(source_id) redis_client.rpush(buffer_key, json.dumps(payload, ensure_ascii=False)) redis_client.ltrim(buffer_key, -BUFFER_LOG_LIMIT, -1) redis_client.expire(buffer_key, BUFFER_LOG_TTL_SECONDS) def parse_timestamp(raw_value: str | None) -> datetime | None: if not raw_value: return None candidate = str(raw_value).strip() if not candidate: return None candidate = candidate.replace("Z", "+00:00") try: parsed = datetime.fromisoformat(candidate) return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) except ValueError: pass for fmt in TIMESTAMP_FORMATS: try: return datetime.strptime(candidate, fmt).replace(tzinfo=UTC) except ValueError: continue return None def parse_prefixed_timestamp(line: str) -> tuple[datetime | None, str]: stripped = line.strip() if not stripped: return None, "" for prefix_length in (35, 32, 29, 26, 23, 19): if len(stripped) < prefix_length: continue prefix = stripped[:prefix_length] timestamp = parse_timestamp(prefix) if timestamp is not None: return timestamp, stripped[prefix_length:].lstrip() first_token = stripped.split(maxsplit=1)[0] timestamp = parse_timestamp(first_token) if timestamp is not None: remainder = stripped[len(first_token):].lstrip() return timestamp, remainder return None, stripped def infer_log_level_from_text(text: str, *, allow_embedded: bool = True) -> str | None: leading_match = LEADING_LEVEL_PATTERN.match(text) if leading_match: return normalize_log_level(leading_match.group(1)) if allow_embedded: embedded_match = EMBEDDED_LEVEL_PATTERN.search(text) if embedded_match: return normalize_log_level(embedded_match.group(1)) upper_text = text.upper() for pattern, normalized in LEVEL_PATTERNS: if f"{pattern}:" in upper_text or f"{pattern} " in upper_text: return normalized return None def build_display_line(timestamp: datetime | None, level: str | None, message: str) -> str: message_part = message.strip() if message else "" parts = [] if timestamp is not None: parts.append(timestamp.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S")) if level: parts.append(level.upper()) if message_part: parts.append(message_part) return " ".join(parts).strip() def sanitize_text_log_line(line: str) -> str: return CONTROL_CHAR_PATTERN.sub("", line) def parse_text_log_entry(line: str) -> StructuredLogEntry: sanitized_line = sanitize_text_log_line(line).rstrip("\n") timestamp, remainder = parse_prefixed_timestamp(sanitized_line) level = infer_log_level_from_text(remainder or sanitized_line, allow_embedded=False) display_line = sanitized_line return StructuredLogEntry( timestamp=timestamp, level=level, display_line=display_line, raw_line=display_line, search_text=display_line.lower(), ) def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry: timestamp = parse_timestamp(str(payload.get("timestamp", "")).strip()) level = normalize_log_level(payload.get("level")) if level == LOG_LEVEL_ALL: level = None message = str(payload.get("message", "")).strip() context = payload.get("context") context_map = context if isinstance(context, dict) else {} context_fragments = [] for key in ("category", "module", "url", "detail"): value = str(context_map.get(key, "")).strip() if value: context_fragments.append(f"{key}={value}") message_with_context = " | ".join([message, *context_fragments]) if context_fragments else message display_line = build_display_line(timestamp, level, message_with_context) search_text = " ".join( [ message, json.dumps(context_map, ensure_ascii=False, sort_keys=True), display_line, ] ).lower() return StructuredLogEntry( timestamp=timestamp, level=level, display_line=display_line, raw_line=json.dumps(payload, ensure_ascii=False, sort_keys=True), search_text=search_text, ) def compact_log_context(context: dict | None) -> str: if not context: return "" allowed = { key: value for key, value in (context or {}).items() if key in { "status", "duration_ms", "provider", "model", "result_provider", "result_model", "collector_name", "datasource_id", "task_id", "snapshot_id", "raw_count", "transformed_count", "saved_count", "created", "updated", "unchanged", "deleted", "result_count", "status_code", "error_type", "error", "route", "module", } } if not allowed: return "" return json.dumps(allowed, ensure_ascii=False, sort_keys=True) def context_search_aliases(context: dict | None) -> str: if not context: return "" aliases: list[str] = [] for key, value in sorted((context or {}).items()): if value is None or isinstance(value, (dict, list, tuple, set)): continue normalized_key = str(key).strip() normalized_value = str(value).strip() if not normalized_key or not normalized_value: continue aliases.append(f"{normalized_key}={normalized_value}") return " ".join(aliases) def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: path = resolve_file_log_path(source) if not path.exists(): return [] with path.open("r", encoding="utf-8", errors="replace") as handle: recent_lines = deque(handle, maxlen=scan_limit) return [ parse_text_log_entry(line) for line in recent_lines if sanitize_text_log_line(line).strip() ] def read_docker_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: if not shutil.which("docker") or not source.container_name: return [] try: completed = subprocess.run( [ "docker", "logs", "--timestamps", "--tail", str(scan_limit), source.container_name, ], capture_output=True, text=True, check=False, ) except OSError: return [] if completed.returncode != 0: return [] return [ parse_text_log_entry(line) for line in completed.stdout.splitlines() if line.strip() ] def read_buffer_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: if not source.buffer_key: return [] try: raw_items = redis_client.lrange(source.buffer_key, -scan_limit, -1) except Exception: return [] entries: list[StructuredLogEntry] = [] for raw_item in raw_items: try: payload = json.loads(raw_item) except json.JSONDecodeError: entries.append(parse_text_log_entry(str(raw_item))) continue if isinstance(payload, dict): entries.append(build_buffer_entry(payload)) else: entries.append(parse_text_log_entry(str(raw_item))) return entries def read_source_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: if source.kind == "file": return read_file_entries(source, scan_limit) if source.kind == "docker": return read_docker_entries(source, scan_limit) if source.kind == "buffer": return read_buffer_entries(source, scan_limit) return [] def _database_event_from_system_record(record: SystemLog) -> LogEvent: record_level = normalize_log_level(record.level) line = " ".join( part for part in [ record.occurred_at.isoformat() if record.occurred_at else "", record_level.upper(), record.source, record.category or "", record.event or "", f"request_id={record.request_id}" if record.request_id else "", record.message, compact_log_context(record.context), ] if part ) search_text = " ".join( [ line, f"id={record.id}", f"user_id={record.user_id}" if record.user_id else "", context_search_aliases(record.context), json.dumps(record.context or {}, ensure_ascii=False, sort_keys=True), ] ).lower() return LogEvent( source_id="system-db", cursor=f"system-db:{record.id}", timestamp=record.occurred_at, level=None if record_level == LOG_LEVEL_ALL else record_level, line=line, search_text=search_text, ) def _database_event_from_audit_record(record: AuditLog) -> LogEvent: line = " ".join( part for part in [ record.occurred_at.isoformat() if record.occurred_at else "", "INFO", record.action, record.target_type or "", record.target_id or "", record.result or "", f"request_id={record.request_id}" if record.request_id else "", ] if part ) search_text = " ".join( [ line, f"id={record.id}", f"actor_id={record.actor_id}" if record.actor_id else "", record.actor_name or "", context_search_aliases(record.details), json.dumps(record.details or {}, ensure_ascii=False, sort_keys=True), ] ).lower() return LogEvent( source_id="audit-db", cursor=f"audit-db:{record.id}", timestamp=record.occurred_at, level=LOG_LEVEL_INFO, line=line, search_text=search_text, ) async def read_database_log_events( source_id: str, *, scan_limit: int, level: str = LOG_LEVEL_ALL, levels: str | None = None, start_date: str | None = None, end_date: str | None = None, search: str | None = None, db: AsyncSession, ) -> list[LogEvent] | None: selected_levels = normalize_log_levels(level, levels) search_query = (search or "").strip() if source_id == "system-db": query = select(SystemLog).order_by(SystemLog.occurred_at.desc().nullslast(), SystemLog.id.desc()).limit(scan_limit) result = await db.execute(query) events = [_database_event_from_system_record(record) for record in result.scalars().all()] elif source_id == "audit-db": query = select(AuditLog).order_by(AuditLog.occurred_at.desc().nullslast(), AuditLog.id.desc()).limit(scan_limit) result = await db.execute(query) events = [_database_event_from_audit_record(record) for record in result.scalars().all()] else: return None events = list(reversed(events)) return [ event for event in events if event_matches_levels(event, selected_levels) and event_matches_search(event, search_query) and event_matches_date_range(event, start_date, end_date) ] async def read_database_log_snapshot( source_id: str, *, limit: int, level: str, levels: str | None, start_date: str | None, end_date: str | None, search: str | None, db: AsyncSession, ) -> dict[str, Any] | None: events = await read_database_log_events( source_id, scan_limit=limit * 5, level=level, levels=levels, start_date=start_date, end_date=end_date, search=search, db=db, ) if events is None: return None visible_events = events[-limit:] selected_levels = normalize_log_levels(level, levels) return { "source_id": source_id, "name": "系统事件" if source_id == "system-db" else "审计事件", "kind": "database", "location": "table://system_logs" if source_id == "system-db" else "table://audit_logs", "description": "数据库持久化日志", "category": "database" if source_id == "system-db" else "audit", "status": "ok" if visible_events else "empty", "level": level, "selected_levels": list(selected_levels), "search_query": search or "", "available_levels": ["all", "error", "warning", "info", "debug"], "daily_markers": build_daily_log_markers_from_events(events), "line_limit": limit, "line_count": len(visible_events), "lines": [event.line for event in visible_events], } def _stable_hash(value: str) -> str: return hashlib.sha1(value.encode("utf-8", errors="replace")).hexdigest()[:16] def build_log_events(source_id: str, entries: list[StructuredLogEntry]) -> list[LogEvent]: events: list[LogEvent] = [] seen: dict[str, int] = {} for entry in entries: stable_value = entry.raw_line or entry.display_line digest = _stable_hash(stable_value) occurrence = seen.get(digest, 0) + 1 seen[digest] = occurrence events.append( LogEvent( source_id=source_id, cursor=f"{source_id}:{digest}:{occurrence}", timestamp=entry.timestamp, level=entry.level, line=entry.display_line, search_text=entry.search_text, ) ) return events def matches_levels(entry: StructuredLogEntry, selected_levels: tuple[str, ...]) -> bool: if not selected_levels: return True return entry.level in selected_levels def matches_date_range( entry: StructuredLogEntry, start_date: str | None, end_date: str | None, ) -> bool: if not start_date and not end_date: return True if entry.timestamp is None: return False date_token = entry.timestamp.astimezone(UTC).date().isoformat() if start_date and date_token < start_date: return False if end_date and date_token > end_date: return False return True def matches_search(entry: StructuredLogEntry, search: str | None) -> bool: if search is None: return True query = search.strip().lower() if not query: return True return query in entry.search_text def event_matches_levels(event: LogEvent, selected_levels: tuple[str, ...]) -> bool: if not selected_levels: return True return event.level in selected_levels def event_matches_date_range(event: LogEvent, start_date: str | None, end_date: str | None) -> bool: if not start_date and not end_date: return True if event.timestamp is None: return False date_token = event.timestamp.astimezone(UTC).date().isoformat() if start_date and date_token < start_date: return False if end_date and date_token > end_date: return False return True def event_matches_search(event: LogEvent, search: str | None) -> bool: if search is None: return True query = search.strip().lower() if not query: return True return query in event.search_text def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str, Any]]: grouped: dict[str, list[StructuredLogEntry]] = {} for entry in entries: if entry.timestamp is None: continue date_token = entry.timestamp.astimezone(UTC).date().isoformat() grouped.setdefault(date_token, []).append(entry) markers: list[DailyLogMarker] = [] for date_token, group in sorted(grouped.items()): level_counts = Counter( entry.level for entry in group if entry.level in SUPPORTED_LOG_LEVELS and entry.level != LOG_LEVEL_ALL ) dominant_level = LOG_LEVEL_INFO if level_counts: dominant_level = sorted( level_counts.items(), key=lambda item: ( -item[1], ("error", "warning", "info", "debug").index(item[0]), ), )[0][0] markers.append( DailyLogMarker( date_token=date_token, total=len(group), dominant_level=dominant_level, ) ) return [marker.__dict__ for marker in markers] def build_daily_log_markers_from_events(events: list[LogEvent]) -> list[dict[str, Any]]: grouped: dict[str, list[LogEvent]] = {} for event in events: if event.timestamp is None: continue date_token = event.timestamp.astimezone(UTC).date().isoformat() grouped.setdefault(date_token, []).append(event) markers: list[DailyLogMarker] = [] for date_token, group in sorted(grouped.items()): level_counts = Counter( event.level for event in group if event.level in SUPPORTED_LOG_LEVELS and event.level != LOG_LEVEL_ALL ) dominant_level = LOG_LEVEL_INFO if level_counts: dominant_level = sorted( level_counts.items(), key=lambda item: ( -item[1], ("error", "warning", "info", "debug").index(item[0]), ), )[0][0] markers.append( DailyLogMarker( date_token=date_token, total=len(group), dominant_level=dominant_level, ) ) return [marker.__dict__ for marker in markers] def read_log_events( source_id: str, scan_limit: int, *, level: str = LOG_LEVEL_ALL, levels: str | None = None, start_date: str | None = None, end_date: str | None = None, search: str | None = None, ) -> list[LogEvent] | None: source = LOG_SOURCES.get(source_id) if source is None: return None selected_levels = normalize_log_levels(level, levels) search_query = (search or "").strip() events = build_log_events(source_id, read_source_entries(source, scan_limit)) marker_events = [ event for event in events if event_matches_levels(event, selected_levels) and event_matches_search(event, search_query) ] return [ event for event in marker_events if event_matches_date_range(event, start_date, end_date) ] def read_log_snapshot( source_id: str, limit: int, *, level: str = LOG_LEVEL_ALL, levels: str | None = None, start_date: str | None = None, end_date: str | None = None, search: str | None = None, ) -> dict[str, Any] | None: source = LOG_SOURCES.get(source_id) if source is None: return None selected_levels = normalize_log_levels(level, levels) search_query = (search or "").strip() scan_limit = max(min(MAX_LOG_LINE_LIMIT * 5, 5000), limit * 5, BUFFER_LOG_LIMIT if source.kind == "buffer" else 1000) all_events = build_log_events(source_id, read_source_entries(source, scan_limit)) marker_events = [ event for event in all_events if event_matches_levels(event, selected_levels) and event_matches_search(event, search_query) ] filtered_events = [ event for event in marker_events if event_matches_date_range(event, start_date, end_date) ] visible_events = filtered_events[-limit:] compatibility_level = selected_levels[0] if len(selected_levels) == 1 else LOG_LEVEL_ALL return { "source_id": source.source_id, "name": source.name, "kind": source.kind, "location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location, "description": source.description, "category": source.category, "status": get_source_status(source), "level": compatibility_level, "selected_levels": list(selected_levels), "search_query": search_query, "available_levels": [ LOG_LEVEL_ALL, LOG_LEVEL_ERROR, LOG_LEVEL_WARNING, LOG_LEVEL_INFO, LOG_LEVEL_DEBUG, ], "daily_markers": build_daily_log_markers_from_events(marker_events), "line_limit": limit, "line_count": len(visible_events), "lines": [event.line for event in visible_events], }