from __future__ import annotations import json import os import re import shutil import subprocess from collections import Counter, deque from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any from app.core.security import redis_client DEFAULT_LOG_LINE_LIMIT = 200 MAX_LOG_LINE_LIMIT = 1000 BUFFER_LOG_LIMIT = 1000 BUFFER_LOG_TTL_SECONDS = 7 * 24 * 60 * 60 LOG_BUFFER_KEY_PREFIX = "planet:system_logs" LOG_LEVEL_ERROR = "error" LOG_LEVEL_WARNING = "warning" LOG_LEVEL_INFO = "info" LOG_LEVEL_DEBUG = "debug" LOG_LEVEL_ALL = "all" SUPPORTED_LOG_LEVELS = { LOG_LEVEL_ALL, LOG_LEVEL_ERROR, LOG_LEVEL_WARNING, LOG_LEVEL_INFO, LOG_LEVEL_DEBUG, } LOG_LEVEL_ALIASES = { "warn": LOG_LEVEL_WARNING, "warning": LOG_LEVEL_WARNING, "err": LOG_LEVEL_ERROR, "error": LOG_LEVEL_ERROR, "info": LOG_LEVEL_INFO, "information": LOG_LEVEL_INFO, "debug": LOG_LEVEL_DEBUG, "trace": LOG_LEVEL_DEBUG, "critical": LOG_LEVEL_ERROR, "fatal": LOG_LEVEL_ERROR, } TIMESTAMP_FORMATS = ( "%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S", ) LEVEL_PATTERNS = ( ("CRITICAL", LOG_LEVEL_ERROR), ("FATAL", LOG_LEVEL_ERROR), ("ERROR", LOG_LEVEL_ERROR), ("WARNING", LOG_LEVEL_WARNING), ("WARN", LOG_LEVEL_WARNING), ("INFO", LOG_LEVEL_INFO), ("DEBUG", LOG_LEVEL_DEBUG), ("TRACE", LOG_LEVEL_DEBUG), ) LEADING_LEVEL_PATTERN = re.compile( r"^\s*(?:\[[^\]]+\]\s*)?(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG|TRACE)\b[:\s-]*", re.IGNORECASE, ) EMBEDDED_LEVEL_PATTERN = re.compile( r"\b(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG|TRACE)\b", re.IGNORECASE, ) CONTROL_CHAR_PATTERN = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") @dataclass(frozen=True) class LogSource: source_id: str name: str kind: str location: str description: str category: str status: str = "ok" buffer_key: str | None = None container_name: str | None = None fallback_locations: tuple[str, ...] = () @dataclass class StructuredLogEntry: timestamp: datetime | None level: str | None display_line: str raw_line: str search_text: str @dataclass class DailyLogMarker: date_token: str total: int dominant_level: str def _planet_state_dir() -> Path: configured = os.getenv("PLANET_STATE_DIR") if configured: return Path(configured).expanduser() xdg_state = os.getenv("XDG_STATE_HOME") if xdg_state: return Path(xdg_state).expanduser() / "planet" return Path.home() / ".local" / "state" / "planet" def _state_log_path(filename: str) -> str: return str(_planet_state_dir() / filename) LOG_SOURCES: dict[str, LogSource] = { "backend": LogSource( source_id="backend", name="后端服务", kind="file", location=_state_log_path("backend.log"), description="FastAPI 后端、调度器和采集任务共享日志。", category="service", fallback_locations=("/tmp/planet_backend.log",), ), "frontend": LogSource( source_id="frontend", name="前端开发服务", kind="file", location=_state_log_path("frontend.log"), description="控制台与 Earth 前端开发服务输出。", category="service", fallback_locations=("/tmp/planet_frontend.log",), ), "ai-provider": LogSource( source_id="ai-provider", name="AI Provider", kind="docker", location="docker://planet_aiprovider", description="AI Provider 容器实时输出日志。", category="service", container_name="planet_aiprovider", ), "earth-client": LogSource( source_id="earth-client", name="Earth 浏览器端", kind="buffer", location="redis://planet:system_logs:earth-client", description="Earth 浏览器端上报的运行时错误与关键业务日志。", category="client", buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:earth-client", ), } def normalize_log_level(level: str | None) -> str: if level is None: return LOG_LEVEL_ALL normalized = str(level).strip().lower() if normalized in {"", LOG_LEVEL_ALL}: return LOG_LEVEL_ALL return LOG_LEVEL_ALIASES.get(normalized, LOG_LEVEL_ALL) def normalize_log_levels(level: str | None = None, levels: str | None = None) -> tuple[str, ...]: normalized_levels: list[str] = [] if levels: for item in str(levels).split(","): normalized = normalize_log_level(item) if normalized != LOG_LEVEL_ALL and normalized not in normalized_levels: normalized_levels.append(normalized) normalized_level = normalize_log_level(level) if normalized_level != LOG_LEVEL_ALL and normalized_level not in normalized_levels: normalized_levels.append(normalized_level) return tuple(normalized_levels) def resolve_file_log_path(source: LogSource) -> Path: primary = Path(source.location).expanduser() candidates = (primary, *(Path(item).expanduser() for item in source.fallback_locations)) for candidate in candidates: if candidate.exists(): return candidate return primary def get_source_status(source: LogSource) -> str: if source.kind == "file": path = resolve_file_log_path(source) if not path.exists(): return "missing" return "ok" if path.stat().st_size > 0 else "empty" if source.kind == "docker": return "ok" if shutil.which("docker") else "docker_unavailable" if source.kind == "buffer": if not source.buffer_key: return "source_unavailable" try: return "ok" if redis_client.llen(source.buffer_key) > 0 else "empty" except Exception: return "source_unavailable" return "source_unavailable" def list_log_sources() -> list[dict[str, str]]: items: list[dict[str, str]] = [] for source in LOG_SOURCES.values(): items.append( { "source_id": source.source_id, "name": source.name, "kind": source.kind, "location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location, "description": source.description, "category": source.category, "status": get_source_status(source), } ) return items def get_buffer_log_key(source_id: str) -> str: return f"{LOG_BUFFER_KEY_PREFIX}:{source_id}" def append_buffer_log( source_id: str, *, level: str, message: str, context: dict[str, Any] | None = None, ) -> None: payload = { "timestamp": datetime.now(tz=UTC).isoformat(), "level": normalize_log_level(level), "message": message, "context": context or {}, } buffer_key = get_buffer_log_key(source_id) redis_client.rpush(buffer_key, json.dumps(payload, ensure_ascii=False)) redis_client.ltrim(buffer_key, -BUFFER_LOG_LIMIT, -1) redis_client.expire(buffer_key, BUFFER_LOG_TTL_SECONDS) def parse_timestamp(raw_value: str | None) -> datetime | None: if not raw_value: return None candidate = str(raw_value).strip() if not candidate: return None candidate = candidate.replace("Z", "+00:00") try: parsed = datetime.fromisoformat(candidate) return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) except ValueError: pass for fmt in TIMESTAMP_FORMATS: try: return datetime.strptime(candidate, fmt).replace(tzinfo=UTC) except ValueError: continue return None def parse_prefixed_timestamp(line: str) -> tuple[datetime | None, str]: stripped = line.strip() if not stripped: return None, "" for prefix_length in (35, 32, 29, 26, 23, 19): if len(stripped) < prefix_length: continue prefix = stripped[:prefix_length] timestamp = parse_timestamp(prefix) if timestamp is not None: return timestamp, stripped[prefix_length:].lstrip() first_token = stripped.split(maxsplit=1)[0] timestamp = parse_timestamp(first_token) if timestamp is not None: remainder = stripped[len(first_token):].lstrip() return timestamp, remainder return None, stripped def infer_log_level_from_text(text: str, *, allow_embedded: bool = True) -> str | None: leading_match = LEADING_LEVEL_PATTERN.match(text) if leading_match: return normalize_log_level(leading_match.group(1)) if allow_embedded: embedded_match = EMBEDDED_LEVEL_PATTERN.search(text) if embedded_match: return normalize_log_level(embedded_match.group(1)) upper_text = text.upper() for pattern, normalized in LEVEL_PATTERNS: if f"{pattern}:" in upper_text or f"{pattern} " in upper_text: return normalized return None def build_display_line(timestamp: datetime | None, level: str | None, message: str) -> str: message_part = message.strip() if message else "" parts = [] if timestamp is not None: parts.append(timestamp.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S")) if level: parts.append(level.upper()) if message_part: parts.append(message_part) return " ".join(parts).strip() def sanitize_text_log_line(line: str) -> str: return CONTROL_CHAR_PATTERN.sub("", line) def parse_text_log_entry(line: str) -> StructuredLogEntry: sanitized_line = sanitize_text_log_line(line).rstrip("\n") timestamp, remainder = parse_prefixed_timestamp(sanitized_line) level = infer_log_level_from_text(remainder or sanitized_line, allow_embedded=False) display_line = sanitized_line return StructuredLogEntry( timestamp=timestamp, level=level, display_line=display_line, raw_line=display_line, search_text=display_line.lower(), ) def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry: timestamp = parse_timestamp(str(payload.get("timestamp", "")).strip()) level = normalize_log_level(payload.get("level")) if level == LOG_LEVEL_ALL: level = None message = str(payload.get("message", "")).strip() context = payload.get("context") context_map = context if isinstance(context, dict) else {} context_fragments = [] for key in ("category", "module", "url", "detail"): value = str(context_map.get(key, "")).strip() if value: context_fragments.append(f"{key}={value}") message_with_context = " | ".join([message, *context_fragments]) if context_fragments else message display_line = build_display_line(timestamp, level, message_with_context) search_text = " ".join( [ message, json.dumps(context_map, ensure_ascii=False, sort_keys=True), display_line, ] ).lower() return StructuredLogEntry( timestamp=timestamp, level=level, display_line=display_line, raw_line=json.dumps(payload, ensure_ascii=False, sort_keys=True), search_text=search_text, ) def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: path = resolve_file_log_path(source) if not path.exists(): return [] with path.open("r", encoding="utf-8", errors="replace") as handle: recent_lines = deque(handle, maxlen=scan_limit) return [ parse_text_log_entry(line) for line in recent_lines if sanitize_text_log_line(line).strip() ] def read_docker_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: if not shutil.which("docker") or not source.container_name: return [] try: completed = subprocess.run( [ "docker", "logs", "--timestamps", "--tail", str(scan_limit), source.container_name, ], capture_output=True, text=True, check=False, ) except OSError: return [] if completed.returncode != 0: return [] return [ parse_text_log_entry(line) for line in completed.stdout.splitlines() if line.strip() ] def read_buffer_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: if not source.buffer_key: return [] try: raw_items = redis_client.lrange(source.buffer_key, -scan_limit, -1) except Exception: return [] entries: list[StructuredLogEntry] = [] for raw_item in raw_items: try: payload = json.loads(raw_item) except json.JSONDecodeError: entries.append(parse_text_log_entry(str(raw_item))) continue if isinstance(payload, dict): entries.append(build_buffer_entry(payload)) else: entries.append(parse_text_log_entry(str(raw_item))) return entries def read_source_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: if source.kind == "file": return read_file_entries(source, scan_limit) if source.kind == "docker": return read_docker_entries(source, scan_limit) if source.kind == "buffer": return read_buffer_entries(source, scan_limit) return [] def matches_levels(entry: StructuredLogEntry, selected_levels: tuple[str, ...]) -> bool: if not selected_levels: return True return entry.level in selected_levels def matches_date_range( entry: StructuredLogEntry, start_date: str | None, end_date: str | None, ) -> bool: if not start_date and not end_date: return True if entry.timestamp is None: return False date_token = entry.timestamp.astimezone(UTC).date().isoformat() if start_date and date_token < start_date: return False if end_date and date_token > end_date: return False return True def matches_search(entry: StructuredLogEntry, search: str | None) -> bool: if search is None: return True query = search.strip().lower() if not query: return True return query in entry.search_text def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str, Any]]: grouped: dict[str, list[StructuredLogEntry]] = {} for entry in entries: if entry.timestamp is None: continue date_token = entry.timestamp.astimezone(UTC).date().isoformat() grouped.setdefault(date_token, []).append(entry) markers: list[DailyLogMarker] = [] for date_token, group in sorted(grouped.items()): level_counts = Counter( entry.level for entry in group if entry.level in SUPPORTED_LOG_LEVELS and entry.level != LOG_LEVEL_ALL ) dominant_level = LOG_LEVEL_INFO if level_counts: dominant_level = sorted( level_counts.items(), key=lambda item: ( -item[1], ("error", "warning", "info", "debug").index(item[0]), ), )[0][0] markers.append( DailyLogMarker( date_token=date_token, total=len(group), dominant_level=dominant_level, ) ) return [marker.__dict__ for marker in markers] def read_log_snapshot( source_id: str, limit: int, *, level: str = LOG_LEVEL_ALL, levels: str | None = None, start_date: str | None = None, end_date: str | None = None, search: str | None = None, ) -> dict[str, Any] | None: source = LOG_SOURCES.get(source_id) if source is None: return None selected_levels = normalize_log_levels(level, levels) search_query = (search or "").strip() scan_limit = max(min(MAX_LOG_LINE_LIMIT * 5, 5000), limit * 5, BUFFER_LOG_LIMIT if source.kind == "buffer" else 1000) all_entries = read_source_entries(source, scan_limit) marker_entries = [ entry for entry in all_entries if matches_levels(entry, selected_levels) and matches_search(entry, search_query) ] filtered_entries = [ entry for entry in marker_entries if matches_date_range(entry, start_date, end_date) ] visible_entries = filtered_entries[-limit:] compatibility_level = selected_levels[0] if len(selected_levels) == 1 else LOG_LEVEL_ALL return { "source_id": source.source_id, "name": source.name, "kind": source.kind, "location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location, "description": source.description, "category": source.category, "status": get_source_status(source), "level": compatibility_level, "selected_levels": list(selected_levels), "search_query": search_query, "available_levels": [ LOG_LEVEL_ALL, LOG_LEVEL_ERROR, LOG_LEVEL_WARNING, LOG_LEVEL_INFO, LOG_LEVEL_DEBUG, ], "daily_markers": build_daily_log_markers(marker_entries), "line_limit": limit, "line_count": len(visible_entries), "lines": [entry.display_line for entry in visible_entries], }