560 lines
17 KiB
Python
560 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
|
|
from collections import Counter, deque
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.core.security import redis_client
|
|
|
|
DEFAULT_LOG_LINE_LIMIT = 200
|
|
MAX_LOG_LINE_LIMIT = 1000
|
|
BUFFER_LOG_LIMIT = 1000
|
|
BUFFER_LOG_TTL_SECONDS = 7 * 24 * 60 * 60
|
|
LOG_BUFFER_KEY_PREFIX = "planet:system_logs"
|
|
|
|
LOG_LEVEL_ERROR = "error"
|
|
LOG_LEVEL_WARNING = "warning"
|
|
LOG_LEVEL_INFO = "info"
|
|
LOG_LEVEL_DEBUG = "debug"
|
|
LOG_LEVEL_ALL = "all"
|
|
|
|
SUPPORTED_LOG_LEVELS = {
|
|
LOG_LEVEL_ALL,
|
|
LOG_LEVEL_ERROR,
|
|
LOG_LEVEL_WARNING,
|
|
LOG_LEVEL_INFO,
|
|
LOG_LEVEL_DEBUG,
|
|
}
|
|
|
|
LOG_LEVEL_ALIASES = {
|
|
"warn": LOG_LEVEL_WARNING,
|
|
"warning": LOG_LEVEL_WARNING,
|
|
"err": LOG_LEVEL_ERROR,
|
|
"error": LOG_LEVEL_ERROR,
|
|
"info": LOG_LEVEL_INFO,
|
|
"information": LOG_LEVEL_INFO,
|
|
"debug": LOG_LEVEL_DEBUG,
|
|
"trace": LOG_LEVEL_DEBUG,
|
|
"critical": LOG_LEVEL_ERROR,
|
|
"fatal": LOG_LEVEL_ERROR,
|
|
}
|
|
|
|
TIMESTAMP_FORMATS = (
|
|
"%Y-%m-%d %H:%M:%S.%f",
|
|
"%Y-%m-%d %H:%M:%S",
|
|
"%Y-%m-%dT%H:%M:%S.%f",
|
|
"%Y-%m-%dT%H:%M:%S",
|
|
)
|
|
|
|
LEVEL_PATTERNS = (
|
|
("CRITICAL", LOG_LEVEL_ERROR),
|
|
("FATAL", LOG_LEVEL_ERROR),
|
|
("ERROR", LOG_LEVEL_ERROR),
|
|
("WARNING", LOG_LEVEL_WARNING),
|
|
("WARN", LOG_LEVEL_WARNING),
|
|
("INFO", LOG_LEVEL_INFO),
|
|
("DEBUG", LOG_LEVEL_DEBUG),
|
|
("TRACE", LOG_LEVEL_DEBUG),
|
|
)
|
|
|
|
LEADING_LEVEL_PATTERN = re.compile(
|
|
r"^\s*(?:\[[^\]]+\]\s*)?(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG|TRACE)\b[:\s-]*",
|
|
re.IGNORECASE,
|
|
)
|
|
EMBEDDED_LEVEL_PATTERN = re.compile(
|
|
r"\b(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG|TRACE)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
CONTROL_CHAR_PATTERN = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LogSource:
|
|
source_id: str
|
|
name: str
|
|
kind: str
|
|
location: str
|
|
description: str
|
|
category: str
|
|
status: str = "ok"
|
|
buffer_key: str | None = None
|
|
container_name: str | None = None
|
|
fallback_locations: tuple[str, ...] = ()
|
|
|
|
|
|
@dataclass
|
|
class StructuredLogEntry:
|
|
timestamp: datetime | None
|
|
level: str | None
|
|
display_line: str
|
|
raw_line: str
|
|
search_text: str
|
|
|
|
|
|
@dataclass
|
|
class DailyLogMarker:
|
|
date_token: str
|
|
total: int
|
|
dominant_level: str
|
|
|
|
|
|
def _planet_state_dir() -> Path:
|
|
configured = os.getenv("PLANET_STATE_DIR")
|
|
if configured:
|
|
return Path(configured).expanduser()
|
|
xdg_state = os.getenv("XDG_STATE_HOME")
|
|
if xdg_state:
|
|
return Path(xdg_state).expanduser() / "planet"
|
|
return Path.home() / ".local" / "state" / "planet"
|
|
|
|
|
|
def _state_log_path(filename: str) -> str:
|
|
return str(_planet_state_dir() / filename)
|
|
|
|
|
|
LOG_SOURCES: dict[str, LogSource] = {
|
|
"backend": LogSource(
|
|
source_id="backend",
|
|
name="后端服务",
|
|
kind="file",
|
|
location=_state_log_path("backend.log"),
|
|
description="FastAPI 后端、调度器和采集任务共享日志。",
|
|
category="service",
|
|
fallback_locations=("/tmp/planet_backend.log",),
|
|
),
|
|
"frontend": LogSource(
|
|
source_id="frontend",
|
|
name="前端开发服务",
|
|
kind="file",
|
|
location=_state_log_path("frontend.log"),
|
|
description="控制台与 Earth 前端开发服务输出。",
|
|
category="service",
|
|
fallback_locations=("/tmp/planet_frontend.log",),
|
|
),
|
|
"ai-provider": LogSource(
|
|
source_id="ai-provider",
|
|
name="AI Provider",
|
|
kind="docker",
|
|
location="docker://planet_aiprovider",
|
|
description="AI Provider 容器实时输出日志。",
|
|
category="service",
|
|
container_name="planet_aiprovider",
|
|
),
|
|
"earth-client": LogSource(
|
|
source_id="earth-client",
|
|
name="Earth 浏览器端",
|
|
kind="buffer",
|
|
location="redis://planet:system_logs:earth-client",
|
|
description="Earth 浏览器端上报的运行时错误与关键业务日志。",
|
|
category="client",
|
|
buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:earth-client",
|
|
),
|
|
}
|
|
|
|
|
|
def normalize_log_level(level: str | None) -> str:
|
|
if level is None:
|
|
return LOG_LEVEL_ALL
|
|
normalized = str(level).strip().lower()
|
|
if normalized in {"", LOG_LEVEL_ALL}:
|
|
return LOG_LEVEL_ALL
|
|
return LOG_LEVEL_ALIASES.get(normalized, LOG_LEVEL_ALL)
|
|
|
|
|
|
def normalize_log_levels(level: str | None = None, levels: str | None = None) -> tuple[str, ...]:
|
|
normalized_levels: list[str] = []
|
|
if levels:
|
|
for item in str(levels).split(","):
|
|
normalized = normalize_log_level(item)
|
|
if normalized != LOG_LEVEL_ALL and normalized not in normalized_levels:
|
|
normalized_levels.append(normalized)
|
|
normalized_level = normalize_log_level(level)
|
|
if normalized_level != LOG_LEVEL_ALL and normalized_level not in normalized_levels:
|
|
normalized_levels.append(normalized_level)
|
|
return tuple(normalized_levels)
|
|
|
|
|
|
def resolve_file_log_path(source: LogSource) -> Path:
|
|
primary = Path(source.location).expanduser()
|
|
candidates = (primary, *(Path(item).expanduser() for item in source.fallback_locations))
|
|
for candidate in candidates:
|
|
if candidate.exists():
|
|
return candidate
|
|
return primary
|
|
|
|
|
|
def get_source_status(source: LogSource) -> str:
|
|
if source.kind == "file":
|
|
path = resolve_file_log_path(source)
|
|
if not path.exists():
|
|
return "missing"
|
|
return "ok" if path.stat().st_size > 0 else "empty"
|
|
if source.kind == "docker":
|
|
return "ok" if shutil.which("docker") else "docker_unavailable"
|
|
if source.kind == "buffer":
|
|
if not source.buffer_key:
|
|
return "source_unavailable"
|
|
try:
|
|
return "ok" if redis_client.llen(source.buffer_key) > 0 else "empty"
|
|
except Exception:
|
|
return "source_unavailable"
|
|
return "source_unavailable"
|
|
|
|
|
|
def list_log_sources() -> list[dict[str, str]]:
|
|
items: list[dict[str, str]] = []
|
|
for source in LOG_SOURCES.values():
|
|
items.append(
|
|
{
|
|
"source_id": source.source_id,
|
|
"name": source.name,
|
|
"kind": source.kind,
|
|
"location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location,
|
|
"description": source.description,
|
|
"category": source.category,
|
|
"status": get_source_status(source),
|
|
}
|
|
)
|
|
return items
|
|
|
|
|
|
def get_buffer_log_key(source_id: str) -> str:
|
|
return f"{LOG_BUFFER_KEY_PREFIX}:{source_id}"
|
|
|
|
|
|
def append_buffer_log(
|
|
source_id: str,
|
|
*,
|
|
level: str,
|
|
message: str,
|
|
context: dict[str, Any] | None = None,
|
|
) -> None:
|
|
payload = {
|
|
"timestamp": datetime.now(tz=UTC).isoformat(),
|
|
"level": normalize_log_level(level),
|
|
"message": message,
|
|
"context": context or {},
|
|
}
|
|
buffer_key = get_buffer_log_key(source_id)
|
|
redis_client.rpush(buffer_key, json.dumps(payload, ensure_ascii=False))
|
|
redis_client.ltrim(buffer_key, -BUFFER_LOG_LIMIT, -1)
|
|
redis_client.expire(buffer_key, BUFFER_LOG_TTL_SECONDS)
|
|
|
|
|
|
def parse_timestamp(raw_value: str | None) -> datetime | None:
|
|
if not raw_value:
|
|
return None
|
|
candidate = str(raw_value).strip()
|
|
if not candidate:
|
|
return None
|
|
candidate = candidate.replace("Z", "+00:00")
|
|
try:
|
|
parsed = datetime.fromisoformat(candidate)
|
|
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
|
except ValueError:
|
|
pass
|
|
for fmt in TIMESTAMP_FORMATS:
|
|
try:
|
|
return datetime.strptime(candidate, fmt).replace(tzinfo=UTC)
|
|
except ValueError:
|
|
continue
|
|
return None
|
|
|
|
|
|
def parse_prefixed_timestamp(line: str) -> tuple[datetime | None, str]:
|
|
stripped = line.strip()
|
|
if not stripped:
|
|
return None, ""
|
|
for prefix_length in (35, 32, 29, 26, 23, 19):
|
|
if len(stripped) < prefix_length:
|
|
continue
|
|
prefix = stripped[:prefix_length]
|
|
timestamp = parse_timestamp(prefix)
|
|
if timestamp is not None:
|
|
return timestamp, stripped[prefix_length:].lstrip()
|
|
first_token = stripped.split(maxsplit=1)[0]
|
|
timestamp = parse_timestamp(first_token)
|
|
if timestamp is not None:
|
|
remainder = stripped[len(first_token):].lstrip()
|
|
return timestamp, remainder
|
|
return None, stripped
|
|
|
|
|
|
def infer_log_level_from_text(text: str, *, allow_embedded: bool = True) -> str | None:
|
|
leading_match = LEADING_LEVEL_PATTERN.match(text)
|
|
if leading_match:
|
|
return normalize_log_level(leading_match.group(1))
|
|
|
|
if allow_embedded:
|
|
embedded_match = EMBEDDED_LEVEL_PATTERN.search(text)
|
|
if embedded_match:
|
|
return normalize_log_level(embedded_match.group(1))
|
|
upper_text = text.upper()
|
|
for pattern, normalized in LEVEL_PATTERNS:
|
|
if f"{pattern}:" in upper_text or f"{pattern} " in upper_text:
|
|
return normalized
|
|
return None
|
|
|
|
|
|
def build_display_line(timestamp: datetime | None, level: str | None, message: str) -> str:
|
|
message_part = message.strip() if message else ""
|
|
parts = []
|
|
if timestamp is not None:
|
|
parts.append(timestamp.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S"))
|
|
if level:
|
|
parts.append(level.upper())
|
|
if message_part:
|
|
parts.append(message_part)
|
|
return " ".join(parts).strip()
|
|
|
|
|
|
def sanitize_text_log_line(line: str) -> str:
|
|
return CONTROL_CHAR_PATTERN.sub("", line)
|
|
|
|
|
|
def parse_text_log_entry(line: str) -> StructuredLogEntry:
|
|
sanitized_line = sanitize_text_log_line(line).rstrip("\n")
|
|
timestamp, remainder = parse_prefixed_timestamp(sanitized_line)
|
|
level = infer_log_level_from_text(remainder or sanitized_line, allow_embedded=False)
|
|
display_line = sanitized_line
|
|
return StructuredLogEntry(
|
|
timestamp=timestamp,
|
|
level=level,
|
|
display_line=display_line,
|
|
raw_line=display_line,
|
|
search_text=display_line.lower(),
|
|
)
|
|
|
|
|
|
def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry:
|
|
timestamp = parse_timestamp(str(payload.get("timestamp", "")).strip())
|
|
level = normalize_log_level(payload.get("level"))
|
|
if level == LOG_LEVEL_ALL:
|
|
level = None
|
|
message = str(payload.get("message", "")).strip()
|
|
context = payload.get("context")
|
|
context_map = context if isinstance(context, dict) else {}
|
|
context_fragments = []
|
|
for key in ("category", "module", "url", "detail"):
|
|
value = str(context_map.get(key, "")).strip()
|
|
if value:
|
|
context_fragments.append(f"{key}={value}")
|
|
message_with_context = " | ".join([message, *context_fragments]) if context_fragments else message
|
|
display_line = build_display_line(timestamp, level, message_with_context)
|
|
search_text = " ".join(
|
|
[
|
|
message,
|
|
json.dumps(context_map, ensure_ascii=False, sort_keys=True),
|
|
display_line,
|
|
]
|
|
).lower()
|
|
return StructuredLogEntry(
|
|
timestamp=timestamp,
|
|
level=level,
|
|
display_line=display_line,
|
|
raw_line=json.dumps(payload, ensure_ascii=False, sort_keys=True),
|
|
search_text=search_text,
|
|
)
|
|
|
|
|
|
def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
|
path = resolve_file_log_path(source)
|
|
if not path.exists():
|
|
return []
|
|
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
recent_lines = deque(handle, maxlen=scan_limit)
|
|
return [
|
|
parse_text_log_entry(line)
|
|
for line in recent_lines
|
|
if sanitize_text_log_line(line).strip()
|
|
]
|
|
|
|
|
|
def read_docker_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
|
if not shutil.which("docker") or not source.container_name:
|
|
return []
|
|
try:
|
|
completed = subprocess.run(
|
|
[
|
|
"docker",
|
|
"logs",
|
|
"--timestamps",
|
|
"--tail",
|
|
str(scan_limit),
|
|
source.container_name,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
except OSError:
|
|
return []
|
|
if completed.returncode != 0:
|
|
return []
|
|
return [
|
|
parse_text_log_entry(line)
|
|
for line in completed.stdout.splitlines()
|
|
if line.strip()
|
|
]
|
|
|
|
|
|
def read_buffer_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
|
if not source.buffer_key:
|
|
return []
|
|
try:
|
|
raw_items = redis_client.lrange(source.buffer_key, -scan_limit, -1)
|
|
except Exception:
|
|
return []
|
|
entries: list[StructuredLogEntry] = []
|
|
for raw_item in raw_items:
|
|
try:
|
|
payload = json.loads(raw_item)
|
|
except json.JSONDecodeError:
|
|
entries.append(parse_text_log_entry(str(raw_item)))
|
|
continue
|
|
if isinstance(payload, dict):
|
|
entries.append(build_buffer_entry(payload))
|
|
else:
|
|
entries.append(parse_text_log_entry(str(raw_item)))
|
|
return entries
|
|
|
|
|
|
def read_source_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
|
if source.kind == "file":
|
|
return read_file_entries(source, scan_limit)
|
|
if source.kind == "docker":
|
|
return read_docker_entries(source, scan_limit)
|
|
if source.kind == "buffer":
|
|
return read_buffer_entries(source, scan_limit)
|
|
return []
|
|
|
|
|
|
def matches_levels(entry: StructuredLogEntry, selected_levels: tuple[str, ...]) -> bool:
|
|
if not selected_levels:
|
|
return True
|
|
return entry.level in selected_levels
|
|
|
|
|
|
def matches_date_range(
|
|
entry: StructuredLogEntry,
|
|
start_date: str | None,
|
|
end_date: str | None,
|
|
) -> bool:
|
|
if not start_date and not end_date:
|
|
return True
|
|
if entry.timestamp is None:
|
|
return False
|
|
date_token = entry.timestamp.astimezone(UTC).date().isoformat()
|
|
if start_date and date_token < start_date:
|
|
return False
|
|
if end_date and date_token > end_date:
|
|
return False
|
|
return True
|
|
|
|
|
|
def matches_search(entry: StructuredLogEntry, search: str | None) -> bool:
|
|
if search is None:
|
|
return True
|
|
query = search.strip().lower()
|
|
if not query:
|
|
return True
|
|
return query in entry.search_text
|
|
|
|
|
|
def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str, Any]]:
|
|
grouped: dict[str, list[StructuredLogEntry]] = {}
|
|
for entry in entries:
|
|
if entry.timestamp is None:
|
|
continue
|
|
date_token = entry.timestamp.astimezone(UTC).date().isoformat()
|
|
grouped.setdefault(date_token, []).append(entry)
|
|
|
|
markers: list[DailyLogMarker] = []
|
|
for date_token, group in sorted(grouped.items()):
|
|
level_counts = Counter(
|
|
entry.level
|
|
for entry in group
|
|
if entry.level in SUPPORTED_LOG_LEVELS and entry.level != LOG_LEVEL_ALL
|
|
)
|
|
dominant_level = LOG_LEVEL_INFO
|
|
if level_counts:
|
|
dominant_level = sorted(
|
|
level_counts.items(),
|
|
key=lambda item: (
|
|
-item[1],
|
|
("error", "warning", "info", "debug").index(item[0]),
|
|
),
|
|
)[0][0]
|
|
markers.append(
|
|
DailyLogMarker(
|
|
date_token=date_token,
|
|
total=len(group),
|
|
dominant_level=dominant_level,
|
|
)
|
|
)
|
|
return [marker.__dict__ for marker in markers]
|
|
|
|
|
|
def read_log_snapshot(
|
|
source_id: str,
|
|
limit: int,
|
|
*,
|
|
level: str = LOG_LEVEL_ALL,
|
|
levels: str | None = None,
|
|
start_date: str | None = None,
|
|
end_date: str | None = None,
|
|
search: str | None = None,
|
|
) -> dict[str, Any] | None:
|
|
source = LOG_SOURCES.get(source_id)
|
|
if source is None:
|
|
return None
|
|
|
|
selected_levels = normalize_log_levels(level, levels)
|
|
search_query = (search or "").strip()
|
|
scan_limit = max(min(MAX_LOG_LINE_LIMIT * 5, 5000), limit * 5, BUFFER_LOG_LIMIT if source.kind == "buffer" else 1000)
|
|
all_entries = read_source_entries(source, scan_limit)
|
|
marker_entries = [
|
|
entry
|
|
for entry in all_entries
|
|
if matches_levels(entry, selected_levels) and matches_search(entry, search_query)
|
|
]
|
|
filtered_entries = [
|
|
entry
|
|
for entry in marker_entries
|
|
if matches_date_range(entry, start_date, end_date)
|
|
]
|
|
visible_entries = filtered_entries[-limit:]
|
|
|
|
compatibility_level = selected_levels[0] if len(selected_levels) == 1 else LOG_LEVEL_ALL
|
|
return {
|
|
"source_id": source.source_id,
|
|
"name": source.name,
|
|
"kind": source.kind,
|
|
"location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location,
|
|
"description": source.description,
|
|
"category": source.category,
|
|
"status": get_source_status(source),
|
|
"level": compatibility_level,
|
|
"selected_levels": list(selected_levels),
|
|
"search_query": search_query,
|
|
"available_levels": [
|
|
LOG_LEVEL_ALL,
|
|
LOG_LEVEL_ERROR,
|
|
LOG_LEVEL_WARNING,
|
|
LOG_LEVEL_INFO,
|
|
LOG_LEVEL_DEBUG,
|
|
],
|
|
"daily_markers": build_daily_log_markers(marker_entries),
|
|
"line_limit": limit,
|
|
"line_count": len(visible_entries),
|
|
"lines": [entry.display_line for entry in visible_entries],
|
|
}
|