release: bump version to 0.38.0
This commit is contained in:
linkong
522
backend/app/services/system_logs.py
Normal file
522
backend/app/services/system_logs.py
Normal file
@@ -0,0 +1,522 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from collections import Counter, deque
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.core.security import redis_client
|
||||
|
||||
DEFAULT_LOG_LINE_LIMIT = 200
|
||||
MAX_LOG_LINE_LIMIT = 1000
|
||||
BUFFER_LOG_LIMIT = 1000
|
||||
BUFFER_LOG_TTL_SECONDS = 7 * 24 * 60 * 60
|
||||
LOG_BUFFER_KEY_PREFIX = "planet:system_logs"
|
||||
|
||||
LOG_LEVEL_ERROR = "error"
|
||||
LOG_LEVEL_WARNING = "warning"
|
||||
LOG_LEVEL_INFO = "info"
|
||||
LOG_LEVEL_DEBUG = "debug"
|
||||
LOG_LEVEL_ALL = "all"
|
||||
|
||||
SUPPORTED_LOG_LEVELS = {
|
||||
LOG_LEVEL_ALL,
|
||||
LOG_LEVEL_ERROR,
|
||||
LOG_LEVEL_WARNING,
|
||||
LOG_LEVEL_INFO,
|
||||
LOG_LEVEL_DEBUG,
|
||||
}
|
||||
|
||||
LOG_LEVEL_ALIASES = {
|
||||
"warn": LOG_LEVEL_WARNING,
|
||||
"warning": LOG_LEVEL_WARNING,
|
||||
"err": LOG_LEVEL_ERROR,
|
||||
"error": LOG_LEVEL_ERROR,
|
||||
"info": LOG_LEVEL_INFO,
|
||||
"information": LOG_LEVEL_INFO,
|
||||
"debug": LOG_LEVEL_DEBUG,
|
||||
"trace": LOG_LEVEL_DEBUG,
|
||||
"critical": LOG_LEVEL_ERROR,
|
||||
"fatal": LOG_LEVEL_ERROR,
|
||||
}
|
||||
|
||||
TIMESTAMP_FORMATS = (
|
||||
"%Y-%m-%d %H:%M:%S.%f",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%dT%H:%M:%S.%f",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
)
|
||||
|
||||
LEVEL_PATTERNS = (
|
||||
("CRITICAL", LOG_LEVEL_ERROR),
|
||||
("FATAL", LOG_LEVEL_ERROR),
|
||||
("ERROR", LOG_LEVEL_ERROR),
|
||||
("WARNING", LOG_LEVEL_WARNING),
|
||||
("WARN", LOG_LEVEL_WARNING),
|
||||
("INFO", LOG_LEVEL_INFO),
|
||||
("DEBUG", LOG_LEVEL_DEBUG),
|
||||
("TRACE", LOG_LEVEL_DEBUG),
|
||||
)
|
||||
|
||||
LEADING_LEVEL_PATTERN = re.compile(
|
||||
r"^\s*(?:\[[^\]]+\]\s*)?(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG|TRACE)\b[:\s-]*",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
EMBEDDED_LEVEL_PATTERN = re.compile(
|
||||
r"\b(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG|TRACE)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LogSource:
|
||||
source_id: str
|
||||
name: str
|
||||
kind: str
|
||||
location: str
|
||||
description: str
|
||||
category: str
|
||||
status: str = "ok"
|
||||
buffer_key: str | None = None
|
||||
container_name: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class StructuredLogEntry:
|
||||
timestamp: datetime | None
|
||||
level: str | None
|
||||
display_line: str
|
||||
raw_line: str
|
||||
search_text: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DailyLogMarker:
|
||||
date_token: str
|
||||
total: int
|
||||
dominant_level: str
|
||||
|
||||
|
||||
LOG_SOURCES: dict[str, LogSource] = {
|
||||
"backend": LogSource(
|
||||
source_id="backend",
|
||||
name="后端服务",
|
||||
kind="file",
|
||||
location="/tmp/planet_backend.log",
|
||||
description="FastAPI 后端、调度器和采集任务共享日志。",
|
||||
category="service",
|
||||
),
|
||||
"frontend": LogSource(
|
||||
source_id="frontend",
|
||||
name="前端开发服务",
|
||||
kind="file",
|
||||
location="/tmp/planet_frontend.log",
|
||||
description="控制台与 Earth 前端开发服务输出。",
|
||||
category="service",
|
||||
),
|
||||
"ai-provider": LogSource(
|
||||
source_id="ai-provider",
|
||||
name="AI Provider",
|
||||
kind="docker",
|
||||
location="docker://planet_aiprovider",
|
||||
description="AI Provider 容器实时输出日志。",
|
||||
category="service",
|
||||
container_name="planet_aiprovider",
|
||||
),
|
||||
"earth-client": LogSource(
|
||||
source_id="earth-client",
|
||||
name="Earth 浏览器端",
|
||||
kind="buffer",
|
||||
location="redis://planet:system_logs:earth-client",
|
||||
description="Earth 浏览器端上报的运行时错误与关键业务日志。",
|
||||
category="client",
|
||||
buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:earth-client",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def normalize_log_level(level: str | None) -> str:
|
||||
if level is None:
|
||||
return LOG_LEVEL_ALL
|
||||
normalized = str(level).strip().lower()
|
||||
if normalized in {"", LOG_LEVEL_ALL}:
|
||||
return LOG_LEVEL_ALL
|
||||
return LOG_LEVEL_ALIASES.get(normalized, LOG_LEVEL_ALL)
|
||||
|
||||
|
||||
def normalize_log_levels(level: str | None = None, levels: str | None = None) -> tuple[str, ...]:
|
||||
normalized_levels: list[str] = []
|
||||
if levels:
|
||||
for item in str(levels).split(","):
|
||||
normalized = normalize_log_level(item)
|
||||
if normalized != LOG_LEVEL_ALL and normalized not in normalized_levels:
|
||||
normalized_levels.append(normalized)
|
||||
normalized_level = normalize_log_level(level)
|
||||
if normalized_level != LOG_LEVEL_ALL and normalized_level not in normalized_levels:
|
||||
normalized_levels.append(normalized_level)
|
||||
return tuple(normalized_levels)
|
||||
|
||||
|
||||
def get_source_status(source: LogSource) -> str:
|
||||
if source.kind == "file":
|
||||
path = Path(source.location)
|
||||
if not path.exists():
|
||||
return "missing"
|
||||
return "ok" if path.stat().st_size > 0 else "empty"
|
||||
if source.kind == "docker":
|
||||
return "ok" if shutil.which("docker") else "docker_unavailable"
|
||||
if source.kind == "buffer":
|
||||
if not source.buffer_key:
|
||||
return "source_unavailable"
|
||||
try:
|
||||
return "ok" if redis_client.llen(source.buffer_key) > 0 else "empty"
|
||||
except Exception:
|
||||
return "source_unavailable"
|
||||
return "source_unavailable"
|
||||
|
||||
|
||||
def list_log_sources() -> list[dict[str, str]]:
|
||||
items: list[dict[str, str]] = []
|
||||
for source in LOG_SOURCES.values():
|
||||
items.append(
|
||||
{
|
||||
"source_id": source.source_id,
|
||||
"name": source.name,
|
||||
"kind": source.kind,
|
||||
"location": source.location,
|
||||
"description": source.description,
|
||||
"category": source.category,
|
||||
"status": get_source_status(source),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def get_buffer_log_key(source_id: str) -> str:
|
||||
return f"{LOG_BUFFER_KEY_PREFIX}:{source_id}"
|
||||
|
||||
|
||||
def append_buffer_log(
|
||||
source_id: str,
|
||||
*,
|
||||
level: str,
|
||||
message: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
payload = {
|
||||
"timestamp": datetime.now(tz=UTC).isoformat(),
|
||||
"level": normalize_log_level(level),
|
||||
"message": message,
|
||||
"context": context or {},
|
||||
}
|
||||
buffer_key = get_buffer_log_key(source_id)
|
||||
redis_client.rpush(buffer_key, json.dumps(payload, ensure_ascii=False))
|
||||
redis_client.ltrim(buffer_key, -BUFFER_LOG_LIMIT, -1)
|
||||
redis_client.expire(buffer_key, BUFFER_LOG_TTL_SECONDS)
|
||||
|
||||
|
||||
def parse_timestamp(raw_value: str | None) -> datetime | None:
|
||||
if not raw_value:
|
||||
return None
|
||||
candidate = str(raw_value).strip()
|
||||
if not candidate:
|
||||
return None
|
||||
candidate = candidate.replace("Z", "+00:00")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(candidate)
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
except ValueError:
|
||||
pass
|
||||
for fmt in TIMESTAMP_FORMATS:
|
||||
try:
|
||||
return datetime.strptime(candidate, fmt).replace(tzinfo=UTC)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def parse_prefixed_timestamp(line: str) -> tuple[datetime | None, str]:
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
return None, ""
|
||||
for prefix_length in (35, 32, 29, 26, 23, 19):
|
||||
if len(stripped) < prefix_length:
|
||||
continue
|
||||
prefix = stripped[:prefix_length]
|
||||
timestamp = parse_timestamp(prefix)
|
||||
if timestamp is not None:
|
||||
return timestamp, stripped[prefix_length:].lstrip()
|
||||
first_token = stripped.split(maxsplit=1)[0]
|
||||
timestamp = parse_timestamp(first_token)
|
||||
if timestamp is not None:
|
||||
remainder = stripped[len(first_token):].lstrip()
|
||||
return timestamp, remainder
|
||||
return None, stripped
|
||||
|
||||
|
||||
def infer_log_level_from_text(text: str) -> str | None:
|
||||
leading_match = LEADING_LEVEL_PATTERN.match(text)
|
||||
if leading_match:
|
||||
return normalize_log_level(leading_match.group(1))
|
||||
|
||||
embedded_match = EMBEDDED_LEVEL_PATTERN.search(text)
|
||||
if embedded_match:
|
||||
return normalize_log_level(embedded_match.group(1))
|
||||
|
||||
upper_text = text.upper()
|
||||
for pattern, normalized in LEVEL_PATTERNS:
|
||||
if f"{pattern}:" in upper_text or f"{pattern} " in upper_text:
|
||||
return normalized
|
||||
return None
|
||||
|
||||
|
||||
def build_display_line(timestamp: datetime | None, level: str | None, message: str) -> str:
|
||||
message_part = message.strip() if message else ""
|
||||
parts = []
|
||||
if timestamp is not None:
|
||||
parts.append(timestamp.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S"))
|
||||
if level:
|
||||
parts.append(level.upper())
|
||||
if message_part:
|
||||
parts.append(message_part)
|
||||
return " ".join(parts).strip()
|
||||
|
||||
|
||||
def parse_text_log_entry(line: str) -> StructuredLogEntry:
|
||||
timestamp, remainder = parse_prefixed_timestamp(line)
|
||||
level = infer_log_level_from_text(remainder or line)
|
||||
display_line = line.rstrip("\n")
|
||||
return StructuredLogEntry(
|
||||
timestamp=timestamp,
|
||||
level=level,
|
||||
display_line=display_line,
|
||||
raw_line=display_line,
|
||||
search_text=display_line.lower(),
|
||||
)
|
||||
|
||||
|
||||
def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry:
|
||||
timestamp = parse_timestamp(str(payload.get("timestamp", "")).strip())
|
||||
level = normalize_log_level(payload.get("level"))
|
||||
if level == LOG_LEVEL_ALL:
|
||||
level = None
|
||||
message = str(payload.get("message", "")).strip()
|
||||
context = payload.get("context")
|
||||
context_map = context if isinstance(context, dict) else {}
|
||||
context_fragments = []
|
||||
for key in ("category", "module", "url", "detail"):
|
||||
value = str(context_map.get(key, "")).strip()
|
||||
if value:
|
||||
context_fragments.append(f"{key}={value}")
|
||||
message_with_context = " | ".join([message, *context_fragments]) if context_fragments else message
|
||||
display_line = build_display_line(timestamp, level, message_with_context)
|
||||
search_text = " ".join(
|
||||
[
|
||||
message,
|
||||
json.dumps(context_map, ensure_ascii=False, sort_keys=True),
|
||||
display_line,
|
||||
]
|
||||
).lower()
|
||||
return StructuredLogEntry(
|
||||
timestamp=timestamp,
|
||||
level=level,
|
||||
display_line=display_line,
|
||||
raw_line=json.dumps(payload, ensure_ascii=False, sort_keys=True),
|
||||
search_text=search_text,
|
||||
)
|
||||
|
||||
|
||||
def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
||||
path = Path(source.location)
|
||||
if not path.exists():
|
||||
return []
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
recent_lines = deque(handle, maxlen=scan_limit)
|
||||
return [parse_text_log_entry(line) for line in recent_lines if line.strip()]
|
||||
|
||||
|
||||
def read_docker_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
||||
if not shutil.which("docker") or not source.container_name:
|
||||
return []
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"logs",
|
||||
"--timestamps",
|
||||
"--tail",
|
||||
str(scan_limit),
|
||||
source.container_name,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError:
|
||||
return []
|
||||
if completed.returncode != 0:
|
||||
return []
|
||||
return [
|
||||
parse_text_log_entry(line)
|
||||
for line in completed.stdout.splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def read_buffer_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
||||
if not source.buffer_key:
|
||||
return []
|
||||
try:
|
||||
raw_items = redis_client.lrange(source.buffer_key, -scan_limit, -1)
|
||||
except Exception:
|
||||
return []
|
||||
entries: list[StructuredLogEntry] = []
|
||||
for raw_item in raw_items:
|
||||
try:
|
||||
payload = json.loads(raw_item)
|
||||
except json.JSONDecodeError:
|
||||
entries.append(parse_text_log_entry(str(raw_item)))
|
||||
continue
|
||||
if isinstance(payload, dict):
|
||||
entries.append(build_buffer_entry(payload))
|
||||
else:
|
||||
entries.append(parse_text_log_entry(str(raw_item)))
|
||||
return entries
|
||||
|
||||
|
||||
def read_source_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
||||
if source.kind == "file":
|
||||
return read_file_entries(source, scan_limit)
|
||||
if source.kind == "docker":
|
||||
return read_docker_entries(source, scan_limit)
|
||||
if source.kind == "buffer":
|
||||
return read_buffer_entries(source, scan_limit)
|
||||
return []
|
||||
|
||||
|
||||
def matches_levels(entry: StructuredLogEntry, selected_levels: tuple[str, ...]) -> bool:
|
||||
if not selected_levels:
|
||||
return True
|
||||
return entry.level in selected_levels
|
||||
|
||||
|
||||
def matches_date_range(
|
||||
entry: StructuredLogEntry,
|
||||
start_date: str | None,
|
||||
end_date: str | None,
|
||||
) -> bool:
|
||||
if not start_date and not end_date:
|
||||
return True
|
||||
if entry.timestamp is None:
|
||||
return False
|
||||
date_token = entry.timestamp.astimezone(UTC).date().isoformat()
|
||||
if start_date and date_token < start_date:
|
||||
return False
|
||||
if end_date and date_token > end_date:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def matches_search(entry: StructuredLogEntry, search: str | None) -> bool:
|
||||
if search is None:
|
||||
return True
|
||||
query = search.strip().lower()
|
||||
if not query:
|
||||
return True
|
||||
return query in entry.search_text
|
||||
|
||||
|
||||
def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str, Any]]:
|
||||
grouped: dict[str, list[StructuredLogEntry]] = {}
|
||||
for entry in entries:
|
||||
if entry.timestamp is None:
|
||||
continue
|
||||
date_token = entry.timestamp.astimezone(UTC).date().isoformat()
|
||||
grouped.setdefault(date_token, []).append(entry)
|
||||
|
||||
markers: list[DailyLogMarker] = []
|
||||
for date_token, group in sorted(grouped.items()):
|
||||
level_counts = Counter(
|
||||
entry.level
|
||||
for entry in group
|
||||
if entry.level in SUPPORTED_LOG_LEVELS and entry.level != LOG_LEVEL_ALL
|
||||
)
|
||||
dominant_level = LOG_LEVEL_INFO
|
||||
if level_counts:
|
||||
dominant_level = sorted(
|
||||
level_counts.items(),
|
||||
key=lambda item: (
|
||||
-item[1],
|
||||
("error", "warning", "info", "debug").index(item[0]),
|
||||
),
|
||||
)[0][0]
|
||||
markers.append(
|
||||
DailyLogMarker(
|
||||
date_token=date_token,
|
||||
total=len(group),
|
||||
dominant_level=dominant_level,
|
||||
)
|
||||
)
|
||||
return [marker.__dict__ for marker in markers]
|
||||
|
||||
|
||||
def read_log_snapshot(
|
||||
source_id: str,
|
||||
limit: int,
|
||||
*,
|
||||
level: str = LOG_LEVEL_ALL,
|
||||
levels: str | None = None,
|
||||
start_date: str | None = None,
|
||||
end_date: str | None = None,
|
||||
search: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
source = LOG_SOURCES.get(source_id)
|
||||
if source is None:
|
||||
return None
|
||||
|
||||
selected_levels = normalize_log_levels(level, levels)
|
||||
search_query = (search or "").strip()
|
||||
scan_limit = max(min(MAX_LOG_LINE_LIMIT * 5, 5000), limit * 5, BUFFER_LOG_LIMIT if source.kind == "buffer" else 1000)
|
||||
all_entries = read_source_entries(source, scan_limit)
|
||||
marker_entries = [
|
||||
entry
|
||||
for entry in all_entries
|
||||
if matches_levels(entry, selected_levels) and matches_search(entry, search_query)
|
||||
]
|
||||
filtered_entries = [
|
||||
entry
|
||||
for entry in marker_entries
|
||||
if matches_date_range(entry, start_date, end_date)
|
||||
]
|
||||
visible_entries = filtered_entries[-limit:]
|
||||
|
||||
compatibility_level = selected_levels[0] if len(selected_levels) == 1 else LOG_LEVEL_ALL
|
||||
return {
|
||||
"source_id": source.source_id,
|
||||
"name": source.name,
|
||||
"kind": source.kind,
|
||||
"location": source.location,
|
||||
"description": source.description,
|
||||
"category": source.category,
|
||||
"status": get_source_status(source),
|
||||
"level": compatibility_level,
|
||||
"selected_levels": list(selected_levels),
|
||||
"search_query": search_query,
|
||||
"available_levels": [
|
||||
LOG_LEVEL_ALL,
|
||||
LOG_LEVEL_ERROR,
|
||||
LOG_LEVEL_WARNING,
|
||||
LOG_LEVEL_INFO,
|
||||
LOG_LEVEL_DEBUG,
|
||||
],
|
||||
"daily_markers": build_daily_log_markers(marker_entries),
|
||||
"line_limit": limit,
|
||||
"line_count": len(visible_entries),
|
||||
"lines": [entry.display_line for entry in visible_entries],
|
||||
}
|
||||
Reference in New Issue
Block a user