dev #12
@@ -25,11 +25,17 @@ FALLBACK_GROUPS = (
|
||||
"starlink",
|
||||
"gps-ops",
|
||||
"galileo",
|
||||
"glonass",
|
||||
"glo-ops",
|
||||
"beidou",
|
||||
"leo",
|
||||
"geo",
|
||||
"iridium-next",
|
||||
"stations",
|
||||
"visual",
|
||||
"weather",
|
||||
"science",
|
||||
"cubesat",
|
||||
"amateur",
|
||||
"last-30-days",
|
||||
)
|
||||
FETCH_RETRY_ATTEMPTS = 3
|
||||
FETCH_RETRY_BASE_DELAY_SECONDS = 0.8
|
||||
@@ -220,27 +226,28 @@ class CelesTrakTLECollector(BaseCollector):
|
||||
try:
|
||||
for group in FALLBACK_GROUPS:
|
||||
group_url = self._group_url(group)
|
||||
try:
|
||||
body_path = await self._downloader.download_file(
|
||||
client,
|
||||
group_url,
|
||||
extension=".json",
|
||||
accept="application/json",
|
||||
validate_existing=self._validate_json_file,
|
||||
)
|
||||
except DownloadHTTPStatusError as exc:
|
||||
if not self._is_not_updated_response(exc):
|
||||
raise RuntimeError(f"CelesTrak fallback group '{group}' download failed: {exc}") from exc
|
||||
cached_path = self._downloader.get_cached_file(
|
||||
group_url,
|
||||
".json",
|
||||
validate_existing=self._validate_json_file,
|
||||
)
|
||||
if cached_path is None:
|
||||
cached_path = self._downloader.get_cached_file(
|
||||
group_url,
|
||||
".json",
|
||||
validate_existing=self._validate_json_file,
|
||||
)
|
||||
if cached_path is not None:
|
||||
body_path = cached_path
|
||||
else:
|
||||
try:
|
||||
body_path = await self._downloader.download_file(
|
||||
client,
|
||||
group_url,
|
||||
extension=".json",
|
||||
accept="application/json",
|
||||
validate_existing=self._validate_json_file,
|
||||
)
|
||||
except DownloadHTTPStatusError as exc:
|
||||
if not self._is_not_updated_response(exc):
|
||||
raise RuntimeError(f"CelesTrak fallback group '{group}' download failed: {exc}") from exc
|
||||
raise RuntimeError(
|
||||
f"CelesTrak fallback group '{group}' has not updated and no local cached copy is available"
|
||||
) from exc
|
||||
body_path = cached_path
|
||||
|
||||
group_records = await self._load_downloaded_payload(
|
||||
body_path,
|
||||
|
||||
@@ -426,6 +426,21 @@ def compact_log_context(context: dict | None) -> str:
|
||||
return json.dumps(allowed, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
def context_search_aliases(context: dict | None) -> str:
|
||||
if not context:
|
||||
return ""
|
||||
aliases: list[str] = []
|
||||
for key, value in sorted((context or {}).items()):
|
||||
if value is None or isinstance(value, (dict, list, tuple, set)):
|
||||
continue
|
||||
normalized_key = str(key).strip()
|
||||
normalized_value = str(value).strip()
|
||||
if not normalized_key or not normalized_value:
|
||||
continue
|
||||
aliases.append(f"{normalized_key}={normalized_value}")
|
||||
return " ".join(aliases)
|
||||
|
||||
|
||||
def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
||||
path = resolve_file_log_path(source)
|
||||
if not path.exists():
|
||||
@@ -519,6 +534,7 @@ def _database_event_from_system_record(record: SystemLog) -> LogEvent:
|
||||
line,
|
||||
f"id={record.id}",
|
||||
f"user_id={record.user_id}" if record.user_id else "",
|
||||
context_search_aliases(record.context),
|
||||
json.dumps(record.context or {}, ensure_ascii=False, sort_keys=True),
|
||||
]
|
||||
).lower()
|
||||
@@ -552,6 +568,7 @@ def _database_event_from_audit_record(record: AuditLog) -> LogEvent:
|
||||
f"id={record.id}",
|
||||
f"actor_id={record.actor_id}" if record.actor_id else "",
|
||||
record.actor_name or "",
|
||||
context_search_aliases(record.details),
|
||||
json.dumps(record.details or {}, ensure_ascii=False, sort_keys=True),
|
||||
]
|
||||
).lower()
|
||||
|
||||
@@ -2,8 +2,10 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.models.system_log import SystemLog
|
||||
from app.services import system_logs
|
||||
|
||||
|
||||
@@ -261,3 +263,22 @@ def test_read_log_snapshot_strips_nul_bytes_from_file_lines(tmp_path: Path, monk
|
||||
"ERROR: bind failed",
|
||||
"2026-04-23 23:41:32 INFO service=backend message=request served",
|
||||
]
|
||||
|
||||
|
||||
def test_database_system_log_search_matches_context_key_value_aliases():
|
||||
record = SystemLog(
|
||||
id=2218,
|
||||
occurred_at=datetime(2026, 5, 28, 9, 14, 50, tzinfo=UTC),
|
||||
source="backend",
|
||||
service="collector",
|
||||
module="app.services.collectors.base",
|
||||
event="collector.run.failed",
|
||||
level="error",
|
||||
message="Collector run failed",
|
||||
context={"collector_name": "celestrak_tle", "datasource_id": 20, "task_id": 26906},
|
||||
)
|
||||
|
||||
event = system_logs._database_event_from_system_record(record)
|
||||
|
||||
assert system_logs.event_matches_search(event, "task_id=26906")
|
||||
assert system_logs.event_matches_search(event, "datasource_id=20")
|
||||
|
||||
@@ -8,6 +8,21 @@ This project follows the repository versioning rule:
|
||||
- `improvement` -> `+0.0.1`(bugfix + 小功能混合)
|
||||
- `bugfix` -> `+0.0.1`
|
||||
|
||||
## [0.68.1] — 2026-05-28
|
||||
|
||||
Released: 2026-05-28
|
||||
|
||||
### Highlights
|
||||
- 修复 CelesTrak active 未更新窗口下清库后无法恢复的问题,fallback 会优先复用本地有效 group 缓存。
|
||||
- 修复数据源任务队列“查看日志”无法按 `task_id=...` 命中数据库结构化日志的问题。
|
||||
|
||||
### Added / Fixed / Improved
|
||||
- CelesTrak fallback group 列表改为公开可用分组,移除失效 group,并在没有 active 缓存时仍可从本地 group 缓存恢复采集。
|
||||
- 数据库日志搜索补充 JSON context 的 `key=value` 别名,支持 `task_id=26906`、`datasource_id=20` 这类控制台跳转查询。
|
||||
- 补充 CelesTrak 缓存边界、数据源任务日志跳转和运维恢复说明的中英文文档。
|
||||
|
||||
---
|
||||
|
||||
## [0.68.0] — 2026-05-28
|
||||
|
||||
Released: 2026-05-28
|
||||
|
||||
@@ -77,6 +77,8 @@ Manual trigger, data clearing, and cache clearing now enter the PostgreSQL data
|
||||
|
||||
Data deletion runs in batches so AIS-scale tables are not locked by one huge statement. A `clear_data` job clears `collected_data`, then source-specific AIS derived tables, and broadcasts `records_processed` as it goes; the console queue renders only user-facing text such as `Deleting data` and `Delete complete`, while internal table names remain in logs and raw task details. After AIS cleanup, the backend runs `ANALYZE ais_raw_observations` so datasource-list estimates converge quickly. The datasource directory uses PostgreSQL statistics for AIS record counts by default to avoid a cold-start `count(*)`; opening a single datasource detail row requests the exact count for that source.
|
||||
|
||||
The CelesTrak TLE collector prefers the complete `active` catalog. If CelesTrak returns the "GP data has not updated" HTTP 403, the backend first reuses the active raw download cache under `$PLANET_CACHE_DIR/downloads/celestrak`; if that cache is missing, it enters fallback group mode. Fallback group mode uses only currently valid public CelesTrak groups, including `starlink`, `gps-ops`, `galileo`, `glo-ops`, `beidou`, `geo`, `iridium-next`, `stations`, `visual`, `weather`, `science`, `cubesat`, `amateur`, and `last-30-days`. Disaster-recovery fallback prefers valid local group caches before touching the network, so small-group update windows or unreliable HEAD metadata do not incorrectly fail recovery. Console `Clear Data` and `Clear Cache` jobs only touch database rows, Earth layer cache, and dashboard cache; they do not remove this raw download cache.
|
||||
|
||||
## III. Collector List
|
||||
|
||||
| Collector | Data type | Content | Frequency |
|
||||
|
||||
@@ -117,6 +117,8 @@ Admin runtime errors are reported through [runtimeLogs.ts](/home/ray/dev/linkong
|
||||
|
||||
The Logs page follows log increments through the `/ws` `logs_tail` channel. File logs and database logs are both normalized into line events by the backend. When adding a new log source, wire it through the backend source registry and tail manager instead of adding a page-local poller.
|
||||
|
||||
The datasource task queue `View Logs` action opens `/logs?source=system-db&search=task_id=<id>`. Backend database-log search indexes must expand simple JSON context fields into `key=value` aliases such as `task_id=26906` and `datasource_id=20`, so historical task logs remain discoverable without rerunning the task.
|
||||
|
||||
## Current Shared Components
|
||||
|
||||
### 1. `Scrollbar`
|
||||
|
||||
@@ -172,6 +172,10 @@ After selecting a task, the page shows the effective prompt, whether it is custo
|
||||
|
||||
The legacy link `/settings?tab=ai` redirects to `/ai?tab=providers`.
|
||||
|
||||
## Datasources and Task Logs
|
||||
|
||||
`/datasources` is the datasource directory. Built-in sources can be filtered by product domain, level, enabled state, latest run state, collected-data state, and keyword. With no rows selected the main action triggers all matching sources; selecting rows changes it to `Trigger Selected N`. The queue button opens a grouped task panel for running, completed, failed, and skipped work. Failed rows can be retried, completed rows can jump back to their datasource detail, and each task can open `/logs` filtered by its task id.
|
||||
|
||||
## System Settings
|
||||
|
||||
`/settings` manages system-level configuration. Sub-tabs:
|
||||
|
||||
@@ -75,7 +75,7 @@ Cleanup order and boundaries:
|
||||
- Docker cleanup targets resources whose Compose project is `planet`, plus the explicit volumes `planet_postgres_data`, `planet_redis_data`, `postgres_data`, and `redis_data`; do not delete unlabeled volumes by a broad `planet_*` pattern, because another local project could own them.
|
||||
- Local build state removes `.venv`, frontend `node_modules` / `dist`, Planet state, and scattered Python / Vite cache directories. `$PLANET_CACHE_DIR/downloads` is preserved so upstream raw downloads such as CelesTrak can survive database resets and local rebuild cleanup.
|
||||
|
||||
After the reset, run `./planet.sh init` again to recreate tables and default seed data. Old collected records are not restored, and Earth OOBE is evaluated from the backend's real collection state on the next visit. When CelesTrak later returns its "GP data has not updated" HTTP 403, the backend first reuses the preserved download cache to repopulate the database; if no cache exists, wait for the next CelesTrak update window or use Space-Track as a fallback.
|
||||
After the reset, run `./planet.sh init` again to recreate tables and default seed data. Old collected records are not restored, and Earth OOBE is evaluated from the backend's real collection state on the next visit. When CelesTrak later returns its "GP data has not updated" HTTP 403, the backend first reuses the preserved download cache to repopulate the database; if no active cache exists, it tries valid CelesTrak fallback group caches; if no download cache exists at all, wait for the next CelesTrak update window or use Space-Track. Datasource `Clear Data` and `Clear Cache` actions in the console do not delete `$PLANET_CACHE_DIR/downloads/celestrak`.
|
||||
|
||||
## Health Check
|
||||
|
||||
|
||||
@@ -77,6 +77,8 @@ async def run(self, db):
|
||||
|
||||
删除数据任务按批次执行,避免 AIS 这类千万级表一次性锁表。`clear_data` 会先清 `collected_data`,再按来源清理 AIS 衍生表,并持续广播 `records_processed`;前端任务队列只展示“正在删除数据 / 删除完成”,内部表名只保留在日志和原始任务详情。删除结束后后端会 `ANALYZE ais_raw_observations`,让数据源列表的估算指标尽快收敛。数据源目录页默认使用 PostgreSQL 统计信息估算 AIS 大表记录数,避免冷启动做 `count(*)`;打开单条详情时再用精确计数校准当前数据源。
|
||||
|
||||
CelesTrak TLE 采集优先拉取完整 `active` 目录。如果 CelesTrak 返回“本轮 GP 数据未更新”的 403,后端先复用 `$PLANET_CACHE_DIR/downloads/celestrak` 下的 active 原始下载缓存;没有 active 缓存时进入 fallback group 模式。fallback group 只使用 CelesTrak 当前公开有效的分组,例如 `starlink`、`gps-ops`、`galileo`、`glo-ops`、`beidou`、`geo`、`iridium-next`、`stations`、`visual`、`weather`、`science`、`cubesat`、`amateur` 和 `last-30-days`。救灾 fallback 会优先使用本地有效 group 缓存,避免 CelesTrak 小分组在未更新窗口或 HEAD 元数据异常时被误判失败。控制台的“删除数据库”和“清理缓存”只处理数据库记录、Earth layer cache 和 dashboard cache,不删除该原始下载缓存。
|
||||
|
||||
## 三、采集器列表
|
||||
|
||||
| 采集器 | 数据类型 | 数据内容 | 采集频率 |
|
||||
|
||||
@@ -117,6 +117,8 @@ Admin 的状态标签统一走 [StatusText](/home/ray/dev/linkong/planet/fronten
|
||||
|
||||
日志页通过 `/ws` 的 `logs_tail` channel 跟随日志增量;文件日志和数据库日志都由后端统一转换成行事件。新增日志源时优先接入后端 source registry 和 tail manager,不要在日志页写独立轮询器。
|
||||
|
||||
数据源任务队列的“查看日志”入口跳转到 `/logs?source=system-db&search=task_id=<id>`。后端数据库日志搜索索引必须把 JSON context 中的简单字段同时展开为 `key=value` 别名,例如 `task_id=26906`、`datasource_id=20`,这样历史任务日志不依赖重新执行任务也能被精确查到。
|
||||
|
||||
## 当前共享组件
|
||||
|
||||
### 1. `Scrollbar`
|
||||
|
||||
@@ -236,7 +236,7 @@ Base URL 输入框尾端的插头图标会触发连接测试。测试通过会
|
||||
|
||||
## 数据探索
|
||||
|
||||
- `/datasources`:数据源目录。`内置源` 支持按产品域、层级、启用状态、最近执行状态、是否已有采集数据和关键词筛选;未勾选时主按钮显示“触发全部”,勾选多行后会变成“触发已选 N”,并只提交所选数据源。右上角队列按钮空态显示队列图标,有任务时显示纯圆环总进度;点击后打开队列浮层,按运行中、完成、失败和跳过分组,失败项可重试,完成项可跳到详情。`实时源` 面向 AISStream / WebSocket 长连接,展示连接健康、累计入库、时间窗统计和启动 / 停止 / 重连操作。接口、凭证、请求头的编辑统一在 `/collection-management` 的"采集器"。
|
||||
- `/datasources`:数据源目录。`内置源` 支持按产品域、层级、启用状态、最近执行状态、是否已有采集数据和关键词筛选;未勾选时主按钮显示“触发全部”,勾选多行后会变成“触发已选 N”,并只提交所选数据源。右上角队列按钮空态显示队列图标,有任务时显示纯圆环总进度;点击后打开队列浮层,按运行中、完成、失败和跳过分组,失败项可重试,完成项可跳到详情或按任务编号打开系统日志。`实时源` 面向 AISStream / WebSocket 长连接,展示连接健康、累计入库、时间窗统计和启动 / 停止 / 重连操作。接口、凭证、请求头的编辑统一在 `/collection-management` 的"采集器"。
|
||||
- `/data`:采集后数据表,适合排查"数据是否已经进入系统"、"更新时间是否符合预期"、"某个数据源是否产出有效记录"
|
||||
- `/bgp`:BGP 专题页面,列表 + 详情 + 研判,与智能星球的 BGP 图层互补
|
||||
- `/alerts/system`、`/alerts/bgp`、`/alerts/situational`:系统、BGP、态势告警
|
||||
|
||||
@@ -75,7 +75,7 @@
|
||||
- Docker 清理只针对 Compose project 为 `planet` 的资源,以及显式列出的 `planet_postgres_data`、`planet_redis_data`、`postgres_data`、`redis_data`;不要按 `planet_*` 模式删除没有 label 的 volume,避免误删同机其他项目。
|
||||
- 本地编译状态会删除 `.venv`、前端 `node_modules` / `dist`、Planet state,以及散落的 Python / Vite 缓存目录;`$PLANET_CACHE_DIR/downloads` 会保留,用于保存 CelesTrak 这类受上游下载窗口限制的原始文件缓存。
|
||||
|
||||
重置后重新执行 `./planet.sh init` 会重建表和默认数据,但不会恢复旧采集结果;首次进入 Earth 时 OOBE 会重新按后端真实采集状态判断。触发 CelesTrak 采集时,如果上游返回“本轮 GP 数据未更新”的 403,后端会优先用保留的下载缓存重新写入数据库;如果下载缓存也不存在,只能等待 CelesTrak 下一次更新窗口或使用 Space-Track 作为 fallback。
|
||||
重置后重新执行 `./planet.sh init` 会重建表和默认数据,但不会恢复旧采集结果;首次进入 Earth 时 OOBE 会重新按后端真实采集状态判断。触发 CelesTrak 采集时,如果上游返回“本轮 GP 数据未更新”的 403,后端会优先用保留的下载缓存重新写入数据库;如果 active 缓存不存在,会尝试有效 CelesTrak 分组缓存作为 fallback;如果下载缓存也不存在,只能等待 CelesTrak 下一次更新窗口或使用 Space-Track。控制台里的数据源“删除数据库”和“清理缓存”不会删除 `$PLANET_CACHE_DIR/downloads/celestrak`。
|
||||
|
||||
## 健康检查
|
||||
|
||||
|
||||
@@ -16,12 +16,13 @@
|
||||
## Current Version
|
||||
|
||||
- `main` 当前主线历史推导到:`0.16.5`
|
||||
- `dev` 当前开发分支历史推导到:`0.68.0`
|
||||
- `dev` 当前开发分支历史推导到:`0.68.1`
|
||||
|
||||
## Timeline
|
||||
|
||||
| Version | Type | Branch | Commit | Summary |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `0.68.1` | bugfix | `dev` | `pending` | 修复 CelesTrak fallback group/cache 恢复链路,并让数据源任务日志可按 task_id / datasource_id 搜索 |
|
||||
| `0.68.0` | feature | `dev` | `pending` | 新增数据源任务队列实时指标、AIS 大表分批删除和智能星球可插拔聚类策略,并让新设备启动前同步前端依赖 |
|
||||
| `0.67.0` | feature | `dev` | `pending` | 新增控制台日志实时跟随和运行时错误上报,重构智能星球 Interactable 聚合、wheel 缩放输入、国界壳半径和开发脚本锁文件保护 |
|
||||
| `0.66.3` | bugfix | `dev` | `pending` | 补上 Admin utility module 并放开前端源码 lib 例外,修复控制台动态导入 500 与 Mermaid 包解析失败 |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "planet-frontend",
|
||||
"version": "0.68.0",
|
||||
"version": "0.68.1",
|
||||
"private": true,
|
||||
"packageManager": "bun@1",
|
||||
"dependencies": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "planet"
|
||||
version = "0.68.0"
|
||||
version = "0.68.1"
|
||||
description = "智能星球计划 - 态势感知系统"
|
||||
requires-python = ">=3.14"
|
||||
dependencies = [
|
||||
|
||||
Reference in New Issue
Block a user