Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d15a9d488a | |||
|
|
eb4c4b7904 | ||
|
|
887fec972e | ||
|
|
5bf5c73ca0 | ||
| e65267fe21 | |||
| ae982e51cd | |||
|
|
65e6a96c0d | ||
|
|
37e92e7572 | ||
|
|
a37d4b6289 | ||
|
|
69789d7505 | ||
|
|
4f124121e7 | ||
|
|
085bdf9a80 | ||
|
|
fbca381512 | ||
|
|
5c65ee24d6 | ||
|
|
81970a1d05 | ||
|
|
9b913a3b83 | ||
|
|
93eb41a9f7 |
@@ -73,6 +73,7 @@ Style:
|
||||
- Use fenced code blocks with language tags.
|
||||
- Prefer tables for comparisons or parameter lists.
|
||||
- Keep snippets concise and relevant.
|
||||
- For UI labels, chart labels, feature names, datasource names, and other terms that may become mixed Chinese/English copy, check `docs/technical/{zh,en}/naming-glossary.md` and use the documented display name. If a confusing term is missing, update the glossary in both languages as part of the docs change.
|
||||
|
||||
### Step 4 — Verify
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ rg -n "class |def |function |export |router|@router|interface |type " <path>
|
||||
- Keep code snippets short and directly relevant.
|
||||
- List related files only when they help future maintainers navigate.
|
||||
- Use the repository’s existing language, heading style, and naming conventions.
|
||||
- For UI labels, chart labels, feature names, datasource names, and other terms that may become mixed Chinese/English copy, check `docs/technical/{zh,en}/naming-glossary.md` and use the documented display name. If a confusing term is missing, update the glossary in both languages as part of the docs change.
|
||||
|
||||
4. Verify:
|
||||
|
||||
|
||||
9
.gitignore
vendored
9
.gitignore
vendored
@@ -8,6 +8,7 @@
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
config/earth-boundary-sources.local.json
|
||||
*.pem
|
||||
*.key
|
||||
*.crt
|
||||
@@ -28,6 +29,8 @@ downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
!frontend/src/admin/lib/
|
||||
!frontend/src/admin/lib/**
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
@@ -150,3 +153,9 @@ temp/
|
||||
# Runtime Data
|
||||
# ----------------------
|
||||
data/ai/bgp-briefs/
|
||||
data/earth-boundary-sources/
|
||||
|
||||
# Generated Earth boundary tile artifacts. Keep source configs and builders in
|
||||
# Git; publish PMTiles/MVT artifacts through release/deploy storage instead of
|
||||
# committing thousands of generated tile files.
|
||||
frontend/public/earth/data/boundaries/
|
||||
|
||||
244
README.md
244
README.md
@@ -8,68 +8,54 @@
|
||||
|
||||
## 系统架构
|
||||
|
||||
当前仓库的核心形态是“Web Earth 可视化 + React 运维台 + FastAPI 数据与 AI 编排后端 + 独立模型适配层”。物理大屏与 UE 客户端仍是长期方向,但不再作为本地开发和当前发布的必需运行单元。
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ 物理大屏展示层 │
|
||||
│ ┌─────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ 偏振片3D大屏 (2m×3m, 4K, 120Hz, 眼镜式) │ │
|
||||
│ │ ┌─────────────────────────────────────────────────────────┐ │ │
|
||||
│ │ │ 虚幻引擎 UE5 客户端 │ │ │
|
||||
│ │ │ ├── 3D地球渲染 (Cesium for UE) │ │ │
|
||||
│ │ │ ├── 算力点可视化 (GPU集群、智算中心) │ │ │
|
||||
│ │ │ ├── 连接弧线 (光缆、路由、数据流向) │ │ │
|
||||
│ │ │ ├── 粒子效果 (数据流动、告警提示) │ │ │
|
||||
│ │ │ └── 自动巡航相机 + 交互控制 │ │ │
|
||||
│ │ └─────────────────────────────────────────────────────────┘ │ │
|
||||
│ └─────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
▲
|
||||
│ WebSocket (实时推送)
|
||||
│ 120Hz 心跳 / 数据帧同步
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ 数据中台服务层 (FastAPI) │
|
||||
│ ┌─────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ API Gateway (Redis 限流) │ │
|
||||
│ └─────────────────────────────────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌───────────────────┬──────────────────────────┬──────────────────┐ │
|
||||
│ │ 数据采集服务 │ 核心业务服务 │ 运维管理服务 │ │
|
||||
│ │ ┌─────────────┐ │ ┌─────────────────┐ │ ┌─────────────┐ │ │
|
||||
│ │ │ 调度中心 │ │ │ WebSocket 服务 │ │ │ 用户管理 │ │ │
|
||||
│ │ │ (Celery) │ │ │ (FastAPI) │ │ │ (JWT Auth) │ │ │
|
||||
│ │ └─────────────┘ │ └─────────────────┘ │ └─────────────┘ │ │
|
||||
│ │ ┌─────────────┐ │ ┌─────────────────┐ │ ┌─────────────┐ │ │
|
||||
│ │ │ 采集器池 │ │ │ 数据查询 API │ │ │ 数据源配置 │ │ │
|
||||
│ │ │ (10+源) │ │ │ (REST) │ │ │ 监控告警 │ │ │
|
||||
│ │ └─────────────┘ │ └─────────────────┘ │ └─────────────┘ │ │
|
||||
│ │ ┌─────────────┐ │ ┌─────────────────┐ │ ┌─────────────┐ │ │
|
||||
│ │ │ 消息队列 │ │ │ 态势分析引擎 │ │ │ 系统配置 │ │ │
|
||||
│ │ │ (Kafka) │ │ │ (计算/聚合) │ │ │ 日志审计 │ │ │
|
||||
│ │ └─────────────┘ │ └─────────────────┘ │ └─────────────┘ │ │
|
||||
│ └───────────────────┴──────────────────────────┴──────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
▲
|
||||
│ 内部 API 调用
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ Web管理端 (React Admin) │
|
||||
│ ┌─────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ 登录页 │ 仪表盘 │ 用户管理 │ 数据源配置 │ 任务监控 │ 系统配置 │ │
|
||||
│ └─────────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
▲
|
||||
│ PostgreSQL / Redis
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ 数据存储层 │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
|
||||
│ │ PostgreSQL │ │ TimescaleDB │ │ Redis │ │ MinIO │ │
|
||||
│ │ (用户/配置) │ │ (时序数据) │ │ (缓存/会话) │ │ (文件存储) │ │
|
||||
│ └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ 浏览器展示与运维层 │
|
||||
│ ┌──────────────────────────────┐ ┌──────────────────────────────┐ │
|
||||
│ │ Web Earth │ │ React 运维台 │ │
|
||||
│ │ frontend/public/earth │ │ frontend/src │ │
|
||||
│ │ Three.js 地球 / HUD / 新闻 │ │ 数据源 / 告警 / AI 设置 │ │
|
||||
│ │ 国界精度 / 品牌内容配置 │ │ 提示词配置 / 用户与系统配置 │ │
|
||||
│ └──────────────────────────────┘ └──────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
│ REST / WebSocket
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ FastAPI 业务与编排后端 │
|
||||
│ ┌────────────────────┐ ┌────────────────────┐ ┌─────────────────┐ │
|
||||
│ │ 数据 API 与认证 │ │ Earth 新闻增强 │ │ 告警与态势简报 │ │
|
||||
│ │ JWT / 权限 / 审计 │ │ 位置推断 / 本地化 │ │ BGP / 告警研判 │ │
|
||||
│ └────────────────────┘ └────────────────────┘ └─────────────────┘ │
|
||||
│ ┌────────────────────┐ ┌────────────────────┐ ┌─────────────────┐ │
|
||||
│ │ 系统运行配置 │ │ 默认提示词注册表 │ │ 未来 Agent Runtime│ │
|
||||
│ │ system_settings │ │ 代码发布 + DB 覆盖 │ │ 工具/证据/工作流 │ │
|
||||
│ └────────────────────┘ └────────────────────┘ └─────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
│ SQLAlchemy / Redis Stream │ 纯净 LLM 调用
|
||||
▼ ▼
|
||||
┌──────────────────────────────┐ ┌──────────────────────────────┐
|
||||
│ PostgreSQL / Redis │ │ aiprovider │
|
||||
│ 用户、配置、采集结果、新闻 │ │ provider + protocol adapter │
|
||||
│ Stream、缓存、运行状态 │ │ OpenAI / MiniMax / Ollama 等 │
|
||||
└──────────────────────────────┘ └──────────────────────────────┘
|
||||
▲
|
||||
│ 采集器 / 外部数据源
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ RSS 新闻、BGP 观测、公开数据源、后续 WebSearch/OCR/语音识别等工具 │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
架构边界:
|
||||
|
||||
- `backend` 负责业务语义、证据收集、提示词选择、AI 任务编排、权限和数据落库。
|
||||
- `aiprovider` 只负责把纯净模型请求适配到不同供应商或协议,不内置具体业务提示词。
|
||||
- 默认提示词随代码发布并保存在 `backend/app/ai_tasks/default_prompts.json`,运维台可在数据库中保存覆盖值,重置时回到当前代码版本的默认提示词。
|
||||
- Earth 新闻保留英文原文,中文展示结果存入 `localizations`,前端默认展示 `zh-CN` 的 `display_title`、`display_summary` 和中文地域/状态文案。
|
||||
- Earth LLM 指令、语音识别、多角色态势研判属于后续 Agent Runtime 方向,计划见 [docs/plans/agents-earth-command-runtime-plan.md](/home/ray/dev/linkong/planet/docs/plans/agents-earth-command-runtime-plan.md)。
|
||||
|
||||
## 四大核心要素
|
||||
|
||||
| 层级 | 要素 | 描述 |
|
||||
@@ -87,11 +73,10 @@
|
||||
|------|------|------|
|
||||
| FastAPI | 0.109+ | Web 框架 |
|
||||
| SQLAlchemy | 2.0+ | ORM |
|
||||
| Alembic | - | 数据库迁移 |
|
||||
| Celery | 5.3+ | 任务队列 |
|
||||
| Redis | 7.0+ | 缓存/消息 |
|
||||
| Kafka | 3.0+ | 事件流 |
|
||||
| uv | - | Python 依赖与命令运行 |
|
||||
| Redis | 7.0+ | 缓存、Stream 与运行协调 |
|
||||
| PyJWT | - | 认证 |
|
||||
| APScheduler / 后台任务 | - | 采集、增强与运行时任务 |
|
||||
|
||||
### 前端 (React Admin)
|
||||
|
||||
@@ -102,6 +87,7 @@
|
||||
| Axios | HTTP 客户端 |
|
||||
| Socket.io-client | WebSocket 客户端 |
|
||||
| ECharts | 统计图表 |
|
||||
| Three.js | Earth 3D 地球渲染 |
|
||||
| Bun | 前端包管理与脚本运行 |
|
||||
|
||||
前端工程统一使用 Bun:
|
||||
@@ -110,22 +96,16 @@
|
||||
- 运行脚本使用 `bun run <script>`
|
||||
- 不使用 `npm`、`pnpm`、`yarn`
|
||||
|
||||
### 虚幻引擎客户端
|
||||
### 大屏与 3D 展示方向
|
||||
|
||||
| 组件 | 版本 | 用途 |
|
||||
|------|------|------|
|
||||
| Unreal Engine 5 | 5.3+ | 3D 渲染引擎 |
|
||||
| Cesium for Unreal | 1.5+ | 地理可视化 |
|
||||
| Niagara | - | 粒子系统 |
|
||||
当前发布优先使用浏览器 Web Earth。UE5 / Cesium for Unreal / Niagara 可作为后续物理大屏方向接入,但不是本地开发闭环的必需组件。
|
||||
|
||||
### 数据库
|
||||
|
||||
| 组件 | 用途 |
|
||||
|------|------|
|
||||
| PostgreSQL 15+ | 关系数据 |
|
||||
| TimescaleDB | 时序数据扩展 |
|
||||
| Redis 7+ | 缓存/会话 |
|
||||
| MinIO | S3 兼容存储 |
|
||||
| Redis 7+ | 缓存、Stream、运行状态 |
|
||||
|
||||
### 部署
|
||||
|
||||
@@ -152,9 +132,9 @@
|
||||
| P0 | Epoch AI | 每小时 |
|
||||
| P0 | Hugging Face | 每 2 小时 |
|
||||
| P0 | GitHub | 每 4 小时 |
|
||||
| P0 每日 |
|
||||
| P0 | 海底光缆 / IXP / 卫星等基础设施数据 | 每日或按源刷新 |
|
||||
| P0 | PeeringDB | 每 2 小时 |
|
||||
| P1 | Cloudflare Radar | | TeleGeography | 每小时 |
|
||||
| P1 | Cloudflare Radar / TeleGeography | 每小时 |
|
||||
| P1 | CAIDA BGPStream | 每 15 分钟 |
|
||||
|
||||
## 项目结构
|
||||
@@ -166,20 +146,18 @@
|
||||
│ │ ├── core/ # 核心配置
|
||||
│ │ ├── models/ # 数据模型
|
||||
│ │ ├── schemas/ # Pydantic 模型
|
||||
│ │ ├── services/ # 业务逻辑
|
||||
│ │ └── tasks/ # Celery 任务
|
||||
│ │ ├── services/ # 业务逻辑与 AI 任务编排
|
||||
│ │ └── ai_tasks/ # 默认提示词与 AI 任务定义
|
||||
│ └── tests/
|
||||
├── aiprovider/ # 独立模型供应商适配层
|
||||
├── frontend/ # React 管理后台
|
||||
│ ├── src/
|
||||
│ │ ├── components/ # 组件
|
||||
│ │ ├── pages/ # 页面
|
||||
│ │ ├── services/ # API 服务
|
||||
│ │ └── store/ # 状态管理
|
||||
│ └── tests/
|
||||
├── unreal/ # UE5 大屏客户端
|
||||
│ ├── Content/
|
||||
│ ├── Source/
|
||||
│ └── Plugins/
|
||||
│ ├── public/earth/ # Web Earth 静态应用
|
||||
│ └── tests/ # 前端测试
|
||||
├── data/ # 数据文件
|
||||
├── docs/ # 文档
|
||||
├── scripts/ # 脚本
|
||||
@@ -191,10 +169,11 @@
|
||||
## 快速启动
|
||||
|
||||
```bash
|
||||
# 新机器首次初始化
|
||||
./scripts/bootstrap-dev.sh
|
||||
# 会自动安装/检查 uv、bun,并同步 Python/前端依赖
|
||||
# 新机器或空项目首次初始化
|
||||
./planet.sh init
|
||||
# 会自动安装/检查 uv、bun,同步 Python/前端依赖
|
||||
# 会在缺少时生成 backend/.env、aiprovider/.env、frontend/.env.local
|
||||
# 会启动 PostgreSQL/Redis,并创建表、默认数据源和本地默认用户
|
||||
|
||||
# 启动前后端服务
|
||||
./planet.sh start
|
||||
@@ -210,6 +189,9 @@
|
||||
|
||||
# 查看服务状态
|
||||
./planet.sh health
|
||||
|
||||
# 删除容器、卷、镜像和本地编译状态,执行前需要输入 Y 确认
|
||||
./planet.sh destroy
|
||||
```
|
||||
|
||||
前端命令约定:
|
||||
@@ -244,7 +226,7 @@ bun run build
|
||||
./planet.sh start --allow-lan
|
||||
```
|
||||
|
||||
这会让前端监听 `0.0.0.0:3000`,后端监听 `0.0.0.0:8000`。
|
||||
这会让前端监听 `0.0.0.0:3000`,后端监听 `0.0.0.0:8000`,AI Provider 通过 Docker 发布到 `0.0.0.0:8010`。启动前脚本会检查这三个端口;如果 WSL/Linux 侧无法释放端口,并检测到 Windows 侧 listener 或旧 `portproxy`,会请求管理员 PowerShell 清理。
|
||||
|
||||
### 2. 先确认 WSL 内部服务正常
|
||||
|
||||
@@ -253,14 +235,16 @@ bun run build
|
||||
```bash
|
||||
curl http://localhost:3000
|
||||
curl http://localhost:8000/health
|
||||
ss -ltnp | grep -E ':3000|:8000'
|
||||
curl http://localhost:8010/health
|
||||
ss -ltnp | grep -E ':3000|:8000|:8010'
|
||||
```
|
||||
|
||||
预期:
|
||||
|
||||
- `3000` 返回前端 HTML
|
||||
- `8000/health` 返回健康检查 JSON
|
||||
- `ss` 中能看到 `0.0.0.0:3000` 和 `0.0.0.0:8000`
|
||||
- `8010/health` 返回 AI Provider 健康检查 JSON
|
||||
- `ss` 中能看到 `0.0.0.0:3000`、`0.0.0.0:8000` 和 `0.0.0.0:8010`,或 Docker 已发布 `8010`
|
||||
|
||||
如果这一步不通,先不要继续做 Windows 转发。
|
||||
|
||||
@@ -271,26 +255,29 @@ ss -ltnp | grep -E ':3000|:8000'
|
||||
```powershell
|
||||
curl http://localhost:3000
|
||||
curl http://localhost:8000/health
|
||||
curl http://localhost:8010/health
|
||||
```
|
||||
|
||||
在常见的 WSL2 开发环境下,Windows 通常可以直接通过 `localhost` 访问 WSL 中的服务。
|
||||
|
||||
### 4. 如果需要让局域网设备访问,使用临时 Windows relay
|
||||
### 4. 如果需要让局域网设备访问,清理端口和防火墙
|
||||
|
||||
`./planet.sh start --allow-lan` 在 WSL 下会优先使用临时 Windows relay:WSL 和 Windows 本机 `localhost` 仍使用 `3000` / `8000`,局域网设备访问 Windows 局域网 IP 的 `3000` / `8000` 时由 relay 转发。relay 由脚本启动和停止;如果 WSL 目标端口断开,relay 会自动退出并释放 Windows 局域网监听端口。
|
||||
`./planet.sh start --allow-lan` 不再启动额外的 Windows 端口转发进程。它直接让开发服务对 `3000` / `8000` / `8010` 开放,并在启动前尝试释放这些端口。端口被 Windows 侧 listener 或旧 `portproxy` 占用时,脚本会请求一次管理员 PowerShell 清理。
|
||||
|
||||
如果以前手动配置过持久 `portproxy`,脚本会在启动前检测 `3000` / `8000` 残留规则,并请求管理员 PowerShell 自动删除。若自动请求被取消,可以手动清理,避免 `iphlpsvc` 继续占用 `3000` / `8000`:
|
||||
如果以前手动配置过持久 `portproxy`,若自动请求被取消,可以手动清理,避免 `iphlpsvc` 继续占用端口:
|
||||
|
||||
```powershell
|
||||
netsh interface portproxy delete v4tov4 listenaddress=0.0.0.0 listenport=3000
|
||||
netsh interface portproxy delete v4tov4 listenaddress=0.0.0.0 listenport=8000
|
||||
netsh interface portproxy delete v4tov4 listenaddress=0.0.0.0 listenport=8010
|
||||
```
|
||||
|
||||
脚本会检测 Windows 防火墙是否已放行 `3000` / `8000`。如果缺少规则,会触发一次 Windows UAC 管理员 PowerShell 请求来自动创建。若自动请求被取消,也可以手动执行:
|
||||
脚本会检测 Windows 防火墙是否已放行 `3000` / `8000` / `8010`。如果缺少规则,会触发一次 Windows UAC 管理员 PowerShell 请求来自动创建。若自动请求被取消,也可以手动执行:
|
||||
|
||||
```powershell
|
||||
New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000
|
||||
New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000
|
||||
New-NetFirewallRule -DisplayName "WSL Planet 8010" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8010
|
||||
```
|
||||
|
||||
### 5. 查 Windows 局域网 IP,并让其他设备访问
|
||||
@@ -307,6 +294,8 @@ ipconfig
|
||||
|
||||
- `http://<Windows局域网IP>:3000/earth`
|
||||
- `http://<Windows局域网IP>:3000/admin`
|
||||
- `http://<Windows局域网IP>:8000/health`
|
||||
- `http://<Windows局域网IP>:8010/health`
|
||||
|
||||
例如:
|
||||
|
||||
@@ -324,10 +313,12 @@ ipconfig
|
||||
|
||||
1. WSL 中执行 `curl http://localhost:3000`
|
||||
2. WSL 中执行 `curl http://localhost:8000/health`
|
||||
3. Windows 中执行 `curl http://localhost:3000`
|
||||
4. Windows 中执行 `curl http://localhost:8000/health`
|
||||
5. 按脚本提示完成 Windows 防火墙 UAC 请求;如果曾手动配置过 `portproxy`,先清理旧规则
|
||||
6. 用手机或其他电脑访问 Windows 对外端口,例如 `http://<Windows局域网IP>:3000/earth`
|
||||
3. WSL 中执行 `curl http://localhost:8010/health`
|
||||
4. Windows 中执行 `curl http://localhost:3000`
|
||||
5. Windows 中执行 `curl http://localhost:8000/health`
|
||||
6. Windows 中执行 `curl http://localhost:8010/health`
|
||||
7. 按脚本提示完成 Windows 防火墙或端口清理 UAC 请求
|
||||
8. 用手机或其他电脑访问 Windows 对外端口,例如 `http://<Windows局域网IP>:3000/earth`
|
||||
|
||||
## 启动容错参数
|
||||
|
||||
@@ -355,14 +346,23 @@ DATABASE_RETRY_INTERVAL=10 \
|
||||
- `FRONTEND_HEALTH_CHECK_ATTEMPTS` / `FRONTEND_HEALTH_CHECK_INTERVAL`: 控制前端 HTTP 可访问检查等待次数与间隔,默认 `10` 次、`2` 秒
|
||||
- `AI_PROVIDER_HEALTH_CHECK_ATTEMPTS` / `AI_PROVIDER_HEALTH_CHECK_INTERVAL`: 控制 `aiprovider` HTTP 健康检查等待次数与间隔,默认 `10` 次、`2` 秒
|
||||
|
||||
## AI 接口预留
|
||||
## AI 与智能体接口
|
||||
|
||||
项目现在采用“两层”设计:
|
||||
项目现在采用“三段式”边界:
|
||||
|
||||
- 主后端暴露稳定业务接口: `GET /api/v1/ai/provider/status`、`POST /api/v1/ai/situational-awareness/analyze`
|
||||
- 独立 `aiprovider` 服务负责适配具体模型供应商
|
||||
- `backend`: 暴露业务接口,负责选择任务提示词、组织证据、调用工具、保存 AI 设置和结果。
|
||||
- `aiprovider`: 暴露模型网关接口,只负责 provider / protocol 适配,不写入 BGP、新闻、告警等业务提示词。
|
||||
- 模型供应商: OpenAI 兼容、MiniMax、Anthropic、Ollama 或其他兼容网关。
|
||||
|
||||
这样前端和业务代码不直接依赖 OpenAI、本地模型网关或其他订阅服务,后续切换部署方式只需要调整环境变量。
|
||||
这样前端和业务代码不直接依赖某个模型供应商,后续增加 Agent Runtime、Earth 一键 LLM 指令、语音识别或多角色态势研判时,也可以把业务工作流放在后端,而不是污染模型适配层。
|
||||
|
||||
当前已落地的 AI 配置能力:
|
||||
|
||||
- 运维台 AI 设置可维护 provider、模型、协议、超时、token 等运行配置。
|
||||
- 运维台 AI 设置中的“提示词”页可选择不同功能入口,手动覆盖提示词,并一键重置到默认值。
|
||||
- 默认提示词随代码发布,位于 [backend/app/ai_tasks/default_prompts.json](/home/ray/dev/linkong/planet/backend/app/ai_tasks/default_prompts.json)。
|
||||
- 覆盖值保存在数据库运行配置中,升级代码后可继续保留现场配置,也可重置到新版本默认提示词。
|
||||
- 态势摘要、告警研判、新闻本地化等入口应使用各自任务提示词;调用 `aiprovider` 时只传递当前任务所需的 `prompt` / `system_prompt`。
|
||||
|
||||
主后端建议配置:
|
||||
|
||||
@@ -375,39 +375,33 @@ AI_PROVIDER_TIMEOUT_SECONDS=60
|
||||
`aiprovider` 服务建议配置:
|
||||
|
||||
```env
|
||||
AI_PROVIDER=openai_compatible
|
||||
AI_BASE_URL=https://api.openai.com/v1
|
||||
AI_PROVIDER=minimax
|
||||
AI_PROVIDER_API=anthropic-messages
|
||||
AI_BASE_URL=https://api.minimaxi.com/anthropic
|
||||
AI_API_KEY=your_api_key
|
||||
AI_MODEL=gpt-4o-mini
|
||||
AI_MODEL=MiniMax-M2.7
|
||||
AI_TIMEOUT_SECONDS=60
|
||||
AI_PROVIDER_SERVICE_TOKEN=change_me
|
||||
```
|
||||
|
||||
OpenAI 兼容场景推荐使用:
|
||||
推荐映射关系:
|
||||
|
||||
- `AI_PROVIDER=openai_compatible`
|
||||
- `vLLM` / `LM Studio` / `One API`: `AI_PROVIDER=openai` + `AI_PROVIDER_API=openai-completions`
|
||||
- `MiniMax`: `AI_PROVIDER=minimax` + `AI_PROVIDER_API=anthropic-messages`
|
||||
- Claude 兼容网关: `AI_PROVIDER=anthropic` + `AI_PROVIDER_API=anthropic-messages`
|
||||
- `Ollama`: `AI_PROVIDER=ollama` + `AI_PROVIDER_API=ollama-generate`
|
||||
|
||||
Claude 兼容场景推荐使用:
|
||||
|
||||
- `AI_PROVIDER=anthropic`
|
||||
- `AI_PROVIDER=anthropic_compatible`
|
||||
- `AI_PROVIDER=claude_compatible`
|
||||
|
||||
Ollama 原生场景推荐使用:
|
||||
|
||||
- `AI_PROVIDER=ollama`
|
||||
|
||||
比如 MiniMax 或其他 Claude 兼容网关,可以这样配置:
|
||||
比如 MiniMax 可以这样配置:
|
||||
|
||||
```env
|
||||
AI_PROVIDER=claude_compatible
|
||||
AI_BASE_URL=https://your-claude-compatible-endpoint.example.com
|
||||
AI_PROVIDER=minimax
|
||||
AI_PROVIDER_API=anthropic-messages
|
||||
AI_BASE_URL=https://api.minimaxi.com/anthropic
|
||||
AI_API_KEY=your_api_key
|
||||
AI_MODEL=your-claude-compatible-model
|
||||
AI_MODEL=MiniMax-M2.7
|
||||
AI_TIMEOUT_SECONDS=60
|
||||
AI_MAX_TOKENS=1200
|
||||
AI_ANTHROPIC_VERSION=2023-06-01
|
||||
AI_PROVIDER_SERVICE_TOKEN=change_me
|
||||
```
|
||||
|
||||
如果你要本地直接起模型适配层,项目里已经补了模板:
|
||||
@@ -415,12 +409,6 @@ AI_PROVIDER_SERVICE_TOKEN=change_me
|
||||
- [aiprovider/.env.example](/home/ray/dev/linkong/planet/aiprovider/.env.example)
|
||||
- [docker-compose.local-model.yml](/home/ray/dev/linkong/planet/docker-compose.local-model.yml)
|
||||
|
||||
推荐映射关系:
|
||||
|
||||
- `vLLM` / `LM Studio` / `One API`: `AI_PROVIDER=openai_compatible`
|
||||
- `MiniMax` / Claude 兼容网关: `AI_PROVIDER=claude_compatible`
|
||||
- `Ollama`: `AI_PROVIDER=ollama`
|
||||
|
||||
运行与调用补充:
|
||||
|
||||
- `./planet.sh start` 默认会启动 `aiprovider`
|
||||
@@ -430,11 +418,13 @@ AI_PROVIDER_SERVICE_TOKEN=change_me
|
||||
|
||||
详细文档:
|
||||
|
||||
- [docs/technical/agents-aiprovider.md](/home/ray/dev/linkong/planet/docs/technical/agents-aiprovider.md)
|
||||
- [docs/technical/zh/agents-aiprovider.md](/home/ray/dev/linkong/planet/docs/technical/zh/agents-aiprovider.md)
|
||||
- [docs/technical/en/agents-aiprovider.md](/home/ray/dev/linkong/planet/docs/technical/en/agents-aiprovider.md)
|
||||
- [aiprovider/README.md](/home/ray/dev/linkong/planet/aiprovider/README.md)
|
||||
- [docs/technical/frontend-layout-guidelines.md](/home/ray/dev/linkong/planet/docs/technical/frontend-layout-guidelines.md)
|
||||
- [docs/plans/frontend-ai-playground-development-plan.md](/home/ray/dev/linkong/planet/docs/plans/frontend-ai-playground-development-plan.md)
|
||||
- [docs/plans/agents-situational-awareness-foundation-plan.md](/home/ray/dev/linkong/planet/docs/plans/agents-situational-awareness-foundation-plan.md)
|
||||
- [docs/plans/agents-earth-command-runtime-plan.md](/home/ray/dev/linkong/planet/docs/plans/agents-earth-command-runtime-plan.md)
|
||||
|
||||
## 前端页面布局规范
|
||||
|
||||
|
||||
20
TODO.md
20
TODO.md
@@ -4,8 +4,13 @@ This file is the active backlog only. Completed history belongs in `docs/CHANGEL
|
||||
|
||||
## Earth
|
||||
|
||||
- [ ] High-precision country boundaries: implement the source-faithful static vector tile pipeline described in [Earth High Precision Boundary Tiles Plan](/home/ray/dev/linkong/planet/docs/plans/earth-high-precision-boundary-tiles-plan.md). Highest zoom must preserve trusted source geometry instead of smoothing or hand-drawing borders.
|
||||
- [ ] Earth boundary data: acquire or generate auditable China POV geometry for Zangnan, Aksai Chin, Taiwan/Penghu, Diaoyu Dao and affiliated islands, Chiwei Yu, South China Sea islands, and the official dashed maritime claim line before implementing visual changes.
|
||||
- [ ] Earth AI command entry: merge natural-language and speech-triggered LLM commands into the existing Earth search panel as described in [Agent Runtime, Earth LLM Command, And Speech Entry Plan](/home/ray/dev/linkong/planet/docs/plans/agents-earth-command-runtime-plan.md).
|
||||
- [ ] Earth action executor: implement safe visualization actions for layer toggles, batch highlights, filters, focus, result panels, and clear-highlight behavior.
|
||||
- [ ] Earth entity matching: support stable entity ids and batch matching for Beidou satellites, mainland China compute centers, BGP, news, vessels, and cables.
|
||||
- [ ] Replace debug GeoJSON boundary tiles with the real `earth-boundaries-china-pov-v1.pmtiles` production artifact after audited admin-0 / coastline / claim-line sources and the PMTiles toolchain are available.
|
||||
- [ ] Import authoritative China POV / coastline / claim-line source packages through the three standard Earth boundary source collectors, then rebuild a versioned PMTiles artifact so highest zoom `8-10` preserves trusted source geometry instead of seed data.
|
||||
- [ ] Earth boundary data: acquire or generate auditable China POV geometry for Zangnan, Aksai Chin, Taiwan/Penghu, Diaoyu Dao and affiliated islands, Chiwei Yu, South China Sea islands, Kosovo, Gaza, and the official dashed maritime claim line before implementing final visual changes.
|
||||
- [ ] Earth high-resolution basemap tiles: implement the viewport-loaded imagery layer described in [Earth High Resolution Basemap Tiles Plan](/home/ray/dev/linkong/planet/docs/plans/earth-high-resolution-basemap-tiles-plan.md), using high-precision coastline as the alignment reference instead of replacing the globe with one huge texture.
|
||||
- [ ] Presentation controller ownership: replace the singleton card fallback in [presentation-controller.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/presentation-controller.js) with a presentation/card token check before BGP/News migrate onto the shared controller, so connectors only attach to their owning card.
|
||||
- [ ] BGP frontend maintainability: split [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) by responsibility into data loading, marker rendering, overlays, and animation once the current interaction behavior is stable.
|
||||
- [ ] Optional BGP marker experiment: evaluate HTML markers for BGP incident/collector points if WebGL marker density or fixed screen-size clickability becomes a real blocker.
|
||||
@@ -27,6 +32,14 @@ This file is the active backlog only. Completed history belongs in `docs/CHANGEL
|
||||
|
||||
## AI Provider And Agents
|
||||
|
||||
- [ ] Unified integration config schema: implement the shared low-code schema engine for datasource, AI Provider, Web Search, and OCR configuration described in [Integration Config Schema System Plan](/home/ray/dev/linkong/planet/docs/plans/integration-config-schema-system-plan.md).
|
||||
- [ ] AI provider routing: finish the OpenClaw-style provider/model routing refactor described in [AI Provider OpenClaw-Style Routing Plan](/home/ray/dev/linkong/planet/docs/plans/ai-provider-openclaw-style-routing-plan.md), so model-specific transport rules live in provider metadata rather than runtime hardcoding.
|
||||
- [ ] AI provider catalog: replace the temporary `model_provider_apis` bridge with structured `models_metadata`, discovery descriptors, and incremental model sync with stale marking.
|
||||
- [ ] AI provider connectivity: keep the plug action as lightweight network/auth/model-directory validation only, and keep real generation tests inside Playground or explicit “trial run” actions.
|
||||
- [ ] Agent runtime foundation: add auditable agent runs, steps, evidence, proposals, and the Agent operations UI described in [Agent Runtime, Earth LLM Command, And Speech Entry Plan](/home/ray/dev/linkong/planet/docs/plans/agents-earth-command-runtime-plan.md).
|
||||
- [ ] Agent tool protocol: add backend JSON tool-call fallback, optional provider-native tool compatibility, tool whitelist validation, and policy-gated proposal application.
|
||||
- [ ] Speech/ASR integration for agents: add provider-neutral transcription settings and API, defaulting to Whisper-compatible API providers while keeping text commands usable when ASR is unavailable.
|
||||
- [ ] Earth voice wake: add device-local configurable wake-word preferences, microphone fallback states, and post-wake instruction upload for Earth commands.
|
||||
- [ ] AI provider compatibility center: move provider/model compatibility rules into a JSON/YAML config read by runtime, instead of continuing to scatter provider-specific branches through Python code.
|
||||
- [ ] Provider compatibility coverage: add explicit config for OpenAI, Anthropic, MiniMax, Ollama, Moonshot, DeepSeek, Qwen, GLM, Gemini, OpenRouter, vLLM, LM Studio, and One API.
|
||||
- [ ] Compatibility schema: cover adapter type, base URL pattern, auth header, thinking/reasoning defaults, stream path, tool-call capability, multimodal capability, and provider-specific request patches.
|
||||
@@ -45,6 +58,9 @@ Archived items stay here so old context is not lost. Completed items remain chec
|
||||
|
||||
### Completed
|
||||
|
||||
- [x] Implemented the high-precision country boundary tile framework from [Earth High Precision Boundary Tiles Plan](/home/ray/dev/linkong/planet/docs/plans/earth-high-precision-boundary-tiles-plan.md): static vector tile builder, versioned seed output, frontend bbox tile loader, debounce, in-flight dedupe, and LRU cache.
|
||||
- [x] Added the `pmtiles-mvt` frontend tile provider contract, MVT decoder dependencies, static PMTiles Nginx handling, collector artifact registration, production readiness check, and user operation docs for Earth boundaries.
|
||||
- [x] Split Earth boundary ingestion into standard source collectors (`earth_admin0_boundaries`, `earth_coastline`, `earth_claim_lines`) plus the downstream `earth_boundary_tiles` PMTiles builder.
|
||||
- [x] Refined BGP observer and anomaly `hover/click` feel.
|
||||
- [x] Added BGP anomaly relationship display with cables / regions.
|
||||
- [x] Added the Earth BGP activity layer so the map still feels alive when incident density is low.
|
||||
|
||||
@@ -2,10 +2,14 @@
|
||||
|
||||
ARG PYTHON_IMAGE=python:3.14-slim
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:latest
|
||||
ARG AI_PROVIDER_BUILD_FINGERPRINT=unknown
|
||||
|
||||
FROM ${UV_IMAGE} AS uv
|
||||
FROM ${PYTHON_IMAGE}
|
||||
|
||||
ARG AI_PROVIDER_BUILD_FINGERPRINT
|
||||
LABEL planet.aiprovider.build-fingerprint="${AI_PROVIDER_BUILD_FINGERPRINT}"
|
||||
|
||||
COPY --from=uv /uv /uvx /bin/
|
||||
|
||||
WORKDIR /app
|
||||
@@ -15,12 +19,15 @@ ENV PYTHONUNBUFFERED=1
|
||||
ENV UV_COMPILE_BYTECODE=1
|
||||
ENV UV_LINK_MODE=copy
|
||||
|
||||
RUN mkdir -p /root/.config/uv
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY pyproject.toml uv.lock /app/
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=secret,id=planet_uv_config,target=/root/.config/uv/uv.toml,required=false \
|
||||
uv sync --frozen --no-dev
|
||||
|
||||
COPY aiprovider /app/aiprovider
|
||||
|
||||
@@ -17,9 +17,6 @@ class Settings(BaseSettings):
|
||||
AI_HTTP_RETRY_ATTEMPTS: int = 2
|
||||
AI_MAX_TOKENS: int = 1200
|
||||
AI_ANTHROPIC_VERSION: str = "2023-06-01"
|
||||
AI_ANALYSIS_SYSTEM_PROMPT: str = (
|
||||
"你是态势感知分析助手。请基于输入的上下文、观测与约束,输出结构化、克制、可执行的分析。"
|
||||
)
|
||||
|
||||
AI_PROVIDER_SERVICE_TOKEN: str = ""
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ def get_provider_service(
|
||||
x_ai_model: str | None = Header(default=None),
|
||||
x_ai_max_tokens: str | None = Header(default=None),
|
||||
x_ai_anthropic_version: str | None = Header(default=None),
|
||||
x_ai_model_provider_apis: str | None = Header(default=None),
|
||||
) -> ProviderService:
|
||||
overrides = {
|
||||
"provider": x_ai_provider,
|
||||
@@ -53,6 +54,7 @@ def get_provider_service(
|
||||
"api_key": x_ai_api_key,
|
||||
"model": x_ai_model,
|
||||
"anthropic_version": x_ai_anthropic_version,
|
||||
"model_provider_apis": x_ai_model_provider_apis,
|
||||
}
|
||||
if x_ai_max_tokens:
|
||||
overrides["max_tokens"] = x_ai_max_tokens
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -14,7 +15,6 @@ from aiprovider.schemas import (
|
||||
SituationalAnalysisResponse,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_provider(value: str) -> str:
|
||||
return (value or "disabled").strip().lower()
|
||||
|
||||
@@ -62,7 +62,9 @@ class ProviderService:
|
||||
self.anthropic_version = str(
|
||||
overrides.get("anthropic_version") or settings.AI_ANTHROPIC_VERSION
|
||||
)
|
||||
self.system_prompt = settings.AI_ANALYSIS_SYSTEM_PROMPT
|
||||
self.model_provider_apis = self._parse_model_provider_apis(
|
||||
overrides.get("model_provider_apis")
|
||||
)
|
||||
|
||||
def get_status(self) -> AIProviderStatusResponse:
|
||||
enabled = self.provider != "disabled"
|
||||
@@ -94,16 +96,23 @@ class ProviderService:
|
||||
|
||||
prompt = self._build_prompt(payload)
|
||||
|
||||
if self.provider_api == "openai-completions":
|
||||
data = await self._request_openai_compatible(model, prompt)
|
||||
provider_api = self._resolve_model_provider_api(model)
|
||||
|
||||
if provider_api == "openai-completions":
|
||||
data = await self._request_openai_compatible(model, prompt, payload.system_prompt)
|
||||
content = self._extract_openai_content(data)
|
||||
content_blocks = self._extract_openai_blocks(data)
|
||||
elif self.provider_api == "anthropic-messages":
|
||||
data = await self._request_anthropic_messages(model, prompt, payload.thinking)
|
||||
elif provider_api == "anthropic-messages":
|
||||
data = await self._request_anthropic_messages(
|
||||
model,
|
||||
prompt,
|
||||
payload.thinking,
|
||||
payload.system_prompt,
|
||||
)
|
||||
content = self._extract_anthropic_content(data)
|
||||
content_blocks = self._extract_anthropic_blocks(data)
|
||||
elif self.provider_api == "ollama-generate":
|
||||
data = await self._request_ollama(model, prompt)
|
||||
elif provider_api == "ollama-generate":
|
||||
data = await self._request_ollama(model, prompt, payload.system_prompt)
|
||||
content = self._extract_ollama_content(data)
|
||||
content_blocks = self._extract_ollama_blocks(data)
|
||||
else:
|
||||
@@ -128,6 +137,26 @@ class ProviderService:
|
||||
def _requires_api_key(self) -> bool:
|
||||
return self.provider_api != "ollama-generate"
|
||||
|
||||
def _resolve_model_provider_api(self, model: str) -> str:
|
||||
return self.model_provider_apis.get(model) or self.provider_api
|
||||
|
||||
def _parse_model_provider_apis(self, value: Any) -> dict[str, str]:
|
||||
if isinstance(value, dict):
|
||||
raw = value
|
||||
elif isinstance(value, str) and value.strip():
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
raw = parsed if isinstance(parsed, dict) else {}
|
||||
else:
|
||||
raw = {}
|
||||
return {
|
||||
str(model): _normalize_provider_api(str(provider_api))
|
||||
for model, provider_api in raw.items()
|
||||
if model and provider_api
|
||||
}
|
||||
|
||||
def _build_prompt(self, payload: SituationalAnalysisRequest) -> str:
|
||||
sections = [
|
||||
f"任务标题:\n{payload.title}",
|
||||
@@ -139,19 +168,28 @@ class ProviderService:
|
||||
sections.append("约束条件:\n" + "\n".join(f"- {item}" for item in payload.constraints))
|
||||
if payload.context:
|
||||
sections.append(f"附加上下文:\n{payload.context}")
|
||||
sections.append(
|
||||
"请输出: 1) 态势摘要 2) 关键风险 3) 研判依据 4) 建议动作 5) 还缺少的数据。"
|
||||
)
|
||||
return "\n\n".join(sections)
|
||||
|
||||
async def _request_openai_compatible(self, model: str, prompt: str) -> dict[str, Any]:
|
||||
def _resolve_system_prompt(self, system_prompt: str | None) -> str | None:
|
||||
resolved = str(system_prompt or "").strip()
|
||||
return resolved or None
|
||||
|
||||
async def _request_openai_compatible(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str,
|
||||
system_prompt: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
messages = []
|
||||
resolved_system_prompt = self._resolve_system_prompt(system_prompt)
|
||||
if resolved_system_prompt:
|
||||
messages.append({"role": "system", "content": resolved_system_prompt})
|
||||
messages.append({"role": "user", "content": prompt})
|
||||
request_body = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": self.system_prompt},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"messages": messages,
|
||||
"temperature": 0.2,
|
||||
"max_tokens": self.max_tokens,
|
||||
}
|
||||
return await self._post(
|
||||
path="/chat/completions",
|
||||
@@ -167,10 +205,10 @@ class ProviderService:
|
||||
model: str,
|
||||
prompt: str,
|
||||
thinking: dict[str, Any] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
request_body = {
|
||||
"model": model,
|
||||
"system": self.system_prompt,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
@@ -185,6 +223,9 @@ class ProviderService:
|
||||
"max_tokens": self.max_tokens,
|
||||
"temperature": 0.2,
|
||||
}
|
||||
resolved_system_prompt = self._resolve_system_prompt(system_prompt)
|
||||
if resolved_system_prompt:
|
||||
request_body["system"] = resolved_system_prompt
|
||||
resolved_thinking = self._resolve_anthropic_thinking(thinking)
|
||||
if resolved_thinking:
|
||||
request_body["thinking"] = resolved_thinking
|
||||
@@ -218,19 +259,28 @@ class ProviderService:
|
||||
model: str,
|
||||
prompt: str,
|
||||
thinking: dict[str, Any] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return await self._request_anthropic_messages(model, prompt, thinking)
|
||||
return await self._request_anthropic_messages(model, prompt, thinking, system_prompt)
|
||||
|
||||
async def _request_ollama(self, model: str, prompt: str) -> dict[str, Any]:
|
||||
async def _request_ollama(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str,
|
||||
system_prompt: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
request_body = {
|
||||
"model": model,
|
||||
"stream": False,
|
||||
"system": self.system_prompt,
|
||||
"prompt": prompt,
|
||||
"options": {
|
||||
"temperature": 0.2,
|
||||
"num_predict": self.max_tokens,
|
||||
},
|
||||
}
|
||||
resolved_system_prompt = self._resolve_system_prompt(system_prompt)
|
||||
if resolved_system_prompt:
|
||||
request_body["system"] = resolved_system_prompt
|
||||
return await self._post(
|
||||
path="/api/generate",
|
||||
headers={
|
||||
@@ -289,13 +339,19 @@ class ProviderService:
|
||||
message = choices[0].get("message") or {}
|
||||
content = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if content:
|
||||
return content
|
||||
reasoning_content = message.get("reasoning_content")
|
||||
return reasoning_content if isinstance(reasoning_content, str) else ""
|
||||
if isinstance(content, list):
|
||||
return "".join(
|
||||
item.get("text", "")
|
||||
for item in content
|
||||
if isinstance(item, dict)
|
||||
)
|
||||
reasoning_content = message.get("reasoning_content")
|
||||
if isinstance(reasoning_content, str):
|
||||
return reasoning_content
|
||||
return ""
|
||||
|
||||
def _extract_openai_blocks(self, payload: dict[str, Any]) -> list[AIContentBlock]:
|
||||
@@ -306,9 +362,14 @@ class ProviderService:
|
||||
message = choices[0].get("message") or {}
|
||||
content = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return [AIContentBlock(type="text", text=content)]
|
||||
blocks = [AIContentBlock(type="text", text=content)] if content else []
|
||||
reasoning_content = message.get("reasoning_content")
|
||||
if isinstance(reasoning_content, str) and reasoning_content:
|
||||
blocks.append(AIContentBlock(type="thinking", thinking=reasoning_content))
|
||||
return blocks
|
||||
if not isinstance(content, list):
|
||||
return []
|
||||
reasoning_content = message.get("reasoning_content")
|
||||
return [AIContentBlock(type="thinking", thinking=reasoning_content)] if isinstance(reasoning_content, str) and reasoning_content else []
|
||||
|
||||
blocks: list[AIContentBlock] = []
|
||||
for item in content:
|
||||
@@ -321,7 +382,11 @@ class ProviderService:
|
||||
metadata={k: v for k, v in item.items() if k not in {"type", "text"}},
|
||||
)
|
||||
)
|
||||
reasoning_content = message.get("reasoning_content")
|
||||
if isinstance(reasoning_content, str) and reasoning_content:
|
||||
blocks.append(AIContentBlock(type="thinking", thinking=reasoning_content))
|
||||
return blocks
|
||||
|
||||
def _extract_anthropic_content(self, payload: dict[str, Any]) -> str:
|
||||
content = payload.get("content")
|
||||
if isinstance(content, str):
|
||||
|
||||
@@ -13,10 +13,11 @@ class AIContentBlock(BaseModel):
|
||||
|
||||
class SituationalAnalysisRequest(BaseModel):
|
||||
title: str = Field(..., min_length=1, max_length=200)
|
||||
objective: str = Field(..., min_length=1, max_length=1000)
|
||||
objective: str = Field(..., min_length=1, max_length=20000)
|
||||
context: dict[str, Any] = Field(default_factory=dict)
|
||||
observations: list[str] = Field(default_factory=list)
|
||||
constraints: list[str] = Field(default_factory=list)
|
||||
system_prompt: str | None = Field(default=None, max_length=8000)
|
||||
preferred_model: str | None = Field(default=None, max_length=200)
|
||||
thinking: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
ARG PYTHON_IMAGE=python:3.14-slim
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:latest
|
||||
|
||||
@@ -14,12 +16,16 @@ ENV UV_COMPILE_BYTECODE=1
|
||||
ENV UV_LINK_MODE=copy
|
||||
ENV PYTHONPATH=/app/backend
|
||||
|
||||
RUN mkdir -p /root/.config/uv
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY pyproject.toml uv.lock /app/
|
||||
RUN uv sync --frozen --no-dev
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=secret,id=planet_uv_config,target=/root/.config/uv/uv.toml,required=false \
|
||||
uv sync --frozen --no-dev
|
||||
|
||||
COPY backend /app/backend
|
||||
COPY VERSION /app/VERSION
|
||||
|
||||
2
backend/app/ai_tasks/__init__.py
Normal file
2
backend/app/ai_tasks/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""AI task prompt registry and runtime helpers."""
|
||||
|
||||
74
backend/app/ai_tasks/default_prompts.json
Normal file
74
backend/app/ai_tasks/default_prompts.json
Normal file
@@ -0,0 +1,74 @@
|
||||
[
|
||||
{
|
||||
"key": "earth.news.enrich",
|
||||
"label": "Earth 新闻汉化与定位",
|
||||
"group": "Earth 新闻",
|
||||
"version": "2026-05-16.2",
|
||||
"system_prompt": "",
|
||||
"prompt": "Return exactly one strict JSON object with a location object and a localizations object. Infer the most likely physical event location and produce a faithful Simplified Chinese title plus a one-sentence newswire-style Chinese summary based only on the supplied RSS headline, description, source, and date. The summary should read like a concise breaking-news lead, not a label, slogan, or keyword headline."
|
||||
},
|
||||
{
|
||||
"key": "alerts.brief",
|
||||
"label": "系统告警研判",
|
||||
"group": "告警研判",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "你是告警研判助手。请基于输入的告警事实、上下文与约束,输出结构化、克制、可执行的值班研判;明确区分事实、推断与建议,不要夸大证据不足的风险。",
|
||||
"prompt": "基于当前告警总量、严重度、状态、数据源分布与最近告警摘录,生成一份面向值班人员的简明告警态势简报,突出待处理风险、告警集中点和优先动作。"
|
||||
},
|
||||
{
|
||||
"key": "alerts.situational.brief",
|
||||
"label": "跨模块态势告警研判",
|
||||
"group": "告警研判",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "你是告警研判助手。请基于输入的告警事实、上下文与约束,输出结构化、克制、可执行的值班研判;明确区分事实、推断与建议,不要夸大证据不足的风险。",
|
||||
"prompt": "综合系统告警、BGP incidents、BGP anomalies 与近期 BGP AI 简报,生成一份面向值班人员的态势告警简报,指出当前最需要关注的风险域、跨模块联动迹象和优先动作。"
|
||||
},
|
||||
{
|
||||
"key": "bgp.brief",
|
||||
"label": "BGP 态势简报",
|
||||
"group": "BGP",
|
||||
"version": "2026-05-16.2",
|
||||
"system_prompt": "你是 BGP 值班分析师。请直接输出面向值班人员的中文 Markdown 简报,只写最终研判内容;不要复述用户需求、提示词、写作计划、字段清单或“我将如何回答”。",
|
||||
"prompt": "基于当前 BGP incidents、anomalies、原始观测事件、观测站覆盖与 prefix geography 证据,生成一份面向操作员的简明态势简报,突出区域热点、观测偏差、当前风险、证据和优先动作。"
|
||||
},
|
||||
{
|
||||
"key": "location.factcheck.normalize",
|
||||
"label": "位置事实核查结构化",
|
||||
"group": "位置解析",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "Convert the supplied location factcheck text into exactly one strict JSON object. Extract only facts present in the text or original query."
|
||||
},
|
||||
{
|
||||
"key": "location.factcheck.resolve",
|
||||
"label": "位置事实核查兜底",
|
||||
"group": "位置解析",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "Return exactly one JSON object for the most likely physical location. Use only fact-checkable public knowledge; return null fields rather than guessing when evidence is weak."
|
||||
},
|
||||
{
|
||||
"key": "datasource.mapping",
|
||||
"label": "数据源映射生成",
|
||||
"group": "采集配置",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "Return only JSON for a deterministic mapping DSL. The JSON must contain source.items_path and fields. Do not include prose or code."
|
||||
},
|
||||
{
|
||||
"key": "credential.guide",
|
||||
"label": "采集器凭据教程",
|
||||
"group": "采集配置",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "生成一份中文采集器凭据配置教程。只能根据 context.search_evidence 中的来源生成教程;如果证据不足,明确说明需要以官方页面为准。"
|
||||
},
|
||||
{
|
||||
"key": "ai.connection_test",
|
||||
"label": "AI Provider 连接测试",
|
||||
"group": "运维测试",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "Reply OK."
|
||||
}
|
||||
]
|
||||
182
backend/app/ai_tasks/prompts.py
Normal file
182
backend/app/ai_tasks/prompts.py
Normal file
@@ -0,0 +1,182 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.system_setting import SystemSetting
|
||||
|
||||
AI_PROMPTS_CATEGORY = "ai_prompts"
|
||||
DEFAULT_PROMPTS_PATH = Path(__file__).with_name("default_prompts.json")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AIPromptDefinition:
|
||||
key: str
|
||||
label: str
|
||||
group: str
|
||||
version: str
|
||||
system_prompt: str
|
||||
prompt: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EffectiveAIPrompt:
|
||||
key: str
|
||||
label: str
|
||||
group: str
|
||||
version: str
|
||||
default_system_prompt: str
|
||||
default_prompt: str
|
||||
system_prompt: str
|
||||
prompt: str
|
||||
is_custom: bool
|
||||
updated_at: str | None = None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def list_prompt_definitions() -> tuple[AIPromptDefinition, ...]:
|
||||
raw_items = json.loads(DEFAULT_PROMPTS_PATH.read_text(encoding="utf-8"))
|
||||
return tuple(
|
||||
AIPromptDefinition(
|
||||
key=str(item["key"]),
|
||||
label=str(item["label"]),
|
||||
group=str(item["group"]),
|
||||
version=str(item["version"]),
|
||||
system_prompt=str(item.get("system_prompt") or ""),
|
||||
prompt=str(item.get("prompt") or ""),
|
||||
)
|
||||
for item in raw_items
|
||||
)
|
||||
|
||||
|
||||
def get_prompt_definition(task_key: str) -> AIPromptDefinition:
|
||||
for definition in list_prompt_definitions():
|
||||
if definition.key == task_key:
|
||||
return definition
|
||||
raise KeyError(task_key)
|
||||
|
||||
|
||||
async def _get_prompt_setting(db: AsyncSession) -> SystemSetting | None:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == AI_PROMPTS_CATEGORY)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _normalize_overrides(payload: dict[str, Any] | None) -> dict[str, dict[str, Any]]:
|
||||
raw = (payload or {}).get("overrides")
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
return {
|
||||
str(key): dict(value)
|
||||
for key, value in raw.items()
|
||||
if isinstance(value, dict)
|
||||
}
|
||||
|
||||
|
||||
async def get_prompt_overrides(db: AsyncSession) -> dict[str, dict[str, Any]]:
|
||||
if not hasattr(db, "execute"):
|
||||
return {}
|
||||
setting = await _get_prompt_setting(db)
|
||||
return _normalize_overrides(setting.payload if setting else None)
|
||||
|
||||
|
||||
def _effective_prompt(
|
||||
definition: AIPromptDefinition,
|
||||
override: dict[str, Any] | None,
|
||||
) -> EffectiveAIPrompt:
|
||||
override = override or {}
|
||||
custom_system = override.get("system_prompt")
|
||||
custom_prompt = override.get("prompt")
|
||||
has_custom_system = isinstance(custom_system, str)
|
||||
has_custom_prompt = isinstance(custom_prompt, str)
|
||||
return EffectiveAIPrompt(
|
||||
key=definition.key,
|
||||
label=definition.label,
|
||||
group=definition.group,
|
||||
version=definition.version,
|
||||
default_system_prompt=definition.system_prompt,
|
||||
default_prompt=definition.prompt,
|
||||
system_prompt=custom_system if has_custom_system else definition.system_prompt,
|
||||
prompt=custom_prompt if has_custom_prompt else definition.prompt,
|
||||
is_custom=has_custom_system or has_custom_prompt,
|
||||
updated_at=str(override.get("updated_at") or "") or None,
|
||||
)
|
||||
|
||||
|
||||
async def list_effective_prompts(db: AsyncSession) -> list[EffectiveAIPrompt]:
|
||||
overrides = await get_prompt_overrides(db)
|
||||
return [
|
||||
_effective_prompt(definition, overrides.get(definition.key))
|
||||
for definition in list_prompt_definitions()
|
||||
]
|
||||
|
||||
|
||||
async def get_effective_prompt(db: AsyncSession | None, task_key: str) -> EffectiveAIPrompt:
|
||||
definition = get_prompt_definition(task_key)
|
||||
if db is None:
|
||||
return _effective_prompt(definition, None)
|
||||
overrides = await get_prompt_overrides(db)
|
||||
return _effective_prompt(definition, overrides.get(task_key))
|
||||
|
||||
|
||||
async def save_prompt_override(
|
||||
db: AsyncSession,
|
||||
task_key: str,
|
||||
*,
|
||||
system_prompt: str,
|
||||
prompt: str,
|
||||
) -> EffectiveAIPrompt:
|
||||
definition = get_prompt_definition(task_key)
|
||||
setting = await _get_prompt_setting(db)
|
||||
payload = dict(setting.payload or {}) if setting else {}
|
||||
overrides = _normalize_overrides(payload)
|
||||
overrides[definition.key] = {
|
||||
"system_prompt": system_prompt,
|
||||
"prompt": prompt,
|
||||
"updated_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
payload["overrides"] = overrides
|
||||
if setting is None:
|
||||
setting = SystemSetting(category=AI_PROMPTS_CATEGORY, payload=payload)
|
||||
db.add(setting)
|
||||
else:
|
||||
setting.payload = payload
|
||||
await db.commit()
|
||||
return _effective_prompt(definition, overrides[definition.key])
|
||||
|
||||
|
||||
async def reset_prompt_override(db: AsyncSession, task_key: str) -> EffectiveAIPrompt:
|
||||
definition = get_prompt_definition(task_key)
|
||||
setting = await _get_prompt_setting(db)
|
||||
if setting is None:
|
||||
return _effective_prompt(definition, None)
|
||||
payload = dict(setting.payload or {})
|
||||
overrides = _normalize_overrides(payload)
|
||||
overrides.pop(definition.key, None)
|
||||
payload["overrides"] = overrides
|
||||
setting.payload = payload
|
||||
await db.commit()
|
||||
return _effective_prompt(definition, None)
|
||||
|
||||
|
||||
def serialize_effective_prompt(prompt: EffectiveAIPrompt) -> dict[str, Any]:
|
||||
return {
|
||||
"key": prompt.key,
|
||||
"label": prompt.label,
|
||||
"group": prompt.group,
|
||||
"version": prompt.version,
|
||||
"default_system_prompt": prompt.default_system_prompt,
|
||||
"default_prompt": prompt.default_prompt,
|
||||
"system_prompt": prompt.system_prompt,
|
||||
"prompt": prompt.prompt,
|
||||
"is_custom": prompt.is_custom,
|
||||
"updated_at": prompt.updated_at,
|
||||
}
|
||||
@@ -6,6 +6,7 @@ from app.api.v1 import (
|
||||
datasource_config,
|
||||
datasources,
|
||||
docs,
|
||||
earth,
|
||||
tasks,
|
||||
dashboard,
|
||||
alerts,
|
||||
@@ -18,6 +19,7 @@ from app.api.v1 import (
|
||||
vessels,
|
||||
bgp,
|
||||
news,
|
||||
interactables,
|
||||
realtime_sources,
|
||||
system_control,
|
||||
tv,
|
||||
@@ -34,6 +36,7 @@ api_router.include_router(
|
||||
api_router.include_router(datasources.router, prefix="/datasources", tags=["datasources"])
|
||||
api_router.include_router(collected_data.router, prefix="/collected", tags=["collected-data"])
|
||||
api_router.include_router(docs.router, prefix="/docs", tags=["docs"])
|
||||
api_router.include_router(earth.router, prefix="/earth", tags=["earth"])
|
||||
api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
|
||||
api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboard"])
|
||||
api_router.include_router(alerts.router, prefix="/alerts", tags=["alerts"])
|
||||
@@ -51,4 +54,5 @@ api_router.include_router(vessels.router, prefix="/vessels", tags=["vessels"])
|
||||
api_router.include_router(bgp.router, prefix="/bgp", tags=["bgp"])
|
||||
api_router.include_router(tv.router, prefix="/tv", tags=["tv"])
|
||||
api_router.include_router(news.router, prefix="/news", tags=["news"])
|
||||
api_router.include_router(interactables.router, prefix="/interactables", tags=["interactables"])
|
||||
api_router.include_router(realtime_sources.router, prefix="/realtime-sources", tags=["realtime-sources"])
|
||||
|
||||
@@ -3,6 +3,7 @@ from uuid import uuid4
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.security import get_current_user
|
||||
from app.db.session import get_db
|
||||
from app.models.user import User
|
||||
@@ -47,8 +48,10 @@ from app.services.playground_chat_service import (
|
||||
stop_message,
|
||||
)
|
||||
from app.services.situational_alert_ai_brief import build_situational_alert_brief_request
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
|
||||
router = APIRouter()
|
||||
logger = get_logger(__name__, service="api")
|
||||
|
||||
|
||||
@router.get("/provider/status", response_model=AIProviderStatusResponse)
|
||||
@@ -122,6 +125,16 @@ async def create_playground_message(
|
||||
provider_client: AIProviderClient = Depends(get_ai_provider_client),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.message.create",
|
||||
message="Playground message creation requested",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"session_key": payload.session_key, "preset": payload.selected_preset_key},
|
||||
)
|
||||
return await create_turn(
|
||||
db,
|
||||
user_id=current_user.id,
|
||||
@@ -136,6 +149,16 @@ async def stop_playground_message(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.message.stop",
|
||||
message="Playground message stop requested",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"session_key": payload.session_key, "message_id": payload.message_id},
|
||||
)
|
||||
return await stop_message(
|
||||
db,
|
||||
user_id=current_user.id,
|
||||
@@ -150,6 +173,16 @@ async def resend_playground_message(
|
||||
provider_client: AIProviderClient = Depends(get_ai_provider_client),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.message.resend",
|
||||
message="Playground message resend requested",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"session_key": payload.session_key, "user_message_id": payload.user_message_id},
|
||||
)
|
||||
return await resend_turn(
|
||||
db,
|
||||
user_id=current_user.id,
|
||||
@@ -214,16 +247,70 @@ async def analyze_bgp_brief(
|
||||
anomaly_limit=payload.anomaly_limit,
|
||||
collector_limit=payload.collector_limit,
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.bgp.facts_collected",
|
||||
message="BGP brief facts collected",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={
|
||||
"incident_limit": payload.incident_limit,
|
||||
"anomaly_limit": payload.anomaly_limit,
|
||||
"collector_limit": payload.collector_limit,
|
||||
"fact_count": len(facts or []),
|
||||
},
|
||||
)
|
||||
brief_request.preferred_model = payload.preferred_model
|
||||
brief_request.thinking = payload.thinking
|
||||
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
return save_bgp_brief_record(
|
||||
analysis,
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.bgp.start",
|
||||
message="BGP brief AI analysis started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
facts=facts,
|
||||
context=context,
|
||||
user_id=current_user.id,
|
||||
context={"preferred_model": payload.preferred_model},
|
||||
)
|
||||
try:
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
record = save_bgp_brief_record(
|
||||
analysis,
|
||||
request_id=request_id,
|
||||
facts=facts,
|
||||
context=context,
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.bgp.completed",
|
||||
message="BGP brief AI analysis saved",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"provider": analysis.provider, "model": analysis.model, "brief_id": record.id},
|
||||
)
|
||||
return record
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.bgp.failed",
|
||||
message="BGP brief AI analysis failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"preferred_model": payload.preferred_model}),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@router.post("/alerts/brief", response_model=AlertBriefResponse)
|
||||
@@ -242,17 +329,65 @@ async def analyze_alert_brief(
|
||||
db,
|
||||
alert_limit=payload.alert_limit,
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.alerts.facts_collected",
|
||||
message="Alert brief facts collected",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"alert_limit": payload.alert_limit, "fact_count": len(facts or [])},
|
||||
)
|
||||
brief_request.preferred_model = payload.preferred_model
|
||||
brief_request.thinking = payload.thinking
|
||||
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
return AlertBriefResponse(
|
||||
**analysis.model_dump(),
|
||||
title=brief_request.title,
|
||||
objective=brief_request.objective,
|
||||
facts=facts,
|
||||
context=context,
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.alerts.start",
|
||||
message="Alert brief AI analysis started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"preferred_model": payload.preferred_model},
|
||||
)
|
||||
try:
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.alerts.completed",
|
||||
message="Alert brief AI analysis completed",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"provider": analysis.provider, "model": analysis.model},
|
||||
)
|
||||
return AlertBriefResponse(
|
||||
**analysis.model_dump(),
|
||||
title=brief_request.title,
|
||||
objective=brief_request.objective,
|
||||
facts=facts,
|
||||
context=context,
|
||||
)
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.alerts.failed",
|
||||
message="Alert brief AI analysis failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"preferred_model": payload.preferred_model}),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@router.post("/situational-alerts/brief", response_model=SituationalAlertBriefResponse)
|
||||
@@ -268,14 +403,62 @@ async def analyze_situational_alert_brief(
|
||||
response.headers["X-Request-ID"] = request_id
|
||||
|
||||
brief_request, facts, context = await build_situational_alert_brief_request(db)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.situational_alerts.facts_collected",
|
||||
message="Situational alert brief facts collected",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"fact_count": len(facts or [])},
|
||||
)
|
||||
brief_request.preferred_model = payload.preferred_model
|
||||
brief_request.thinking = payload.thinking
|
||||
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
return SituationalAlertBriefResponse(
|
||||
**analysis.model_dump(),
|
||||
title=brief_request.title,
|
||||
objective=brief_request.objective,
|
||||
facts=facts,
|
||||
context=context,
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.situational_alerts.start",
|
||||
message="Situational alert brief AI analysis started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"preferred_model": payload.preferred_model},
|
||||
)
|
||||
try:
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.situational_alerts.completed",
|
||||
message="Situational alert brief AI analysis completed",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"provider": analysis.provider, "model": analysis.model},
|
||||
)
|
||||
return SituationalAlertBriefResponse(
|
||||
**analysis.model_dump(),
|
||||
title=brief_request.title,
|
||||
objective=brief_request.objective,
|
||||
facts=facts,
|
||||
context=context,
|
||||
)
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.situational_alerts.failed",
|
||||
message="Situational alert brief AI analysis failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"preferred_model": payload.preferred_model}),
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -341,6 +341,7 @@ async def collect_bgp_collector_location(
|
||||
provider_client=provider_client,
|
||||
query=query,
|
||||
entity_type="bgp_collector",
|
||||
db=db,
|
||||
attempted_queries=attempted_queries,
|
||||
search_evidence=search_result.evidence,
|
||||
)
|
||||
|
||||
@@ -102,7 +102,12 @@ def build_search_rank_sql(search: Optional[str]) -> str:
|
||||
"""
|
||||
|
||||
|
||||
def serialize_collected_row(row, source_name_map: dict[str, str] | None = None) -> dict:
|
||||
def serialize_collected_row(
|
||||
row,
|
||||
source_name_map: dict[str, str] | None = None,
|
||||
*,
|
||||
include_metadata: bool = True,
|
||||
) -> dict:
|
||||
metadata = row[7]
|
||||
source = row[1]
|
||||
return {
|
||||
@@ -120,7 +125,7 @@ def serialize_collected_row(row, source_name_map: dict[str, str] | None = None)
|
||||
"longitude": get_metadata_field(metadata, "longitude"),
|
||||
"value": get_metadata_field(metadata, "value"),
|
||||
"unit": get_metadata_field(metadata, "unit"),
|
||||
"metadata": metadata,
|
||||
"metadata": metadata if include_metadata else None,
|
||||
"cores": get_metadata_field(metadata, "cores"),
|
||||
"rmax": get_metadata_field(metadata, "rmax"),
|
||||
"rpeak": get_metadata_field(metadata, "rpeak"),
|
||||
@@ -145,6 +150,7 @@ async def list_collected_data(
|
||||
search: Optional[str] = Query(None, description="搜索名称"),
|
||||
page: int = Query(1, ge=1, description="页码"),
|
||||
page_size: int = Query(20, ge=1, le=100, description="每页数量"),
|
||||
include_metadata: bool = Query(True, description="是否返回完整 metadata 字段"),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
@@ -201,7 +207,7 @@ async def list_collected_data(
|
||||
|
||||
data = []
|
||||
for row in rows:
|
||||
data.append(serialize_collected_row(row[:11], source_name_map))
|
||||
data.append(serialize_collected_row(row[:11], source_name_map, include_metadata=include_metadata))
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
|
||||
@@ -5,7 +5,7 @@ from datetime import datetime
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from sqlalchemy import delete, select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from pydantic import BaseModel, Field
|
||||
@@ -22,6 +22,7 @@ from app.models.vessel import AISRawObservation, AISSourceHealth
|
||||
from app.core.security import get_current_user
|
||||
from app.core.cache import cache
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient, get_ai_provider_client
|
||||
from app.services.datasource_mapping import (
|
||||
@@ -41,15 +42,82 @@ from app.services.custom_datasource_runtime import (
|
||||
stop_custom_stream,
|
||||
test_websocket_config,
|
||||
)
|
||||
|
||||
DATASOURCE_MAPPING_PROMPT_KEY = "datasource.mapping"
|
||||
from app.services.datasource_connectivity import (
|
||||
_resolve_aisstream_api_key,
|
||||
_resolve_spacetrack_credentials_with_override,
|
||||
get_builtin_connection_status,
|
||||
save_connectivity_success,
|
||||
strip_connectivity_validation,
|
||||
test_builtin_connectivity,
|
||||
)
|
||||
from app.services.barentswatch import resolve_barentswatch_config
|
||||
from app.services.persistent_logs import record_audit_log
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
|
||||
|
||||
|
||||
def _user_role_value(user: User) -> str:
|
||||
role = getattr(user, "role", "")
|
||||
return str(getattr(role, "value", role) or "").lower()
|
||||
|
||||
|
||||
def _user_display_name(user: User) -> str:
|
||||
return str(getattr(user, "username", None) or getattr(user, "email", None) or getattr(user, "id", ""))
|
||||
|
||||
|
||||
async def _record_datasource_secret_reveal(
|
||||
*,
|
||||
current_user: User,
|
||||
request: Request,
|
||||
target_id: str,
|
||||
result: str,
|
||||
details: dict[str, Any],
|
||||
) -> None:
|
||||
await record_audit_log(
|
||||
action="datasource_config.secret.reveal",
|
||||
actor_id=getattr(current_user, "id", None),
|
||||
actor_name=_user_display_name(current_user),
|
||||
target_type="datasource_config_secret",
|
||||
target_id=target_id,
|
||||
result=result,
|
||||
ip=request.client.host if request.client else None,
|
||||
details=details,
|
||||
)
|
||||
|
||||
|
||||
async def _ensure_datasource_secret_reveal_allowed(
|
||||
current_user: User,
|
||||
request: Request,
|
||||
target_id: str,
|
||||
details: dict[str, Any],
|
||||
) -> None:
|
||||
if _user_role_value(current_user) in SECRET_REVEAL_ROLES:
|
||||
return
|
||||
await _record_datasource_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=target_id,
|
||||
result="denied",
|
||||
details={**details, "role": _user_role_value(current_user)},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Only administrators can reveal datasource credentials",
|
||||
)
|
||||
|
||||
def _default_builtin_config(name: str) -> dict[str, Any]:
|
||||
return {"timeout": 30, "retry": 3}
|
||||
|
||||
|
||||
def _default_builtin_source_type(name: str) -> str:
|
||||
if name == "aisstream_vessels":
|
||||
return "websocket"
|
||||
return "http"
|
||||
|
||||
|
||||
class DataSourceConfigCreate(BaseModel):
|
||||
name: str = Field(..., min_length=1, max_length=100)
|
||||
@@ -364,7 +432,7 @@ async def list_all_datasources(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List all data sources: YAML defaults + DB overrides"""
|
||||
from app.core.data_sources import COLLECTOR_URL_KEYS, get_data_sources_config
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
|
||||
config = get_data_sources_config()
|
||||
|
||||
@@ -372,38 +440,144 @@ async def list_all_datasources(
|
||||
db_configs = {c.name: c for c in db_query.scalars().all()}
|
||||
|
||||
result = []
|
||||
for name, yaml_key in COLLECTOR_URL_KEYS.items():
|
||||
for name, metadata in DEFAULT_DATASOURCES.items():
|
||||
yaml_url = config.get_yaml_url(name)
|
||||
db_config = db_configs.get(name)
|
||||
default_config = _default_builtin_config(name)
|
||||
default_url = yaml_url
|
||||
db_auth_config = db_config.auth_config or {} if db_config else {}
|
||||
|
||||
result.append(
|
||||
{
|
||||
"name": name,
|
||||
"default_url": yaml_url,
|
||||
"endpoint": db_config.endpoint if db_config else yaml_url,
|
||||
"requires_credentials": bool(metadata.get("requires_credentials", False)),
|
||||
"credential_provider": metadata.get("credential_provider"),
|
||||
"credential_status": metadata.get("credential_status", "none"),
|
||||
"default_url": default_url,
|
||||
"endpoint": db_config.endpoint if db_config else default_url,
|
||||
"is_overridden": db_config is not None and db_config.endpoint != yaml_url
|
||||
if yaml_url
|
||||
if default_url
|
||||
else db_config is not None,
|
||||
"is_active": db_config.is_active if db_config else True,
|
||||
"source_type": db_config.source_type if db_config else "http",
|
||||
"source_type": db_config.source_type if db_config else _default_builtin_source_type(name),
|
||||
"auth_type": db_config.auth_type if db_config else "none",
|
||||
"auth_config": {
|
||||
"client_id": db_auth_config.get("client_id") or "",
|
||||
"username": db_auth_config.get("username") or "",
|
||||
"key_name": db_auth_config.get("key_name") or db_auth_config.get("param_name") or "",
|
||||
"param_name": db_auth_config.get("param_name") or db_auth_config.get("key_name") or "",
|
||||
"location": db_auth_config.get("location") or db_auth_config.get("in") or "",
|
||||
"in": db_auth_config.get("in") or db_auth_config.get("location") or "",
|
||||
},
|
||||
"auth_configured": {
|
||||
"api_key": bool((db_config.auth_config or {}).get("api_key"))
|
||||
if db_config
|
||||
else False,
|
||||
"api_key": bool(db_auth_config.get("api_key")),
|
||||
"client_id": bool(db_auth_config.get("client_id")),
|
||||
"client_secret": bool(db_auth_config.get("client_secret")),
|
||||
"username": bool(db_auth_config.get("username")),
|
||||
"password": bool(db_auth_config.get("password")),
|
||||
},
|
||||
"headers": db_config.headers if db_config else {},
|
||||
"config": strip_connectivity_validation(db_config.config if db_config else {}),
|
||||
"config": strip_connectivity_validation(db_config.config if db_config else default_config),
|
||||
"config_id": db_config.id if db_config else None,
|
||||
"description": db_config.description
|
||||
if db_config
|
||||
else f"Data source from YAML: {yaml_key}",
|
||||
else f"内置采集器默认配置:{metadata.get('display_name') or metadata.get('name') or name}",
|
||||
}
|
||||
)
|
||||
|
||||
return {"total": len(result), "data": result}
|
||||
|
||||
|
||||
@router.get("/configs/secrets")
|
||||
async def reveal_builtin_config_secrets(
|
||||
request: Request,
|
||||
name: str = Query(..., min_length=1),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Reveal configured built-in datasource credentials for admin editing."""
|
||||
source = name.strip()
|
||||
metadata = DEFAULT_DATASOURCES.get(source)
|
||||
if not metadata or not metadata.get("requires_credentials"):
|
||||
raise HTTPException(status_code=404, detail="Credentialed datasource config not found")
|
||||
|
||||
provider = str(metadata.get("credential_provider") or "")
|
||||
target_id = f"datasource_config:{source}"
|
||||
await _ensure_datasource_secret_reveal_allowed(
|
||||
current_user,
|
||||
request,
|
||||
target_id,
|
||||
{"source": source, "provider": provider},
|
||||
)
|
||||
|
||||
result = await db.execute(select(DataSourceConfig).where(DataSourceConfig.name == source))
|
||||
record = result.scalar_one_or_none()
|
||||
auth_config = dict(record.auth_config or {}) if record else {}
|
||||
payload: dict[str, Any] = {
|
||||
"name": source,
|
||||
"provider": provider,
|
||||
}
|
||||
details: dict[str, Any] = {"source": source, "provider": provider}
|
||||
|
||||
if provider == "barentswatch":
|
||||
resolved = await resolve_barentswatch_config(db)
|
||||
client_id = str(auth_config.get("client_id") or resolved.client_id or "")
|
||||
client_secret = str(auth_config.get("client_secret") or resolved.client_secret or "")
|
||||
source_label = "datasource_config" if auth_config.get("client_id") or auth_config.get("client_secret") else resolved.credential_source
|
||||
payload.update(
|
||||
{
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"client_id_source": source_label if client_id else "missing",
|
||||
"client_secret_source": source_label if client_secret else "missing",
|
||||
}
|
||||
)
|
||||
details.update(
|
||||
{
|
||||
"client_id_configured": bool(client_id),
|
||||
"client_secret_configured": bool(client_secret),
|
||||
"credential_source": source_label,
|
||||
}
|
||||
)
|
||||
elif provider == "aisstream":
|
||||
api_key, api_key_source = await _resolve_aisstream_api_key(db)
|
||||
payload.update({"api_key": api_key, "api_key_source": api_key_source})
|
||||
details.update({"api_key_configured": bool(api_key), "api_key_source": api_key_source})
|
||||
elif provider == "spacetrack":
|
||||
if auth_config.get("username") or auth_config.get("password"):
|
||||
username = str(auth_config.get("username") or "")
|
||||
password = str(auth_config.get("password") or "")
|
||||
credential_source = "datasource_config"
|
||||
else:
|
||||
username, password, credential_source = _resolve_spacetrack_credentials_with_override()
|
||||
payload.update(
|
||||
{
|
||||
"username": username,
|
||||
"password": password,
|
||||
"username_source": credential_source if username else "missing",
|
||||
"password_source": credential_source if password else "missing",
|
||||
}
|
||||
)
|
||||
details.update(
|
||||
{
|
||||
"username_configured": bool(username),
|
||||
"password_configured": bool(password),
|
||||
"credential_source": credential_source,
|
||||
}
|
||||
)
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="Datasource credential provider is not supported")
|
||||
|
||||
await _record_datasource_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=target_id,
|
||||
result="success",
|
||||
details=details,
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
@router.get("/configs/{config_id}")
|
||||
async def get_config(
|
||||
config_id: int,
|
||||
@@ -757,14 +931,12 @@ async def propose_datasource_mapping(
|
||||
generated_by = "heuristic"
|
||||
if payload.use_ai:
|
||||
try:
|
||||
prompt = await get_effective_prompt(db, DATASOURCE_MAPPING_PROMPT_KEY)
|
||||
response = await ai_client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title=f"Generate datasource mapping for {schema.key}",
|
||||
objective=(
|
||||
"Return only JSON for a deterministic mapping DSL. "
|
||||
"The JSON must contain source.items_path and fields. "
|
||||
"Do not include prose or code."
|
||||
),
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"target_schema": schema.to_dict(),
|
||||
"sample_payload": redacted_sample,
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
@@ -7,6 +6,7 @@ from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, or_, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.security import get_current_user
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
@@ -21,15 +21,26 @@ from app.models.user import User
|
||||
from app.models.vessel import AISRawObservation
|
||||
from app.services.vessel_ais_aggregation import VESSEL_AIS_SCHEMA
|
||||
from app.services.scheduler import (
|
||||
cancel_running_collector_now,
|
||||
get_latest_task_id_for_datasource,
|
||||
run_collector_now,
|
||||
sync_datasource_job,
|
||||
)
|
||||
from app.services.data_jobs import (
|
||||
JOB_STATUS_CANCELLING,
|
||||
JOB_STATUS_QUEUED,
|
||||
JOB_STATUS_RUNNING,
|
||||
JOB_TYPE_CLEAR_CACHE,
|
||||
JOB_TYPE_CLEAR_DATA,
|
||||
JOB_TYPE_COLLECT,
|
||||
enqueue_datasource_job,
|
||||
get_active_datasource_job,
|
||||
request_cancel_datasource_task,
|
||||
)
|
||||
from app.services.business_logs import emit_business_log
|
||||
|
||||
router = APIRouter()
|
||||
logger = get_logger(__name__, service="api")
|
||||
STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90
|
||||
|
||||
|
||||
PRODUCT_SOURCE_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
("vessels", ("vessel", "ais")),
|
||||
("cables", ("cable", "landing", "telegeography", "arcgis", "fao")),
|
||||
@@ -113,7 +124,7 @@ async def _load_latest_running_tasks(
|
||||
_task_rank_column(CollectionTask.started_at),
|
||||
)
|
||||
.where(CollectionTask.datasource_id.in_(datasource_ids))
|
||||
.where(CollectionTask.status == "running")
|
||||
.where(CollectionTask.status.in_((JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)))
|
||||
.subquery()
|
||||
)
|
||||
result = await db.execute(
|
||||
@@ -124,10 +135,10 @@ async def _load_latest_running_tasks(
|
||||
return {task.datasource_id: task for task in result.scalars().all()}
|
||||
|
||||
|
||||
async def _load_latest_task_ids(
|
||||
async def _load_latest_tasks(
|
||||
db: AsyncSession,
|
||||
datasource_ids: list[int],
|
||||
) -> dict[int, int]:
|
||||
) -> dict[int, CollectionTask]:
|
||||
if not datasource_ids:
|
||||
return {}
|
||||
|
||||
@@ -137,17 +148,18 @@ async def _load_latest_task_ids(
|
||||
CollectionTask.datasource_id.label("datasource_id"),
|
||||
func.row_number().over(
|
||||
partition_by=CollectionTask.datasource_id,
|
||||
order_by=CollectionTask.id.desc(),
|
||||
order_by=(CollectionTask.created_at.desc().nullslast(), CollectionTask.id.desc()),
|
||||
).label("row_num"),
|
||||
)
|
||||
.where(CollectionTask.datasource_id.in_(datasource_ids))
|
||||
.subquery()
|
||||
)
|
||||
result = await db.execute(
|
||||
select(ranked_tasks.c.datasource_id, ranked_tasks.c.task_id)
|
||||
select(CollectionTask)
|
||||
.join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id)
|
||||
.where(ranked_tasks.c.row_num == 1)
|
||||
)
|
||||
return {datasource_id: task_id for datasource_id, task_id in result.all()}
|
||||
return {task.datasource_id: task for task in result.scalars().all()}
|
||||
|
||||
|
||||
async def _load_collected_record_counts(
|
||||
@@ -208,7 +220,9 @@ async def _load_datasource_endpoint_overrides(
|
||||
async def _load_datasource_list_context(
|
||||
db: AsyncSession,
|
||||
datasources: list[DataSource],
|
||||
) -> tuple[dict[int, CollectionTask], dict[str, str]]:
|
||||
*,
|
||||
include_endpoint: bool = True,
|
||||
) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, str]]:
|
||||
datasource_ids = [datasource.id for datasource in datasources]
|
||||
sources = [datasource.source for datasource in datasources]
|
||||
|
||||
@@ -232,8 +246,67 @@ async def _load_datasource_list_context(
|
||||
if stale_datasource_ids:
|
||||
running_tasks = await _load_latest_running_tasks(db, datasource_ids)
|
||||
|
||||
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
|
||||
return running_tasks, endpoint_overrides
|
||||
latest_tasks = await _load_latest_tasks(db, datasource_ids)
|
||||
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources) if include_endpoint else {}
|
||||
return running_tasks, latest_tasks, endpoint_overrides
|
||||
|
||||
|
||||
def serialize_datasource_row(
|
||||
datasource: DataSource,
|
||||
*,
|
||||
running_tasks: dict[int, CollectionTask],
|
||||
latest_tasks: dict[int, CollectionTask],
|
||||
record_counts: dict[str, int],
|
||||
endpoint_overrides: dict[str, str],
|
||||
config,
|
||||
include_endpoint: bool,
|
||||
) -> dict:
|
||||
running_task = running_tasks.get(datasource.id)
|
||||
latest_task = latest_tasks.get(datasource.id)
|
||||
display_task = running_task or latest_task
|
||||
endpoint = None
|
||||
if include_endpoint:
|
||||
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
|
||||
last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None)
|
||||
last_status = datasource.last_status or (latest_task.status if latest_task else None)
|
||||
collected_records = record_counts.get(datasource.source, 0)
|
||||
|
||||
row = {
|
||||
"id": datasource.id,
|
||||
"source": datasource.source,
|
||||
"name": datasource.name,
|
||||
**datasource_metadata(datasource.source),
|
||||
"product": datasource_product_key(datasource),
|
||||
"module": datasource.module,
|
||||
"priority": datasource.priority,
|
||||
"frequency": format_frequency_label(datasource.frequency_minutes),
|
||||
"frequency_minutes": datasource.frequency_minutes,
|
||||
"is_active": datasource.is_active,
|
||||
"collector_class": datasource.collector_class,
|
||||
"last_run": to_iso8601_utc(last_run_at),
|
||||
"last_run_at": to_iso8601_utc(last_run_at),
|
||||
"last_status": last_status,
|
||||
"is_running": running_task is not None and running_task.task_type == JOB_TYPE_COLLECT,
|
||||
"is_task_active": running_task is not None,
|
||||
"task_status": running_task.status if running_task else None,
|
||||
"task_id": display_task.id if display_task else None,
|
||||
"task_type": display_task.task_type if display_task else None,
|
||||
"progress": display_task.progress if display_task else None,
|
||||
"phase": display_task.phase if display_task else None,
|
||||
"phase_progress": display_task.phase_progress if display_task else None,
|
||||
"phase_message": display_task.phase_message if display_task else None,
|
||||
"phase_current": display_task.phase_current if display_task else None,
|
||||
"phase_total": display_task.phase_total if display_task else None,
|
||||
"phase_unit": display_task.phase_unit if display_task else None,
|
||||
"records_processed": display_task.records_processed if display_task else None,
|
||||
"total_records": display_task.total_records if display_task else None,
|
||||
"error_message": display_task.error_message if display_task else None,
|
||||
"collected_records": collected_records,
|
||||
"has_collected_data": collected_records > 0,
|
||||
}
|
||||
if include_endpoint:
|
||||
row["endpoint"] = endpoint
|
||||
return row
|
||||
|
||||
|
||||
def _apply_datasource_query_filters(
|
||||
@@ -251,11 +324,6 @@ def _apply_datasource_query_filters(
|
||||
query = query.where(DataSource.is_active == is_active)
|
||||
if priority:
|
||||
query = query.where(DataSource.priority == priority)
|
||||
if run_status and run_status not in {"running", "collected", "uncollected"}:
|
||||
if run_status == "not_run":
|
||||
query = query.where(DataSource.last_status.is_(None))
|
||||
else:
|
||||
query = query.where(DataSource.last_status == run_status)
|
||||
if q:
|
||||
like_value = f"%{q.strip()}%"
|
||||
query = query.where(
|
||||
@@ -272,15 +340,25 @@ def _filter_datasources_in_memory(
|
||||
datasources: list[DataSource],
|
||||
*,
|
||||
running_tasks: dict[int, CollectionTask],
|
||||
latest_tasks: dict[int, CollectionTask] | None = None,
|
||||
record_counts: dict[str, int],
|
||||
product: Optional[str] = None,
|
||||
run_status: Optional[str] = None,
|
||||
collected: Optional[bool] = None,
|
||||
credential_status: Optional[str] = None,
|
||||
) -> list[DataSource]:
|
||||
latest_tasks = latest_tasks or {}
|
||||
filtered: list[DataSource] = []
|
||||
for datasource in datasources:
|
||||
record_count = record_counts.get(datasource.source, 0)
|
||||
latest_task = latest_tasks.get(datasource.id)
|
||||
effective_status = (
|
||||
"running"
|
||||
if datasource.id in running_tasks
|
||||
else latest_task.status
|
||||
if latest_task is not None
|
||||
else datasource.last_status
|
||||
)
|
||||
if product and datasource_product_key(datasource) != product:
|
||||
continue
|
||||
if collected is not None and (record_count > 0) != collected:
|
||||
@@ -291,6 +369,10 @@ def _filter_datasources_in_memory(
|
||||
continue
|
||||
if run_status == "running" and datasource.id not in running_tasks:
|
||||
continue
|
||||
if run_status == "not_run" and effective_status is not None:
|
||||
continue
|
||||
if run_status not in {None, "running", "not_run", "collected", "uncollected"} and effective_status != run_status:
|
||||
continue
|
||||
if run_status == "collected" and record_count <= 0:
|
||||
continue
|
||||
if run_status == "uncollected" and record_count > 0:
|
||||
@@ -304,9 +386,25 @@ async def _trigger_datasource_batch(
|
||||
datasources: list[DataSource],
|
||||
*,
|
||||
force: bool,
|
||||
actor_id: int | None = None,
|
||||
trigger_kind: str = "batch",
|
||||
) -> dict:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=f"collector.trigger.{trigger_kind}.start",
|
||||
message="Datasource batch trigger started",
|
||||
category="collector",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=actor_id,
|
||||
context={
|
||||
"trigger_kind": trigger_kind,
|
||||
"force": force,
|
||||
"requested_count": len(datasources),
|
||||
},
|
||||
)
|
||||
if not datasources:
|
||||
return {
|
||||
result = {
|
||||
"status": "noop",
|
||||
"message": "No matching data sources to trigger",
|
||||
"force": force,
|
||||
@@ -314,8 +412,18 @@ async def _trigger_datasource_batch(
|
||||
"skipped": [],
|
||||
"failed": [],
|
||||
}
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=f"collector.trigger.{trigger_kind}.completed",
|
||||
message="Datasource batch trigger completed with no matching sources",
|
||||
category="collector",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=actor_id,
|
||||
context={"trigger_kind": trigger_kind, "force": force, "status": "noop", "triggered_count": 0},
|
||||
)
|
||||
return result
|
||||
|
||||
previous_task_ids: dict[int, Optional[int]] = {}
|
||||
triggered_sources: list[dict] = []
|
||||
skipped_sources: list[dict] = []
|
||||
failed_sources: list[dict] = []
|
||||
@@ -350,9 +458,11 @@ async def _trigger_datasource_batch(
|
||||
}
|
||||
)
|
||||
continue
|
||||
cancelled = await cancel_running_collector_now(datasource.source)
|
||||
if not cancelled:
|
||||
await rollback_orphaned_running_task(db, datasource, running_task)
|
||||
await request_cancel_datasource_task(
|
||||
db,
|
||||
running_task,
|
||||
reason="superseded_by_forced_collection",
|
||||
)
|
||||
|
||||
if not force and not is_due_for_collection(datasource, now):
|
||||
skipped_sources.append(
|
||||
@@ -369,57 +479,51 @@ async def _trigger_datasource_batch(
|
||||
)
|
||||
continue
|
||||
|
||||
previous_task_ids[datasource.id] = None
|
||||
success = run_collector_now(datasource.source)
|
||||
if not success:
|
||||
failed_sources.append(
|
||||
{
|
||||
"id": datasource.id,
|
||||
"source": datasource.source,
|
||||
"name": datasource.name,
|
||||
"reason": "trigger_failed",
|
||||
}
|
||||
)
|
||||
continue
|
||||
task = await enqueue_datasource_job(
|
||||
db,
|
||||
datasource,
|
||||
JOB_TYPE_COLLECT,
|
||||
payload={"force": force, "trigger": "batch"},
|
||||
)
|
||||
|
||||
triggered_sources.append(
|
||||
{
|
||||
"id": datasource.id,
|
||||
"source": datasource.source,
|
||||
"name": datasource.name,
|
||||
"task_id": None,
|
||||
"task_id": task.id,
|
||||
}
|
||||
)
|
||||
|
||||
latest_task_ids = await _load_latest_task_ids(
|
||||
db,
|
||||
[datasource.id for datasource in datasources],
|
||||
)
|
||||
for datasource_id in previous_task_ids:
|
||||
previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id)
|
||||
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0.1)
|
||||
pending = [item for item in triggered_sources if item["task_id"] is None]
|
||||
if not pending:
|
||||
break
|
||||
latest_task_ids = await _load_latest_task_ids(
|
||||
db,
|
||||
[item["id"] for item in pending],
|
||||
)
|
||||
for item in pending:
|
||||
task_id = latest_task_ids.get(item["id"])
|
||||
if task_id is not None and task_id != previous_task_ids.get(item["id"]):
|
||||
item["task_id"] = task_id
|
||||
|
||||
return {
|
||||
"status": "triggered" if triggered_sources else "partial",
|
||||
"message": f"Triggered {len(triggered_sources)} data sources",
|
||||
result = {
|
||||
"status": "queued" if triggered_sources else "partial",
|
||||
"message": f"Queued {len(triggered_sources)} data source jobs",
|
||||
"force": force,
|
||||
"triggered": triggered_sources,
|
||||
"skipped": skipped_sources,
|
||||
"failed": failed_sources,
|
||||
}
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=f"collector.trigger.{trigger_kind}.completed",
|
||||
message="Datasource batch trigger completed",
|
||||
category="collector",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=actor_id,
|
||||
context={
|
||||
"trigger_kind": trigger_kind,
|
||||
"force": force,
|
||||
"status": result["status"],
|
||||
"requested_count": len(datasources),
|
||||
"triggered_count": len(triggered_sources),
|
||||
"skipped_count": len(skipped_sources),
|
||||
"failed_count": len(failed_sources),
|
||||
"triggered_sources": [item["source"] for item in triggered_sources],
|
||||
"skipped_reasons": [item["reason"] for item in skipped_sources],
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
|
||||
@@ -621,6 +725,7 @@ async def list_datasources(
|
||||
collected: Optional[bool] = None,
|
||||
credential_status: Optional[str] = None,
|
||||
q: Optional[str] = None,
|
||||
include_endpoint: bool = True,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
@@ -639,11 +744,16 @@ async def list_datasources(
|
||||
|
||||
collector_list = []
|
||||
config = get_data_sources_config()
|
||||
running_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
|
||||
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(
|
||||
db,
|
||||
datasources,
|
||||
include_endpoint=include_endpoint,
|
||||
)
|
||||
record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
|
||||
datasources = _filter_datasources_in_memory(
|
||||
datasources,
|
||||
running_tasks=running_tasks,
|
||||
latest_tasks=latest_tasks,
|
||||
record_counts=record_counts,
|
||||
product=product,
|
||||
run_status=run_status,
|
||||
@@ -651,43 +761,16 @@ async def list_datasources(
|
||||
credential_status=credential_status,
|
||||
)
|
||||
for datasource in datasources:
|
||||
running_task = running_tasks.get(datasource.id)
|
||||
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
|
||||
last_run_at = datasource.last_run_at
|
||||
last_status = datasource.last_status
|
||||
collected_records = record_counts.get(datasource.source, 0)
|
||||
|
||||
collector_list.append(
|
||||
{
|
||||
"id": datasource.id,
|
||||
"source": datasource.source,
|
||||
"name": datasource.name,
|
||||
**datasource_metadata(datasource.source),
|
||||
"product": datasource_product_key(datasource),
|
||||
"module": datasource.module,
|
||||
"priority": datasource.priority,
|
||||
"frequency": format_frequency_label(datasource.frequency_minutes),
|
||||
"frequency_minutes": datasource.frequency_minutes,
|
||||
"is_active": datasource.is_active,
|
||||
"collector_class": datasource.collector_class,
|
||||
"endpoint": endpoint,
|
||||
"last_run": to_iso8601_utc(last_run_at),
|
||||
"last_run_at": to_iso8601_utc(last_run_at),
|
||||
"last_status": last_status,
|
||||
"is_running": running_task is not None,
|
||||
"task_id": running_task.id if running_task else None,
|
||||
"progress": running_task.progress if running_task else None,
|
||||
"phase": running_task.phase if running_task else None,
|
||||
"phase_progress": running_task.phase_progress if running_task else None,
|
||||
"phase_message": running_task.phase_message if running_task else None,
|
||||
"phase_current": running_task.phase_current if running_task else None,
|
||||
"phase_total": running_task.phase_total if running_task else None,
|
||||
"phase_unit": running_task.phase_unit if running_task else None,
|
||||
"records_processed": running_task.records_processed if running_task else None,
|
||||
"total_records": running_task.total_records if running_task else None,
|
||||
"collected_records": collected_records,
|
||||
"has_collected_data": collected_records > 0,
|
||||
}
|
||||
serialize_datasource_row(
|
||||
datasource,
|
||||
running_tasks=running_tasks,
|
||||
latest_tasks=latest_tasks,
|
||||
record_counts=record_counts,
|
||||
endpoint_overrides=endpoint_overrides,
|
||||
config=config,
|
||||
include_endpoint=include_endpoint,
|
||||
)
|
||||
)
|
||||
|
||||
return {"total": len(collector_list), "data": collector_list}
|
||||
@@ -705,7 +788,13 @@ async def trigger_all_datasources(
|
||||
.order_by(DataSource.module, DataSource.id)
|
||||
)
|
||||
datasources = result.scalars().all()
|
||||
return await _trigger_datasource_batch(db, datasources, force=force)
|
||||
return await _trigger_datasource_batch(
|
||||
db,
|
||||
datasources,
|
||||
force=force,
|
||||
actor_id=current_user.id,
|
||||
trigger_kind="all",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/trigger-batch")
|
||||
@@ -729,18 +818,70 @@ async def trigger_datasource_batch(
|
||||
|
||||
result = await db.execute(query)
|
||||
datasources = result.scalars().all()
|
||||
running_tasks, _ = await _load_datasource_list_context(db, datasources)
|
||||
running_tasks, latest_tasks, _ = await _load_datasource_list_context(db, datasources)
|
||||
record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
|
||||
datasources = _filter_datasources_in_memory(
|
||||
datasources,
|
||||
running_tasks=running_tasks,
|
||||
latest_tasks=latest_tasks,
|
||||
record_counts=record_counts,
|
||||
product=None if payload.source_ids else payload.product,
|
||||
run_status=None if payload.source_ids else payload.run_status,
|
||||
collected=None if payload.source_ids else payload.collected,
|
||||
credential_status=None if payload.source_ids else payload.credential_status,
|
||||
)
|
||||
return await _trigger_datasource_batch(db, datasources, force=payload.force)
|
||||
return await _trigger_datasource_batch(
|
||||
db,
|
||||
datasources,
|
||||
force=payload.force,
|
||||
actor_id=current_user.id,
|
||||
trigger_kind="batch",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/snapshots")
|
||||
async def list_datasource_snapshots(
|
||||
source_id: Optional[str] = None,
|
||||
current_only: Optional[bool] = None,
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
query = (
|
||||
select(DataSnapshot, DataSource.name, DataSource.module)
|
||||
.outerjoin(DataSource, DataSource.id == DataSnapshot.datasource_id)
|
||||
.order_by(DataSnapshot.created_at.desc().nullslast(), DataSnapshot.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if source_id:
|
||||
query = query.where(DataSnapshot.source == source_id)
|
||||
if current_only is not None:
|
||||
query = query.where(DataSnapshot.is_current.is_(current_only))
|
||||
|
||||
result = await db.execute(query)
|
||||
rows = []
|
||||
for snapshot, datasource_name, datasource_module in result.all():
|
||||
rows.append(
|
||||
{
|
||||
"id": snapshot.id,
|
||||
"datasource_id": snapshot.datasource_id,
|
||||
"datasource_name": datasource_name,
|
||||
"module": datasource_module,
|
||||
"task_id": snapshot.task_id,
|
||||
"source": snapshot.source,
|
||||
"snapshot_key": snapshot.snapshot_key,
|
||||
"reference_date": to_iso8601_utc(snapshot.reference_date),
|
||||
"started_at": to_iso8601_utc(snapshot.started_at),
|
||||
"completed_at": to_iso8601_utc(snapshot.completed_at),
|
||||
"record_count": snapshot.record_count,
|
||||
"status": snapshot.status,
|
||||
"is_current": snapshot.is_current,
|
||||
"parent_snapshot_id": snapshot.parent_snapshot_id,
|
||||
"summary": snapshot.summary or {},
|
||||
"created_at": to_iso8601_utc(snapshot.created_at),
|
||||
}
|
||||
)
|
||||
return {"total": len(rows), "data": rows}
|
||||
|
||||
|
||||
@router.get("/{source_id}")
|
||||
@@ -771,6 +912,37 @@ async def get_datasource(
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{source_id}/row")
|
||||
async def get_datasource_row(
|
||||
source_id: str,
|
||||
include_endpoint: bool = True,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
datasource = await get_datasource_record(db, source_id)
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
config = get_data_sources_config()
|
||||
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(
|
||||
db,
|
||||
[datasource],
|
||||
include_endpoint=include_endpoint,
|
||||
)
|
||||
record_counts = await _load_collected_record_counts(db, [datasource.source])
|
||||
return {
|
||||
"data": serialize_datasource_row(
|
||||
datasource,
|
||||
running_tasks=running_tasks,
|
||||
latest_tasks=latest_tasks,
|
||||
record_counts=record_counts,
|
||||
endpoint_overrides=endpoint_overrides,
|
||||
config=config,
|
||||
include_endpoint=include_endpoint,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/enable")
|
||||
async def enable_datasource(
|
||||
source_id: str,
|
||||
@@ -840,8 +1012,24 @@ async def trigger_datasource(
|
||||
if not datasource.is_active:
|
||||
raise HTTPException(status_code=400, detail="Data source is disabled")
|
||||
|
||||
running_task = await get_running_task(db, datasource.id)
|
||||
running_task = await get_active_datasource_job(db, datasource.id, task_types=(JOB_TYPE_COLLECT,))
|
||||
if running_task is not None and not force:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.trigger.single.skipped_already_running",
|
||||
message="Datasource trigger skipped because a task is already running",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={
|
||||
"collector_name": datasource.source,
|
||||
"datasource_id": datasource.id,
|
||||
"task_id": running_task.id,
|
||||
"status": "skipped",
|
||||
},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
@@ -861,31 +1049,42 @@ async def trigger_datasource(
|
||||
)
|
||||
|
||||
if running_task is not None and force:
|
||||
cancelled = await cancel_running_collector_now(datasource.source)
|
||||
if not cancelled:
|
||||
await rollback_orphaned_running_task(db, datasource, running_task)
|
||||
await request_cancel_datasource_task(
|
||||
db,
|
||||
running_task,
|
||||
reason="superseded_by_forced_collection",
|
||||
)
|
||||
|
||||
previous_task_id = await get_latest_task_id_for_datasource(datasource.id)
|
||||
success = run_collector_now(datasource.source)
|
||||
if not success:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to trigger collector '{datasource.source}'")
|
||||
|
||||
task_id = None
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0.1)
|
||||
task_id = await get_latest_task_id_for_datasource(datasource.id)
|
||||
if task_id is not None and task_id != previous_task_id:
|
||||
break
|
||||
if task_id == previous_task_id:
|
||||
task_id = None
|
||||
task = await enqueue_datasource_job(
|
||||
db,
|
||||
datasource,
|
||||
JOB_TYPE_COLLECT,
|
||||
payload={"force": force, "trigger": "single"},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.trigger.single.completed",
|
||||
message="Datasource trigger queued",
|
||||
category="collector",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={
|
||||
"collector_name": datasource.source,
|
||||
"datasource_id": datasource.id,
|
||||
"task_id": task.id,
|
||||
"force": force,
|
||||
"status": "queued",
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "triggered",
|
||||
"status": "queued",
|
||||
"source_id": datasource.id,
|
||||
"task_id": task_id,
|
||||
"task_id": task.id,
|
||||
"collector_name": datasource.source,
|
||||
"force": force,
|
||||
"message": f"Collector '{datasource.source}' has been triggered",
|
||||
"message": f"Collector '{datasource.source}' has been queued",
|
||||
}
|
||||
|
||||
|
||||
@@ -899,22 +1098,81 @@ async def clear_datasource_data(
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
result = await db.execute(
|
||||
select(func.count(CollectedData.id)).where(CollectedData.source == datasource.source)
|
||||
active_task = await get_active_datasource_job(db, datasource.id)
|
||||
if active_task is not None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"reason": "datasource_job_in_progress",
|
||||
"message": "当前数据源已有任务在执行,请等待完成或先取消任务。",
|
||||
"task_id": active_task.id,
|
||||
"task_type": active_task.task_type,
|
||||
"status": active_task.status,
|
||||
},
|
||||
)
|
||||
task = await enqueue_datasource_job(
|
||||
db,
|
||||
datasource,
|
||||
JOB_TYPE_CLEAR_DATA,
|
||||
payload={"source": datasource.source},
|
||||
)
|
||||
count = result.scalar() or 0
|
||||
|
||||
if count == 0:
|
||||
return {"status": "success", "message": "No data to clear", "deleted_count": 0}
|
||||
|
||||
delete_query = CollectedData.__table__.delete().where(CollectedData.source == datasource.source)
|
||||
await db.execute(delete_query)
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Cleared {count} records for data source '{datasource.name}'",
|
||||
"deleted_count": count,
|
||||
"status": "queued",
|
||||
"message": f"Queued data clearing for data source '{datasource.name}'",
|
||||
"task_id": task.id,
|
||||
"deleted_count": None,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{source_id}/cache")
|
||||
async def clear_datasource_cache(
|
||||
source_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
datasource = await get_datasource_record(db, source_id)
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
task = await enqueue_datasource_job(
|
||||
db,
|
||||
datasource,
|
||||
JOB_TYPE_CLEAR_CACHE,
|
||||
payload={"source": datasource.source},
|
||||
dedupe_key=f"clear_cache:{datasource.source}",
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "queued",
|
||||
"message": f"Queued cache clearing for data source '{datasource.name}'",
|
||||
"task_id": task.id,
|
||||
"deleted_count": None,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/tasks/{task_id}/cancel")
|
||||
async def cancel_datasource_task(
|
||||
source_id: str,
|
||||
task_id: int,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
datasource = await get_datasource_record(db, source_id)
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if not task or task.datasource_id != datasource.id:
|
||||
raise HTTPException(status_code=404, detail="Task not found")
|
||||
|
||||
task = await request_cancel_datasource_task(db, task)
|
||||
return {
|
||||
"status": "cancelled" if task.completed_at else "cancelling",
|
||||
"task_id": task.id,
|
||||
"task_type": task.task_type,
|
||||
"phase": task.phase,
|
||||
"requested_cancel_at": to_iso8601_utc(task.requested_cancel_at),
|
||||
}
|
||||
|
||||
|
||||
@@ -934,7 +1192,15 @@ async def get_task_status(
|
||||
if not task or task.datasource_id != datasource.id:
|
||||
raise HTTPException(status_code=404, detail="Task not found")
|
||||
else:
|
||||
task = await get_running_task(db, datasource.id)
|
||||
task = await get_active_datasource_job(db, datasource.id)
|
||||
if task is None:
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.datasource_id == datasource.id)
|
||||
.order_by(CollectionTask.created_at.desc().nullslast(), CollectionTask.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
task = result.scalar_one_or_none()
|
||||
|
||||
if not task:
|
||||
return {
|
||||
@@ -951,8 +1217,12 @@ async def get_task_status(
|
||||
}
|
||||
|
||||
return {
|
||||
"is_running": task.status == "running",
|
||||
"is_running": task.status in {JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING}
|
||||
and task.task_type == JOB_TYPE_COLLECT,
|
||||
"is_task_active": task.status in {JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING},
|
||||
"task_id": task.id,
|
||||
"task_type": task.task_type,
|
||||
"task_status": task.status,
|
||||
"progress": task.progress,
|
||||
"phase": task.phase,
|
||||
"phase_progress": task.phase_progress,
|
||||
@@ -963,4 +1233,6 @@ async def get_task_status(
|
||||
"records_processed": task.records_processed,
|
||||
"total_records": task.total_records,
|
||||
"status": task.status,
|
||||
"requested_cancel_at": to_iso8601_utc(task.requested_cancel_at),
|
||||
"error_message": task.error_message,
|
||||
}
|
||||
|
||||
441
backend/app/api/v1/earth.py
Normal file
441
backend/app/api/v1/earth.py
Normal file
@@ -0,0 +1,441 @@
|
||||
"""Earth asset management APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile, status
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import delete, func, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.security import decode_token, get_current_user, redis_client
|
||||
from app.db.session import get_db
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.models.user import User
|
||||
from app.services.tv_streams import get_tv_settings_payload
|
||||
from app.services.earth_boundaries import (
|
||||
EarthBoundaryBuildError,
|
||||
get_boundary_build_status,
|
||||
get_boundary_status,
|
||||
save_boundary_config,
|
||||
start_boundary_build_job,
|
||||
)
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
optional_bearer = HTTPBearer(auto_error=False)
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
EARTH_BRAND_ASSET_DIR = REPO_ROOT / "data" / "earth-brand"
|
||||
EARTH_BRAND_ASSET_URL_PREFIX = "/earth-brand-assets"
|
||||
EARTH_BRAND_CATEGORY = "earth_brand"
|
||||
EARTH_ABOUT_CATEGORY = "earth_about"
|
||||
SYSTEM_SETTINGS_CATEGORY = "system"
|
||||
MAX_EARTH_BRAND_ASSET_BYTES = 3 * 1024 * 1024
|
||||
ALLOWED_EARTH_BRAND_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".svg"}
|
||||
|
||||
|
||||
def _app_version_label() -> str:
|
||||
version = str(app_settings.VERSION or "").strip() or "0.0.0"
|
||||
return version if version.startswith("v") else f"v{version}"
|
||||
|
||||
|
||||
DEFAULT_EARTH_BRAND = {
|
||||
"logo_src": "/earth/assets/brand/earth-logo.png",
|
||||
"title_src": "/earth/assets/brand/title-zh.png",
|
||||
"title_text": "智能星球计划",
|
||||
"subtitle": "现实层宇宙全息感知系统",
|
||||
"description": "卫星 · 海底光缆 · 算力基础设施",
|
||||
"aria_label": "智能星球计划品牌标识",
|
||||
"title_alt": "智能星球计划",
|
||||
}
|
||||
|
||||
DEFAULT_EARTH_ABOUT = {
|
||||
"logo_src": "/earth/assets/brand/lim-logo.png",
|
||||
"kicker": "About",
|
||||
"title": "智能星球计划",
|
||||
"version": _app_version_label(),
|
||||
"description": "面向临空场景下的智能媒体研究、全球态势感知与多源开放数据巡航,提供可视化观测、事件聚合与交互式探索能力。",
|
||||
"meta": [
|
||||
{"label": "出品方", "value": "浙江大学临空智能媒体研究院"},
|
||||
{"label": "策划人", "value": "方兴东、黄柳青"},
|
||||
{"label": "产品兼开发者", "value": "钱坤、张鸽、齐鹏"},
|
||||
],
|
||||
}
|
||||
EARTH_ABOUT_LEGACY_PLANNER_VALUE = "黄柳青"
|
||||
|
||||
|
||||
class EarthBoundaryConfigPayload(BaseModel):
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class EarthBrandPayload(BaseModel):
|
||||
logo_src: str = Field(default=DEFAULT_EARTH_BRAND["logo_src"], max_length=1000)
|
||||
title_src: str = Field(default=DEFAULT_EARTH_BRAND["title_src"], max_length=1000)
|
||||
title_text: str = Field(default=DEFAULT_EARTH_BRAND["title_text"], max_length=120)
|
||||
subtitle: str = Field(default=DEFAULT_EARTH_BRAND["subtitle"], max_length=160)
|
||||
description: str = Field(default=DEFAULT_EARTH_BRAND["description"], max_length=200)
|
||||
aria_label: str = Field(default=DEFAULT_EARTH_BRAND["aria_label"], max_length=200)
|
||||
title_alt: str = Field(default=DEFAULT_EARTH_BRAND["title_alt"], max_length=200)
|
||||
|
||||
|
||||
class EarthAboutMetaItem(BaseModel):
|
||||
label: str = Field(default="", max_length=80)
|
||||
value: str = Field(default="", max_length=240)
|
||||
|
||||
|
||||
class EarthAboutPayload(BaseModel):
|
||||
logo_src: str = Field(default=DEFAULT_EARTH_ABOUT["logo_src"], max_length=1000)
|
||||
kicker: str = Field(default=DEFAULT_EARTH_ABOUT["kicker"], max_length=80)
|
||||
title: str = Field(default=DEFAULT_EARTH_ABOUT["title"], max_length=160)
|
||||
version: str = Field(default=DEFAULT_EARTH_ABOUT["version"], max_length=80)
|
||||
description: str = Field(default=DEFAULT_EARTH_ABOUT["description"], max_length=800)
|
||||
meta: list[EarthAboutMetaItem] = Field(default_factory=list)
|
||||
|
||||
|
||||
def _normalize_earth_brand_payload(payload: dict[str, Any] | None) -> dict[str, str]:
|
||||
merged = DEFAULT_EARTH_BRAND.copy()
|
||||
if payload:
|
||||
for key in DEFAULT_EARTH_BRAND:
|
||||
value = payload.get(key)
|
||||
if value is not None:
|
||||
merged[key] = str(value).strip()
|
||||
|
||||
if not merged["title_text"]:
|
||||
merged["title_text"] = DEFAULT_EARTH_BRAND["title_text"]
|
||||
if not merged["aria_label"]:
|
||||
merged["aria_label"] = merged["title_text"]
|
||||
if not merged["title_alt"]:
|
||||
merged["title_alt"] = merged["title_text"]
|
||||
return merged
|
||||
|
||||
|
||||
def _normalize_earth_about_payload(payload: dict[str, Any] | None) -> dict[str, Any]:
|
||||
merged: dict[str, Any] = {
|
||||
key: value
|
||||
for key, value in DEFAULT_EARTH_ABOUT.items()
|
||||
if key != "meta"
|
||||
}
|
||||
raw_meta = DEFAULT_EARTH_ABOUT["meta"]
|
||||
if payload:
|
||||
for key in ("logo_src", "kicker", "title", "description"):
|
||||
value = payload.get(key)
|
||||
if value is not None:
|
||||
merged[key] = str(value).strip()
|
||||
raw_meta = payload.get("meta") if isinstance(payload.get("meta"), list) else raw_meta
|
||||
merged["version"] = _app_version_label()
|
||||
|
||||
for key, default_value in DEFAULT_EARTH_ABOUT.items():
|
||||
if key == "meta":
|
||||
continue
|
||||
if not merged.get(key):
|
||||
merged[key] = default_value
|
||||
|
||||
normalized_meta: list[dict[str, str]] = []
|
||||
for item in raw_meta:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
label = str(item.get("label") or "").strip()
|
||||
value = str(item.get("value") or "").strip()
|
||||
if label == "策划人" and value == EARTH_ABOUT_LEGACY_PLANNER_VALUE:
|
||||
value = "方兴东、黄柳青"
|
||||
if label or value:
|
||||
normalized_meta.append({"label": label, "value": value})
|
||||
if not normalized_meta:
|
||||
normalized_meta = [dict(item) for item in DEFAULT_EARTH_ABOUT["meta"]]
|
||||
merged["meta"] = normalized_meta
|
||||
return merged
|
||||
|
||||
|
||||
def _is_demo_mode_enabled(payload: Any) -> bool:
|
||||
return bool(payload.get("demo_mode")) if isinstance(payload, dict) else False
|
||||
|
||||
|
||||
async def _get_earth_brand_record(db: AsyncSession) -> SystemSetting | None:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == EARTH_BRAND_CATEGORY)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _get_earth_brand_payload(db: AsyncSession) -> dict[str, Any]:
|
||||
record = await _get_earth_brand_record(db)
|
||||
return {
|
||||
"brand": _normalize_earth_brand_payload(record.payload if record else None),
|
||||
"is_default": record is None,
|
||||
}
|
||||
|
||||
|
||||
async def _get_earth_about_record(db: AsyncSession) -> SystemSetting | None:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == EARTH_ABOUT_CATEGORY)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _get_earth_about_payload(db: AsyncSession) -> dict[str, Any]:
|
||||
record = await _get_earth_about_record(db)
|
||||
return {
|
||||
"about": _normalize_earth_about_payload(record.payload if record else None),
|
||||
"is_default": record is None,
|
||||
}
|
||||
|
||||
|
||||
async def _get_optional_current_user(
|
||||
credentials: HTTPAuthorizationCredentials | None = Depends(optional_bearer),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> User | None:
|
||||
if credentials is None:
|
||||
return None
|
||||
token = credentials.credentials
|
||||
if redis_client.sismember("blacklisted_tokens", token):
|
||||
return None
|
||||
payload = decode_token(token)
|
||||
if payload is None or payload.get("type") != "access":
|
||||
return None
|
||||
user_id = payload.get("sub")
|
||||
if user_id is None:
|
||||
return None
|
||||
result = await db.execute(
|
||||
text(
|
||||
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id"
|
||||
),
|
||||
{"id": int(user_id)},
|
||||
)
|
||||
row = result.fetchone()
|
||||
if row is None or not row[5]:
|
||||
return None
|
||||
user = User()
|
||||
user.id = row[0]
|
||||
user.username = row[1]
|
||||
user.email = row[2]
|
||||
user.password_hash = row[3]
|
||||
user.role = row[4]
|
||||
user.is_active = row[5]
|
||||
user.gatekeeper_groups = row[6] or []
|
||||
return user
|
||||
|
||||
|
||||
@router.get("/brand")
|
||||
async def get_earth_brand(db: AsyncSession = Depends(get_db)):
|
||||
return await _get_earth_brand_payload(db)
|
||||
|
||||
|
||||
@router.put("/brand")
|
||||
async def update_earth_brand(
|
||||
payload: EarthBrandPayload,
|
||||
_current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
normalized = _normalize_earth_brand_payload(payload.model_dump())
|
||||
record = await _get_earth_brand_record(db)
|
||||
if record is None:
|
||||
record = SystemSetting(category=EARTH_BRAND_CATEGORY, payload=normalized)
|
||||
db.add(record)
|
||||
else:
|
||||
record.payload = normalized
|
||||
await db.commit()
|
||||
await db.refresh(record)
|
||||
return {"status": "updated", "brand": _normalize_earth_brand_payload(record.payload), "is_default": False}
|
||||
|
||||
|
||||
@router.delete("/brand")
|
||||
@router.post("/brand/reset")
|
||||
async def reset_earth_brand(
|
||||
_current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await db.execute(delete(SystemSetting).where(SystemSetting.category == EARTH_BRAND_CATEGORY))
|
||||
await db.commit()
|
||||
return {"status": "reset", "brand": DEFAULT_EARTH_BRAND.copy(), "is_default": True}
|
||||
|
||||
|
||||
@router.post("/brand/assets")
|
||||
async def upload_earth_brand_asset(
|
||||
file: UploadFile = File(...),
|
||||
_current_user: User = Depends(get_current_user),
|
||||
):
|
||||
original_name = file.filename or ""
|
||||
extension = Path(original_name).suffix.lower()
|
||||
if extension not in ALLOWED_EARTH_BRAND_EXTENSIONS:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "unsupported_file_type",
|
||||
"message": "Only png, jpg, jpeg, webp, and svg brand assets are supported.",
|
||||
},
|
||||
)
|
||||
|
||||
content = await file.read(MAX_EARTH_BRAND_ASSET_BYTES + 1)
|
||||
if len(content) > MAX_EARTH_BRAND_ASSET_BYTES:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "file_too_large",
|
||||
"message": "Brand asset must be 3 MB or smaller.",
|
||||
},
|
||||
)
|
||||
|
||||
EARTH_BRAND_ASSET_DIR.mkdir(parents=True, exist_ok=True)
|
||||
safe_name = f"{uuid4().hex}{extension}"
|
||||
destination = EARTH_BRAND_ASSET_DIR / safe_name
|
||||
destination.write_bytes(content)
|
||||
asset_url = f"{EARTH_BRAND_ASSET_URL_PREFIX}/{safe_name}"
|
||||
return {"url": asset_url, "filename": safe_name, "content_type": file.content_type}
|
||||
|
||||
|
||||
@router.get("/about")
|
||||
async def get_earth_about(db: AsyncSession = Depends(get_db)):
|
||||
return await _get_earth_about_payload(db)
|
||||
|
||||
|
||||
@router.put("/about")
|
||||
async def update_earth_about(
|
||||
payload: EarthAboutPayload,
|
||||
_current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
normalized = _normalize_earth_about_payload(payload.model_dump())
|
||||
record = await _get_earth_about_record(db)
|
||||
if record is None:
|
||||
record = SystemSetting(category=EARTH_ABOUT_CATEGORY, payload=normalized)
|
||||
db.add(record)
|
||||
else:
|
||||
record.payload = normalized
|
||||
await db.commit()
|
||||
await db.refresh(record)
|
||||
return {"status": "updated", "about": _normalize_earth_about_payload(record.payload), "is_default": False}
|
||||
|
||||
|
||||
@router.delete("/about")
|
||||
async def reset_earth_about(
|
||||
_current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await db.execute(delete(SystemSetting).where(SystemSetting.category == EARTH_ABOUT_CATEGORY))
|
||||
await db.commit()
|
||||
return {"status": "reset", "about": _normalize_earth_about_payload(None), "is_default": True}
|
||||
|
||||
|
||||
@router.get("/oobe-status")
|
||||
async def get_earth_oobe_status(
|
||||
current_user: User | None = Depends(_get_optional_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
current_count_result = await db.execute(
|
||||
select(func.count(CollectedData.id)).where(CollectedData.is_current.is_(True))
|
||||
)
|
||||
current_record_count = int(current_count_result.scalar() or 0)
|
||||
system_result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == SYSTEM_SETTINGS_CATEGORY)
|
||||
)
|
||||
system_record = system_result.scalar_one_or_none()
|
||||
demo_mode = _is_demo_mode_enabled(system_record.payload if system_record else None)
|
||||
|
||||
datasource_count_result = await db.execute(select(func.count(DataSource.id)))
|
||||
datasource_count = int(datasource_count_result.scalar() or 0)
|
||||
active_datasource_count_result = await db.execute(
|
||||
select(func.count(DataSource.id)).where(DataSource.is_active.is_(True))
|
||||
)
|
||||
active_datasource_count = int(active_datasource_count_result.scalar() or 0)
|
||||
config_result = await db.execute(select(func.count(DataSourceConfig.id)))
|
||||
custom_config_count = int(config_result.scalar() or 0)
|
||||
|
||||
tv_payload = await get_tv_settings_payload(db)
|
||||
tv_sources = tv_payload.get("sources") if isinstance(tv_payload, dict) else []
|
||||
tv_source_count = len(tv_sources) if isinstance(tv_sources, list) else 0
|
||||
|
||||
boundary_status = get_boundary_status()
|
||||
has_core_layers = bool(boundary_status.get("ready") or boundary_status.get("available") or boundary_status.get("status") in {"ready", "built", "ok"})
|
||||
has_collected_data = current_record_count > 0
|
||||
ready = has_collected_data
|
||||
|
||||
suggestions: list[str] = []
|
||||
if demo_mode:
|
||||
suggestions.append("演示模式已开启")
|
||||
if not current_user:
|
||||
suggestions.append("登录控制台")
|
||||
if not has_collected_data:
|
||||
suggestions.append("触发数据源采集")
|
||||
if not custom_config_count:
|
||||
suggestions.append("确认采集器配置")
|
||||
if not has_core_layers:
|
||||
suggestions.append("构建或启用 Earth 图层")
|
||||
|
||||
return {
|
||||
"ready": ready,
|
||||
"demo_mode": demo_mode,
|
||||
"authenticated": current_user is not None,
|
||||
"needs_login": current_user is None and not ready and not demo_mode,
|
||||
"has_collected_data": has_collected_data,
|
||||
"has_tv_sources": tv_source_count > 0,
|
||||
"has_core_layers": has_core_layers,
|
||||
"current_record_count": current_record_count,
|
||||
"datasource_count": datasource_count,
|
||||
"active_datasource_count": active_datasource_count,
|
||||
"custom_config_count": custom_config_count,
|
||||
"tv_source_count": tv_source_count,
|
||||
"suggestions": suggestions,
|
||||
"login_url": "/login?next=/datasources",
|
||||
"datasources_url": "/datasources",
|
||||
"collection_url": "/collection-management",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/boundaries/status")
|
||||
async def get_earth_boundary_status():
|
||||
return get_boundary_status()
|
||||
|
||||
def _is_loopback_request(request: Request) -> bool:
|
||||
host = request.client.host if request.client else ""
|
||||
return host in {"127.0.0.1", "::1", "localhost"} or host.startswith("127.")
|
||||
|
||||
|
||||
def _require_local_or_user(request: Request, user: User | None) -> None:
|
||||
if user is not None or _is_loopback_request(request):
|
||||
return
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Authentication required outside localhost",
|
||||
)
|
||||
|
||||
|
||||
@router.put("/boundaries/config")
|
||||
async def update_earth_boundary_config(
|
||||
payload: EarthBoundaryConfigPayload,
|
||||
_current_user: User = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
return save_boundary_config(payload.config)
|
||||
except EarthBoundaryBuildError as exc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": exc.code, "message": str(exc), "details": exc.details},
|
||||
) from exc
|
||||
|
||||
|
||||
@router.post("/boundaries/build")
|
||||
async def build_earth_boundary_assets(
|
||||
request: Request,
|
||||
current_user: User | None = Depends(_get_optional_current_user),
|
||||
):
|
||||
_require_local_or_user(request, current_user)
|
||||
try:
|
||||
return await start_boundary_build_job()
|
||||
except EarthBoundaryBuildError as exc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": exc.code, "message": str(exc), "details": exc.details},
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get("/boundaries/build/status")
|
||||
async def get_earth_boundary_build_status():
|
||||
return get_boundary_build_status()
|
||||
190
backend/app/api/v1/interactables.py
Normal file
190
backend/app/api/v1/interactables.py
Normal file
@@ -0,0 +1,190 @@
|
||||
"""CRUD APIs for persistent Earth interactables."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.security import get_current_user
|
||||
from app.db.session import get_db
|
||||
from app.models.earth_interactable import EarthInteractable
|
||||
from app.models.user import User
|
||||
from app.services.earth_interactables import (
|
||||
build_interactable_event,
|
||||
interactables_to_geojson,
|
||||
invalidate_interactable_cache,
|
||||
list_interactables,
|
||||
normalize_interactable_id,
|
||||
publish_interactable_event,
|
||||
serialize_interactable,
|
||||
)
|
||||
from app.services.earth_layer_cache import (
|
||||
EarthLayerCachePolicy,
|
||||
earth_layer_cache,
|
||||
get_or_build_layer_payload,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
INTERACTABLE_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
fresh_ttl_seconds=60,
|
||||
stale_ttl_seconds=10 * 60,
|
||||
max_features=5000,
|
||||
)
|
||||
|
||||
|
||||
class InteractableCreate(BaseModel):
|
||||
id: str | None = Field(default=None, max_length=160)
|
||||
layer: str = Field(default="default", min_length=1, max_length=80)
|
||||
kind: str = Field(default="default", min_length=1, max_length=80)
|
||||
label: str = Field(default="", max_length=255)
|
||||
description: str = Field(default="", max_length=4000)
|
||||
latitude: float = Field(ge=-90, le=90)
|
||||
longitude: float = Field(ge=-180, le=180)
|
||||
altitude: float | None = None
|
||||
properties: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("layer", "kind")
|
||||
@classmethod
|
||||
def normalize_key(cls, value: str) -> str:
|
||||
normalized = str(value or "").strip()
|
||||
if not normalized:
|
||||
raise ValueError("must not be empty")
|
||||
return normalized
|
||||
|
||||
|
||||
class InteractableUpdate(BaseModel):
|
||||
layer: str | None = Field(default=None, min_length=1, max_length=80)
|
||||
kind: str | None = Field(default=None, min_length=1, max_length=80)
|
||||
label: str | None = Field(default=None, max_length=255)
|
||||
description: str | None = Field(default=None, max_length=4000)
|
||||
latitude: float | None = Field(default=None, ge=-90, le=90)
|
||||
longitude: float | None = Field(default=None, ge=-180, le=180)
|
||||
altitude: float | None = None
|
||||
properties: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def get_interactables(
|
||||
response: Response,
|
||||
layer: str | None = Query(default=None),
|
||||
include_deleted: bool = Query(default=False),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
items = await list_interactables(db, layer=layer, include_deleted=include_deleted)
|
||||
response.headers["X-Planet-Interactables-Count"] = str(len(items))
|
||||
return {"items": [serialize_interactable(item) for item in items]}
|
||||
|
||||
|
||||
@router.get("/geojson")
|
||||
async def get_interactables_geojson(
|
||||
response: Response,
|
||||
layer: str | None = Query(default=None),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
async def build_payload() -> dict[str, Any]:
|
||||
items = await list_interactables(db, layer=layer)
|
||||
return interactables_to_geojson(items)
|
||||
|
||||
payload = await get_or_build_layer_payload(
|
||||
key=earth_layer_cache.key("interactables", layer=layer or "all"),
|
||||
policy=INTERACTABLE_CACHE_POLICY,
|
||||
builder=build_payload,
|
||||
response=response,
|
||||
)
|
||||
response.headers["X-Planet-Interactables-Count"] = str(len(payload.get("features") or []))
|
||||
return payload
|
||||
|
||||
|
||||
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||
async def create_interactable(
|
||||
payload: InteractableCreate,
|
||||
_current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
record_id = normalize_interactable_id(payload.id)
|
||||
existing = await db.get(EarthInteractable, record_id)
|
||||
if existing and not existing.is_deleted:
|
||||
raise HTTPException(status_code=409, detail="Interactable already exists")
|
||||
|
||||
if existing is None:
|
||||
record = EarthInteractable(id=record_id)
|
||||
db.add(record)
|
||||
else:
|
||||
record = existing
|
||||
record.is_deleted = False
|
||||
record.deleted_at = None
|
||||
record.revision += 1
|
||||
|
||||
record.layer = payload.layer
|
||||
record.kind = payload.kind
|
||||
record.label = payload.label
|
||||
record.description = payload.description
|
||||
record.latitude = payload.latitude
|
||||
record.longitude = payload.longitude
|
||||
record.altitude = payload.altitude
|
||||
record.properties = payload.properties
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(record)
|
||||
invalidate_interactable_cache(record.layer)
|
||||
await publish_interactable_event("created", record)
|
||||
return {"item": serialize_interactable(record)}
|
||||
|
||||
|
||||
@router.get("/{interactable_id}")
|
||||
async def get_interactable(interactable_id: str, db: AsyncSession = Depends(get_db)):
|
||||
record = await db.get(EarthInteractable, interactable_id)
|
||||
if record is None or record.is_deleted:
|
||||
raise HTTPException(status_code=404, detail="Interactable not found")
|
||||
return {"item": serialize_interactable(record)}
|
||||
|
||||
|
||||
@router.patch("/{interactable_id}")
|
||||
async def update_interactable(
|
||||
interactable_id: str,
|
||||
payload: InteractableUpdate,
|
||||
_current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
record = await db.get(EarthInteractable, interactable_id)
|
||||
if record is None or record.is_deleted:
|
||||
raise HTTPException(status_code=404, detail="Interactable not found")
|
||||
|
||||
previous_layer = record.layer
|
||||
patch = payload.model_dump(exclude_unset=True)
|
||||
for key, value in patch.items():
|
||||
setattr(record, key, value)
|
||||
record.revision += 1
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(record)
|
||||
invalidate_interactable_cache(previous_layer)
|
||||
if record.layer != previous_layer:
|
||||
invalidate_interactable_cache(record.layer)
|
||||
await publish_interactable_event("updated", record)
|
||||
return {"item": serialize_interactable(record)}
|
||||
|
||||
|
||||
@router.delete("/{interactable_id}")
|
||||
async def delete_interactable(
|
||||
interactable_id: str,
|
||||
_current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
record = await db.get(EarthInteractable, interactable_id)
|
||||
if record is None or record.is_deleted:
|
||||
raise HTTPException(status_code=404, detail="Interactable not found")
|
||||
|
||||
record.is_deleted = True
|
||||
record.deleted_at = datetime.now(UTC)
|
||||
record.revision += 1
|
||||
await db.commit()
|
||||
await db.refresh(record)
|
||||
invalidate_interactable_cache(record.layer)
|
||||
await publish_interactable_event("deleted", record)
|
||||
event = build_interactable_event(action="deleted", record=record, include_item=True)
|
||||
return {"deleted": True, "event": event}
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.v1.visualization import (
|
||||
@@ -110,6 +110,7 @@ async def get_vessel_layer_snapshot(
|
||||
vessel_type: Optional[str] = Query(None, alias="type"),
|
||||
since_minutes: int = Query(60, ge=1, le=1440),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
response: Response = None,
|
||||
):
|
||||
parsed_bbox = _parse_layer_bbox(bbox)
|
||||
return await build_vessel_snapshot_response(
|
||||
@@ -119,6 +120,7 @@ async def get_vessel_layer_snapshot(
|
||||
limit=limit,
|
||||
type_filter=vessel_type,
|
||||
since_minutes=since_minutes,
|
||||
response=response,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from fastapi import APIRouter, Query
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.services.earth_news import get_earth_news_payload
|
||||
|
||||
router = APIRouter()
|
||||
@@ -9,5 +11,6 @@ router = APIRouter()
|
||||
async def get_earth_feed(
|
||||
lat: float | None = Query(None, description="Current Earth view center latitude"),
|
||||
lon: float | None = Query(None, description="Current Earth view center longitude"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await get_earth_news_payload(lat=lat, lon=lon)
|
||||
return await get_earth_news_payload(lat=lat, lon=lon, db=db)
|
||||
|
||||
@@ -4,17 +4,26 @@ import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
import httpx
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
from dotenv import dotenv_values
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.security import get_current_user
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.ai_tasks.prompts import (
|
||||
get_effective_prompt,
|
||||
list_effective_prompts,
|
||||
reset_prompt_override,
|
||||
save_prompt_override,
|
||||
serialize_effective_prompt,
|
||||
)
|
||||
from app.db.session import get_db
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
@@ -57,8 +66,14 @@ from app.services.llm_provider_catalog import (
|
||||
)
|
||||
from app.services.scheduler import sync_datasource_job
|
||||
from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings
|
||||
from app.services.persistent_logs import record_audit_log
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
|
||||
router = APIRouter()
|
||||
logger = get_logger(__name__, service="api")
|
||||
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5
|
||||
AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test"
|
||||
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
"system": {
|
||||
@@ -67,6 +82,7 @@ DEFAULT_SETTINGS = {
|
||||
"auto_refresh": True,
|
||||
"data_retention_days": 30,
|
||||
"max_concurrent_tasks": 5,
|
||||
"demo_mode": False,
|
||||
},
|
||||
"notifications": {
|
||||
"email_enabled": False,
|
||||
@@ -121,12 +137,77 @@ DEFAULT_SETTINGS = {
|
||||
}
|
||||
|
||||
|
||||
def _user_role_value(user: User) -> str:
|
||||
role = getattr(user, "role", "")
|
||||
return role.value if hasattr(role, "value") else str(role or "")
|
||||
|
||||
|
||||
def _user_display_name(user: User) -> str | None:
|
||||
return getattr(user, "username", None) or getattr(user, "email", None)
|
||||
|
||||
|
||||
def _request_client_ip(request: Request | None) -> str | None:
|
||||
if request is None or request.client is None:
|
||||
return None
|
||||
return request.client.host
|
||||
|
||||
|
||||
def _can_reveal_integration_secrets(user: User) -> bool:
|
||||
return _user_role_value(user) in SECRET_REVEAL_ROLES
|
||||
|
||||
|
||||
async def _record_integration_secret_reveal(
|
||||
*,
|
||||
current_user: User,
|
||||
request: Request | None,
|
||||
target_id: str,
|
||||
result: str,
|
||||
details: dict,
|
||||
) -> None:
|
||||
await record_audit_log(
|
||||
action="settings.integration_secret.reveal",
|
||||
actor_id=getattr(current_user, "id", None),
|
||||
actor_name=_user_display_name(current_user),
|
||||
target_type="integration_secret",
|
||||
target_id=target_id,
|
||||
result=result,
|
||||
ip=_request_client_ip(request),
|
||||
details=details,
|
||||
)
|
||||
|
||||
|
||||
async def _ensure_secret_reveal_allowed(
|
||||
*,
|
||||
current_user: User,
|
||||
request: Request | None,
|
||||
target_id: str,
|
||||
details: dict | None = None,
|
||||
) -> None:
|
||||
if _can_reveal_integration_secrets(current_user):
|
||||
return
|
||||
await _record_integration_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=target_id,
|
||||
result="denied",
|
||||
details={
|
||||
**(details or {}),
|
||||
"role": _user_role_value(current_user),
|
||||
},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Only administrators can reveal integration secrets",
|
||||
)
|
||||
|
||||
|
||||
class SystemSettingsUpdate(BaseModel):
|
||||
system_name: str = "智能星球"
|
||||
refresh_interval: int = Field(default=60, ge=10, le=3600)
|
||||
auto_refresh: bool = True
|
||||
data_retention_days: int = Field(default=30, ge=1, le=3650)
|
||||
max_concurrent_tasks: int = Field(default=5, ge=1, le=50)
|
||||
demo_mode: bool = False
|
||||
|
||||
|
||||
class NotificationSettingsUpdate(BaseModel):
|
||||
@@ -250,6 +331,11 @@ class OCRIntegrationUpdate(BaseModel):
|
||||
output_format: str = Field(default="markdown", pattern="^(markdown|text|json)$")
|
||||
|
||||
|
||||
class AIPromptUpdate(BaseModel):
|
||||
system_prompt: str = Field(default="", max_length=8000)
|
||||
prompt: str = Field(min_length=1, max_length=20000)
|
||||
|
||||
|
||||
class ExternalIntegrationsUpdate(BaseModel):
|
||||
ai_provider: AIProviderIntegrationUpdate
|
||||
barentswatch: BarentsWatchIntegrationUpdate
|
||||
@@ -339,9 +425,10 @@ def _get_provider_preset(provider: str) -> dict:
|
||||
"provider": provider,
|
||||
"provider_api": "openai-completions",
|
||||
"base_url": "",
|
||||
"model": "",
|
||||
"models": [],
|
||||
"api_key_env": "",
|
||||
"model": "",
|
||||
"models": [],
|
||||
"model_provider_apis": {},
|
||||
"api_key_env": "",
|
||||
}
|
||||
|
||||
|
||||
@@ -363,6 +450,9 @@ def _resolve_env_secret(*names: str) -> tuple[str, str]:
|
||||
value = env_file_values.get(name)
|
||||
if value:
|
||||
return value, "env_file"
|
||||
value = os.environ.get(name)
|
||||
if value:
|
||||
return value, "env"
|
||||
return "", ""
|
||||
|
||||
|
||||
@@ -405,9 +495,16 @@ def _provider_defaults(provider: str) -> dict:
|
||||
1200 if preset.get("provider_api") == "anthropic-messages" else 4096
|
||||
),
|
||||
"anthropic_version": "2023-06-01",
|
||||
"model_provider_apis": preset.get("model_provider_apis") or {},
|
||||
}
|
||||
|
||||
|
||||
def _selected_ai_env_provider() -> str:
|
||||
env_file_values = _read_ai_provider_env_file()
|
||||
provider = env_file_values.get("AI_PROVIDER") or os.environ.get("AI_PROVIDER") or "minimax"
|
||||
return _normalize_provider_id(provider)
|
||||
|
||||
|
||||
def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict:
|
||||
raw = dict(ai_payload or {})
|
||||
default_provider = _normalize_provider_id(raw.get("default_provider") or raw.get("provider"))
|
||||
@@ -426,6 +523,7 @@ def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict:
|
||||
"api_key",
|
||||
"max_tokens",
|
||||
"anthropic_version",
|
||||
"model_provider_apis",
|
||||
)
|
||||
if raw.get(key) not in (None, "")
|
||||
}
|
||||
@@ -463,7 +561,12 @@ def _resolve_provider_api_key(provider: str, provider_config: dict) -> tuple[str
|
||||
return str(saved_key), "runtime"
|
||||
preset = _get_provider_preset(provider)
|
||||
api_key_env = preset.get("api_key_env") or ""
|
||||
return _resolve_env_secret(api_key_env, "AI_API_KEY")
|
||||
value, source = _resolve_env_secret(api_key_env)
|
||||
if value:
|
||||
return value, source
|
||||
if _normalize_provider_id(provider) == _selected_ai_env_provider():
|
||||
return _resolve_env_secret("AI_API_KEY")
|
||||
return "", ""
|
||||
|
||||
|
||||
def _resolve_service_token(ai_payload: dict) -> tuple[str, str]:
|
||||
@@ -484,12 +587,23 @@ def _is_secret_placeholder(value: Optional[str], current_preview: str = "") -> b
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return True
|
||||
return text == current_preview or text.startswith("••••") or "*" in text
|
||||
if text == current_preview or text.startswith("••••"):
|
||||
return True
|
||||
if "-" in text:
|
||||
_prefix, masked = text.split("-", 1)
|
||||
if masked and all(char in {"*", "•", " ", "\t"} for char in masked):
|
||||
return True
|
||||
return all(char in {"*", "•", " ", "\t"} for char in text)
|
||||
|
||||
|
||||
def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrationUpdate) -> dict:
|
||||
current_ai = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
|
||||
provider_id = _normalize_provider_id(update.default_provider or update.provider)
|
||||
provider_id = _normalize_provider_id(update.provider)
|
||||
default_provider = (
|
||||
_normalize_provider_id(update.default_provider)
|
||||
if update.default_provider is not None
|
||||
else current_ai["default_provider"]
|
||||
)
|
||||
current_providers = {
|
||||
provider: dict(config or {})
|
||||
for provider, config in current_ai.get("providers", {}).items()
|
||||
@@ -523,7 +637,7 @@ def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrat
|
||||
"service_url": update.service_url.strip()
|
||||
or app_settings.AI_PROVIDER_SERVICE_URL,
|
||||
"service_token": current_ai.get("service_token") or "",
|
||||
"default_provider": provider_id,
|
||||
"default_provider": default_provider,
|
||||
"providers": current_providers,
|
||||
"timeout_seconds": update.timeout_seconds,
|
||||
"retry_attempts": update.retry_attempts,
|
||||
@@ -557,10 +671,240 @@ def _runtime_config_from_ai_payload(ai_payload: dict) -> dict:
|
||||
"api_key": api_key,
|
||||
"max_tokens": int(provider_config.get("max_tokens") or 1200),
|
||||
"anthropic_version": provider_config.get("anthropic_version") or "2023-06-01",
|
||||
"model_provider_apis": provider_config.get("model_provider_apis") or {},
|
||||
"preset_models": _get_provider_preset(default_provider).get("models") or [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _ai_provider_runtime_fingerprint(ai_payload: dict) -> dict:
|
||||
runtime_config = _runtime_config_from_ai_payload(ai_payload)
|
||||
llm_config = runtime_config.get("llm_config") or {}
|
||||
return {
|
||||
"service_url": runtime_config.get("service_url") or "",
|
||||
"service_token": runtime_config.get("service_token") or "",
|
||||
"timeout_seconds": int(runtime_config.get("timeout_seconds") or 0),
|
||||
"retry_attempts": int(runtime_config.get("retry_attempts") or 0),
|
||||
"provider": llm_config.get("provider") or "",
|
||||
"provider_api": llm_config.get("provider_api") or "",
|
||||
"base_url": llm_config.get("base_url") or "",
|
||||
"model": llm_config.get("model") or "",
|
||||
"api_key": llm_config.get("api_key") or "",
|
||||
"max_tokens": int(llm_config.get("max_tokens") or 0),
|
||||
"anthropic_version": llm_config.get("anthropic_version") or "",
|
||||
}
|
||||
|
||||
|
||||
async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict:
|
||||
runtime_config = _runtime_config_from_ai_payload(ai_payload)
|
||||
client = AIProviderClient(
|
||||
service_url=runtime_config["service_url"],
|
||||
service_token=runtime_config["service_token"],
|
||||
timeout=runtime_config["timeout_seconds"],
|
||||
retry_attempts=runtime_config["retry_attempts"],
|
||||
llm_config=runtime_config.get("llm_config") or {},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.full_connection.start",
|
||||
message="AI provider full connection validation started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
context={
|
||||
"provider": runtime_config.get("llm_config", {}).get("provider"),
|
||||
"model": runtime_config.get("llm_config", {}).get("model"),
|
||||
},
|
||||
)
|
||||
status_result = await client.get_status()
|
||||
if not status_result.configured:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="AI Provider 可访问,但当前 provider/model/key 未完整配置。",
|
||||
)
|
||||
prompt = await get_effective_prompt(None, AI_CONNECTION_TEST_PROMPT_KEY)
|
||||
analysis_result = await client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title="保存前完整连接测试",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=["这是保存 AI Provider 配置前的完整 LLM 调用测试。"],
|
||||
constraints=["回复尽量简短。"],
|
||||
)
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.full_connection.success",
|
||||
message="AI provider full connection validation completed",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
context={
|
||||
"provider": analysis_result.provider,
|
||||
"model": analysis_result.model,
|
||||
"configured": status_result.configured,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"status": status_result.model_dump(),
|
||||
"provider": analysis_result.provider,
|
||||
"model": analysis_result.model,
|
||||
}
|
||||
|
||||
|
||||
def _join_provider_url(base_url: str, path: str) -> str:
|
||||
return f"{base_url.rstrip('/')}/{path.lstrip('/')}"
|
||||
|
||||
|
||||
def _extract_model_ids(payload: dict) -> list[str]:
|
||||
data = payload.get("data") if isinstance(payload, dict) else None
|
||||
if isinstance(data, list):
|
||||
return [
|
||||
str(item.get("id"))
|
||||
for item in data
|
||||
if isinstance(item, dict) and item.get("id")
|
||||
]
|
||||
models = payload.get("models") if isinstance(payload, dict) else None
|
||||
if isinstance(models, list):
|
||||
return [
|
||||
str(item.get("name") or item.get("model") or item.get("id") or item)
|
||||
for item in models
|
||||
if item
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
def _contains_model(model_ids: list[str], model: str) -> bool:
|
||||
normalized_model = model.strip().lower()
|
||||
return any(str(item).strip().lower() == normalized_model for item in model_ids)
|
||||
|
||||
|
||||
async def _check_ai_provider_lightweight(llm_config: dict, timeout_seconds: int) -> dict:
|
||||
provider = _normalize_provider_id(llm_config.get("provider") or "")
|
||||
configured_api = str(llm_config.get("provider_api") or "").strip() or "openai-completions"
|
||||
model = str(llm_config.get("model") or "").strip()
|
||||
base_url = str(llm_config.get("base_url") or "").strip().rstrip("/")
|
||||
api_key = str(llm_config.get("api_key") or "").strip()
|
||||
provider_api = configured_api
|
||||
model_provider_apis = llm_config.get("model_provider_apis")
|
||||
if isinstance(model_provider_apis, dict):
|
||||
provider_api = str(model_provider_apis.get(model) or provider_api)
|
||||
preset_models = [
|
||||
str(item)
|
||||
for item in (llm_config.get("preset_models") or [])
|
||||
if str(item).strip()
|
||||
]
|
||||
|
||||
if not provider or not base_url or not model:
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": "当前 provider/base_url/model 未完整配置。",
|
||||
"mode": "lightweight_config",
|
||||
}
|
||||
if provider_api != "ollama-generate" and not api_key:
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": "当前 provider 未配置 API Key。",
|
||||
"mode": "lightweight_config",
|
||||
}
|
||||
|
||||
if provider == "opencode-go":
|
||||
url = _join_provider_url(base_url, "/models")
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
elif provider_api == "ollama-generate":
|
||||
url = _join_provider_url(base_url, "/api/tags")
|
||||
headers: dict[str, str] = {}
|
||||
elif provider_api == "openai-completions":
|
||||
url = _join_provider_url(base_url, "/models")
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
elif provider_api == "anthropic-messages":
|
||||
url = _join_provider_url(base_url, "/models")
|
||||
headers = {
|
||||
"x-api-key": api_key,
|
||||
"anthropic-version": str(llm_config.get("anthropic_version") or "2023-06-01"),
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": f"当前 provider_api 不支持轻量连通性测试: {provider_api}",
|
||||
"mode": "lightweight_unsupported",
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=min(timeout_seconds, AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS)) as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
detail = exc.response.text or exc.response.reason_phrase
|
||||
if exc.response.status_code == 404 and _contains_model(preset_models, model):
|
||||
return {
|
||||
"success": True,
|
||||
"connected": True,
|
||||
"message": "轻量连通性测试通过;当前 provider 不提供可用的模型目录,已按内置模型预设确认。",
|
||||
"mode": "lightweight_preset",
|
||||
"provider": provider,
|
||||
"provider_api": provider_api,
|
||||
"model": model,
|
||||
"url": url,
|
||||
}
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": f"轻量连通性测试失败: HTTP {exc.response.status_code} {detail}",
|
||||
"mode": "lightweight_models",
|
||||
"url": url,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": f"轻量连通性测试失败: {exc}",
|
||||
"mode": "lightweight_models",
|
||||
"url": url,
|
||||
}
|
||||
|
||||
model_ids = _extract_model_ids(payload)
|
||||
if model_ids and not _contains_model(model_ids, model):
|
||||
if _contains_model(preset_models, model):
|
||||
return {
|
||||
"success": True,
|
||||
"connected": True,
|
||||
"message": "轻量连通性测试通过;provider 模型目录未返回当前别名,已按内置模型预设确认。",
|
||||
"mode": "lightweight_models_with_preset_alias",
|
||||
"provider": provider,
|
||||
"provider_api": provider_api,
|
||||
"model": model,
|
||||
"models_count": len(model_ids),
|
||||
"url": url,
|
||||
}
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": f"连接可用,但模型目录中没有当前模型: {model}",
|
||||
"mode": "lightweight_models",
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"models_count": len(model_ids),
|
||||
"url": url,
|
||||
}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"connected": True,
|
||||
"message": "轻量连通性测试通过",
|
||||
"mode": "lightweight_models",
|
||||
"provider": provider,
|
||||
"provider_api": provider_api,
|
||||
"model": model,
|
||||
"models_count": len(model_ids),
|
||||
"url": url,
|
||||
}
|
||||
|
||||
|
||||
def _web_search_provider_defaults(provider: str) -> dict:
|
||||
return web_search_provider_defaults(provider).model_dump()
|
||||
|
||||
@@ -621,12 +965,21 @@ def _normalize_web_search_payload(web_search_payload: dict | None) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _resolve_web_search_api_key(provider: str, provider_config: dict) -> tuple[str, str]:
|
||||
def _resolve_web_search_api_key(
|
||||
provider: str,
|
||||
provider_config: dict,
|
||||
default_provider: str | None = None,
|
||||
) -> tuple[str, str]:
|
||||
saved_key = provider_config.get("api_key") or ""
|
||||
if saved_key:
|
||||
return str(saved_key), "runtime"
|
||||
preset = get_web_search_provider_preset(provider)
|
||||
return _resolve_web_search_env_secret(preset.get("api_key_env") or "", "WEB_SEARCH_API_KEY")
|
||||
value, source = _resolve_web_search_env_secret(preset.get("api_key_env") or "")
|
||||
if value:
|
||||
return value, source
|
||||
if normalize_web_search_provider(provider) == normalize_web_search_provider(default_provider or "tavily"):
|
||||
return _resolve_web_search_env_secret("WEB_SEARCH_API_KEY")
|
||||
return "", ""
|
||||
|
||||
|
||||
def _build_web_search_payload(
|
||||
@@ -636,13 +989,22 @@ def _build_web_search_payload(
|
||||
current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {})
|
||||
if update is None:
|
||||
return current_web_search
|
||||
provider_id = normalize_web_search_provider(update.default_provider or update.provider)
|
||||
provider_id = normalize_web_search_provider(update.provider)
|
||||
default_provider = (
|
||||
normalize_web_search_provider(update.default_provider)
|
||||
if update.default_provider is not None
|
||||
else current_web_search["default_provider"]
|
||||
)
|
||||
current_providers = {
|
||||
provider: dict(config or {})
|
||||
for provider, config in current_web_search.get("providers", {}).items()
|
||||
}
|
||||
current_provider = current_providers.get(provider_id) or _web_search_provider_defaults(provider_id)
|
||||
current_key, current_key_source = _resolve_web_search_api_key(provider_id, current_provider)
|
||||
current_key, current_key_source = _resolve_web_search_api_key(
|
||||
provider_id,
|
||||
current_provider,
|
||||
current_web_search["default_provider"],
|
||||
)
|
||||
current_key_preview = _mask_secret(current_key, current_key_source)["preview"]
|
||||
provider_payload = {
|
||||
**_web_search_provider_defaults(provider_id),
|
||||
@@ -675,7 +1037,7 @@ def _build_web_search_payload(
|
||||
current_providers[provider_id] = provider_payload
|
||||
return {
|
||||
"enabled": update.enabled,
|
||||
"default_provider": provider_id,
|
||||
"default_provider": default_provider,
|
||||
"providers": current_providers,
|
||||
}
|
||||
|
||||
@@ -684,12 +1046,12 @@ def _runtime_config_from_web_search_payload(web_search_payload: dict) -> WebSear
|
||||
normalized = _normalize_web_search_payload(web_search_payload)
|
||||
provider_id = normalized["default_provider"]
|
||||
provider_config = normalized["providers"].get(provider_id) or _web_search_provider_defaults(provider_id)
|
||||
api_key, _source = _resolve_web_search_api_key(provider_id, provider_config)
|
||||
api_key, _source = _resolve_web_search_api_key(provider_id, provider_config, provider_id)
|
||||
provider_models = {
|
||||
provider: WebSearchProviderConfig(**{
|
||||
**config,
|
||||
"api_key": (
|
||||
api_key if provider == provider_id else _resolve_web_search_api_key(provider, config)[0]
|
||||
api_key if provider == provider_id else _resolve_web_search_api_key(provider, config, provider_id)[0]
|
||||
),
|
||||
})
|
||||
for provider, config in normalized["providers"].items()
|
||||
@@ -823,7 +1185,11 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
|
||||
normalized_web_search["providers"].get(provider_id)
|
||||
or _web_search_provider_defaults(provider_id)
|
||||
)
|
||||
api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config)
|
||||
api_key, api_key_source = _resolve_web_search_api_key(
|
||||
provider_id,
|
||||
provider_config,
|
||||
normalized_web_search["default_provider"],
|
||||
)
|
||||
web_search_providers_payload[provider_id] = {
|
||||
**{
|
||||
key: value
|
||||
@@ -1158,6 +1524,47 @@ async def get_external_integrations(
|
||||
return {"integrations": await serialize_external_integrations(db)}
|
||||
|
||||
|
||||
@router.get("/ai-prompts")
|
||||
async def get_ai_prompts(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
prompts = await list_effective_prompts(db)
|
||||
return {"data": [serialize_effective_prompt(prompt) for prompt in prompts]}
|
||||
|
||||
|
||||
@router.put("/ai-prompts/{task_key}")
|
||||
async def update_ai_prompt(
|
||||
task_key: str,
|
||||
payload: AIPromptUpdate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
try:
|
||||
prompt = await save_prompt_override(
|
||||
db,
|
||||
task_key,
|
||||
system_prompt=payload.system_prompt,
|
||||
prompt=payload.prompt,
|
||||
)
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=404, detail="未知 AI 提示词任务") from None
|
||||
return {"data": serialize_effective_prompt(prompt)}
|
||||
|
||||
|
||||
@router.post("/ai-prompts/{task_key}/reset")
|
||||
async def reset_ai_prompt(
|
||||
task_key: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
try:
|
||||
prompt = await reset_prompt_override(db, task_key)
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=404, detail="未知 AI 提示词任务") from None
|
||||
return {"data": serialize_effective_prompt(prompt)}
|
||||
|
||||
|
||||
@router.get("/integrations/barentswatch/connectivity")
|
||||
async def get_barentswatch_connectivity(
|
||||
current_user: User = Depends(get_current_user),
|
||||
@@ -1217,48 +1624,83 @@ async def connect_ai_provider_integration(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
# Connection testing should validate the provider being edited, not the
|
||||
# currently saved default provider. This is a transient draft only and is
|
||||
# intentionally not persisted.
|
||||
payload = payload.model_copy(update={"default_provider": payload.provider})
|
||||
draft_ai_payload = _build_ai_provider_payload(current_payload, payload)
|
||||
runtime_config = _runtime_config_from_ai_payload(draft_ai_payload)
|
||||
quick_llm_config = {
|
||||
**(runtime_config.get("llm_config") or {}),
|
||||
"max_tokens": 1,
|
||||
}
|
||||
client = AIProviderClient(
|
||||
service_url=runtime_config["service_url"],
|
||||
service_token=runtime_config["service_token"],
|
||||
timeout=runtime_config["timeout_seconds"],
|
||||
retry_attempts=runtime_config["retry_attempts"],
|
||||
llm_config=runtime_config.get("llm_config") or {},
|
||||
timeout=min(int(runtime_config["timeout_seconds"] or 60), AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS),
|
||||
retry_attempts=1,
|
||||
llm_config=quick_llm_config,
|
||||
)
|
||||
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.connect.start",
|
||||
message="AI provider connection test started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={
|
||||
"provider": payload.provider,
|
||||
"model": payload.model,
|
||||
"timeout_seconds": min(int(runtime_config["timeout_seconds"] or 60), AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS),
|
||||
},
|
||||
)
|
||||
try:
|
||||
status_result = await client.get_status()
|
||||
if not status_result.configured:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.connect.failed",
|
||||
message="AI provider connection test failed because provider is incomplete",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"provider": payload.provider, "model": payload.model, "configured": False},
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": "AI Provider 可访问,但当前 provider/model/key 未完整配置。",
|
||||
"status": status_result.model_dump(),
|
||||
}
|
||||
analysis_result = await client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title="连接测试",
|
||||
objective="请用一句话回复连接可用。",
|
||||
observations=["这是配置中心发起的 LLM 连接测试。"],
|
||||
constraints=["回复尽量简短。"],
|
||||
)
|
||||
lightweight_result = await _check_ai_provider_lightweight(
|
||||
quick_llm_config,
|
||||
timeout_seconds=min(
|
||||
int(runtime_config["timeout_seconds"] or 60),
|
||||
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS,
|
||||
),
|
||||
)
|
||||
current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {})
|
||||
current_ocr = _normalize_ocr_payload(current_payload.get("ocr") or {})
|
||||
await save_setting_payload(
|
||||
db,
|
||||
"external_integrations",
|
||||
{"ai_provider": draft_ai_payload, "web_search": current_web_search, "ocr": current_ocr},
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.connect.success",
|
||||
message="AI provider connection test completed",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={
|
||||
"provider": payload.provider,
|
||||
"model": payload.model,
|
||||
"configured": True,
|
||||
"lightweight_status": lightweight_result.get("status"),
|
||||
},
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"connected": True,
|
||||
"message": "AI Provider 连接成功,已保存为全局默认配置。",
|
||||
**lightweight_result,
|
||||
"status": status_result.model_dump(),
|
||||
"provider": analysis_result.provider,
|
||||
"model": analysis_result.model,
|
||||
"integrations": await serialize_external_integrations(db),
|
||||
}
|
||||
except HTTPException as exc:
|
||||
return {
|
||||
@@ -1267,6 +1709,17 @@ async def connect_ai_provider_integration(
|
||||
"message": str(exc.detail),
|
||||
}
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.connect.failed",
|
||||
message="AI provider connection test failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": payload.provider, "model": payload.model}),
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
@@ -1276,16 +1729,38 @@ async def connect_ai_provider_integration(
|
||||
|
||||
@router.get("/integrations/ai-provider/secrets")
|
||||
async def reveal_ai_provider_secrets(
|
||||
request: Request,
|
||||
provider: str = Query(default=""),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
requested_provider = _normalize_provider_id(provider) if provider else "default"
|
||||
await _ensure_secret_reveal_allowed(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=f"ai_provider:{requested_provider}",
|
||||
details={"kind": "ai_provider", "provider": requested_provider},
|
||||
)
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
|
||||
provider_id = _normalize_provider_id(provider or ai_payload["default_provider"])
|
||||
provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id)
|
||||
api_key, api_key_source = _resolve_provider_api_key(provider_id, provider_config)
|
||||
service_token, service_token_source = _resolve_service_token(ai_payload)
|
||||
await _record_integration_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=f"ai_provider:{provider_id}",
|
||||
result="success",
|
||||
details={
|
||||
"kind": "ai_provider",
|
||||
"provider": provider_id,
|
||||
"api_key_configured": bool(api_key),
|
||||
"api_key_source": api_key_source,
|
||||
"service_token_configured": bool(service_token),
|
||||
"service_token_source": service_token_source,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"provider": provider_id,
|
||||
"api_key": api_key,
|
||||
@@ -1304,10 +1779,18 @@ async def get_web_search_presets(
|
||||
|
||||
@router.get("/integrations/web-search/secrets")
|
||||
async def reveal_web_search_secrets(
|
||||
request: Request,
|
||||
provider: str = Query(default=""),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
requested_provider = normalize_web_search_provider(provider) if provider else "default"
|
||||
await _ensure_secret_reveal_allowed(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=f"web_search:{requested_provider}",
|
||||
details={"kind": "web_search", "provider": requested_provider},
|
||||
)
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
web_search_payload = _normalize_web_search_payload(current_payload.get("web_search") or {})
|
||||
provider_id = normalize_web_search_provider(provider or web_search_payload["default_provider"])
|
||||
@@ -1315,7 +1798,23 @@ async def reveal_web_search_secrets(
|
||||
web_search_payload["providers"].get(provider_id)
|
||||
or _web_search_provider_defaults(provider_id)
|
||||
)
|
||||
api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config)
|
||||
api_key, api_key_source = _resolve_web_search_api_key(
|
||||
provider_id,
|
||||
provider_config,
|
||||
web_search_payload["default_provider"],
|
||||
)
|
||||
await _record_integration_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=f"web_search:{provider_id}",
|
||||
result="success",
|
||||
details={
|
||||
"kind": "web_search",
|
||||
"provider": provider_id,
|
||||
"api_key_configured": bool(api_key),
|
||||
"api_key_source": api_key_source,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"provider": provider_id,
|
||||
"api_key": api_key,
|
||||
@@ -1325,12 +1824,31 @@ async def reveal_web_search_secrets(
|
||||
|
||||
@router.get("/integrations/ocr/secrets")
|
||||
async def reveal_ocr_secrets(
|
||||
request: Request,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await _ensure_secret_reveal_allowed(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id="ocr:default",
|
||||
details={"kind": "ocr", "provider": "default"},
|
||||
)
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
ocr_payload = _normalize_ocr_payload(current_payload.get("ocr") or {})
|
||||
api_key, api_key_source = _resolve_ocr_api_key(ocr_payload)
|
||||
await _record_integration_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=f"ocr:{ocr_payload['provider']}",
|
||||
result="success",
|
||||
details={
|
||||
"kind": "ocr",
|
||||
"provider": ocr_payload["provider"],
|
||||
"api_key_configured": bool(api_key),
|
||||
"api_key_source": api_key_source,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"provider": ocr_payload["provider"],
|
||||
"api_key": api_key,
|
||||
@@ -1349,8 +1867,28 @@ async def connect_web_search_integration(
|
||||
runtime_config = _runtime_config_from_web_search_payload(draft_web_search_payload)
|
||||
client = WebSearchClient(runtime_config)
|
||||
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.web_search.connect.start",
|
||||
message="WebSearch connection test started",
|
||||
category="ai_tool",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"provider": runtime_config.default_provider},
|
||||
)
|
||||
try:
|
||||
results = await client.test_connection()
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.web_search.connect.success",
|
||||
message="WebSearch connection test completed",
|
||||
category="ai_tool",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"provider": runtime_config.default_provider, "result_count": len(results)},
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"connected": True,
|
||||
@@ -1359,18 +1897,51 @@ async def connect_web_search_integration(
|
||||
"results": [item.model_dump(mode="json") for item in results[:3]],
|
||||
}
|
||||
except WebSearchConfigurationError as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.web_search.connect.failed",
|
||||
message="WebSearch connection test failed because configuration is incomplete",
|
||||
category="ai_tool",
|
||||
level="warning",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": runtime_config.default_provider}),
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": str(exc),
|
||||
}
|
||||
except WebSearchError as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.web_search.connect.failed",
|
||||
message="WebSearch connection test failed",
|
||||
category="ai_tool",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": runtime_config.default_provider}),
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": str(exc),
|
||||
}
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.web_search.connect.failed",
|
||||
message="WebSearch connection test failed",
|
||||
category="ai_tool",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": runtime_config.default_provider}),
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
@@ -1398,17 +1969,60 @@ async def generate_provider_credential_guide(
|
||||
ai_client: AIProviderClient = Depends(get_ai_provider_client),
|
||||
):
|
||||
try:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.credential_guide.generate.start",
|
||||
message="Credential guide generation started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"provider": provider},
|
||||
)
|
||||
web_search_client = await get_web_search_client(db)
|
||||
return {
|
||||
"guide": await generate_credential_guide(
|
||||
guide = await generate_credential_guide(
|
||||
db,
|
||||
provider,
|
||||
ai_client,
|
||||
web_search_client,
|
||||
)
|
||||
}
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.credential_guide.generate.success",
|
||||
message="Credential guide generation completed",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"provider": provider},
|
||||
)
|
||||
return {"guide": guide}
|
||||
except ValueError as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.credential_guide.generate.failed",
|
||||
message="Credential guide generation failed",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": provider}),
|
||||
)
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.credential_guide.generate.failed",
|
||||
message="Credential guide generation failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": provider}),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@router.post("/credential-guides/{provider}/reset")
|
||||
@@ -1434,9 +2048,17 @@ async def get_ai_provider_presets(
|
||||
async def refresh_ai_provider_preset(
|
||||
provider: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
try:
|
||||
return {"data": await refresh_llm_provider_preset(provider)}
|
||||
provider_id = _normalize_provider_id(provider)
|
||||
api_key = None
|
||||
if provider_id == "opencode-go":
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
|
||||
provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id)
|
||||
api_key, _api_key_source = _resolve_provider_api_key(provider_id, provider_config)
|
||||
return {"data": await refresh_llm_provider_preset(provider_id, api_key=api_key)}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
@@ -8,9 +9,13 @@ from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import ROOT_DIR
|
||||
from app.core.security import get_current_user
|
||||
from app.db.session import get_db
|
||||
from app.models.system_log import AuditLog, SystemLog
|
||||
from app.models.user import User
|
||||
from app.services.persistent_logs import record_audit_log, record_system_log
|
||||
from app.services.system_control import (
|
||||
@@ -35,10 +40,47 @@ from app.services.system_logs import (
|
||||
normalize_log_level,
|
||||
read_log_snapshot,
|
||||
)
|
||||
from app.services.earth_layer_cache import earth_layer_cache
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _compact_log_context(context: dict | None) -> str:
|
||||
if not context:
|
||||
return ""
|
||||
allowed = {
|
||||
key: value
|
||||
for key, value in (context or {}).items()
|
||||
if key
|
||||
in {
|
||||
"status",
|
||||
"duration_ms",
|
||||
"provider",
|
||||
"model",
|
||||
"result_provider",
|
||||
"result_model",
|
||||
"collector_name",
|
||||
"datasource_id",
|
||||
"task_id",
|
||||
"snapshot_id",
|
||||
"raw_count",
|
||||
"transformed_count",
|
||||
"saved_count",
|
||||
"created",
|
||||
"updated",
|
||||
"unchanged",
|
||||
"deleted",
|
||||
"result_count",
|
||||
"status_code",
|
||||
"error_type",
|
||||
"error",
|
||||
}
|
||||
}
|
||||
if not allowed:
|
||||
return ""
|
||||
return json.dumps(allowed, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
class RestartTaskCreate(BaseModel):
|
||||
action: str
|
||||
|
||||
@@ -112,6 +154,17 @@ class EarthClientLogEventResponse(BaseModel):
|
||||
level: str
|
||||
|
||||
|
||||
class EarthLayerCacheStatusResponse(BaseModel):
|
||||
prefix: str
|
||||
key_count: int
|
||||
memory_bytes: int
|
||||
layers: dict[str, dict[str, int]]
|
||||
|
||||
|
||||
class EarthLayerCacheClearResponse(BaseModel):
|
||||
deleted: int
|
||||
|
||||
|
||||
def ensure_super_admin(current_user: User) -> None:
|
||||
if not require_super_admin(current_user.role):
|
||||
raise HTTPException(
|
||||
@@ -132,6 +185,34 @@ def validate_log_date(raw_value: str | None, field_name: str) -> str | None:
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get("/cache/earth-layers", response_model=EarthLayerCacheStatusResponse)
|
||||
async def get_earth_layer_cache_status(
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
ensure_super_admin(current_user)
|
||||
try:
|
||||
return earth_layer_cache.status()
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=f"Unable to read Earth layer cache status: {exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.delete("/cache/earth-layers", response_model=EarthLayerCacheClearResponse)
|
||||
async def clear_earth_layer_cache(
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
ensure_super_admin(current_user)
|
||||
try:
|
||||
return {"deleted": earth_layer_cache.delete_pattern()}
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=f"Unable to clear Earth layer cache: {exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.post("/restart-tasks", response_model=RestartTaskResponse)
|
||||
async def create_restart_task(
|
||||
payload: RestartTaskCreate,
|
||||
@@ -270,7 +351,123 @@ async def get_system_log_sources(
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
ensure_super_admin(current_user)
|
||||
return {"items": list_log_sources()}
|
||||
return {
|
||||
"items": [
|
||||
*list_log_sources(),
|
||||
{
|
||||
"source_id": "system-db",
|
||||
"name": "系统事件",
|
||||
"kind": "database",
|
||||
"location": "table://system_logs",
|
||||
"description": "后端持久化系统事件、AI 和采集器操作日志。",
|
||||
"category": "database",
|
||||
"status": "ok",
|
||||
},
|
||||
{
|
||||
"source_id": "audit-db",
|
||||
"name": "审计事件",
|
||||
"kind": "database",
|
||||
"location": "table://audit_logs",
|
||||
"description": "管理员敏感操作和密钥 reveal 审计记录。",
|
||||
"category": "audit",
|
||||
"status": "ok",
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
async def read_database_log_snapshot(
|
||||
source_id: str,
|
||||
*,
|
||||
limit: int,
|
||||
level: str,
|
||||
levels: str | None,
|
||||
start_date: str | None,
|
||||
end_date: str | None,
|
||||
search: str | None,
|
||||
db: AsyncSession,
|
||||
) -> dict | None:
|
||||
selected_levels = set(normalize_log_level(item) for item in (levels or level).split(",") if item.strip())
|
||||
selected_levels.discard("all")
|
||||
search_query = (search or "").strip().lower()
|
||||
lines: list[str] = []
|
||||
|
||||
if source_id == "system-db":
|
||||
query = select(SystemLog).order_by(SystemLog.occurred_at.desc().nullslast(), SystemLog.id.desc()).limit(limit * 5)
|
||||
result = await db.execute(query)
|
||||
records = result.scalars().all()
|
||||
for record in records:
|
||||
record_level = normalize_log_level(record.level)
|
||||
if selected_levels and record_level not in selected_levels:
|
||||
continue
|
||||
occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else ""
|
||||
if start_date and occurred_at and occurred_at < start_date:
|
||||
continue
|
||||
if end_date and occurred_at and occurred_at > end_date:
|
||||
continue
|
||||
line = " ".join(
|
||||
part
|
||||
for part in [
|
||||
record.occurred_at.isoformat() if record.occurred_at else "",
|
||||
record_level.upper(),
|
||||
record.source,
|
||||
record.category or "",
|
||||
record.event or "",
|
||||
f"request_id={record.request_id}" if record.request_id else "",
|
||||
record.message,
|
||||
_compact_log_context(record.context),
|
||||
]
|
||||
if part
|
||||
)
|
||||
if search_query and search_query not in line.lower():
|
||||
continue
|
||||
lines.append(line)
|
||||
elif source_id == "audit-db":
|
||||
query = select(AuditLog).order_by(AuditLog.occurred_at.desc().nullslast(), AuditLog.id.desc()).limit(limit * 5)
|
||||
result = await db.execute(query)
|
||||
records = result.scalars().all()
|
||||
for record in records:
|
||||
occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else ""
|
||||
if start_date and occurred_at and occurred_at < start_date:
|
||||
continue
|
||||
if end_date and occurred_at and occurred_at > end_date:
|
||||
continue
|
||||
line = " ".join(
|
||||
part
|
||||
for part in [
|
||||
record.occurred_at.isoformat() if record.occurred_at else "",
|
||||
"INFO",
|
||||
record.action,
|
||||
record.target_type or "",
|
||||
record.target_id or "",
|
||||
record.result or "",
|
||||
]
|
||||
if part
|
||||
)
|
||||
if search_query and search_query not in line.lower():
|
||||
continue
|
||||
lines.append(line)
|
||||
else:
|
||||
return None
|
||||
|
||||
lines = list(reversed(lines[:limit]))
|
||||
return {
|
||||
"source_id": source_id,
|
||||
"name": "系统事件" if source_id == "system-db" else "审计事件",
|
||||
"kind": "database",
|
||||
"location": "table://system_logs" if source_id == "system-db" else "table://audit_logs",
|
||||
"description": "数据库持久化日志",
|
||||
"category": "database" if source_id == "system-db" else "audit",
|
||||
"status": "ok" if lines else "empty",
|
||||
"level": level,
|
||||
"selected_levels": sorted(selected_levels),
|
||||
"search_query": search or "",
|
||||
"available_levels": ["all", "error", "warning", "info", "debug"],
|
||||
"daily_markers": [],
|
||||
"line_limit": limit,
|
||||
"line_count": len(lines),
|
||||
"lines": lines,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse)
|
||||
@@ -283,6 +480,7 @@ async def get_system_log_snapshot(
|
||||
end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"),
|
||||
search: str | None = Query(None, description="Case-insensitive substring search"),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
ensure_super_admin(current_user)
|
||||
|
||||
@@ -305,15 +503,26 @@ async def get_system_log_snapshot(
|
||||
if normalized_start_date and normalized_end_date and normalized_start_date > normalized_end_date:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="start_date must be earlier than or equal to end_date")
|
||||
|
||||
snapshot = read_log_snapshot(
|
||||
snapshot = await read_database_log_snapshot(
|
||||
source_id,
|
||||
limit,
|
||||
limit=limit,
|
||||
level=level,
|
||||
levels=levels,
|
||||
start_date=normalized_start_date,
|
||||
end_date=normalized_end_date,
|
||||
search=search,
|
||||
db=db,
|
||||
)
|
||||
if snapshot is None:
|
||||
snapshot = read_log_snapshot(
|
||||
source_id,
|
||||
limit,
|
||||
level=level,
|
||||
levels=levels,
|
||||
start_date=normalized_start_date,
|
||||
end_date=normalized_end_date,
|
||||
search=search,
|
||||
)
|
||||
if snapshot is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Log source not found")
|
||||
return snapshot
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.v1.visualization import _parse_bbox, build_vessel_snapshot_response
|
||||
@@ -23,6 +23,7 @@ async def get_vessel_snapshot(
|
||||
limit: int = Query(1000, ge=1, le=MAX_SNAPSHOT_LIMIT),
|
||||
since_minutes: int = Query(60, ge=1, le=1440),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
response: Response = None,
|
||||
):
|
||||
if not bbox:
|
||||
raise HTTPException(status_code=400, detail="bbox is required")
|
||||
@@ -36,4 +37,5 @@ async def get_vessel_snapshot(
|
||||
type_filter=type,
|
||||
limit=limit,
|
||||
since_minutes=since_minutes,
|
||||
response=response,
|
||||
)
|
||||
|
||||
@@ -38,7 +38,7 @@ from app.services.compute_center_locations import (
|
||||
upsert_compute_center_location,
|
||||
)
|
||||
from app.services.ai_client import get_ai_provider_client
|
||||
from app.api.v1.settings import get_web_search_client
|
||||
from app.api.v1.settings import get_runtime_web_search_config, get_web_search_client
|
||||
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
|
||||
from app.services.location.llm_fallback import (
|
||||
collect_llm_location_fallback_candidate,
|
||||
@@ -56,6 +56,13 @@ from app.services.vessel_ais_aggregation import (
|
||||
get_vessel_raw_observations,
|
||||
MAX_SNAPSHOT_LIMIT,
|
||||
)
|
||||
from app.services.earth_layer_cache import (
|
||||
EarthLayerCachePolicy,
|
||||
earth_layer_cache,
|
||||
format_bbox_key,
|
||||
get_or_build_layer_payload,
|
||||
quantize_bbox,
|
||||
)
|
||||
from app.core.logging import get_logger
|
||||
|
||||
router = APIRouter()
|
||||
@@ -69,6 +76,68 @@ TERRAIN_TILE_BATCH_CONCURRENCY = 16
|
||||
_terrain_tile_cache: OrderedDict[tuple[int, int, int], tuple[bytes, str, dict[str, str]]] = OrderedDict()
|
||||
VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE)
|
||||
VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED = True
|
||||
SECONDS_PER_MINUTE = 60
|
||||
BYTES_PER_MIB = 1024 * 1024
|
||||
CABLE_CACHE_FRESH_SECONDS = 6 * 60 * SECONDS_PER_MINUTE
|
||||
CABLE_CACHE_STALE_SECONDS = 24 * 60 * SECONDS_PER_MINUTE
|
||||
SATELLITE_CACHE_FRESH_SECONDS = 15 * SECONDS_PER_MINUTE
|
||||
SATELLITE_CACHE_STALE_SECONDS = 2 * 60 * SECONDS_PER_MINUTE
|
||||
COMPUTE_CENTER_CACHE_FRESH_SECONDS = 10 * SECONDS_PER_MINUTE
|
||||
COMPUTE_CENTER_CACHE_STALE_SECONDS = 60 * SECONDS_PER_MINUTE
|
||||
BGP_CACHE_FRESH_SECONDS = 60
|
||||
BGP_EVENT_CACHE_FRESH_SECONDS = 30
|
||||
BGP_CACHE_STALE_SECONDS = 10 * SECONDS_PER_MINUTE
|
||||
VESSEL_SNAPSHOT_CACHE_FRESH_SECONDS = 5
|
||||
VESSEL_SNAPSHOT_CACHE_STALE_SECONDS = 30
|
||||
|
||||
CABLE_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
CABLE_CACHE_FRESH_SECONDS,
|
||||
CABLE_CACHE_STALE_SECONDS,
|
||||
max_features=6000,
|
||||
max_bytes=10 * BYTES_PER_MIB,
|
||||
)
|
||||
LANDING_POINT_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
CABLE_CACHE_FRESH_SECONDS,
|
||||
CABLE_CACHE_STALE_SECONDS,
|
||||
max_features=6000,
|
||||
max_bytes=8 * BYTES_PER_MIB,
|
||||
)
|
||||
SATELLITE_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
SATELLITE_CACHE_FRESH_SECONDS,
|
||||
SATELLITE_CACHE_STALE_SECONDS,
|
||||
max_features=25000,
|
||||
max_bytes=32 * BYTES_PER_MIB,
|
||||
)
|
||||
COMPUTE_CENTER_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
COMPUTE_CENTER_CACHE_FRESH_SECONDS,
|
||||
COMPUTE_CENTER_CACHE_STALE_SECONDS,
|
||||
max_features=1000,
|
||||
max_bytes=4 * BYTES_PER_MIB,
|
||||
)
|
||||
BGP_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
BGP_CACHE_FRESH_SECONDS,
|
||||
BGP_CACHE_STALE_SECONDS,
|
||||
max_features=1000,
|
||||
max_bytes=3 * BYTES_PER_MIB,
|
||||
)
|
||||
BGP_EVENT_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
BGP_EVENT_CACHE_FRESH_SECONDS,
|
||||
BGP_CACHE_STALE_SECONDS,
|
||||
max_features=1000,
|
||||
max_bytes=3 * BYTES_PER_MIB,
|
||||
)
|
||||
SUMMARY_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
BGP_EVENT_CACHE_FRESH_SECONDS,
|
||||
BGP_CACHE_STALE_SECONDS,
|
||||
max_features=0,
|
||||
max_bytes=512 * 1024,
|
||||
)
|
||||
VESSEL_SNAPSHOT_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
VESSEL_SNAPSHOT_CACHE_FRESH_SECONDS,
|
||||
VESSEL_SNAPSHOT_CACHE_STALE_SECONDS,
|
||||
max_features=1500,
|
||||
max_bytes=3 * BYTES_PER_MIB,
|
||||
)
|
||||
|
||||
|
||||
class TerrariumTileRequest(BaseModel):
|
||||
@@ -1010,7 +1079,40 @@ async def build_vessel_snapshot_response(
|
||||
type_filter: str | None,
|
||||
limit: int | None,
|
||||
since_minutes: int = 60,
|
||||
response: Response | None = None,
|
||||
use_cache: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
if use_cache and bbox is not None:
|
||||
safe_limit_for_key = _safe_vessel_limit(limit)
|
||||
safe_since_for_key = min(max(int(since_minutes or 60), 1), 1440)
|
||||
cache_key = earth_layer_cache.key(
|
||||
"vessels-snapshot",
|
||||
bbox=format_bbox_key(quantize_bbox(bbox)),
|
||||
zoom=zoom or "none",
|
||||
type=type_filter or "all",
|
||||
limit=safe_limit_for_key,
|
||||
since=safe_since_for_key,
|
||||
)
|
||||
|
||||
async def build_uncached() -> dict[str, Any]:
|
||||
return await build_vessel_snapshot_response(
|
||||
db,
|
||||
bbox=bbox,
|
||||
zoom=zoom,
|
||||
type_filter=type_filter,
|
||||
limit=limit,
|
||||
since_minutes=since_minutes,
|
||||
response=None,
|
||||
use_cache=False,
|
||||
)
|
||||
|
||||
return await get_or_build_layer_payload(
|
||||
key=cache_key,
|
||||
policy=VESSEL_SNAPSHOT_CACHE_POLICY,
|
||||
builder=build_uncached,
|
||||
response=response,
|
||||
)
|
||||
|
||||
requested_types = _requested_vessel_types(type_filter)
|
||||
safe_limit = _safe_vessel_limit(limit)
|
||||
safe_since_minutes = min(max(int(since_minutes or 60), 1), 1440)
|
||||
@@ -1444,20 +1546,24 @@ def convert_bgp_incidents_to_geojson(
|
||||
|
||||
|
||||
@router.get("/geo/cables")
|
||||
async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
|
||||
async def get_cables_geojson(db: AsyncSession = Depends(get_db), response: Response = None):
|
||||
"""获取海底电缆 GeoJSON 数据 (LineString)"""
|
||||
async def build_payload() -> dict[str, Any]:
|
||||
return await _build_cables_geojson(db)
|
||||
|
||||
return await get_or_build_layer_payload(
|
||||
key=earth_layer_cache.key("cables"),
|
||||
policy=CABLE_CACHE_POLICY,
|
||||
builder=build_payload,
|
||||
response=response,
|
||||
)
|
||||
|
||||
|
||||
async def _build_cables_geojson(db: AsyncSession) -> dict[str, Any]:
|
||||
try:
|
||||
records = await _load_current_collected_data(db, "arcgis_cables")
|
||||
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="No cable data found. Please run the arcgis_cables collector first.",
|
||||
)
|
||||
|
||||
return convert_cable_to_geojson(records)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception_event(
|
||||
"Failed to build cables GeoJSON response",
|
||||
@@ -1478,7 +1584,19 @@ async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
|
||||
|
||||
|
||||
@router.get("/geo/landing-points")
|
||||
async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
|
||||
async def get_landing_points_geojson(db: AsyncSession = Depends(get_db), response: Response = None):
|
||||
async def build_payload() -> dict[str, Any]:
|
||||
return await _build_landing_points_geojson(db)
|
||||
|
||||
return await get_or_build_layer_payload(
|
||||
key=earth_layer_cache.key("landing-points"),
|
||||
policy=LANDING_POINT_CACHE_POLICY,
|
||||
builder=build_payload,
|
||||
response=response,
|
||||
)
|
||||
|
||||
|
||||
async def _build_landing_points_geojson(db: AsyncSession) -> dict[str, Any]:
|
||||
try:
|
||||
records_by_source = await _load_current_collected_data_by_sources(
|
||||
db,
|
||||
@@ -1499,16 +1617,8 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
|
||||
relation_records,
|
||||
cable_records,
|
||||
)
|
||||
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="No landing point data found. Please run the arcgis_landing_points collector first.",
|
||||
)
|
||||
|
||||
|
||||
return convert_landing_point_to_geojson(records, city_to_cable_ids_map, cable_id_to_name_map)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception_event(
|
||||
"Failed to build landing points GeoJSON response",
|
||||
@@ -1731,8 +1841,25 @@ async def get_satellites_geojson(
|
||||
description="Maximum number of satellites to return. Omit for no limit.",
|
||||
),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
response: Response = None,
|
||||
):
|
||||
"""获取卫星 TLE GeoJSON 数据"""
|
||||
async def build_payload() -> dict[str, Any]:
|
||||
return await _build_satellites_geojson(limit=limit, db=db)
|
||||
|
||||
return await get_or_build_layer_payload(
|
||||
key=earth_layer_cache.key("satellites", limit=limit or "all"),
|
||||
policy=SATELLITE_CACHE_POLICY,
|
||||
builder=build_payload,
|
||||
response=response,
|
||||
)
|
||||
|
||||
|
||||
async def _build_satellites_geojson(
|
||||
*,
|
||||
limit: int | None,
|
||||
db: AsyncSession,
|
||||
) -> dict[str, Any]:
|
||||
records = await _load_current_or_latest_task_data(
|
||||
db,
|
||||
"celestrak_tle",
|
||||
@@ -1800,8 +1927,25 @@ async def get_gpu_clusters_geojson(
|
||||
async def get_compute_centers_geojson(
|
||||
limit: int = Query(200, ge=1, le=1000),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
response: Response = None,
|
||||
):
|
||||
"""获取统一算力中心 GeoJSON 数据"""
|
||||
async def build_payload() -> dict[str, Any]:
|
||||
return await _build_compute_centers_geojson(limit=limit, db=db)
|
||||
|
||||
return await get_or_build_layer_payload(
|
||||
key=earth_layer_cache.key("compute-centers", limit=limit),
|
||||
policy=COMPUTE_CENTER_CACHE_POLICY,
|
||||
builder=build_payload,
|
||||
response=response,
|
||||
)
|
||||
|
||||
|
||||
async def _build_compute_centers_geojson(
|
||||
*,
|
||||
limit: int,
|
||||
db: AsyncSession,
|
||||
) -> dict[str, Any]:
|
||||
records_by_source = await _load_current_collected_data_by_sources(
|
||||
db,
|
||||
["top500", "epoch_ai_gpu"],
|
||||
@@ -1911,6 +2055,44 @@ class SaveComputeCenterLocationRequest(BaseModel):
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
async def _compute_center_location_web_search_capability(db: AsyncSession) -> Dict[str, Any]:
|
||||
try:
|
||||
config = await get_runtime_web_search_config(db)
|
||||
except Exception as exc:
|
||||
return {
|
||||
"enabled": False,
|
||||
"provider": None,
|
||||
"reason": f"WebSearch 配置读取失败:{exc}",
|
||||
}
|
||||
provider_config = config.active_provider_config
|
||||
has_api_key = bool((provider_config.api_key or "").strip())
|
||||
if not config.enabled:
|
||||
return {
|
||||
"enabled": False,
|
||||
"provider": config.default_provider,
|
||||
"reason": "WebSearch 未开启,无法进行事实核查定位。",
|
||||
}
|
||||
if not has_api_key:
|
||||
return {
|
||||
"enabled": False,
|
||||
"provider": config.default_provider,
|
||||
"reason": f"WebSearch Provider {config.default_provider} 未配置 API Key。",
|
||||
}
|
||||
return {
|
||||
"enabled": True,
|
||||
"provider": config.default_provider,
|
||||
"reason": "",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/compute-centers/location-capability")
|
||||
async def get_compute_center_location_capability(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Return whether fact-checked compute-center location collection can run."""
|
||||
return await _compute_center_location_web_search_capability(db)
|
||||
|
||||
|
||||
@router.post("/compute-centers/{source_id}/collect-location")
|
||||
async def collect_compute_center_location(
|
||||
source_id: str,
|
||||
@@ -1929,6 +2111,9 @@ async def collect_compute_center_location(
|
||||
"""
|
||||
if not source_id or not source_id.strip():
|
||||
raise HTTPException(status_code=400, detail="source_id is required")
|
||||
capability = await _compute_center_location_web_search_capability(db)
|
||||
if not capability.get("enabled"):
|
||||
raise HTTPException(status_code=409, detail=capability)
|
||||
|
||||
record = await _load_compute_center_record(db, source_id)
|
||||
name = payload.name or (record.name if record else None)
|
||||
@@ -1982,6 +2167,7 @@ async def collect_compute_center_location(
|
||||
provider_client=provider_client,
|
||||
query=query,
|
||||
entity_type="compute_center",
|
||||
db=db,
|
||||
attempted_queries=attempted_queries,
|
||||
search_evidence=search_result.evidence,
|
||||
)
|
||||
@@ -1998,6 +2184,24 @@ async def collect_compute_center_location(
|
||||
llm_failure_reason = llm_result.failure_reason
|
||||
|
||||
if not candidates:
|
||||
logger.warning_event(
|
||||
"Compute center location collection returned no candidates",
|
||||
event="visualization.compute_center.location_collect.completed",
|
||||
context={
|
||||
"source_id": source_id,
|
||||
"record_id": record_id,
|
||||
"name": name,
|
||||
"success": False,
|
||||
"llm_failure_reason": llm_failure_reason,
|
||||
"attempted_queries": list(attempted_queries),
|
||||
"context": {
|
||||
"operator": operator,
|
||||
"site": site,
|
||||
"city": city,
|
||||
"country": country,
|
||||
},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"source_id": source_id,
|
||||
"record_id": record_id,
|
||||
@@ -2019,13 +2223,34 @@ async def collect_compute_center_location(
|
||||
},
|
||||
}
|
||||
|
||||
best_candidate = candidates[0].to_dict()
|
||||
logger.info_event(
|
||||
"Compute center location collection returned candidates",
|
||||
event="visualization.compute_center.location_collect.completed",
|
||||
context={
|
||||
"source_id": source_id,
|
||||
"record_id": record_id,
|
||||
"name": name,
|
||||
"success": True,
|
||||
"candidate_count": len(candidates),
|
||||
"best_candidate": best_candidate,
|
||||
"llm_failure_reason": llm_failure_reason,
|
||||
"attempted_queries": list(attempted_queries),
|
||||
"context": {
|
||||
"operator": operator,
|
||||
"site": site,
|
||||
"city": city,
|
||||
"country": country,
|
||||
},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"source_id": source_id,
|
||||
"record_id": record_id,
|
||||
"name": name,
|
||||
"success": True,
|
||||
"candidates": [candidate.to_dict() for candidate in candidates],
|
||||
"best_candidate": candidates[0].to_dict(),
|
||||
"best_candidate": best_candidate,
|
||||
"attempted_queries": list(attempted_queries),
|
||||
"context": {
|
||||
"name": name,
|
||||
@@ -2396,7 +2621,31 @@ async def get_bgp_anomalies_geojson(
|
||||
status: Optional[str] = Query("active"),
|
||||
limit: int = Query(200, ge=1, le=1000),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
response: Response = None,
|
||||
):
|
||||
async def build_payload() -> dict[str, Any]:
|
||||
return await _build_bgp_anomalies_geojson(
|
||||
severity=severity,
|
||||
status=status,
|
||||
limit=limit,
|
||||
db=db,
|
||||
)
|
||||
|
||||
return await get_or_build_layer_payload(
|
||||
key=earth_layer_cache.key("bgp-anomalies", severity=severity or "all", status=status or "all", limit=limit),
|
||||
policy=BGP_EVENT_CACHE_POLICY,
|
||||
builder=build_payload,
|
||||
response=response,
|
||||
)
|
||||
|
||||
|
||||
async def _build_bgp_anomalies_geojson(
|
||||
*,
|
||||
severity: str | None,
|
||||
status: str | None,
|
||||
limit: int,
|
||||
db: AsyncSession,
|
||||
) -> dict[str, Any]:
|
||||
stmt = select(BGPAnomaly).order_by(BGPAnomaly.created_at.desc()).limit(limit)
|
||||
if severity:
|
||||
stmt = stmt.where(BGPAnomaly.severity == severity)
|
||||
@@ -2416,7 +2665,31 @@ async def get_bgp_incidents_geojson(
|
||||
status: Optional[str] = Query("active"),
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
response: Response = None,
|
||||
):
|
||||
async def build_payload() -> dict[str, Any]:
|
||||
return await _build_bgp_incidents_geojson(
|
||||
severity=severity,
|
||||
status=status,
|
||||
limit=limit,
|
||||
db=db,
|
||||
)
|
||||
|
||||
return await get_or_build_layer_payload(
|
||||
key=earth_layer_cache.key("bgp-incidents", severity=severity or "all", status=status or "all", limit=limit),
|
||||
policy=BGP_EVENT_CACHE_POLICY,
|
||||
builder=build_payload,
|
||||
response=response,
|
||||
)
|
||||
|
||||
|
||||
async def _build_bgp_incidents_geojson(
|
||||
*,
|
||||
severity: str | None,
|
||||
status: str | None,
|
||||
limit: int,
|
||||
db: AsyncSession,
|
||||
) -> dict[str, Any]:
|
||||
stmt = select(BGPIncident).order_by(BGPIncident.created_at.desc()).limit(limit)
|
||||
if severity:
|
||||
stmt = stmt.where(BGPIncident.severity == severity)
|
||||
@@ -2431,11 +2704,25 @@ async def get_bgp_incidents_geojson(
|
||||
|
||||
|
||||
@router.get("/geo/bgp-collectors")
|
||||
async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db)):
|
||||
async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db), response: Response = None):
|
||||
async def build_payload() -> dict[str, Any]:
|
||||
return await _build_bgp_collectors_geojson(db)
|
||||
|
||||
return await get_or_build_layer_payload(
|
||||
key=earth_layer_cache.key("bgp-collectors"),
|
||||
policy=BGP_CACHE_POLICY,
|
||||
builder=build_payload,
|
||||
response=response,
|
||||
)
|
||||
|
||||
|
||||
async def _build_bgp_collectors_geojson(db: AsyncSession) -> dict[str, Any]:
|
||||
coverage = await build_bgp_collector_coverage(
|
||||
db,
|
||||
source_filter=("ris_live_bgp", "bgpstream_bgp"),
|
||||
)
|
||||
if not any(int(item.get("observation_count") or 0) > 0 for item in coverage):
|
||||
return {"type": "FeatureCollection", "features": [], "count": 0}
|
||||
coverage_by_collector = {
|
||||
item["collector"]: item
|
||||
for item in coverage
|
||||
@@ -2446,8 +2733,20 @@ async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db)):
|
||||
|
||||
|
||||
@router.get("/geo/summary")
|
||||
async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
|
||||
async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db), response: Response = None):
|
||||
"""Return lightweight Earth HUD counts without loading layer GeoJSON payloads."""
|
||||
async def build_payload() -> dict[str, Any]:
|
||||
return await _build_visualization_geo_summary(db)
|
||||
|
||||
return await get_or_build_layer_payload(
|
||||
key=earth_layer_cache.key("summary"),
|
||||
policy=SUMMARY_CACHE_POLICY,
|
||||
builder=build_payload,
|
||||
response=response,
|
||||
)
|
||||
|
||||
|
||||
async def _build_visualization_geo_summary(db: AsyncSession) -> dict[str, Any]:
|
||||
cable_count = await _count_current_or_latest_task_data(db, "arcgis_cables")
|
||||
landing_point_count = await _count_current_or_latest_task_data(db, "arcgis_landing_points")
|
||||
satellite_count = await _count_current_or_latest_task_data(
|
||||
|
||||
@@ -14,6 +14,7 @@ from app.core.websocket.manager import manager
|
||||
|
||||
logger = get_logger(__name__, service="api")
|
||||
router = APIRouter()
|
||||
EARTH_UPDATES_CHANNEL = "earth_updates"
|
||||
|
||||
|
||||
async def authenticate_token(token: str) -> Optional[dict]:
|
||||
@@ -58,7 +59,7 @@ async def websocket_endpoint(
|
||||
|
||||
is_anonymous = payload is None
|
||||
user_id = str(payload.get("sub")) if payload else f"anonymous:{id(websocket)}"
|
||||
supported_channels = ["vessels"] if is_anonymous else [
|
||||
supported_channels = ["vessels", "earth_news", EARTH_UPDATES_CHANNEL] if is_anonymous else [
|
||||
"gpu_clusters",
|
||||
"submarine_cables",
|
||||
"ixp_nodes",
|
||||
@@ -66,6 +67,8 @@ async def websocket_endpoint(
|
||||
"dashboard",
|
||||
"datasource_tasks",
|
||||
"vessels",
|
||||
"earth_news",
|
||||
EARTH_UPDATES_CHANNEL,
|
||||
]
|
||||
await manager.connect(websocket, user_id)
|
||||
|
||||
|
||||
@@ -257,6 +257,16 @@ DEFAULT_DATASOURCES = {
|
||||
"credential_provider": "aisstream",
|
||||
"credential_status": "supported",
|
||||
},
|
||||
"media_news_archive": {
|
||||
"id": 33,
|
||||
"name": "Media News Archive",
|
||||
"display_name": "媒体新闻归档",
|
||||
"module": "L4",
|
||||
"priority": "P2",
|
||||
"frequency_minutes": 720,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
}
|
||||
|
||||
ID_TO_COLLECTOR = {info["id"]: name for name, info in DEFAULT_DATASOURCES.items()}
|
||||
|
||||
@@ -8,6 +8,8 @@ from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.manager import manager
|
||||
|
||||
|
||||
EARTH_UPDATES_CHANNEL = "earth_updates"
|
||||
|
||||
|
||||
class DataBroadcaster:
|
||||
"""Periodically broadcasts data to connected WebSocket clients"""
|
||||
@@ -83,6 +85,10 @@ class DataBroadcaster:
|
||||
channel=channel,
|
||||
)
|
||||
|
||||
async def broadcast_earth_update(self, data: Dict[str, Any]):
|
||||
"""Broadcast Earth visualization refresh hints to connected clients."""
|
||||
await self.broadcast_custom(EARTH_UPDATES_CHANNEL, data)
|
||||
|
||||
def enqueue_vessel_update(self, data: Dict[str, Any]):
|
||||
vessels = data.get("vessels") if isinstance(data, dict) else None
|
||||
if not isinstance(vessels, list):
|
||||
@@ -144,7 +150,7 @@ class DataBroadcaster:
|
||||
"timestamp": to_iso8601_utc(datetime.now(UTC)),
|
||||
"payload": data,
|
||||
},
|
||||
channel="all",
|
||||
channel="datasource_tasks",
|
||||
)
|
||||
|
||||
def start(self):
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy import bindparam, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
from sqlalchemy.orm import declarative_base
|
||||
|
||||
@@ -72,6 +72,74 @@ async def seed_default_datasources(session: AsyncSession):
|
||||
await session.commit()
|
||||
|
||||
|
||||
LEGACY_EARTH_BOUNDARY_SOURCES = (
|
||||
"earth_admin0_boundaries",
|
||||
"earth_coastline",
|
||||
"earth_claim_lines",
|
||||
"earth_boundary_tiles",
|
||||
)
|
||||
LEGACY_EARTH_BOUNDARY_DATATYPES = (
|
||||
"earth_boundary_source",
|
||||
"earth_boundary_tiles",
|
||||
)
|
||||
LEGACY_EARTH_BOUNDARY_IDS = (29, 30, 31, 32)
|
||||
|
||||
|
||||
async def purge_legacy_earth_boundary_datasources(session: AsyncSession) -> None:
|
||||
source_names = tuple(LEGACY_EARTH_BOUNDARY_SOURCES)
|
||||
source_ids = tuple(LEGACY_EARTH_BOUNDARY_IDS)
|
||||
data_types = tuple(LEGACY_EARTH_BOUNDARY_DATATYPES)
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
DELETE FROM datasource_mapping_templates
|
||||
WHERE target_schema IN :data_types
|
||||
OR datasource_config_id IN (
|
||||
SELECT id FROM datasource_configs WHERE name IN :source_names
|
||||
)
|
||||
"""
|
||||
).bindparams(bindparam("source_names", expanding=True), bindparam("data_types", expanding=True)),
|
||||
{"source_names": list(source_names), "data_types": list(data_types)},
|
||||
)
|
||||
await session.execute(
|
||||
text("DELETE FROM datasource_configs WHERE name IN :source_names").bindparams(
|
||||
bindparam("source_names", expanding=True)
|
||||
),
|
||||
{"source_names": list(source_names)},
|
||||
)
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
DELETE FROM collected_data
|
||||
WHERE source IN :source_names OR data_type IN :data_types
|
||||
"""
|
||||
).bindparams(bindparam("source_names", expanding=True), bindparam("data_types", expanding=True)),
|
||||
{"source_names": list(source_names), "data_types": list(data_types)},
|
||||
)
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
DELETE FROM data_snapshots
|
||||
WHERE source IN :source_names OR datasource_id IN :source_ids
|
||||
"""
|
||||
).bindparams(bindparam("source_names", expanding=True), bindparam("source_ids", expanding=True)),
|
||||
{"source_names": list(source_names), "source_ids": list(source_ids)},
|
||||
)
|
||||
await session.execute(
|
||||
text("DELETE FROM collection_tasks WHERE datasource_id IN :source_ids").bindparams(
|
||||
bindparam("source_ids", expanding=True)
|
||||
),
|
||||
{"source_ids": list(source_ids)},
|
||||
)
|
||||
await session.execute(
|
||||
text("DELETE FROM data_sources WHERE source IN :source_names OR id IN :source_ids").bindparams(
|
||||
bindparam("source_names", expanding=True), bindparam("source_ids", expanding=True)
|
||||
),
|
||||
{"source_names": list(source_names), "source_ids": list(source_ids)},
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
DEFAULT_LOGIN_USERS = (
|
||||
{
|
||||
"username": "admin",
|
||||
@@ -82,7 +150,7 @@ DEFAULT_LOGIN_USERS = (
|
||||
{
|
||||
"username": "linkong",
|
||||
"email": "linkong@planet.local",
|
||||
"password": "12345678",
|
||||
"password": "LK12345678",
|
||||
"role": "super_admin",
|
||||
},
|
||||
)
|
||||
@@ -134,6 +202,8 @@ async def init_db():
|
||||
import app.models.vessel # noqa: F401
|
||||
import app.models.vessel_enrichment # noqa: F401
|
||||
import app.models.datasource_mapping # noqa: F401
|
||||
import app.models.earth_news # noqa: F401
|
||||
import app.models.earth_interactable # noqa: F401
|
||||
|
||||
logger.warning_event(
|
||||
"Database pool settings active",
|
||||
@@ -189,6 +259,406 @@ async def init_db():
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS earth_data_change_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
table_name VARCHAR(128) NOT NULL,
|
||||
operation VARCHAR(16) NOT NULL,
|
||||
source VARCHAR(128),
|
||||
entity_key VARCHAR(255),
|
||||
payload JSONB NOT NULL,
|
||||
occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
consumed_at TIMESTAMPTZ
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_data_change_events_unconsumed
|
||||
ON earth_data_change_events (consumed_at, id)
|
||||
WHERE consumed_at IS NULL
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_emit_earth_data_changed_statement(
|
||||
change_table TEXT,
|
||||
change_operation TEXT,
|
||||
change_source TEXT,
|
||||
source_record_count INTEGER,
|
||||
source_entity_keys TEXT[]
|
||||
)
|
||||
RETURNS VOID AS $$
|
||||
DECLARE
|
||||
change_event_id BIGINT;
|
||||
change_payload JSONB;
|
||||
BEGIN
|
||||
change_payload := jsonb_build_object(
|
||||
'event', 'earth.layer.changed',
|
||||
'table', change_table,
|
||||
'operation', change_operation,
|
||||
'source', change_source,
|
||||
'entity_key', NULL,
|
||||
'entity_keys', COALESCE(to_jsonb(source_entity_keys), '[]'::jsonb),
|
||||
'records_processed', COALESCE(source_record_count, 0),
|
||||
'occurred_at', NOW()
|
||||
);
|
||||
|
||||
INSERT INTO earth_data_change_events (
|
||||
table_name,
|
||||
operation,
|
||||
source,
|
||||
entity_key,
|
||||
payload,
|
||||
occurred_at
|
||||
) VALUES (
|
||||
change_table,
|
||||
change_operation,
|
||||
change_source,
|
||||
NULL,
|
||||
change_payload,
|
||||
NOW()
|
||||
)
|
||||
RETURNING id INTO change_event_id;
|
||||
|
||||
change_payload := change_payload || jsonb_build_object(
|
||||
'event_id', change_event_id
|
||||
);
|
||||
|
||||
UPDATE earth_data_change_events
|
||||
SET payload = change_payload
|
||||
WHERE id = change_event_id;
|
||||
|
||||
PERFORM pg_notify(
|
||||
'planet_earth_data_changes',
|
||||
change_payload::text
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_emit_collected_data_changed_statement(
|
||||
change_operation TEXT,
|
||||
change_source TEXT,
|
||||
source_record_count INTEGER,
|
||||
source_entity_keys TEXT[]
|
||||
)
|
||||
RETURNS VOID AS $$
|
||||
BEGIN
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
'collected_data',
|
||||
change_operation,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
RETURNS trigger AS $$
|
||||
DECLARE
|
||||
change_source TEXT;
|
||||
source_record_count INTEGER;
|
||||
source_entity_keys TEXT[];
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) changed_rows
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(
|
||||
NULLIF(row_data->>'entity_key', ''),
|
||||
NULLIF(row_data->>'source_id', ''),
|
||||
NULLIF(row_data->>'incident_key', ''),
|
||||
NULLIF(row_data->>'id', ''),
|
||||
NULLIF(row_data->>'mmsi', '')
|
||||
)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) rows_for_keys
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) rows_for_count
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
||||
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
TG_TABLE_NAME,
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'DELETE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) changed_rows
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(
|
||||
NULLIF(row_data->>'entity_key', ''),
|
||||
NULLIF(row_data->>'source_id', ''),
|
||||
NULLIF(row_data->>'incident_key', ''),
|
||||
NULLIF(row_data->>'id', ''),
|
||||
NULLIF(row_data->>'mmsi', '')
|
||||
)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) rows_for_keys
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) rows_for_count
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
||||
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
TG_TABLE_NAME,
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'UPDATE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
||||
FROM (
|
||||
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
||||
UNION ALL
|
||||
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
||||
) changed_rows
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(
|
||||
NULLIF(row_data->>'entity_key', ''),
|
||||
NULLIF(row_data->>'source_id', ''),
|
||||
NULLIF(row_data->>'incident_key', ''),
|
||||
NULLIF(row_data->>'id', ''),
|
||||
NULLIF(row_data->>'mmsi', '')
|
||||
)
|
||||
FROM (
|
||||
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
||||
UNION ALL
|
||||
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
||||
) rows_for_keys
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (
|
||||
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
||||
UNION ALL
|
||||
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
||||
) rows_for_count
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
||||
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
TG_TABLE_NAME,
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
RETURNS trigger AS $$
|
||||
DECLARE
|
||||
change_source TEXT;
|
||||
source_record_count INTEGER;
|
||||
source_entity_keys TEXT[];
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT source FROM new_rows WHERE source IS NOT NULL
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
||||
FROM new_rows
|
||||
WHERE source = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM new_rows
|
||||
WHERE source = change_source;
|
||||
|
||||
PERFORM planet_emit_collected_data_changed_statement(
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'DELETE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT source FROM old_rows WHERE source IS NOT NULL
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
||||
FROM old_rows
|
||||
WHERE source = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM old_rows
|
||||
WHERE source = change_source;
|
||||
|
||||
PERFORM planet_emit_collected_data_changed_statement(
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'UPDATE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT source FROM (
|
||||
SELECT source FROM new_rows
|
||||
UNION
|
||||
SELECT source FROM old_rows
|
||||
) changed_sources
|
||||
WHERE source IS NOT NULL
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
||||
FROM (
|
||||
SELECT id, source_id, entity_key, source FROM new_rows
|
||||
UNION ALL
|
||||
SELECT id, source_id, entity_key, source FROM old_rows
|
||||
) changed_rows
|
||||
WHERE source = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (
|
||||
SELECT id, source_id, entity_key, source FROM new_rows
|
||||
UNION ALL
|
||||
SELECT id, source_id, entity_key, source FROM old_rows
|
||||
) changed_rows
|
||||
WHERE source = change_source;
|
||||
|
||||
PERFORM planet_emit_collected_data_changed_statement(
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
for statement in (
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed ON collected_data",
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_insert ON collected_data",
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_update ON collected_data",
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_delete ON collected_data",
|
||||
"DROP FUNCTION IF EXISTS planet_notify_collected_data_changed()",
|
||||
"""
|
||||
CREATE TRIGGER tr_planet_collected_data_changed_insert
|
||||
AFTER INSERT ON collected_data
|
||||
REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER tr_planet_collected_data_changed_update
|
||||
AFTER UPDATE ON collected_data
|
||||
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER tr_planet_collected_data_changed_delete
|
||||
AFTER DELETE ON collected_data
|
||||
REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
""",
|
||||
):
|
||||
await conn.execute(text(statement))
|
||||
for table_name in (
|
||||
"bgp_observations",
|
||||
"bgp_anomalies",
|
||||
"bgp_incidents",
|
||||
"bgp_collector_locations",
|
||||
"vessel_static",
|
||||
"vessel_position",
|
||||
"ais_raw_observations",
|
||||
"ais_source_health",
|
||||
"compute_center_locations",
|
||||
"earth_interactables",
|
||||
"earth_news_items",
|
||||
):
|
||||
for statement in (
|
||||
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_insert ON {table_name}",
|
||||
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_update ON {table_name}",
|
||||
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_delete ON {table_name}",
|
||||
f"""
|
||||
CREATE TRIGGER tr_planet_{table_name}_changed_insert
|
||||
AFTER INSERT ON {table_name}
|
||||
REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
""",
|
||||
f"""
|
||||
CREATE TRIGGER tr_planet_{table_name}_changed_update
|
||||
AFTER UPDATE ON {table_name}
|
||||
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
""",
|
||||
f"""
|
||||
CREATE TRIGGER tr_planet_{table_name}_changed_delete
|
||||
AFTER DELETE ON {table_name}
|
||||
REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
""",
|
||||
):
|
||||
await conn.execute(text(statement))
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -198,7 +668,39 @@ async def init_db():
|
||||
ADD COLUMN IF NOT EXISTS phase_message VARCHAR(255),
|
||||
ADD COLUMN IF NOT EXISTS phase_current BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS phase_total BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30)
|
||||
ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30),
|
||||
ADD COLUMN IF NOT EXISTS source VARCHAR(100),
|
||||
ADD COLUMN IF NOT EXISTS task_type VARCHAR(30) NOT NULL DEFAULT 'collect',
|
||||
ADD COLUMN IF NOT EXISTS payload JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
ADD COLUMN IF NOT EXISTS rollback_policy VARCHAR(40) NOT NULL DEFAULT 'keep_committed_batches',
|
||||
ADD COLUMN IF NOT EXISTS dedupe_key VARCHAR(180),
|
||||
ADD COLUMN IF NOT EXISTS worker_id VARCHAR(120),
|
||||
ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS requested_cancel_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS cancel_reason TEXT
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
ALTER TABLE earth_news_items
|
||||
ADD COLUMN IF NOT EXISTS content_language VARCHAR(32) NOT NULL DEFAULT 'en',
|
||||
ADD COLUMN IF NOT EXISTS localizations JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
ADD COLUMN IF NOT EXISTS enrichment_status VARCHAR(80) NOT NULL DEFAULT 'pending',
|
||||
ADD COLUMN IF NOT EXISTS enrichment_error TEXT,
|
||||
ADD COLUMN IF NOT EXISTS enriched_at TIMESTAMPTZ
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
ALTER TABLE earth_interactables
|
||||
ADD COLUMN IF NOT EXISTS altitude DOUBLE PRECISION,
|
||||
ADD COLUMN IF NOT EXISTS revision INTEGER NOT NULL DEFAULT 1,
|
||||
ADD COLUMN IF NOT EXISTS is_deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ
|
||||
"""
|
||||
)
|
||||
)
|
||||
@@ -210,6 +712,64 @@ async def init_db():
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_news_enrichment_status
|
||||
ON earth_news_items (enrichment_status)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_news_enriched_at
|
||||
ON earth_news_items (enriched_at)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_interactables_layer_deleted
|
||||
ON earth_interactables (layer, is_deleted)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_interactables_updated_at
|
||||
ON earth_interactables (updated_at)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_collection_tasks_source_status
|
||||
ON collection_tasks (source, status)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_collection_tasks_queue
|
||||
ON collection_tasks (status, created_at, id)
|
||||
WHERE status = 'queued'
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_collection_tasks_dedupe
|
||||
ON collection_tasks (dedupe_key)
|
||||
WHERE dedupe_key IS NOT NULL
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -284,4 +844,5 @@ async def init_db():
|
||||
await seed_default_bgp_collector_locations(session)
|
||||
await seed_compute_center_locations_from_source_coords(session)
|
||||
await seed_default_datasources(session)
|
||||
await purge_legacy_earth_boundary_datasources(session)
|
||||
await ensure_default_admin_user(session)
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
from app.api.main import api_router
|
||||
@@ -18,6 +20,15 @@ from app.services.scheduler import (
|
||||
stop_scheduler,
|
||||
sync_scheduler_with_datasources,
|
||||
)
|
||||
from app.services.earth_news_worker import (
|
||||
start_earth_news_target_worker,
|
||||
stop_earth_news_target_worker,
|
||||
)
|
||||
from app.services.earth_db_change_listener import (
|
||||
start_earth_db_change_listener,
|
||||
stop_earth_db_change_listener,
|
||||
)
|
||||
from app.services.data_jobs import start_data_job_worker, stop_data_job_worker
|
||||
|
||||
|
||||
configure_logging()
|
||||
@@ -53,7 +64,13 @@ async def lifespan(app: FastAPI):
|
||||
start_scheduler()
|
||||
await sync_scheduler_with_datasources()
|
||||
broadcaster.start()
|
||||
start_data_job_worker()
|
||||
start_earth_db_change_listener()
|
||||
start_earth_news_target_worker()
|
||||
yield
|
||||
await stop_earth_news_target_worker()
|
||||
await stop_earth_db_change_listener()
|
||||
await stop_data_job_worker()
|
||||
broadcaster.stop()
|
||||
stop_scheduler()
|
||||
|
||||
@@ -82,6 +99,14 @@ app.add_middleware(WebSocketCORSMiddleware)
|
||||
app.include_router(api_router, prefix="/api/v1")
|
||||
app.include_router(websocket.router)
|
||||
|
||||
EARTH_BRAND_ASSET_DIR = Path(__file__).resolve().parents[2] / "data" / "earth-brand"
|
||||
EARTH_BRAND_ASSET_DIR.mkdir(parents=True, exist_ok=True)
|
||||
app.mount(
|
||||
"/earth-brand-assets",
|
||||
StaticFiles(directory=str(EARTH_BRAND_ASSET_DIR)),
|
||||
name="earth-brand-assets",
|
||||
)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
|
||||
@@ -16,6 +16,8 @@ from app.models.playground_message import PlaygroundMessage
|
||||
from app.models.system_log import SystemLog, AuditLog
|
||||
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth, VesselPosition, VesselStatic
|
||||
from app.models.datasource_mapping import DataSourceMappingTemplate
|
||||
from app.models.earth_news import EarthNewsItem
|
||||
from app.models.earth_interactable import EarthInteractable
|
||||
|
||||
__all__ = [
|
||||
"User",
|
||||
@@ -43,4 +45,6 @@ __all__ = [
|
||||
"AISConflictRecord",
|
||||
"AISSourceHealth",
|
||||
"DataSourceMappingTemplate",
|
||||
"EarthNewsItem",
|
||||
"EarthInteractable",
|
||||
]
|
||||
|
||||
30
backend/app/models/earth_interactable.py
Normal file
30
backend/app/models/earth_interactable.py
Normal file
@@ -0,0 +1,30 @@
|
||||
"""Persistent Earth interactable objects."""
|
||||
|
||||
from sqlalchemy import Boolean, Column, DateTime, Float, Index, Integer, JSON, String, Text
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.db.session import Base
|
||||
|
||||
|
||||
class EarthInteractable(Base):
|
||||
__tablename__ = "earth_interactables"
|
||||
|
||||
id = Column(String(160), primary_key=True)
|
||||
layer = Column(String(80), nullable=False, default="interactables", index=True)
|
||||
kind = Column(String(80), nullable=False, default="default", index=True)
|
||||
label = Column(String(255), nullable=False, default="")
|
||||
description = Column(Text, nullable=False, default="")
|
||||
latitude = Column(Float, nullable=False)
|
||||
longitude = Column(Float, nullable=False)
|
||||
altitude = Column(Float, nullable=True)
|
||||
revision = Column(Integer, nullable=False, default=1)
|
||||
properties = Column(JSON, nullable=False, default=dict)
|
||||
is_deleted = Column(Boolean, nullable=False, default=False, index=True)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
deleted_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_earth_interactables_layer_deleted", "layer", "is_deleted"),
|
||||
Index("idx_earth_interactables_updated_at", "updated_at"),
|
||||
)
|
||||
40
backend/app/models/earth_news.py
Normal file
40
backend/app/models/earth_news.py
Normal file
@@ -0,0 +1,40 @@
|
||||
from sqlalchemy import Boolean, Column, DateTime, Float, Index, JSON, String, Text
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.db.session import Base
|
||||
|
||||
|
||||
class EarthNewsItem(Base):
|
||||
__tablename__ = "earth_news_items"
|
||||
|
||||
id = Column(String(160), primary_key=True)
|
||||
title = Column(String(500), nullable=False)
|
||||
summary = Column(Text, nullable=False, default="")
|
||||
content_language = Column(String(32), nullable=False, default="en")
|
||||
localizations = Column(JSON, nullable=False, default=dict)
|
||||
url = Column(Text, nullable=False)
|
||||
source = Column(String(255), nullable=False, default="")
|
||||
feed_name = Column(String(255), nullable=False, default="")
|
||||
region = Column(String(80), nullable=False, index=True)
|
||||
homepage_url = Column(Text, nullable=False, default="")
|
||||
published_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
|
||||
latitude = Column(Float, nullable=False)
|
||||
longitude = Column(Float, nullable=False)
|
||||
location_label = Column(String(255), nullable=False)
|
||||
location_source = Column(String(80), nullable=False, default="region_anchor")
|
||||
verified = Column(Boolean, nullable=False, default=False, index=True)
|
||||
location_meta = Column(JSON, nullable=False, default=dict)
|
||||
|
||||
first_seen_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
|
||||
last_seen_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False, index=True)
|
||||
resolved_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
enrichment_status = Column(String(80), nullable=False, default="pending", index=True)
|
||||
enrichment_error = Column(Text, nullable=True)
|
||||
enriched_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_earth_news_region_published", "region", "published_at"),
|
||||
Index("idx_earth_news_region_seen", "region", "last_seen_at"),
|
||||
)
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Collection Task model"""
|
||||
"""Datasource job model."""
|
||||
|
||||
from sqlalchemy import BigInteger, Column, DateTime, Integer, String, Text, Float
|
||||
from sqlalchemy import BigInteger, Column, DateTime, Float, Integer, JSON, String, Text
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.db.session import Base
|
||||
@@ -11,7 +11,9 @@ class CollectionTask(Base):
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
datasource_id = Column(Integer, nullable=False, index=True)
|
||||
status = Column(String(20), nullable=False) # pending, running, success, failed, cancelled
|
||||
source = Column(String(100), nullable=True, index=True)
|
||||
task_type = Column(String(30), nullable=False, default="collect", index=True)
|
||||
status = Column(String(20), nullable=False) # queued, running, cancelling, success, failed, cancelled
|
||||
phase = Column(String(30), default="queued")
|
||||
phase_progress = Column(Float)
|
||||
phase_message = Column(String(255))
|
||||
@@ -24,6 +26,13 @@ class CollectionTask(Base):
|
||||
total_records = Column(Integer, default=0) # Total records to process
|
||||
progress = Column(Float, default=0.0) # Progress percentage (0-100)
|
||||
error_message = Column(Text)
|
||||
payload = Column(JSON, default=dict)
|
||||
rollback_policy = Column(String(40), nullable=False, default="keep_committed_batches")
|
||||
dedupe_key = Column(String(180), nullable=True, index=True)
|
||||
worker_id = Column(String(120), nullable=True, index=True)
|
||||
locked_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
requested_cancel_at = Column(DateTime(timezone=True), nullable=True)
|
||||
cancel_reason = Column(Text)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
def __repr__(self):
|
||||
|
||||
@@ -13,10 +13,11 @@ class AIContentBlock(BaseModel):
|
||||
|
||||
class SituationalAnalysisRequest(BaseModel):
|
||||
title: str = Field(..., min_length=1, max_length=200)
|
||||
objective: str = Field(..., min_length=1, max_length=1000)
|
||||
objective: str = Field(..., min_length=1, max_length=20000)
|
||||
context: dict[str, Any] = Field(default_factory=dict)
|
||||
observations: list[str] = Field(default_factory=list)
|
||||
constraints: list[str] = Field(default_factory=list)
|
||||
system_prompt: str | None = Field(default=None, max_length=8000)
|
||||
preferred_model: str | None = Field(default=None, max_length=200)
|
||||
thinking: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@@ -1,18 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from time import perf_counter
|
||||
|
||||
import httpx
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import get_db
|
||||
from app.schemas.ai import (
|
||||
AIProviderStatusResponse,
|
||||
SituationalAnalysisRequest,
|
||||
SituationalAnalysisResponse,
|
||||
)
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="ai")
|
||||
|
||||
|
||||
class AIProviderClient:
|
||||
@@ -57,10 +64,25 @@ class AIProviderClient:
|
||||
value = self.llm_config.get(key)
|
||||
if value not in (None, ""):
|
||||
headers[header_name] = str(value)
|
||||
model_provider_apis = self.llm_config.get("model_provider_apis")
|
||||
if isinstance(model_provider_apis, dict) and model_provider_apis:
|
||||
headers["X-AI-Model-Provider-APIs"] = json.dumps(model_provider_apis)
|
||||
return headers
|
||||
|
||||
async def get_status(self, request_id: str | None = None) -> AIProviderStatusResponse:
|
||||
context = self._base_log_context(operation="status")
|
||||
if not self.service_url:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.status.failed",
|
||||
message="AI provider status skipped because service URL is not configured",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context={**context, "status": "unconfigured"},
|
||||
)
|
||||
return AIProviderStatusResponse(
|
||||
provider="unconfigured",
|
||||
enabled=False,
|
||||
@@ -69,27 +91,133 @@ class AIProviderClient:
|
||||
base_url=None,
|
||||
)
|
||||
|
||||
data = await self._request("GET", "/v1/provider/status", request_id=request_id)
|
||||
return AIProviderStatusResponse.model_validate(data)
|
||||
started_at = perf_counter()
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.status.start",
|
||||
message="AI provider status request started",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context=context,
|
||||
)
|
||||
try:
|
||||
data = await self._request("GET", "/v1/provider/status", request_id=request_id, operation="status")
|
||||
result = AIProviderStatusResponse.model_validate(data)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.status.success",
|
||||
message="AI provider status request completed",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context={
|
||||
**context,
|
||||
"status": "success",
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
"result_provider": result.provider,
|
||||
"result_model": result.model,
|
||||
"configured": result.configured,
|
||||
"enabled": result.enabled,
|
||||
},
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.status.failed",
|
||||
message="AI provider status request failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context=exception_context(exc, {**context, "status": "failed", "duration_ms": self._duration_ms(started_at)}),
|
||||
)
|
||||
raise
|
||||
|
||||
async def analyze(
|
||||
self,
|
||||
payload: SituationalAnalysisRequest,
|
||||
request_id: str | None = None,
|
||||
) -> SituationalAnalysisResponse:
|
||||
context = self._base_log_context(
|
||||
operation="analyze",
|
||||
preferred_model=payload.preferred_model,
|
||||
input_summary=self._summarize_analysis_payload(payload),
|
||||
)
|
||||
if not self.service_url:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.analyze.failed",
|
||||
message="AI provider analyze skipped because service URL is not configured",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context={**context, "status": "unconfigured"},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="AI provider service URL is not configured.",
|
||||
)
|
||||
|
||||
data = await self._request(
|
||||
"POST",
|
||||
"/v1/analyze",
|
||||
json=payload.model_dump(),
|
||||
started_at = perf_counter()
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.analyze.start",
|
||||
message="AI provider analyze request started",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context=context,
|
||||
)
|
||||
return SituationalAnalysisResponse.model_validate(data)
|
||||
try:
|
||||
data = await self._request(
|
||||
"POST",
|
||||
"/v1/analyze",
|
||||
json=payload.model_dump(),
|
||||
request_id=request_id,
|
||||
operation="analyze",
|
||||
payload_summary=context["input_summary"],
|
||||
)
|
||||
result = SituationalAnalysisResponse.model_validate(data)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.analyze.success",
|
||||
message="AI provider analyze request completed",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context={
|
||||
**context,
|
||||
"status": "success",
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
"result_provider": result.provider,
|
||||
"result_model": result.model,
|
||||
"content_block_count": len(result.content_blocks or []),
|
||||
"thinking_block_count": len(result.thinking_blocks or []),
|
||||
},
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.analyze.failed",
|
||||
message="AI provider analyze request failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context=exception_context(exc, {**context, "status": "failed", "duration_ms": self._duration_ms(started_at)}),
|
||||
)
|
||||
raise
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
@@ -97,9 +225,12 @@ class AIProviderClient:
|
||||
path: str,
|
||||
json: dict | None = None,
|
||||
request_id: str | None = None,
|
||||
operation: str = "request",
|
||||
payload_summary: dict | None = None,
|
||||
) -> dict:
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, self.retry_attempts + 1):
|
||||
attempt_started_at = perf_counter()
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.request(
|
||||
@@ -113,6 +244,15 @@ class AIProviderClient:
|
||||
except httpx.HTTPStatusError as exc:
|
||||
last_error = exc
|
||||
if attempt < self.retry_attempts and exc.response.status_code >= 500:
|
||||
await self._log_retry(
|
||||
operation=operation,
|
||||
request_id=request_id,
|
||||
attempt=attempt,
|
||||
status_code=exc.response.status_code,
|
||||
duration_ms=self._duration_ms(attempt_started_at),
|
||||
error=exc,
|
||||
payload_summary=payload_summary,
|
||||
)
|
||||
await asyncio.sleep(0.3 * attempt)
|
||||
continue
|
||||
detail = exc.response.text or "AI provider service returned an error"
|
||||
@@ -123,6 +263,14 @@ class AIProviderClient:
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = exc
|
||||
if attempt < self.retry_attempts:
|
||||
await self._log_retry(
|
||||
operation=operation,
|
||||
request_id=request_id,
|
||||
attempt=attempt,
|
||||
duration_ms=self._duration_ms(attempt_started_at),
|
||||
error=exc,
|
||||
payload_summary=payload_summary,
|
||||
)
|
||||
await asyncio.sleep(0.3 * attempt)
|
||||
continue
|
||||
raise HTTPException(
|
||||
@@ -135,6 +283,71 @@ class AIProviderClient:
|
||||
detail=f"AI provider service request failed: {last_error}",
|
||||
)
|
||||
|
||||
def _base_log_context(self, **extra: object) -> dict:
|
||||
llm_provider_apis = self.llm_config.get("model_provider_apis")
|
||||
return {
|
||||
"provider": self.llm_config.get("provider") or "",
|
||||
"provider_api": self.llm_config.get("provider_api") or "",
|
||||
"model": self.llm_config.get("model") or "",
|
||||
"base_url_configured": bool(self.llm_config.get("base_url")),
|
||||
"service_url_configured": bool(self.service_url),
|
||||
"timeout_seconds": self.timeout,
|
||||
"retry_attempts": self.retry_attempts,
|
||||
"model_provider_api_count": len(llm_provider_apis or {}) if isinstance(llm_provider_apis, dict) else 0,
|
||||
**extra,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _duration_ms(started_at: float) -> int:
|
||||
return int((perf_counter() - started_at) * 1000)
|
||||
|
||||
@staticmethod
|
||||
def _summarize_analysis_payload(payload: SituationalAnalysisRequest) -> dict:
|
||||
context = payload.context if isinstance(payload.context, dict) else {}
|
||||
thinking = payload.thinking if isinstance(payload.thinking, dict) else payload.thinking
|
||||
return {
|
||||
"title_length": len(payload.title or ""),
|
||||
"objective_length": len(payload.objective or ""),
|
||||
"observation_count": len(payload.observations or []),
|
||||
"constraint_count": len(payload.constraints or []),
|
||||
"has_system_prompt": bool(payload.system_prompt),
|
||||
"thinking_enabled": bool(thinking),
|
||||
"context_keys": sorted(str(key) for key in context.keys()),
|
||||
}
|
||||
|
||||
async def _log_retry(
|
||||
self,
|
||||
*,
|
||||
operation: str,
|
||||
request_id: str | None,
|
||||
attempt: int,
|
||||
duration_ms: int,
|
||||
error: BaseException,
|
||||
status_code: int | None = None,
|
||||
payload_summary: dict | None = None,
|
||||
) -> None:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=f"ai.provider.{operation}.retry",
|
||||
message="AI provider request will retry",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context=exception_context(
|
||||
error,
|
||||
{
|
||||
**self._base_log_context(operation=operation),
|
||||
"attempt": attempt,
|
||||
"next_attempt": attempt + 1,
|
||||
"status_code": status_code,
|
||||
"duration_ms": duration_ms,
|
||||
"input_summary": payload_summary,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def get_ai_provider_client(db: AsyncSession = Depends(get_db)) -> AIProviderClient:
|
||||
from app.api.v1.settings import get_runtime_ai_provider_config
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from time import perf_counter
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.services.ai_tools.schemas import FetchedEvidence
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="ai_tool")
|
||||
|
||||
|
||||
class WebFetchError(RuntimeError):
|
||||
@@ -29,8 +36,33 @@ async def fetch_url_evidence(
|
||||
timeout_seconds: int = 20,
|
||||
max_bytes: int = 1_500_000,
|
||||
) -> FetchedEvidence:
|
||||
started_at = perf_counter()
|
||||
if not url:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_fetch.failed",
|
||||
message="WebFetch failed because URL is empty",
|
||||
category="ai_tool",
|
||||
level="warning",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={"reason": "empty_url"},
|
||||
)
|
||||
raise WebFetchError("url is required")
|
||||
request_host = urlparse(url).netloc
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_fetch.start",
|
||||
message="WebFetch request started",
|
||||
category="ai_tool",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={
|
||||
"url_host": request_host,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
"max_bytes": max_bytes,
|
||||
},
|
||||
)
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout_seconds,
|
||||
@@ -41,10 +73,45 @@ async def fetch_url_evidence(
|
||||
response.raise_for_status()
|
||||
content = response.content[:max_bytes]
|
||||
except httpx.HTTPError as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_fetch.failed",
|
||||
message="WebFetch request failed",
|
||||
category="ai_tool",
|
||||
level="error",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"url_host": request_host,
|
||||
"status": "failed",
|
||||
"duration_ms": int((perf_counter() - started_at) * 1000),
|
||||
},
|
||||
),
|
||||
)
|
||||
raise WebFetchError(f"failed to fetch page: {exc}") from exc
|
||||
|
||||
title, text = _extract_title_and_text(content.decode(response.encoding or "utf-8", errors="ignore"))
|
||||
content_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_fetch.success",
|
||||
message="WebFetch request completed",
|
||||
category="ai_tool",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={
|
||||
"url_host": request_host,
|
||||
"final_url_host": urlparse(str(response.url)).netloc,
|
||||
"status": "success",
|
||||
"status_code": response.status_code,
|
||||
"bytes_read": len(content),
|
||||
"content_hash": content_hash,
|
||||
"duration_ms": int((perf_counter() - started_at) * 1000),
|
||||
"extractor": "beautifulsoup_basic",
|
||||
},
|
||||
)
|
||||
return FetchedEvidence(
|
||||
url=url,
|
||||
final_url=str(response.url),
|
||||
@@ -53,4 +120,3 @@ async def fetch_url_evidence(
|
||||
content_hash=content_hash,
|
||||
extractor="beautifulsoup_basic",
|
||||
)
|
||||
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
import hashlib
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.services.ai_tools.schemas import SearchEvidence, WebSearchConfig, WebSearchProviderConfig
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="ai_tool")
|
||||
|
||||
|
||||
WEB_SEARCH_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
|
||||
@@ -120,29 +127,108 @@ class WebSearchClient:
|
||||
domains: list[str] | None = None,
|
||||
freshness_days: int | None = None,
|
||||
) -> list[SearchEvidence]:
|
||||
started_at = perf_counter()
|
||||
if not self.config.enabled:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_search.unavailable",
|
||||
message="WebSearch skipped because integration is disabled",
|
||||
category="ai_tool",
|
||||
level="warning",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={"provider": self.config.default_provider, "reason": "disabled"},
|
||||
)
|
||||
raise WebSearchConfigurationError("WebSearch is disabled.")
|
||||
provider_config = self.config.active_provider_config
|
||||
provider = normalize_web_search_provider(provider_config.provider)
|
||||
if provider != "searxng" and not provider_config.api_key:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_search.unavailable",
|
||||
message="WebSearch skipped because API key is not configured",
|
||||
category="ai_tool",
|
||||
level="warning",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={"provider": provider, "reason": "missing_api_key"},
|
||||
)
|
||||
raise WebSearchConfigurationError(f"{provider} API key is not configured.")
|
||||
query = " ".join(str(query or "").split())
|
||||
if not query:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_search.failed",
|
||||
message="WebSearch failed because query is empty",
|
||||
category="ai_tool",
|
||||
level="warning",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={"provider": provider, "reason": "empty_query"},
|
||||
)
|
||||
raise WebSearchConfigurationError("search query is required.")
|
||||
limit = max_results or provider_config.max_results
|
||||
if provider == "tavily":
|
||||
return await self._search_tavily(provider_config, query, limit, domains, freshness_days)
|
||||
if provider == "brave":
|
||||
return await self._search_brave(provider_config, query, limit, domains)
|
||||
if provider == "serpapi":
|
||||
return await self._search_serpapi(provider_config, query, limit)
|
||||
if provider == "exa":
|
||||
return await self._search_exa(provider_config, query, limit, domains)
|
||||
if provider == "firecrawl":
|
||||
return await self._search_firecrawl(provider_config, query, limit)
|
||||
if provider == "searxng":
|
||||
return await self._search_searxng(provider_config, query, limit, domains)
|
||||
raise WebSearchConfigurationError(f"Unsupported web search provider: {provider}")
|
||||
context = {
|
||||
"provider": provider,
|
||||
"query_hash": hashlib.sha256(query.encode("utf-8")).hexdigest(),
|
||||
"query_length": len(query),
|
||||
"max_results": limit,
|
||||
"domain_count": len(domains or []),
|
||||
"freshness_days": freshness_days,
|
||||
}
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_search.start",
|
||||
message="WebSearch request started",
|
||||
category="ai_tool",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context=context,
|
||||
)
|
||||
try:
|
||||
if provider == "tavily":
|
||||
results = await self._search_tavily(provider_config, query, limit, domains, freshness_days)
|
||||
elif provider == "brave":
|
||||
results = await self._search_brave(provider_config, query, limit, domains)
|
||||
elif provider == "serpapi":
|
||||
results = await self._search_serpapi(provider_config, query, limit)
|
||||
elif provider == "exa":
|
||||
results = await self._search_exa(provider_config, query, limit, domains)
|
||||
elif provider == "firecrawl":
|
||||
results = await self._search_firecrawl(provider_config, query, limit)
|
||||
elif provider == "searxng":
|
||||
results = await self._search_searxng(provider_config, query, limit, domains)
|
||||
else:
|
||||
raise WebSearchConfigurationError(f"Unsupported web search provider: {provider}")
|
||||
event = "ai_tool.web_search.success" if results else "ai_tool.web_search.empty"
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=event,
|
||||
message="WebSearch request completed" if results else "WebSearch returned no results",
|
||||
category="ai_tool",
|
||||
level="info" if results else "warning",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={
|
||||
**context,
|
||||
"status": "success" if results else "empty",
|
||||
"result_count": len(results),
|
||||
"duration_ms": int((perf_counter() - started_at) * 1000),
|
||||
},
|
||||
)
|
||||
return results
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_search.failed",
|
||||
message="WebSearch request failed",
|
||||
category="ai_tool",
|
||||
level="error",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context=exception_context(exc, {**context, "status": "failed", "duration_ms": int((perf_counter() - started_at) * 1000)}),
|
||||
)
|
||||
raise
|
||||
|
||||
async def test_connection(self) -> list[SearchEvidence]:
|
||||
return await self.search("Planet WebSearch connectivity test", max_results=1)
|
||||
@@ -388,4 +474,3 @@ def _float_or_none(value: Any) -> float | None:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
@@ -8,6 +8,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.alert import Alert, AlertSeverity, AlertStatus
|
||||
from app.schemas.ai import AlertBriefRequest, SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
|
||||
ALERT_BRIEF_PROMPT_KEY = "alerts.brief"
|
||||
|
||||
|
||||
def _format_counter(counter: Counter[str], empty_text: str = "无") -> str:
|
||||
@@ -84,11 +87,13 @@ async def build_alert_brief_request(
|
||||
"top_datasources": dict(datasource_counts.most_common(6)),
|
||||
"top_active_datasources": dict(active_datasource_counts.most_common(5)),
|
||||
}
|
||||
prompt = await get_effective_prompt(db, ALERT_BRIEF_PROMPT_KEY)
|
||||
|
||||
return (
|
||||
SituationalAnalysisRequest(
|
||||
title="告警态势 AI 简报",
|
||||
objective="基于当前告警总量、严重度、状态、数据源分布与最近告警摘录,生成一份面向值班人员的简明告警态势简报,突出待处理风险、告警集中点和优先动作。",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=facts,
|
||||
constraints=[
|
||||
"明确区分事实、推断与建议。",
|
||||
|
||||
@@ -11,9 +11,12 @@ from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.models.bgp_observation import BGPObservation
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.bgp_collectors import build_bgp_collector_coverage
|
||||
from app.services.bgp_enrichment import lookup_prefix_geography
|
||||
|
||||
BGP_BRIEF_PROMPT_KEY = "bgp.brief"
|
||||
|
||||
|
||||
def _format_counter(counter: dict[str, int], empty_text: str = "无") -> str:
|
||||
if not counter:
|
||||
@@ -243,12 +246,15 @@ async def build_bgp_brief_request(
|
||||
for prefix, item in list(prefix_geographies.items())[:8]
|
||||
},
|
||||
}
|
||||
prompt = await get_effective_prompt(db, BGP_BRIEF_PROMPT_KEY)
|
||||
|
||||
return SituationalAnalysisRequest(
|
||||
title="BGP 态势 AI 简报",
|
||||
objective="基于当前 BGP incidents、anomalies、原始观测事件、观测站覆盖与 prefix geography 证据,生成一份面向操作员的简明态势简报,突出区域热点、观测偏差、当前风险、证据和优先动作。",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=observations_lines,
|
||||
constraints=[
|
||||
"直接输出中文 Markdown 简报正文,不要输出英文写作计划、提示词复述、字段说明或元评论。",
|
||||
"明确区分事实、推断与建议。",
|
||||
"优先指出需要立即关注的高严重度 incident 或异常模式。",
|
||||
"需要单独指出哪些区域结论来自 prefix geography / affected regions,哪些可能受 collector coverage 偏差影响。",
|
||||
|
||||
117
backend/app/services/business_logs.py
Normal file
117
backend/app/services/business_logs.py
Normal file
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from app.core.logging import PlanetLoggerAdapter, sanitize_log_value
|
||||
from app.core.request_context import get_request_id
|
||||
from app.services.persistent_logs import record_system_log
|
||||
|
||||
|
||||
LEVEL_METHODS = {
|
||||
"debug": "debug_event",
|
||||
"info": "info_event",
|
||||
"warning": "warning_event",
|
||||
"error": "error_event",
|
||||
}
|
||||
|
||||
|
||||
def normalize_business_level(level: str | None) -> str:
|
||||
normalized = str(level or "info").strip().lower()
|
||||
if normalized in {"warn", "warning"}:
|
||||
return "warning"
|
||||
if normalized in {"err", "error", "critical", "fatal"}:
|
||||
return "error"
|
||||
if normalized == "debug":
|
||||
return "debug"
|
||||
return "info"
|
||||
|
||||
|
||||
def build_business_context(
|
||||
context: Mapping[str, Any] | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any]:
|
||||
payload = dict(context or {})
|
||||
for key, value in fields.items():
|
||||
if value is not None:
|
||||
payload[key] = value
|
||||
return sanitize_log_value(payload)
|
||||
|
||||
|
||||
async def emit_business_log(
|
||||
logger: PlanetLoggerAdapter,
|
||||
*,
|
||||
event: str,
|
||||
message: str,
|
||||
category: str,
|
||||
level: str = "info",
|
||||
source: str = "backend",
|
||||
service: str | None = None,
|
||||
module: str | None = None,
|
||||
request_id: str | None = None,
|
||||
user_id: int | None = None,
|
||||
context: Mapping[str, Any] | None = None,
|
||||
) -> None:
|
||||
normalized_level = normalize_business_level(level)
|
||||
safe_context = build_business_context(context)
|
||||
log_method = getattr(logger, LEVEL_METHODS[normalized_level])
|
||||
log_method(message, event=event, context=safe_context)
|
||||
await record_system_log(
|
||||
source=source,
|
||||
level=normalized_level,
|
||||
message=message,
|
||||
service=service,
|
||||
module=module,
|
||||
event=event,
|
||||
request_id=request_id or get_request_id(),
|
||||
user_id=user_id,
|
||||
category=category,
|
||||
context=safe_context,
|
||||
)
|
||||
|
||||
|
||||
def emit_business_log_background(
|
||||
logger: PlanetLoggerAdapter,
|
||||
*,
|
||||
event: str,
|
||||
message: str,
|
||||
category: str,
|
||||
level: str = "info",
|
||||
source: str = "backend",
|
||||
service: str | None = None,
|
||||
module: str | None = None,
|
||||
request_id: str | None = None,
|
||||
user_id: int | None = None,
|
||||
context: Mapping[str, Any] | None = None,
|
||||
) -> None:
|
||||
normalized_level = normalize_business_level(level)
|
||||
safe_context = build_business_context(context)
|
||||
log_method = getattr(logger, LEVEL_METHODS[normalized_level])
|
||||
log_method(message, event=event, context=safe_context)
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return
|
||||
loop.create_task(
|
||||
record_system_log(
|
||||
source=source,
|
||||
level=normalized_level,
|
||||
message=message,
|
||||
service=service,
|
||||
module=module,
|
||||
event=event,
|
||||
request_id=request_id or get_request_id(),
|
||||
user_id=user_id,
|
||||
category=category,
|
||||
context=safe_context,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def exception_context(exc: BaseException, context: Mapping[str, Any] | None = None) -> dict[str, Any]:
|
||||
return build_business_context(
|
||||
context,
|
||||
error_type=type(exc).__name__,
|
||||
error=str(exc),
|
||||
)
|
||||
@@ -36,6 +36,7 @@ from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
|
||||
from app.services.collectors.opengeofeed import OpenGeoFeedPrefixGeoCollector
|
||||
from app.services.collectors.nro_delegated import NRODelegatedPrefixGeoCollector
|
||||
from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
|
||||
from app.services.collectors.media_news_archive import MediaNewsArchiveCollector
|
||||
from app.services.collectors.aisstream import AISStreamCollector
|
||||
from app.services.collectors.vessel_ais import VesselAISCollector
|
||||
|
||||
@@ -65,6 +66,7 @@ collector_registry.register(IPtoASNPrefixGeoCollector())
|
||||
collector_registry.register(OpenGeoFeedPrefixGeoCollector())
|
||||
collector_registry.register(NRODelegatedPrefixGeoCollector())
|
||||
collector_registry.register(NewsLiveStreamsCollector())
|
||||
collector_registry.register(MediaNewsArchiveCollector())
|
||||
collector_registry.register(VesselAISCollector())
|
||||
collector_registry.register(AISStreamCollector())
|
||||
|
||||
@@ -100,6 +102,7 @@ __all__ = [
|
||||
"OpenGeoFeedPrefixGeoCollector",
|
||||
"NRODelegatedPrefixGeoCollector",
|
||||
"NewsLiveStreamsCollector",
|
||||
"MediaNewsArchiveCollector",
|
||||
"VesselAISCollector",
|
||||
"AISStreamCollector",
|
||||
]
|
||||
|
||||
@@ -4,15 +4,22 @@ import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, List, Any, Optional
|
||||
from datetime import UTC, datetime
|
||||
from time import perf_counter
|
||||
from urllib.parse import urlparse
|
||||
import httpx
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.collected_data_fields import build_dynamic_metadata, get_record_field
|
||||
from app.core.config import settings
|
||||
from app.core.countries import normalize_country
|
||||
from app.core.logging import get_logger
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
from app.services.earth_layer_adapters import get_earth_update_layers_for_source
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="collector")
|
||||
|
||||
|
||||
class BaseCollector(ABC):
|
||||
@@ -31,6 +38,7 @@ class BaseCollector(ABC):
|
||||
self._datasource_id = 1
|
||||
self._resolved_url: Optional[str] = None
|
||||
self._last_broadcast_progress: Optional[int] = None
|
||||
self._last_save_summary: dict[str, int] = {}
|
||||
|
||||
async def resolve_url(self, db: AsyncSession) -> None:
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
@@ -186,7 +194,7 @@ class BaseCollector(ABC):
|
||||
|
||||
result = await db.execute(
|
||||
select(DataSnapshot)
|
||||
.where(DataSnapshot.source == self.name, DataSnapshot.is_current == True)
|
||||
.where(DataSnapshot.source == self.name, DataSnapshot.is_current.is_(True))
|
||||
.order_by(DataSnapshot.completed_at.desc().nullslast(), DataSnapshot.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
@@ -272,19 +280,39 @@ class BaseCollector(ABC):
|
||||
from app.models.data_snapshot import DataSnapshot
|
||||
|
||||
start_time = datetime.now(UTC)
|
||||
started_at = perf_counter()
|
||||
datasource_id = getattr(self, "_datasource_id", 1)
|
||||
snapshot_id: Optional[int] = None
|
||||
|
||||
if not collector_registry.is_active(self.name):
|
||||
await self._log_collection_event(
|
||||
"collector.run.skipped_disabled",
|
||||
"Collector skipped because it is disabled",
|
||||
level="info",
|
||||
context={"status": "skipped", "reason": "disabled"},
|
||||
)
|
||||
return {"status": "skipped", "reason": "Collector is disabled"}
|
||||
|
||||
task = CollectionTask(
|
||||
datasource_id=datasource_id,
|
||||
status="running",
|
||||
phase="queued",
|
||||
started_at=start_time,
|
||||
)
|
||||
db.add(task)
|
||||
task = self._current_task if isinstance(self._current_task, CollectionTask) else None
|
||||
if task is None:
|
||||
task = CollectionTask(
|
||||
datasource_id=datasource_id,
|
||||
source=self.name,
|
||||
task_type="collect",
|
||||
status="running",
|
||||
phase="queued",
|
||||
started_at=start_time,
|
||||
)
|
||||
db.add(task)
|
||||
else:
|
||||
task.datasource_id = datasource_id
|
||||
task.source = task.source or self.name
|
||||
task.task_type = task.task_type or "collect"
|
||||
task.status = "running"
|
||||
task.phase = "queued"
|
||||
task.started_at = task.started_at or start_time
|
||||
task.completed_at = None
|
||||
task.error_message = None
|
||||
await db.commit()
|
||||
task_id = task.id
|
||||
|
||||
@@ -294,23 +322,76 @@ class BaseCollector(ABC):
|
||||
|
||||
await self.resolve_url(db)
|
||||
await self._publish_task_update(force=True)
|
||||
await self._log_collection_event(
|
||||
"collector.run.started",
|
||||
"Collector run started",
|
||||
context={"status": "running", "task_id": task_id},
|
||||
)
|
||||
|
||||
try:
|
||||
phase_started_at = perf_counter()
|
||||
await self.set_phase("fetching", message="正在拉取原始数据")
|
||||
await self._log_collection_event(
|
||||
"collector.phase.fetching.start",
|
||||
"Collector fetch phase started",
|
||||
context={"task_id": task_id, "snapshot_id": snapshot_id},
|
||||
)
|
||||
raw_data = await self.fetch()
|
||||
task.total_records = len(raw_data)
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
await self._log_collection_event(
|
||||
"collector.phase.fetching.success",
|
||||
"Collector fetch phase completed",
|
||||
context={
|
||||
"task_id": task_id,
|
||||
"raw_count": len(raw_data),
|
||||
"duration_ms": self._duration_ms(phase_started_at),
|
||||
},
|
||||
)
|
||||
|
||||
if self.fail_on_empty and not raw_data:
|
||||
raise RuntimeError(f"Collector {self.name} returned no data")
|
||||
|
||||
phase_started_at = perf_counter()
|
||||
await self.set_phase("transforming", message="正在转换采集数据")
|
||||
await self._log_collection_event(
|
||||
"collector.phase.transforming.start",
|
||||
"Collector transform phase started",
|
||||
context={"task_id": task_id, "raw_count": len(raw_data)},
|
||||
)
|
||||
data = self.transform(raw_data)
|
||||
await self._log_collection_event(
|
||||
"collector.phase.transforming.success",
|
||||
"Collector transform phase completed",
|
||||
context={
|
||||
"task_id": task_id,
|
||||
"raw_count": len(raw_data),
|
||||
"transformed_count": len(data),
|
||||
"duration_ms": self._duration_ms(phase_started_at),
|
||||
},
|
||||
)
|
||||
snapshot_id = await self._create_snapshot(db, task_id, data, start_time)
|
||||
|
||||
phase_started_at = perf_counter()
|
||||
await self.set_phase("saving", message="正在保存采集数据")
|
||||
await self._log_collection_event(
|
||||
"collector.phase.saving.start",
|
||||
"Collector save phase started",
|
||||
context={"task_id": task_id, "snapshot_id": snapshot_id, "transformed_count": len(data)},
|
||||
)
|
||||
records_count = await self._save_data(db, data, task_id=task_id, snapshot_id=snapshot_id)
|
||||
await self._log_collection_event(
|
||||
"collector.phase.saving.success",
|
||||
"Collector save phase completed",
|
||||
context={
|
||||
"task_id": task_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"saved_count": records_count,
|
||||
**self._last_save_summary,
|
||||
"duration_ms": self._duration_ms(phase_started_at),
|
||||
},
|
||||
)
|
||||
|
||||
task.status = "success"
|
||||
task.phase = "completed"
|
||||
@@ -324,6 +405,20 @@ class BaseCollector(ABC):
|
||||
task.completed_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
await self._log_collection_event(
|
||||
"collector.run.completed",
|
||||
"Collector run completed",
|
||||
context={
|
||||
"status": "success",
|
||||
"task_id": task_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"raw_count": len(raw_data),
|
||||
"transformed_count": len(data),
|
||||
"saved_count": records_count,
|
||||
**self._last_save_summary,
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
@@ -347,6 +442,17 @@ class BaseCollector(ABC):
|
||||
)
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
await self._log_collection_event(
|
||||
"collector.run.cancelled",
|
||||
"Collector run cancelled",
|
||||
level="warning",
|
||||
context={
|
||||
"status": "cancelled",
|
||||
"task_id": task_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
@@ -363,6 +469,20 @@ class BaseCollector(ABC):
|
||||
snapshot.summary = {"error": str(e)}
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
await self._log_collection_event(
|
||||
"collector.run.failed",
|
||||
"Collector run failed",
|
||||
level="error",
|
||||
context=exception_context(
|
||||
e,
|
||||
{
|
||||
"status": "failed",
|
||||
"task_id": task_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "failed",
|
||||
@@ -383,6 +503,7 @@ class BaseCollector(ABC):
|
||||
from app.models.data_snapshot import DataSnapshot
|
||||
|
||||
if not data:
|
||||
self._last_save_summary = {"created": 0, "updated": 0, "unchanged": 0, "deleted": 0}
|
||||
if snapshot_id is not None:
|
||||
snapshot = await db.get(DataSnapshot, snapshot_id)
|
||||
if snapshot:
|
||||
@@ -405,7 +526,7 @@ class BaseCollector(ABC):
|
||||
select(CollectedData)
|
||||
.where(
|
||||
CollectedData.source == self.name,
|
||||
CollectedData.is_current == True,
|
||||
CollectedData.is_current.is_(True),
|
||||
)
|
||||
.order_by(CollectedData.entity_key.asc(), CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
|
||||
)
|
||||
@@ -529,11 +650,51 @@ class BaseCollector(ABC):
|
||||
"unchanged": unchanged_count,
|
||||
"deleted": len(deleted_keys),
|
||||
}
|
||||
self._last_save_summary = {
|
||||
"created": created_count,
|
||||
"updated": updated_count,
|
||||
"unchanged": unchanged_count,
|
||||
"deleted": len(deleted_keys),
|
||||
}
|
||||
else:
|
||||
self._last_save_summary = {
|
||||
"created": created_count,
|
||||
"updated": updated_count,
|
||||
"unchanged": unchanged_count,
|
||||
"deleted": 0,
|
||||
}
|
||||
|
||||
await db.commit()
|
||||
await self.update_progress(len(data), force=True)
|
||||
return records_added
|
||||
|
||||
@staticmethod
|
||||
def _duration_ms(started_at: float) -> int:
|
||||
return int((perf_counter() - started_at) * 1000)
|
||||
|
||||
async def _log_collection_event(
|
||||
self,
|
||||
event: str,
|
||||
message: str,
|
||||
*,
|
||||
level: str = "info",
|
||||
context: Dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=event,
|
||||
message=message,
|
||||
category="collector",
|
||||
level=level,
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
**(context or {}),
|
||||
},
|
||||
)
|
||||
|
||||
async def save(self, db: AsyncSession, data: List[Dict[str, Any]]) -> int:
|
||||
"""Save data to database (legacy method, use _save_data instead)"""
|
||||
return await self._save_data(db, data)
|
||||
@@ -546,10 +707,65 @@ class HTTPCollector(BaseCollector):
|
||||
headers: Dict[str, str] = {}
|
||||
|
||||
async def fetch(self) -> List[Dict[str, Any]]:
|
||||
started_at = perf_counter()
|
||||
request_host = urlparse(self.base_url).netloc
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.http.fetch.start",
|
||||
message="Collector HTTP request started",
|
||||
category="collector",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"url_host": request_host,
|
||||
},
|
||||
)
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.get(self.base_url, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
return self.parse_response(response.json())
|
||||
try:
|
||||
response = await client.get(self.base_url, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
parsed = self.parse_response(payload)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.http.fetch.success",
|
||||
message="Collector HTTP request completed",
|
||||
category="collector",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"url_host": request_host,
|
||||
"status_code": response.status_code,
|
||||
"response_bytes": len(response.content or b""),
|
||||
"parsed_count": len(parsed),
|
||||
"duration_ms": BaseCollector._duration_ms(started_at),
|
||||
},
|
||||
)
|
||||
return parsed
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.http.fetch.failed",
|
||||
message="Collector HTTP request failed",
|
||||
category="collector",
|
||||
level="error",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"url_host": request_host,
|
||||
"duration_ms": BaseCollector._duration_ms(started_at),
|
||||
},
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
@abstractmethod
|
||||
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -19,6 +19,7 @@ from app.services.bgp_collector_locations import (
|
||||
)
|
||||
from app.services.bgp_event_locations import resolve_bgp_event_geo_dict
|
||||
from app.services.bgp_incidents import create_bgp_incidents_for_anomalies
|
||||
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
|
||||
from app.services.bgp_detectors import (
|
||||
detect_mass_withdrawal_anomalies,
|
||||
detect_more_specific_burst_anomalies,
|
||||
@@ -223,6 +224,8 @@ async def save_bgp_observations_for_batch(
|
||||
|
||||
if created:
|
||||
await db.commit()
|
||||
for source in {"ris_live_bgp", "bgpstream_bgp"}:
|
||||
invalidate_earth_layer_cache_for_source(source)
|
||||
|
||||
return created
|
||||
|
||||
|
||||
@@ -1,15 +1,39 @@
|
||||
"""CelesTrak TLE Collector
|
||||
"""CelesTrak TLE Collector.
|
||||
|
||||
Collects satellite TLE (Two-Line Element) data from CelesTrak.org.
|
||||
Free, no authentication required.
|
||||
Collects the full active satellite GP element set from CelesTrak.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Dict, Any, List
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from typing import Any, Dict, List
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.satellite_tle import build_tle_lines_from_elements
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
from app.services.collectors.base import BaseCollector
|
||||
from app.services.collectors.downloads import DownloadHTTPStatusError, ResumableFileDownloader
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="collector")
|
||||
ACTIVE_GROUP = "active"
|
||||
FALLBACK_GROUPS = (
|
||||
"starlink",
|
||||
"gps-ops",
|
||||
"galileo",
|
||||
"glonass",
|
||||
"beidou",
|
||||
"leo",
|
||||
"geo",
|
||||
"iridium-next",
|
||||
)
|
||||
FETCH_RETRY_ATTEMPTS = 3
|
||||
FETCH_RETRY_BASE_DELAY_SECONDS = 0.8
|
||||
CELESTRAK_NOT_UPDATED_MARKER = "GP data has not updated since your last successful"
|
||||
|
||||
|
||||
class CelesTrakTLECollector(BaseCollector):
|
||||
@@ -18,55 +42,359 @@ class CelesTrakTLECollector(BaseCollector):
|
||||
module = "L3"
|
||||
frequency_hours = 24
|
||||
data_type = "satellite_tle"
|
||||
_downloader = ResumableFileDownloader(
|
||||
cache_namespace="celestrak",
|
||||
default_accept="application/json",
|
||||
)
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return self._resolved_url or ""
|
||||
|
||||
def _active_url(self) -> str:
|
||||
return self._group_url(ACTIVE_GROUP)
|
||||
|
||||
def _group_url(self, group: str) -> str:
|
||||
if not self.base_url:
|
||||
raise RuntimeError("CelesTrak base URL is not configured")
|
||||
return f"{self.base_url}?{urlencode({'GROUP': group, 'FORMAT': 'json'})}"
|
||||
|
||||
async def fetch(self) -> List[Dict[str, Any]]:
|
||||
satellite_groups = [
|
||||
"starlink",
|
||||
"gps-ops",
|
||||
"galileo",
|
||||
"glonass",
|
||||
"beidou",
|
||||
"leo",
|
||||
"geo",
|
||||
"iridium-next",
|
||||
]
|
||||
url = self._active_url()
|
||||
last_error: Exception | None = None
|
||||
|
||||
all_satellites = []
|
||||
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
for group in satellite_groups:
|
||||
async with httpx.AsyncClient(timeout=180.0, follow_redirects=True) as client:
|
||||
for attempt in range(1, FETCH_RETRY_ATTEMPTS + 1):
|
||||
started_at = perf_counter()
|
||||
try:
|
||||
url = f"{self.base_url}?GROUP={group}&FORMAT=json"
|
||||
response = await client.get(url)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.download.start",
|
||||
message="CelesTrak active satellite download started",
|
||||
category="collector",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"group": ACTIVE_GROUP,
|
||||
"attempt": attempt,
|
||||
"url_host": urlparse(url).netloc,
|
||||
},
|
||||
)
|
||||
body_path = await self._downloader.download_file(
|
||||
client,
|
||||
url,
|
||||
extension=".json",
|
||||
accept="application/json",
|
||||
progress_callback=self._report_download_progress,
|
||||
validate_existing=self._validate_json_file,
|
||||
)
|
||||
data = await self._load_downloaded_payload(body_path, url)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.download.success",
|
||||
message="CelesTrak active satellite download completed",
|
||||
category="collector",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"group": ACTIVE_GROUP,
|
||||
"attempt": attempt,
|
||||
"record_count": len(data),
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
)
|
||||
return data
|
||||
except DownloadHTTPStatusError as exc:
|
||||
if self._is_not_updated_response(exc):
|
||||
cached_path = self._downloader.get_cached_file(
|
||||
url,
|
||||
".json",
|
||||
validate_existing=self._validate_json_file,
|
||||
)
|
||||
if cached_path is not None:
|
||||
data = await self._load_downloaded_payload(cached_path, url)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.download.cached_not_updated",
|
||||
message="CelesTrak active satellite data has not changed; using cached download",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"group": ACTIVE_GROUP,
|
||||
"attempt": attempt,
|
||||
"record_count": len(data),
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
)
|
||||
return data
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.download.not_updated_no_cache",
|
||||
message="CelesTrak active satellite data has not changed; trying fallback groups",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"group": ACTIVE_GROUP,
|
||||
"attempt": attempt,
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
),
|
||||
)
|
||||
return await self._fetch_fallback_groups(client, active_error=exc)
|
||||
raise
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
is_final_attempt = attempt >= FETCH_RETRY_ATTEMPTS
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=(
|
||||
"collector.celestrak.download.failed"
|
||||
if is_final_attempt
|
||||
else "collector.celestrak.download.retry"
|
||||
),
|
||||
message=(
|
||||
"CelesTrak active satellite download failed"
|
||||
if is_final_attempt
|
||||
else "CelesTrak active satellite download will retry"
|
||||
),
|
||||
category="collector",
|
||||
level="error" if is_final_attempt else "warning",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"group": ACTIVE_GROUP,
|
||||
"attempt": attempt,
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
),
|
||||
)
|
||||
if not is_final_attempt:
|
||||
await asyncio.sleep(FETCH_RETRY_BASE_DELAY_SECONDS * attempt)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if isinstance(data, list):
|
||||
for item in data:
|
||||
if isinstance(item, dict):
|
||||
item["_celestrak_group"] = group
|
||||
all_satellites.extend(data)
|
||||
print(f"CelesTrak: Fetched {len(data)} satellites from group '{group}'")
|
||||
except Exception as e:
|
||||
print(f"CelesTrak: Error fetching group '{group}': {e}")
|
||||
raise RuntimeError(f"CelesTrak active satellite download failed after retries: {last_error}")
|
||||
|
||||
if not all_satellites:
|
||||
return self._get_sample_data()
|
||||
async def _fetch_fallback_groups(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
*,
|
||||
active_error: DownloadHTTPStatusError,
|
||||
) -> List[Dict[str, Any]]:
|
||||
started_at = perf_counter()
|
||||
records_by_norad: dict[str, Dict[str, Any]] = {}
|
||||
group_counts: dict[str, int] = {}
|
||||
|
||||
print(f"CelesTrak: Total satellites fetched: {len(all_satellites)}")
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.fallback_groups.start",
|
||||
message="CelesTrak fallback group download started",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"groups": list(FALLBACK_GROUPS),
|
||||
"reason": "active_not_updated_without_cache",
|
||||
},
|
||||
)
|
||||
|
||||
# Return raw data - base.run() will call transform()
|
||||
return all_satellites
|
||||
try:
|
||||
for group in FALLBACK_GROUPS:
|
||||
group_url = self._group_url(group)
|
||||
try:
|
||||
body_path = await self._downloader.download_file(
|
||||
client,
|
||||
group_url,
|
||||
extension=".json",
|
||||
accept="application/json",
|
||||
validate_existing=self._validate_json_file,
|
||||
)
|
||||
except DownloadHTTPStatusError as exc:
|
||||
if not self._is_not_updated_response(exc):
|
||||
raise RuntimeError(f"CelesTrak fallback group '{group}' download failed: {exc}") from exc
|
||||
cached_path = self._downloader.get_cached_file(
|
||||
group_url,
|
||||
".json",
|
||||
validate_existing=self._validate_json_file,
|
||||
)
|
||||
if cached_path is None:
|
||||
raise RuntimeError(
|
||||
f"CelesTrak fallback group '{group}' has not updated and no local cached copy is available"
|
||||
) from exc
|
||||
body_path = cached_path
|
||||
|
||||
group_records = await self._load_downloaded_payload(
|
||||
body_path,
|
||||
group_url,
|
||||
query_group=group,
|
||||
constellation_group=group,
|
||||
)
|
||||
group_counts[group] = len(group_records)
|
||||
for item in group_records:
|
||||
norad_cat_id = item.get("NORAD_CAT_ID")
|
||||
if norad_cat_id is None:
|
||||
continue
|
||||
records_by_norad.setdefault(str(norad_cat_id), item)
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.fallback_groups.failed",
|
||||
message="CelesTrak fallback group download failed",
|
||||
category="collector",
|
||||
level="error",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"groups": list(FALLBACK_GROUPS),
|
||||
"completed_groups": list(group_counts),
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
),
|
||||
)
|
||||
raise RuntimeError(
|
||||
"CelesTrak active data has not updated since this network's last successful download, "
|
||||
"no active cache is available, and fallback group mode failed. Wait until CelesTrak "
|
||||
"publishes the next GP update, restore the Planet download cache, or use Space-Track."
|
||||
) from active_error
|
||||
|
||||
records = list(records_by_norad.values())
|
||||
if not records:
|
||||
raise RuntimeError("CelesTrak fallback group mode produced no satellite records")
|
||||
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.fallback_groups.success",
|
||||
message="CelesTrak fallback group download completed",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"groups": list(FALLBACK_GROUPS),
|
||||
"group_counts": group_counts,
|
||||
"record_count": len(records),
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
)
|
||||
return records
|
||||
|
||||
async def _load_downloaded_payload(
|
||||
self,
|
||||
body_path: Path,
|
||||
url: str,
|
||||
*,
|
||||
query_group: str = ACTIVE_GROUP,
|
||||
constellation_group: str | None = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
data = self._load_active_payload(body_path)
|
||||
except RuntimeError as exc:
|
||||
await self._log_parse_failure(exc)
|
||||
raise
|
||||
for item in data:
|
||||
item["_celestrak_query_group"] = query_group
|
||||
item["_celestrak_source_url"] = url
|
||||
if constellation_group:
|
||||
item["_celestrak_group"] = constellation_group
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _is_not_updated_response(exc: DownloadHTTPStatusError) -> bool:
|
||||
return exc.status_code == 403 and CELESTRAK_NOT_UPDATED_MARKER in exc.body
|
||||
|
||||
@staticmethod
|
||||
def _duration_ms(started_at: float) -> int:
|
||||
return int((perf_counter() - started_at) * 1000)
|
||||
|
||||
async def _report_download_progress(self, downloaded: int, total: int | None) -> None:
|
||||
if total and total > 0:
|
||||
await self.update_phase_progress(
|
||||
current=min(downloaded, total),
|
||||
total=total,
|
||||
unit="bytes",
|
||||
message=f"正在下载 CelesTrak active 卫星数据 {downloaded}/{total} bytes",
|
||||
commit=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_json_file(path: Path) -> bool:
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, UnicodeDecodeError):
|
||||
return False
|
||||
return isinstance(data, list)
|
||||
|
||||
async def _log_parse_failure(self, exc: Exception) -> None:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.parse.failed",
|
||||
message="CelesTrak active satellite JSON parsing failed",
|
||||
category="collector",
|
||||
level="error",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"group": ACTIVE_GROUP,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
def _load_active_payload(self, path: Path) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
raise RuntimeError(f"CelesTrak active payload is not valid JSON: {exc}") from exc
|
||||
if not isinstance(raw, list):
|
||||
raise RuntimeError("CelesTrak active payload is not a JSON array")
|
||||
|
||||
records: List[Dict[str, Any]] = []
|
||||
invalid_count = 0
|
||||
for item in raw:
|
||||
if isinstance(item, dict) and item.get("NORAD_CAT_ID") is not None:
|
||||
records.append(item)
|
||||
else:
|
||||
invalid_count += 1
|
||||
if invalid_count:
|
||||
raise RuntimeError(f"CelesTrak active payload contains {invalid_count} invalid record(s)")
|
||||
if not records:
|
||||
raise RuntimeError("CelesTrak active payload contains no satellite records")
|
||||
return records
|
||||
|
||||
def transform(self, raw_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
transformed = []
|
||||
for item in raw_data:
|
||||
norad_cat_id = item.get("NORAD_CAT_ID")
|
||||
tle_line1, tle_line2 = build_tle_lines_from_elements(
|
||||
norad_cat_id=item.get("NORAD_CAT_ID"),
|
||||
norad_cat_id=norad_cat_id,
|
||||
epoch=item.get("EPOCH"),
|
||||
inclination=item.get("INCLINATION"),
|
||||
raan=item.get("RA_OF_ASC_NODE"),
|
||||
@@ -75,14 +403,18 @@ class CelesTrakTLECollector(BaseCollector):
|
||||
mean_anomaly=item.get("MEAN_ANOMALY"),
|
||||
mean_motion=item.get("MEAN_MOTION"),
|
||||
)
|
||||
constellation_group = self._infer_constellation_group(item)
|
||||
|
||||
transformed.append(
|
||||
{
|
||||
"source_id": str(norad_cat_id),
|
||||
"name": item.get("OBJECT_NAME", "Unknown"),
|
||||
"reference_date": item.get("EPOCH", ""),
|
||||
"metadata": {
|
||||
"constellation_group": item.get("_celestrak_group"),
|
||||
"norad_cat_id": item.get("NORAD_CAT_ID"),
|
||||
"constellation_group": constellation_group,
|
||||
"celestrak_query_group": item.get("_celestrak_query_group") or ACTIVE_GROUP,
|
||||
"celestrak_source_url": item.get("_celestrak_source_url"),
|
||||
"norad_cat_id": norad_cat_id,
|
||||
"international_designator": item.get("OBJECT_ID"),
|
||||
"epoch": item.get("EPOCH"),
|
||||
"mean_motion": item.get("MEAN_MOTION"),
|
||||
@@ -105,6 +437,19 @@ class CelesTrakTLECollector(BaseCollector):
|
||||
)
|
||||
return transformed
|
||||
|
||||
@staticmethod
|
||||
def _infer_constellation_group(item: Dict[str, Any]) -> str | None:
|
||||
explicit_group = str(item.get("_celestrak_group") or "").strip().lower()
|
||||
if explicit_group and explicit_group != ACTIVE_GROUP:
|
||||
return explicit_group
|
||||
|
||||
name = str(item.get("OBJECT_NAME") or "").strip().upper()
|
||||
if name.startswith("STARLINK"):
|
||||
return "starlink"
|
||||
if name.startswith("IRIDIUM"):
|
||||
return "iridium-next"
|
||||
return None
|
||||
|
||||
def _get_sample_data(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import tempfile
|
||||
import os
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
@@ -17,6 +17,31 @@ ProgressCallback = Callable[[int, int | None], Awaitable[None]]
|
||||
ValidateCallback = Callable[[Path], bool]
|
||||
|
||||
|
||||
class DownloadHTTPStatusError(RuntimeError):
|
||||
"""HTTP status error that keeps the upstream response body for caller-specific handling."""
|
||||
|
||||
def __init__(self, *, url: str, status_code: int, body: str) -> None:
|
||||
self.url = url
|
||||
self.status_code = status_code
|
||||
self.body = body
|
||||
preview = body.strip().replace("\r", " ").replace("\n", " ")[:240]
|
||||
suffix = f": {preview}" if preview else ""
|
||||
super().__init__(f"HTTP {status_code} while downloading {url}{suffix}")
|
||||
|
||||
|
||||
def default_download_cache_root() -> Path:
|
||||
configured = os.getenv("PLANET_DOWNLOAD_CACHE_DIR")
|
||||
if configured:
|
||||
return Path(configured).expanduser()
|
||||
planet_cache = os.getenv("PLANET_CACHE_DIR")
|
||||
if planet_cache:
|
||||
return Path(planet_cache).expanduser() / "downloads"
|
||||
xdg_cache = os.getenv("XDG_CACHE_HOME")
|
||||
if xdg_cache:
|
||||
return Path(xdg_cache).expanduser() / "planet" / "downloads"
|
||||
return Path.home() / ".cache" / "planet" / "downloads"
|
||||
|
||||
|
||||
class ResumableFileDownloader:
|
||||
"""Download files with cache validators and byte-range resume support."""
|
||||
|
||||
@@ -26,8 +51,9 @@ class ResumableFileDownloader:
|
||||
cache_namespace: str,
|
||||
user_agent: str = "Planet-Intelligence-System/1.0 (Python/collector)",
|
||||
default_accept: str = "*/*",
|
||||
cache_root: Path | None = None,
|
||||
) -> None:
|
||||
self._cache_dir = Path(tempfile.gettempdir()) / "planet-download-cache" / cache_namespace
|
||||
self._cache_dir = (cache_root or default_download_cache_root()) / cache_namespace
|
||||
self._user_agent = user_agent
|
||||
self._default_accept = default_accept
|
||||
|
||||
@@ -43,6 +69,25 @@ class ResumableFileDownloader:
|
||||
meta_path = self._cache_dir / f"{key}.meta.json"
|
||||
return final_path, part_path, meta_path
|
||||
|
||||
def cached_file_path(self, url: str, extension: str) -> Path:
|
||||
final_path, _, _ = self._cache_paths(url, extension)
|
||||
return final_path
|
||||
|
||||
def get_cached_file(
|
||||
self,
|
||||
url: str,
|
||||
extension: str,
|
||||
*,
|
||||
validate_existing: ValidateCallback | None = None,
|
||||
) -> Path | None:
|
||||
final_path = self.cached_file_path(url, extension)
|
||||
if not final_path.exists():
|
||||
return None
|
||||
if validate_existing and not validate_existing(final_path):
|
||||
final_path.unlink(missing_ok=True)
|
||||
return None
|
||||
return final_path
|
||||
|
||||
@staticmethod
|
||||
def _load_meta(meta_path: Path) -> dict[str, Any]:
|
||||
if not meta_path.exists():
|
||||
@@ -140,7 +185,9 @@ class ResumableFileDownloader:
|
||||
if progress_callback and expected_size and expected_size > 0:
|
||||
await progress_callback(expected_size, expected_size)
|
||||
return final_path
|
||||
response.raise_for_status()
|
||||
if response.status_code >= 400:
|
||||
body = (await response.aread()).decode("utf-8", errors="replace")
|
||||
raise DownloadHTTPStatusError(url=url, status_code=response.status_code, body=body)
|
||||
|
||||
if response.status_code == 206 and resume_from > 0:
|
||||
mode = "ab"
|
||||
|
||||
57
backend/app/services/collectors/media_news_archive.py
Normal file
57
backend/app/services/collectors/media_news_archive.py
Normal file
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.services.collectors.base import BaseCollector
|
||||
from app.services.earth_news_store import list_all_earth_news_records
|
||||
|
||||
|
||||
class MediaNewsArchiveCollector(BaseCollector):
|
||||
name = "media_news_archive"
|
||||
priority = "P2"
|
||||
module = "L4"
|
||||
frequency_hours = 12
|
||||
data_type = "news_item"
|
||||
fail_on_empty = False
|
||||
|
||||
async def fetch(self) -> list[dict[str, Any]]:
|
||||
if not self._db_session:
|
||||
return []
|
||||
|
||||
records = await list_all_earth_news_records(self._db_session)
|
||||
items: list[dict[str, Any]] = []
|
||||
for record in records:
|
||||
location_meta = dict(record.location_meta or {})
|
||||
target = location_meta.get("target") if isinstance(location_meta.get("target"), dict) else {}
|
||||
country = target.get("country")
|
||||
city = target.get("city")
|
||||
items.append(
|
||||
{
|
||||
"id": record.id,
|
||||
"source_id": record.id,
|
||||
"name": record.title,
|
||||
"title": record.title,
|
||||
"description": record.summary,
|
||||
"country": country,
|
||||
"city": city,
|
||||
"latitude": record.latitude,
|
||||
"longitude": record.longitude,
|
||||
"reference_date": record.published_at,
|
||||
"metadata": {
|
||||
"url": record.url,
|
||||
"source": record.source,
|
||||
"feed_name": record.feed_name,
|
||||
"region": record.region,
|
||||
"homepage_url": record.homepage_url,
|
||||
"published_at": record.published_at.isoformat() if record.published_at else None,
|
||||
"location_label": record.location_label,
|
||||
"location_source": record.location_source,
|
||||
"verified": record.verified,
|
||||
"location_meta": location_meta,
|
||||
"first_seen_at": record.first_seen_at.isoformat() if record.first_seen_at else None,
|
||||
"last_seen_at": record.last_seen_at.isoformat() if record.last_seen_at else None,
|
||||
"resolved_at": record.resolved_at.isoformat() if record.resolved_at else None,
|
||||
},
|
||||
}
|
||||
)
|
||||
return items
|
||||
@@ -163,32 +163,64 @@ class TeleGeographyLandingPointCollector(BaseCollector):
|
||||
data_type = "landing_point"
|
||||
|
||||
async def fetch(self) -> List[Dict[str, Any]]:
|
||||
"""Fetch landing point data from GitHub mirror"""
|
||||
url = self._resolved_url or ""
|
||||
"""Fetch landing point data, falling back when the old mirror disappears."""
|
||||
config = get_data_sources_config()
|
||||
sources = [
|
||||
self._resolved_url or "",
|
||||
str(config.get_yaml_value("telegeography.landing_point_url") or ""),
|
||||
str(config.get_yaml_value("arcgis.landing_point_url") or ""),
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
return self.parse_response(response.json())
|
||||
last_error: Exception | None = None
|
||||
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
|
||||
for url in dict.fromkeys(source for source in sources if source):
|
||||
try:
|
||||
params = (
|
||||
{"where": "1=1", "outFields": "*", "returnGeometry": "true", "f": "geojson"}
|
||||
if "FeatureServer" in url or url.endswith("/query")
|
||||
else None
|
||||
)
|
||||
response = await client.get(url, params=params)
|
||||
response.raise_for_status()
|
||||
records = self.parse_response(response.json())
|
||||
if records:
|
||||
return records
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
continue
|
||||
|
||||
def parse_response(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
if last_error:
|
||||
raise last_error
|
||||
return self._get_sample_data()
|
||||
|
||||
def parse_response(self, data: List[Dict[str, Any]] | Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Parse landing point data"""
|
||||
result = []
|
||||
items = data.get("features", []) if isinstance(data, dict) else data
|
||||
|
||||
for item in data:
|
||||
for item in items:
|
||||
props = item.get("properties", {}) if isinstance(item, dict) else {}
|
||||
geometry = item.get("geometry", {}) if isinstance(item, dict) else {}
|
||||
source = props or item
|
||||
coords = geometry.get("coordinates", []) if isinstance(geometry, dict) else []
|
||||
longitude = coords[0] if len(coords) > 0 else source.get("longitude")
|
||||
latitude = coords[1] if len(coords) > 1 else source.get("latitude")
|
||||
source_id = source.get("id") or source.get("OBJECTID") or source.get("city_id") or ""
|
||||
try:
|
||||
entry = {
|
||||
"source_id": f"telegeo_lp_{item.get('id', '')}",
|
||||
"name": item.get("name", "Unknown"),
|
||||
"country": item.get("country", "Unknown"),
|
||||
"city": item.get("city", item.get("name", "")),
|
||||
"latitude": str(item.get("latitude", "")),
|
||||
"longitude": str(item.get("longitude", "")),
|
||||
"source_id": f"telegeo_lp_{source_id}",
|
||||
"name": source.get("name", source.get("Name", "Unknown")),
|
||||
"country": source.get("country", "Unknown"),
|
||||
"city": source.get("city", source.get("Name", source.get("name", ""))),
|
||||
"latitude": str(latitude or ""),
|
||||
"longitude": str(longitude or ""),
|
||||
"value": "",
|
||||
"unit": "",
|
||||
"metadata": {
|
||||
"cable_count": len(item.get("cables", [])),
|
||||
"url": item.get("url"),
|
||||
"cable_count": len(source.get("cables", [])),
|
||||
"url": source.get("url"),
|
||||
"objectid": source.get("OBJECTID"),
|
||||
"city_id": source.get("city_id"),
|
||||
},
|
||||
"reference_date": datetime.now(UTC).strftime("%Y-%m-%d"),
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ startup, so it must stay local and deterministic.
|
||||
|
||||
For the full design and the reason behind the abstraction (compute centers,
|
||||
BGP collectors, BGP events, and future entities all share one pipeline),
|
||||
see ``docs/plans/location-resolver-shared-pipeline-plan.md``.
|
||||
see ``docs/technical/zh/location-pipeline-development.md``.
|
||||
|
||||
The ``ComputeCenterLocation`` dataclass and the public function signatures are
|
||||
preserved verbatim so existing callers and tests do not need to change.
|
||||
|
||||
@@ -3,10 +3,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm.attributes import flag_modified
|
||||
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient
|
||||
@@ -15,6 +18,7 @@ from app.services.ai_tools.web_search import WebSearchClient, WebSearchError
|
||||
|
||||
|
||||
CREDENTIAL_GUIDES_CATEGORY = "collector_credential_guides"
|
||||
CREDENTIAL_GUIDE_PROMPT_KEY = "credential.guide"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -133,33 +137,57 @@ DEFAULT_CREDENTIAL_GUIDES = {
|
||||
}
|
||||
|
||||
|
||||
def _normalize_provider(provider: str) -> str:
|
||||
return provider.strip().lower().replace(" ", "_")
|
||||
|
||||
|
||||
def _credential_guide_default(provider: str) -> CredentialGuideDefault:
|
||||
normalized = _normalize_provider(provider)
|
||||
known = DEFAULT_CREDENTIAL_GUIDES.get(normalized)
|
||||
if known is not None:
|
||||
return known
|
||||
title = f"{normalized or 'collector'} 凭证配置教程"
|
||||
return CredentialGuideDefault(
|
||||
provider=normalized,
|
||||
title=title,
|
||||
prompt=(
|
||||
f"请生成一份中文教程,指导开发者为 Planet 采集器配置 {normalized} 凭证。"
|
||||
"教程要面向已经有本地开发环境的人,包含官方入口或文档查找方式、"
|
||||
"获取 API Key / Token / Client credentials 的通用步骤、在 Planet 采集器配置中"
|
||||
"填写凭证字段、连接测试、保存、常见失败排查。不要编造具体页面按钮文案;"
|
||||
"如果公开资料不足,必须明确提醒以 provider 官方文档和当前控制台页面为准。"
|
||||
),
|
||||
markdown="",
|
||||
)
|
||||
|
||||
|
||||
async def _get_guide_store(db) -> tuple[SystemSetting | None, dict[str, Any]]:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == CREDENTIAL_GUIDES_CATEGORY)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
payload = dict(record.payload or {}) if record and isinstance(record.payload, dict) else {}
|
||||
payload = deepcopy(record.payload) if record and isinstance(record.payload, dict) else {}
|
||||
return record, payload
|
||||
|
||||
|
||||
async def get_credential_guide(db, provider: str) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
provider = _normalize_provider(provider)
|
||||
default = _credential_guide_default(provider)
|
||||
|
||||
_record, store = await _get_guide_store(db)
|
||||
custom = store.get(provider) if isinstance(store.get(provider), dict) else None
|
||||
has_default_markdown = bool(default.markdown.strip())
|
||||
return {
|
||||
"provider": provider,
|
||||
"title": custom.get("title") if custom else default.title,
|
||||
"markdown": custom.get("markdown") if custom else default.markdown,
|
||||
"prompt": default.prompt,
|
||||
"source": "ai" if custom else "default",
|
||||
"source": "ai" if custom else "default" if has_default_markdown else "missing",
|
||||
"sources": custom.get("sources", []) if custom else [],
|
||||
"verification_status": (
|
||||
custom.get("verification_status", "verified_with_search_evidence")
|
||||
if custom
|
||||
else "default_unverified"
|
||||
else "default_unverified" if has_default_markdown else "missing"
|
||||
),
|
||||
"verification_error": custom.get("verification_error") if custom else None,
|
||||
}
|
||||
@@ -175,9 +203,8 @@ async def save_credential_guide(
|
||||
verification_status: str = "verified_with_search_evidence",
|
||||
verification_error: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
provider = _normalize_provider(provider)
|
||||
default = _credential_guide_default(provider)
|
||||
|
||||
record, store = await _get_guide_store(db)
|
||||
store[provider] = {
|
||||
@@ -190,21 +217,21 @@ async def save_credential_guide(
|
||||
if record is None:
|
||||
db.add(SystemSetting(category=CREDENTIAL_GUIDES_CATEGORY, payload=store))
|
||||
else:
|
||||
record.payload = store
|
||||
record.payload = deepcopy(store)
|
||||
flag_modified(record, "payload")
|
||||
await db.commit()
|
||||
return await get_credential_guide(db, provider)
|
||||
|
||||
|
||||
async def reset_credential_guide(db, provider: str) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
provider = _normalize_provider(provider)
|
||||
|
||||
record, store = await _get_guide_store(db)
|
||||
if provider in store:
|
||||
store.pop(provider, None)
|
||||
if record is not None:
|
||||
record.payload = store
|
||||
record.payload = deepcopy(store)
|
||||
flag_modified(record, "payload")
|
||||
await db.commit()
|
||||
return await get_credential_guide(db, provider)
|
||||
|
||||
@@ -215,9 +242,8 @@ async def generate_credential_guide(
|
||||
ai_client: AIProviderClient,
|
||||
web_search_client: WebSearchClient | None = None,
|
||||
) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
provider = _normalize_provider(provider)
|
||||
default = _credential_guide_default(provider)
|
||||
|
||||
search_evidence: list[dict[str, Any]] = []
|
||||
search_error: str | None = None
|
||||
@@ -240,14 +266,12 @@ async def generate_credential_guide(
|
||||
guide["sources"] = []
|
||||
return guide
|
||||
|
||||
prompt = await get_effective_prompt(db, CREDENTIAL_GUIDE_PROMPT_KEY)
|
||||
response = await ai_client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title=f"Generate credential guide for {provider}",
|
||||
objective=(
|
||||
default.prompt
|
||||
+ "\n只能根据 context.search_evidence 中的来源生成教程;"
|
||||
+ "如果证据不足,明确说明需要以官方页面为准。"
|
||||
),
|
||||
objective=f"{default.prompt}\n{prompt.prompt}",
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"provider": provider,
|
||||
"current_default_guide": default.markdown,
|
||||
|
||||
597
backend/app/services/data_jobs.py
Normal file
597
backend/app/services/data_jobs.py
Normal file
@@ -0,0 +1,597 @@
|
||||
"""Kafka-ready datasource job queue backed by PostgreSQL for v1."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import bindparam, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.cache import cache
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.db.session import async_session_factory
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.data_snapshot import DataSnapshot
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.task import CollectionTask
|
||||
from app.services.collectors.registry import collector_registry
|
||||
from app.services.datasource_connectivity import (
|
||||
build_builtin_connectivity_checksum,
|
||||
get_builtin_effective_candidate,
|
||||
save_connectivity_success,
|
||||
)
|
||||
from app.services.earth_layer_adapters import (
|
||||
clear_derived_datasource_data,
|
||||
get_earth_refresh_strategy_for_change,
|
||||
get_earth_update_layers_for_source,
|
||||
)
|
||||
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
|
||||
from app.services.scheduler import sync_datasource_job
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
JOB_TYPE_COLLECT = "collect"
|
||||
JOB_TYPE_CLEAR_DATA = "clear_data"
|
||||
JOB_TYPE_CLEAR_CACHE = "clear_cache"
|
||||
JOB_TYPE_EARTH_REFRESH = "earth_refresh"
|
||||
|
||||
JOB_STATUS_QUEUED = "queued"
|
||||
JOB_STATUS_RUNNING = "running"
|
||||
JOB_STATUS_CANCELLING = "cancelling"
|
||||
JOB_STATUS_SUCCESS = "success"
|
||||
JOB_STATUS_FAILED = "failed"
|
||||
JOB_STATUS_CANCELLED = "cancelled"
|
||||
|
||||
ACTIVE_JOB_STATUSES = (JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)
|
||||
TERMINAL_JOB_STATUSES = (JOB_STATUS_SUCCESS, JOB_STATUS_FAILED, JOB_STATUS_CANCELLED)
|
||||
DATA_WRITE_JOB_TYPES = (JOB_TYPE_COLLECT, JOB_TYPE_CLEAR_DATA, JOB_TYPE_CLEAR_CACHE)
|
||||
SOURCE_LOCK_JOB_STATUSES = (JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)
|
||||
QUEUE_POLL_SECONDS = 0.35
|
||||
JOB_STALE_LOCK_MINUTES = 90
|
||||
DEFAULT_WORKER_CONCURRENCY = 2
|
||||
|
||||
RUNNING_DATA_JOB_TASKS: dict[int, asyncio.Task[Any]] = {}
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _job_worker_id() -> str:
|
||||
return f"{settings.PROJECT_NAME}:data-job-worker:{uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def is_terminal_job_status(status: str | None) -> bool:
|
||||
return status in TERMINAL_JOB_STATUSES
|
||||
|
||||
|
||||
async def enqueue_datasource_job(
|
||||
db: AsyncSession,
|
||||
datasource: DataSource,
|
||||
task_type: str,
|
||||
*,
|
||||
payload: dict[str, Any] | None = None,
|
||||
rollback_policy: str = "keep_committed_batches",
|
||||
dedupe_key: str | None = None,
|
||||
) -> CollectionTask:
|
||||
if dedupe_key:
|
||||
existing = await _get_active_job_by_dedupe_key(db, dedupe_key)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
task = CollectionTask(
|
||||
datasource_id=datasource.id,
|
||||
source=datasource.source,
|
||||
task_type=task_type,
|
||||
status=JOB_STATUS_QUEUED,
|
||||
phase="queued",
|
||||
phase_message="任务已进入队列",
|
||||
payload=payload or {},
|
||||
rollback_policy=rollback_policy,
|
||||
dedupe_key=dedupe_key,
|
||||
)
|
||||
db.add(task)
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
await _broadcast_task_update(task)
|
||||
return task
|
||||
|
||||
|
||||
async def enqueue_earth_refresh_job(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
) -> CollectionTask | None:
|
||||
layers = list((payload or {}).get("layers") or get_earth_update_layers_for_source(source))
|
||||
if not layers:
|
||||
return None
|
||||
|
||||
datasource = await _get_or_create_virtual_datasource(db, source)
|
||||
refresh_payload = {
|
||||
"source": source,
|
||||
"layers": layers,
|
||||
"refresh_strategy": (payload or {}).get("refresh_strategy")
|
||||
or get_earth_refresh_strategy_for_change((payload or {}).get("table"), source)
|
||||
or "clear_then_reload",
|
||||
**(payload or {}),
|
||||
}
|
||||
return await enqueue_datasource_job(
|
||||
db,
|
||||
datasource,
|
||||
JOB_TYPE_EARTH_REFRESH,
|
||||
payload=refresh_payload,
|
||||
dedupe_key=f"earth_refresh:{source}",
|
||||
)
|
||||
|
||||
|
||||
async def enqueue_earth_refresh_from_update(payload: dict[str, Any]) -> None:
|
||||
source = str(payload.get("source") or "").strip()
|
||||
if not source:
|
||||
return
|
||||
async with async_session_factory() as db:
|
||||
await enqueue_earth_refresh_job(db, source=source, payload=payload)
|
||||
|
||||
|
||||
async def request_cancel_datasource_task(
|
||||
db: AsyncSession,
|
||||
task: CollectionTask,
|
||||
*,
|
||||
reason: str = "cancelled_by_operator",
|
||||
) -> CollectionTask:
|
||||
if is_terminal_job_status(task.status):
|
||||
return task
|
||||
|
||||
running_task = RUNNING_DATA_JOB_TASKS.get(task.id)
|
||||
if task.status == JOB_STATUS_QUEUED or (
|
||||
running_task is None
|
||||
and (
|
||||
task.status == JOB_STATUS_CANCELLING
|
||||
or (task.status == JOB_STATUS_RUNNING and task.task_type != JOB_TYPE_COLLECT)
|
||||
)
|
||||
):
|
||||
return await _cancel_task_without_runner(db, task, reason=reason)
|
||||
|
||||
task.status = JOB_STATUS_CANCELLING
|
||||
task.phase = JOB_STATUS_CANCELLING
|
||||
task.phase_message = "正在停止任务"
|
||||
task.requested_cancel_at = _utcnow()
|
||||
task.cancel_reason = reason
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
|
||||
if running_task is not None and not running_task.done():
|
||||
running_task.cancel()
|
||||
|
||||
await _broadcast_task_update(task)
|
||||
return task
|
||||
|
||||
|
||||
async def _cancel_task_without_runner(
|
||||
db: AsyncSession,
|
||||
task: CollectionTask,
|
||||
*,
|
||||
reason: str,
|
||||
) -> CollectionTask:
|
||||
if task.task_type == JOB_TYPE_COLLECT:
|
||||
await db.execute(CollectedData.__table__.delete().where(CollectedData.task_id == task.id))
|
||||
snapshot_result = await db.execute(select(DataSnapshot).where(DataSnapshot.task_id == task.id))
|
||||
for snapshot in snapshot_result.scalars().all():
|
||||
snapshot.status = JOB_STATUS_CANCELLED
|
||||
snapshot.completed_at = _utcnow()
|
||||
snapshot.error_message = "Cancelled after operator stop request; no active worker handle remained"
|
||||
datasource = await db.get(DataSource, task.datasource_id)
|
||||
if datasource is not None:
|
||||
datasource.last_status = JOB_STATUS_CANCELLED
|
||||
|
||||
task.status = JOB_STATUS_CANCELLED
|
||||
task.phase = JOB_STATUS_CANCELLED
|
||||
task.phase_message = "任务已停止"
|
||||
task.completed_at = _utcnow()
|
||||
task.requested_cancel_at = task.requested_cancel_at or _utcnow()
|
||||
task.cancel_reason = reason
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
await _broadcast_task_update(task)
|
||||
return task
|
||||
|
||||
|
||||
async def get_active_datasource_job(
|
||||
db: AsyncSession,
|
||||
datasource_id: int,
|
||||
*,
|
||||
task_types: tuple[str, ...] = DATA_WRITE_JOB_TYPES,
|
||||
) -> CollectionTask | None:
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.datasource_id == datasource_id)
|
||||
.where(CollectionTask.task_type.in_(task_types))
|
||||
.where(CollectionTask.status.in_(ACTIVE_JOB_STATUSES))
|
||||
.order_by(CollectionTask.created_at.desc().nullslast(), CollectionTask.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _get_active_job_by_dedupe_key(db: AsyncSession, dedupe_key: str) -> CollectionTask | None:
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.dedupe_key == dedupe_key)
|
||||
.where(CollectionTask.status.in_(ACTIVE_JOB_STATUSES))
|
||||
.order_by(CollectionTask.created_at.desc().nullslast(), CollectionTask.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _get_or_create_virtual_datasource(db: AsyncSession, source: str) -> DataSource:
|
||||
result = await db.execute(select(DataSource).where(DataSource.source == source))
|
||||
datasource = result.scalar_one_or_none()
|
||||
if datasource is not None:
|
||||
return datasource
|
||||
|
||||
datasource = DataSource(
|
||||
name=f"Earth refresh: {source}",
|
||||
source=source,
|
||||
module="SYS",
|
||||
collector_class="EarthRefreshJob",
|
||||
is_active=True,
|
||||
)
|
||||
db.add(datasource)
|
||||
await db.commit()
|
||||
await db.refresh(datasource)
|
||||
return datasource
|
||||
|
||||
|
||||
async def _broadcast_task_update(task: CollectionTask) -> None:
|
||||
await broadcaster.broadcast_datasource_task_update(
|
||||
{
|
||||
"datasource_id": task.datasource_id,
|
||||
"collector_name": task.source,
|
||||
"task_id": task.id,
|
||||
"task_type": task.task_type,
|
||||
"status": task.status,
|
||||
"phase": task.phase,
|
||||
"phase_progress": task.phase_progress,
|
||||
"phase_message": task.phase_message,
|
||||
"phase_current": task.phase_current,
|
||||
"phase_total": task.phase_total,
|
||||
"phase_unit": task.phase_unit,
|
||||
"progress": task.progress,
|
||||
"records_processed": task.records_processed,
|
||||
"total_records": task.total_records,
|
||||
"started_at": to_iso8601_utc(task.started_at),
|
||||
"completed_at": to_iso8601_utc(task.completed_at),
|
||||
"requested_cancel_at": to_iso8601_utc(task.requested_cancel_at),
|
||||
"error_message": task.error_message,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class DataJobWorker:
|
||||
def __init__(self, *, concurrency: int = DEFAULT_WORKER_CONCURRENCY) -> None:
|
||||
self.worker_id = _job_worker_id()
|
||||
self.concurrency = max(1, concurrency)
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._stop_event: asyncio.Event | None = None
|
||||
self._running: set[asyncio.Task[Any]] = set()
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._stop_event = asyncio.Event()
|
||||
self._task = asyncio.create_task(self._run(), name="data-job-worker")
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._stop_event:
|
||||
self._stop_event.set()
|
||||
for task in list(self._running):
|
||||
task.cancel()
|
||||
if self._task:
|
||||
await asyncio.gather(self._task, return_exceptions=True)
|
||||
if self._running:
|
||||
await asyncio.gather(*self._running, return_exceptions=True)
|
||||
|
||||
async def _run(self) -> None:
|
||||
assert self._stop_event is not None
|
||||
await self._recover_stale_running_jobs()
|
||||
while not self._stop_event.is_set():
|
||||
self._running = {task for task in self._running if not task.done()}
|
||||
if len(self._running) >= self.concurrency:
|
||||
await asyncio.sleep(QUEUE_POLL_SECONDS)
|
||||
continue
|
||||
|
||||
task_id = await self._claim_next_job()
|
||||
if task_id is None:
|
||||
await asyncio.sleep(QUEUE_POLL_SECONDS)
|
||||
continue
|
||||
|
||||
runner = asyncio.create_task(self._run_claimed_job(task_id), name=f"data-job:{task_id}")
|
||||
self._running.add(runner)
|
||||
|
||||
async def _recover_stale_running_jobs(self) -> None:
|
||||
cutoff = _utcnow() - timedelta(minutes=JOB_STALE_LOCK_MINUTES)
|
||||
async with async_session_factory() as db:
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.status.in_((JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)))
|
||||
.where(CollectionTask.locked_at.is_not(None))
|
||||
.where(CollectionTask.locked_at < cutoff)
|
||||
)
|
||||
stale_jobs = list(result.scalars().all())
|
||||
for job in stale_jobs:
|
||||
job.status = JOB_STATUS_FAILED
|
||||
job.phase = JOB_STATUS_FAILED
|
||||
job.completed_at = _utcnow()
|
||||
job.error_message = "Marked failed after stale data job lock timeout"
|
||||
if stale_jobs:
|
||||
await db.commit()
|
||||
|
||||
async def _claim_next_job(self) -> int | None:
|
||||
async with async_session_factory() as db:
|
||||
row = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT queued.id
|
||||
FROM collection_tasks AS queued
|
||||
WHERE queued.status = :queued_status
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM collection_tasks AS active
|
||||
WHERE active.source = queued.source
|
||||
AND active.id <> queued.id
|
||||
AND active.status IN :active_statuses
|
||||
)
|
||||
ORDER BY queued.created_at ASC NULLS FIRST, queued.id ASC
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
"""
|
||||
).bindparams(bindparam("active_statuses", expanding=True)),
|
||||
{
|
||||
"queued_status": JOB_STATUS_QUEUED,
|
||||
"active_statuses": SOURCE_LOCK_JOB_STATUSES,
|
||||
},
|
||||
)
|
||||
task_id = row.scalar_one_or_none()
|
||||
if task_id is None:
|
||||
return None
|
||||
|
||||
task = await db.get(CollectionTask, int(task_id))
|
||||
if task is None:
|
||||
return None
|
||||
task.status = JOB_STATUS_RUNNING
|
||||
task.phase = "starting"
|
||||
task.phase_message = "任务开始执行"
|
||||
task.started_at = task.started_at or _utcnow()
|
||||
task.worker_id = self.worker_id
|
||||
task.locked_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
return int(task_id)
|
||||
|
||||
async def _run_claimed_job(self, task_id: int) -> None:
|
||||
current_task = asyncio.current_task()
|
||||
if current_task is not None:
|
||||
RUNNING_DATA_JOB_TASKS[task_id] = current_task
|
||||
try:
|
||||
async with async_session_factory() as db:
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if task is None:
|
||||
return
|
||||
await self._execute_job(db, task)
|
||||
except asyncio.CancelledError:
|
||||
async with async_session_factory() as db:
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if task is not None and not is_terminal_job_status(task.status):
|
||||
task.status = JOB_STATUS_CANCELLED
|
||||
task.phase = JOB_STATUS_CANCELLED
|
||||
task.phase_message = "任务已停止"
|
||||
task.completed_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception_event(
|
||||
"Data job failed",
|
||||
event="data_jobs.job_failed",
|
||||
context={"task_id": task_id, "error": str(exc)},
|
||||
)
|
||||
async with async_session_factory() as db:
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if task is not None:
|
||||
task.status = JOB_STATUS_FAILED
|
||||
task.phase = JOB_STATUS_FAILED
|
||||
task.phase_message = str(exc)
|
||||
task.error_message = str(exc)
|
||||
task.completed_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
finally:
|
||||
RUNNING_DATA_JOB_TASKS.pop(task_id, None)
|
||||
|
||||
async def _execute_job(self, db: AsyncSession, task: CollectionTask) -> None:
|
||||
if task.task_type == JOB_TYPE_COLLECT:
|
||||
await _run_collect_job(db, task)
|
||||
elif task.task_type == JOB_TYPE_CLEAR_DATA:
|
||||
await _run_clear_data_job(db, task)
|
||||
elif task.task_type == JOB_TYPE_CLEAR_CACHE:
|
||||
await _run_clear_cache_job(db, task)
|
||||
elif task.task_type == JOB_TYPE_EARTH_REFRESH:
|
||||
await _run_earth_refresh_job(db, task)
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported data job type: {task.task_type}")
|
||||
|
||||
|
||||
async def _run_collect_job(db: AsyncSession, task: CollectionTask) -> None:
|
||||
datasource = await db.get(DataSource, task.datasource_id)
|
||||
if datasource is None:
|
||||
raise RuntimeError("Data source not found")
|
||||
|
||||
collector = collector_registry.get(datasource.source)
|
||||
if collector is None:
|
||||
raise RuntimeError(f"Collector '{datasource.source}' not found")
|
||||
if not datasource.is_active:
|
||||
raise RuntimeError("Data source is disabled")
|
||||
|
||||
collector._datasource_id = datasource.id
|
||||
collector._current_task = task
|
||||
collector._db_session = db
|
||||
result = await collector.run(db)
|
||||
|
||||
datasource.last_run_at = _utcnow()
|
||||
datasource.last_status = result.get("status")
|
||||
if datasource.last_status == JOB_STATUS_SUCCESS:
|
||||
effective_candidate = await get_builtin_effective_candidate(db, datasource.source)
|
||||
checksum, _credential_context = await build_builtin_connectivity_checksum(
|
||||
datasource.source,
|
||||
effective_candidate["endpoint"],
|
||||
effective_candidate["auth_type"],
|
||||
effective_candidate["headers"],
|
||||
effective_candidate["config"],
|
||||
db,
|
||||
)
|
||||
await save_connectivity_success(
|
||||
db,
|
||||
datasource.source,
|
||||
checksum,
|
||||
{"status_code": None},
|
||||
connected_by="collection",
|
||||
)
|
||||
await db.commit()
|
||||
await sync_datasource_job(datasource.id)
|
||||
|
||||
|
||||
async def _run_clear_data_job(db: AsyncSession, task: CollectionTask) -> None:
|
||||
source = str(task.source or (task.payload or {}).get("source") or "").strip()
|
||||
if not source:
|
||||
raise RuntimeError("Clear data job has no source")
|
||||
|
||||
task.phase = "clearing_data"
|
||||
task.phase_message = "正在删除数据库数据"
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
|
||||
count_result = await db.execute(
|
||||
select(CollectedData.id).where(CollectedData.source == source)
|
||||
)
|
||||
collected_ids = [row[0] for row in count_result.all()]
|
||||
derived_deleted_counts = await clear_derived_datasource_data(db, source)
|
||||
if collected_ids:
|
||||
await db.execute(CollectedData.__table__.delete().where(CollectedData.id.in_(collected_ids)))
|
||||
deleted_count = len(collected_ids)
|
||||
derived_deleted_count = sum(derived_deleted_counts.values())
|
||||
|
||||
task.records_processed = deleted_count + derived_deleted_count
|
||||
task.total_records = task.records_processed
|
||||
task.progress = 100.0
|
||||
task.phase_progress = 100.0
|
||||
task.phase_current = task.records_processed
|
||||
task.phase_total = task.records_processed
|
||||
task.phase_unit = "records"
|
||||
task.payload = {
|
||||
**(task.payload or {}),
|
||||
"deleted_count": deleted_count,
|
||||
"derived_deleted_count": derived_deleted_count,
|
||||
"derived_deleted_counts": derived_deleted_counts,
|
||||
}
|
||||
task.status = JOB_STATUS_SUCCESS
|
||||
task.phase = "completed"
|
||||
task.phase_message = "数据库数据已清理"
|
||||
task.completed_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
|
||||
|
||||
async def _run_clear_cache_job(db: AsyncSession, task: CollectionTask) -> None:
|
||||
source = str(task.source or (task.payload or {}).get("source") or "").strip()
|
||||
if not source:
|
||||
raise RuntimeError("Clear cache job has no source")
|
||||
|
||||
earth_deleted_count = invalidate_earth_layer_cache_for_source(source)
|
||||
dashboard_deleted_count = int(cache.delete("dashboard:stats")) + int(cache.delete("dashboard:summary"))
|
||||
deleted_count = earth_deleted_count + dashboard_deleted_count
|
||||
|
||||
task.records_processed = deleted_count
|
||||
task.total_records = deleted_count
|
||||
task.progress = 100.0
|
||||
task.phase_progress = 100.0
|
||||
task.phase = "completed"
|
||||
task.phase_message = "缓存已清理"
|
||||
task.payload = {
|
||||
**(task.payload or {}),
|
||||
"earth_layer_deleted_count": earth_deleted_count,
|
||||
"dashboard_deleted_count": dashboard_deleted_count,
|
||||
}
|
||||
task.status = JOB_STATUS_SUCCESS
|
||||
task.completed_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
await enqueue_earth_refresh_job(db, source=source, payload={"operation": "CACHE_INVALIDATED"})
|
||||
|
||||
|
||||
async def _run_earth_refresh_job(db: AsyncSession, task: CollectionTask) -> None:
|
||||
payload = task.payload or {}
|
||||
source = str(payload.get("source") or task.source or "").strip()
|
||||
layers = list(payload.get("layers") or get_earth_update_layers_for_source(source))
|
||||
if not source or not layers:
|
||||
task.status = JOB_STATUS_SUCCESS
|
||||
task.phase = "completed"
|
||||
task.phase_message = "没有需要刷新的 Earth 图层"
|
||||
task.completed_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
return
|
||||
|
||||
deleted_cache_entries = invalidate_earth_layer_cache_for_source(source)
|
||||
update_payload = {
|
||||
"event": "earth.layer.changed",
|
||||
"action": "database_changed",
|
||||
"source": source,
|
||||
"table": payload.get("table"),
|
||||
"data_type": source,
|
||||
"layers": layers,
|
||||
"refresh_strategy": payload.get("refresh_strategy") or "clear_then_reload",
|
||||
"records_processed": payload.get("records_processed", 0),
|
||||
"operations": payload.get("operations") or [payload.get("operation") or "CHANGE"],
|
||||
"operation": payload.get("operation"),
|
||||
"cache_entries_invalidated": deleted_cache_entries,
|
||||
"timestamp": to_iso8601_utc(_utcnow()),
|
||||
}
|
||||
if payload.get("entity") == "interactable":
|
||||
update_payload.update(
|
||||
{
|
||||
"entity": "interactable",
|
||||
"action": payload.get("action") or "changed",
|
||||
"ids": payload.get("ids") or payload.get("entity_keys") or [],
|
||||
"item": payload.get("item"),
|
||||
}
|
||||
)
|
||||
await broadcaster.broadcast_earth_update(update_payload)
|
||||
|
||||
task.records_processed = int(payload.get("records_processed") or 0)
|
||||
task.progress = 100.0
|
||||
task.phase_progress = 100.0
|
||||
task.phase = "completed"
|
||||
task.phase_message = "Earth 图层刷新通知已发送"
|
||||
task.status = JOB_STATUS_SUCCESS
|
||||
task.completed_at = _utcnow()
|
||||
task.payload = {**payload, "cache_entries_invalidated": deleted_cache_entries}
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
|
||||
|
||||
_worker = DataJobWorker()
|
||||
|
||||
|
||||
def start_data_job_worker() -> None:
|
||||
_worker.start()
|
||||
|
||||
|
||||
async def stop_data_job_worker() -> None:
|
||||
await _worker.stop()
|
||||
@@ -18,6 +18,7 @@ from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.services.barentswatch import (
|
||||
BarentsWatchConfig,
|
||||
_read_zshrc_env,
|
||||
fetch_barentswatch_access_token,
|
||||
resolve_barentswatch_config,
|
||||
@@ -44,6 +45,20 @@ def _resolve_spacetrack_credentials() -> tuple[str, str, str]:
|
||||
return username, password, source or "missing"
|
||||
|
||||
|
||||
def _resolve_spacetrack_credentials_with_override(
|
||||
credential_override: dict[str, str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
if credential_override and (
|
||||
credential_override.get("username") or credential_override.get("password")
|
||||
):
|
||||
return (
|
||||
str(credential_override.get("username") or ""),
|
||||
str(credential_override.get("password") or ""),
|
||||
"draft",
|
||||
)
|
||||
return _resolve_spacetrack_credentials()
|
||||
|
||||
|
||||
async def _resolve_aisstream_api_key(
|
||||
db=None,
|
||||
credential_override: dict[str, str] | None = None,
|
||||
@@ -126,7 +141,9 @@ async def build_builtin_connectivity_checksum(
|
||||
}
|
||||
)
|
||||
elif credential_provider == "spacetrack":
|
||||
username, password, credential_source = _resolve_spacetrack_credentials()
|
||||
username, password, credential_source = _resolve_spacetrack_credentials_with_override(
|
||||
credential_override
|
||||
)
|
||||
has_credentials = bool(username and password)
|
||||
credential_fingerprint = _sha256_json(
|
||||
{
|
||||
@@ -230,7 +247,16 @@ async def test_builtin_connectivity(
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
if credential_context["credential_provider"] == "barentswatch":
|
||||
barentswatch_config = await resolve_barentswatch_config(db)
|
||||
if credential_override:
|
||||
barentswatch_config = BarentsWatchConfig(
|
||||
endpoint=str(request_endpoint or ""),
|
||||
client_id=str(credential_override.get("client_id") or ""),
|
||||
client_secret=str(credential_override.get("client_secret") or ""),
|
||||
credential_source="draft",
|
||||
endpoint_source="draft",
|
||||
)
|
||||
else:
|
||||
barentswatch_config = await resolve_barentswatch_config(db)
|
||||
token = await fetch_barentswatch_access_token(client, barentswatch_config)
|
||||
if not token:
|
||||
return {
|
||||
@@ -243,7 +269,9 @@ async def test_builtin_connectivity(
|
||||
}
|
||||
request_headers["Authorization"] = f"Bearer {token}"
|
||||
elif credential_context["credential_provider"] == "spacetrack":
|
||||
username, password, _source = _resolve_spacetrack_credentials()
|
||||
username, password, _source = _resolve_spacetrack_credentials_with_override(
|
||||
credential_override
|
||||
)
|
||||
login_url = "https://www.space-track.org/ajaxauth/login"
|
||||
login_response = await client.post(
|
||||
login_url,
|
||||
|
||||
@@ -8,6 +8,7 @@ import re
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.target_schema_registry import TargetSchema, get_target_schema
|
||||
@@ -254,6 +255,12 @@ def _best_field_match(field_name: str, candidates: list[str]) -> str | None:
|
||||
"lat": ("lat", "latitude", "y"),
|
||||
"lon": ("lon", "lng", "longitude", "x"),
|
||||
"mmsi": ("mmsi",),
|
||||
"geometry": ("geometry", "geom"),
|
||||
"properties": ("properties", "props"),
|
||||
"source_kind": ("source_kind", "kind", "type"),
|
||||
"feature_count": ("feature_count", "features_count", "count"),
|
||||
"artifact_path": ("artifact_path", "path", "file"),
|
||||
"sha256": ("sha256", "hash", "checksum"),
|
||||
"sog": ("sog", "speed", "speedOverGround"),
|
||||
"cog": ("cog", "course", "courseOverGround"),
|
||||
"received_at": ("received_at", "timestamp", "time", "updated_at"),
|
||||
|
||||
@@ -32,28 +32,31 @@ class DocsMetadata:
|
||||
|
||||
DOCS_METADATA: tuple[DocsMetadata, ...] = (
|
||||
DocsMetadata(DOCS_README_FILENAME, DEFAULT_DOCS_SLUG, "public", "Overview", 0, "技术文档", "Technical Docs"),
|
||||
DocsMetadata("quickstart.md", "quickstart", "public", "Manual", 1, "快速开始", "Quickstart"),
|
||||
DocsMetadata("manual.md", "manual", "public", "Manual", 2, "Planet 使用手册", "Planet Manual"),
|
||||
DocsMetadata("manual.md", "manual", "public", "Manual", 1, "智能星球使用手册", "Intelligent Planet Manual"),
|
||||
DocsMetadata("quickstart.md", "quickstart", "public", "Manual", 2, "快速开始", "Quickstart"),
|
||||
DocsMetadata("faq.md", "faq", "public", "Manual", 3, "常见问题", "FAQ"),
|
||||
DocsMetadata("location-pipeline-user.md", "location-pipeline-user", "public", "Manual", 4, "Earth 位置候选采集使用手册", "Earth Location Candidate Collection User Guide"),
|
||||
DocsMetadata("earth-frontend-context.md", "earth-frontend-context", "docs_developer", "Earth", 10, "Earth 前端结构", "Earth Frontend Context"),
|
||||
DocsMetadata("earth-layer-style-reference.md", "earth-layer-style-reference", "docs_developer", "Earth", 11, "Earth 图层样式属性索引", "Earth Layer Style Reference"),
|
||||
DocsMetadata("earth-render-layer-order.md", "earth-render-layer-order", "docs_developer", "Earth", 12, "Earth 渲染图层顺序", "Earth Render Layer Order"),
|
||||
DocsMetadata("earth-satellite-footprint-policy.md", "earth-satellite-footprint-policy", "docs_developer", "Earth", 13, "Earth 卫星覆盖策略", "Earth Satellite Footprint Policy"),
|
||||
DocsMetadata("platform-data-flows.md", "platform-data-flows", "docs_developer", "Architecture", 5, "业务架构与数据流转", "Business Architecture and Data Flows"),
|
||||
DocsMetadata("naming-glossary.md", "naming-glossary", "docs_developer", "Architecture", 6, "命名与术语对照", "Naming Glossary"),
|
||||
DocsMetadata("earth-frontend-context.md", "earth-frontend-context", "docs_developer", "Earth", 10, "智能星球前端结构", "Intelligent Planet Frontend Context"),
|
||||
DocsMetadata("earth-layer-style-reference.md", "earth-layer-style-reference", "docs_developer", "Earth", 11, "智能星球图层样式属性索引", "Intelligent Planet Layer Style Reference"),
|
||||
DocsMetadata("earth-render-layer-order.md", "earth-render-layer-order", "docs_developer", "Earth", 12, "智能星球渲染图层顺序", "Intelligent Planet Render Layer Order"),
|
||||
DocsMetadata("earth-satellite-footprint-policy.md", "earth-satellite-footprint-policy", "docs_developer", "Earth", 13, "智能星球卫星覆盖策略", "Intelligent Planet Satellite Footprint Policy"),
|
||||
DocsMetadata("earth-bgp-context.md", "earth-bgp-context", "docs_developer", "Earth", 14, "BGP 态势上下文", "BGP Context"),
|
||||
DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Earth", 15, "新闻直播采集格式", "News Live Streams Collector Format"),
|
||||
DocsMetadata("earth-interactable-usage.md", "earth-interactable-usage", "docs_developer", "Earth", 16, "Earth 可交互图标接入", "Earth Interactable Usage"),
|
||||
DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 17, "Earth 工具栏与浮层协同", "Earth Toolbar and Overlay Coordination"),
|
||||
DocsMetadata("earth-interactable-usage.md", "earth-interactable-usage", "docs_developer", "Earth", 16, "智能星球可交互图标接入", "Intelligent Planet Interactable Usage"),
|
||||
DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 17, "智能星球工具栏与浮层协同", "Intelligent Planet Toolbar and Overlay Coordination"),
|
||||
DocsMetadata("frontend-admin-frontend-context.md", "frontend-admin-frontend-context", "docs_developer", "Frontend", 20, "控制台前端结构", "Admin Frontend Context"),
|
||||
DocsMetadata("frontend-layout-guidelines.md", "frontend-layout-guidelines", "docs_developer", "Frontend", 21, "前端布局指南", "Frontend Layout Guidelines"),
|
||||
DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Frontend", 22, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
|
||||
DocsMetadata("tactile-ui-components.md", "tactile-ui-components", "docs_developer", "Frontend", 24, "Tactile UI 组件库", "Tactile UI Components"),
|
||||
DocsMetadata("backend-collectors.md", "backend-collectors", "docs_developer", "Backend", 30, "数据采集系统", "Data Collectors"),
|
||||
DocsMetadata("backend-system-service-control.md", "backend-system-service-control", "docs_admin", "Backend", 31, "系统服务控制", "System Service Control"),
|
||||
DocsMetadata("datasource-collector-settings-connectivity.md", "datasource-collector-settings-connectivity", "docs_developer", "Backend", 32, "数据源、采集器设置与连接验证", "Datasource Collector Settings and Connectivity"),
|
||||
DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"),
|
||||
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 34, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
|
||||
DocsMetadata("data-job-earth-sync-architecture.md", "data-job-earth-sync-architecture", "docs_developer", "Backend", 34, "数据作业与 Outbox 技术架构", "Data Jobs and Outbox Architecture"),
|
||||
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 35, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
|
||||
DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Backend", 36, "新闻直播采集格式", "News Live Streams Collector Format"),
|
||||
DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Backend", 37, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
|
||||
DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"),
|
||||
DocsMetadata("ops-runbook.md", "ops-runbook", "docs_admin", "Ops", 49, "Planet 运维手册", "Planet Ops Runbook"),
|
||||
DocsMetadata("ops-runbook.md", "ops-runbook", "docs_admin", "Ops", 49, "智能星球运维手册", "Intelligent Planet Ops Runbook"),
|
||||
DocsMetadata("ops-docker-compose-buildx-upgrade.md", "ops-docker-compose-buildx-upgrade", "docs_admin", "Ops", 50, "Docker + Compose + Buildx 升级", "Docker + Compose + Buildx Upgrade"),
|
||||
DocsMetadata("ops-planet-sh-startup.md", "ops-planet-sh-startup", "docs_admin", "Ops", 51, "planet.sh 启动机制", "planet.sh Startup"),
|
||||
)
|
||||
|
||||
671
backend/app/services/earth_boundaries.py
Normal file
671
backend/app/services/earth_boundaries.py
Normal file
@@ -0,0 +1,671 @@
|
||||
"""Earth boundary static asset service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
|
||||
SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
|
||||
BUILD_RESULT_PATH = SOURCE_OUTPUT_DIR / "build-result.json"
|
||||
BUILD_JOB_PATH = SOURCE_OUTPUT_DIR / "build-job.json"
|
||||
BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
|
||||
BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
|
||||
PMTILES_ARTIFACT_PATH = (
|
||||
REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
|
||||
)
|
||||
LEGACY_GEOJSON_PATH = REPO_ROOT / "frontend/public/earth/data/countries-admin0.min.geojson"
|
||||
POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
|
||||
LOCAL_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.local.json"
|
||||
EXAMPLE_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.example.json"
|
||||
|
||||
BOUNDARY_SOURCE_KINDS = {
|
||||
"earth_admin0_boundaries": "admin0-boundaries",
|
||||
"earth_coastline": "coastline",
|
||||
"earth_claim_lines": "claim-lines",
|
||||
}
|
||||
|
||||
DEFAULT_PUBLIC_BOUNDARY_SOURCES = {
|
||||
"earth_admin0_boundaries": {
|
||||
"displayName": "Natural Earth Admin-0 Countries",
|
||||
"sourceKind": "admin0-boundaries",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_countries.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
"earth_coastline": {
|
||||
"displayName": "Natural Earth Coastline",
|
||||
"sourceKind": "coastline",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_coastline.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
"earth_claim_lines": {
|
||||
"displayName": "Natural Earth Disputed Boundaries",
|
||||
"sourceKind": "claim-lines",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_boundary_lines_disputed_areas.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
}
|
||||
|
||||
BUILD_CONFIG = {
|
||||
"builder": "scripts/build_earth_boundary_pmtiles.py",
|
||||
"format": "pmtiles+mvt",
|
||||
"production_target": "pmtiles-mvt",
|
||||
}
|
||||
|
||||
|
||||
class EarthBoundaryBuildError(RuntimeError):
|
||||
def __init__(self, message: str, *, code: str = "build_failed", details: Any = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.details = details
|
||||
|
||||
|
||||
_build_job_lock = asyncio.Lock()
|
||||
_build_task: asyncio.Task | None = None
|
||||
_build_job_state: dict[str, Any] = {}
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _public_job_state() -> dict[str, Any]:
|
||||
if _build_job_state:
|
||||
return dict(_build_job_state)
|
||||
return _read_json(BUILD_JOB_PATH)
|
||||
|
||||
|
||||
def get_boundary_build_status() -> dict[str, Any]:
|
||||
return {"job": _public_job_state()}
|
||||
|
||||
|
||||
def _set_job_state(**updates: Any) -> dict[str, Any]:
|
||||
global _build_job_state
|
||||
current = dict(_build_job_state)
|
||||
current.update(updates)
|
||||
current["updated_at"] = _utc_now_iso()
|
||||
_build_job_state = current
|
||||
_write_json(BUILD_JOB_PATH, current)
|
||||
return current
|
||||
|
||||
|
||||
def _append_job_log(message: str) -> None:
|
||||
logs = list(_build_job_state.get("logs") or [])
|
||||
logs.append({"time": _utc_now_iso(), "message": message})
|
||||
_set_job_state(logs=logs[-40:])
|
||||
|
||||
|
||||
def _update_job_progress(progress: float, phase: str, message: str, **extra: Any) -> None:
|
||||
bounded_progress = max(0, min(100, int(round(progress))))
|
||||
_set_job_state(
|
||||
status="running",
|
||||
progress=bounded_progress,
|
||||
phase=phase,
|
||||
message=message,
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def _sha256_bytes(payload: bytes) -> str:
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _stable_json_hash(payload: Any) -> str:
|
||||
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
|
||||
suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
|
||||
if suffix in {".json", ".geojson", ".zip", ".pbf"}:
|
||||
return suffix
|
||||
if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
|
||||
return ".geojson"
|
||||
if "json" in content_type:
|
||||
return ".json"
|
||||
return ".dat"
|
||||
|
||||
|
||||
def _json_feature_count(payload: Any) -> int:
|
||||
if isinstance(payload, dict) and isinstance(payload.get("features"), list):
|
||||
return len(payload["features"])
|
||||
if isinstance(payload, list):
|
||||
return len(payload)
|
||||
return 1 if payload else 0
|
||||
|
||||
|
||||
def _directory_stats(path: Path) -> dict[str, int]:
|
||||
if not path.exists():
|
||||
return {"file_count": 0, "size_bytes": 0}
|
||||
files = [item for item in path.rglob("*") if item.is_file()]
|
||||
return {"file_count": len(files), "size_bytes": sum(item.stat().st_size for item in files)}
|
||||
|
||||
|
||||
def _load_source_feature_collection(source: dict[str, Any]) -> dict[str, Any]:
|
||||
path = REPO_ROOT / source["path"]
|
||||
payload = _read_json(path)
|
||||
features = payload.get("features") if isinstance(payload, dict) else None
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": features if isinstance(features, list) else [],
|
||||
}
|
||||
|
||||
|
||||
def _write_high_precision_geojson_manifest(
|
||||
sources: list[dict[str, Any]],
|
||||
build_input_hash: str,
|
||||
missing_tools: list[str],
|
||||
) -> dict[str, Any]:
|
||||
BOUNDARY_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
|
||||
coastline = next(source for source in sources if source["kind"] == "coastline")
|
||||
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
|
||||
|
||||
admin0_payload = _load_source_feature_collection(admin0)
|
||||
coastline_payload = _load_source_feature_collection(coastline)
|
||||
claim_payload = _load_source_feature_collection(claim_lines)
|
||||
for feature in coastline_payload["features"]:
|
||||
props = feature.setdefault("properties", {})
|
||||
if isinstance(props, dict):
|
||||
props["PLANET_LAYER"] = "coastline"
|
||||
|
||||
base_payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [*admin0_payload["features"], *coastline_payload["features"]],
|
||||
}
|
||||
base_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-base.geojson"
|
||||
hover_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-hover.geojson"
|
||||
claim_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-claims.geojson"
|
||||
_write_json(base_path, base_payload)
|
||||
_write_json(hover_path, admin0_payload)
|
||||
_write_json(claim_path, claim_payload)
|
||||
|
||||
manifest = {
|
||||
"version": "natural-earth-v1",
|
||||
"builtAt": _utc_now_iso(),
|
||||
"tileProvider": "geojson-high-precision",
|
||||
"format": "geojson-directory",
|
||||
"buildInputHash": build_input_hash,
|
||||
"base": base_path.name,
|
||||
"hoverIndex": hover_path.name,
|
||||
"claimLine": claim_path.name,
|
||||
"sourceFeatureCount": {
|
||||
"admin0": len(admin0_payload["features"]),
|
||||
"coastline": len(coastline_payload["features"]),
|
||||
"claimLines": len(claim_payload["features"]),
|
||||
},
|
||||
"pmtiles": None,
|
||||
"missingTools": missing_tools,
|
||||
}
|
||||
_write_json(BOUNDARY_MANIFEST_PATH, manifest)
|
||||
return manifest
|
||||
|
||||
|
||||
def _relative(path: Path) -> str:
|
||||
return str(path.relative_to(REPO_ROOT))
|
||||
|
||||
|
||||
def load_boundary_config() -> tuple[dict[str, Any], str]:
|
||||
if LOCAL_CONFIG_PATH.exists():
|
||||
return _read_json(LOCAL_CONFIG_PATH), "local"
|
||||
return _read_json(EXAMPLE_CONFIG_PATH), "example"
|
||||
|
||||
|
||||
def save_boundary_config(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
if not isinstance(payload, dict):
|
||||
raise EarthBoundaryBuildError("Earth boundary config must be a JSON object", code="invalid_config")
|
||||
_write_json(LOCAL_CONFIG_PATH, payload)
|
||||
return get_boundary_status()
|
||||
|
||||
|
||||
def _source_configs(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
raw_sources = payload.get("collectorConfigs") or payload.get("sources") or {}
|
||||
return raw_sources if isinstance(raw_sources, dict) else {}
|
||||
|
||||
|
||||
def _is_placeholder_endpoint(endpoint: Any) -> bool:
|
||||
value = str(endpoint or "").strip()
|
||||
return not value or "example.com" in value
|
||||
|
||||
|
||||
def _source_configs_with_defaults(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
raw_sources = _source_configs(payload)
|
||||
merged: dict[str, Any] = {}
|
||||
for source_key, default_config in DEFAULT_PUBLIC_BOUNDARY_SOURCES.items():
|
||||
configured = raw_sources.get(source_key)
|
||||
if not isinstance(configured, dict) or _is_placeholder_endpoint(configured.get("endpoint")):
|
||||
merged[source_key] = dict(default_config)
|
||||
else:
|
||||
merged[source_key] = {**default_config, **configured}
|
||||
for source_key, source_config in raw_sources.items():
|
||||
if source_key not in merged:
|
||||
merged[source_key] = source_config
|
||||
return merged
|
||||
|
||||
|
||||
def _build_input_hash(source_manifest: dict[str, Any]) -> str:
|
||||
return _stable_json_hash(
|
||||
{
|
||||
"source_manifest_schema": source_manifest.get("schema"),
|
||||
"sources": [
|
||||
{
|
||||
"id": source.get("id"),
|
||||
"sha256": source.get("sha256"),
|
||||
"kind": source.get("kind"),
|
||||
}
|
||||
for source in source_manifest.get("sources", [])
|
||||
],
|
||||
"pov_policy": source_manifest.get("povPolicy"),
|
||||
"build_config": BUILD_CONFIG,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
|
||||
return (
|
||||
bool(boundary_manifest)
|
||||
and boundary_manifest.get("buildInputHash") == build_input_hash
|
||||
and boundary_manifest.get("tileProvider") == "pmtiles-mvt"
|
||||
and PMTILES_ARTIFACT_PATH.exists()
|
||||
)
|
||||
|
||||
|
||||
def get_boundary_status() -> dict[str, Any]:
|
||||
config_payload, config_source = load_boundary_config()
|
||||
effective_source_configs = _source_configs_with_defaults(config_payload)
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
|
||||
manifest_exists = BOUNDARY_MANIFEST_PATH.exists()
|
||||
high_precision_ready = (
|
||||
manifest_exists
|
||||
and (
|
||||
(
|
||||
boundary_manifest.get("tileProvider") == "pmtiles-mvt"
|
||||
and pmtiles_exists
|
||||
)
|
||||
or boundary_manifest.get("tileProvider") == "geojson-high-precision"
|
||||
)
|
||||
)
|
||||
legacy_exists = LEGACY_GEOJSON_PATH.exists()
|
||||
provider = (
|
||||
boundary_manifest.get("tileProvider")
|
||||
if high_precision_ready
|
||||
else "legacy-geojson" if legacy_exists else "missing"
|
||||
)
|
||||
return {
|
||||
"provider": provider,
|
||||
"high_precision_ready": high_precision_ready,
|
||||
"fallback_available": legacy_exists,
|
||||
"config_source": config_source,
|
||||
"config_path": _relative(LOCAL_CONFIG_PATH),
|
||||
"config_exists": LOCAL_CONFIG_PATH.exists(),
|
||||
"config": config_payload,
|
||||
"effective_default_sources": [
|
||||
source_key
|
||||
for source_key, source_config in effective_source_configs.items()
|
||||
if source_key in DEFAULT_PUBLIC_BOUNDARY_SOURCES
|
||||
and source_config.get("endpoint") == DEFAULT_PUBLIC_BOUNDARY_SOURCES[source_key]["endpoint"]
|
||||
],
|
||||
"manifest": {
|
||||
"path": _relative(BOUNDARY_MANIFEST_PATH),
|
||||
"exists": manifest_exists,
|
||||
"tileProvider": boundary_manifest.get("tileProvider"),
|
||||
"buildInputHash": boundary_manifest.get("buildInputHash"),
|
||||
"builtAt": boundary_manifest.get("builtAt"),
|
||||
},
|
||||
"pmtiles": {
|
||||
"path": _relative(PMTILES_ARTIFACT_PATH),
|
||||
"exists": pmtiles_exists,
|
||||
"size_bytes": PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0,
|
||||
},
|
||||
"legacy": {
|
||||
"path": _relative(LEGACY_GEOJSON_PATH),
|
||||
"exists": legacy_exists,
|
||||
"size_bytes": LEGACY_GEOJSON_PATH.stat().st_size if legacy_exists else 0,
|
||||
},
|
||||
"source_manifest": {
|
||||
"path": _relative(SOURCE_MANIFEST_PATH),
|
||||
"exists": SOURCE_MANIFEST_PATH.exists(),
|
||||
},
|
||||
"last_build": _read_json(BUILD_RESULT_PATH),
|
||||
"current_job": _public_job_state(),
|
||||
}
|
||||
|
||||
|
||||
async def _download_source(
|
||||
source_key: str,
|
||||
source_config: dict[str, Any],
|
||||
progress_callback: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
endpoint = str(source_config.get("endpoint") or "").strip()
|
||||
if _is_placeholder_endpoint(endpoint):
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} endpoint is not configured",
|
||||
code="source_not_configured",
|
||||
details={"source": source_key},
|
||||
)
|
||||
method = str(source_config.get("method") or "GET").upper()
|
||||
if method not in {"GET", "POST"}:
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} method must be GET or POST",
|
||||
code="invalid_config",
|
||||
details={"source": source_key, "method": method},
|
||||
)
|
||||
|
||||
if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
|
||||
payload = Path(endpoint.removeprefix("file://")).expanduser().read_bytes()
|
||||
content_type = "application/octet-stream"
|
||||
if progress_callback:
|
||||
progress_callback(1, len(payload), len(payload))
|
||||
else:
|
||||
timeout = float(source_config.get("timeout") or 120)
|
||||
headers = source_config.get("headers") if isinstance(source_config.get("headers"), dict) else {}
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
async with client.stream(method, endpoint, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
content_type = response.headers.get("content-type", "")
|
||||
total = int(response.headers.get("content-length") or 0)
|
||||
chunks = []
|
||||
downloaded = 0
|
||||
async for chunk in response.aiter_bytes():
|
||||
if not chunk:
|
||||
continue
|
||||
chunks.append(chunk)
|
||||
downloaded += len(chunk)
|
||||
if progress_callback:
|
||||
progress_callback(
|
||||
(downloaded / total) if total else None,
|
||||
downloaded,
|
||||
total,
|
||||
)
|
||||
payload = b"".join(chunks)
|
||||
|
||||
extension = _artifact_extension(endpoint, content_type, payload)
|
||||
parsed: Any = None
|
||||
if extension in {".json", ".geojson"}:
|
||||
parsed = json.loads(payload.decode("utf-8"))
|
||||
feature_count = _json_feature_count(parsed)
|
||||
if feature_count <= 0:
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} downloaded payload contains no features",
|
||||
code="empty_source",
|
||||
details={"source": source_key},
|
||||
)
|
||||
|
||||
sha256 = _sha256_bytes(payload)
|
||||
source_dir = SOURCE_OUTPUT_DIR / source_key
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact_path = source_dir / f"{sha256}{extension}"
|
||||
artifact_path.write_bytes(payload)
|
||||
return {
|
||||
"id": source_key,
|
||||
"kind": source_config.get("sourceKind") or BOUNDARY_SOURCE_KINDS[source_key],
|
||||
"path": _relative(artifact_path),
|
||||
"sha256": sha256,
|
||||
"featureCount": feature_count,
|
||||
"license": source_config.get("license"),
|
||||
}
|
||||
|
||||
|
||||
async def _run_step(args: list[str]) -> dict[str, Any]:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
*args,
|
||||
cwd=REPO_ROOT,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout_bytes, stderr_bytes = await process.communicate()
|
||||
stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
|
||||
stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
|
||||
payload: dict[str, Any] = {"stdout": stdout, "stderr": stderr, "returncode": process.returncode}
|
||||
last_line = stdout.splitlines()[-1:] or []
|
||||
if last_line:
|
||||
try:
|
||||
payload["result"] = json.loads(last_line[0])
|
||||
except json.JSONDecodeError:
|
||||
payload["result"] = last_line[0]
|
||||
if process.returncode != 0:
|
||||
raise EarthBoundaryBuildError(
|
||||
stderr or stdout or f"command failed: {' '.join(args)}",
|
||||
code="build_command_failed",
|
||||
details=payload,
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
async def build_boundary_assets(progress_callback: Any = None) -> dict[str, Any]:
|
||||
config_payload, config_source = load_boundary_config()
|
||||
|
||||
source_configs = _source_configs_with_defaults(config_payload)
|
||||
missing = [source for source in BOUNDARY_SOURCE_KINDS if source not in source_configs]
|
||||
if missing:
|
||||
raise EarthBoundaryBuildError(
|
||||
f"Missing Earth boundary source configs: {', '.join(missing)}",
|
||||
code="missing_sources",
|
||||
details={"missing": missing},
|
||||
)
|
||||
|
||||
sources = []
|
||||
source_keys = list(BOUNDARY_SOURCE_KINDS)
|
||||
for index, source_key in enumerate(source_keys):
|
||||
source_config = source_configs[source_key]
|
||||
if not isinstance(source_config, dict):
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} config must be an object",
|
||||
code="invalid_config",
|
||||
details={"source": source_key},
|
||||
)
|
||||
source_start = 8 + index * 18
|
||||
source_end = source_start + 18
|
||||
if progress_callback:
|
||||
progress_callback(source_start, "download", f"正在下载 {source_key}")
|
||||
|
||||
def report_download_progress(ratio: float | None, downloaded: int, total: int) -> None:
|
||||
if not progress_callback:
|
||||
return
|
||||
if ratio is None:
|
||||
progress_callback(source_start + 8, "download", f"{source_key} 已下载 {downloaded} bytes")
|
||||
return
|
||||
progress_callback(
|
||||
source_start + (source_end - source_start) * ratio,
|
||||
"download",
|
||||
f"{source_key} 下载 {int(ratio * 100)}%",
|
||||
downloaded_bytes=downloaded,
|
||||
total_bytes=total,
|
||||
)
|
||||
|
||||
sources.append(await _download_source(source_key, source_config, report_download_progress))
|
||||
|
||||
source_manifest = {
|
||||
"schema": "planet-earth-boundary-sources/v2",
|
||||
"sources": sources,
|
||||
"povPolicy": _read_json(POV_POLICY_PATH),
|
||||
}
|
||||
if progress_callback:
|
||||
progress_callback(65, "manifest", "正在写入边界源 manifest")
|
||||
_write_json(SOURCE_MANIFEST_PATH, source_manifest)
|
||||
build_input_hash = _build_input_hash(source_manifest)
|
||||
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
|
||||
missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
|
||||
if missing_tools and not build_skipped:
|
||||
if progress_callback:
|
||||
progress_callback(82, "build", "缺少 PMTiles 工具,正在生成 GeoJSON 高清包")
|
||||
boundary_manifest = _write_high_precision_geojson_manifest(
|
||||
sources,
|
||||
build_input_hash,
|
||||
missing_tools,
|
||||
)
|
||||
result = {
|
||||
"status": "built_geojson_fallback",
|
||||
"code": "missing_tools",
|
||||
"missing_tools": missing_tools,
|
||||
"sources": sources,
|
||||
"boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH),
|
||||
"manifest": boundary_manifest,
|
||||
}
|
||||
_write_json(BUILD_RESULT_PATH, result)
|
||||
if progress_callback:
|
||||
progress_callback(96, "finalize", "GeoJSON 高清国界包已生成")
|
||||
return {**get_boundary_status(), "build": result}
|
||||
|
||||
if build_skipped:
|
||||
if progress_callback:
|
||||
progress_callback(96, "unchanged", "高精国界已是最新")
|
||||
build_result = {
|
||||
"status": "unchanged",
|
||||
"reason": "source manifest and build config hash unchanged",
|
||||
"buildInputHash": build_input_hash,
|
||||
}
|
||||
else:
|
||||
if progress_callback:
|
||||
progress_callback(72, "build", "正在构建 PMTiles/MVT")
|
||||
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
|
||||
coastline = next(source for source in sources if source["kind"] == "coastline")
|
||||
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
|
||||
build_result = await _run_step(
|
||||
[
|
||||
"scripts/build_earth_boundary_pmtiles.py",
|
||||
"--admin0-source",
|
||||
admin0["path"],
|
||||
"--coastline-source",
|
||||
coastline["path"],
|
||||
"--claims-source",
|
||||
claim_lines["path"],
|
||||
"--output",
|
||||
_relative(PMTILES_ARTIFACT_PATH),
|
||||
"--manifest",
|
||||
_relative(BOUNDARY_MANIFEST_PATH),
|
||||
"--build-input-hash",
|
||||
build_input_hash,
|
||||
"--pov-policy",
|
||||
_relative(POV_POLICY_PATH),
|
||||
]
|
||||
)
|
||||
if progress_callback:
|
||||
progress_callback(95, "finalize", "正在校验构建产物")
|
||||
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
|
||||
result = {
|
||||
"status": "unchanged" if build_skipped else "built",
|
||||
"sources": sources,
|
||||
"source_manifest": _relative(SOURCE_MANIFEST_PATH),
|
||||
"boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH),
|
||||
"pmtiles_artifact": _relative(PMTILES_ARTIFACT_PATH),
|
||||
"pmtiles_exists": PMTILES_ARTIFACT_PATH.exists(),
|
||||
"boundary_stats": boundary_stats,
|
||||
"manifest": boundary_manifest,
|
||||
"build_result": build_result,
|
||||
}
|
||||
_write_json(BUILD_RESULT_PATH, result)
|
||||
return {**get_boundary_status(), "build": result}
|
||||
|
||||
|
||||
async def _run_boundary_build_job(job_id: str) -> None:
|
||||
def report(progress: float, phase: str, message: str, **extra: Any) -> None:
|
||||
if _build_job_state.get("id") != job_id:
|
||||
return
|
||||
_update_job_progress(progress, phase, message, **extra)
|
||||
|
||||
try:
|
||||
report(3, "prepare", "正在准备高精国界构建")
|
||||
result = await build_boundary_assets(report)
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="succeeded",
|
||||
progress=100,
|
||||
phase="complete",
|
||||
message="高精国界构建完成",
|
||||
finished_at=_utc_now_iso(),
|
||||
result={
|
||||
"provider": result.get("provider"),
|
||||
"high_precision_ready": result.get("high_precision_ready"),
|
||||
"pmtiles": result.get("pmtiles"),
|
||||
"manifest": result.get("manifest"),
|
||||
},
|
||||
)
|
||||
_append_job_log("高精国界构建完成")
|
||||
except EarthBoundaryBuildError as exc:
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="failed",
|
||||
progress=_build_job_state.get("progress", 0),
|
||||
phase="failed",
|
||||
message=str(exc),
|
||||
code=exc.code,
|
||||
details=exc.details,
|
||||
finished_at=_utc_now_iso(),
|
||||
)
|
||||
_append_job_log(str(exc))
|
||||
except Exception as exc: # pragma: no cover - defensive guard for background task
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="failed",
|
||||
progress=_build_job_state.get("progress", 0),
|
||||
phase="failed",
|
||||
message=str(exc),
|
||||
code="build_failed",
|
||||
finished_at=_utc_now_iso(),
|
||||
)
|
||||
_append_job_log(str(exc))
|
||||
|
||||
|
||||
async def start_boundary_build_job() -> dict[str, Any]:
|
||||
global _build_task
|
||||
async with _build_job_lock:
|
||||
if _build_task and not _build_task.done():
|
||||
return {"accepted": False, "job": _public_job_state()}
|
||||
job_id = uuid4().hex
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="queued",
|
||||
progress=0,
|
||||
phase="queued",
|
||||
message="高精国界构建已加入队列",
|
||||
logs=[],
|
||||
started_at=_utc_now_iso(),
|
||||
finished_at=None,
|
||||
code=None,
|
||||
details=None,
|
||||
)
|
||||
_append_job_log("高精国界构建已启动")
|
||||
_build_task = asyncio.create_task(_run_boundary_build_job(job_id))
|
||||
return {"accepted": True, "job": _public_job_state()}
|
||||
482
backend/app/services/earth_db_change_listener.py
Normal file
482
backend/app/services/earth_db_change_listener.py
Normal file
@@ -0,0 +1,482 @@
|
||||
"""PostgreSQL LISTEN/NOTIFY bridge for Earth layer refresh events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from time import monotonic
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.services.earth_layer_adapters import (
|
||||
get_earth_refresh_strategy_for_change,
|
||||
get_earth_update_layers_for_change,
|
||||
get_earth_update_layers_for_source,
|
||||
)
|
||||
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
EARTH_DATA_CHANGES_CHANNEL = "planet_earth_data_changes"
|
||||
DEFAULT_DEBOUNCE_SECONDS = 0.25
|
||||
DEFAULT_MAX_WAIT_SECONDS = 1.5
|
||||
DELETE_FAST_FLUSH_SECONDS = 0.05
|
||||
LISTEN_KEEPALIVE_SECONDS = 5.0
|
||||
OUTBOX_POLL_LIMIT = 5000
|
||||
MAX_ENTITY_KEY_SAMPLES = 20
|
||||
MAX_SEEN_EVENT_IDS = 20000
|
||||
|
||||
BroadcastFn = Callable[[dict[str, Any]], Awaitable[None]]
|
||||
InvalidateFn = Callable[[str], int]
|
||||
|
||||
|
||||
def normalize_asyncpg_dsn(dsn: str) -> str:
|
||||
"""Convert SQLAlchemy asyncpg URLs into asyncpg-compatible URLs."""
|
||||
return dsn.replace("postgresql+asyncpg://", "postgresql://", 1)
|
||||
|
||||
|
||||
def build_earth_update_from_db_payload(payload: dict[str, Any]) -> dict[str, Any] | None:
|
||||
table = payload.get("table")
|
||||
source = str(payload.get("source") or "").strip()
|
||||
table_name = str(table or "").strip()
|
||||
if not source and not table_name:
|
||||
return None
|
||||
layers = get_earth_update_layers_for_change(table_name, source)
|
||||
if not layers:
|
||||
return None
|
||||
refresh_strategy = get_earth_refresh_strategy_for_change(table_name, source) or "clear_then_reload"
|
||||
source_has_adapter = bool(get_earth_update_layers_for_source(source))
|
||||
effective_source = source if source_has_adapter else (table_name if table_name else source)
|
||||
operation = payload.get("operation")
|
||||
update: dict[str, Any] = {
|
||||
"event": "earth.layer.changed",
|
||||
"action": "database_changed",
|
||||
"source": effective_source,
|
||||
"original_source": source or None,
|
||||
"table": table_name or None,
|
||||
"data_type": effective_source,
|
||||
"layers": layers,
|
||||
"refresh_strategy": refresh_strategy,
|
||||
"operation": operation,
|
||||
"entity_key": payload.get("entity_key"),
|
||||
"timestamp": to_iso8601_utc(datetime.now(UTC)),
|
||||
}
|
||||
if table_name == "earth_interactables" and refresh_strategy == "delta":
|
||||
ids = payload.get("entity_keys")
|
||||
if not isinstance(ids, list):
|
||||
ids = [payload.get("entity_key")] if payload.get("entity_key") else []
|
||||
update.update(
|
||||
{
|
||||
"entity": "interactable",
|
||||
"action": "deleted" if operation == "DELETE" else "changed",
|
||||
"ids": [str(item) for item in ids if item],
|
||||
"item": None,
|
||||
}
|
||||
)
|
||||
return {
|
||||
**update,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PendingEarthDbChange:
|
||||
source: str
|
||||
layers: list[str]
|
||||
table: str | None = None
|
||||
refresh_strategy: str = "clear_then_reload"
|
||||
entity: str | None = None
|
||||
action: str = "database_changed"
|
||||
records_processed: int = 0
|
||||
operations: set[str] = field(default_factory=set)
|
||||
entity_keys: list[str] = field(default_factory=list)
|
||||
first_occurred_at: str | None = None
|
||||
last_occurred_at: str | None = None
|
||||
first_seen_monotonic: float = field(default_factory=monotonic)
|
||||
last_seen_monotonic: float = field(default_factory=monotonic)
|
||||
|
||||
def add(self, payload: dict[str, Any]) -> None:
|
||||
self.last_seen_monotonic = monotonic()
|
||||
records_processed = payload.get("records_processed", 1)
|
||||
try:
|
||||
records_processed = int(records_processed)
|
||||
except (TypeError, ValueError):
|
||||
records_processed = 1
|
||||
self.records_processed += max(records_processed, 1)
|
||||
operation = payload.get("operation")
|
||||
if operation:
|
||||
self.operations.add(str(operation))
|
||||
entity_keys = payload.get("entity_keys")
|
||||
if not isinstance(entity_keys, list):
|
||||
entity_key = payload.get("entity_key")
|
||||
entity_keys = [entity_key] if entity_key else []
|
||||
for entity_key in entity_keys:
|
||||
if entity_key and len(self.entity_keys) < MAX_ENTITY_KEY_SAMPLES:
|
||||
self.entity_keys.append(str(entity_key))
|
||||
occurred_at = payload.get("occurred_at")
|
||||
if occurred_at:
|
||||
occurred_at = str(occurred_at)
|
||||
self.first_occurred_at = self.first_occurred_at or occurred_at
|
||||
self.last_occurred_at = occurred_at
|
||||
|
||||
|
||||
class EarthDbChangeDispatcher:
|
||||
"""Debounces database notifications and broadcasts Earth refresh hints."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
broadcast_earth_update: BroadcastFn | None = None,
|
||||
invalidate_cache: InvalidateFn | None = None,
|
||||
debounce_seconds: float = DEFAULT_DEBOUNCE_SECONDS,
|
||||
max_wait_seconds: float = DEFAULT_MAX_WAIT_SECONDS,
|
||||
) -> None:
|
||||
self._broadcast_earth_update = broadcast_earth_update or broadcaster.broadcast_earth_update
|
||||
self._invalidate_cache = invalidate_cache or invalidate_earth_layer_cache_for_source
|
||||
self._debounce_seconds = debounce_seconds
|
||||
self._max_wait_seconds = max(max_wait_seconds, debounce_seconds)
|
||||
self._pending: dict[str, PendingEarthDbChange] = {}
|
||||
self._flush_tasks: dict[str, asyncio.Task[None]] = {}
|
||||
self._seen_event_ids: set[int] = set()
|
||||
self._seen_event_order: deque[int] = deque()
|
||||
|
||||
def handle_notification(self, payload_text: str) -> bool:
|
||||
try:
|
||||
payload = json.loads(payload_text)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning_event(
|
||||
"Ignoring malformed Earth database change notification",
|
||||
event="earth.db_changes.notification_malformed",
|
||||
)
|
||||
return False
|
||||
if not isinstance(payload, dict):
|
||||
return False
|
||||
return self.handle_payload(payload)
|
||||
|
||||
def handle_payload(self, payload: dict[str, Any]) -> bool:
|
||||
event_id = payload.get("event_id")
|
||||
if event_id is not None:
|
||||
try:
|
||||
normalized_event_id = int(event_id)
|
||||
except (TypeError, ValueError):
|
||||
normalized_event_id = None
|
||||
if normalized_event_id is not None:
|
||||
if normalized_event_id in self._seen_event_ids:
|
||||
return False
|
||||
self._remember_event_id(normalized_event_id)
|
||||
|
||||
update = build_earth_update_from_db_payload(payload)
|
||||
if not update:
|
||||
return False
|
||||
|
||||
source = update["source"]
|
||||
pending = self._pending.get(source)
|
||||
if pending is None:
|
||||
pending = PendingEarthDbChange(
|
||||
source=source,
|
||||
layers=list(update["layers"]),
|
||||
table=update.get("table"),
|
||||
refresh_strategy=str(update.get("refresh_strategy") or "clear_then_reload"),
|
||||
entity=update.get("entity"),
|
||||
action=str(update.get("action") or "database_changed"),
|
||||
)
|
||||
self._pending[source] = pending
|
||||
pending.add(payload)
|
||||
|
||||
task = self._flush_tasks.pop(source, None)
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
self._flush_tasks[source] = asyncio.create_task(
|
||||
self._flush_later(source, delay_seconds=self._next_flush_delay(pending))
|
||||
)
|
||||
return True
|
||||
|
||||
def _next_flush_delay(self, pending: PendingEarthDbChange) -> float:
|
||||
if "DELETE" in pending.operations and pending.refresh_strategy == "clear_then_reload":
|
||||
return DELETE_FAST_FLUSH_SECONDS
|
||||
elapsed = max(0.0, monotonic() - pending.first_seen_monotonic)
|
||||
remaining = self._max_wait_seconds - elapsed
|
||||
if remaining <= 0:
|
||||
return 0.0
|
||||
return min(self._debounce_seconds, remaining)
|
||||
|
||||
def _remember_event_id(self, event_id: int) -> None:
|
||||
self._seen_event_ids.add(event_id)
|
||||
self._seen_event_order.append(event_id)
|
||||
while len(self._seen_event_order) > MAX_SEEN_EVENT_IDS:
|
||||
expired_event_id = self._seen_event_order.popleft()
|
||||
self._seen_event_ids.discard(expired_event_id)
|
||||
|
||||
async def _flush_later(self, source: str, *, delay_seconds: float) -> None:
|
||||
try:
|
||||
if delay_seconds > 0:
|
||||
await asyncio.sleep(delay_seconds)
|
||||
await self.flush_source(source)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception_event(
|
||||
"Failed to broadcast debounced Earth database change",
|
||||
event="earth.db_changes.flush_failed",
|
||||
context={"source": source, "error": str(exc)},
|
||||
)
|
||||
finally:
|
||||
current = self._flush_tasks.get(source)
|
||||
if current is asyncio.current_task():
|
||||
self._flush_tasks.pop(source, None)
|
||||
|
||||
async def flush_source(self, source: str) -> None:
|
||||
pending = self._pending.get(source)
|
||||
if pending is None:
|
||||
return
|
||||
|
||||
flushed_at = datetime.now(UTC)
|
||||
deleted_cache_entries = self._invalidate_cache(source)
|
||||
operations = sorted(pending.operations)
|
||||
payload: dict[str, Any] = {
|
||||
"event": "earth.layer.changed",
|
||||
"action": "database_changed",
|
||||
"source": source,
|
||||
"table": pending.table,
|
||||
"data_type": source,
|
||||
"layers": pending.layers,
|
||||
"refresh_strategy": pending.refresh_strategy,
|
||||
"records_processed": pending.records_processed,
|
||||
"operations": operations,
|
||||
"operation": operations[-1] if len(operations) == 1 else None,
|
||||
"entity_keys": pending.entity_keys,
|
||||
"entity_key_sample_size": len(pending.entity_keys),
|
||||
"cache_entries_invalidated": deleted_cache_entries,
|
||||
"first_occurred_at": pending.first_occurred_at,
|
||||
"last_occurred_at": pending.last_occurred_at,
|
||||
"debounce_ms": int((monotonic() - pending.first_seen_monotonic) * 1000),
|
||||
"total_latency_ms": self._total_latency_ms(pending, flushed_at),
|
||||
"timestamp": to_iso8601_utc(flushed_at),
|
||||
}
|
||||
if pending.entity == "interactable":
|
||||
payload.update(
|
||||
{
|
||||
"entity": "interactable",
|
||||
"action": "deleted" if "DELETE" in pending.operations else "changed",
|
||||
"ids": pending.entity_keys,
|
||||
"item": None,
|
||||
}
|
||||
)
|
||||
await self._broadcast_earth_update(payload)
|
||||
self._pending.pop(source, None)
|
||||
logger.info_event(
|
||||
"Broadcasted Earth database change",
|
||||
event="earth.db_changes.broadcasted",
|
||||
context={
|
||||
"source": source,
|
||||
"layers": pending.layers,
|
||||
"records_processed": pending.records_processed,
|
||||
"cache_entries_invalidated": deleted_cache_entries,
|
||||
"debounce_ms": int((monotonic() - pending.first_seen_monotonic) * 1000),
|
||||
"total_latency_ms": payload["total_latency_ms"],
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _total_latency_ms(pending: PendingEarthDbChange, flushed_at: datetime) -> int | None:
|
||||
occurred_at = pending.first_occurred_at
|
||||
if not occurred_at:
|
||||
return None
|
||||
try:
|
||||
normalized = occurred_at.replace("Z", "+00:00")
|
||||
occurred = datetime.fromisoformat(normalized)
|
||||
if occurred.tzinfo is None:
|
||||
occurred = occurred.replace(tzinfo=UTC)
|
||||
return max(0, int((flushed_at - occurred.astimezone(UTC)).total_seconds() * 1000))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
async def flush_all(self) -> None:
|
||||
sources = list(self._pending)
|
||||
for source in sources:
|
||||
task = self._flush_tasks.pop(source, None)
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
await self.flush_source(source)
|
||||
|
||||
async def stop(self) -> None:
|
||||
tasks = [task for task in self._flush_tasks.values() if not task.done()]
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
self._flush_tasks.clear()
|
||||
await self.flush_all()
|
||||
|
||||
|
||||
class EarthDbChangeListener:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dsn: str,
|
||||
dispatcher: EarthDbChangeDispatcher,
|
||||
channel: str = EARTH_DATA_CHANGES_CHANNEL,
|
||||
) -> None:
|
||||
self._dsn = normalize_asyncpg_dsn(dsn)
|
||||
self._dispatcher = dispatcher
|
||||
self._channel = channel
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._stop_event: asyncio.Event | None = None
|
||||
self._connection: asyncpg.Connection | None = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._loop = asyncio.get_running_loop()
|
||||
self._stop_event = asyncio.Event()
|
||||
self._task = asyncio.create_task(self._run())
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._stop_event:
|
||||
self._stop_event.set()
|
||||
if self._connection:
|
||||
await self._connection.close()
|
||||
if self._task:
|
||||
await asyncio.gather(self._task, return_exceptions=True)
|
||||
await self._dispatcher.stop()
|
||||
|
||||
async def _run(self) -> None:
|
||||
backoff_seconds = 1.0
|
||||
assert self._stop_event is not None
|
||||
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self._connection = await asyncpg.connect(self._dsn)
|
||||
await self._connection.add_listener(self._channel, self._on_notification)
|
||||
logger.info_event(
|
||||
"Earth database change listener connected",
|
||||
event="earth.db_changes.connected",
|
||||
context={"channel": self._channel},
|
||||
)
|
||||
backoff_seconds = 1.0
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._stop_event.wait(),
|
||||
timeout=LISTEN_KEEPALIVE_SECONDS,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
await self._poll_outbox()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception_event(
|
||||
"Earth database change listener failed",
|
||||
event="earth.db_changes.listener_failed",
|
||||
context={"channel": self._channel, "error": str(exc)},
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(self._stop_event.wait(), timeout=backoff_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
backoff_seconds = min(backoff_seconds * 2, 30.0)
|
||||
finally:
|
||||
if self._connection:
|
||||
try:
|
||||
await self._connection.remove_listener(self._channel, self._on_notification)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await self._connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._connection = None
|
||||
|
||||
async def _poll_outbox(self) -> None:
|
||||
if self._connection is None:
|
||||
return
|
||||
|
||||
rows = await self._connection.fetch(
|
||||
"""
|
||||
SELECT id, payload
|
||||
FROM earth_data_change_events
|
||||
WHERE consumed_at IS NULL
|
||||
ORDER BY id
|
||||
LIMIT $1
|
||||
""",
|
||||
OUTBOX_POLL_LIMIT,
|
||||
)
|
||||
accepted_count = 0
|
||||
consumed_ids: list[int] = []
|
||||
for row in rows:
|
||||
payload = row["payload"]
|
||||
if isinstance(payload, str):
|
||||
try:
|
||||
payload = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
consumed_ids.append(int(row["id"]))
|
||||
continue
|
||||
if isinstance(payload, dict):
|
||||
if self._dispatcher.handle_payload(payload):
|
||||
accepted_count += 1
|
||||
consumed_ids.append(int(row["id"]))
|
||||
else:
|
||||
consumed_ids.append(int(row["id"]))
|
||||
if consumed_ids:
|
||||
await self._dispatcher.flush_all()
|
||||
if consumed_ids:
|
||||
await self._connection.execute(
|
||||
"""
|
||||
UPDATE earth_data_change_events
|
||||
SET consumed_at = NOW()
|
||||
WHERE id = ANY($1::bigint[])
|
||||
AND consumed_at IS NULL
|
||||
""",
|
||||
consumed_ids,
|
||||
)
|
||||
if rows:
|
||||
logger.info_event(
|
||||
"Polled Earth database change outbox",
|
||||
event="earth.db_changes.outbox_polled",
|
||||
context={"events": len(rows), "accepted": accepted_count},
|
||||
)
|
||||
|
||||
def _on_notification(
|
||||
self,
|
||||
_connection: asyncpg.Connection,
|
||||
_pid: int,
|
||||
_channel: str,
|
||||
payload: str,
|
||||
) -> None:
|
||||
if self._loop and self._loop.is_running():
|
||||
self._loop.call_soon_threadsafe(self._dispatcher.handle_notification, payload)
|
||||
return
|
||||
self._dispatcher.handle_notification(payload)
|
||||
|
||||
|
||||
_dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcaster.broadcast_earth_update,
|
||||
invalidate_cache=invalidate_earth_layer_cache_for_source,
|
||||
)
|
||||
_listener: EarthDbChangeListener | None = None
|
||||
|
||||
|
||||
def start_earth_db_change_listener() -> None:
|
||||
global _listener
|
||||
if _listener is not None:
|
||||
return
|
||||
_listener = EarthDbChangeListener(dsn=settings.DATABASE_URL, dispatcher=_dispatcher)
|
||||
_listener.start()
|
||||
|
||||
|
||||
async def stop_earth_db_change_listener() -> None:
|
||||
global _listener
|
||||
if _listener is None:
|
||||
await _dispatcher.stop()
|
||||
return
|
||||
listener = _listener
|
||||
_listener = None
|
||||
await listener.stop()
|
||||
113
backend/app/services/earth_interactables.py
Normal file
113
backend/app/services/earth_interactables.py
Normal file
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.models.earth_interactable import EarthInteractable
|
||||
from app.services.earth_layer_cache import EARTH_LAYER_CACHE_PREFIX, earth_layer_cache
|
||||
|
||||
INTERACTABLE_ENTITY = "interactable"
|
||||
INTERACTABLE_LAYER = "interactables"
|
||||
|
||||
|
||||
def normalize_interactable_id(value: str | None = None) -> str:
|
||||
raw = str(value or "").strip()
|
||||
return raw or f"interactable-{uuid4().hex}"
|
||||
|
||||
|
||||
def serialize_interactable(record: EarthInteractable) -> dict[str, Any]:
|
||||
return {
|
||||
"id": record.id,
|
||||
"layer": record.layer,
|
||||
"kind": record.kind,
|
||||
"label": record.label,
|
||||
"description": record.description,
|
||||
"latitude": record.latitude,
|
||||
"longitude": record.longitude,
|
||||
"altitude": record.altitude,
|
||||
"revision": record.revision,
|
||||
"properties": record.properties or {},
|
||||
"is_deleted": bool(record.is_deleted),
|
||||
"created_at": to_iso8601_utc(record.created_at),
|
||||
"updated_at": to_iso8601_utc(record.updated_at),
|
||||
"deleted_at": to_iso8601_utc(record.deleted_at),
|
||||
}
|
||||
|
||||
|
||||
def interactables_to_geojson(items: list[EarthInteractable]) -> dict[str, Any]:
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": item.id,
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [item.longitude, item.latitude],
|
||||
},
|
||||
"properties": serialize_interactable(item),
|
||||
}
|
||||
for item in items
|
||||
if not item.is_deleted
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def invalidate_interactable_cache(layer: str | None = None) -> int:
|
||||
layer_key = str(layer or "*").strip() or "*"
|
||||
deleted = earth_layer_cache.delete_pattern(
|
||||
f"{EARTH_LAYER_CACHE_PREFIX}:interactables:layer:{layer_key}*"
|
||||
)
|
||||
if layer_key != "all":
|
||||
deleted += earth_layer_cache.delete_pattern(
|
||||
f"{EARTH_LAYER_CACHE_PREFIX}:interactables:layer:all*"
|
||||
)
|
||||
return deleted
|
||||
|
||||
|
||||
def build_interactable_event(
|
||||
*,
|
||||
action: str,
|
||||
record: EarthInteractable,
|
||||
include_item: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
item = serialize_interactable(record)
|
||||
return {
|
||||
"entity": INTERACTABLE_ENTITY,
|
||||
"action": action,
|
||||
"layer": record.layer,
|
||||
"layers": [INTERACTABLE_LAYER],
|
||||
"ids": [record.id],
|
||||
"revision": record.revision,
|
||||
"changed_at": item["deleted_at"] or item["updated_at"] or to_iso8601_utc(datetime.now(UTC)),
|
||||
"item": item if include_item else None,
|
||||
"source": "earth_interactables",
|
||||
}
|
||||
|
||||
|
||||
async def publish_interactable_event(action: str, record: EarthInteractable, *, include_item: bool = True) -> None:
|
||||
await broadcaster.broadcast_earth_update(
|
||||
build_interactable_event(action=action, record=record, include_item=include_item)
|
||||
)
|
||||
|
||||
|
||||
async def list_interactables(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
layer: str | None = None,
|
||||
include_deleted: bool = False,
|
||||
) -> list[EarthInteractable]:
|
||||
stmt = select(EarthInteractable)
|
||||
if layer:
|
||||
stmt = stmt.where(EarthInteractable.layer == layer)
|
||||
if not include_deleted:
|
||||
stmt = stmt.where(EarthInteractable.is_deleted.is_(False))
|
||||
stmt = stmt.order_by(EarthInteractable.updated_at.desc(), EarthInteractable.id.asc())
|
||||
result = await db.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
179
backend/app/services/earth_layer_adapters.py
Normal file
179
backend/app/services/earth_layer_adapters.py
Normal file
@@ -0,0 +1,179 @@
|
||||
"""Earth layer adapter registry for datasource-backed refresh behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EarthLayerAdapter:
|
||||
sources: frozenset[str]
|
||||
layers: tuple[str, ...]
|
||||
cache_patterns: tuple[str, ...]
|
||||
tables: frozenset[str] = field(default_factory=frozenset)
|
||||
derived_models: tuple[str, ...] = field(default_factory=tuple)
|
||||
refresh_strategy: str = "clear_then_reload"
|
||||
|
||||
|
||||
EARTH_LAYER_ADAPTERS: tuple[EarthLayerAdapter, ...] = (
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"barentswatch_vessels", "aisstream_vessels", "vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
|
||||
tables=frozenset({"vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
|
||||
layers=("vessels",),
|
||||
cache_patterns=("vessels*", "summary*"),
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset(
|
||||
{
|
||||
"telegeography_cables",
|
||||
"telegeography_landing",
|
||||
"telegeography_landing_points",
|
||||
"telegeography_systems",
|
||||
"telegeography_cable_systems",
|
||||
"arcgis_cables",
|
||||
"arcgis_landing_points",
|
||||
"arcgis_cable_landing_relation",
|
||||
"arcgis_cable_landing_relations",
|
||||
"fao_landing_points",
|
||||
}
|
||||
),
|
||||
tables=frozenset({"collected_data"}),
|
||||
layers=("cables",),
|
||||
cache_patterns=("cables*", "landing-points*", "summary*"),
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"celestrak_tle", "spacetrack_tle"}),
|
||||
tables=frozenset({"collected_data"}),
|
||||
layers=("satellites",),
|
||||
cache_patterns=("satellites*", "summary*"),
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset(
|
||||
{
|
||||
"top500",
|
||||
"top500_supercomputers",
|
||||
"epoch_ai_gpu",
|
||||
"huggingface_models",
|
||||
"huggingface_datasets",
|
||||
"huggingface_spaces",
|
||||
"compute_center_locations",
|
||||
}
|
||||
),
|
||||
tables=frozenset({"compute_center_locations"}),
|
||||
layers=("computeCenters",),
|
||||
cache_patterns=("compute-centers*", "summary*"),
|
||||
refresh_strategy="reload",
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset(
|
||||
{
|
||||
"ris_live_bgp",
|
||||
"bgpstream_bgp",
|
||||
"iptoasn_prefix_geo",
|
||||
"opengeofeed_prefix_geo",
|
||||
"nro_delegated_prefix_geo",
|
||||
"bgp_observations",
|
||||
"bgp_anomalies",
|
||||
"bgp_incidents",
|
||||
"bgp_collector_locations",
|
||||
}
|
||||
),
|
||||
tables=frozenset({"bgp_observations", "bgp_anomalies", "bgp_incidents", "bgp_collector_locations"}),
|
||||
layers=("bgp",),
|
||||
cache_patterns=("bgp*", "summary*"),
|
||||
derived_models=("bgp_observations", "bgp_anomalies", "bgp_incidents"),
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"news_live_streams"}),
|
||||
tables=frozenset({"collected_data"}),
|
||||
layers=("media",),
|
||||
cache_patterns=("summary*",),
|
||||
refresh_strategy="reload",
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"media_news_archive", "earth_news_items"}),
|
||||
tables=frozenset({"earth_news_items"}),
|
||||
layers=("news",),
|
||||
cache_patterns=("summary*",),
|
||||
refresh_strategy="reload",
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"earth_interactables"}),
|
||||
tables=frozenset({"earth_interactables"}),
|
||||
layers=("interactables",),
|
||||
cache_patterns=("interactables*", "summary*"),
|
||||
refresh_strategy="delta",
|
||||
),
|
||||
)
|
||||
|
||||
_ADAPTERS_BY_SOURCE = {
|
||||
source: adapter
|
||||
for adapter in EARTH_LAYER_ADAPTERS
|
||||
for source in adapter.sources
|
||||
}
|
||||
_ADAPTERS_BY_TABLE = {
|
||||
table: adapter
|
||||
for adapter in EARTH_LAYER_ADAPTERS
|
||||
for table in adapter.tables
|
||||
}
|
||||
|
||||
|
||||
def get_earth_layer_adapter_for_source(source: str | None) -> EarthLayerAdapter | None:
|
||||
return _ADAPTERS_BY_SOURCE.get(str(source or "").strip())
|
||||
|
||||
|
||||
def get_earth_layer_adapter_for_change(table: str | None, source: str | None) -> EarthLayerAdapter | None:
|
||||
table_key = str(table or "").strip()
|
||||
source_key = str(source or "").strip()
|
||||
if table_key and table_key != "collected_data":
|
||||
adapter = _ADAPTERS_BY_TABLE.get(table_key)
|
||||
if adapter is not None:
|
||||
return adapter
|
||||
return get_earth_layer_adapter_for_source(source_key)
|
||||
|
||||
|
||||
def get_earth_update_layers_for_source(source: str | None) -> list[str]:
|
||||
adapter = get_earth_layer_adapter_for_source(source)
|
||||
return list(adapter.layers) if adapter else []
|
||||
|
||||
|
||||
def get_earth_update_layers_for_change(table: str | None, source: str | None) -> list[str]:
|
||||
adapter = get_earth_layer_adapter_for_change(table, source)
|
||||
return list(adapter.layers) if adapter else []
|
||||
|
||||
|
||||
def get_earth_refresh_strategy_for_change(table: str | None, source: str | None) -> str | None:
|
||||
adapter = get_earth_layer_adapter_for_change(table, source)
|
||||
return adapter.refresh_strategy if adapter else None
|
||||
|
||||
|
||||
def get_earth_cache_patterns_for_source(source: str | None) -> list[str]:
|
||||
adapter = get_earth_layer_adapter_for_source(source)
|
||||
return list(adapter.cache_patterns) if adapter else []
|
||||
|
||||
|
||||
async def clear_derived_datasource_data(db: AsyncSession, source: str) -> dict[str, int]:
|
||||
adapter = get_earth_layer_adapter_for_source(source)
|
||||
if adapter is None or not adapter.derived_models:
|
||||
return {}
|
||||
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.models.bgp_observation import BGPObservation
|
||||
|
||||
model_by_key: dict[str, Any] = {
|
||||
"bgp_observations": BGPObservation,
|
||||
"bgp_anomalies": BGPAnomaly,
|
||||
"bgp_incidents": BGPIncident,
|
||||
}
|
||||
deleted_counts: dict[str, int] = {}
|
||||
for key in adapter.derived_models:
|
||||
model = model_by_key.get(key)
|
||||
if model is None:
|
||||
continue
|
||||
result = await db.execute(model.__table__.delete().where(model.source == source))
|
||||
deleted_counts[key] = int(result.rowcount or 0)
|
||||
return deleted_counts
|
||||
384
backend/app/services/earth_layer_cache.py
Normal file
384
backend/app/services/earth_layer_cache.py
Normal file
@@ -0,0 +1,384 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Response
|
||||
|
||||
from app.core.cache import _RedisClient
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="earth_layer_cache")
|
||||
|
||||
EARTH_LAYER_CACHE_PREFIX = "earth:layer:v1"
|
||||
EARTH_LAYER_LOCK_PREFIX = "earth:layer:lock:v1"
|
||||
DEFAULT_LOCK_TTL_SECONDS = 10
|
||||
DEFAULT_LOCK_WAIT_SECONDS = 0.2
|
||||
DEFAULT_MAX_FEATURES = 5000
|
||||
DEFAULT_MAX_BYTES = 5 * 1024 * 1024
|
||||
DEFAULT_BBOX_PRECISION_DEGREES = 0.1
|
||||
DEV_CACHE_KEY_HEADER = {"development", "dev", "test", "testing", "local"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EarthLayerCachePolicy:
|
||||
fresh_ttl_seconds: int
|
||||
stale_ttl_seconds: int
|
||||
max_features: int = DEFAULT_MAX_FEATURES
|
||||
max_bytes: int = DEFAULT_MAX_BYTES
|
||||
lock_ttl_seconds: int = DEFAULT_LOCK_TTL_SECONDS
|
||||
lock_wait_seconds: float = DEFAULT_LOCK_WAIT_SECONDS
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EarthLayerCacheResult:
|
||||
payload: dict[str, Any]
|
||||
state: str
|
||||
key: str
|
||||
features: int
|
||||
bytes: int
|
||||
|
||||
|
||||
class EarthLayerCache:
|
||||
def __init__(self) -> None:
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
if self._client is None:
|
||||
self._client = _RedisClient.get_client()
|
||||
return self._client
|
||||
|
||||
@staticmethod
|
||||
def key(layer: str, **params: Any) -> str:
|
||||
parts = [EARTH_LAYER_CACHE_PREFIX, _safe_key_part(layer)]
|
||||
for name in sorted(params):
|
||||
value = params[name]
|
||||
if value is None:
|
||||
value = "none"
|
||||
parts.append(f"{_safe_key_part(name)}:{_safe_key_part(value)}")
|
||||
return ":".join(parts)
|
||||
|
||||
@staticmethod
|
||||
def stale_key(key: str) -> str:
|
||||
return f"{key}:stale"
|
||||
|
||||
@staticmethod
|
||||
def lock_key(key: str) -> str:
|
||||
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:32]
|
||||
return f"{EARTH_LAYER_LOCK_PREFIX}:{digest}"
|
||||
|
||||
def get_json(self, key: str) -> dict[str, Any] | None:
|
||||
raw = self.client.get(key)
|
||||
if not raw:
|
||||
return None
|
||||
value = json.loads(raw)
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
def set_json(self, key: str, payload: dict[str, Any], ttl_seconds: int) -> None:
|
||||
self.client.setex(key, ttl_seconds, json.dumps(payload, ensure_ascii=False, default=str))
|
||||
|
||||
def acquire_lock(self, key: str, ttl_seconds: int) -> bool:
|
||||
return bool(self.client.set(self.lock_key(key), "1", nx=True, ex=ttl_seconds))
|
||||
|
||||
def release_lock(self, key: str) -> None:
|
||||
try:
|
||||
self.client.delete(self.lock_key(key))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def delete_pattern(self, pattern: str = f"{EARTH_LAYER_CACHE_PREFIX}:*") -> int:
|
||||
keys = list(self.client.scan_iter(match=pattern))
|
||||
if not keys:
|
||||
return 0
|
||||
return int(self.client.delete(*keys))
|
||||
|
||||
def status(self) -> dict[str, Any]:
|
||||
keys = list(self.client.scan_iter(match=f"{EARTH_LAYER_CACHE_PREFIX}:*"))
|
||||
by_layer: dict[str, dict[str, Any]] = {}
|
||||
total_memory = 0
|
||||
for key in keys:
|
||||
key_str = key.decode("utf-8") if isinstance(key, bytes) else str(key)
|
||||
layer = _layer_from_key(key_str)
|
||||
entry = by_layer.setdefault(layer, {"keys": 0, "stale_keys": 0, "memory_bytes": 0})
|
||||
entry["keys"] += 1
|
||||
if key_str.endswith(":stale"):
|
||||
entry["stale_keys"] += 1
|
||||
try:
|
||||
memory = int(self.client.memory_usage(key) or 0)
|
||||
except Exception:
|
||||
memory = 0
|
||||
entry["memory_bytes"] += memory
|
||||
total_memory += memory
|
||||
return {
|
||||
"prefix": EARTH_LAYER_CACHE_PREFIX,
|
||||
"key_count": len(keys),
|
||||
"memory_bytes": total_memory,
|
||||
"layers": by_layer,
|
||||
}
|
||||
|
||||
|
||||
earth_layer_cache = EarthLayerCache()
|
||||
|
||||
|
||||
def quantize_bbox(
|
||||
bbox: tuple[float, float, float, float],
|
||||
*,
|
||||
precision: float = DEFAULT_BBOX_PRECISION_DEGREES,
|
||||
) -> tuple[float, float, float, float]:
|
||||
return tuple(round(value / precision) * precision for value in bbox) # type: ignore[return-value]
|
||||
|
||||
|
||||
def format_bbox_key(bbox: tuple[float, float, float, float]) -> str:
|
||||
return ",".join(f"{value:.1f}" for value in bbox)
|
||||
|
||||
|
||||
def apply_cache_headers(response: Response | None, result: EarthLayerCacheResult) -> None:
|
||||
if response is None:
|
||||
return
|
||||
response.headers["X-Planet-Cache"] = result.state
|
||||
response.headers["X-Planet-Cache-Features"] = str(result.features)
|
||||
response.headers["X-Planet-Cache-Bytes"] = str(result.bytes)
|
||||
env_name = str(getattr(settings, "ENVIRONMENT", "") or "development").lower()
|
||||
if env_name in DEV_CACHE_KEY_HEADER:
|
||||
response.headers["X-Planet-Cache-Key"] = result.key
|
||||
|
||||
|
||||
async def get_or_build_layer_payload(
|
||||
*,
|
||||
key: str,
|
||||
policy: EarthLayerCachePolicy,
|
||||
builder: Callable[[], Awaitable[dict[str, Any]]],
|
||||
response: Response | None = None,
|
||||
) -> dict[str, Any]:
|
||||
result = await resolve_layer_payload(key=key, policy=policy, builder=builder)
|
||||
apply_cache_headers(response, result)
|
||||
return result.payload
|
||||
|
||||
|
||||
async def resolve_layer_payload(
|
||||
*,
|
||||
key: str,
|
||||
policy: EarthLayerCachePolicy,
|
||||
builder: Callable[[], Awaitable[dict[str, Any]]],
|
||||
) -> EarthLayerCacheResult:
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
cached = earth_layer_cache.get_json(key)
|
||||
if cached is not None:
|
||||
return _result(cached, state="hit", key=key)
|
||||
|
||||
lock_acquired = earth_layer_cache.acquire_lock(key, policy.lock_ttl_seconds)
|
||||
if lock_acquired:
|
||||
try:
|
||||
payload = await _build_budgeted_payload(builder, policy)
|
||||
_write_fresh_and_stale(key, payload, policy)
|
||||
_log_cache_event("refresh", key, payload, started)
|
||||
return _result(payload, state="refresh", key=key)
|
||||
except Exception as exc:
|
||||
stale = _read_stale(key)
|
||||
if stale is not None:
|
||||
logger.warning_event(
|
||||
"Earth layer cache builder failed; returning stale payload",
|
||||
event="earth_layer_cache.stale_after_builder_error",
|
||||
context={"key": key, "error": str(exc)},
|
||||
)
|
||||
return _result(stale, state="stale", key=key)
|
||||
raise
|
||||
finally:
|
||||
earth_layer_cache.release_lock(key)
|
||||
|
||||
stale = _read_stale(key)
|
||||
if stale is not None:
|
||||
return _result(stale, state="stale", key=key)
|
||||
|
||||
await asyncio.sleep(policy.lock_wait_seconds)
|
||||
cached_after_wait = earth_layer_cache.get_json(key)
|
||||
if cached_after_wait is not None:
|
||||
return _result(cached_after_wait, state="hit", key=key)
|
||||
|
||||
payload = await _build_budgeted_payload(builder, policy)
|
||||
_log_cache_event("miss", key, payload, started)
|
||||
return _result(payload, state="miss", key=key)
|
||||
except Exception as exc:
|
||||
try:
|
||||
payload = await _build_budgeted_payload(builder, policy)
|
||||
except Exception:
|
||||
raise exc
|
||||
logger.warning_event(
|
||||
"Earth layer cache bypassed",
|
||||
event="earth_layer_cache.bypass",
|
||||
context={"key": key, "error": str(exc)},
|
||||
)
|
||||
return _result(payload, state="bypass", key=key)
|
||||
|
||||
|
||||
def apply_payload_budget(payload: dict[str, Any], policy: EarthLayerCachePolicy) -> dict[str, Any]:
|
||||
budgeted = _truncate_features(payload, policy.max_features, "feature_budget")
|
||||
size = _payload_size(budgeted)
|
||||
if size <= policy.max_bytes:
|
||||
return budgeted
|
||||
|
||||
features = budgeted.get("features")
|
||||
if not isinstance(features, list):
|
||||
return _with_budget_diagnostics(
|
||||
budgeted,
|
||||
truncated=True,
|
||||
reason="byte_budget",
|
||||
bytes_before=size,
|
||||
bytes_after=size,
|
||||
)
|
||||
|
||||
low = 0
|
||||
high = len(features)
|
||||
best = []
|
||||
best_size = _payload_size({**budgeted, "features": best})
|
||||
while low <= high:
|
||||
mid = (low + high) // 2
|
||||
candidate_features = features[:mid]
|
||||
candidate = _with_budget_diagnostics(
|
||||
{**budgeted, "features": candidate_features},
|
||||
truncated=mid < len(features),
|
||||
reason="byte_budget",
|
||||
bytes_before=size,
|
||||
bytes_after=0,
|
||||
)
|
||||
candidate_size = _payload_size(candidate)
|
||||
if candidate_size <= policy.max_bytes:
|
||||
best = candidate_features
|
||||
best_size = candidate_size
|
||||
low = mid + 1
|
||||
else:
|
||||
high = mid - 1
|
||||
|
||||
return _with_budget_diagnostics(
|
||||
{**budgeted, "features": best},
|
||||
truncated=True,
|
||||
reason="byte_budget",
|
||||
bytes_before=size,
|
||||
bytes_after=best_size,
|
||||
)
|
||||
|
||||
|
||||
def invalidate_earth_layer_cache_for_source(source: str) -> int:
|
||||
from app.services.earth_layer_adapters import get_earth_cache_patterns_for_source
|
||||
|
||||
source_key = str(source or "").strip()
|
||||
patterns = get_earth_cache_patterns_for_source(source_key)
|
||||
deleted = 0
|
||||
for layer_pattern in patterns:
|
||||
deleted += earth_layer_cache.delete_pattern(f"{EARTH_LAYER_CACHE_PREFIX}:{layer_pattern}")
|
||||
return deleted
|
||||
|
||||
|
||||
async def _build_budgeted_payload(
|
||||
builder: Callable[[], Awaitable[dict[str, Any]]],
|
||||
policy: EarthLayerCachePolicy,
|
||||
) -> dict[str, Any]:
|
||||
payload = await builder()
|
||||
return apply_payload_budget(payload, policy)
|
||||
|
||||
|
||||
def _write_fresh_and_stale(key: str, payload: dict[str, Any], policy: EarthLayerCachePolicy) -> None:
|
||||
earth_layer_cache.set_json(key, payload, policy.fresh_ttl_seconds)
|
||||
earth_layer_cache.set_json(earth_layer_cache.stale_key(key), payload, policy.stale_ttl_seconds)
|
||||
|
||||
|
||||
def _read_stale(key: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
return earth_layer_cache.get_json(earth_layer_cache.stale_key(key))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _truncate_features(payload: dict[str, Any], max_features: int, reason: str) -> dict[str, Any]:
|
||||
features = payload.get("features")
|
||||
if not isinstance(features, list) or len(features) <= max_features:
|
||||
return payload
|
||||
return _with_budget_diagnostics(
|
||||
{**payload, "features": features[:max_features]},
|
||||
truncated=True,
|
||||
reason=reason,
|
||||
original_feature_count=len(features),
|
||||
)
|
||||
|
||||
|
||||
def _with_budget_diagnostics(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
truncated: bool,
|
||||
reason: str,
|
||||
original_feature_count: int | None = None,
|
||||
bytes_before: int | None = None,
|
||||
bytes_after: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
diagnostics = dict(payload.get("diagnostics") or {})
|
||||
diagnostics.update(
|
||||
{
|
||||
"truncated": bool(truncated or diagnostics.get("truncated")),
|
||||
"limit_reason": reason,
|
||||
}
|
||||
)
|
||||
if original_feature_count is not None:
|
||||
diagnostics["original_feature_count"] = original_feature_count
|
||||
if bytes_before is not None:
|
||||
diagnostics["bytes_before_budget"] = bytes_before
|
||||
if bytes_after is not None:
|
||||
diagnostics["bytes_after_budget"] = bytes_after
|
||||
return {**payload, "diagnostics": diagnostics}
|
||||
|
||||
|
||||
def _result(payload: dict[str, Any], *, state: str, key: str) -> EarthLayerCacheResult:
|
||||
return EarthLayerCacheResult(
|
||||
payload=payload,
|
||||
state=state,
|
||||
key=key,
|
||||
features=_feature_count(payload),
|
||||
bytes=_payload_size(payload),
|
||||
)
|
||||
|
||||
|
||||
def _feature_count(payload: dict[str, Any]) -> int:
|
||||
features = payload.get("features")
|
||||
if isinstance(features, list):
|
||||
return len(features)
|
||||
count = payload.get("count")
|
||||
return int(count) if isinstance(count, int) else 0
|
||||
|
||||
|
||||
def _payload_size(payload: dict[str, Any]) -> int:
|
||||
return len(json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8"))
|
||||
|
||||
|
||||
def _safe_key_part(value: Any) -> str:
|
||||
raw = str(value).strip().lower()
|
||||
return "".join(char if char.isalnum() or char in {"-", "_", ".", ","} else "_" for char in raw)[:160]
|
||||
|
||||
|
||||
def _layer_from_key(key: str) -> str:
|
||||
prefix = f"{EARTH_LAYER_CACHE_PREFIX}:"
|
||||
if not key.startswith(prefix):
|
||||
return "unknown"
|
||||
remainder = key[len(prefix):]
|
||||
return remainder.split(":", 1)[0]
|
||||
|
||||
|
||||
def _log_cache_event(state: str, key: str, payload: dict[str, Any], started: float) -> None:
|
||||
logger.info_event(
|
||||
"Earth layer cache resolved",
|
||||
event="earth_layer_cache.resolved",
|
||||
context={
|
||||
"state": state,
|
||||
"key": key,
|
||||
"features": _feature_count(payload),
|
||||
"bytes": _payload_size(payload),
|
||||
"duration_ms": round((time.perf_counter() - started) * 1000, 2),
|
||||
},
|
||||
)
|
||||
@@ -1,11 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
@@ -13,6 +15,13 @@ import xml.etree.ElementTree as ET
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
|
||||
from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder
|
||||
|
||||
|
||||
USER_AGENT = "PlanetEarthNewsBoard/1.0 (+https://planet.local)"
|
||||
@@ -20,6 +29,11 @@ REQUEST_TIMEOUT = 12.0
|
||||
MAX_ITEMS_PER_SOURCE = 6
|
||||
MAX_ITEMS_TOTAL = 12
|
||||
STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
|
||||
RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS
|
||||
MAX_TARGET_INFERENCE_CONCURRENCY = 3
|
||||
TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0
|
||||
DEFAULT_NEWS_LOCALE = "zh-CN"
|
||||
NEWS_ENRICH_PROMPT_KEY = "earth.news.enrich"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -49,6 +63,17 @@ class NewsFeedSource:
|
||||
priority: int = 100
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NewsTargetLocation:
|
||||
latitude: float
|
||||
longitude: float
|
||||
label: str
|
||||
source: str
|
||||
confidence: float | None = None
|
||||
country: str | None = None
|
||||
city: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedNewsItem:
|
||||
id: str
|
||||
@@ -60,6 +85,18 @@ class ParsedNewsItem:
|
||||
feed_region: str
|
||||
homepage_url: str
|
||||
published_at: datetime | None
|
||||
content_language: str = "en"
|
||||
localizations: dict[str, dict[str, str]] = field(default_factory=dict)
|
||||
enrichment_status: str = "pending"
|
||||
enrichment_error: str | None = None
|
||||
enriched_at: datetime | None = None
|
||||
target_location: NewsTargetLocation | None = None
|
||||
target_resolution_stage: str = "unresolved"
|
||||
target_ai_attempted: bool = False
|
||||
target_ai_status: str = "not_attempted"
|
||||
target_ai_error: str | None = None
|
||||
target_debug_note: str | None = None
|
||||
location_patch: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -236,6 +273,17 @@ NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = (
|
||||
|
||||
|
||||
_REGION_CACHE: dict[str, CachedRegionFeed] = {}
|
||||
_news_target_geocode = build_default_nominatim_geocoder(user_agent=USER_AGENT)
|
||||
_CITY_HINTS: tuple[dict[str, str | None], ...] = (
|
||||
{"name": "Beijing", "country": "中国"},
|
||||
{"name": "Havana", "country": "古巴"},
|
||||
{"name": "Kyiv", "country": "乌克兰"},
|
||||
{"name": "Bangkok", "country": "泰国"},
|
||||
{"name": "Tehran", "country": "伊朗"},
|
||||
{"name": "Moscow", "country": "俄罗斯"},
|
||||
{"name": "Taipei", "country": "中国(台湾)"},
|
||||
{"name": "Hong Kong", "country": "中国(香港)"},
|
||||
)
|
||||
|
||||
|
||||
def determine_focus_region(lat: float | None, lon: float | None) -> str:
|
||||
@@ -258,6 +306,462 @@ def get_region_anchor(region: str) -> RegionAnchor:
|
||||
return REGION_ANCHORS.get(region, REGION_ANCHORS["global"])
|
||||
|
||||
|
||||
def _coerce_str(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
value = str(value)
|
||||
cleaned = re.sub(r"\s+", " ", value).strip()
|
||||
return cleaned or None
|
||||
|
||||
|
||||
def _contains_location_alias(text: str, alias: str) -> bool:
|
||||
normalized_alias = _coerce_str(alias)
|
||||
if not normalized_alias:
|
||||
return False
|
||||
if re.search(r"[A-Za-z]", normalized_alias):
|
||||
pattern = r"(?<![A-Za-z])" + re.escape(normalized_alias) + r"(?![A-Za-z])"
|
||||
return re.search(pattern, text, flags=re.IGNORECASE) is not None
|
||||
return normalized_alias in text
|
||||
|
||||
|
||||
def _iter_searchable_country_variants(
|
||||
canonical: str,
|
||||
variants: list[str],
|
||||
) -> tuple[str, ...]:
|
||||
searchable: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for variant in (canonical, *variants):
|
||||
normalized = _coerce_str(variant)
|
||||
if not normalized:
|
||||
continue
|
||||
if re.fullmatch(r"[A-Z]{2,3}", normalized):
|
||||
continue
|
||||
if len(normalized) <= 2:
|
||||
continue
|
||||
key = normalized.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
searchable.append(normalized)
|
||||
return tuple(searchable)
|
||||
|
||||
|
||||
def _coerce_float(value: Any) -> float | None:
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not math.isfinite(parsed):
|
||||
return None
|
||||
return parsed
|
||||
|
||||
|
||||
def _first_json_object(text: str) -> dict[str, Any] | None:
|
||||
if not text:
|
||||
return None
|
||||
decoder = json.JSONDecoder()
|
||||
for index, char in enumerate(text):
|
||||
if char != "{":
|
||||
continue
|
||||
try:
|
||||
payload, _ = decoder.raw_decode(text[index:])
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(payload, dict):
|
||||
return payload
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_localizations(value: Any) -> dict[str, dict[str, str]]:
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
normalized: dict[str, dict[str, str]] = {}
|
||||
for locale, payload in value.items():
|
||||
locale_key = _coerce_str(locale)
|
||||
if not locale_key or not isinstance(payload, dict):
|
||||
continue
|
||||
title = _coerce_str(payload.get("title"))
|
||||
summary = _coerce_str(payload.get("summary"))
|
||||
entry: dict[str, str] = {}
|
||||
if title:
|
||||
entry["title"] = title
|
||||
if summary:
|
||||
entry["summary"] = summary
|
||||
if entry:
|
||||
normalized[locale_key] = entry
|
||||
return normalized
|
||||
|
||||
|
||||
def _get_locale_text(
|
||||
item: ParsedNewsItem,
|
||||
key: str,
|
||||
*,
|
||||
locale: str = DEFAULT_NEWS_LOCALE,
|
||||
) -> str:
|
||||
localized = item.localizations.get(locale)
|
||||
if isinstance(localized, dict):
|
||||
value = _coerce_str(localized.get(key))
|
||||
if value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def _has_default_localization(item: ParsedNewsItem) -> bool:
|
||||
localized = item.localizations.get(DEFAULT_NEWS_LOCALE)
|
||||
if not isinstance(localized, dict):
|
||||
return False
|
||||
return bool(_coerce_str(localized.get("title")) and _coerce_str(localized.get("summary")))
|
||||
|
||||
|
||||
def apply_enrichment_patch_to_item(
|
||||
item: ParsedNewsItem,
|
||||
patch: dict[str, Any],
|
||||
) -> ParsedNewsItem:
|
||||
item.location_patch = patch
|
||||
if "content_language" in patch:
|
||||
item.content_language = _coerce_str(patch.get("content_language")) or item.content_language
|
||||
if "localizations" in patch:
|
||||
item.localizations = _normalize_localizations(patch.get("localizations"))
|
||||
if "enrichment_status" in patch:
|
||||
item.enrichment_status = _coerce_str(patch.get("enrichment_status")) or item.enrichment_status
|
||||
if "enrichment_error" in patch:
|
||||
item.enrichment_error = _coerce_str(patch.get("enrichment_error"))
|
||||
if "enriched_at" in patch:
|
||||
item.enriched_at = _parse_datetime(_coerce_str(patch.get("enriched_at")))
|
||||
return item
|
||||
|
||||
|
||||
async def _geocode_target_location(query: str) -> dict[str, Any] | None:
|
||||
return await asyncio.to_thread(_news_target_geocode, query)
|
||||
|
||||
|
||||
async def _build_target_location_from_payload(
|
||||
payload: dict[str, Any],
|
||||
) -> NewsTargetLocation | None:
|
||||
country = normalize_country(payload.get("country"))
|
||||
city = _coerce_str(payload.get("city"))
|
||||
matched_location_name = _coerce_str(payload.get("matched_location_name"))
|
||||
confidence = _coerce_float(payload.get("confidence"))
|
||||
if confidence is not None:
|
||||
confidence = max(0.0, min(confidence, 1.0))
|
||||
|
||||
latitude = _coerce_float(payload.get("latitude"))
|
||||
longitude = _coerce_float(payload.get("longitude"))
|
||||
if latitude not in (None, 0.0) and longitude not in (None, 0.0):
|
||||
label = matched_location_name or ", ".join(part for part in (city, country) if part) or "关联位置"
|
||||
return NewsTargetLocation(
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
label=label,
|
||||
source="ai_inferred_target",
|
||||
confidence=confidence,
|
||||
country=country,
|
||||
city=city,
|
||||
)
|
||||
|
||||
geocode_queries: list[str] = []
|
||||
for value in (
|
||||
", ".join(part for part in (city, country) if part),
|
||||
matched_location_name,
|
||||
city,
|
||||
country,
|
||||
):
|
||||
normalized = _coerce_str(value)
|
||||
if normalized and normalized not in geocode_queries:
|
||||
geocode_queries.append(normalized)
|
||||
|
||||
for query in geocode_queries:
|
||||
try:
|
||||
result = await _geocode_target_location(query)
|
||||
except Exception:
|
||||
continue
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
latitude = _coerce_float(result.get("lat"))
|
||||
longitude = _coerce_float(result.get("lon"))
|
||||
if latitude in (None, 0.0) or longitude in (None, 0.0):
|
||||
continue
|
||||
label = (
|
||||
_coerce_str(result.get("display_name"))
|
||||
or matched_location_name
|
||||
or ", ".join(part for part in (city, country) if part)
|
||||
or query
|
||||
)
|
||||
return NewsTargetLocation(
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
label=label,
|
||||
source="ai_inferred_target",
|
||||
confidence=confidence,
|
||||
country=country,
|
||||
city=city,
|
||||
)
|
||||
|
||||
centroid = get_country_centroid(country)
|
||||
if centroid:
|
||||
label = matched_location_name or city or country or "关联位置"
|
||||
return NewsTargetLocation(
|
||||
latitude=centroid["latitude"],
|
||||
longitude=centroid["longitude"],
|
||||
label=label,
|
||||
source="ai_inferred_target",
|
||||
confidence=confidence,
|
||||
country=country,
|
||||
city=city,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def _extract_target_location_from_text(item: ParsedNewsItem) -> NewsTargetLocation | None:
|
||||
combined_text = " ".join(part for part in (item.title, item.summary) if part).strip()
|
||||
if not combined_text:
|
||||
return None
|
||||
|
||||
for hint in _CITY_HINTS:
|
||||
city_name = _coerce_str(hint.get("name"))
|
||||
if not city_name or not _contains_location_alias(combined_text, city_name):
|
||||
continue
|
||||
country = normalize_country(hint.get("country"))
|
||||
geocode_query = ", ".join(part for part in (city_name, country) if part)
|
||||
try:
|
||||
result = await _geocode_target_location(geocode_query)
|
||||
except Exception:
|
||||
result = None
|
||||
if isinstance(result, dict):
|
||||
latitude = _coerce_float(result.get("lat"))
|
||||
longitude = _coerce_float(result.get("lon"))
|
||||
if latitude not in (None, 0.0) and longitude not in (None, 0.0):
|
||||
return NewsTargetLocation(
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
label=_coerce_str(result.get("display_name")) or geocode_query,
|
||||
source="headline_location_hint",
|
||||
confidence=0.78,
|
||||
country=country,
|
||||
city=city_name,
|
||||
)
|
||||
centroid = get_country_centroid(country)
|
||||
if centroid:
|
||||
return NewsTargetLocation(
|
||||
latitude=centroid["latitude"],
|
||||
longitude=centroid["longitude"],
|
||||
label=geocode_query,
|
||||
source="headline_location_hint",
|
||||
confidence=0.68,
|
||||
country=country,
|
||||
city=city_name,
|
||||
)
|
||||
|
||||
for canonical, variants in COUNTRY_VARIANTS_MAP.items():
|
||||
if not get_country_centroid(canonical):
|
||||
continue
|
||||
searchable_variants = _iter_searchable_country_variants(canonical, variants)
|
||||
if not any(_contains_location_alias(combined_text, variant) for variant in searchable_variants):
|
||||
continue
|
||||
centroid = get_country_centroid(canonical)
|
||||
if not centroid:
|
||||
continue
|
||||
return NewsTargetLocation(
|
||||
latitude=centroid["latitude"],
|
||||
longitude=centroid["longitude"],
|
||||
label=canonical,
|
||||
source="headline_country_hint",
|
||||
confidence=0.62,
|
||||
country=canonical,
|
||||
city=None,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def _infer_news_target_location(
|
||||
item: ParsedNewsItem,
|
||||
*,
|
||||
provider_client: AIProviderClient | None,
|
||||
prompt: EffectiveAIPrompt | None = None,
|
||||
) -> NewsTargetLocation | None:
|
||||
target, _localizations = await _infer_news_enrichment(
|
||||
item,
|
||||
provider_client=provider_client,
|
||||
prompt=prompt,
|
||||
)
|
||||
return target
|
||||
|
||||
|
||||
async def _infer_news_enrichment(
|
||||
item: ParsedNewsItem,
|
||||
*,
|
||||
provider_client: AIProviderClient | None,
|
||||
prompt: EffectiveAIPrompt | None = None,
|
||||
) -> tuple[NewsTargetLocation | None, dict[str, dict[str, str]]]:
|
||||
text_hint = await _extract_target_location_from_text(item)
|
||||
content_error: str | None = None
|
||||
if text_hint is not None and text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source
|
||||
item.target_ai_attempted = False
|
||||
item.target_ai_status = "skipped_text_hint"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = f"text hint matched {text_hint.label}"
|
||||
localizations: dict[str, dict[str, str]] = {}
|
||||
|
||||
if provider_client is None:
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_attempted = False
|
||||
item.target_ai_status = "unavailable"
|
||||
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
|
||||
item.target_debug_note = (
|
||||
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
||||
)
|
||||
item.enrichment_status = "unavailable"
|
||||
item.enrichment_error = "AI provider is not configured or unavailable for earth-feed."
|
||||
return text_hint, localizations
|
||||
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_ai_attempted = True
|
||||
item.target_ai_status = "attempted"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = (
|
||||
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
||||
)
|
||||
item.enrichment_status = "attempted"
|
||||
item.enrichment_error = None
|
||||
|
||||
prompt = prompt or await get_effective_prompt(None, NEWS_ENRICH_PROMPT_KEY)
|
||||
request = SituationalAnalysisRequest(
|
||||
title="Enrich Earth news item with event location and zh-CN content",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"news_item": {
|
||||
"title": item.title,
|
||||
"summary": item.summary,
|
||||
"source": item.source,
|
||||
"feed_name": item.feed_name,
|
||||
"feed_region": item.feed_region,
|
||||
"url": item.url,
|
||||
"published_at": (
|
||||
item.published_at.isoformat().replace("+00:00", "Z")
|
||||
if item.published_at
|
||||
else None
|
||||
),
|
||||
},
|
||||
"required_json_schema": {
|
||||
"location": {
|
||||
"country": "string|null",
|
||||
"city": "string|null",
|
||||
"matched_location_name": "string|null",
|
||||
"latitude": "number|null",
|
||||
"longitude": "number|null",
|
||||
"confidence": "number from 0 to 1",
|
||||
"reasoning_summary": "short string",
|
||||
},
|
||||
"localizations": {
|
||||
"zh-CN": {
|
||||
"title": "faithful Simplified Chinese title",
|
||||
"summary": "one-sentence newswire-style Simplified Chinese lead summary",
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
constraints=[
|
||||
"Return only strict JSON. Do not wrap it in markdown.",
|
||||
"For localizations, do not add facts that are absent from the RSS headline, description, source, or date.",
|
||||
"Write zh-CN summary as one concise newswire-style sentence, like a breaking-news lead.",
|
||||
"If the RSS description is thin, write a conservative one-sentence summary that says only what is supported.",
|
||||
"Keep zh-CN summary factual, non-promotional, and avoid colon-heavy keyword labels.",
|
||||
"Prefer the event location, not the newsroom or publisher headquarters.",
|
||||
"When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.",
|
||||
"Use null for unknown fields instead of inventing details.",
|
||||
"Calibrate confidence conservatively: 0.75+ only when the city is strongly supported, 0.55-0.74 for country-level or likely city inference, below 0.55 when weak.",
|
||||
],
|
||||
)
|
||||
try:
|
||||
response = await provider_client.analyze(request)
|
||||
except Exception as exc:
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "provider_error"
|
||||
item.target_ai_error = str(exc)
|
||||
item.enrichment_status = "provider_error"
|
||||
item.enrichment_error = str(exc)
|
||||
return text_hint, localizations
|
||||
|
||||
payload = _first_json_object(response.content)
|
||||
if not isinstance(payload, dict):
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "parse_error"
|
||||
item.target_ai_error = "AI response did not contain a parseable JSON object."
|
||||
item.enrichment_status = "parse_error"
|
||||
item.enrichment_error = "AI response did not contain a parseable JSON object."
|
||||
return text_hint, localizations
|
||||
|
||||
localizations = _normalize_localizations(payload.get("localizations"))
|
||||
if not localizations:
|
||||
content_error = "AI returned no usable localizations."
|
||||
|
||||
location_payload = payload.get("location") if isinstance(payload.get("location"), dict) else payload
|
||||
if text_hint is not None and text_hint.city:
|
||||
target = text_hint
|
||||
else:
|
||||
target = await _build_target_location_from_payload(location_payload)
|
||||
if target is None:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "no_result"
|
||||
item.target_ai_error = "AI returned no usable target coordinates or geocodeable location."
|
||||
target = text_hint
|
||||
elif target.confidence is not None and target.confidence < 0.45:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "low_confidence"
|
||||
item.target_ai_error = f"AI target confidence too low: {target.confidence:.2f}"
|
||||
target = text_hint
|
||||
else:
|
||||
item.target_resolution_stage = target.source
|
||||
item.target_ai_status = "success"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = f"ai inferred {target.label}"
|
||||
|
||||
item.localizations = localizations
|
||||
if localizations and item.target_ai_status in {"success", "skipped_text_hint"}:
|
||||
item.enrichment_status = "success"
|
||||
item.enrichment_error = None
|
||||
elif localizations:
|
||||
item.enrichment_status = "content_only"
|
||||
item.enrichment_error = item.target_ai_error
|
||||
else:
|
||||
item.enrichment_status = "location_only" if target is not None else "no_result"
|
||||
item.enrichment_error = content_error or item.target_ai_error
|
||||
item.enriched_at = datetime.now(UTC) if localizations else None
|
||||
return target, localizations
|
||||
|
||||
|
||||
async def _enrich_items_with_target_locations(
|
||||
items: list[ParsedNewsItem],
|
||||
*,
|
||||
provider_client: AIProviderClient | None,
|
||||
prompt: EffectiveAIPrompt | None = None,
|
||||
) -> list[ParsedNewsItem]:
|
||||
if not items:
|
||||
return items
|
||||
|
||||
semaphore = asyncio.Semaphore(MAX_TARGET_INFERENCE_CONCURRENCY)
|
||||
|
||||
async def enrich(item: ParsedNewsItem) -> ParsedNewsItem:
|
||||
async with semaphore:
|
||||
target = await _infer_news_target_location(
|
||||
item,
|
||||
provider_client=provider_client,
|
||||
prompt=prompt,
|
||||
)
|
||||
item.target_location = target
|
||||
return item
|
||||
|
||||
return list(await asyncio.gather(*(enrich(item) for item in items)))
|
||||
|
||||
|
||||
def get_sources_for_region(region: str) -> list[NewsFeedSource]:
|
||||
return sorted(
|
||||
[source for source in NEWS_FEED_SOURCES if source.region in {"global", region}],
|
||||
@@ -349,7 +853,7 @@ def _parse_feed_entries(xml_text: str, source: NewsFeedSource) -> list[ParsedNew
|
||||
if not clean_title or not link:
|
||||
continue
|
||||
|
||||
item_source = _normalize_source_name(clean_title, source.name)
|
||||
item_source = source.name
|
||||
display_title = clean_title
|
||||
if source.source_type == "aggregated" and " - " in clean_title:
|
||||
parts = clean_title.rsplit(" - ", 1)
|
||||
@@ -385,23 +889,170 @@ def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]:
|
||||
]
|
||||
|
||||
|
||||
def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, Any]:
|
||||
published_at = item.published_at
|
||||
def _serialize_anchor(anchor: RegionAnchor) -> dict[str, Any]:
|
||||
return {
|
||||
"region": anchor.region,
|
||||
"label": anchor.label,
|
||||
"latitude": anchor.latitude,
|
||||
"longitude": anchor.longitude,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_target(target: NewsTargetLocation | None) -> dict[str, Any] | None:
|
||||
if target is None:
|
||||
return None
|
||||
return {
|
||||
"latitude": target.latitude,
|
||||
"longitude": target.longitude,
|
||||
"label": target.label,
|
||||
"source": target.source,
|
||||
"confidence": target.confidence,
|
||||
"country": target.country,
|
||||
"city": target.city,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_enriched_at(value: datetime | None) -> str | None:
|
||||
return value.isoformat().replace("+00:00", "Z") if value else None
|
||||
|
||||
|
||||
def _content_patch(item: ParsedNewsItem) -> dict[str, Any]:
|
||||
return {
|
||||
"content_language": item.content_language,
|
||||
"localizations": item.localizations,
|
||||
"enrichment_status": item.enrichment_status,
|
||||
"enrichment_error": item.enrichment_error,
|
||||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||||
}
|
||||
|
||||
|
||||
def build_anchor_location_patch(
|
||||
item: ParsedNewsItem,
|
||||
*,
|
||||
queued: bool = False,
|
||||
queue_available: bool | None = None,
|
||||
) -> dict[str, Any]:
|
||||
anchor = get_region_anchor(item.feed_region)
|
||||
if queued:
|
||||
resolution_stage = "queued"
|
||||
ai_status = "queued"
|
||||
debug_note = "queued for async target location inference"
|
||||
else:
|
||||
resolution_stage = item.target_resolution_stage
|
||||
ai_status = item.target_ai_status
|
||||
debug_note = item.target_debug_note
|
||||
content_patch = _content_patch(item)
|
||||
if queued and content_patch["enrichment_status"] == "pending":
|
||||
content_patch["enrichment_status"] = "queued"
|
||||
return {
|
||||
"latitude": anchor.latitude,
|
||||
"longitude": anchor.longitude,
|
||||
"location_label": anchor.label,
|
||||
"location_source": "region_anchor",
|
||||
"verified": False,
|
||||
"location_meta": {
|
||||
"resolution_stage": resolution_stage,
|
||||
"ai_attempted": item.target_ai_attempted,
|
||||
"ai_status": ai_status,
|
||||
"ai_error": item.target_ai_error,
|
||||
"debug_note": debug_note,
|
||||
"queue_available": queue_available,
|
||||
"target": None,
|
||||
"anchor": _serialize_anchor(anchor),
|
||||
},
|
||||
**content_patch,
|
||||
}
|
||||
|
||||
|
||||
def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation | None) -> dict[str, Any]:
|
||||
if target is None:
|
||||
return build_anchor_location_patch(item)
|
||||
anchor = get_region_anchor(item.feed_region)
|
||||
return {
|
||||
"latitude": target.latitude,
|
||||
"longitude": target.longitude,
|
||||
"location_label": target.label,
|
||||
"location_source": target.source,
|
||||
"verified": True,
|
||||
"location_meta": {
|
||||
"resolution_stage": item.target_resolution_stage,
|
||||
"ai_attempted": item.target_ai_attempted,
|
||||
"ai_status": item.target_ai_status,
|
||||
"ai_error": item.target_ai_error,
|
||||
"debug_note": item.target_debug_note,
|
||||
"target": _serialize_target(target),
|
||||
"anchor": _serialize_anchor(anchor),
|
||||
},
|
||||
**_content_patch(item),
|
||||
}
|
||||
|
||||
|
||||
def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
|
||||
published_at = item.published_at
|
||||
return {
|
||||
"id": item.id,
|
||||
"title": item.title,
|
||||
"summary": item.summary,
|
||||
"content_language": item.content_language,
|
||||
"localizations": item.localizations,
|
||||
"enrichment_status": item.enrichment_status,
|
||||
"enrichment_error": item.enrichment_error,
|
||||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||||
"url": item.url,
|
||||
"source": item.source,
|
||||
"feed_name": item.feed_name,
|
||||
"feed_region": item.feed_region,
|
||||
"homepage_url": item.homepage_url,
|
||||
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
|
||||
}
|
||||
|
||||
|
||||
def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem:
|
||||
return ParsedNewsItem(
|
||||
id=str(payload.get("id") or ""),
|
||||
title=str(payload.get("title") or ""),
|
||||
summary=str(payload.get("summary") or ""),
|
||||
content_language=str(payload.get("content_language") or "en"),
|
||||
localizations=_normalize_localizations(payload.get("localizations")),
|
||||
enrichment_status=str(payload.get("enrichment_status") or "pending"),
|
||||
enrichment_error=_coerce_str(payload.get("enrichment_error")),
|
||||
enriched_at=_parse_datetime(_coerce_str(payload.get("enriched_at"))),
|
||||
url=str(payload.get("url") or ""),
|
||||
source=str(payload.get("source") or ""),
|
||||
feed_name=str(payload.get("feed_name") or ""),
|
||||
feed_region=str(payload.get("feed_region") or "global"),
|
||||
homepage_url=str(payload.get("homepage_url") or ""),
|
||||
published_at=_parse_datetime(_coerce_str(payload.get("published_at"))),
|
||||
)
|
||||
|
||||
|
||||
def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, Any]:
|
||||
published_at = item.published_at
|
||||
location_patch = item.location_patch or build_target_location_patch(item, item.target_location)
|
||||
return {
|
||||
"id": item.id,
|
||||
"title": item.title,
|
||||
"summary": item.summary,
|
||||
"content_language": item.content_language,
|
||||
"localizations": item.localizations,
|
||||
"display_title": _get_locale_text(item, "title"),
|
||||
"display_summary": _get_locale_text(item, "summary"),
|
||||
"url": item.url,
|
||||
"source": item.source,
|
||||
"feed_name": item.feed_name,
|
||||
"region": item.feed_region,
|
||||
"display_region": get_region_anchor(item.feed_region).label,
|
||||
"homepage_url": item.homepage_url,
|
||||
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
|
||||
"latitude": anchor.latitude,
|
||||
"longitude": anchor.longitude,
|
||||
"location_label": anchor.label,
|
||||
"location_inferred": True,
|
||||
"latitude": location_patch["latitude"],
|
||||
"longitude": location_patch["longitude"],
|
||||
"location_label": location_patch["location_label"],
|
||||
"location_source": location_patch["location_source"],
|
||||
"verified": location_patch["verified"],
|
||||
"location_meta": location_patch["location_meta"],
|
||||
"enrichment_status": item.enrichment_status,
|
||||
"enrichment_error": item.enrichment_error,
|
||||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||||
"is_focus_match": item.feed_region == active_region,
|
||||
}
|
||||
|
||||
@@ -412,6 +1063,7 @@ def _build_payload(
|
||||
lon: float | None,
|
||||
active_region: str,
|
||||
items: list[ParsedNewsItem],
|
||||
cruise_items: list[ParsedNewsItem] | None = None,
|
||||
sources: list[NewsFeedSource],
|
||||
errors: list[str],
|
||||
stale: bool,
|
||||
@@ -426,10 +1078,15 @@ def _build_payload(
|
||||
"lon": lon,
|
||||
"region": active_region,
|
||||
"label": profile.label,
|
||||
"display_region": get_region_anchor(active_region).label,
|
||||
"accent": profile.accent,
|
||||
},
|
||||
"sources": _serialize_sources(sources),
|
||||
"items": [_serialize_item(item, active_region=active_region) for item in items],
|
||||
"cruise_items": [
|
||||
_serialize_item(item, active_region=active_region)
|
||||
for item in (cruise_items if cruise_items is not None else items)
|
||||
],
|
||||
"errors": errors,
|
||||
"stale": stale,
|
||||
}
|
||||
@@ -472,6 +1129,68 @@ def _store_region_cache(region: str, *, items: list[ParsedNewsItem], sources: li
|
||||
)
|
||||
|
||||
|
||||
async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> list[ParsedNewsItem]:
|
||||
if not items:
|
||||
return items
|
||||
|
||||
from app.services.earth_news_queue import (
|
||||
enqueue_target_location_job,
|
||||
get_cached_target_location_patch,
|
||||
)
|
||||
|
||||
async def enqueue_item(item: ParsedNewsItem, *, force: bool = False) -> bool:
|
||||
return await enqueue_target_location_job(build_target_location_job_payload(item), force=force)
|
||||
|
||||
async def apply_location(item: ParsedNewsItem) -> ParsedNewsItem:
|
||||
cached_patch = await get_cached_target_location_patch(item.id)
|
||||
if cached_patch:
|
||||
apply_enrichment_patch_to_item(item, cached_patch)
|
||||
if not _has_default_localization(item):
|
||||
queued = await enqueue_item(item, force=True)
|
||||
if queued and item.enrichment_status in {
|
||||
"pending",
|
||||
"unavailable",
|
||||
"provider_error",
|
||||
"parse_error",
|
||||
"no_result",
|
||||
"location_only",
|
||||
}:
|
||||
item.enrichment_status = "queued"
|
||||
return item
|
||||
|
||||
queued = await enqueue_item(item)
|
||||
item.location_patch = build_anchor_location_patch(
|
||||
item,
|
||||
queued=queued,
|
||||
queue_available=queued,
|
||||
)
|
||||
return item
|
||||
|
||||
return list(await asyncio.gather(*(apply_location(item) for item in items)))
|
||||
|
||||
|
||||
async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None:
|
||||
if not items:
|
||||
return
|
||||
|
||||
from app.services.earth_news_queue import enqueue_target_location_job
|
||||
|
||||
await asyncio.gather(
|
||||
*(
|
||||
enqueue_target_location_job(
|
||||
build_target_location_job_payload(item),
|
||||
force=not _has_default_localization(item),
|
||||
)
|
||||
for item in items
|
||||
if (
|
||||
item.location_patch is None
|
||||
or item.location_patch.get("verified") is False
|
||||
or not _has_default_localization(item)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_source(
|
||||
client: httpx.AsyncClient,
|
||||
source: NewsFeedSource,
|
||||
@@ -484,11 +1203,10 @@ async def _fetch_source(
|
||||
return source, [], str(exc)
|
||||
|
||||
|
||||
async def get_earth_news_payload(lat: float | None = None, lon: float | None = None) -> dict[str, Any]:
|
||||
active_region = determine_focus_region(lat, lon)
|
||||
sources = get_sources_for_region(active_region)
|
||||
async def _fetch_rss_items_for_sources(
|
||||
sources: list[NewsFeedSource],
|
||||
) -> tuple[list[ParsedNewsItem], list[str]]:
|
||||
errors: list[str] = []
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
@@ -502,9 +1220,29 @@ async def get_earth_news_payload(lat: float | None = None, lon: float | None = N
|
||||
errors.append(f"{source.name}: {error}")
|
||||
continue
|
||||
fetched_items.extend(items)
|
||||
return fetched_items, errors
|
||||
|
||||
|
||||
def _needs_rss_supplement(*, item_count: int, newest_at: datetime | None) -> bool:
|
||||
if item_count < MAX_ITEMS_TOTAL:
|
||||
return True
|
||||
if newest_at is None:
|
||||
return True
|
||||
age_seconds = (datetime.now(UTC) - newest_at).total_seconds()
|
||||
return age_seconds > RSS_SUPPLEMENT_MAX_AGE_SECONDS
|
||||
|
||||
|
||||
async def _get_earth_news_payload_from_rss_only(
|
||||
*,
|
||||
lat: float | None,
|
||||
lon: float | None,
|
||||
active_region: str,
|
||||
sources: list[NewsFeedSource],
|
||||
) -> dict[str, Any]:
|
||||
fetched_items, errors = await _fetch_rss_items_for_sources(sources)
|
||||
ranked_items = _rank_and_trim_items(fetched_items, active_region=active_region)
|
||||
if ranked_items:
|
||||
ranked_items = await _apply_cached_locations_and_enqueue(ranked_items)
|
||||
_store_region_cache(active_region, items=ranked_items, sources=sources)
|
||||
return _build_payload(
|
||||
lat=lat,
|
||||
@@ -518,6 +1256,7 @@ async def get_earth_news_payload(lat: float | None = None, lon: float | None = N
|
||||
|
||||
cached = _get_cached_region_feed(active_region)
|
||||
if cached:
|
||||
cached.items = await _apply_cached_locations_and_enqueue(cached.items)
|
||||
return _build_payload(
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
@@ -538,3 +1277,64 @@ async def get_earth_news_payload(lat: float | None = None, lon: float | None = N
|
||||
errors=errors,
|
||||
stale=False,
|
||||
)
|
||||
|
||||
|
||||
async def get_earth_news_payload(
|
||||
lat: float | None = None,
|
||||
lon: float | None = None,
|
||||
*,
|
||||
provider_client: AIProviderClient | None = None,
|
||||
db: AsyncSession | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del provider_client
|
||||
active_region = determine_focus_region(lat, lon)
|
||||
sources = get_sources_for_region(active_region)
|
||||
|
||||
if db is None:
|
||||
return await _get_earth_news_payload_from_rss_only(
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
active_region=active_region,
|
||||
sources=sources,
|
||||
)
|
||||
|
||||
from app.services.earth_news_store import (
|
||||
get_earth_news_freshness,
|
||||
list_earth_news_cruise_items,
|
||||
list_earth_news_items,
|
||||
upsert_earth_news_items,
|
||||
)
|
||||
|
||||
errors: list[str] = []
|
||||
item_count, newest_at = await get_earth_news_freshness(db, active_region=active_region)
|
||||
should_supplement = _needs_rss_supplement(item_count=item_count, newest_at=newest_at)
|
||||
if should_supplement:
|
||||
fetched_items, errors = await _fetch_rss_items_for_sources(sources)
|
||||
ranked_fetched_items = _rank_and_trim_items(fetched_items, active_region=active_region)
|
||||
await upsert_earth_news_items(db, ranked_fetched_items)
|
||||
|
||||
items = await list_earth_news_items(
|
||||
db,
|
||||
active_region=active_region,
|
||||
limit=MAX_ITEMS_TOTAL,
|
||||
)
|
||||
if hasattr(db, "execute"):
|
||||
cruise_items = await list_earth_news_cruise_items(
|
||||
db,
|
||||
limit=MAX_ITEMS_TOTAL * len(REGION_ANCHORS),
|
||||
)
|
||||
else:
|
||||
cruise_items = items
|
||||
await _enqueue_unverified_locations(items)
|
||||
stale = bool(errors and items)
|
||||
|
||||
return _build_payload(
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
active_region=active_region,
|
||||
items=items,
|
||||
cruise_items=cruise_items,
|
||||
sources=sources,
|
||||
errors=errors,
|
||||
stale=stale,
|
||||
)
|
||||
|
||||
234
backend/app/services/earth_news_queue.py
Normal file
234
backend/app/services/earth_news_queue.py
Normal file
@@ -0,0 +1,234 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
from typing import Any, Protocol
|
||||
|
||||
import redis.asyncio as redis
|
||||
from redis.exceptions import ResponseError
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="earth_news")
|
||||
|
||||
TARGET_LOCATION_STREAM = "earth_news:target_location:jobs"
|
||||
TARGET_LOCATION_GROUP = "earth_news_target_location"
|
||||
TARGET_LOCATION_DEAD_LETTER_STREAM = "earth_news:target_location:dead"
|
||||
TARGET_LOCATION_RESULT_TTL_SECONDS = 60 * 60 * 12
|
||||
TARGET_LOCATION_JOB_DEDUP_TTL_SECONDS = 60 * 60 * 6
|
||||
TARGET_LOCATION_MAX_ATTEMPTS = 3
|
||||
|
||||
_redis_client: redis.Redis | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NewsTargetLocationMessage:
|
||||
message_id: str
|
||||
item_id: str
|
||||
payload: dict[str, Any]
|
||||
attempts: int = 0
|
||||
|
||||
|
||||
class NewsTargetLocationQueue(Protocol):
|
||||
async def enqueue(self, *, item_id: str, payload: dict[str, Any], force: bool = False) -> bool:
|
||||
...
|
||||
|
||||
async def consume_batch(
|
||||
self,
|
||||
*,
|
||||
consumer_name: str,
|
||||
count: int,
|
||||
block_ms: int,
|
||||
) -> list[NewsTargetLocationMessage]:
|
||||
...
|
||||
|
||||
async def ack(self, message_id: str) -> None:
|
||||
...
|
||||
|
||||
async def retry_or_dead_letter(
|
||||
self,
|
||||
message: NewsTargetLocationMessage,
|
||||
*,
|
||||
error: str,
|
||||
) -> None:
|
||||
...
|
||||
|
||||
|
||||
def _get_redis_client() -> redis.Redis:
|
||||
global _redis_client
|
||||
if _redis_client is None:
|
||||
_redis_client = redis.from_url(settings.REDIS_URL, decode_responses=True)
|
||||
return _redis_client
|
||||
|
||||
|
||||
def _result_key(item_id: str) -> str:
|
||||
return f"earth_news:target_location:result:{item_id}"
|
||||
|
||||
|
||||
def _queued_key(item_id: str) -> str:
|
||||
return f"earth_news:target_location:queued:{item_id}"
|
||||
|
||||
|
||||
class RedisStreamsNewsTargetLocationQueue:
|
||||
def __init__(self, client: redis.Redis | None = None) -> None:
|
||||
self.client = client or _get_redis_client()
|
||||
self._group_ready = False
|
||||
|
||||
async def _ensure_group(self) -> None:
|
||||
if self._group_ready:
|
||||
return
|
||||
try:
|
||||
await self.client.xgroup_create(
|
||||
TARGET_LOCATION_STREAM,
|
||||
TARGET_LOCATION_GROUP,
|
||||
id="0",
|
||||
mkstream=True,
|
||||
)
|
||||
except ResponseError as exc:
|
||||
if "BUSYGROUP" not in str(exc):
|
||||
raise
|
||||
self._group_ready = True
|
||||
|
||||
async def enqueue(self, *, item_id: str, payload: dict[str, Any], force: bool = False) -> bool:
|
||||
await self._ensure_group()
|
||||
if force:
|
||||
await self.client.delete(_result_key(item_id), _queued_key(item_id))
|
||||
elif await self.client.exists(_result_key(item_id)):
|
||||
return False
|
||||
queued = await self.client.set(
|
||||
_queued_key(item_id),
|
||||
"1",
|
||||
nx=True,
|
||||
ex=TARGET_LOCATION_JOB_DEDUP_TTL_SECONDS,
|
||||
)
|
||||
if not queued:
|
||||
return bool(await self.client.exists(_queued_key(item_id)))
|
||||
await self.client.xadd(
|
||||
TARGET_LOCATION_STREAM,
|
||||
{
|
||||
"item_id": item_id,
|
||||
"attempts": "0",
|
||||
"payload": json.dumps(payload, ensure_ascii=False),
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
async def consume_batch(
|
||||
self,
|
||||
*,
|
||||
consumer_name: str,
|
||||
count: int,
|
||||
block_ms: int,
|
||||
) -> list[NewsTargetLocationMessage]:
|
||||
await self._ensure_group()
|
||||
streams = await self.client.xreadgroup(
|
||||
TARGET_LOCATION_GROUP,
|
||||
consumer_name,
|
||||
{TARGET_LOCATION_STREAM: ">"},
|
||||
count=count,
|
||||
block=block_ms,
|
||||
)
|
||||
messages: list[NewsTargetLocationMessage] = []
|
||||
for _stream_name, stream_messages in streams:
|
||||
for message_id, fields in stream_messages:
|
||||
raw_payload = fields.get("payload")
|
||||
item_id = fields.get("item_id")
|
||||
if not raw_payload or not item_id:
|
||||
await self.ack(message_id)
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(raw_payload)
|
||||
except json.JSONDecodeError:
|
||||
await self.ack(message_id)
|
||||
continue
|
||||
attempts = int(fields.get("attempts") or 0)
|
||||
messages.append(
|
||||
NewsTargetLocationMessage(
|
||||
message_id=message_id,
|
||||
item_id=item_id,
|
||||
payload=payload,
|
||||
attempts=attempts,
|
||||
)
|
||||
)
|
||||
return messages
|
||||
|
||||
async def ack(self, message_id: str) -> None:
|
||||
await self.client.xack(TARGET_LOCATION_STREAM, TARGET_LOCATION_GROUP, message_id)
|
||||
|
||||
async def retry_or_dead_letter(
|
||||
self,
|
||||
message: NewsTargetLocationMessage,
|
||||
*,
|
||||
error: str,
|
||||
) -> None:
|
||||
await self.ack(message.message_id)
|
||||
if message.attempts + 1 >= TARGET_LOCATION_MAX_ATTEMPTS:
|
||||
await self.client.xadd(
|
||||
TARGET_LOCATION_DEAD_LETTER_STREAM,
|
||||
{
|
||||
"item_id": message.item_id,
|
||||
"attempts": str(message.attempts + 1),
|
||||
"error": error,
|
||||
"payload": json.dumps(message.payload, ensure_ascii=False),
|
||||
},
|
||||
)
|
||||
return
|
||||
await self.client.xadd(
|
||||
TARGET_LOCATION_STREAM,
|
||||
{
|
||||
"item_id": message.item_id,
|
||||
"attempts": str(message.attempts + 1),
|
||||
"payload": json.dumps(message.payload, ensure_ascii=False),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def get_news_target_location_queue() -> NewsTargetLocationQueue:
|
||||
return RedisStreamsNewsTargetLocationQueue()
|
||||
|
||||
|
||||
async def enqueue_target_location_job(payload: dict[str, Any], *, force: bool = False) -> bool:
|
||||
item_id = str(payload.get("id") or "")
|
||||
if not item_id:
|
||||
return False
|
||||
try:
|
||||
queue = get_news_target_location_queue()
|
||||
return await queue.enqueue(item_id=item_id, payload=payload, force=force)
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"Failed to enqueue Earth news target location job",
|
||||
event="earth_news.target_location.enqueue_failed",
|
||||
context={"item_id": item_id, "error": str(exc)},
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
async def get_cached_target_location_patch(item_id: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
raw_value = await _get_redis_client().get(_result_key(item_id))
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"Failed to read Earth news target location cache",
|
||||
event="earth_news.target_location.cache_read_failed",
|
||||
context={"item_id": item_id, "error": str(exc)},
|
||||
)
|
||||
return None
|
||||
if not raw_value:
|
||||
return None
|
||||
try:
|
||||
value = json.loads(raw_value)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
|
||||
async def save_target_location_patch(item_id: str, patch: dict[str, Any]) -> None:
|
||||
client = _get_redis_client()
|
||||
await client.setex(
|
||||
_result_key(item_id),
|
||||
TARGET_LOCATION_RESULT_TTL_SECONDS,
|
||||
json.dumps(patch, ensure_ascii=False),
|
||||
)
|
||||
await client.delete(_queued_key(item_id))
|
||||
262
backend/app/services/earth_news_store.py
Normal file
262
backend/app/services/earth_news_store.py
Normal file
@@ -0,0 +1,262 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.earth_news import EarthNewsItem
|
||||
from app.services.earth_news import (
|
||||
ParsedNewsItem,
|
||||
apply_enrichment_patch_to_item,
|
||||
build_anchor_location_patch,
|
||||
)
|
||||
|
||||
|
||||
def _coerce_datetime(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=UTC)
|
||||
return value.astimezone(UTC)
|
||||
|
||||
|
||||
def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]:
|
||||
return {
|
||||
"latitude": record.latitude,
|
||||
"longitude": record.longitude,
|
||||
"location_label": record.location_label,
|
||||
"location_source": record.location_source,
|
||||
"verified": record.verified,
|
||||
"location_meta": dict(record.location_meta or {}),
|
||||
}
|
||||
|
||||
|
||||
def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
|
||||
item = ParsedNewsItem(
|
||||
id=record.id,
|
||||
title=record.title,
|
||||
summary=record.summary or "",
|
||||
url=record.url,
|
||||
source=record.source or "",
|
||||
feed_name=record.feed_name or "",
|
||||
feed_region=record.region or "global",
|
||||
homepage_url=record.homepage_url or "",
|
||||
published_at=_coerce_datetime(record.published_at),
|
||||
content_language=record.content_language or "en",
|
||||
localizations=dict(record.localizations or {}),
|
||||
enrichment_status=record.enrichment_status or "pending",
|
||||
enrichment_error=record.enrichment_error,
|
||||
enriched_at=_coerce_datetime(record.enriched_at),
|
||||
)
|
||||
return apply_enrichment_patch_to_item(item, _location_patch_from_record(record))
|
||||
|
||||
|
||||
def _query_sort_key(active_region: str):
|
||||
return (
|
||||
EarthNewsItem.region != active_region,
|
||||
EarthNewsItem.published_at.is_(None),
|
||||
EarthNewsItem.published_at.desc().nullslast(),
|
||||
EarthNewsItem.feed_name.asc(),
|
||||
)
|
||||
|
||||
|
||||
async def list_earth_news_items(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
active_region: str,
|
||||
limit: int,
|
||||
) -> list[ParsedNewsItem]:
|
||||
regions = {"global", active_region}
|
||||
result = await db.execute(
|
||||
select(EarthNewsItem)
|
||||
.where(EarthNewsItem.region.in_(regions))
|
||||
.order_by(*_query_sort_key(active_region))
|
||||
.limit(limit)
|
||||
)
|
||||
return [record_to_parsed_news_item(record) for record in result.scalars().all()]
|
||||
|
||||
|
||||
async def list_earth_news_cruise_items(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
limit: int,
|
||||
) -> list[ParsedNewsItem]:
|
||||
result = await db.execute(
|
||||
select(EarthNewsItem)
|
||||
.order_by(
|
||||
EarthNewsItem.region.asc(),
|
||||
EarthNewsItem.published_at.desc().nullslast(),
|
||||
EarthNewsItem.last_seen_at.desc(),
|
||||
EarthNewsItem.feed_name.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
return [record_to_parsed_news_item(record) for record in result.scalars().all()]
|
||||
|
||||
|
||||
async def get_earth_news_freshness(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
active_region: str,
|
||||
) -> tuple[int, datetime | None]:
|
||||
regions = {"global", active_region}
|
||||
result = await db.execute(
|
||||
select(
|
||||
func.count(EarthNewsItem.id),
|
||||
func.max(func.coalesce(EarthNewsItem.published_at, EarthNewsItem.last_seen_at)),
|
||||
).where(EarthNewsItem.region.in_(regions))
|
||||
)
|
||||
count, newest = result.one()
|
||||
item_count = int(count or 0)
|
||||
if item_count == 0:
|
||||
return 0, None
|
||||
return item_count, _coerce_datetime(newest)
|
||||
|
||||
|
||||
async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem]) -> int:
|
||||
if not items:
|
||||
return 0
|
||||
now = datetime.now(UTC)
|
||||
existing_result = await db.execute(
|
||||
select(EarthNewsItem).where(EarthNewsItem.id.in_([item.id for item in items]))
|
||||
)
|
||||
existing = {record.id: record for record in existing_result.scalars().all()}
|
||||
changed = 0
|
||||
for item in items:
|
||||
record = existing.get(item.id)
|
||||
if record is None:
|
||||
patch = build_anchor_location_patch(item)
|
||||
record = EarthNewsItem(
|
||||
id=item.id,
|
||||
title=item.title,
|
||||
summary=item.summary,
|
||||
content_language=item.content_language,
|
||||
localizations=dict(item.localizations or {}),
|
||||
url=item.url,
|
||||
source=item.source,
|
||||
feed_name=item.feed_name,
|
||||
region=item.feed_region,
|
||||
homepage_url=item.homepage_url,
|
||||
published_at=item.published_at,
|
||||
latitude=patch["latitude"],
|
||||
longitude=patch["longitude"],
|
||||
location_label=patch["location_label"],
|
||||
location_source=patch["location_source"],
|
||||
verified=patch["verified"],
|
||||
location_meta=patch["location_meta"],
|
||||
first_seen_at=now,
|
||||
last_seen_at=now,
|
||||
enrichment_status=item.enrichment_status,
|
||||
enrichment_error=item.enrichment_error,
|
||||
enriched_at=item.enriched_at,
|
||||
)
|
||||
db.add(record)
|
||||
changed += 1
|
||||
continue
|
||||
|
||||
record.title = item.title
|
||||
record.summary = item.summary
|
||||
record.url = item.url
|
||||
record.source = item.source
|
||||
record.feed_name = item.feed_name
|
||||
record.region = item.feed_region
|
||||
record.homepage_url = item.homepage_url
|
||||
record.published_at = item.published_at
|
||||
record.last_seen_at = now
|
||||
if item.localizations:
|
||||
record.content_language = item.content_language
|
||||
record.localizations = dict(item.localizations or {})
|
||||
record.enrichment_status = item.enrichment_status
|
||||
record.enrichment_error = item.enrichment_error
|
||||
record.enriched_at = item.enriched_at
|
||||
changed += 1
|
||||
await db.flush()
|
||||
return changed
|
||||
|
||||
|
||||
async def update_earth_news_item_location(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
item_id: str,
|
||||
patch: dict[str, Any],
|
||||
) -> bool:
|
||||
record = await db.get(EarthNewsItem, item_id)
|
||||
if record is None:
|
||||
return False
|
||||
record.latitude = float(patch["latitude"])
|
||||
record.longitude = float(patch["longitude"])
|
||||
record.location_label = str(patch["location_label"])
|
||||
record.location_source = str(patch["location_source"])
|
||||
record.verified = bool(patch["verified"])
|
||||
record.location_meta = dict(patch.get("location_meta") or {})
|
||||
record.resolved_at = datetime.now(UTC) if record.verified else None
|
||||
await db.flush()
|
||||
return True
|
||||
|
||||
|
||||
async def update_earth_news_item_enrichment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
item_id: str,
|
||||
patch: dict[str, Any],
|
||||
) -> bool:
|
||||
record = await db.get(EarthNewsItem, item_id)
|
||||
if record is None:
|
||||
return False
|
||||
if "latitude" in patch:
|
||||
record.latitude = float(patch["latitude"])
|
||||
record.longitude = float(patch["longitude"])
|
||||
record.location_label = str(patch["location_label"])
|
||||
record.location_source = str(patch["location_source"])
|
||||
record.verified = bool(patch["verified"])
|
||||
record.location_meta = dict(patch.get("location_meta") or {})
|
||||
record.resolved_at = datetime.now(UTC) if record.verified else None
|
||||
if "content_language" in patch:
|
||||
record.content_language = str(patch.get("content_language") or "en")
|
||||
if "localizations" in patch:
|
||||
record.localizations = dict(patch.get("localizations") or {})
|
||||
if "enrichment_status" in patch:
|
||||
record.enrichment_status = str(patch.get("enrichment_status") or "pending")
|
||||
if "enrichment_error" in patch:
|
||||
record.enrichment_error = patch.get("enrichment_error")
|
||||
if patch.get("enriched_at"):
|
||||
try:
|
||||
parsed_enriched_at = datetime.fromisoformat(
|
||||
str(patch["enriched_at"]).replace("Z", "+00:00")
|
||||
)
|
||||
except ValueError:
|
||||
parsed_enriched_at = datetime.now(UTC)
|
||||
record.enriched_at = _coerce_datetime(parsed_enriched_at)
|
||||
elif patch.get("localizations"):
|
||||
record.enriched_at = datetime.now(UTC)
|
||||
await db.flush()
|
||||
return True
|
||||
|
||||
|
||||
async def list_unverified_earth_news_items(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
active_region: str,
|
||||
limit: int,
|
||||
) -> list[ParsedNewsItem]:
|
||||
regions = {"global", active_region}
|
||||
result = await db.execute(
|
||||
select(EarthNewsItem)
|
||||
.where(EarthNewsItem.region.in_(regions))
|
||||
.where(EarthNewsItem.verified.is_(False))
|
||||
.order_by(*_query_sort_key(active_region))
|
||||
.limit(limit)
|
||||
)
|
||||
return [record_to_parsed_news_item(record) for record in result.scalars().all()]
|
||||
|
||||
|
||||
async def list_all_earth_news_records(db: AsyncSession) -> list[EarthNewsItem]:
|
||||
result = await db.execute(
|
||||
select(EarthNewsItem).order_by(
|
||||
EarthNewsItem.published_at.desc().nullslast(),
|
||||
EarthNewsItem.last_seen_at.desc(),
|
||||
)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
142
backend/app/services/earth_news_worker.py
Normal file
142
backend/app/services/earth_news_worker.py
Normal file
@@ -0,0 +1,142 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from contextlib import suppress
|
||||
from socket import gethostname
|
||||
from typing import Any
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.db.session import async_session_factory
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.earth_news import (
|
||||
NEWS_ENRICH_PROMPT_KEY,
|
||||
_infer_news_enrichment,
|
||||
build_target_location_patch,
|
||||
parsed_news_item_from_job_payload,
|
||||
)
|
||||
from app.services.earth_news_queue import (
|
||||
NewsTargetLocationMessage,
|
||||
get_news_target_location_queue,
|
||||
save_target_location_patch,
|
||||
)
|
||||
from app.services.earth_news_store import update_earth_news_item_enrichment as update_earth_news_item_location
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="earth_news")
|
||||
|
||||
WORKER_BATCH_SIZE = 4
|
||||
WORKER_BLOCK_MS = 5000
|
||||
WORKER_BACKOFF_SECONDS = 5.0
|
||||
|
||||
_worker_task: asyncio.Task | None = None
|
||||
|
||||
|
||||
async def _build_provider_client() -> AIProviderClient | None:
|
||||
try:
|
||||
from app.api.v1.settings import get_runtime_ai_provider_config
|
||||
|
||||
async with async_session_factory() as session:
|
||||
runtime_config = await get_runtime_ai_provider_config(session)
|
||||
return AIProviderClient(
|
||||
service_url=runtime_config["service_url"],
|
||||
service_token=runtime_config["service_token"],
|
||||
timeout=runtime_config["timeout_seconds"],
|
||||
retry_attempts=runtime_config["retry_attempts"],
|
||||
llm_config=runtime_config.get("llm_config") or {},
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"Failed to build Earth news AI provider client",
|
||||
event="earth_news.target_location.provider_unavailable",
|
||||
context={"error": str(exc)},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def process_target_location_message(
|
||||
message: NewsTargetLocationMessage,
|
||||
*,
|
||||
provider_client: AIProviderClient | None,
|
||||
) -> dict[str, Any]:
|
||||
item = parsed_news_item_from_job_payload(message.payload)
|
||||
async with async_session_factory() as session:
|
||||
prompt = await get_effective_prompt(session, NEWS_ENRICH_PROMPT_KEY)
|
||||
target, localizations = await _infer_news_enrichment(
|
||||
item,
|
||||
provider_client=provider_client,
|
||||
prompt=prompt,
|
||||
)
|
||||
item.target_location = target
|
||||
item.localizations = localizations or item.localizations
|
||||
patch = build_target_location_patch(item, target)
|
||||
await save_target_location_patch(item.id, patch)
|
||||
async with async_session_factory() as session:
|
||||
await update_earth_news_item_location(session, item_id=item.id, patch=patch)
|
||||
await session.commit()
|
||||
await broadcaster.broadcast_custom(
|
||||
"earth_news",
|
||||
{
|
||||
"item_id": item.id,
|
||||
"patch": patch,
|
||||
},
|
||||
)
|
||||
return patch
|
||||
|
||||
|
||||
async def _run_target_location_worker() -> None:
|
||||
consumer_name = f"{gethostname()}:{id(asyncio.current_task())}"
|
||||
queue = get_news_target_location_queue()
|
||||
while True:
|
||||
try:
|
||||
messages = await queue.consume_batch(
|
||||
consumer_name=consumer_name,
|
||||
count=WORKER_BATCH_SIZE,
|
||||
block_ms=WORKER_BLOCK_MS,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"Earth news target location worker queue read failed",
|
||||
event="earth_news.target_location.worker_read_failed",
|
||||
context={"error": str(exc)},
|
||||
)
|
||||
await asyncio.sleep(WORKER_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
if not messages:
|
||||
continue
|
||||
provider_client = await _build_provider_client()
|
||||
for message in messages:
|
||||
try:
|
||||
await process_target_location_message(message, provider_client=provider_client)
|
||||
await queue.ack(message.message_id)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"Earth news target location worker job failed",
|
||||
event="earth_news.target_location.worker_job_failed",
|
||||
context={"item_id": message.item_id, "error": str(exc)},
|
||||
)
|
||||
with suppress(Exception):
|
||||
await queue.retry_or_dead_letter(message, error=str(exc))
|
||||
|
||||
|
||||
def start_earth_news_target_worker() -> None:
|
||||
global _worker_task
|
||||
if _worker_task is None or _worker_task.done():
|
||||
_worker_task = asyncio.create_task(_run_target_location_worker())
|
||||
|
||||
|
||||
async def stop_earth_news_target_worker() -> None:
|
||||
global _worker_task
|
||||
task = _worker_task
|
||||
if task is None:
|
||||
return
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
_worker_task = None
|
||||
@@ -7,6 +7,26 @@ from typing import Any
|
||||
import httpx
|
||||
|
||||
MODELS_DEV_URL = "https://models.dev/api.json"
|
||||
OPENCODE_GO_MODELS_URL = "https://opencode.ai/zen/go/v1/models"
|
||||
|
||||
OPENCODE_GO_MODEL_PROVIDER_APIS = {
|
||||
"minimax-m2.7": "anthropic-messages",
|
||||
"minimax-m2.5": "anthropic-messages",
|
||||
}
|
||||
OPENCODE_GO_FALLBACK_MODELS = [
|
||||
"minimax-m2.7",
|
||||
"minimax-m2.5",
|
||||
"kimi-k2.6",
|
||||
"kimi-k2.5",
|
||||
"glm-5.1",
|
||||
"glm-5",
|
||||
"deepseek-v4-pro",
|
||||
"deepseek-v4-flash",
|
||||
"qwen3.6-plus",
|
||||
"qwen3.5-plus",
|
||||
"mimo-v2.5-pro",
|
||||
"mimo-v2.5",
|
||||
]
|
||||
|
||||
|
||||
FALLBACK_LLM_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
|
||||
@@ -80,6 +100,17 @@ FALLBACK_LLM_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
|
||||
"api_key_env": "OPENROUTER_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"opencode-go": {
|
||||
"provider": "opencode-go",
|
||||
"label": "OpenCode Go",
|
||||
"provider_api": "openai-completions",
|
||||
"base_url": "https://opencode.ai/zen/go/v1",
|
||||
"model": "glm-5.1",
|
||||
"models": OPENCODE_GO_FALLBACK_MODELS,
|
||||
"model_provider_apis": OPENCODE_GO_MODEL_PROVIDER_APIS,
|
||||
"api_key_env": "OPENCODE_GO_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"ollama": {
|
||||
"provider": "ollama",
|
||||
"label": "Ollama Local",
|
||||
@@ -114,8 +145,43 @@ def get_fallback_llm_provider_preset(provider: str) -> dict[str, Any]:
|
||||
return dict(FALLBACK_LLM_PROVIDER_PRESETS[key])
|
||||
|
||||
|
||||
async def refresh_llm_provider_preset(provider: str) -> dict[str, Any]:
|
||||
def _opencode_go_model_provider_apis(model_ids: list[str]) -> dict[str, str]:
|
||||
return {
|
||||
model_id: OPENCODE_GO_MODEL_PROVIDER_APIS.get(model_id, "openai-completions")
|
||||
for model_id in model_ids
|
||||
}
|
||||
|
||||
|
||||
async def refresh_llm_provider_preset(provider: str, api_key: str | None = None) -> dict[str, Any]:
|
||||
fallback = get_fallback_llm_provider_preset(provider)
|
||||
if fallback["provider"] == "opencode-go":
|
||||
headers = {"User-Agent": "Planet/1.0"}
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
|
||||
response = await client.get(
|
||||
OPENCODE_GO_MODELS_URL,
|
||||
headers=headers,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
|
||||
data = payload.get("data") if isinstance(payload, dict) else []
|
||||
model_ids = [
|
||||
str(item.get("id"))
|
||||
for item in data
|
||||
if isinstance(item, dict) and item.get("id")
|
||||
][:120]
|
||||
if not model_ids:
|
||||
model_ids = fallback["models"]
|
||||
return {
|
||||
**fallback,
|
||||
"model": fallback["model"] if fallback["model"] in model_ids else model_ids[0],
|
||||
"models": model_ids,
|
||||
"model_provider_apis": _opencode_go_model_provider_apis(model_ids),
|
||||
"source": OPENCODE_GO_MODELS_URL,
|
||||
}
|
||||
|
||||
models_dev_key = MODELS_DEV_PROVIDER_KEYS.get(fallback["provider"])
|
||||
if not models_dev_key:
|
||||
return fallback
|
||||
|
||||
@@ -7,8 +7,12 @@ import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.countries import COUNTRY_ENTRIES, normalize_country
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.services.ai_tools.evidence_store import normalize_search_evidence
|
||||
from app.services.ai_tools.web_search import WebSearchClient, WebSearchError
|
||||
@@ -23,7 +27,12 @@ from app.services.location.text import (
|
||||
|
||||
VALID_LLM_PRECISIONS = {"precise", "site", "city"}
|
||||
DEFAULT_MIN_CONFIDENCE = 0.55
|
||||
LOCATION_NORMALIZE_PROMPT_KEY = "location.factcheck.normalize"
|
||||
LOCATION_RESOLVE_PROMPT_KEY = "location.factcheck.resolve"
|
||||
MODEL_CONFIDENCE_WEIGHT = 0.25
|
||||
LOG_TEXT_LIMIT = 1200
|
||||
LOG_EVIDENCE_LIMIT = 5
|
||||
logger = get_logger(__name__, service="location")
|
||||
_geocode_llm_city = build_default_nominatim_geocoder()
|
||||
_LLM_LOCATION_NAME_KEYS = (
|
||||
"matched_location_name",
|
||||
@@ -92,6 +101,35 @@ class LocationEvidenceScore:
|
||||
summary: str
|
||||
|
||||
|
||||
def _truncate_log_text(value: Any, limit: int = LOG_TEXT_LIMIT) -> str:
|
||||
text = coerce_str(value)
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return f"{text[:limit]}…"
|
||||
|
||||
|
||||
def _summarize_search_evidence(evidence: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
for item in (evidence or [])[:LOG_EVIDENCE_LIMIT]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"title": _truncate_log_text(item.get("title"), 180),
|
||||
"source": _truncate_log_text(item.get("source") or item.get("name"), 120),
|
||||
"url": _truncate_log_text(item.get("url"), 240),
|
||||
"snippet": _truncate_log_text(
|
||||
item.get("snippet")
|
||||
or item.get("content")
|
||||
or item.get("text")
|
||||
or item.get("summary"),
|
||||
360,
|
||||
),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def _first_json_object(text: str) -> dict[str, Any] | None:
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
@@ -157,6 +195,64 @@ def _evidence_label(item: Any) -> str:
|
||||
return coerce_str(item)
|
||||
|
||||
|
||||
def _evidence_text(item: dict[str, Any]) -> str:
|
||||
return " ".join(
|
||||
coerce_str(item.get(key))
|
||||
for key in ("title", "source", "name", "url", "snippet", "content", "text", "quote", "summary")
|
||||
if coerce_str(item.get(key))
|
||||
)
|
||||
|
||||
|
||||
def _search_evidence_entity_match(item: dict[str, Any], query: LocationQuery) -> bool:
|
||||
haystack = normalize_text(_evidence_text(item))
|
||||
if not haystack:
|
||||
return False
|
||||
needles = [
|
||||
coerce_str(query.name),
|
||||
*[coerce_str(alias) for alias in query.aliases],
|
||||
]
|
||||
return any(normalize_text(needle) and normalize_text(needle) in haystack for needle in needles)
|
||||
|
||||
|
||||
def _evidence_has_location_assertion(item: dict[str, Any], city: str) -> bool:
|
||||
normalized_city = normalize_text(city)
|
||||
text = normalize_text(_evidence_text(item))
|
||||
if not normalized_city or normalized_city not in text:
|
||||
return False
|
||||
assertion_terms = (
|
||||
"located",
|
||||
"situated",
|
||||
"built",
|
||||
"hosted",
|
||||
"deployed",
|
||||
"installed",
|
||||
"facility",
|
||||
"campus",
|
||||
"site",
|
||||
"data center",
|
||||
"datacenter",
|
||||
"supercomputer center",
|
||||
"位于",
|
||||
"位於",
|
||||
"坐落",
|
||||
"建置",
|
||||
"設置",
|
||||
"设置",
|
||||
)
|
||||
return any(term in text for term in assertion_terms)
|
||||
|
||||
|
||||
def _city_is_unsupported_name_hint(payload: dict[str, Any], query: LocationQuery, evidence_items: list[dict[str, Any]]) -> bool:
|
||||
city = coerce_str(payload.get("city") or query.city)
|
||||
if not city:
|
||||
return False
|
||||
normalized_city = normalize_text(city)
|
||||
normalized_name = normalize_text(query.name)
|
||||
if not normalized_city or not normalized_name or normalized_city not in normalized_name:
|
||||
return False
|
||||
return not any(_evidence_has_location_assertion(item, city) for item in evidence_items)
|
||||
|
||||
|
||||
def _normalize_llm_precision(value: Any) -> str:
|
||||
text = coerce_str(value).lower()
|
||||
return LLM_PRECISION_ALIASES.get(text, text)
|
||||
@@ -583,6 +679,7 @@ def _weak_evidence_penalty(
|
||||
payload: dict[str, Any],
|
||||
evidence_items: list[dict[str, Any]],
|
||||
*,
|
||||
query: LocationQuery,
|
||||
entity_match: float,
|
||||
geography_match: float,
|
||||
conflict_penalty: float,
|
||||
@@ -593,6 +690,8 @@ def _weak_evidence_penalty(
|
||||
penalty += 0.20
|
||||
if any(_truthy_evidence_field(item, "ambiguous") for item in evidence_items):
|
||||
penalty += 0.15
|
||||
if _city_is_unsupported_name_hint(payload, query, evidence_items):
|
||||
penalty += 0.10
|
||||
if conflict_penalty == 0.0 and entity_match > 0 and geography_match >= 0.20:
|
||||
return min(penalty, 0.15)
|
||||
return min(penalty, 0.30)
|
||||
@@ -615,6 +714,7 @@ def _score_llm_location_payload(
|
||||
weak_evidence_penalty = _weak_evidence_penalty(
|
||||
payload,
|
||||
evidence_items,
|
||||
query=query,
|
||||
entity_match=entity_match,
|
||||
geography_match=geography_match,
|
||||
conflict_penalty=conflict_penalty,
|
||||
@@ -631,6 +731,8 @@ def _score_llm_location_payload(
|
||||
- weak_evidence_penalty
|
||||
)
|
||||
score = min(max(score, 0.0), 1.0)
|
||||
if _city_is_unsupported_name_hint(payload, query, evidence_items):
|
||||
score = min(score, 0.54)
|
||||
summary = (
|
||||
f"combined={score:.2f}; model={model_confidence:.2f}; "
|
||||
f"source={source_quality:.2f}; entity={entity_match:.2f}; "
|
||||
@@ -842,15 +944,43 @@ async def collect_location_search_evidence(
|
||||
) -> LocationSearchEvidenceResult:
|
||||
search_query = _location_search_query(query, entity_type)
|
||||
attempt = f"web_search:{entity_type}:{search_query}"
|
||||
logger.info_event(
|
||||
"Collecting location search evidence",
|
||||
event="location.factcheck.web_search.start",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"search_query": search_query,
|
||||
"location_query": _query_context(query),
|
||||
"max_results": max_results,
|
||||
},
|
||||
)
|
||||
try:
|
||||
evidence = await web_search_client.search(search_query, max_results=max_results)
|
||||
except WebSearchError as exc:
|
||||
logger.warning_event(
|
||||
"Location search evidence failed",
|
||||
event="location.factcheck.web_search.failed",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"search_query": search_query,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
return LocationSearchEvidenceResult(
|
||||
evidence=[],
|
||||
attempted_queries=[attempt],
|
||||
failure_reason=f"WebSearch location evidence failed: {exc}",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"Location search evidence unavailable",
|
||||
event="location.factcheck.web_search.unavailable",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"search_query": search_query,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
return LocationSearchEvidenceResult(
|
||||
evidence=[],
|
||||
attempted_queries=[attempt],
|
||||
@@ -858,11 +988,29 @@ async def collect_location_search_evidence(
|
||||
)
|
||||
normalized = normalize_search_evidence(evidence, limit=max_results)
|
||||
if not normalized:
|
||||
logger.warning_event(
|
||||
"Location search returned no usable evidence",
|
||||
event="location.factcheck.web_search.empty",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"search_query": search_query,
|
||||
},
|
||||
)
|
||||
return LocationSearchEvidenceResult(
|
||||
evidence=[],
|
||||
attempted_queries=[attempt],
|
||||
failure_reason="WebSearch returned no usable location evidence.",
|
||||
)
|
||||
logger.info_event(
|
||||
"Collected location search evidence",
|
||||
event="location.factcheck.web_search.result",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"search_query": search_query,
|
||||
"evidence_count": len(normalized),
|
||||
"evidence": _summarize_search_evidence(normalized),
|
||||
},
|
||||
)
|
||||
return LocationSearchEvidenceResult(
|
||||
evidence=normalized,
|
||||
attempted_queries=[attempt],
|
||||
@@ -876,6 +1024,7 @@ async def _repair_location_payload_from_text(
|
||||
raw_text: str,
|
||||
query: LocationQuery,
|
||||
entity_type: str,
|
||||
db: AsyncSession | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Second-pass structure repair for models that answer in prose.
|
||||
|
||||
@@ -884,12 +1033,11 @@ async def _repair_location_payload_from_text(
|
||||
"""
|
||||
if not coerce_str(raw_text):
|
||||
return None
|
||||
prompt = await get_effective_prompt(db, LOCATION_NORMALIZE_PROMPT_KEY)
|
||||
request = SituationalAnalysisRequest(
|
||||
title=f"Normalize location factcheck for {entity_type}",
|
||||
objective=(
|
||||
"Convert the supplied location factcheck text into exactly one strict "
|
||||
"JSON object. Extract only facts present in the text or original query."
|
||||
),
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"location_query": _query_context(query),
|
||||
@@ -929,6 +1077,7 @@ async def collect_llm_location_fallback_candidate(
|
||||
provider_client: AIProviderClient,
|
||||
query: LocationQuery,
|
||||
entity_type: str,
|
||||
db: AsyncSession | None = None,
|
||||
attempted_queries: Iterable[str] = (),
|
||||
search_evidence: list[dict[str, Any]] | None = None,
|
||||
min_confidence: float = DEFAULT_MIN_CONFIDENCE,
|
||||
@@ -941,18 +1090,25 @@ async def collect_llm_location_fallback_candidate(
|
||||
"""
|
||||
attempt = f"llm_factcheck:{entity_type}:{coerce_str(query.name) or 'unknown'}"
|
||||
if search_evidence is not None and not search_evidence:
|
||||
logger.warning_event(
|
||||
"Skipping LLM location factcheck because search evidence is empty",
|
||||
event="location.factcheck.llm.skipped_no_evidence",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"location_query": _query_context(query),
|
||||
},
|
||||
)
|
||||
return LocationLLMFallbackResult(
|
||||
candidates=[],
|
||||
attempted_queries=[attempt],
|
||||
failure_reason="LLM location factcheck skipped: no WebSearch evidence.",
|
||||
)
|
||||
prompt = await get_effective_prompt(db, LOCATION_RESOLVE_PROMPT_KEY)
|
||||
request = SituationalAnalysisRequest(
|
||||
title=f"Location factcheck fallback for {entity_type}",
|
||||
objective=(
|
||||
"Return exactly one JSON object for the most likely physical location. "
|
||||
"Use only fact-checkable public knowledge; return null fields rather "
|
||||
"than guessing when evidence is weak."
|
||||
),
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"location_query": _query_context(query),
|
||||
@@ -982,31 +1138,86 @@ async def collect_llm_location_fallback_candidate(
|
||||
"Return evidence as objects when possible, including source, url, source_type, and entity_match.",
|
||||
"Include source names or URLs in evidence when known. The backend will recompute the final confidence from model confidence plus evidence quality.",
|
||||
"If search_evidence is provided, use only that evidence as factual support.",
|
||||
"Do not treat a website footer, office address, publisher address, or contact address as the entity's physical location.",
|
||||
"If the entity name contains a city name, do not choose that city unless evidence explicitly says the entity/facility/supercomputer is located, hosted, built, deployed, or installed there.",
|
||||
"Prefer the facility/site if known; otherwise use the best supported city.",
|
||||
],
|
||||
)
|
||||
logger.info_event(
|
||||
"Sending location factcheck request to LLM",
|
||||
event="location.factcheck.llm.request",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"title": request.title,
|
||||
"objective": request.objective,
|
||||
"location_query": request.context.get("location_query"),
|
||||
"observations": request.observations,
|
||||
"constraints": request.constraints,
|
||||
"search_evidence_count": len(search_evidence or []),
|
||||
"search_evidence": _summarize_search_evidence(search_evidence),
|
||||
},
|
||||
)
|
||||
try:
|
||||
response = await provider_client.analyze(request)
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"LLM location factcheck failed",
|
||||
event="location.factcheck.llm.failed",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
return LocationLLMFallbackResult(
|
||||
candidates=[],
|
||||
attempted_queries=[attempt],
|
||||
failure_reason=f"LLM location factcheck failed: {exc}",
|
||||
)
|
||||
|
||||
logger.info_event(
|
||||
"Received location factcheck response from LLM",
|
||||
event="location.factcheck.llm.response",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"provider": response.provider,
|
||||
"model": response.model,
|
||||
"content": _truncate_log_text(response.content, 2000),
|
||||
},
|
||||
)
|
||||
payload = _first_json_object(response.content)
|
||||
if payload is None:
|
||||
logger.warning_event(
|
||||
"LLM location factcheck response was not strict JSON; attempting repair",
|
||||
event="location.factcheck.llm.non_json",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"content": _truncate_log_text(response.content, 1200),
|
||||
},
|
||||
)
|
||||
payload = await _repair_location_payload_from_text(
|
||||
provider_client=provider_client,
|
||||
raw_text=response.content,
|
||||
query=query,
|
||||
entity_type=entity_type,
|
||||
db=db,
|
||||
)
|
||||
if payload is None:
|
||||
payload = _payload_from_free_text(response.content, query=query)
|
||||
if payload is None:
|
||||
if payload is None and entity_type != "compute_center":
|
||||
payload = _payload_from_query_name_geocode(query)
|
||||
if payload is None:
|
||||
logger.warning_event(
|
||||
"LLM location factcheck produced no parseable payload",
|
||||
event="location.factcheck.llm.unparseable",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
},
|
||||
)
|
||||
return LocationLLMFallbackResult(
|
||||
candidates=[],
|
||||
attempted_queries=[attempt],
|
||||
@@ -1027,7 +1238,7 @@ async def collect_llm_location_fallback_candidate(
|
||||
"url": item.get("url"),
|
||||
"text": item.get("snippet") or item.get("content"),
|
||||
"source_type": "web_search",
|
||||
"entity_match": True,
|
||||
"entity_match": _search_evidence_entity_match(item, query),
|
||||
}
|
||||
for item in search_evidence
|
||||
if isinstance(item, dict)
|
||||
@@ -1049,6 +1260,18 @@ async def collect_llm_location_fallback_candidate(
|
||||
if candidate is None:
|
||||
if city_geocode_failure and rejection_reason == "missing, invalid, or zero latitude/longitude":
|
||||
rejection_reason = f"{rejection_reason}; {city_geocode_failure}"
|
||||
logger.warning_event(
|
||||
"Rejected LLM location factcheck candidate",
|
||||
event="location.factcheck.llm.rejected",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"reason": rejection_reason,
|
||||
"payload": payload,
|
||||
"search_evidence_count": len(search_evidence or []),
|
||||
"search_evidence": _summarize_search_evidence(search_evidence),
|
||||
},
|
||||
)
|
||||
return LocationLLMFallbackResult(
|
||||
candidates=[],
|
||||
attempted_queries=[attempt],
|
||||
@@ -1057,6 +1280,18 @@ async def collect_llm_location_fallback_candidate(
|
||||
+ (f": {rejection_reason}." if rejection_reason else ".")
|
||||
),
|
||||
)
|
||||
logger.info_event(
|
||||
"Accepted LLM location factcheck candidate",
|
||||
event="location.factcheck.llm.accepted",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"candidate": candidate.to_dict(),
|
||||
"payload": payload,
|
||||
"search_evidence_count": len(search_evidence or []),
|
||||
"search_evidence": _summarize_search_evidence(search_evidence),
|
||||
},
|
||||
)
|
||||
return LocationLLMFallbackResult(
|
||||
candidates=[candidate],
|
||||
attempted_queries=[attempt],
|
||||
|
||||
@@ -10,6 +10,7 @@ from fastapi import HTTPException, status
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import async_session_factory
|
||||
from app.models.playground_message import PlaygroundMessage
|
||||
from app.models.playground_session import PlaygroundSession
|
||||
@@ -27,9 +28,11 @@ from app.schemas.ai import (
|
||||
SituationalAnalysisRequest,
|
||||
)
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
from app.services.playground_session_store import _to_response as session_to_response
|
||||
from app.services.playground_session_store import upsert_playground_session
|
||||
|
||||
logger = get_logger(__name__, service="ai")
|
||||
STREAM_CHUNK_SIZE = 24
|
||||
STREAM_INTERVAL_SECONDS = 0.08
|
||||
THINKING_PREVIEW_SECONDS = 2.6
|
||||
@@ -624,7 +627,42 @@ async def _run_assistant_message(
|
||||
thinking={"type": "enabled"},
|
||||
)
|
||||
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.run.start",
|
||||
message="Playground AI run started",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=user_id,
|
||||
context={
|
||||
"session_id": session_id,
|
||||
"session_key": session_key,
|
||||
"user_message_id": user_message_id,
|
||||
"assistant_message_id": assistant_message_id,
|
||||
"preset": payload.selected_preset_key,
|
||||
},
|
||||
)
|
||||
analysis = await provider_client.analyze(request_payload, request_id=request_id)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.run.success",
|
||||
message="Playground AI run completed",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=user_id,
|
||||
context={
|
||||
"session_id": session_id,
|
||||
"session_key": session_key,
|
||||
"provider": analysis.provider,
|
||||
"model": analysis.model,
|
||||
"content_block_count": len(analysis.content_blocks or []),
|
||||
"thinking_block_count": len(analysis.thinking_blocks or []),
|
||||
},
|
||||
)
|
||||
|
||||
async with async_session_factory() as db:
|
||||
assistant_message = await _mark_message_state(
|
||||
@@ -704,6 +742,23 @@ async def _run_assistant_message(
|
||||
await db.flush()
|
||||
await db.commit()
|
||||
except asyncio.CancelledError:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.run.cancelled",
|
||||
message="Playground AI run cancelled",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=user_id,
|
||||
context={
|
||||
"session_id": session_id,
|
||||
"session_key": session_key,
|
||||
"assistant_message_id": assistant_message_id,
|
||||
"duration_ms": round((perf_counter() - started_at) * 1000),
|
||||
},
|
||||
)
|
||||
async with async_session_factory() as db:
|
||||
result = await db.execute(select(PlaygroundMessage).where(PlaygroundMessage.id == assistant_message_id))
|
||||
message = result.scalar_one_or_none()
|
||||
@@ -715,6 +770,26 @@ async def _run_assistant_message(
|
||||
await db.commit()
|
||||
raise
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.run.failed",
|
||||
message="Playground AI run failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=user_id,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"session_id": session_id,
|
||||
"session_key": session_key,
|
||||
"assistant_message_id": assistant_message_id,
|
||||
"duration_ms": round((perf_counter() - started_at) * 1000),
|
||||
},
|
||||
),
|
||||
)
|
||||
error_message = _format_run_exception(exc)
|
||||
async with async_session_factory() as db:
|
||||
result = await db.execute(select(PlaygroundMessage).where(PlaygroundMessage.id == assistant_message_id))
|
||||
|
||||
@@ -13,6 +13,7 @@ from app.db.session import async_session_factory
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.task import CollectionTask
|
||||
from app.services.business_logs import emit_business_log, emit_business_log_background, exception_context
|
||||
from app.services.collectors.registry import collector_registry
|
||||
from app.services.datasource_connectivity import (
|
||||
build_builtin_connectivity_checksum,
|
||||
@@ -124,6 +125,15 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.skipped_disabled",
|
||||
context={"collector_name": collector_name},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.skipped_disabled",
|
||||
message="Skipping disabled collector",
|
||||
category="collector",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={"collector_name": collector_name, "datasource_id": datasource.id, "status": "skipped"},
|
||||
)
|
||||
return
|
||||
|
||||
running_result = await db.execute(
|
||||
@@ -152,6 +162,21 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.skipped_already_running",
|
||||
context={"collector_name": collector_name, "task_id": existing_running.id},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.skipped_already_running",
|
||||
message="Skipping collector trigger because task is already running",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": collector_name,
|
||||
"datasource_id": datasource.id,
|
||||
"task_id": existing_running.id,
|
||||
"status": "skipped",
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
existing_error = (existing_running.error_message or "").strip()
|
||||
@@ -173,21 +198,55 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.stale_task_failed",
|
||||
context={"collector_name": collector_name, "task_id": existing_running.id},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.stale_task_failed",
|
||||
message="Marked stale running task as failed before rerun",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": collector_name,
|
||||
"datasource_id": datasource.id,
|
||||
"task_id": existing_running.id,
|
||||
"status": "failed",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
collector._datasource_id = datasource.id
|
||||
datasource_id = datasource.id
|
||||
datasource_source = datasource.source
|
||||
collector._datasource_id = datasource_id
|
||||
logger.info_event(
|
||||
"Running collector",
|
||||
event="collector.run.started",
|
||||
context={"collector_name": collector_name, "datasource_id": datasource.id},
|
||||
context={"collector_name": collector_name, "datasource_id": datasource_id},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.scheduled_started",
|
||||
message="Scheduler started collector run",
|
||||
category="collector",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={"collector_name": collector_name, "datasource_id": datasource_id, "status": "running"},
|
||||
)
|
||||
task_result = await collector.run(db)
|
||||
datasource = await db.get(DataSource, datasource_id)
|
||||
if datasource is None:
|
||||
logger.error_event(
|
||||
"Datasource disappeared after collector run",
|
||||
event="collector.run.datasource_missing_after_run",
|
||||
context={"collector_name": collector_name, "datasource_id": datasource_id},
|
||||
)
|
||||
return
|
||||
datasource.last_run_at = datetime.now(UTC)
|
||||
datasource.last_status = task_result.get("status")
|
||||
if datasource.last_status == "success":
|
||||
effective_candidate = await get_builtin_effective_candidate(db, datasource.source)
|
||||
effective_candidate = await get_builtin_effective_candidate(db, datasource_source)
|
||||
checksum, _credential_context = await build_builtin_connectivity_checksum(
|
||||
datasource.source,
|
||||
datasource_source,
|
||||
effective_candidate["endpoint"],
|
||||
effective_candidate["auth_type"],
|
||||
effective_candidate["headers"],
|
||||
@@ -196,7 +255,7 @@ async def run_collector_task(collector_name: str):
|
||||
)
|
||||
await save_connectivity_success(
|
||||
db,
|
||||
datasource.source,
|
||||
datasource_source,
|
||||
checksum,
|
||||
{"status_code": None},
|
||||
connected_by="collection",
|
||||
@@ -205,9 +264,25 @@ async def run_collector_task(collector_name: str):
|
||||
logger.info_event(
|
||||
"Collector completed",
|
||||
event="collector.run.completed",
|
||||
context={"collector_name": collector_name, "datasource_id": datasource.id, "result": task_result},
|
||||
context={"collector_name": collector_name, "datasource_id": datasource_id, "result": task_result},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.scheduled_completed",
|
||||
message="Scheduler completed collector run",
|
||||
category="collector",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": collector_name,
|
||||
"datasource_id": datasource_id,
|
||||
"status": task_result.get("status"),
|
||||
"result": task_result,
|
||||
},
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
await db.rollback()
|
||||
datasource = await db.get(DataSource, datasource_id)
|
||||
datasource.last_run_at = datetime.now(UTC)
|
||||
datasource.last_status = "cancelled"
|
||||
await db.commit()
|
||||
@@ -216,8 +291,20 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.cancelled",
|
||||
context={"collector_name": collector_name, "datasource_id": datasource.id},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.cancelled",
|
||||
message="Collector cancelled by operator",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={"collector_name": collector_name, "datasource_id": datasource.id, "status": "cancelled"},
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
await db.rollback()
|
||||
datasource = await db.get(DataSource, datasource_id)
|
||||
datasource.last_run_at = datetime.now(UTC)
|
||||
datasource.last_status = "failed"
|
||||
await db.commit()
|
||||
@@ -226,6 +313,19 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.failed",
|
||||
context={"collector_name": collector_name, "datasource_id": datasource.id, "error": str(exc)},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.failed",
|
||||
message="Collector failed",
|
||||
category="collector",
|
||||
level="error",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{"collector_name": collector_name, "datasource_id": datasource.id, "status": "failed"},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def cleanup_stale_running_tasks(max_age_hours: int = 2) -> int:
|
||||
@@ -347,6 +447,16 @@ def run_collector_now(collector_name: str) -> bool:
|
||||
event="collector.trigger.skipped_already_running",
|
||||
context={"collector_name": collector_name},
|
||||
)
|
||||
emit_business_log_background(
|
||||
logger,
|
||||
event="collector.trigger.skipped_already_running",
|
||||
message="Collector is already running in-memory; skipping duplicate trigger",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={"collector_name": collector_name, "status": "skipped"},
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
@@ -364,6 +474,15 @@ def run_collector_now(collector_name: str) -> bool:
|
||||
event="collector.trigger.started",
|
||||
context={"collector_name": collector_name},
|
||||
)
|
||||
emit_business_log_background(
|
||||
logger,
|
||||
event="collector.trigger.started",
|
||||
message="Triggered collector",
|
||||
category="collector",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={"collector_name": collector_name, "status": "queued"},
|
||||
)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error_event(
|
||||
@@ -371,6 +490,16 @@ def run_collector_now(collector_name: str) -> bool:
|
||||
event="collector.trigger.failed",
|
||||
context={"collector_name": collector_name, "error": str(exc)},
|
||||
)
|
||||
emit_business_log_background(
|
||||
logger,
|
||||
event="collector.trigger.failed",
|
||||
message="Failed to trigger collector",
|
||||
category="collector",
|
||||
level="error",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context=exception_context(exc, {"collector_name": collector_name, "status": "failed"}),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
|
||||
@@ -10,8 +10,11 @@ from app.models.alert import Alert, AlertSeverity, AlertStatus
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.bgp_ai_brief_store import get_latest_bgp_brief_record
|
||||
|
||||
SITUATIONAL_ALERT_BRIEF_PROMPT_KEY = "alerts.situational.brief"
|
||||
|
||||
|
||||
def _format_pairs(pairs: list[tuple[str, int]], empty_text: str = "无") -> str:
|
||||
if not pairs:
|
||||
@@ -96,18 +99,24 @@ async def build_situational_alert_brief_request(
|
||||
(str(item[0] or "未命名数据源"), item[1])
|
||||
for item in alert_source_result.fetchall()
|
||||
]
|
||||
total_alerts = total_alerts_result.scalar() or 0
|
||||
active_alerts = active_alerts_result.scalar() or 0
|
||||
total_incidents = total_incidents_result.scalar() or 0
|
||||
active_incidents = active_incidents_result.scalar() or 0
|
||||
total_anomalies = total_anomalies_result.scalar() or 0
|
||||
active_anomalies = active_anomalies_result.scalar() or 0
|
||||
|
||||
facts = [
|
||||
(
|
||||
f"系统告警侧:总告警 {total_alerts_result.scalar() or 0} 条,active {active_alerts_result.scalar() or 0} 条;"
|
||||
f"系统告警侧:总告警 {total_alerts} 条,active {active_alerts} 条;"
|
||||
f"活跃告警严重度分布为 {_format_pairs(active_alert_severities)}。"
|
||||
),
|
||||
(
|
||||
f"BGP态势侧:累计 incidents {total_incidents_result.scalar() or 0} 条,active incidents {active_incidents_result.scalar() or 0} 条;"
|
||||
f"BGP态势侧:累计 incidents {total_incidents} 条,active incidents {active_incidents} 条;"
|
||||
f"活跃 incidents 严重度分布为 {_format_pairs(active_bgp_severities)}。"
|
||||
),
|
||||
(
|
||||
f"BGP异常侧:累计 anomalies {total_anomalies_result.scalar() or 0} 条,active anomalies {active_anomalies_result.scalar() or 0} 条;"
|
||||
f"BGP异常侧:累计 anomalies {total_anomalies} 条,active anomalies {active_anomalies} 条;"
|
||||
f"活跃 anomaly 类型分布为 {_format_pairs(active_anomaly_types)}。"
|
||||
),
|
||||
]
|
||||
@@ -147,21 +156,23 @@ async def build_situational_alert_brief_request(
|
||||
|
||||
context = {
|
||||
"source": "situational-alerts",
|
||||
"active_system_alerts": active_alerts_result.scalar() or 0,
|
||||
"active_system_alerts": active_alerts,
|
||||
"active_system_alert_severities": dict(active_alert_severities),
|
||||
"top_system_alert_sources": dict(active_alert_sources),
|
||||
"active_bgp_incidents": active_incidents_result.scalar() or 0,
|
||||
"active_bgp_incidents": active_incidents,
|
||||
"active_bgp_incident_severities": dict(active_bgp_severities),
|
||||
"active_bgp_anomalies": active_anomalies_result.scalar() or 0,
|
||||
"active_bgp_anomalies": active_anomalies,
|
||||
"active_bgp_anomaly_types": dict(active_anomaly_types),
|
||||
"bgp_hot_regions": dict(bgp_region_counter.most_common(5)),
|
||||
"latest_bgp_brief_id": latest_bgp_brief.id if latest_bgp_brief else None,
|
||||
"latest_bgp_brief_generated_at": latest_bgp_brief.generated_at if latest_bgp_brief else None,
|
||||
}
|
||||
prompt = await get_effective_prompt(db, SITUATIONAL_ALERT_BRIEF_PROMPT_KEY)
|
||||
|
||||
request = SituationalAnalysisRequest(
|
||||
title="态势告警 AI 简报",
|
||||
objective="综合系统告警、BGP incidents、BGP anomalies 与近期 BGP AI 简报,生成一份面向值班人员的态势告警简报,指出当前最需要关注的风险域、跨模块联动迹象和优先动作。",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=facts,
|
||||
constraints=[
|
||||
"明确区分事实、推断与建议。",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -86,6 +87,7 @@ class LogSource:
|
||||
status: str = "ok"
|
||||
buffer_key: str | None = None
|
||||
container_name: str | None = None
|
||||
fallback_locations: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -104,22 +106,38 @@ class DailyLogMarker:
|
||||
dominant_level: str
|
||||
|
||||
|
||||
def _planet_state_dir() -> Path:
|
||||
configured = os.getenv("PLANET_STATE_DIR")
|
||||
if configured:
|
||||
return Path(configured).expanduser()
|
||||
xdg_state = os.getenv("XDG_STATE_HOME")
|
||||
if xdg_state:
|
||||
return Path(xdg_state).expanduser() / "planet"
|
||||
return Path.home() / ".local" / "state" / "planet"
|
||||
|
||||
|
||||
def _state_log_path(filename: str) -> str:
|
||||
return str(_planet_state_dir() / filename)
|
||||
|
||||
|
||||
LOG_SOURCES: dict[str, LogSource] = {
|
||||
"backend": LogSource(
|
||||
source_id="backend",
|
||||
name="后端服务",
|
||||
kind="file",
|
||||
location="/tmp/planet_backend.log",
|
||||
location=_state_log_path("backend.log"),
|
||||
description="FastAPI 后端、调度器和采集任务共享日志。",
|
||||
category="service",
|
||||
fallback_locations=("/tmp/planet_backend.log",),
|
||||
),
|
||||
"frontend": LogSource(
|
||||
source_id="frontend",
|
||||
name="前端开发服务",
|
||||
kind="file",
|
||||
location="/tmp/planet_frontend.log",
|
||||
location=_state_log_path("frontend.log"),
|
||||
description="控制台与 Earth 前端开发服务输出。",
|
||||
category="service",
|
||||
fallback_locations=("/tmp/planet_frontend.log",),
|
||||
),
|
||||
"ai-provider": LogSource(
|
||||
source_id="ai-provider",
|
||||
@@ -164,9 +182,18 @@ def normalize_log_levels(level: str | None = None, levels: str | None = None) ->
|
||||
return tuple(normalized_levels)
|
||||
|
||||
|
||||
def resolve_file_log_path(source: LogSource) -> Path:
|
||||
primary = Path(source.location).expanduser()
|
||||
candidates = (primary, *(Path(item).expanduser() for item in source.fallback_locations))
|
||||
for candidate in candidates:
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return primary
|
||||
|
||||
|
||||
def get_source_status(source: LogSource) -> str:
|
||||
if source.kind == "file":
|
||||
path = Path(source.location)
|
||||
path = resolve_file_log_path(source)
|
||||
if not path.exists():
|
||||
return "missing"
|
||||
return "ok" if path.stat().st_size > 0 else "empty"
|
||||
@@ -190,7 +217,7 @@ def list_log_sources() -> list[dict[str, str]]:
|
||||
"source_id": source.source_id,
|
||||
"name": source.name,
|
||||
"kind": source.kind,
|
||||
"location": source.location,
|
||||
"location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location,
|
||||
"description": source.description,
|
||||
"category": source.category,
|
||||
"status": get_source_status(source),
|
||||
@@ -339,7 +366,7 @@ def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry:
|
||||
|
||||
|
||||
def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
||||
path = Path(source.location)
|
||||
path = resolve_file_log_path(source)
|
||||
if not path.exists():
|
||||
return []
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
@@ -511,7 +538,7 @@ def read_log_snapshot(
|
||||
"source_id": source.source_id,
|
||||
"name": source.name,
|
||||
"kind": source.kind,
|
||||
"location": source.location,
|
||||
"location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location,
|
||||
"description": source.description,
|
||||
"category": source.category,
|
||||
"status": get_source_status(source),
|
||||
|
||||
@@ -385,13 +385,18 @@ def build_public_tv_payload(
|
||||
settings_payload: dict[str, Any],
|
||||
collected_sources: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
configured_by_id = {
|
||||
source["id"]: source
|
||||
for source in settings_payload["sources"]
|
||||
if source.get("id")
|
||||
}
|
||||
configured_sources = [
|
||||
source for source in settings_payload["sources"] if source["is_enabled"]
|
||||
]
|
||||
|
||||
merged_by_id = {source["id"]: source for source in configured_sources}
|
||||
for source in collected_sources:
|
||||
if source["id"] in merged_by_id or not source["is_enabled"]:
|
||||
if source["id"] in configured_by_id or not source["is_enabled"]:
|
||||
continue
|
||||
merged_by_id[source["id"]] = source
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ async def create_admin():
|
||||
|
||||
if existing_user:
|
||||
print("用户 linkong 已存在,更新密码...")
|
||||
existing_user.set_password("12345678")
|
||||
existing_user.set_password("LK12345678")
|
||||
existing_user.role = "super_admin"
|
||||
existing_user.email = "linkong@planet.local"
|
||||
else:
|
||||
@@ -26,7 +26,7 @@ async def create_admin():
|
||||
user = User(
|
||||
username="linkong",
|
||||
email="linkong@planet.local",
|
||||
password_hash=get_password_hash("12345678"),
|
||||
password_hash=get_password_hash("LK12345678"),
|
||||
role="super_admin",
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
@@ -19,7 +19,7 @@ DEFAULT_LOGIN_USERS = (
|
||||
{
|
||||
"username": "linkong",
|
||||
"email": "linkong@planet.local",
|
||||
"password": "12345678",
|
||||
"password": "LK12345678",
|
||||
"role": "super_admin",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -22,7 +22,7 @@ DEFAULT_LOGIN_USERS = (
|
||||
{
|
||||
"username": "linkong",
|
||||
"email": "linkong@planet.local",
|
||||
"password": "12345678",
|
||||
"password": "LK12345678",
|
||||
"role": "super_admin",
|
||||
},
|
||||
)
|
||||
|
||||
128
backend/tests/test_ai_observability.py
Normal file
128
backend/tests/test_ai_observability.py
Normal file
@@ -0,0 +1,128 @@
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_tools import web_search as web_search_module
|
||||
from app.services.ai_tools.schemas import SearchEvidence, WebSearchConfig, WebSearchProviderConfig
|
||||
from app.services.ai_tools.web_search import WebSearchClient
|
||||
from app.services import ai_client as ai_client_module
|
||||
from app.services.ai_client import AIProviderClient
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ai_client_analyze_logs_summary_without_prompt(monkeypatch):
|
||||
events = []
|
||||
|
||||
async def fake_emit_business_log(_logger, **payload):
|
||||
events.append(payload)
|
||||
|
||||
async def fake_request(self, method, path, json=None, request_id=None, operation="request", payload_summary=None):
|
||||
return {
|
||||
"provider": "test-provider",
|
||||
"model": "test-model",
|
||||
"content": "ok",
|
||||
"content_blocks": [],
|
||||
"text_blocks": ["ok"],
|
||||
"thinking_blocks": [],
|
||||
"raw_response": {},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(ai_client_module, "emit_business_log", fake_emit_business_log)
|
||||
monkeypatch.setattr(AIProviderClient, "_request", fake_request)
|
||||
|
||||
client = AIProviderClient(
|
||||
service_url="http://provider.test",
|
||||
llm_config={"provider": "openai", "provider_api": "openai-completions", "model": "gpt-test", "api_key": "sk-secret"},
|
||||
)
|
||||
result = await client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title="Sensitive title",
|
||||
objective="Do not store this full prompt",
|
||||
observations=["secret observation"],
|
||||
constraints=["secret constraint"],
|
||||
context={"source": "test", "private": "value"},
|
||||
),
|
||||
request_id="req-ai-test",
|
||||
)
|
||||
|
||||
assert result.model == "test-model"
|
||||
assert [event["event"] for event in events] == [
|
||||
"ai.provider.analyze.start",
|
||||
"ai.provider.analyze.success",
|
||||
]
|
||||
serialized = str(events)
|
||||
assert "Do not store this full prompt" not in serialized
|
||||
assert "secret observation" not in serialized
|
||||
assert "sk-secret" not in serialized
|
||||
start_context = events[0]["context"]
|
||||
assert start_context["model"] == "gpt-test"
|
||||
assert start_context["input_summary"]["objective_length"] == len("Do not store this full prompt")
|
||||
assert start_context["input_summary"]["observation_count"] == 1
|
||||
assert start_context["input_summary"]["context_keys"] == ["private", "source"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ai_client_analyze_logs_failure(monkeypatch):
|
||||
events = []
|
||||
|
||||
async def fake_emit_business_log(_logger, **payload):
|
||||
events.append(payload)
|
||||
|
||||
async def fake_request(self, method, path, json=None, request_id=None, operation="request", payload_summary=None):
|
||||
raise HTTPException(status_code=502, detail="provider failed")
|
||||
|
||||
monkeypatch.setattr(ai_client_module, "emit_business_log", fake_emit_business_log)
|
||||
monkeypatch.setattr(AIProviderClient, "_request", fake_request)
|
||||
|
||||
client = AIProviderClient(service_url="http://provider.test", llm_config={"provider": "openai", "model": "gpt-test"})
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
await client.analyze(
|
||||
SituationalAnalysisRequest(title="T", objective="O", observations=["one"]),
|
||||
request_id="req-ai-fail",
|
||||
)
|
||||
|
||||
assert events[-1]["event"] == "ai.provider.analyze.failed"
|
||||
assert events[-1]["level"] == "error"
|
||||
assert events[-1]["context"]["error_type"] == "HTTPException"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_search_logs_query_hash_without_query(monkeypatch):
|
||||
events = []
|
||||
|
||||
async def fake_emit_business_log(_logger, **payload):
|
||||
events.append(payload)
|
||||
|
||||
async def fake_search_tavily(self, config, query, max_results, domains, freshness_days):
|
||||
return [
|
||||
SearchEvidence(
|
||||
title="Example",
|
||||
url="https://example.test",
|
||||
snippet="result",
|
||||
source_provider="tavily",
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(web_search_module, "emit_business_log", fake_emit_business_log)
|
||||
monkeypatch.setattr(WebSearchClient, "_search_tavily", fake_search_tavily)
|
||||
|
||||
client = WebSearchClient(
|
||||
WebSearchConfig(
|
||||
enabled=True,
|
||||
default_provider="tavily",
|
||||
providers={"tavily": WebSearchProviderConfig(provider="tavily", api_key="secret-key")},
|
||||
)
|
||||
)
|
||||
results = await client.search("secret query text", max_results=1)
|
||||
|
||||
assert len(results) == 1
|
||||
assert [event["event"] for event in events] == [
|
||||
"ai_tool.web_search.start",
|
||||
"ai_tool.web_search.success",
|
||||
]
|
||||
serialized = str(events)
|
||||
assert "secret query text" not in serialized
|
||||
assert "secret-key" not in serialized
|
||||
assert events[0]["context"]["query_length"] == len("secret query text")
|
||||
assert events[1]["context"]["result_count"] == 1
|
||||
@@ -5,10 +5,12 @@ from datetime import datetime
|
||||
from unittest.mock import patch, AsyncMock
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
|
||||
from app.api.v1 import earth as earth_api
|
||||
from app.main import app
|
||||
from app.core.config import settings
|
||||
from app.core.security import create_access_token
|
||||
from app.db.session import get_db
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.models.user import User
|
||||
from app.schemas.ai import (
|
||||
AIProviderStatusResponse,
|
||||
@@ -61,6 +63,148 @@ async def test_root_endpoint():
|
||||
assert data["version"] == settings.VERSION
|
||||
|
||||
|
||||
class _ScalarOneOrNoneResult:
|
||||
def __init__(self, value=None):
|
||||
self._value = value
|
||||
|
||||
def scalar_one_or_none(self):
|
||||
return self._value
|
||||
|
||||
|
||||
class _FakeEarthBrandSession:
|
||||
def __init__(self, record=None):
|
||||
self.record = record
|
||||
self.added = None
|
||||
self.deleted = False
|
||||
self.committed = False
|
||||
|
||||
async def execute(self, statement):
|
||||
if statement.__class__.__name__ == "Delete":
|
||||
self.deleted = True
|
||||
self.record = None
|
||||
return _ScalarOneOrNoneResult(None)
|
||||
return _ScalarOneOrNoneResult(self.record)
|
||||
|
||||
def add(self, record):
|
||||
self.added = record
|
||||
self.record = record
|
||||
|
||||
async def commit(self):
|
||||
self.committed = True
|
||||
|
||||
async def refresh(self, _record):
|
||||
return None
|
||||
|
||||
|
||||
def _override_admin_user():
|
||||
return User(id=1, username="testuser", email="test@example.com", role="admin", is_active=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_earth_brand_returns_static_defaults():
|
||||
async def override_get_db():
|
||||
yield _FakeEarthBrandSession()
|
||||
|
||||
app.dependency_overrides[get_db] = override_get_db
|
||||
try:
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/api/v1/earth/brand")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["is_default"] is True
|
||||
assert data["brand"]["logo_src"] == "/earth/assets/brand/earth-logo.png"
|
||||
assert data["brand"]["title_src"] == "/earth/assets/brand/title-zh.png"
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_and_reset_earth_brand(auth_headers):
|
||||
session = _FakeEarthBrandSession()
|
||||
|
||||
async def override_get_db():
|
||||
yield session
|
||||
|
||||
app.dependency_overrides.update(
|
||||
{
|
||||
__import__("app.core.security", fromlist=["get_current_user"]).get_current_user: _override_admin_user,
|
||||
get_db: override_get_db,
|
||||
}
|
||||
)
|
||||
try:
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
update_response = await client.put(
|
||||
"/api/v1/earth/brand",
|
||||
headers=auth_headers,
|
||||
json={
|
||||
"logo_src": "/earth-brand-assets/custom.png",
|
||||
"title_src": "",
|
||||
"title_text": "Custom Earth",
|
||||
"subtitle": "Custom subtitle",
|
||||
"description": "Custom description",
|
||||
"aria_label": "",
|
||||
"title_alt": "",
|
||||
},
|
||||
)
|
||||
reset_response = await client.delete("/api/v1/earth/brand", headers=auth_headers)
|
||||
|
||||
assert update_response.status_code == 200
|
||||
updated = update_response.json()
|
||||
assert updated["is_default"] is False
|
||||
assert updated["brand"]["title_text"] == "Custom Earth"
|
||||
assert updated["brand"]["aria_label"] == "Custom Earth"
|
||||
assert isinstance(session.added, SystemSetting)
|
||||
assert reset_response.status_code == 200
|
||||
reset = reset_response.json()
|
||||
assert reset["is_default"] is True
|
||||
assert reset["brand"]["logo_src"] == "/earth/assets/brand/earth-logo.png"
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_earth_brand_asset_rejects_invalid_type(auth_headers):
|
||||
app.dependency_overrides[
|
||||
__import__("app.core.security", fromlist=["get_current_user"]).get_current_user
|
||||
] = _override_admin_user
|
||||
try:
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.post(
|
||||
"/api/v1/earth/brand/assets",
|
||||
headers=auth_headers,
|
||||
files={"file": ("brand.txt", b"nope", "text/plain")},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert response.json()["detail"]["code"] == "unsupported_file_type"
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_earth_brand_asset_saves_file(auth_headers, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(earth_api, "EARTH_BRAND_ASSET_DIR", tmp_path)
|
||||
app.dependency_overrides[
|
||||
__import__("app.core.security", fromlist=["get_current_user"]).get_current_user
|
||||
] = _override_admin_user
|
||||
try:
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.post(
|
||||
"/api/v1/earth/brand/assets",
|
||||
headers=auth_headers,
|
||||
files={"file": ("brand.png", b"png-bytes", "image/png")},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["url"].startswith("/earth-brand-assets/")
|
||||
assert (tmp_path / data["filename"]).read_bytes() == b"png-bytes"
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_stats_without_auth():
|
||||
"""Test dashboard stats requires authentication"""
|
||||
@@ -528,6 +672,77 @@ async def test_ingest_earth_client_log_accepts_public_events():
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_layer_cache_status_requires_super_admin(auth_headers, monkeypatch):
|
||||
def override_get_current_user():
|
||||
return User(
|
||||
id=1,
|
||||
username="root",
|
||||
email="root@example.com",
|
||||
password_hash="hashed",
|
||||
role="super_admin",
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.api.v1.system_control.earth_layer_cache.status",
|
||||
lambda: {
|
||||
"prefix": "earth:layer:v1",
|
||||
"key_count": 2,
|
||||
"memory_bytes": 42,
|
||||
"layers": {"cables": {"keys": 2, "stale_keys": 1, "memory_bytes": 42}},
|
||||
},
|
||||
)
|
||||
|
||||
app.dependency_overrides = {
|
||||
__import__("app.core.security", fromlist=["get_current_user"]).get_current_user: override_get_current_user,
|
||||
}
|
||||
transport = ASGITransport(app=app)
|
||||
try:
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/api/v1/system/cache/earth-layers", headers=auth_headers)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["prefix"] == "earth:layer:v1"
|
||||
assert data["layers"]["cables"]["stale_keys"] == 1
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_earth_layer_cache_deletes_only_earth_layer_prefix(auth_headers, monkeypatch):
|
||||
def override_get_current_user():
|
||||
return User(
|
||||
id=1,
|
||||
username="root",
|
||||
email="root@example.com",
|
||||
password_hash="hashed",
|
||||
role="super_admin",
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_delete_pattern(pattern="earth:layer:v1:*"):
|
||||
captured["pattern"] = pattern
|
||||
return 3
|
||||
|
||||
monkeypatch.setattr("app.api.v1.system_control.earth_layer_cache.delete_pattern", fake_delete_pattern)
|
||||
|
||||
app.dependency_overrides = {
|
||||
__import__("app.core.security", fromlist=["get_current_user"]).get_current_user: override_get_current_user,
|
||||
}
|
||||
transport = ASGITransport(app=app)
|
||||
try:
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.delete("/api/v1/system/cache/earth-layers", headers=auth_headers)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["deleted"] == 3
|
||||
assert captured["pattern"] == "earth:layer:v1:*"
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_id_header_is_echoed_when_provided():
|
||||
transport = ASGITransport(app=app)
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
"""Unit tests for data collectors"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.services.collectors.celestrak import CelesTrakTLECollector
|
||||
from app.services.collectors.downloads import DownloadHTTPStatusError, ResumableFileDownloader
|
||||
from app.services.credential_guides import DEFAULT_CREDENTIAL_GUIDES
|
||||
from app.services.collectors.top500 import TOP500Collector
|
||||
from app.services.collectors.registry import collector_registry
|
||||
@@ -149,6 +153,172 @@ class TestHTTPCollector:
|
||||
assert callable(collector.parse_response)
|
||||
|
||||
|
||||
class TestCelesTrakTLECollector:
|
||||
def test_transform_uses_norad_as_source_id_and_preserves_starlink_group(self):
|
||||
collector = CelesTrakTLECollector()
|
||||
result = collector.transform([
|
||||
{
|
||||
"NORAD_CAT_ID": 44720,
|
||||
"OBJECT_NAME": "STARLINK-1000",
|
||||
"OBJECT_ID": "2019-029AZ",
|
||||
"EPOCH": "2026-03-13T00:00:00Z",
|
||||
"MEAN_MOTION": 15.79234567,
|
||||
"ECCENTRICITY": 0.0001234,
|
||||
"INCLINATION": 53.0,
|
||||
"RA_OF_ASC_NODE": 10.0,
|
||||
"ARG_OF_PERICENTER": 20.0,
|
||||
"MEAN_ANOMALY": 30.0,
|
||||
"_celestrak_query_group": "active",
|
||||
"_celestrak_source_url": "https://celestrak.example/gp.php?GROUP=active&FORMAT=json",
|
||||
}
|
||||
])
|
||||
|
||||
assert result[0]["source_id"] == "44720"
|
||||
assert result[0]["metadata"]["constellation_group"] == "starlink"
|
||||
assert result[0]["metadata"]["celestrak_query_group"] == "active"
|
||||
assert result[0]["metadata"]["norad_cat_id"] == 44720
|
||||
assert result[0]["metadata"]["tle_line1"]
|
||||
assert result[0]["metadata"]["tle_line2"]
|
||||
|
||||
def test_load_active_payload_rejects_invalid_records(self, tmp_path):
|
||||
collector = CelesTrakTLECollector()
|
||||
payload_path = tmp_path / "active.json"
|
||||
payload_path.write_text(json.dumps([{"OBJECT_NAME": "missing norad"}]), encoding="utf-8")
|
||||
|
||||
with pytest.raises(RuntimeError, match="invalid record"):
|
||||
collector._load_active_payload(payload_path)
|
||||
|
||||
def test_load_active_payload_accepts_complete_array(self, tmp_path):
|
||||
collector = CelesTrakTLECollector()
|
||||
payload_path = tmp_path / "active.json"
|
||||
payload_path.write_text(
|
||||
json.dumps([{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
records = collector._load_active_payload(payload_path)
|
||||
|
||||
assert records == [{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_retries_and_raises_instead_of_returning_partial_data(self, monkeypatch, tmp_path):
|
||||
collector = CelesTrakTLECollector()
|
||||
collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php"
|
||||
attempts = 0
|
||||
|
||||
async def fake_download_file(*args, **kwargs):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
raise RuntimeError("network interrupted")
|
||||
|
||||
async def fake_emit_business_log(*args, **kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(collector._downloader, "download_file", fake_download_file)
|
||||
monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log)
|
||||
monkeypatch.setattr("app.services.collectors.celestrak.asyncio.sleep", AsyncMock())
|
||||
|
||||
with pytest.raises(RuntimeError, match="failed after retries"):
|
||||
await collector.fetch()
|
||||
|
||||
assert attempts == 3
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_uses_cache_when_celestrak_reports_not_updated(self, monkeypatch, tmp_path):
|
||||
collector = CelesTrakTLECollector()
|
||||
collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php"
|
||||
collector._downloader = ResumableFileDownloader(cache_namespace="celestrak-test", cache_root=tmp_path)
|
||||
url = collector._active_url()
|
||||
cached_path = collector._downloader.cached_file_path(url, ".json")
|
||||
cached_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
cached_path.write_text(
|
||||
json.dumps([{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
async def fake_download_file(*args, **kwargs):
|
||||
raise DownloadHTTPStatusError(
|
||||
url=url,
|
||||
status_code=403,
|
||||
body="GP data has not updated since your last successful download of GROUP=active.",
|
||||
)
|
||||
|
||||
async def fake_emit_business_log(*args, **kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(collector._downloader, "download_file", fake_download_file)
|
||||
monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log)
|
||||
|
||||
records = await collector.fetch()
|
||||
|
||||
assert records[0]["NORAD_CAT_ID"] == 25544
|
||||
assert records[0]["_celestrak_query_group"] == "active"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_not_updated_without_cache_does_not_retry(self, monkeypatch, tmp_path):
|
||||
collector = CelesTrakTLECollector()
|
||||
collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php"
|
||||
collector._downloader = ResumableFileDownloader(cache_namespace="celestrak-test", cache_root=tmp_path)
|
||||
attempts = 0
|
||||
|
||||
async def fake_download_file(*args, **kwargs):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
raise DownloadHTTPStatusError(
|
||||
url=collector._active_url(),
|
||||
status_code=403,
|
||||
body="GP data has not updated since your last successful download of GROUP=active.",
|
||||
)
|
||||
|
||||
async def fake_emit_business_log(*args, **kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(collector._downloader, "download_file", fake_download_file)
|
||||
monkeypatch.setattr("app.services.collectors.celestrak.FALLBACK_GROUPS", ("starlink",))
|
||||
monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log)
|
||||
monkeypatch.setattr("app.services.collectors.celestrak.asyncio.sleep", AsyncMock())
|
||||
|
||||
with pytest.raises(RuntimeError, match="fallback group mode failed"):
|
||||
await collector.fetch()
|
||||
|
||||
assert attempts == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_falls_back_to_all_groups_when_active_not_updated_without_cache(self, monkeypatch, tmp_path):
|
||||
collector = CelesTrakTLECollector()
|
||||
collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php"
|
||||
collector._downloader = ResumableFileDownloader(cache_namespace="celestrak-test", cache_root=tmp_path)
|
||||
payload_by_group = {
|
||||
"starlink": [{"NORAD_CAT_ID": 100, "OBJECT_NAME": "STARLINK-100"}],
|
||||
"gps-ops": [{"NORAD_CAT_ID": 200, "OBJECT_NAME": "GPS BIIR-2"}],
|
||||
}
|
||||
|
||||
async def fake_download_file(_client, url, **_kwargs):
|
||||
if "GROUP=active" in url:
|
||||
raise DownloadHTTPStatusError(
|
||||
url=url,
|
||||
status_code=403,
|
||||
body="GP data has not updated since your last successful download of GROUP=active.",
|
||||
)
|
||||
group = "starlink" if "GROUP=starlink" in url else "gps-ops"
|
||||
path = tmp_path / f"{group}.json"
|
||||
path.write_text(json.dumps(payload_by_group[group]), encoding="utf-8")
|
||||
return path
|
||||
|
||||
async def fake_emit_business_log(*args, **kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(collector._downloader, "download_file", fake_download_file)
|
||||
monkeypatch.setattr("app.services.collectors.celestrak.FALLBACK_GROUPS", tuple(payload_by_group))
|
||||
monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log)
|
||||
|
||||
records = await collector.fetch()
|
||||
|
||||
assert [item["NORAD_CAT_ID"] for item in records] == [100, 200]
|
||||
assert records[0]["_celestrak_query_group"] == "starlink"
|
||||
assert records[1]["_celestrak_group"] == "gps-ops"
|
||||
|
||||
|
||||
def test_aisstream_collector_is_registered():
|
||||
collector = collector_registry.get("aisstream_vessels")
|
||||
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.api.v1 import datasources as datasources_api
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.task import CollectionTask
|
||||
from app.services import data_jobs
|
||||
from app.services import earth_layer_cache as earth_cache
|
||||
|
||||
|
||||
def make_datasource(
|
||||
@@ -53,34 +56,166 @@ def test_filter_datasources_by_product_status_and_collected_state():
|
||||
assert filtered == [vessels]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_datasource_batch_skips_disabled_and_frequency_window(monkeypatch):
|
||||
now = datetime.now(timezone.utc)
|
||||
disabled = make_datasource(1, "aisstream_vessels", is_active=False)
|
||||
not_due = make_datasource(2, "telegeography_cables", last_run_at=now, frequency_minutes=120)
|
||||
due = make_datasource(3, "ris_live_bgp", last_run_at=now - timedelta(hours=2))
|
||||
triggered_sources: list[str] = []
|
||||
def test_serialize_datasource_row_can_skip_endpoint_resolution():
|
||||
datasource = make_datasource(7, "arcgis_cables", last_status="success", module="L2")
|
||||
|
||||
async def fake_running_tasks(_db, _ids):
|
||||
return {}
|
||||
class ExplodingConfig:
|
||||
def get_yaml_url(self, _source):
|
||||
raise AssertionError("endpoint resolution should be skipped")
|
||||
|
||||
async def fake_latest_task_ids(_db, _ids):
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(datasources_api, "_load_latest_running_tasks", fake_running_tasks)
|
||||
monkeypatch.setattr(datasources_api, "_load_latest_task_ids", fake_latest_task_ids)
|
||||
monkeypatch.setattr(
|
||||
datasources_api,
|
||||
"run_collector_now",
|
||||
lambda source: triggered_sources.append(source) or True,
|
||||
row = datasources_api.serialize_datasource_row(
|
||||
datasource,
|
||||
running_tasks={},
|
||||
latest_tasks={},
|
||||
record_counts={"arcgis_cables": 343},
|
||||
endpoint_overrides={},
|
||||
config=ExplodingConfig(),
|
||||
include_endpoint=False,
|
||||
)
|
||||
|
||||
result = await datasources_api._trigger_datasource_batch(
|
||||
object(),
|
||||
[disabled, not_due, due],
|
||||
force=False,
|
||||
assert row["id"] == 7
|
||||
assert row["source"] == "arcgis_cables"
|
||||
assert row["module"] == "L2"
|
||||
assert row["last_status"] == "success"
|
||||
assert row["collected_records"] == 343
|
||||
assert row["has_collected_data"] is True
|
||||
assert "endpoint" not in row
|
||||
|
||||
|
||||
def test_serialize_datasource_row_includes_endpoint_when_requested():
|
||||
datasource = make_datasource(8, "arcgis_landing_points")
|
||||
|
||||
class Config:
|
||||
def get_yaml_url(self, source):
|
||||
return f"https://example.test/{source}"
|
||||
|
||||
row = datasources_api.serialize_datasource_row(
|
||||
datasource,
|
||||
running_tasks={},
|
||||
latest_tasks={},
|
||||
record_counts={},
|
||||
endpoint_overrides={},
|
||||
config=Config(),
|
||||
include_endpoint=True,
|
||||
)
|
||||
|
||||
assert [item["source"] for item in result["triggered"]] == ["ris_live_bgp"]
|
||||
assert {item["reason"] for item in result["skipped"]} == {"disabled", "within_frequency_window"}
|
||||
assert triggered_sources == ["ris_live_bgp"]
|
||||
assert row["endpoint"] == "https://example.test/arcgis_landing_points"
|
||||
|
||||
|
||||
def test_serialize_datasource_row_separates_collector_running_from_delete_task():
|
||||
datasource = make_datasource(9, "top500", last_status="success")
|
||||
task = CollectionTask(
|
||||
id=44,
|
||||
datasource_id=9,
|
||||
source="top500",
|
||||
task_type="clear_data",
|
||||
status="running",
|
||||
phase="clearing_data",
|
||||
)
|
||||
|
||||
row = datasources_api.serialize_datasource_row(
|
||||
datasource,
|
||||
running_tasks={9: task},
|
||||
latest_tasks={9: task},
|
||||
record_counts={"top500": 500},
|
||||
endpoint_overrides={},
|
||||
config=object(),
|
||||
include_endpoint=False,
|
||||
)
|
||||
|
||||
assert row["is_task_active"] is True
|
||||
assert row["is_running"] is False
|
||||
assert row["task_type"] == "clear_data"
|
||||
assert row["task_status"] == "running"
|
||||
|
||||
|
||||
def test_cancel_queued_delete_task_finishes_immediately(monkeypatch):
|
||||
task = CollectionTask(
|
||||
id=45,
|
||||
datasource_id=9,
|
||||
source="top500",
|
||||
task_type="clear_data",
|
||||
status="queued",
|
||||
phase="queued",
|
||||
)
|
||||
|
||||
class FakeDb:
|
||||
async def commit(self):
|
||||
return None
|
||||
|
||||
async def refresh(self, _task):
|
||||
return None
|
||||
|
||||
async def fake_broadcast(_task):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(data_jobs, "_broadcast_task_update", fake_broadcast)
|
||||
|
||||
async def run():
|
||||
return await data_jobs.request_cancel_datasource_task(FakeDb(), task)
|
||||
|
||||
result = asyncio.run(run())
|
||||
|
||||
assert result.status == "cancelled"
|
||||
assert result.phase == "cancelled"
|
||||
assert result.completed_at is not None
|
||||
|
||||
|
||||
def test_clear_data_job_relies_on_db_outbox_instead_of_extra_earth_refresh_task():
|
||||
source = inspect.getsource(data_jobs._run_clear_data_job)
|
||||
|
||||
assert "enqueue_earth_refresh_job" not in source
|
||||
|
||||
|
||||
def test_invalidate_earth_layer_cache_for_source_covers_datasource_aliases(monkeypatch):
|
||||
patterns: list[str] = []
|
||||
|
||||
def fake_delete_pattern(pattern: str) -> int:
|
||||
patterns.append(pattern)
|
||||
return 1
|
||||
|
||||
monkeypatch.setattr(earth_cache.earth_layer_cache, "delete_pattern", fake_delete_pattern)
|
||||
|
||||
deleted = earth_cache.invalidate_earth_layer_cache_for_source("telegeography_cables")
|
||||
|
||||
assert deleted == 3
|
||||
assert patterns == [
|
||||
"earth:layer:v1:cables*",
|
||||
"earth:layer:v1:landing-points*",
|
||||
"earth:layer:v1:summary*",
|
||||
]
|
||||
|
||||
|
||||
def test_trigger_datasource_batch_skips_disabled_and_frequency_window(monkeypatch):
|
||||
async def run():
|
||||
now = datetime.now(timezone.utc)
|
||||
disabled = make_datasource(1, "aisstream_vessels", is_active=False)
|
||||
not_due = make_datasource(2, "telegeography_cables", last_run_at=now, frequency_minutes=120)
|
||||
due = make_datasource(3, "ris_live_bgp", last_run_at=now - timedelta(hours=2))
|
||||
queued_sources: list[str] = []
|
||||
|
||||
async def fake_running_tasks(_db, _ids):
|
||||
return {}
|
||||
|
||||
async def fake_enqueue(_db, datasource, task_type, **_kwargs):
|
||||
queued_sources.append(datasource.source)
|
||||
return CollectionTask(id=100 + datasource.id, datasource_id=datasource.id, source=datasource.source, task_type=task_type, status="queued")
|
||||
|
||||
monkeypatch.setattr(datasources_api, "_load_latest_running_tasks", fake_running_tasks)
|
||||
monkeypatch.setattr(
|
||||
datasources_api,
|
||||
"enqueue_datasource_job",
|
||||
fake_enqueue,
|
||||
)
|
||||
|
||||
result = await datasources_api._trigger_datasource_batch(
|
||||
object(),
|
||||
[disabled, not_due, due],
|
||||
force=False,
|
||||
)
|
||||
|
||||
assert [item["source"] for item in result["triggered"]] == ["ris_live_bgp"]
|
||||
assert {item["reason"] for item in result["skipped"]} == {"disabled", "within_frequency_window"}
|
||||
assert queued_sources == ["ris_live_bgp"]
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
"""Docs Gatekeeper API tests."""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.api.v1 import docs as docs_api
|
||||
from app.main import app
|
||||
from app.models.user import User
|
||||
from app.services.docs_gatekeeper import DOCS_METADATA
|
||||
|
||||
|
||||
def make_user(role: str = "viewer", groups: list[str] | None = None) -> User:
|
||||
@@ -43,22 +47,45 @@ async def test_public_catalog_only_for_anonymous_user():
|
||||
assert response.status_code == 200
|
||||
items = response.json()["items"]
|
||||
assert {item["access"] for item in items} == {"public"}
|
||||
assert {item["slug"] for item in items if item["lang"] == "zh"} == {
|
||||
zh_items = [item for item in items if item["lang"] == "zh"]
|
||||
assert [item["slug"] for item in zh_items] == [
|
||||
"overview",
|
||||
"quickstart",
|
||||
"manual",
|
||||
"quickstart",
|
||||
"faq",
|
||||
"location-pipeline-user",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_developer_catalog_includes_architecture_and_frontend_reference_docs():
|
||||
response = await get_json(
|
||||
"/api/v1/docs/catalog",
|
||||
make_user(role="viewer", groups=["docs_developer"]),
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
zh_slugs = {item["slug"] for item in response.json()["items"] if item["lang"] == "zh"}
|
||||
zh_items = [item for item in response.json()["items"] if item["lang"] == "zh"]
|
||||
assert [item["group"] for item in zh_items[:4]] == ["Overview", "Manual", "Manual", "Manual"]
|
||||
assert zh_items[4]["group"] == "Architecture"
|
||||
assert "platform-data-flows" in zh_slugs
|
||||
assert "naming-glossary" in zh_slugs
|
||||
assert "tactile-ui-components" in zh_slugs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_can_read_public_doc():
|
||||
response = await get_json("/api/v1/docs/zh/quickstart")
|
||||
manual_response = await get_json("/api/v1/docs/zh/manual")
|
||||
overview_response = await get_json("/api/v1/docs/zh/overview")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["access"] == "public"
|
||||
assert "快速开始" in response.json()["markdown"]
|
||||
assert manual_response.status_code == 200
|
||||
assert manual_response.json()["access"] == "public"
|
||||
assert overview_response.status_code == 200
|
||||
assert overview_response.json()["access"] == "public"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -83,10 +110,16 @@ async def test_developer_group_can_read_developer_but_not_admin_doc():
|
||||
user = make_user(role="viewer", groups=["docs_developer"])
|
||||
|
||||
developer_response = await get_json("/api/v1/docs/zh/backend-collectors", user)
|
||||
tactile_response = await get_json("/api/v1/docs/zh/tactile-ui-components", user)
|
||||
glossary_response = await get_json("/api/v1/docs/zh/naming-glossary", user)
|
||||
admin_response = await get_json("/api/v1/docs/zh/backend-system-service-control", user)
|
||||
|
||||
assert developer_response.status_code == 200
|
||||
assert developer_response.json()["access"] == "docs_developer"
|
||||
assert tactile_response.status_code == 200
|
||||
assert tactile_response.json()["access"] == "docs_developer"
|
||||
assert glossary_response.status_code == 200
|
||||
assert glossary_response.json()["access"] == "docs_developer"
|
||||
assert admin_response.status_code == 403
|
||||
|
||||
|
||||
@@ -114,3 +147,29 @@ async def test_unknown_language_slug_and_path_traversal_do_not_read_files():
|
||||
assert bad_lang.status_code == 404
|
||||
assert bad_slug.status_code == 404
|
||||
assert traversal.status_code == 404
|
||||
|
||||
|
||||
def test_public_docs_markdown_links_do_not_create_missing_docs_routes():
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
technical_root = repo_root / "docs" / "technical"
|
||||
registered_filenames = {entry.filename for entry in DOCS_METADATA}
|
||||
problems: list[str] = []
|
||||
|
||||
for markdown_path in sorted(technical_root.glob("*/*.md")):
|
||||
markdown = markdown_path.read_text(encoding="utf-8")
|
||||
for match in re.finditer(r"\[([^\]]+)]\(([^)]+\.md(?:#[^)]+)?)\)", markdown):
|
||||
label, href = match.group(1), match.group(2)
|
||||
if href.startswith(("http://", "https://", "mailto:")):
|
||||
continue
|
||||
|
||||
href_without_hash = href.split("#", 1)[0].replace("\\", "/")
|
||||
filename = Path(href_without_hash).name
|
||||
if "/docs/technical/" in href_without_hash:
|
||||
if filename not in registered_filenames:
|
||||
problems.append(f"{markdown_path.relative_to(repo_root)} links unregistered public doc {href!r} ({label})")
|
||||
continue
|
||||
|
||||
if href_without_hash.endswith(".md"):
|
||||
problems.append(f"{markdown_path.relative_to(repo_root)} links non-public markdown {href!r} ({label})")
|
||||
|
||||
assert problems == []
|
||||
|
||||
177
backend/tests/test_earth_boundaries.py
Normal file
177
backend/tests/test_earth_boundaries.py
Normal file
@@ -0,0 +1,177 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.services.collectors.registry import collector_registry
|
||||
from app.services import earth_boundaries
|
||||
|
||||
|
||||
def write_geojson(path, name="Test"):
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"name": name},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def patch_paths(monkeypatch, tmp_path):
|
||||
repo = tmp_path
|
||||
source_dir = repo / "data/earth-boundary-sources"
|
||||
boundary_dir = repo / "frontend/public/earth/data/boundaries/v1"
|
||||
pmtiles = repo / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
|
||||
legacy = repo / "frontend/public/earth/data/countries-admin0.min.geojson"
|
||||
config = repo / "config/earth-boundary-sources.local.json"
|
||||
example = repo / "config/earth-boundary-sources.example.json"
|
||||
policy = repo / "config/earth-boundary-pov-policy.china-v1.json"
|
||||
for path in (source_dir, boundary_dir, pmtiles.parent, legacy.parent, config.parent):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
policy.write_text('{"productionTileFormat":"pmtiles+mvt"}\n', encoding="utf-8")
|
||||
example.write_text('{"collectorConfigs":{}}\n', encoding="utf-8")
|
||||
monkeypatch.setattr(earth_boundaries, "REPO_ROOT", repo)
|
||||
monkeypatch.setattr(earth_boundaries, "SOURCE_OUTPUT_DIR", source_dir)
|
||||
monkeypatch.setattr(earth_boundaries, "SOURCE_MANIFEST_PATH", source_dir / "manifest.json")
|
||||
monkeypatch.setattr(earth_boundaries, "BUILD_RESULT_PATH", source_dir / "build-result.json")
|
||||
monkeypatch.setattr(earth_boundaries, "BUILD_JOB_PATH", source_dir / "build-job.json")
|
||||
monkeypatch.setattr(earth_boundaries, "BOUNDARY_OUTPUT_DIR", boundary_dir)
|
||||
monkeypatch.setattr(earth_boundaries, "BOUNDARY_MANIFEST_PATH", boundary_dir / "manifest.json")
|
||||
monkeypatch.setattr(earth_boundaries, "PMTILES_ARTIFACT_PATH", pmtiles)
|
||||
monkeypatch.setattr(earth_boundaries, "LEGACY_GEOJSON_PATH", legacy)
|
||||
monkeypatch.setattr(earth_boundaries, "LOCAL_CONFIG_PATH", config)
|
||||
monkeypatch.setattr(earth_boundaries, "EXAMPLE_CONFIG_PATH", example)
|
||||
monkeypatch.setattr(earth_boundaries, "POV_POLICY_PATH", policy)
|
||||
return {
|
||||
"repo": repo,
|
||||
"config": config,
|
||||
"legacy": legacy,
|
||||
"pmtiles": pmtiles,
|
||||
"manifest": boundary_dir / "manifest.json",
|
||||
}
|
||||
|
||||
|
||||
def test_boundary_status_uses_legacy_provider_when_pmtiles_missing(monkeypatch, tmp_path):
|
||||
paths = patch_paths(monkeypatch, tmp_path)
|
||||
write_geojson(paths["legacy"])
|
||||
|
||||
status = earth_boundaries.get_boundary_status()
|
||||
|
||||
assert status["provider"] == "legacy-geojson"
|
||||
assert status["fallback_available"] is True
|
||||
assert status["high_precision_ready"] is False
|
||||
|
||||
|
||||
def test_boundary_status_prefers_high_precision_when_manifest_and_pmtiles_exist(monkeypatch, tmp_path):
|
||||
paths = patch_paths(monkeypatch, tmp_path)
|
||||
write_geojson(paths["legacy"])
|
||||
paths["pmtiles"].write_bytes(b"pmtiles")
|
||||
paths["manifest"].write_text('{"tileProvider":"pmtiles-mvt"}\n', encoding="utf-8")
|
||||
|
||||
status = earth_boundaries.get_boundary_status()
|
||||
|
||||
assert status["provider"] == "pmtiles-mvt"
|
||||
assert status["high_precision_ready"] is True
|
||||
|
||||
|
||||
def test_save_boundary_config_writes_local_config(monkeypatch, tmp_path):
|
||||
paths = patch_paths(monkeypatch, tmp_path)
|
||||
payload = {"collectorConfigs": {"earth_admin0_boundaries": {"endpoint": "file:///tmp/a.geojson"}}}
|
||||
|
||||
status = earth_boundaries.save_boundary_config(payload)
|
||||
|
||||
assert paths["config"].exists()
|
||||
assert status["config_source"] == "local"
|
||||
assert status["config"] == payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_reports_missing_tools_after_source_artifacts(monkeypatch, tmp_path):
|
||||
paths = patch_paths(monkeypatch, tmp_path)
|
||||
source_files = {}
|
||||
for source in earth_boundaries.BOUNDARY_SOURCE_KINDS:
|
||||
source_path = paths["repo"] / f"{source}.geojson"
|
||||
write_geojson(source_path, name=source)
|
||||
source_files[source] = source_path
|
||||
paths["config"].write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collectorConfigs": {
|
||||
source: {
|
||||
"sourceKind": kind,
|
||||
"endpoint": str(source_files[source]),
|
||||
"method": "GET",
|
||||
}
|
||||
for source, kind in earth_boundaries.BOUNDARY_SOURCE_KINDS.items()
|
||||
}
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(earth_boundaries.shutil, "which", lambda _tool: None)
|
||||
progress_events = []
|
||||
|
||||
status = await earth_boundaries.build_boundary_assets(
|
||||
lambda progress, phase, message, **_extra: progress_events.append((progress, phase, message))
|
||||
)
|
||||
|
||||
assert status["provider"] == "geojson-high-precision"
|
||||
assert status["high_precision_ready"] is True
|
||||
assert (paths["repo"] / "data/earth-boundary-sources/manifest.json").exists()
|
||||
assert paths["manifest"].exists()
|
||||
assert any(phase == "download" for _progress, phase, _message in progress_events)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_boundary_build_job_records_geojson_fallback_success(monkeypatch, tmp_path):
|
||||
paths = patch_paths(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(earth_boundaries, "_build_task", None)
|
||||
monkeypatch.setattr(earth_boundaries, "_build_job_state", {})
|
||||
source_files = {}
|
||||
for source in earth_boundaries.BOUNDARY_SOURCE_KINDS:
|
||||
source_path = paths["repo"] / f"{source}.geojson"
|
||||
write_geojson(source_path, name=source)
|
||||
source_files[source] = source_path
|
||||
paths["config"].write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collectorConfigs": {
|
||||
source: {
|
||||
"sourceKind": kind,
|
||||
"endpoint": str(source_files[source]),
|
||||
"method": "GET",
|
||||
}
|
||||
for source, kind in earth_boundaries.BOUNDARY_SOURCE_KINDS.items()
|
||||
}
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(earth_boundaries.shutil, "which", lambda _tool: None)
|
||||
|
||||
response = await earth_boundaries.start_boundary_build_job()
|
||||
await earth_boundaries._build_task
|
||||
status = earth_boundaries.get_boundary_build_status()
|
||||
|
||||
assert response["accepted"] is True
|
||||
assert status["job"]["status"] == "succeeded"
|
||||
assert status["job"]["result"]["provider"] == "geojson-high-precision"
|
||||
|
||||
|
||||
def test_earth_boundary_collectors_are_not_registered_as_datasources():
|
||||
removed = set(earth_boundaries.BOUNDARY_SOURCE_KINDS) | {"earth_boundary_tiles"}
|
||||
|
||||
assert removed.isdisjoint(DEFAULT_DATASOURCES)
|
||||
for source in removed:
|
||||
assert collector_registry.get(source) is None
|
||||
454
backend/tests/test_earth_db_change_listener.py
Normal file
454
backend/tests/test_earth_db_change_listener.py
Normal file
@@ -0,0 +1,454 @@
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from app.services.earth_db_change_listener import (
|
||||
EarthDbChangeDispatcher,
|
||||
EarthDbChangeListener,
|
||||
build_earth_update_from_db_payload,
|
||||
normalize_asyncpg_dsn,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_asyncpg_dsn_strips_sqlalchemy_driver():
|
||||
assert (
|
||||
normalize_asyncpg_dsn("postgresql+asyncpg://postgres:postgres@localhost:5432/planet_db")
|
||||
== "postgresql://postgres:postgres@localhost:5432/planet_db"
|
||||
)
|
||||
|
||||
|
||||
def test_build_earth_update_maps_known_sources_to_layers():
|
||||
satellite_update = build_earth_update_from_db_payload(
|
||||
{"source": "celestrak_tle", "operation": "DELETE", "entity_key": "25544"}
|
||||
)
|
||||
cable_update = build_earth_update_from_db_payload(
|
||||
{"source": "arcgis_cables", "operation": "UPDATE", "entity_key": "cable-1"}
|
||||
)
|
||||
compute_update = build_earth_update_from_db_payload(
|
||||
{"source": "top500", "operation": "DELETE", "entity_key": None}
|
||||
)
|
||||
landing_update = build_earth_update_from_db_payload(
|
||||
{"source": "telegeography_landing", "operation": "DELETE", "entity_key": None}
|
||||
)
|
||||
|
||||
assert satellite_update is not None
|
||||
assert satellite_update["layers"] == ["satellites"]
|
||||
assert cable_update is not None
|
||||
assert cable_update["layers"] == ["cables"]
|
||||
assert compute_update is not None
|
||||
assert compute_update["layers"] == ["computeCenters"]
|
||||
assert landing_update is not None
|
||||
assert landing_update["layers"] == ["cables"]
|
||||
assert landing_update["refresh_strategy"] == "clear_then_reload"
|
||||
|
||||
|
||||
def test_build_earth_update_maps_derived_tables_to_layers():
|
||||
bgp_update = build_earth_update_from_db_payload(
|
||||
{"table": "bgp_anomalies", "source": "ris_live_bgp", "operation": "DELETE", "entity_key": "a1"}
|
||||
)
|
||||
compute_update = build_earth_update_from_db_payload(
|
||||
{"table": "compute_center_locations", "source": "top500", "operation": "UPDATE", "entity_key": "top500_1"}
|
||||
)
|
||||
vessel_update = build_earth_update_from_db_payload(
|
||||
{"table": "vessel_position", "operation": "DELETE", "entity_key": "123456789"}
|
||||
)
|
||||
|
||||
assert bgp_update is not None
|
||||
assert bgp_update["source"] == "ris_live_bgp"
|
||||
assert bgp_update["table"] == "bgp_anomalies"
|
||||
assert bgp_update["layers"] == ["bgp"]
|
||||
assert bgp_update["refresh_strategy"] == "clear_then_reload"
|
||||
assert compute_update is not None
|
||||
assert compute_update["layers"] == ["computeCenters"]
|
||||
assert compute_update["refresh_strategy"] == "reload"
|
||||
assert vessel_update is not None
|
||||
assert vessel_update["source"] == "vessel_position"
|
||||
assert vessel_update["layers"] == ["vessels"]
|
||||
|
||||
|
||||
def test_build_earth_update_maps_interactable_delete_to_delta():
|
||||
update = build_earth_update_from_db_payload(
|
||||
{
|
||||
"table": "earth_interactables",
|
||||
"operation": "DELETE",
|
||||
"entity_keys": ["note-1"],
|
||||
"records_processed": 1,
|
||||
}
|
||||
)
|
||||
|
||||
assert update is not None
|
||||
assert update["entity"] == "interactable"
|
||||
assert update["action"] == "deleted"
|
||||
assert update["ids"] == ["note-1"]
|
||||
assert update["layers"] == ["interactables"]
|
||||
assert update["refresh_strategy"] == "delta"
|
||||
|
||||
|
||||
def test_build_earth_update_ignores_unmapped_sources():
|
||||
assert build_earth_update_from_db_payload({"source": "not_for_earth"}) is None
|
||||
|
||||
|
||||
def test_dispatcher_debounces_same_source_notifications():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
invalidated = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
def invalidate(source):
|
||||
invalidated.append(source)
|
||||
return 2
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=invalidate,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"table": "collected_data",
|
||||
"operation": "INSERT",
|
||||
"source": "arcgis_cables",
|
||||
"entity_key": "cable-1",
|
||||
"occurred_at": "2026-05-22T00:00:00Z",
|
||||
}
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"table": "collected_data",
|
||||
"operation": "UPDATE",
|
||||
"source": "arcgis_cables",
|
||||
"entity_key": "cable-2",
|
||||
"occurred_at": "2026-05-22T00:00:01Z",
|
||||
}
|
||||
)
|
||||
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert invalidated == ["arcgis_cables"]
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["action"] == "database_changed"
|
||||
assert broadcasts[0]["source"] == "arcgis_cables"
|
||||
assert broadcasts[0]["layers"] == ["cables"]
|
||||
assert broadcasts[0]["records_processed"] == 2
|
||||
assert broadcasts[0]["operations"] == ["INSERT", "UPDATE"]
|
||||
assert broadcasts[0]["entity_keys"] == ["cable-1", "cable-2"]
|
||||
assert broadcasts[0]["cache_entries_invalidated"] == 2
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_handles_delete_notification_as_earth_update():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 1,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
accepted = dispatcher.handle_notification(
|
||||
json.dumps(
|
||||
{
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"source": "celestrak_tle",
|
||||
"entity_key": "sat:25544",
|
||||
"occurred_at": "2026-05-22T00:00:00Z",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert accepted is True
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["action"] == "database_changed"
|
||||
assert broadcasts[0]["source"] == "celestrak_tle"
|
||||
assert broadcasts[0]["layers"] == ["satellites"]
|
||||
assert broadcasts[0]["operation"] == "DELETE"
|
||||
assert broadcasts[0]["refresh_strategy"] == "clear_then_reload"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_debounces_bgp_derived_table_events_by_source():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 200,
|
||||
"table": "bgp_observations",
|
||||
"operation": "DELETE",
|
||||
"source": "ris_live_bgp",
|
||||
"records_processed": 2,
|
||||
}
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 201,
|
||||
"table": "bgp_anomalies",
|
||||
"operation": "DELETE",
|
||||
"source": "ris_live_bgp",
|
||||
"records_processed": 3,
|
||||
}
|
||||
)
|
||||
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["source"] == "ris_live_bgp"
|
||||
assert broadcasts[0]["layers"] == ["bgp"]
|
||||
assert broadcasts[0]["records_processed"] == 5
|
||||
assert broadcasts[0]["refresh_strategy"] == "clear_then_reload"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_deduplicates_notify_and_outbox_by_event_id():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
payload = {
|
||||
"event_id": 42,
|
||||
"table": "collected_data",
|
||||
"operation": "INSERT",
|
||||
"source": "celestrak_tle",
|
||||
"entity_key": "sat:42",
|
||||
"occurred_at": "2026-05-23T00:00:00Z",
|
||||
}
|
||||
|
||||
assert dispatcher.handle_payload(payload) is True
|
||||
assert dispatcher.handle_payload(dict(payload)) is False
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["records_processed"] == 1
|
||||
assert broadcasts[0]["entity_keys"] == ["sat:42"]
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_flushes_continuous_events_at_max_wait():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
sleeps = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
max_wait_seconds=0.01,
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 300,
|
||||
"table": "collected_data",
|
||||
"operation": "INSERT",
|
||||
"source": "arcgis_cables",
|
||||
"records_processed": 1,
|
||||
}
|
||||
)
|
||||
await asyncio.sleep(0.02)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 301,
|
||||
"table": "collected_data",
|
||||
"operation": "UPDATE",
|
||||
"source": "arcgis_cables",
|
||||
"records_processed": 1,
|
||||
}
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["records_processed"] == 2
|
||||
assert broadcasts[0]["debounce_ms"] >= 0
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_fast_flushes_delete_clear_then_reload():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
max_wait_seconds=10,
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 310,
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"source": "celestrak_tle",
|
||||
"records_processed": 10,
|
||||
}
|
||||
)
|
||||
await asyncio.sleep(0.08)
|
||||
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["source"] == "celestrak_tle"
|
||||
assert broadcasts[0]["refresh_strategy"] == "clear_then_reload"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_outbox_rows_are_consumed_after_successful_flush():
|
||||
async def run():
|
||||
class FakeConnection:
|
||||
def __init__(self):
|
||||
self.consumed_ids = []
|
||||
|
||||
async def fetch(self, _query, _limit):
|
||||
return [
|
||||
{
|
||||
"id": 501,
|
||||
"payload": {
|
||||
"event_id": 501,
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"source": "celestrak_tle",
|
||||
"records_processed": 1,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
async def execute(self, _query, consumed_ids):
|
||||
self.consumed_ids.extend(consumed_ids)
|
||||
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
listener = EarthDbChangeListener(
|
||||
dsn="postgresql://example/db",
|
||||
dispatcher=dispatcher,
|
||||
)
|
||||
connection = FakeConnection()
|
||||
listener._connection = connection
|
||||
|
||||
await listener._poll_outbox()
|
||||
|
||||
assert connection.consumed_ids == [501]
|
||||
assert len(broadcasts) == 1
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_outbox_rows_remain_unconsumed_when_flush_fails():
|
||||
async def run():
|
||||
class FakeConnection:
|
||||
def __init__(self):
|
||||
self.consumed_ids = []
|
||||
|
||||
async def fetch(self, _query, _limit):
|
||||
return [
|
||||
{
|
||||
"id": 601,
|
||||
"payload": {
|
||||
"event_id": 601,
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"source": "celestrak_tle",
|
||||
"records_processed": 1,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
async def execute(self, _query, consumed_ids):
|
||||
self.consumed_ids.extend(consumed_ids)
|
||||
|
||||
async def broadcast(_payload):
|
||||
raise RuntimeError("ws down")
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
listener = EarthDbChangeListener(
|
||||
dsn="postgresql://example/db",
|
||||
dispatcher=dispatcher,
|
||||
)
|
||||
connection = FakeConnection()
|
||||
listener._connection = connection
|
||||
|
||||
try:
|
||||
await listener._poll_outbox()
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
assert connection.consumed_ids == []
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_uses_aggregated_statement_record_count():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 99,
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"source": "top500",
|
||||
"records_processed": 100,
|
||||
"entity_keys": ["top500:1", "top500:2"],
|
||||
"occurred_at": "2026-05-23T00:00:00Z",
|
||||
}
|
||||
)
|
||||
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["source"] == "top500"
|
||||
assert broadcasts[0]["layers"] == ["computeCenters"]
|
||||
assert broadcasts[0]["records_processed"] == 100
|
||||
assert broadcasts[0]["entity_keys"] == ["top500:1", "top500:2"]
|
||||
|
||||
asyncio.run(run())
|
||||
79
backend/tests/test_earth_interactables.py
Normal file
79
backend/tests/test_earth_interactables.py
Normal file
@@ -0,0 +1,79 @@
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.models.earth_interactable import EarthInteractable
|
||||
from app.services import earth_interactables
|
||||
|
||||
|
||||
def make_interactable(**overrides):
|
||||
values = {
|
||||
"id": "poi-1",
|
||||
"layer": "places",
|
||||
"kind": "note",
|
||||
"label": "Test POI",
|
||||
"description": "A point on Earth",
|
||||
"latitude": 30.25,
|
||||
"longitude": 120.15,
|
||||
"altitude": None,
|
||||
"revision": 3,
|
||||
"properties": {"owner": "test"},
|
||||
"is_deleted": False,
|
||||
"created_at": datetime(2026, 5, 22, 1, 0, tzinfo=UTC),
|
||||
"updated_at": datetime(2026, 5, 22, 1, 5, tzinfo=UTC),
|
||||
"deleted_at": None,
|
||||
}
|
||||
values.update(overrides)
|
||||
return EarthInteractable(**values)
|
||||
|
||||
|
||||
def test_interactable_event_uses_object_delta_contract():
|
||||
record = make_interactable()
|
||||
|
||||
event = earth_interactables.build_interactable_event(
|
||||
action="updated",
|
||||
record=record,
|
||||
)
|
||||
|
||||
assert event["entity"] == "interactable"
|
||||
assert event["action"] == "updated"
|
||||
assert event["layer"] == "places"
|
||||
assert event["layers"] == ["interactables"]
|
||||
assert event["ids"] == ["poi-1"]
|
||||
assert event["revision"] == 3
|
||||
assert event["item"]["latitude"] == 30.25
|
||||
assert event["item"]["properties"] == {"owner": "test"}
|
||||
|
||||
|
||||
def test_interactable_geojson_omits_deleted_records():
|
||||
active = make_interactable(id="active")
|
||||
deleted = make_interactable(id="deleted", is_deleted=True)
|
||||
|
||||
payload = earth_interactables.interactables_to_geojson([active, deleted])
|
||||
|
||||
assert payload["type"] == "FeatureCollection"
|
||||
assert [feature["id"] for feature in payload["features"]] == ["active"]
|
||||
assert payload["features"][0]["geometry"] == {
|
||||
"type": "Point",
|
||||
"coordinates": [120.15, 30.25],
|
||||
}
|
||||
|
||||
|
||||
def test_interactable_cache_invalidation_clears_layer_and_all(monkeypatch):
|
||||
patterns = []
|
||||
|
||||
def fake_delete_pattern(pattern):
|
||||
patterns.append(pattern)
|
||||
return 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
earth_interactables.earth_layer_cache,
|
||||
"delete_pattern",
|
||||
fake_delete_pattern,
|
||||
)
|
||||
|
||||
deleted = earth_interactables.invalidate_interactable_cache("places")
|
||||
|
||||
assert deleted == 2
|
||||
assert patterns == [
|
||||
"earth:layer:v1:interactables:layer:places*",
|
||||
"earth:layer:v1:interactables:layer:all*",
|
||||
]
|
||||
254
backend/tests/test_earth_layer_cache.py
Normal file
254
backend/tests/test_earth_layer_cache.py
Normal file
@@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from fastapi import Response
|
||||
|
||||
from app.api.v1 import visualization
|
||||
from app.services.earth_layer_cache import (
|
||||
EarthLayerCachePolicy,
|
||||
apply_payload_budget,
|
||||
earth_layer_cache,
|
||||
format_bbox_key,
|
||||
quantize_bbox,
|
||||
resolve_layer_payload,
|
||||
)
|
||||
|
||||
|
||||
class FakeRedis:
|
||||
def __init__(self, *, fail: bool = False) -> None:
|
||||
self.store: dict[str, str] = {}
|
||||
self.fail = fail
|
||||
self.lock_claimed = False
|
||||
|
||||
def _maybe_fail(self) -> None:
|
||||
if self.fail:
|
||||
raise RuntimeError("redis unavailable")
|
||||
|
||||
def get(self, key: str):
|
||||
self._maybe_fail()
|
||||
return self.store.get(key)
|
||||
|
||||
def set(self, key: str, value: str, nx: bool = False, ex: int | None = None):
|
||||
self._maybe_fail()
|
||||
if nx and key in self.store:
|
||||
return False
|
||||
self.store[key] = value
|
||||
return True
|
||||
|
||||
def setex(self, key: str, _seconds: int, value: str):
|
||||
self._maybe_fail()
|
||||
self.store[key] = value
|
||||
return True
|
||||
|
||||
def delete(self, *keys: str):
|
||||
self._maybe_fail()
|
||||
deleted = 0
|
||||
for key in keys:
|
||||
deleted += 1 if self.store.pop(key, None) is not None else 0
|
||||
return deleted
|
||||
|
||||
def scan_iter(self, match: str):
|
||||
self._maybe_fail()
|
||||
prefix = match.rstrip("*")
|
||||
for key in list(self.store):
|
||||
if key.startswith(prefix):
|
||||
yield key
|
||||
|
||||
def memory_usage(self, key: str):
|
||||
value = self.store.get(key, "")
|
||||
return len(value.encode("utf-8"))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fake_cache_client():
|
||||
previous = earth_layer_cache._client
|
||||
fake = FakeRedis()
|
||||
earth_layer_cache._client = fake
|
||||
try:
|
||||
yield fake
|
||||
finally:
|
||||
earth_layer_cache._client = previous
|
||||
|
||||
|
||||
def test_quantized_bbox_key_is_stable_for_small_movements():
|
||||
first = format_bbox_key(quantize_bbox((10.01, 59.04, 10.96, 60.02)))
|
||||
second = format_bbox_key(quantize_bbox((10.04, 59.01, 10.99, 60.04)))
|
||||
|
||||
assert first == second
|
||||
assert first == "10.0,59.0,11.0,60.0"
|
||||
|
||||
|
||||
def test_payload_budget_truncates_features():
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [{"id": index} for index in range(5)],
|
||||
}
|
||||
policy = EarthLayerCachePolicy(60, 120, max_features=2, max_bytes=1024)
|
||||
|
||||
result = apply_payload_budget(payload, policy)
|
||||
|
||||
assert len(result["features"]) == 2
|
||||
assert result["diagnostics"]["truncated"] is True
|
||||
assert result["diagnostics"]["limit_reason"] == "feature_budget"
|
||||
assert result["diagnostics"]["original_feature_count"] == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_layer_payload_writes_fresh_and_stale(fake_cache_client):
|
||||
calls = 0
|
||||
|
||||
async def builder():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return {"type": "FeatureCollection", "features": [{"id": "a"}]}
|
||||
|
||||
key = earth_layer_cache.key("satellites", limit="all")
|
||||
policy = EarthLayerCachePolicy(60, 120)
|
||||
|
||||
first = await resolve_layer_payload(key=key, policy=policy, builder=builder)
|
||||
second = await resolve_layer_payload(key=key, policy=policy, builder=builder)
|
||||
|
||||
assert first.state == "refresh"
|
||||
assert second.state == "hit"
|
||||
assert calls == 1
|
||||
assert key in fake_cache_client.store
|
||||
assert f"{key}:stale" in fake_cache_client.store
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_layer_payload_returns_stale_when_builder_fails(fake_cache_client):
|
||||
key = earth_layer_cache.key("bgp-incidents", status="active")
|
||||
fake_cache_client.store[f"{key}:stale"] = json.dumps({"type": "FeatureCollection", "features": []})
|
||||
|
||||
async def builder():
|
||||
raise RuntimeError("db exploded")
|
||||
|
||||
result = await resolve_layer_payload(
|
||||
key=key,
|
||||
policy=EarthLayerCachePolicy(60, 120),
|
||||
builder=builder,
|
||||
)
|
||||
|
||||
assert result.state == "stale"
|
||||
assert result.payload["features"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_layer_payload_uses_stale_during_lock_contention(fake_cache_client):
|
||||
key = earth_layer_cache.key("cables")
|
||||
fake_cache_client.store[earth_layer_cache.lock_key(key)] = "1"
|
||||
fake_cache_client.store[f"{key}:stale"] = json.dumps(
|
||||
{"type": "FeatureCollection", "features": [{"id": "stale-cable"}]}
|
||||
)
|
||||
calls = 0
|
||||
|
||||
async def builder():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return {"type": "FeatureCollection", "features": [{"id": "fresh-cable"}]}
|
||||
|
||||
result = await resolve_layer_payload(
|
||||
key=key,
|
||||
policy=EarthLayerCachePolicy(60, 120),
|
||||
builder=builder,
|
||||
)
|
||||
|
||||
assert result.state == "stale"
|
||||
assert result.payload["features"][0]["id"] == "stale-cable"
|
||||
assert calls == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_layer_payload_bypasses_redis_failure():
|
||||
previous = earth_layer_cache._client
|
||||
earth_layer_cache._client = FakeRedis(fail=True)
|
||||
try:
|
||||
async def builder():
|
||||
return {"type": "FeatureCollection", "features": [{"id": "safe"}]}
|
||||
|
||||
result = await resolve_layer_payload(
|
||||
key=earth_layer_cache.key("cables"),
|
||||
policy=EarthLayerCachePolicy(60, 120),
|
||||
builder=builder,
|
||||
)
|
||||
|
||||
assert result.state == "bypass"
|
||||
assert result.payload["features"][0]["id"] == "safe"
|
||||
finally:
|
||||
earth_layer_cache._client = previous
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_visualization_endpoint_sets_cache_headers(fake_cache_client, monkeypatch):
|
||||
calls = 0
|
||||
|
||||
async def fake_build_satellites_geojson(*, limit, db):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return {"type": "FeatureCollection", "features": [{"id": f"sat-{limit}"}], "count": 1}
|
||||
|
||||
monkeypatch.setattr(visualization, "_build_satellites_geojson", fake_build_satellites_geojson)
|
||||
|
||||
first_response = Response()
|
||||
first = await visualization.get_satellites_geojson(limit=25, db=object(), response=first_response)
|
||||
second_response = Response()
|
||||
second = await visualization.get_satellites_geojson(limit=25, db=object(), response=second_response)
|
||||
|
||||
assert first == second
|
||||
assert calls == 1
|
||||
assert first_response.headers["X-Planet-Cache"] == "refresh"
|
||||
assert second_response.headers["X-Planet-Cache"] == "hit"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vessel_snapshot_uses_short_cache(fake_cache_client, monkeypatch):
|
||||
calls = 0
|
||||
|
||||
async def fake_load_raw_vessel_snapshot_features(db, *, bbox, limit, observed_since):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return (
|
||||
[
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [10.1, 59.1]},
|
||||
"properties": {"mmsi": 123, "vessel_type_name": "Cargo"},
|
||||
}
|
||||
],
|
||||
{"raw_feature_count": 1},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
visualization,
|
||||
"_load_raw_vessel_snapshot_features",
|
||||
fake_load_raw_vessel_snapshot_features,
|
||||
)
|
||||
|
||||
first_response = Response()
|
||||
first = await visualization.build_vessel_snapshot_response(
|
||||
object(),
|
||||
bbox=(10.01, 59.04, 10.96, 60.02),
|
||||
zoom=12,
|
||||
type_filter=None,
|
||||
limit=1000,
|
||||
since_minutes=60,
|
||||
response=first_response,
|
||||
)
|
||||
second_response = Response()
|
||||
second = await visualization.build_vessel_snapshot_response(
|
||||
object(),
|
||||
bbox=(10.04, 59.01, 10.99, 60.04),
|
||||
zoom=12,
|
||||
type_filter=None,
|
||||
limit=1000,
|
||||
since_minutes=60,
|
||||
response=second_response,
|
||||
)
|
||||
|
||||
assert first["count"] == 1
|
||||
assert second == first
|
||||
assert calls == 1
|
||||
assert first_response.headers["X-Planet-Cache"] == "refresh"
|
||||
assert second_response.headers["X-Planet-Cache"] == "hit"
|
||||
@@ -1,6 +1,21 @@
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.services.earth_news import ParsedNewsItem, _serialize_item
|
||||
import pytest
|
||||
|
||||
from app.services.earth_news import (
|
||||
NewsFeedSource,
|
||||
NewsTargetLocation,
|
||||
ParsedNewsItem,
|
||||
_enrich_items_with_target_locations,
|
||||
_extract_target_location_from_text,
|
||||
_parse_feed_entries,
|
||||
_serialize_item,
|
||||
get_earth_news_payload,
|
||||
)
|
||||
from app.services.earth_news_queue import NewsTargetLocationMessage
|
||||
from app.services.earth_news_worker import process_target_location_message
|
||||
from app.services.collectors.media_news_archive import MediaNewsArchiveCollector
|
||||
|
||||
|
||||
def test_serialize_item_includes_region_anchor_for_cruise():
|
||||
@@ -21,7 +36,10 @@ def test_serialize_item_includes_region_anchor_for_cruise():
|
||||
assert payload["latitude"] == 1.3521
|
||||
assert payload["longitude"] == 103.8198
|
||||
assert payload["location_label"] == "亚太"
|
||||
assert payload["location_inferred"] is True
|
||||
assert payload["location_source"] == "region_anchor"
|
||||
assert payload["verified"] is False
|
||||
assert payload["location_meta"]["target"] is None
|
||||
assert payload["location_meta"]["anchor"]["region"] == "asia-pacific"
|
||||
assert payload["is_focus_match"] is True
|
||||
assert payload["published_at"] == "2026-04-23T02:30:00Z"
|
||||
|
||||
@@ -44,6 +62,797 @@ def test_serialize_item_falls_back_to_global_anchor():
|
||||
assert payload["latitude"] == 20.0
|
||||
assert payload["longitude"] == 0.0
|
||||
assert payload["location_label"] == "全球"
|
||||
assert payload["location_inferred"] is True
|
||||
assert payload["location_source"] == "region_anchor"
|
||||
assert payload["verified"] is False
|
||||
assert payload["location_meta"]["anchor"]["region"] == "global"
|
||||
assert payload["is_focus_match"] is False
|
||||
assert payload["published_at"] is None
|
||||
|
||||
|
||||
def test_serialize_item_includes_inferred_target_location():
|
||||
item = ParsedNewsItem(
|
||||
id="bbc-world:f55310fb667b",
|
||||
title="Watch: What happened on day one of Trump's China visit?",
|
||||
summary=(
|
||||
"China welcomed US President Donald Trump with cheering children "
|
||||
"and a troop parade."
|
||||
),
|
||||
url="https://example.com/china-visit",
|
||||
source="BBC World",
|
||||
feed_name="BBC World",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 14, 7, 13, 24, tzinfo=UTC),
|
||||
target_location=NewsTargetLocation(
|
||||
latitude=39.9042,
|
||||
longitude=116.4074,
|
||||
label="Beijing, China",
|
||||
source="ai_inferred_target",
|
||||
confidence=0.88,
|
||||
country="中国",
|
||||
city="Beijing",
|
||||
),
|
||||
)
|
||||
|
||||
payload = _serialize_item(item, active_region="global")
|
||||
|
||||
assert payload["latitude"] == 39.9042
|
||||
assert payload["longitude"] == 116.4074
|
||||
assert payload["location_label"] == "Beijing, China"
|
||||
assert payload["location_source"] == "ai_inferred_target"
|
||||
assert payload["verified"] is True
|
||||
assert payload["location_meta"]["target"]["confidence"] == 0.88
|
||||
assert payload["location_meta"]["target"]["country"] == "中国"
|
||||
assert payload["location_meta"]["target"]["city"] == "Beijing"
|
||||
assert payload["location_meta"]["resolution_stage"] == "unresolved"
|
||||
assert payload["location_meta"]["ai_attempted"] is False
|
||||
assert payload["location_meta"]["ai_status"] == "not_attempted"
|
||||
assert payload["location_meta"]["ai_error"] is None
|
||||
|
||||
|
||||
def test_parse_plain_rss_uses_feed_name_as_source():
|
||||
source = NewsFeedSource(
|
||||
id="bbc-world",
|
||||
name="BBC World",
|
||||
region="global",
|
||||
feed_url="https://feeds.bbci.co.uk/news/world/rss.xml",
|
||||
homepage_url="https://www.bbc.com/news/world",
|
||||
)
|
||||
xml = """
|
||||
<rss>
|
||||
<channel>
|
||||
<item>
|
||||
<title>This may be the last time you hear my voice: Political executions surge in Iran since start of war</title>
|
||||
<description>Story summary</description>
|
||||
<link>https://www.bbc.com/news/example</link>
|
||||
<pubDate>Fri, 15 May 2026 03:00:00 GMT</pubDate>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
|
||||
items = _parse_feed_entries(xml, source)
|
||||
|
||||
assert items[0].title == "This may be the last time you hear my voice: Political executions surge in Iran since start of war"
|
||||
assert items[0].source == "BBC World"
|
||||
|
||||
|
||||
def test_parse_aggregated_rss_splits_publisher_from_title():
|
||||
source = NewsFeedSource(
|
||||
id="global-scan",
|
||||
name="Global Monitor / World",
|
||||
region="global",
|
||||
feed_url="https://news.google.com/rss",
|
||||
homepage_url="https://news.google.com/",
|
||||
source_type="aggregated",
|
||||
)
|
||||
xml = """
|
||||
<rss>
|
||||
<channel>
|
||||
<item>
|
||||
<title>Example headline - Reuters</title>
|
||||
<description>Story summary</description>
|
||||
<link>https://news.google.com/example</link>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
|
||||
items = _parse_feed_entries(xml, source)
|
||||
|
||||
assert items[0].title == "Example headline"
|
||||
assert items[0].source == "Reuters"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enrich_items_with_target_locations_uses_ai_and_geocode(monkeypatch):
|
||||
item = ParsedNewsItem(
|
||||
id="bbc-world:f55310fb667b",
|
||||
title="Watch: What happened on day one of Trump's China visit?",
|
||||
summary="China welcomed US President Donald Trump before a long meeting with Xi Jinping.",
|
||||
url="https://example.com/china-visit",
|
||||
source="BBC World",
|
||||
feed_name="BBC World",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 14, 7, 13, 24, tzinfo=UTC),
|
||||
)
|
||||
|
||||
async def fake_geocode(_query: str):
|
||||
return {
|
||||
"lat": "39.9042",
|
||||
"lon": "116.4074",
|
||||
"display_name": "Beijing, China",
|
||||
}
|
||||
|
||||
class FakeProviderClient:
|
||||
async def analyze(self, _request):
|
||||
class Response:
|
||||
content = (
|
||||
'{"country":"China","city":"Beijing","matched_location_name":"Beijing, China",'
|
||||
'"latitude":null,"longitude":null,"confidence":0.88}'
|
||||
)
|
||||
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news._geocode_target_location", fake_geocode)
|
||||
|
||||
enriched = await _enrich_items_with_target_locations(
|
||||
[item],
|
||||
provider_client=FakeProviderClient(),
|
||||
)
|
||||
|
||||
assert len(enriched) == 1
|
||||
assert enriched[0].target_location is not None
|
||||
assert enriched[0].target_location.latitude == 39.9042
|
||||
assert enriched[0].target_location.longitude == 116.4074
|
||||
assert enriched[0].target_location.label == "Beijing, China"
|
||||
assert enriched[0].target_resolution_stage == "ai_inferred_target"
|
||||
assert enriched[0].target_ai_attempted is True
|
||||
assert enriched[0].target_ai_status == "success"
|
||||
assert enriched[0].target_ai_error is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enrich_items_with_target_locations_adds_localizations(monkeypatch):
|
||||
item = ParsedNewsItem(
|
||||
id="global-scan:localized",
|
||||
title="Global leaders meet to discuss energy security",
|
||||
summary="Officials said the talks focused on supply chains and grid resilience.",
|
||||
url="https://example.com/energy-security",
|
||||
source="Example Source",
|
||||
feed_name="Global Monitor / World",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 14, 7, 13, 24, tzinfo=UTC),
|
||||
)
|
||||
|
||||
async def fake_geocode(_query: str):
|
||||
return {
|
||||
"lat": "50.1109",
|
||||
"lon": "8.6821",
|
||||
"display_name": "Frankfurt am Main, Germany",
|
||||
}
|
||||
|
||||
class FakeProviderClient:
|
||||
async def analyze(self, _request):
|
||||
class Response:
|
||||
content = (
|
||||
'{"location":{"country":"Germany","city":"Frankfurt",'
|
||||
'"matched_location_name":"Frankfurt, Germany",'
|
||||
'"latitude":null,"longitude":null,"confidence":0.77},'
|
||||
'"localizations":{"zh-CN":{"title":"全球领导人讨论能源安全",'
|
||||
'"summary":"官员表示,会谈聚焦供应链和电网韧性。"}}}'
|
||||
)
|
||||
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news._geocode_target_location", fake_geocode)
|
||||
|
||||
enriched = await _enrich_items_with_target_locations(
|
||||
[item],
|
||||
provider_client=FakeProviderClient(),
|
||||
)
|
||||
payload = _serialize_item(enriched[0], active_region="global")
|
||||
|
||||
assert payload["title"] == "Global leaders meet to discuss energy security"
|
||||
assert payload["summary"] == "Officials said the talks focused on supply chains and grid resilience."
|
||||
assert payload["localizations"]["zh-CN"]["title"] == "全球领导人讨论能源安全"
|
||||
assert payload["display_title"] == "全球领导人讨论能源安全"
|
||||
assert payload["display_summary"] == "官员表示,会谈聚焦供应链和电网韧性。"
|
||||
assert payload["enrichment_status"] == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_target_location_from_text_uses_country_hint(monkeypatch):
|
||||
item = ParsedNewsItem(
|
||||
id="bbc-world:country-hint",
|
||||
title="Giant new dinosaur identified from fossils in Thailand",
|
||||
summary="The nagatitan is the largest dinosaur found in South-East Asia.",
|
||||
url="https://example.com/thailand-dinosaur",
|
||||
source="BBC World",
|
||||
feed_name="BBC World",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 14, 17, 28, 56, tzinfo=UTC),
|
||||
)
|
||||
|
||||
target = await _extract_target_location_from_text(item)
|
||||
|
||||
assert target is not None
|
||||
assert target.country == "泰国"
|
||||
assert target.latitude == 15.87
|
||||
assert target.longitude == 100.9925
|
||||
assert target.source == "headline_country_hint"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enrich_items_with_target_locations_records_ai_provider_error():
|
||||
item = ParsedNewsItem(
|
||||
id="global-scan:no-hint",
|
||||
title="The New Geopolitics of Power: Whoever Controls Electrons Wins the Decade",
|
||||
summary="A broad analysis of industrial policy and energy systems.",
|
||||
url="https://example.com/geopolitics-power",
|
||||
source="Example Source",
|
||||
feed_name="Global Monitor / World",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 14, 17, 3, 10, tzinfo=UTC),
|
||||
)
|
||||
|
||||
class FailingProviderClient:
|
||||
async def analyze(self, _request):
|
||||
raise RuntimeError("upstream ai timeout")
|
||||
|
||||
enriched = await _enrich_items_with_target_locations(
|
||||
[item],
|
||||
provider_client=FailingProviderClient(),
|
||||
)
|
||||
|
||||
assert len(enriched) == 1
|
||||
assert enriched[0].target_location is None
|
||||
assert enriched[0].target_resolution_stage == "unresolved"
|
||||
assert enriched[0].target_ai_attempted is True
|
||||
assert enriched[0].target_ai_status == "provider_error"
|
||||
assert enriched[0].target_ai_error == "upstream ai timeout"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_news_payload_returns_anchor_items_and_enqueues_location_job(monkeypatch):
|
||||
source = NewsFeedSource(
|
||||
id="test-feed",
|
||||
name="Test Feed",
|
||||
region="global",
|
||||
homepage_url="https://example.com",
|
||||
feed_url="https://example.com/rss.xml",
|
||||
)
|
||||
item = ParsedNewsItem(
|
||||
id="test-feed:timeout",
|
||||
title="Example story",
|
||||
summary="Example summary",
|
||||
url="https://example.com/story",
|
||||
source="Test Feed",
|
||||
feed_name="Test Feed",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
)
|
||||
|
||||
async def fake_fetch_source(_client, feed_source):
|
||||
return feed_source, [item], None
|
||||
|
||||
async def fake_get_cached_target_location_patch(_item_id):
|
||||
return None
|
||||
|
||||
enqueued_payloads = []
|
||||
|
||||
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
||||
enqueued_payloads.append(payload)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
|
||||
monkeypatch.setattr("app.services.earth_news._fetch_source", fake_fetch_source)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_queue.get_cached_target_location_patch",
|
||||
fake_get_cached_target_location_patch,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_queue.enqueue_target_location_job",
|
||||
fake_enqueue_target_location_job,
|
||||
)
|
||||
|
||||
payload = await get_earth_news_payload(provider_client=None)
|
||||
|
||||
assert len(payload["items"]) == 1
|
||||
assert payload["items"][0]["id"] == "test-feed:timeout"
|
||||
assert payload["items"][0]["display_title"] == ""
|
||||
assert payload["items"][0]["display_summary"] == ""
|
||||
assert payload["items"][0]["latitude"] == 20.0
|
||||
assert payload["items"][0]["longitude"] == 0.0
|
||||
assert payload["items"][0]["location_source"] == "region_anchor"
|
||||
assert payload["items"][0]["verified"] is False
|
||||
assert payload["items"][0]["location_meta"]["ai_status"] == "queued"
|
||||
assert enqueued_payloads[0]["id"] == "test-feed:timeout"
|
||||
assert payload["errors"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_news_payload_uses_fresh_database_items_without_rss(monkeypatch):
|
||||
db = object()
|
||||
item = ParsedNewsItem(
|
||||
id="db:fresh",
|
||||
title="Fresh database story",
|
||||
summary="Stored summary",
|
||||
url="https://example.com/fresh",
|
||||
source="Stored Source",
|
||||
feed_name="Stored Feed",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
)
|
||||
item.location_patch = {
|
||||
"latitude": 39.9057136,
|
||||
"longitude": 116.3912972,
|
||||
"location_label": "北京市, 中国",
|
||||
"location_source": "headline_location_hint",
|
||||
"verified": True,
|
||||
"location_meta": {"target": {"city": "Beijing"}, "anchor": {"region": "global"}},
|
||||
}
|
||||
|
||||
async def fake_get_earth_news_freshness(_db, *, active_region):
|
||||
return 12, datetime.now(UTC)
|
||||
|
||||
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
||||
assert limit == 12
|
||||
return [item]
|
||||
|
||||
async def fail_fetch(_sources):
|
||||
raise AssertionError("fresh database items should not fetch RSS")
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
|
||||
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_items", fake_list_earth_news_items)
|
||||
monkeypatch.setattr("app.services.earth_news._fetch_rss_items_for_sources", fail_fetch)
|
||||
|
||||
payload = await get_earth_news_payload(db=db)
|
||||
|
||||
assert payload["items"][0]["id"] == "db:fresh"
|
||||
assert payload["items"][0]["verified"] is True
|
||||
assert payload["items"][0]["latitude"] == 39.9057136
|
||||
assert payload["stale"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_news_payload_keeps_current_items_and_all_cruise_items(monkeypatch):
|
||||
class FakeDb:
|
||||
execute = object()
|
||||
|
||||
current_item = ParsedNewsItem(
|
||||
id="db:current",
|
||||
title="Current region story",
|
||||
summary="Current summary",
|
||||
url="https://example.com/current",
|
||||
source="Stored Source",
|
||||
feed_name="Stored Feed",
|
||||
feed_region="americas",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
)
|
||||
cruise_item = ParsedNewsItem(
|
||||
id="db:apac",
|
||||
title="APAC story",
|
||||
summary="APAC summary",
|
||||
url="https://example.com/apac",
|
||||
source="Stored Source",
|
||||
feed_name="Stored Feed",
|
||||
feed_region="asia-pacific",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 15, 4, 0, tzinfo=UTC),
|
||||
)
|
||||
|
||||
async def fake_get_earth_news_freshness(_db, *, active_region):
|
||||
return 12, datetime.now(UTC)
|
||||
|
||||
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
||||
return [current_item]
|
||||
|
||||
async def fake_list_earth_news_cruise_items(_db, *, limit):
|
||||
return [current_item, cruise_item]
|
||||
|
||||
async def fake_enqueue_target_location_job(_payload, **_kwargs):
|
||||
return True
|
||||
|
||||
async def fail_fetch(_sources):
|
||||
raise AssertionError("fresh database items should not fetch RSS")
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
|
||||
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_items", fake_list_earth_news_items)
|
||||
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_cruise_items", fake_list_earth_news_cruise_items)
|
||||
monkeypatch.setattr("app.services.earth_news_queue.enqueue_target_location_job", fake_enqueue_target_location_job)
|
||||
monkeypatch.setattr("app.services.earth_news._fetch_rss_items_for_sources", fail_fetch)
|
||||
|
||||
payload = await get_earth_news_payload(lat=35.0, lon=-100.0, db=FakeDb())
|
||||
|
||||
assert [item["id"] for item in payload["items"]] == ["db:current"]
|
||||
assert [item["id"] for item in payload["cruise_items"]] == ["db:current", "db:apac"]
|
||||
assert payload["cruise_items"][1]["region"] == "asia-pacific"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_news_payload_initializes_empty_database_from_rss(monkeypatch):
|
||||
db = object()
|
||||
source = NewsFeedSource(
|
||||
id="test-feed",
|
||||
name="Test Feed",
|
||||
region="global",
|
||||
homepage_url="https://example.com",
|
||||
feed_url="https://example.com/rss.xml",
|
||||
)
|
||||
item = ParsedNewsItem(
|
||||
id="test-feed:init",
|
||||
title="Initial RSS story",
|
||||
summary="Initial summary",
|
||||
url="https://example.com/init",
|
||||
source="Test Feed",
|
||||
feed_name="Test Feed",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
)
|
||||
item.location_patch = {
|
||||
"latitude": 20.0,
|
||||
"longitude": 0.0,
|
||||
"location_label": "全球",
|
||||
"location_source": "region_anchor",
|
||||
"verified": False,
|
||||
"location_meta": {"target": None, "anchor": {"region": "global"}},
|
||||
}
|
||||
upserted = []
|
||||
enqueued = []
|
||||
|
||||
async def fake_get_earth_news_freshness(_db, *, active_region):
|
||||
return 0, None
|
||||
|
||||
async def fake_fetch_rss_items_for_sources(_sources):
|
||||
return [item], []
|
||||
|
||||
async def fake_upsert_earth_news_items(_db, items):
|
||||
upserted.extend(items)
|
||||
return len(items)
|
||||
|
||||
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
||||
return [item]
|
||||
|
||||
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
||||
enqueued.append(payload)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
|
||||
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
|
||||
monkeypatch.setattr("app.services.earth_news._fetch_rss_items_for_sources", fake_fetch_rss_items_for_sources)
|
||||
monkeypatch.setattr("app.services.earth_news_store.upsert_earth_news_items", fake_upsert_earth_news_items)
|
||||
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_items", fake_list_earth_news_items)
|
||||
monkeypatch.setattr("app.services.earth_news_queue.enqueue_target_location_job", fake_enqueue_target_location_job)
|
||||
|
||||
payload = await get_earth_news_payload(db=db)
|
||||
|
||||
assert upserted[0].id == "test-feed:init"
|
||||
assert payload["items"][0]["id"] == "test-feed:init"
|
||||
assert payload["items"][0]["verified"] is False
|
||||
assert enqueued[0]["id"] == "test-feed:init"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_news_payload_supplements_stale_database_items(monkeypatch):
|
||||
db = object()
|
||||
old_item = ParsedNewsItem(
|
||||
id="db:old",
|
||||
title="Old story",
|
||||
summary="Old summary",
|
||||
url="https://example.com/old",
|
||||
source="Stored Source",
|
||||
feed_name="Stored Feed",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 14, 3, 0, tzinfo=UTC),
|
||||
)
|
||||
old_item.location_patch = {
|
||||
"latitude": 20.0,
|
||||
"longitude": 0.0,
|
||||
"location_label": "全球",
|
||||
"location_source": "region_anchor",
|
||||
"verified": False,
|
||||
"location_meta": {"target": None, "anchor": {"region": "global"}},
|
||||
}
|
||||
fetched = []
|
||||
|
||||
async def fake_get_earth_news_freshness(_db, *, active_region):
|
||||
return 12, datetime(2026, 5, 14, 3, 0, tzinfo=UTC)
|
||||
|
||||
async def fake_fetch_rss_items_for_sources(_sources):
|
||||
fetched.append(True)
|
||||
return [old_item], []
|
||||
|
||||
async def fake_upsert_earth_news_items(_db, items):
|
||||
return len(items)
|
||||
|
||||
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
||||
return [old_item]
|
||||
|
||||
async def fake_enqueue_target_location_job(_payload, **_kwargs):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
|
||||
monkeypatch.setattr("app.services.earth_news._fetch_rss_items_for_sources", fake_fetch_rss_items_for_sources)
|
||||
monkeypatch.setattr("app.services.earth_news_store.upsert_earth_news_items", fake_upsert_earth_news_items)
|
||||
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_items", fake_list_earth_news_items)
|
||||
monkeypatch.setattr("app.services.earth_news_queue.enqueue_target_location_job", fake_enqueue_target_location_job)
|
||||
|
||||
payload = await get_earth_news_payload(db=db)
|
||||
|
||||
assert fetched == [True]
|
||||
assert payload["items"][0]["id"] == "db:old"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_news_payload_merges_cached_location_patch(monkeypatch):
|
||||
source = NewsFeedSource(
|
||||
id="test-feed",
|
||||
name="Test Feed",
|
||||
region="global",
|
||||
homepage_url="https://example.com",
|
||||
feed_url="https://example.com/rss.xml",
|
||||
)
|
||||
item = ParsedNewsItem(
|
||||
id="test-feed:cached",
|
||||
title="Cached story",
|
||||
summary="Cached summary",
|
||||
url="https://example.com/cached",
|
||||
source="Test Feed",
|
||||
feed_name="Test Feed",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
)
|
||||
cached_patch = {
|
||||
"latitude": 39.9057136,
|
||||
"longitude": 116.3912972,
|
||||
"location_label": "北京市, 中国",
|
||||
"location_source": "headline_location_hint",
|
||||
"verified": True,
|
||||
"location_meta": {
|
||||
"resolution_stage": "headline_location_hint",
|
||||
"ai_attempted": False,
|
||||
"ai_status": "skipped_text_hint",
|
||||
"ai_error": None,
|
||||
"debug_note": "text hint matched 北京市, 中国",
|
||||
"target": {"city": "Beijing"},
|
||||
"anchor": {"region": "global"},
|
||||
},
|
||||
}
|
||||
|
||||
async def fake_fetch_source(_client, feed_source):
|
||||
return feed_source, [item], None
|
||||
|
||||
async def fake_get_cached_target_location_patch(_item_id):
|
||||
return cached_patch
|
||||
|
||||
enqueued = []
|
||||
|
||||
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
||||
enqueued.append(payload)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
|
||||
monkeypatch.setattr("app.services.earth_news._fetch_source", fake_fetch_source)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_queue.get_cached_target_location_patch",
|
||||
fake_get_cached_target_location_patch,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_queue.enqueue_target_location_job",
|
||||
fake_enqueue_target_location_job,
|
||||
)
|
||||
|
||||
payload = await get_earth_news_payload(provider_client=None)
|
||||
|
||||
assert payload["items"][0]["latitude"] == 39.9057136
|
||||
assert payload["items"][0]["longitude"] == 116.3912972
|
||||
assert payload["items"][0]["verified"] is True
|
||||
assert payload["items"][0]["location_source"] == "headline_location_hint"
|
||||
assert enqueued[0]["id"] == "test-feed:cached"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_news_payload_requeues_cached_failed_localization(monkeypatch):
|
||||
source = NewsFeedSource(
|
||||
id="test-feed",
|
||||
name="Test Feed",
|
||||
region="global",
|
||||
homepage_url="https://example.com",
|
||||
feed_url="https://example.com/rss.xml",
|
||||
)
|
||||
item = ParsedNewsItem(
|
||||
id="test-feed:failed-localization",
|
||||
title="Failed localization story",
|
||||
summary="English source summary.",
|
||||
url="https://example.com/failed-localization",
|
||||
source="Test Feed",
|
||||
feed_name="Test Feed",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
)
|
||||
cached_patch = {
|
||||
"latitude": 20.0,
|
||||
"longitude": 0.0,
|
||||
"location_label": "全球",
|
||||
"location_source": "region_anchor",
|
||||
"verified": False,
|
||||
"location_meta": {"target": None, "anchor": {"region": "global"}},
|
||||
"content_language": "en",
|
||||
"localizations": {},
|
||||
"enrichment_status": "parse_error",
|
||||
"enrichment_error": "AI response did not contain a parseable JSON object.",
|
||||
"enriched_at": None,
|
||||
}
|
||||
enqueued = []
|
||||
|
||||
async def fake_fetch_source(_client, feed_source):
|
||||
return feed_source, [item], None
|
||||
|
||||
async def fake_get_cached_target_location_patch(_item_id):
|
||||
return cached_patch
|
||||
|
||||
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
||||
enqueued.append(payload)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
|
||||
monkeypatch.setattr("app.services.earth_news._fetch_source", fake_fetch_source)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_queue.get_cached_target_location_patch",
|
||||
fake_get_cached_target_location_patch,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_queue.enqueue_target_location_job",
|
||||
fake_enqueue_target_location_job,
|
||||
)
|
||||
|
||||
payload = await get_earth_news_payload(provider_client=None)
|
||||
|
||||
assert enqueued[0]["id"] == "test-feed:failed-localization"
|
||||
assert payload["items"][0]["display_title"] == ""
|
||||
assert payload["items"][0]["enrichment_status"] == "queued"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_processes_target_location_message_and_returns_patch(monkeypatch):
|
||||
message = NewsTargetLocationMessage(
|
||||
message_id="1-0",
|
||||
item_id="bbc-world:worker",
|
||||
payload={
|
||||
"id": "bbc-world:worker",
|
||||
"title": "Ukraine rescuers pull dead from rubble of Kyiv flats",
|
||||
"summary": "Massive Russian drone and missile attacks in Ukraine's capital.",
|
||||
"url": "https://example.com/kyiv",
|
||||
"source": "BBC World",
|
||||
"feed_name": "BBC World",
|
||||
"feed_region": "global",
|
||||
"homepage_url": "https://www.bbc.com/news/world",
|
||||
"published_at": "2026-05-14T13:16:32Z",
|
||||
},
|
||||
)
|
||||
|
||||
async def fake_geocode(_query: str):
|
||||
return {
|
||||
"lat": "50.4500336",
|
||||
"lon": "30.5241361",
|
||||
"display_name": "Київ, Україна",
|
||||
}
|
||||
|
||||
saved = {}
|
||||
broadcasted = {}
|
||||
|
||||
async def fake_save_target_location_patch(item_id, patch):
|
||||
saved["item_id"] = item_id
|
||||
saved["patch"] = patch
|
||||
|
||||
async def fake_update_earth_news_item_location(_session, *, item_id, patch):
|
||||
saved["db_item_id"] = item_id
|
||||
saved["db_patch"] = patch
|
||||
return True
|
||||
|
||||
async def fake_broadcast_custom(channel, data):
|
||||
broadcasted["channel"] = channel
|
||||
broadcasted["data"] = data
|
||||
|
||||
class FakeSession:
|
||||
async def commit(self):
|
||||
saved["committed"] = True
|
||||
|
||||
class FakeSessionFactory:
|
||||
async def __aenter__(self):
|
||||
return FakeSession()
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news._geocode_target_location", fake_geocode)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_worker.save_target_location_patch",
|
||||
fake_save_target_location_patch,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_worker.update_earth_news_item_location",
|
||||
fake_update_earth_news_item_location,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_worker.async_session_factory",
|
||||
lambda: FakeSessionFactory(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_worker.broadcaster.broadcast_custom",
|
||||
fake_broadcast_custom,
|
||||
)
|
||||
|
||||
patch = await process_target_location_message(message, provider_client=None)
|
||||
|
||||
assert patch["latitude"] == 50.4500336
|
||||
assert patch["longitude"] == 30.5241361
|
||||
assert patch["location_source"] == "headline_location_hint"
|
||||
assert patch["verified"] is True
|
||||
assert saved["item_id"] == "bbc-world:worker"
|
||||
assert saved["db_item_id"] == "bbc-world:worker"
|
||||
assert saved["committed"] is True
|
||||
assert broadcasted["channel"] == "earth_news"
|
||||
assert broadcasted["data"]["item_id"] == "bbc-world:worker"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_media_news_archive_collector_maps_news_items(monkeypatch):
|
||||
collector = MediaNewsArchiveCollector()
|
||||
collector._db_session = object()
|
||||
record = SimpleNamespace(
|
||||
id="bbc-world:archive",
|
||||
title="Archived news",
|
||||
summary="Archived summary",
|
||||
url="https://example.com/archive",
|
||||
source="BBC World",
|
||||
feed_name="BBC World",
|
||||
region="global",
|
||||
homepage_url="https://www.bbc.com/news/world",
|
||||
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
latitude=39.9057136,
|
||||
longitude=116.3912972,
|
||||
location_label="北京市, 中国",
|
||||
location_source="headline_location_hint",
|
||||
verified=True,
|
||||
location_meta={"target": {"country": "中国", "city": "Beijing"}},
|
||||
content_language="en",
|
||||
localizations={"zh-CN": {"title": "归档新闻", "summary": "归档概要"}},
|
||||
enrichment_status="success",
|
||||
enrichment_error=None,
|
||||
enriched_at=datetime(2026, 5, 15, 3, 6, tzinfo=UTC),
|
||||
first_seen_at=datetime(2026, 5, 15, 2, 0, tzinfo=UTC),
|
||||
last_seen_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
resolved_at=datetime(2026, 5, 15, 3, 5, tzinfo=UTC),
|
||||
)
|
||||
|
||||
async def fake_list_all_earth_news_records(_db):
|
||||
return [record]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.collectors.media_news_archive.list_all_earth_news_records",
|
||||
fake_list_all_earth_news_records,
|
||||
)
|
||||
|
||||
items = await collector.fetch()
|
||||
|
||||
assert items[0]["source_id"] == "bbc-world:archive"
|
||||
assert collector.data_type == "news_item"
|
||||
assert items[0]["country"] == "中国"
|
||||
assert items[0]["city"] == "Beijing"
|
||||
assert items[0]["latitude"] == 39.9057136
|
||||
assert items[0]["metadata"]["verified"] is True
|
||||
assert "localizations" not in items[0]["metadata"]
|
||||
assert "enrichment_status" not in items[0]["metadata"]
|
||||
|
||||
@@ -700,7 +700,7 @@ async def test_llm_location_fallback_repairs_non_json_answer(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_accepts_taipei_name_hint_with_weak_wording(monkeypatch):
|
||||
async def test_llm_location_fallback_rejects_city_from_name_without_location_evidence(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
llm_fallback,
|
||||
"_geocode_llm_city",
|
||||
@@ -740,18 +740,60 @@ async def test_llm_location_fallback_accepts_taipei_name_hint_with_weak_wording(
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.failure_reason is None
|
||||
candidate = result.candidates[0]
|
||||
assert candidate.city == "Taipei"
|
||||
assert candidate.confidence >= 0.55
|
||||
breakdown = candidate.suggested_registry_entry["llm_score_breakdown"]
|
||||
assert breakdown["weak_evidence_penalty"] <= 0.15
|
||||
assert breakdown["conflict_penalty"] == 0
|
||||
assert breakdown["name_location_hint"] > 0
|
||||
assert result.candidates == []
|
||||
assert result.failure_reason is not None
|
||||
assert "below minimum" in result.failure_reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_geocodes_city_from_entity_name_when_llm_unparseable(monkeypatch):
|
||||
async def test_llm_location_fallback_accepts_explicit_facility_location_for_name_city_conflict(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
llm_fallback,
|
||||
"_geocode_llm_city",
|
||||
lambda query: {
|
||||
"lat": "22.6048",
|
||||
"lon": "120.3000",
|
||||
"display_name": "Kaohsiung, Taiwan",
|
||||
"address": {"city": "Kaohsiung", "country": "Taiwan"},
|
||||
},
|
||||
)
|
||||
client = _FakeAIProviderClient(
|
||||
json.dumps(
|
||||
{
|
||||
"latitude": None,
|
||||
"longitude": None,
|
||||
"precision": "city",
|
||||
"confidence": 0.72,
|
||||
"city": "Kaohsiung",
|
||||
"country": "Taiwan",
|
||||
"matched_location_name": "Kaohsiung, Taiwan",
|
||||
"evidence": [
|
||||
{
|
||||
"source": "Taiwan News",
|
||||
"source_type": "news",
|
||||
"entity_match": True,
|
||||
"text": "Nvidia's first AI supercomputer center, Taipei-1, is located in Kaohsiung.",
|
||||
}
|
||||
],
|
||||
"reasoning_summary": "Explicit facility location evidence overrides the city-like system name.",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(name="TAIPEI-1", country="Taiwan"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.failure_reason is None
|
||||
candidate = result.candidates[0]
|
||||
assert candidate.city == "Kaohsiung"
|
||||
assert candidate.confidence >= 0.55
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_rejects_compute_center_city_from_entity_name_when_llm_unparseable(monkeypatch):
|
||||
def _fake_geocode(query):
|
||||
if query != "Taipei, 中国(台湾)":
|
||||
return None
|
||||
@@ -772,12 +814,9 @@ async def test_llm_location_fallback_geocodes_city_from_entity_name_when_llm_unp
|
||||
)
|
||||
|
||||
assert client.calls == 2
|
||||
assert result.failure_reason is None
|
||||
candidate = result.candidates[0]
|
||||
assert candidate.city == "Taipei"
|
||||
assert candidate.latitude == pytest.approx(25.033)
|
||||
assert candidate.longitude == pytest.approx(121.5654)
|
||||
assert "Entity name city hint" in candidate.source_note
|
||||
assert result.candidates == []
|
||||
assert result.failure_reason is not None
|
||||
assert "parseable city-level location fact" in result.failure_reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -4,8 +4,11 @@ import logging
|
||||
|
||||
from io import StringIO
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.logging import PlanetContextFilter, PlanetFormatter, get_logger
|
||||
from app.core.request_context import set_request_id
|
||||
from app.services import business_logs
|
||||
|
||||
|
||||
def _capture_output(callback):
|
||||
@@ -76,3 +79,43 @@ def test_structured_logger_redacts_sensitive_text_and_context():
|
||||
assert "hunter2" not in output
|
||||
assert "[REDACTED]" in output
|
||||
assert '"safe": "visible"' in output
|
||||
|
||||
|
||||
def test_business_context_redacts_nested_sensitive_values():
|
||||
context = business_logs.build_business_context(
|
||||
{
|
||||
"provider": "openai",
|
||||
"api_key": "sk-secret",
|
||||
"nested": {
|
||||
"token": "plain-token",
|
||||
"safe": "visible",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert context["api_key"] == "[REDACTED]"
|
||||
assert context["nested"]["token"] == "[REDACTED]"
|
||||
assert context["nested"]["safe"] == "visible"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_emit_business_log_persists_sanitized_system_event(monkeypatch):
|
||||
events = []
|
||||
|
||||
async def fake_record_system_log(**payload):
|
||||
events.append(payload)
|
||||
|
||||
monkeypatch.setattr(business_logs, "record_system_log", fake_record_system_log)
|
||||
|
||||
await business_logs.emit_business_log(
|
||||
get_logger("tests.business"),
|
||||
event="ai.provider.analyze.success",
|
||||
message="AI request completed",
|
||||
category="ai",
|
||||
context={"model": "gpt-test", "api_key": "sk-secret"},
|
||||
)
|
||||
|
||||
assert events[0]["event"] == "ai.provider.analyze.success"
|
||||
assert events[0]["category"] == "ai"
|
||||
assert events[0]["context"]["model"] == "gpt-test"
|
||||
assert events[0]["context"]["api_key"] == "[REDACTED]"
|
||||
|
||||
92
backend/tests/test_settings_ai_prompts.py
Normal file
92
backend/tests/test_settings_ai_prompts.py
Normal file
@@ -0,0 +1,92 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.ai_tasks.prompts import (
|
||||
get_effective_prompt,
|
||||
list_effective_prompts,
|
||||
reset_prompt_override,
|
||||
save_prompt_override,
|
||||
)
|
||||
|
||||
|
||||
class _ScalarResult:
|
||||
def __init__(self, value):
|
||||
self._value = value
|
||||
|
||||
def scalar_one_or_none(self):
|
||||
return self._value
|
||||
|
||||
|
||||
class _PromptSettingsDB:
|
||||
def __init__(self, payload=None):
|
||||
self.record = SimpleNamespace(category="ai_prompts", payload=payload) if payload is not None else None
|
||||
self.added = None
|
||||
self.commits = 0
|
||||
|
||||
async def execute(self, _statement):
|
||||
return _ScalarResult(self.record)
|
||||
|
||||
def add(self, record):
|
||||
self.record = record
|
||||
self.added = record
|
||||
|
||||
async def commit(self):
|
||||
self.commits += 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_defaults_are_loaded_without_override():
|
||||
db = _PromptSettingsDB()
|
||||
|
||||
prompt = await get_effective_prompt(db, "earth.news.enrich")
|
||||
|
||||
assert prompt.key == "earth.news.enrich"
|
||||
assert prompt.is_custom is False
|
||||
assert "strict JSON" in prompt.prompt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_override_save_and_reset():
|
||||
db = _PromptSettingsDB()
|
||||
|
||||
saved = await save_prompt_override(
|
||||
db,
|
||||
"alerts.brief",
|
||||
system_prompt="system custom",
|
||||
prompt="prompt custom",
|
||||
)
|
||||
|
||||
assert saved.is_custom is True
|
||||
assert saved.system_prompt == "system custom"
|
||||
assert saved.prompt == "prompt custom"
|
||||
assert db.commits == 1
|
||||
|
||||
effective = await get_effective_prompt(db, "alerts.brief")
|
||||
assert effective.prompt == "prompt custom"
|
||||
|
||||
reset = await reset_prompt_override(db, "alerts.brief")
|
||||
assert reset.is_custom is False
|
||||
assert reset.prompt != "prompt custom"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_list_marks_custom_items():
|
||||
db = _PromptSettingsDB(
|
||||
{
|
||||
"overrides": {
|
||||
"bgp.brief": {
|
||||
"system_prompt": "",
|
||||
"prompt": "custom bgp prompt",
|
||||
"updated_at": "2026-05-16T00:00:00Z",
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
prompts = await list_effective_prompts(db)
|
||||
by_key = {prompt.key: prompt for prompt in prompts}
|
||||
|
||||
assert by_key["bgp.brief"].is_custom is True
|
||||
assert by_key["bgp.brief"].prompt == "custom bgp prompt"
|
||||
assert by_key["earth.news.enrich"].is_custom is False
|
||||
@@ -5,21 +5,42 @@ import pytest
|
||||
from app.api.v1 import settings as settings_api
|
||||
from app.api.v1.settings import (
|
||||
AIProviderIntegrationUpdate,
|
||||
BarentsWatchIntegrationUpdate,
|
||||
ExternalIntegrationsUpdate,
|
||||
OCRIntegrationUpdate,
|
||||
WebSearchIntegrationUpdate,
|
||||
_build_ai_provider_payload,
|
||||
_build_ocr_payload,
|
||||
_can_reveal_integration_secrets,
|
||||
_ensure_secret_reveal_allowed,
|
||||
_mask_secret,
|
||||
_normalize_ai_provider_payload,
|
||||
_normalize_ocr_payload,
|
||||
_record_integration_secret_reveal,
|
||||
_resolve_provider_api_key,
|
||||
get_runtime_ai_provider_config,
|
||||
save_external_integrations_payload,
|
||||
)
|
||||
from app.services.llm_provider_catalog import get_fallback_llm_provider_preset
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_ai_provider_env_file(monkeypatch, tmp_path):
|
||||
env_file = tmp_path / ".env"
|
||||
monkeypatch.setattr(settings_api, "AI_PROVIDER_ENV_FILE", env_file)
|
||||
for name in (
|
||||
"AI_PROVIDER",
|
||||
"AI_API_KEY",
|
||||
"MINIMAX_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"ANTHROPIC_API_KEY",
|
||||
"DEEPSEEK_API_KEY",
|
||||
"DASHSCOPE_API_KEY",
|
||||
"MOONSHOT_API_KEY",
|
||||
"OPENROUTER_API_KEY",
|
||||
"OPENCODE_GO_API_KEY",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
return env_file
|
||||
|
||||
|
||||
@@ -56,7 +77,7 @@ def test_provider_key_prefers_specific_env_file_key(isolated_ai_provider_env_fil
|
||||
|
||||
def test_provider_key_falls_back_to_generic_ai_api_key(isolated_ai_provider_env_file):
|
||||
isolated_ai_provider_env_file.write_text(
|
||||
"AI_API_KEY=generic-env-file-key\n",
|
||||
"AI_PROVIDER=openai\nAI_API_KEY=generic-env-file-key\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
@@ -66,11 +87,101 @@ def test_provider_key_falls_back_to_generic_ai_api_key(isolated_ai_provider_env_
|
||||
assert source == "env_file"
|
||||
|
||||
|
||||
def test_generic_ai_api_key_only_applies_to_selected_env_provider(isolated_ai_provider_env_file):
|
||||
isolated_ai_provider_env_file.write_text(
|
||||
"AI_PROVIDER=minimax\nAI_API_KEY=generic-env-file-key\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
openai_value, openai_source = _resolve_provider_api_key("openai", {"api_key": ""})
|
||||
minimax_value, minimax_source = _resolve_provider_api_key("minimax", {"api_key": ""})
|
||||
|
||||
assert openai_value == ""
|
||||
assert openai_source == ""
|
||||
assert minimax_value == "generic-env-file-key"
|
||||
assert minimax_source == "env_file"
|
||||
|
||||
|
||||
def test_opencode_go_provider_preset_is_openai_compatible():
|
||||
preset = get_fallback_llm_provider_preset("opencode-go")
|
||||
|
||||
assert preset["label"] == "OpenCode Go"
|
||||
assert preset["provider_api"] == "openai-completions"
|
||||
assert preset["base_url"] == "https://opencode.ai/zen/go/v1"
|
||||
assert preset["model"] == "glm-5.1"
|
||||
assert "glm-5.1" in preset["models"]
|
||||
assert "deepseek-v4-flash" in preset["models"]
|
||||
assert preset["model_provider_apis"]["minimax-m2.7"] == "anthropic-messages"
|
||||
assert preset["api_key_env"] == "OPENCODE_GO_API_KEY"
|
||||
|
||||
|
||||
def test_mask_secret_without_prefix_is_fully_masked():
|
||||
assert _mask_secret("plainsecret")["preview"] == "***********"
|
||||
assert _mask_secret("sk-prefixed")["preview"] == "sk-********"
|
||||
|
||||
|
||||
def test_secret_reveal_permission_is_admin_only():
|
||||
assert _can_reveal_integration_secrets(SimpleNamespace(role="super_admin")) is True
|
||||
assert _can_reveal_integration_secrets(SimpleNamespace(role="admin")) is True
|
||||
assert _can_reveal_integration_secrets(SimpleNamespace(role="viewer")) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_secret_reveal_denial_is_audited(monkeypatch):
|
||||
events = []
|
||||
|
||||
async def fake_record_audit_log(**payload):
|
||||
events.append(payload)
|
||||
|
||||
monkeypatch.setattr(settings_api, "record_audit_log", fake_record_audit_log)
|
||||
user = SimpleNamespace(id=7, username="viewer", email="viewer@example.test", role="viewer")
|
||||
|
||||
with pytest.raises(settings_api.HTTPException) as exc:
|
||||
await _ensure_secret_reveal_allowed(
|
||||
current_user=user,
|
||||
request=None,
|
||||
target_id="ai_provider:openai",
|
||||
details={"kind": "ai_provider", "provider": "openai"},
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 403
|
||||
assert events[0]["action"] == "settings.integration_secret.reveal"
|
||||
assert events[0]["result"] == "denied"
|
||||
assert events[0]["actor_id"] == 7
|
||||
assert events[0]["target_id"] == "ai_provider:openai"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_secret_reveal_audit_does_not_store_plaintext(monkeypatch):
|
||||
events = []
|
||||
|
||||
async def fake_record_audit_log(**payload):
|
||||
events.append(payload)
|
||||
|
||||
monkeypatch.setattr(settings_api, "record_audit_log", fake_record_audit_log)
|
||||
user = SimpleNamespace(id=1, username="admin", email="admin@example.test", role="admin")
|
||||
|
||||
await _record_integration_secret_reveal(
|
||||
current_user=user,
|
||||
request=None,
|
||||
target_id="ai_provider:minimax",
|
||||
result="success",
|
||||
details={
|
||||
"kind": "ai_provider",
|
||||
"provider": "minimax",
|
||||
"api_key_configured": True,
|
||||
"api_key_source": "env_file",
|
||||
"service_token_configured": True,
|
||||
"service_token_source": "runtime",
|
||||
},
|
||||
)
|
||||
|
||||
serialized = str(events[0])
|
||||
assert "secret-value" not in serialized
|
||||
assert events[0]["details"]["api_key_source"] == "env_file"
|
||||
assert events[0]["details"]["service_token_configured"] is True
|
||||
|
||||
|
||||
def test_build_payload_updates_only_selected_provider_key():
|
||||
current = {
|
||||
"ai_provider": {
|
||||
@@ -109,6 +220,30 @@ def test_build_payload_updates_only_selected_provider_key():
|
||||
assert payload["providers"]["minimax"]["api_key"] == "minimax-old-key"
|
||||
|
||||
|
||||
def test_build_payload_saves_provider_without_changing_default():
|
||||
current = {
|
||||
"ai_provider": {
|
||||
"default_provider": "minimax",
|
||||
"providers": {
|
||||
"minimax": {"provider": "minimax", "api_key": "minimax-key"},
|
||||
"openai": {"provider": "openai", "api_key": ""},
|
||||
},
|
||||
}
|
||||
}
|
||||
update = AIProviderIntegrationUpdate(
|
||||
provider="openai",
|
||||
provider_api="openai-completions",
|
||||
base_url="https://api.openai.com/v1",
|
||||
model="gpt-test",
|
||||
api_key="openai-new-key",
|
||||
)
|
||||
|
||||
payload = _build_ai_provider_payload(current, update)
|
||||
|
||||
assert payload["default_provider"] == "minimax"
|
||||
assert payload["providers"]["openai"]["api_key"] == "openai-new-key"
|
||||
|
||||
|
||||
def test_build_payload_keeps_saved_key_when_preview_submitted():
|
||||
current = {
|
||||
"ai_provider": {
|
||||
@@ -207,3 +342,78 @@ async def test_runtime_config_uses_default_provider_specific_key(monkeypatch):
|
||||
assert runtime_config["llm_config"]["provider"] == "minimax"
|
||||
assert runtime_config["llm_config"]["api_key"] == "minimax-key"
|
||||
assert runtime_config["llm_config"]["model"] == "MiniMax-test"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_saving_ai_provider_does_not_run_connection_validation(monkeypatch):
|
||||
calls = {"validated": 0, "saved": 0}
|
||||
|
||||
async def fake_get_setting_payload(_db, category):
|
||||
assert category == "external_integrations"
|
||||
return {
|
||||
"ai_provider": {
|
||||
"default_provider": "minimax",
|
||||
"providers": {
|
||||
"minimax": {
|
||||
"provider": "minimax",
|
||||
"api_key": "minimax-key",
|
||||
},
|
||||
},
|
||||
},
|
||||
"web_search": {},
|
||||
"ocr": {},
|
||||
}
|
||||
|
||||
async def fake_validate(_payload):
|
||||
calls["validated"] += 1
|
||||
raise AssertionError("save should not run AI provider connection validation")
|
||||
|
||||
async def fake_save_setting_payload(_db, category, payload):
|
||||
assert category == "external_integrations"
|
||||
calls["saved"] += 1
|
||||
return payload
|
||||
|
||||
async def fake_get_barentswatch_config_record(_db):
|
||||
return SimpleNamespace(
|
||||
endpoint="",
|
||||
auth_config={},
|
||||
)
|
||||
|
||||
async def fake_serialize_external_integrations(_db):
|
||||
return {"ai_provider": {"default_provider": "minimax"}}
|
||||
|
||||
monkeypatch.setattr(settings_api, "get_setting_payload", fake_get_setting_payload)
|
||||
monkeypatch.setattr(settings_api, "_validate_ai_provider_full_connection", fake_validate)
|
||||
monkeypatch.setattr(settings_api, "save_setting_payload", fake_save_setting_payload)
|
||||
monkeypatch.setattr(settings_api, "get_barentswatch_config_record", fake_get_barentswatch_config_record)
|
||||
monkeypatch.setattr(settings_api, "serialize_external_integrations", fake_serialize_external_integrations)
|
||||
|
||||
update = ExternalIntegrationsUpdate(
|
||||
ai_provider=AIProviderIntegrationUpdate(
|
||||
provider="minimax",
|
||||
default_provider="minimax",
|
||||
provider_api="anthropic-messages",
|
||||
base_url="https://api.minimaxi.com/anthropic",
|
||||
model="MiniMax-M2.7",
|
||||
api_key="sk-new-key",
|
||||
),
|
||||
barentswatch=BarentsWatchIntegrationUpdate(),
|
||||
web_search=WebSearchIntegrationUpdate(),
|
||||
ocr=OCRIntegrationUpdate(),
|
||||
)
|
||||
|
||||
async def fake_commit():
|
||||
return None
|
||||
|
||||
async def fake_refresh(_record):
|
||||
return None
|
||||
|
||||
db = SimpleNamespace(
|
||||
add=lambda _record: None,
|
||||
commit=fake_commit,
|
||||
refresh=fake_refresh,
|
||||
)
|
||||
|
||||
await save_external_integrations_payload(db, update)
|
||||
|
||||
assert calls == {"validated": 0, "saved": 1}
|
||||
|
||||
12
backend/tests/test_settings_secret_placeholder.py
Normal file
12
backend/tests/test_settings_secret_placeholder.py
Normal file
@@ -0,0 +1,12 @@
|
||||
from app.api.v1.settings import _is_secret_placeholder
|
||||
|
||||
|
||||
def test_secret_placeholder_treats_masked_values_as_placeholder():
|
||||
assert _is_secret_placeholder("••••••••", "••••1234") is True
|
||||
assert _is_secret_placeholder("********", "") is True
|
||||
assert _is_secret_placeholder(" * * * ", "") is True
|
||||
|
||||
|
||||
def test_secret_placeholder_preserves_real_keys_with_asterisks():
|
||||
assert _is_secret_placeholder("sk-live-*real-key*", "") is False
|
||||
assert _is_secret_placeholder("token_with*embedded*star", "") is False
|
||||
71
backend/tests/test_situational_alert_ai_brief.py
Normal file
71
backend/tests/test_situational_alert_ai_brief.py
Normal file
@@ -0,0 +1,71 @@
|
||||
import pytest
|
||||
|
||||
from app.services.situational_alert_ai_brief import build_situational_alert_brief_request
|
||||
|
||||
|
||||
class _SingleUseScalarResult:
|
||||
def __init__(self, value=0, rows=None):
|
||||
self.value = value
|
||||
self.rows = rows or []
|
||||
self.scalar_calls = 0
|
||||
|
||||
def scalar(self):
|
||||
self.scalar_calls += 1
|
||||
if self.scalar_calls > 1:
|
||||
raise AssertionError("scalar result was consumed more than once")
|
||||
return self.value
|
||||
|
||||
def fetchall(self):
|
||||
return self.rows
|
||||
|
||||
def scalar_one_or_none(self):
|
||||
return None
|
||||
|
||||
def scalars(self):
|
||||
rows = self.rows
|
||||
|
||||
class _Scalars:
|
||||
def all(self):
|
||||
return rows
|
||||
|
||||
return _Scalars()
|
||||
|
||||
|
||||
class _FakeBriefSession:
|
||||
def __init__(self):
|
||||
self._results = [
|
||||
_SingleUseScalarResult(3),
|
||||
_SingleUseScalarResult(2),
|
||||
_SingleUseScalarResult(rows=[]),
|
||||
_SingleUseScalarResult(rows=[]),
|
||||
_SingleUseScalarResult(rows=[]),
|
||||
_SingleUseScalarResult(4),
|
||||
_SingleUseScalarResult(1),
|
||||
_SingleUseScalarResult(rows=[]),
|
||||
_SingleUseScalarResult(rows=[]),
|
||||
_SingleUseScalarResult(5),
|
||||
_SingleUseScalarResult(2),
|
||||
_SingleUseScalarResult(rows=[]),
|
||||
_SingleUseScalarResult(),
|
||||
]
|
||||
|
||||
async def execute(self, _query):
|
||||
return self._results.pop(0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_situational_alert_brief_builder_reuses_counts_without_reconsuming_results(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"app.services.situational_alert_ai_brief.get_latest_bgp_brief_record",
|
||||
lambda: None,
|
||||
)
|
||||
|
||||
request, facts, context = await build_situational_alert_brief_request(_FakeBriefSession())
|
||||
|
||||
assert request.title == "态势告警 AI 简报"
|
||||
assert "总告警 3 条,active 2 条" in facts[0]
|
||||
assert "累计 incidents 4 条,active incidents 1 条" in facts[1]
|
||||
assert "累计 anomalies 5 条,active anomalies 2 条" in facts[2]
|
||||
assert context["active_system_alerts"] == 2
|
||||
assert context["active_bgp_incidents"] == 1
|
||||
assert context["active_bgp_anomalies"] == 2
|
||||
@@ -816,6 +816,27 @@ async def test_visualization_geo_summary_returns_counts(monkeypatch):
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_cable_layers_return_empty_feature_collections():
|
||||
class _ScalarResult:
|
||||
def scalars(self):
|
||||
class _Scalars:
|
||||
def all(self):
|
||||
return []
|
||||
|
||||
return _Scalars()
|
||||
|
||||
class _FakeSession:
|
||||
async def execute(self, _query):
|
||||
return _ScalarResult()
|
||||
|
||||
cables = await visualization_api._build_cables_geojson(_FakeSession())
|
||||
landing_points = await visualization_api._build_landing_points_geojson(_FakeSession())
|
||||
|
||||
assert cables == {"type": "FeatureCollection", "features": []}
|
||||
assert landing_points == {"type": "FeatureCollection", "features": []}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_location_endpoint_returns_candidates_for_known_record(monkeypatch):
|
||||
def _fake_ror(query):
|
||||
|
||||
@@ -46,6 +46,21 @@ def test_web_search_key_prefers_provider_env(isolated_web_search_env_files):
|
||||
assert source == "env_file"
|
||||
|
||||
|
||||
def test_web_search_generic_key_only_applies_to_default_provider(isolated_web_search_env_files):
|
||||
isolated_web_search_env_files.write_text(
|
||||
"WEB_SEARCH_API_KEY=generic-search-key\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
default_value, default_source = _resolve_web_search_api_key("tavily", {"api_key": ""}, "tavily")
|
||||
other_value, other_source = _resolve_web_search_api_key("brave", {"api_key": ""}, "tavily")
|
||||
|
||||
assert default_value == "generic-search-key"
|
||||
assert default_source == "env_file"
|
||||
assert other_value == ""
|
||||
assert other_source == ""
|
||||
|
||||
|
||||
def test_build_web_search_payload_keeps_saved_key_when_preview_submitted():
|
||||
current = {
|
||||
"web_search": {
|
||||
@@ -71,6 +86,35 @@ def test_build_web_search_payload_keeps_saved_key_when_preview_submitted():
|
||||
assert payload["providers"]["tavily"]["api_key"] == "tvly-old-secret"
|
||||
|
||||
|
||||
def test_build_web_search_payload_saves_provider_without_changing_default():
|
||||
current = {
|
||||
"web_search": {
|
||||
"default_provider": "tavily",
|
||||
"providers": {
|
||||
"tavily": {
|
||||
"provider": "tavily",
|
||||
"api_key": "tvly-key",
|
||||
},
|
||||
"brave": {
|
||||
"provider": "brave",
|
||||
"api_key": "",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
update = WebSearchIntegrationUpdate(
|
||||
enabled=True,
|
||||
provider="brave",
|
||||
base_url="https://api.search.brave.com/res/v1/web/search",
|
||||
api_key="brave-new-key",
|
||||
)
|
||||
|
||||
payload = _build_web_search_payload(current, update)
|
||||
|
||||
assert payload["default_provider"] == "tavily"
|
||||
assert payload["providers"]["brave"]["api_key"] == "brave-new-key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tavily_adapter_normalizes_results(monkeypatch):
|
||||
config = WebSearchConfig(
|
||||
|
||||
92
config/earth-boundary-pov-policy.china-v1.json
Normal file
92
config/earth-boundary-pov-policy.china-v1.json
Normal file
@@ -0,0 +1,92 @@
|
||||
{
|
||||
"schema": "planet-earth-boundary-pov-policy/v1",
|
||||
"profile": "china-pov-v1",
|
||||
"description": "Product boundary policy for the China POV Earth boundary build. This file declares intent only; geometry must come from audited source packages and be applied offline before PMTiles/MVT generation.",
|
||||
"defaultCountryHandling": "source-admin0-with-reviewed-overrides",
|
||||
"rules": [
|
||||
{
|
||||
"id": "china-zangnan",
|
||||
"name": "Zangnan / South Tibet",
|
||||
"action": "union_to_country",
|
||||
"targetIsoA3": "CHN",
|
||||
"subtractFromIsoA3": ["IND"],
|
||||
"hoverIsoA3": "CHN",
|
||||
"labelPolicy": "show_country_only"
|
||||
},
|
||||
{
|
||||
"id": "china-aksai-chin",
|
||||
"name": "Aksai Chin",
|
||||
"action": "union_to_country",
|
||||
"targetIsoA3": "CHN",
|
||||
"subtractFromIsoA3": ["IND"],
|
||||
"hoverIsoA3": "CHN",
|
||||
"labelPolicy": "show_country_only"
|
||||
},
|
||||
{
|
||||
"id": "china-taiwan-penghu",
|
||||
"name": "Taiwan and Penghu",
|
||||
"action": "union_to_country",
|
||||
"targetIsoA3": "CHN",
|
||||
"hoverIsoA3": "CHN",
|
||||
"labelPolicy": "show_country_only"
|
||||
},
|
||||
{
|
||||
"id": "china-diaoyu-chiwei",
|
||||
"name": "Diaoyu Dao, affiliated islands, and Chiwei Yu",
|
||||
"action": "union_to_country",
|
||||
"targetIsoA3": "CHN",
|
||||
"hoverIsoA3": "CHN",
|
||||
"labelPolicy": "show_country_only"
|
||||
},
|
||||
{
|
||||
"id": "china-south-china-sea-islands",
|
||||
"name": "Dongsha, Xisha, Zhongsha, Nansha, Huangyan Dao, Zengmu Ansha and related islands/reefs",
|
||||
"action": "union_to_country",
|
||||
"targetIsoA3": "CHN",
|
||||
"hoverIsoA3": "CHN",
|
||||
"labelPolicy": "show_country_only"
|
||||
},
|
||||
{
|
||||
"id": "china-maritime-claim-line",
|
||||
"name": "South China Sea dashed maritime claim line",
|
||||
"action": "render_claim_line",
|
||||
"targetIsoA3": "CHN",
|
||||
"geometryRole": "claim_line_only",
|
||||
"landPolygonEffect": "none"
|
||||
},
|
||||
{
|
||||
"id": "kosovo",
|
||||
"name": "Kosovo",
|
||||
"action": "render_as_disputed_with_parent",
|
||||
"parentIsoA3": "SRB",
|
||||
"hoverIsoA3": "SRB",
|
||||
"boundaryStyle": "disputed_internal",
|
||||
"labelPolicy": "show_parent_country"
|
||||
},
|
||||
{
|
||||
"id": "gaza",
|
||||
"name": "Gaza Strip",
|
||||
"action": "render_as_region_of_country",
|
||||
"targetIsoA3": "PSE",
|
||||
"hoverIsoA3": "PSE",
|
||||
"boundaryStyle": "admin_or_disputed",
|
||||
"labelPolicy": "show_country_only"
|
||||
}
|
||||
],
|
||||
"sourceRequirements": {
|
||||
"geometryMustBeAudited": true,
|
||||
"noHandDrawnClaimLines": true,
|
||||
"noFrontendRuntimePovPatch": true,
|
||||
"artifactIsolation": "one PMTiles/MVT artifact per POV profile"
|
||||
},
|
||||
"officialPositionNotes": [
|
||||
{
|
||||
"id": "kosovo",
|
||||
"note": "China has emphasized respect for Serbia's sovereignty and territorial integrity and the framework of UNSC Resolution 1244."
|
||||
},
|
||||
{
|
||||
"id": "gaza",
|
||||
"note": "China supports the two-state solution and an independent State of Palestine based on the 1967 borders with East Jerusalem as its capital; Gaza governance should follow Palestinians governing Palestine."
|
||||
}
|
||||
]
|
||||
}
|
||||
112
config/earth-boundary-sources.example.json
Normal file
112
config/earth-boundary-sources.example.json
Normal file
@@ -0,0 +1,112 @@
|
||||
{
|
||||
"policy": {
|
||||
"runtimeFetch": false,
|
||||
"profile": "china-pov-v1",
|
||||
"povPolicyPath": "config/earth-boundary-pov-policy.china-v1.json",
|
||||
"productionTileFormat": "pmtiles+mvt",
|
||||
"debugTileFormat": "geojson-directory",
|
||||
"description": "Default Earth boundary update sources. These public Natural Earth endpoints make local high-precision boundary download work out of the box; replace with audited internal sources for production if needed."
|
||||
},
|
||||
"collectorConfigs": {
|
||||
"earth_admin0_boundaries": {
|
||||
"displayName": "Earth Admin-0 国界源",
|
||||
"sourceKind": "admin0-boundaries",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_countries.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
"mapping_json": {
|
||||
"source": {
|
||||
"items_path": "$.features[*]"
|
||||
},
|
||||
"fields": {
|
||||
"source_id": {
|
||||
"path": "$.properties.id",
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"path": "$.properties.name",
|
||||
"type": "string"
|
||||
},
|
||||
"geometry": {
|
||||
"path": "$.geometry",
|
||||
"type": "object"
|
||||
},
|
||||
"properties": {
|
||||
"path": "$.properties",
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"earth_coastline": {
|
||||
"displayName": "Earth 海岸线源",
|
||||
"sourceKind": "coastline",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_coastline.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
"mapping_json": {
|
||||
"source": {
|
||||
"items_path": "$.features[*]"
|
||||
},
|
||||
"fields": {
|
||||
"source_id": {
|
||||
"path": "$.properties.id",
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"path": "$.properties.name",
|
||||
"type": "string"
|
||||
},
|
||||
"geometry": {
|
||||
"path": "$.geometry",
|
||||
"type": "object"
|
||||
},
|
||||
"properties": {
|
||||
"path": "$.properties",
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"earth_claim_lines": {
|
||||
"displayName": "Earth 主张线源",
|
||||
"sourceKind": "claim-lines",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_boundary_lines_disputed_areas.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
"mapping_json": {
|
||||
"source": {
|
||||
"items_path": "$.features[*]"
|
||||
},
|
||||
"fields": {
|
||||
"source_id": {
|
||||
"path": "$.properties.id",
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"path": "$.properties.name",
|
||||
"type": "string"
|
||||
},
|
||||
"geometry": {
|
||||
"path": "$.geometry",
|
||||
"type": "object"
|
||||
},
|
||||
"properties": {
|
||||
"path": "$.properties",
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"notes": [
|
||||
"Earth can download these sources directly from the toolbar settings when no local source override exists.",
|
||||
"If tippecanoe/pmtiles are unavailable, the backend generates a GeoJSON high-precision package so the feature remains usable."
|
||||
]
|
||||
}
|
||||
@@ -15,12 +15,16 @@ services:
|
||||
retries: 10
|
||||
|
||||
aiprovider:
|
||||
image: ${AI_PROVIDER_IMAGE_NAME:-planet-aiprovider:latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: aiprovider/Dockerfile
|
||||
args:
|
||||
PYTHON_IMAGE: ${PYTHON_IMAGE:-python:3.14-slim}
|
||||
UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:latest}
|
||||
AI_PROVIDER_BUILD_FINGERPRINT: ${AI_PROVIDER_BUILD_FINGERPRINT:-unknown}
|
||||
secrets:
|
||||
- planet_uv_config
|
||||
container_name: planet_aiprovider
|
||||
ports:
|
||||
- "8010:8010"
|
||||
@@ -43,3 +47,7 @@ services:
|
||||
|
||||
volumes:
|
||||
ollama_data:
|
||||
|
||||
secrets:
|
||||
planet_uv_config:
|
||||
file: ${PLANET_UV_CONFIG_FILE:-/dev/null}
|
||||
|
||||
@@ -2,12 +2,16 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
aiprovider:
|
||||
image: ${AI_PROVIDER_IMAGE_NAME:-planet-aiprovider:latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: aiprovider/Dockerfile
|
||||
args:
|
||||
PYTHON_IMAGE: ${PYTHON_IMAGE:-python:3.14-slim}
|
||||
UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:latest}
|
||||
AI_PROVIDER_BUILD_FINGERPRINT: ${AI_PROVIDER_BUILD_FINGERPRINT:-unknown}
|
||||
secrets:
|
||||
- planet_uv_config
|
||||
env_file:
|
||||
- ./aiprovider/.env
|
||||
- ${PLANET_AI_PROVIDER_RUNTIME_ENV_FILE:-./aiprovider/.env}
|
||||
@@ -43,3 +47,7 @@ services:
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
|
||||
secrets:
|
||||
planet_uv_config:
|
||||
file: ${PLANET_UV_CONFIG_FILE:-/dev/null}
|
||||
|
||||
@@ -2,12 +2,16 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
aiprovider:
|
||||
image: ${AI_PROVIDER_IMAGE_NAME:-planet-aiprovider:latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: aiprovider/Dockerfile
|
||||
args:
|
||||
PYTHON_IMAGE: ${PYTHON_IMAGE:-python:3.14-slim}
|
||||
UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:latest}
|
||||
AI_PROVIDER_BUILD_FINGERPRINT: ${AI_PROVIDER_BUILD_FINGERPRINT:-unknown}
|
||||
secrets:
|
||||
- planet_uv_config
|
||||
env_file:
|
||||
- ./aiprovider/.env
|
||||
- ${PLANET_AI_PROVIDER_RUNTIME_ENV_FILE:-./aiprovider/.env}
|
||||
@@ -53,3 +57,7 @@ services:
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
|
||||
secrets:
|
||||
planet_uv_config:
|
||||
file: ${PLANET_UV_CONFIG_FILE:-/dev/null}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user