Compare commits

..
Author SHA1 Message Date
rayd1o 455b8360d0 release: bump version to 0.50.0 2026-05-10 22:06:01 +08:00
linkongandClaude Sonnet 4.6 e1984c7a35 release: bump version to 0.49.0
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-08 17:42:27 +08:00
linkong bb9183b8a4 release: bump version to 0.48.0 2026-05-07 18:06:06 +08:00
linkong 421234301a release: bump version to 0.47.0 2026-04-30 16:56:37 +08:00
linkong f22079d33a release: bump version to 0.46.3 2026-04-30 14:46:19 +08:00
linkong 9f737fdb89 release: bump version to 0.46.2 2026-04-30 14:30:12 +08:00
linkong 7418ce2fc1 release: bump version to 0.46.1 2026-04-30 09:41:08 +08:00
rayd1o b1a5934b80 release: bump version to 0.46.0 2026-04-30 04:42:29 +08:00
rayd1o ba54545ac7 release: bump version to 0.45.0 2026-04-29 23:43:54 +08:00
linkong 9dafbf4f6e release: bump version to 0.44.2 2026-04-29 18:11:37 +08:00
linkong a87537e903 release: bump version to 0.44.1 2026-04-29 18:07:35 +08:00
linkong 87594a95ff release: bump version to 0.44.0 2026-04-29 17:27:44 +08:00
linkong 2da25376bd release: bump version to 0.43.1 2026-04-28 16:21:33 +08:00
linkong ac69d5d354 release: bump version to 0.43.0 2026-04-28 16:10:17 +08:00
rayd1o 1cd2dab0ee release: bump version to 0.42.2 2026-04-28 04:35:13 +08:00
rayd1o 42d019af36 release: bump version to 0.42.1 2026-04-28 04:29:44 +08:00
rayd1o b4e8afb272 release: bump version to 0.42.0 2026-04-28 04:27:18 +08:00
rayd1o eeee788530 release: bump version to 0.41.2 2026-04-27 23:23:23 +08:00
linkongandClaude Sonnet 4.6 655e2a7d2d release: bump version to 0.41.1
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-27 16:31:34 +08:00
linkong 3ea99a9529 release: bump version to 0.41.0 2026-04-27 13:58:29 +08:00
rayd1oandClaude Sonnet 4.6 f9c1334365 release: bump version to 0.40.5
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-26 05:03:30 +08:00
rayd1o 5f47ec1659 release: bump version to 0.40.4 2026-04-26 01:41:29 +08:00
rayd1o 229be0bced release: bump version to 0.40.3 2026-04-25 23:02:22 +08:00
linkong 50a417ca83 release: bump version to 0.40.2 2026-04-24 17:50:43 +08:00
linkong e9464a9833 release: bump version to 0.40.1 2026-04-24 17:28:03 +08:00
linkong 86807f6af6 release: bump version to 0.40.0 2026-04-24 15:41:42 +08:00
rayd1o 8b8f7138c0 release: bump version to 0.39.0 2026-04-24 00:48:33 +08:00
linkong d5f3784ffb release: bump version to 0.38.0 2026-04-23 17:57:35 +08:00
linkong 195a8bf71c release: bump version to 0.37.2 2026-04-23 11:12:49 +08:00
linkong 987c378f99 release: bump version to 0.37.1 2026-04-23 11:05:30 +08:00
rayd1o 67f82dc41c release: bump version to 0.37.0 2026-04-23 07:56:10 +08:00
rayd1o abe04030fb release: bump version to 0.36.0 2026-04-22 23:42:10 +08:00
linkong 6a5f9f7ad4 release: bump version to 0.35.1 2026-04-22 18:04:54 +08:00
linkong 439a512148 docs: add earth mobile drawer UI plan and Claude Code/Codex toolchain
- Add earth-mobile-drawer-ui-plan documenting mobile drawer UX decisions
- Add goal-driven.md Claude Code command for autonomous task execution
- Add .codex/ config with OpenAI model definitions and goal-driven agent
- Add SKILL.md, openai.yaml, and prompt-template for Codex integration
2026-04-22 17:37:00 +08:00
linkong f73fa1ea6d release: bump version to 0.35.0 2026-04-22 17:29:24 +08:00
linkong 5b623a6385 release: bump version to 0.34.0 2026-04-22 12:49:37 +08:00
rayd1o 0082cf3fbd release: bump version to 0.33.0 2026-04-22 05:28:54 +08:00
rayd1o 3ae4acdff8 release: bump version to 0.32.0 2026-04-22 04:41:39 +08:00
rayd1o 437efc848c release: bump version to 0.31.3 2026-04-22 03:52:09 +08:00
rayd1o 003a46ac30 release: bump version to 0.31.2 2026-04-21 23:50:35 +08:00
rayd1o 4b0be4cb76 release: bump version to 0.31.1 2026-04-21 22:49:39 +08:00
linkong b7647379de release: bump version to 0.31.0 2026-04-21 18:35:40 +08:00
linkong 0f89372d71 release: bump version to 0.30.0 2026-04-21 12:28:04 +08:00
linkong 2b0d4cfc49 release: bump version to 0.29.2 2026-04-21 10:43:48 +08:00
rayd1o e6d0332fba release: bump version to 0.29.1 2026-04-20 22:12:19 +08:00
linkong fe45a99cbd release: bump version to 0.29.0 2026-04-20 17:43:59 +08:00
linkong ae77b06c3c docs: expand earth celestial background implementation plan 2026-04-20 16:29:03 +08:00
linkong b5dd4f12f8 release: bump version to 0.28.2 2026-04-20 16:00:54 +08:00
linkong 75cb214f23 release: bump version to 0.28.1 2026-04-20 15:14:53 +08:00
linkong 4c21973197 release: bump version to 0.28.0 2026-04-20 14:10:27 +08:00
linkong 51ae5e6ec9 release: bump version to 0.27.8 2026-04-20 12:03:09 +08:00
linkong 1cf1f32ddd feat: refine earth hud panel behaviors and news board 2026-04-20 11:23:05 +08:00
linkong 8f3ab88743 release: bump version to 0.27.7 2026-04-16 10:04:14 +08:00
rayd1o f8b43a995b release: bump version to 0.27.6 2026-04-15 07:37:41 +08:00
linkong d9adaf4134 release: bump version to 0.27.5 2026-04-14 17:34:04 +08:00
linkongandClaude Sonnet 4.6 40e51d5b20 release: bump version to 0.27.4
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-14 13:11:39 +08:00
linkongandClaude Sonnet 4.6 93c1c1e550 release: bump version to 0.27.3
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-14 12:52:05 +08:00
linkongandClaude Sonnet 4.6 48eb13b993 release: bump version to 0.27.2
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-14 10:55:04 +08:00
linkong 11179e7e67 release: bump version to 0.27.1 2026-04-14 10:22:05 +08:00
rayd1oandClaude Sonnet 4.6 07e26d6d5a chore: add cleanup and release skills for claude and codex
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-14 07:46:19 +08:00
rayd1oandClaude Sonnet 4.6 7cd29cf9c0 release: bump version to 0.27.0
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-14 07:37:45 +08:00
rayd1o 2ee4773f4f release: bump version to 0.26.2 2026-04-12 09:27:01 +08:00
rayd1o b1d0624061 release: bump version to 0.26.1 2026-04-12 05:20:23 +08:00
rayd1o 812c825dc6 release: bump version to 0.26.0 2026-04-12 04:36:38 +08:00
rayd1o a359d94127 refactor: clean up earth brand component wiring 2026-04-11 03:58:28 +08:00
rayd1o c92be9c054 chore: remove unused earth icon assets 2026-04-11 03:46:59 +08:00
rayd1o 10e2bae8c2 release: bump version to 0.25.3 2026-04-11 03:42:35 +08:00
rayd1o 60ed88b609 release: bump version to 0.25.2 2026-04-10 23:43:56 +08:00
linkong e85a9fc614 fix: refine earth settings modal and hud interactions 2026-04-10 22:53:09 +08:00
linkong a2210f0f78 release: bump version to 0.25.1 2026-04-10 16:19:57 +08:00
linkong 62ad09e816 release: bump version to 0.25.0 2026-04-10 16:04:50 +08:00
linkong 89a71e6f29 feat: ship persistent ai playground and alerts foundation 2026-04-10 15:57:34 +08:00
rayd1o 60f5ff9bab fix: move bgp brief markdown into a modal workspace 2026-04-10 03:57:13 +08:00
rayd1o fbb6adfbf5 docs: formalize release workflow and frontend layout guardrails 2026-04-10 03:08:39 +08:00
rayd1o 749e6e76b6 fix: optimize visualization hot paths and refine bgp brief workspace 2026-04-10 02:12:29 +08:00
rayd1o 83839b8b11 fix: ship persistent bgp ai briefs and optimize bgp queries 2026-04-10 00:33:01 +08:00
rayd1o ed898aef9c docs: add docker compose and buildx upgrade guide
- add a standalone docs guide for reinstalling Docker, Compose, and Buildx from the official Ubuntu repository
- refine planet.sh compose/build diagnostics so docker compose is the primary wording and missing buildx / fallback cases report more accurately
- keep the startup failure output aligned with the existing terminal tone while reducing misleading compose-version messaging
2026-04-09 23:08:31 +08:00
linkong abe0b5c11b fix: polish ai provider rebuild feedback 2026-04-09 16:52:21 +08:00
linkong 306ba7f850 fix: expand playground diagnostics and compose fallback 2026-04-09 16:38:55 +08:00
linkong 39f90bd575 fix: polish playground routing and bundle splits 2026-04-09 15:23:10 +08:00
rayd1o c4ea918fac fix: refine earth hud structure and bun startup flow
- refactor the /earth HUD into class-first CSS layers with dedicated base, hud, and toolbar responsibilities
- clean up Earth markup and runtime DOM hooks so shared panel, toolbar, tooltip, and status classes stay consistent after dynamic updates
- keep HUD scaling configurable through extracted constants and remove leftover legacy panel/toolbar styling paths
- make planet.sh self-bootstrap Bun and uv by prepending local runtime bins to PATH and auto-installing missing tools in fresh environments
- document Bun as the frontend package manager of record and sync repository version metadata to 0.24.1
2026-04-09 02:12:22 +08:00
rayd1o 34d94a6b6b Redirect login success to admin 2026-04-09 00:37:02 +08:00
rayd1o ef65acd49c Add AI playground and frontend layout guidelines 2026-04-09 00:35:38 +08:00
linkong 5639546990 fix: stabilize bgp overview table viewport 2026-04-08 17:48:32 +08:00
linkong c8fe8cad59 fix: refine planet startup script UX 2026-04-08 16:54:45 +08:00
linkong d395769df6 docs: add agent runtime planning docs 2026-04-08 12:49:49 +08:00
linkong 8bd9d34376 feat: configure collector endpoints and health plan 2026-04-08 10:15:33 +08:00
linkong 2d43263b9e fix: tighten datasource trigger guards 2026-04-08 09:33:48 +08:00
rayd1o 2da6ed166b Bump version to 0.23.2 2026-04-08 03:24:08 +08:00
rayd1o da587398d9 Refine datasource trigger flow and CLI output 2026-04-08 03:22:08 +08:00
rayd1o f5308340af Add force rerun recovery and polish CLI startup 2026-04-08 02:49:29 +08:00
rayd1o 981617ee80 Bump version to 0.23.1 2026-04-07 23:51:44 +08:00
rayd1o 7abf391c74 Harden startup retry and recovery flow 2026-04-07 23:25:13 +08:00
linkong f12719914d feat: align aiprovider with adapter-based compatibility 2026-04-07 17:54:44 +08:00
linkong bc90e00e25 Merge branch 'codex/aiprovider-foundation' into dev 2026-04-07 17:33:37 +08:00
404 changed files with 95731 additions and 5798 deletions

No files matched your search

+139
View File
@@ -0,0 +1,139 @@
---
description: 审查当前工作区未提交代码中的垃圾代码,并在不影响逻辑的前提下自动清理
argument-hint: 可选:指定要检查的文件或目录(默认检查所有未提交修改)
allowed-tools: ["Read", "Edit", "Bash", "Grep", "Glob"]
---
# /cleanup — 垃圾代码审查与清理
分析当前工作区(git diff)中的未提交代码,找出并修复常见垃圾代码,**不得改变任何运行逻辑**。
## 检查范围
若 `$ARGUMENTS` 非空,则只检查指定文件/目录;否则检查所有未提交修改(`git diff HEAD`)。
## 节省上下文规则
优先用确定性的 CLI 检查缩小范围,不要一上来把完整文件或大 diff 读入上下文:
```bash
git diff --name-only HEAD
git diff --unified=0 HEAD -- <path>
git diff --check
rg -n "TODO|FIXME|console\.log|debugger|print\(" <changed-paths>
```
只有 focused diff 不足以安全判断或修改时,才读取完整文件。
## 审查清单
按优先级检查以下问题(只报告在本次 diff 中**新增或修改**的代码里存在的问题):
### 1. 重复逻辑 (Duplicate Logic)
- 完全相同或高度相似的代码块在多处出现
- 同一函数/方法被多个地方各自实现,已有公共版本未被复用
- 相同的 DOM 查询、正则、模板字符串在同一文件重复
### 2. Magic Numbers / Magic Strings
- 裸数字直接参与计算(如偏移量、时间、尺寸、阈值),没有命名常量
- 硬编码字符串(如 id 名、状态值、URL 片段)散落在逻辑中
- 例外:`0`, `1`, `-1`, `100`, `""` 等语义明确的惯用值不算
### 3. 命名问题
- 含义不明的缩写变量(如 `or_`, `tmp2`, `x2`)
- 命名与实际用途不符
- 同一概念在不同地方用不同名字表达
### 4. 死代码 / 无效代码
- 注释掉的旧代码块(3行以上)
- 声明后从未使用的变量/参数/导入
- 永远不会执行的条件分支
### 5. 代码风格问题
- 尾部空白字符(trailing whitespace)
- 同一文件内风格不一致(如混用单双引号、缩进不统一)
- 空行使用不一致(连续多个空行等)
### 6. 其他常见问题
- 私有辅助函数应被 export 但没有,导致调用方重复实现
- 类型/接口重复定义
- 过于冗长的条件表达式可以简化(不改逻辑)
## 执行步骤
### Step 1 — 获取待检查文件列表
```bash
# 无参数时:获取所有未提交修改
git diff HEAD --name-only
# 有参数时:用 $ARGUMENTS 过滤
```
### Step 2 — 逐文件阅读并分析
先从 focused diff 开始:
```bash
git diff --unified=0 HEAD -- <file>
```
用 `rg`、`git diff --check`、编译器或 linter 输出确认确定性问题。只有需要上下文时才用 Read 读取完整文件。对照审查清单,记录每个问题:文件名、行号、问题类型、建议修复方式。
### Step 3 — 报告问题清单
在修改前,先以列表形式输出所有发现的问题:
```
发现 N 个问题:
[文件] js/foo.js
· L34, L78: 重复逻辑 — 两处都实现了相同的 DOM 查询,可提取到 getPanel()
· L91: Magic number — 硬编码 14 作为偏移量,应命名为 TOOLTIP_OFFSET
[文件] js/bar.js
· L12: 命名问题 — 变量 `or_` 语义不明,应命名为 outerR/outerG/outerB
...
```
如果没有发现问题,直接输出"未发现垃圾代码,当前代码质量良好。"并停止。
### Step 4 — 执行修复
对每个问题,使用 Edit 工具进行**最小化修改**:
- **重复逻辑**:提取为共享常量/函数,更新所有调用点
- **Magic number**:在文件顶部或逻辑附近声明 `const NAME = value`,替换所有引用
- **命名问题**:重命名变量,更新所有使用处
- **死代码**:直接删除
- **尾部空白/风格**:修正
- **未 export 的函数**:添加 `export`,在调用方改为导入(不重复实现)
**修复原则:**
- 只改在审查清单中发现的问题,不做额外优化
- 每次 Edit 只修改确实有问题的行,保持 diff 最小
- 改完后用 `grep` 验证旧的坏代码已消失
- 优先做精确补丁;只有仓库已有对应格式化流程时,才运行格式化工具
### Step 5 — 输出总结
```
清理完成:
修复了 N 个问题:
✓ earth.js — 提取重复 vertexShader 为 ATMOS_VERTEX_SHADER 常量
✓ main.js — 提取 TOOLTIP_CURSOR_OFFSET = 14(4处引用)
✓ controls.js — export updateLayerButtonState,移除 main.js 中的重复实现
...
未修改的问题(需人工确认):
! foo.js L45 — 注释代码块较长,建议手动确认是否可删除
```
## 约束
- **禁止**改变函数签名、接口定义、导出 API(除非问题正是私有函数应被 export)
- **禁止**添加新功能、新抽象、新参数
- **禁止**修改注释内容(只删除注释掉的死代码)
- **禁止**修改测试文件逻辑
- 如果一个 Magic number 的语义不完全确定,**跳过**,在总结中标记为"需人工确认"
+93
View File
@@ -0,0 +1,93 @@
---
description: Create or update repository documentation from current code changes
argument-hint: Optional: topic to document, or leave empty to infer from git diff
allowed-tools: ["Read", "Edit", "Write", "Bash", "Glob", "Grep"]
---
# /docs — Documentation Workflow
## Goal
Create or update documentation that explains why a change exists, how it behaves, and what maintainers need to know. Keep this command generic. Repository-specific coverage rules live in the repository and must be loaded separately.
## Repository Rules
Before deciding scope, check whether the repository has a documentation rules file:
```bash
test -f docs/documentation-coverage-rules.md && sed -n '1,240p' docs/documentation-coverage-rules.md
```
If it exists, apply it as the project-specific coverage checklist. If it does not exist, continue with the generic workflow below.
## Workflow
### Step 1 — Understand The Change
```bash
git diff HEAD --stat
git diff HEAD --name-only
git log --oneline -10
rg --files docs
```
If `$ARGUMENTS` specifies a topic, focus on that topic. Otherwise infer the documentation topic from the changed files. Do not read the full repository diff by default; inspect focused files only:
```bash
git diff HEAD -- <path>
rg -n "class |def |function |export |router|@router|interface |type " <path>
```
### Step 2 — Decide Scope
- Prefer updating an existing relevant document over creating a duplicate.
- Use one document for one coherent topic.
- Split documents only when the change crosses meaningful domains.
- Keep filenames lowercase and hyphenated.
- Apply the repository-specific rules file before writing.
For ambiguous or large documentation changes, briefly state the intended doc plan before editing. For clear small changes, proceed directly.
### Step 3 — Write
Explain:
- Background/problem: what was wrong or missing before.
- Core design decisions and rationale.
- Operational or user-facing impact.
- Relevant code paths, only when useful for future maintainers.
Style:
- Follow the repository’s existing language and heading conventions.
- Use fenced code blocks with language tags.
- Prefer tables for comparisons or parameter lists.
- Keep snippets concise and relevant.
### Step 4 — Verify
- Read the completed docs once for clarity and stale statements.
- Verify referenced paths exist with `test -e` or `rg --files`.
- Run applicable checks from `docs/documentation-coverage-rules.md`.
- Check Markdown links use readable user-facing titles unless repository rules allow otherwise.
### Step 5 — Report
Summarize changed docs and verification:
```md
Updated:
- path/to/doc.md — what changed
Verified:
- checks that passed
- checks that could not be run, if any
```
## Hard Constraints
- Do not leave placeholder docs.
- Do not duplicate bilingual files byte-for-byte.
- Do not reference PR numbers, issue numbers, or the current conversation unless explicitly requested.
- Do not write changelog-style lists without the reasoning and tradeoffs behind the change.
- Keep docs maintainable and concise.
+93
View File
@@ -0,0 +1,93 @@
---
description: 用 goal-driven 方法推动一个复杂任务持续执行,直到明确成功标准被满足
argument-hint: 建议填写任务目标;若同时给出成功标准更好
allowed-tools: ["Read", "Edit", "Bash", "Grep", "Glob"]
---
# /goal-driven — 目标驱动执行模式
使用 `lidangzzz/goal-driven` 的核心思想来推进复杂任务:先固定目标与成功标准,再持续执行和反复验收,直到标准真正满足。
适用场景:
- 长周期实现任务
- 高复杂度工程任务
- 可被明确验收的研究、实现、迁移、验证类工作
不适用场景:
- 纯脑暴
- 无法定义成功标准的模糊任务
- 很小的一次性修改
## 输入要求
若 `$ARGUMENTS` 只包含目标,没有成功标准,先补全一版可执行的成功标准再开始。
启动时先输出:
```md
Goal
- ...
Criteria for success
- ...
Plan
1. ...
2. ...
3. ...
Verification
- ...
```
## 执行规则
1. 先把任务固化为两个核心块:
- `Goal`
- `Criteria for success`
2. 成功标准必须尽量客观,可验证,可落地。
优先写成:
- 需要交付什么
- 需要通过哪些测试或验证
- 如何判断结果真的完成
3. 进入持续执行循环:
- 完成一个阶段
- 检查当前结果是否满足成功标准
- 若未满足,明确剩余差距并继续推进
4. 任何“完成了”“差不多了”“已实现”之类的结论,都必须经过验证,不能直接接受。
5. 如果验证失败:
- 明确指出哪条成功标准没满足
- 继续工作,不要把阶段性进展误判为完成
6. 只有在以下情况之一才能停止:
- 成功标准已满足
- 用户明确要求停止
## 执行风格
- 重证据,轻口头判断
- 优先使用确定性工具证据:`rg`、`git diff --stat`、`git diff -- <path>`、测试、构建、lint、`curl`、数据库查询等能直接证明成功标准的方式
- 不把大段命令输出粘进回复;保留在工具调用里,回复只总结关键证据
- 重验收,轻自我感觉
- 优先用测试、日志、产物、对比结果来证明完成
- 对长期任务保持“未达标就继续”的节奏
## 简版模板
```md
Goal: [[[[[在此填写最终目标]]]]]
Criteria for success: [[[[[在此填写成功标准]]]]]
循环执行:
1. 推进任务
2. 检查是否满足成功标准
3. 若未满足,继续工作
4. 直到满足标准或用户明确停止
```
+160
View File
@@ -0,0 +1,160 @@
---
description: 发版工作流:根据变更类型决定版本号,更新所有版本文件和 changelog,运行验证,commit 并 push
argument-hint: 可选:feature | bugfix | 或直接描述本次发布内容
allowed-tools: ["Read", "Edit", "Bash", "Glob", "Grep"]
---
# /release — Planet 发版工作流
## 版本号规则
| 变更类型 | 版本跳动 | 适用场景 |
|---------|---------|---------|
| `feature` | `+0.1.0` | 纯新功能,无 bugfix |
| `improvement` | `+0.0.1` | UI 调整、小功能增强、bugfix 混合,或以 UI/体验改进为主的迭代 |
| `bugfix` | `+0.0.1` | 纯 bug 修复,无新功能 |
| `docs` / `maintenance` / `refactor` | 默认不发版,除非用户明确要求 |
意图混合时以用户明确描述为准;bugfix + 小 feature 混合默认判定为 `improvement`(`+0.0.1`)。
## 必须同步更新的文件
使用 `git rev-parse --show-toplevel` 获取仓库根目录,以下路径均相对于根目录:
- `VERSION`
- `frontend/package.json`(`"version"` 字段)
- `pyproject.toml`(`version =` 字段)
- `uv.lock`(**不要手动编辑**,通过 `uv lock` 重新生成)
- `docs/CHANGELOG.md`
- `docs/version-history.md`
## 节省上下文规则
发版判断应以确定性 CLI 证据为主,优先使用紧凑命令和定点读取:
```bash
git status --short
git diff --stat HEAD
git diff --name-only HEAD
rg -n "version|^## |^Released:|当前开发版本|current" VERSION frontend/package.json pyproject.toml docs/CHANGELOG.md docs/version-history.md
```
除非需要判断某个代码变更是否属于本次发版,否则不要读取完整 diff。
## 执行步骤
### Step 1 — 环境检查
```bash
git branch --show-current # 确认在 dev 分支
git status --short # 检查是否有无关的未暂存修改
cat VERSION # 读取当前版本
```
若当前**不在 `dev` 分支**,停下来告知用户,不要继续。
若存在无关的未暂存修改,列出并询问用户是否一并提交,或先 stash。
### Step 2 — 确定发版类型与新版本号
- 若 `$ARGUMENTS` 提供了明确类型(`feature` / `bugfix`),直接使用
- 否则根据 `git diff --stat HEAD`、`git diff --name-only HEAD`、必要的 focused diff 和 `git log` 推断
- 计算新版本号(例:`0.26.2` → bugfix → `0.26.3`)
- **先输出发版计划供用户确认**:
```
发版计划:
类型:bugfix
版本:0.26.2 → 0.26.3
分支:dev
将更新:VERSION, frontend/package.json, pyproject.toml, uv.lock, CHANGELOG.md, version-history.md
```
### Step 3 — 更新版本号文件
按顺序更新(每步用 Edit 工具,精确替换,不要重写整个文件):
1. `VERSION` — 直接替换全部内容为新版本号
2. `frontend/package.json` — 替换 `"version": "x.x.x"` 行
3. `pyproject.toml` — 替换 `version = "x.x.x"` 行
4. 运行 `uv lock` 重新生成 `uv.lock`(在仓库根目录下执行)
### Step 4 — 更新 CHANGELOG.md
在文件顶部插入新条目,格式:
```markdown
## [x.x.x] — YYYY-MM-DD
### ✨ Features / 🐛 Fixes / 🔧 Improvements
- ...(只列高信号条目,最多 5 条)
- ...
---
```
日期使用 `date +%Y-%m-%d` 获取今天的日期。
### Step 5 — 更新 docs/version-history.md
- 更新文件头部的"当前开发版本"字段
- 在时间线表格顶部插入新行:`| vx.x.x | YYYY-MM-DD | 一句话摘要 |`
### Step 6 — 验证
针对本次变更范围做最小验证:
- Python 文件有修改:先用 `git diff --name-only HEAD -- '*.py'` 列出,再运行 `python3 -m py_compile <changed_files>`
- Frontend 文件有修改:先用 `git diff --name-only HEAD -- frontend` 判断范围,再运行项目标准检查(若无则跳过并说明)
- 版本号一致性检查:用 grep 确认 VERSION、package.json、pyproject.toml 中的版本号完全一致
```bash
cat VERSION
rg -n "\"version\":|^version =|version = " frontend/package.json pyproject.toml uv.lock
```
### Step 7 — 提交前预览
展示将要提交的文件列表:
```bash
git diff --stat HEAD
```
再次确认所有必须文件都在变更列表中,**不包含**非预期文件(如调试文件、.env 等)。
### Step 8 — Commit & Push(用户确认后)
```bash
git add VERSION frontend/package.json pyproject.toml uv.lock docs/CHANGELOG.md docs/version-history.md
# 若有代码变更也一并 stage
git add <code_files>
git commit -m "release: bump version to x.x.x"
git tag vx.x.x
git push origin dev
git push origin vx.x.x
```
commit message 固定格式:`release: bump version to x.x.x`
### Step 9 — 完成确认
输出摘要:
```
✓ 版本号已更新:0.26.2 → 0.26.3
✓ CHANGELOG 已更新
✓ version-history 已更新
✓ uv.lock 已重新生成
✓ 验证通过
✓ commit: release: bump version to 0.26.3
✓ tag: v0.26.3
✓ 已 push 到 origin/dev
```
## 注意事项
- `uv.lock` 只能通过 `uv lock` 生成,绝不手动编辑
- 发版 commit 只包含版本文件 + 本次功能代码,不混入无关改动
- 若环境中 `uv` 不可用,说明原因并跳过 lockfile 更新,提醒用户手动运行
+3
View File
@@ -0,0 +1,3 @@
approval_policy = "never"
sandbox_mode = "danger-full-access"
+144
View File
@@ -0,0 +1,144 @@
---
name: cleanup
description: Use when the user asks to clean up, lint, or review uncommitted code for common code smells — duplicate logic, magic numbers, unclear naming, dead code, style inconsistencies. Fixes issues without changing any runtime behavior.
---
# Cleanup
Review and fix code quality issues in the current working tree without altering any logic or behavior.
## When To Use
- The user asks to clean up, tidy, or lint uncommitted changes
- The user wants a code smell review before releasing or committing
- The user mentions magic numbers, duplicate logic, dead code, or naming issues
Do not refactor architecture, add features, or change behavior.
## Scope
If the user specifies a file or directory, check only that. Otherwise check all uncommitted changes (`git diff HEAD`).
Only report issues present in **newly added or modified** lines of this diff — do not audit unchanged code.
## Token-Saving Rule
Prefer deterministic CLI checks before reading files into model context:
```bash
git diff --name-only HEAD
git diff --unified=0 HEAD -- <path>
git diff --check
rg -n "TODO|FIXME|console\.log|debugger|print\(" <changed-paths>
```
Read full files only when the focused diff does not provide enough surrounding context to make a safe edit.
## Checklist
### 1. Duplicate Logic
- Identical or near-identical code blocks appearing in multiple places
- A function/helper that already exists but is re-implemented elsewhere instead of being reused
- Repeated DOM queries, regex literals, or template strings within the same file
### 2. Magic Numbers / Magic Strings
- Bare numeric literals used in calculations (offsets, timeouts, sizes, thresholds) without a named constant
- Hardcoded strings (IDs, status values, URL fragments) scattered through logic
- Exceptions: `0`, `1`, `-1`, `100`, `""` and other idiomatically clear values are fine
### 3. Naming Issues
- Cryptic abbreviations (`or_`, `tmp2`, `x2`)
- Names that do not match actual behavior
- The same concept referred to by different names in different places
### 4. Dead Code
- Commented-out code blocks (3+ lines)
- Variables, parameters, or imports declared but never used
- Branches that can never execute
### 5. Style Inconsistencies
- Trailing whitespace
- Mixed quote styles or indentation within the same file
- Inconsistent blank-line usage (multiple consecutive blank lines, etc.)
### 6. Other
- Private helper functions that should be exported but are not, causing callers to duplicate the implementation
- Overly verbose conditions that can be simplified without changing logic
## Steps
### Step 1 — Get the file list
```bash
git diff HEAD --name-only
```
Filter to the user-specified path if one was provided.
### Step 2 — Read and analyze each file
Start with focused diffs:
```bash
git diff --unified=0 HEAD -- <file>
```
Use `rg`, `git diff --check`, and compiler/linter output for deterministic findings. Read the full file only for files that need surrounding context. For each issue found, record filename, line number, category, and suggested fix.
### Step 3 — Report findings before touching anything
Print a structured list:
```
Found N issues:
[file] js/foo.js
· L34, L78: Duplicate logic — same DOM query implemented twice; extract to getPanel()
· L91: Magic number — bare 14 used as pixel offset; name it TOOLTIP_OFFSET
[file] js/bar.js
· L12: Naming — variable `or_` is unclear; rename to outerR, outerG, outerB
...
```
If no issues are found, output "No code smells detected. Code quality looks good." and stop.
### Step 4 — Fix each issue
Use the Edit tool for **minimal, targeted changes**:
- **Duplicate logic**: extract to a shared constant or function; update all call sites
- **Magic number/string**: declare `const NAME = value` near the top of the relevant scope; replace all usages
- **Naming**: rename the variable/function; update all references
- **Dead code**: delete it
- **Trailing whitespace / style**: fix in place
- **Unexported helper**: add `export`; update callers to import instead of re-implementing
Principles:
- Only fix issues identified in the checklist — no extra improvements
- Keep each Edit as small as possible
- After fixing, verify the old bad pattern is gone with grep
- Prefer `apply_patch` for targeted edits; use formatters only when the repository already uses them for the touched file type
### Step 5 — Summary
```
Cleanup complete:
Fixed N issues:
✓ earth.js — extracted duplicate vertexShader into ATMOS_VERTEX_SHADER constant
✓ main.js — extracted TOOLTIP_CURSOR_OFFSET = 14 (4 references updated)
✓ controls.js — exported updateLayerButtonState; removed duplicate implementation in main.js
...
Skipped (needs manual review):
! foo.js L45 — large commented-out block; confirm it is safe to delete
```
## Constraints
- **Do not** change function signatures, exported interfaces, or public APIs (unless the issue is a missing export)
- **Do not** add new features, abstractions, or parameters
- **Do not** rewrite comments (only delete commented-out dead code)
- **Do not** touch test file logic
- If a magic number's intent is uncertain, skip it and flag it in the summary
+82
View File
@@ -0,0 +1,82 @@
---
name: docs
description: Create or update repository documentation from current code changes. Use when the user asks to write docs, update docs, summarize implementation changes into docs, or check documentation coverage. Load repository-specific coverage rules from docs/documentation-coverage-rules.md when present.
---
# Docs
Use this skill when the task is documentation work: creating, updating, checking, or summarizing docs for code or behavior changes.
## Goal
Write documentation that explains why a change exists, how it behaves, and what maintainers need to know. Keep the skill generic; repository-specific rules belong in the repository, not in this skill.
## Repository Rules
Before deciding scope, check whether the repository has a documentation rules file:
```bash
test -f docs/documentation-coverage-rules.md && sed -n '1,240p' docs/documentation-coverage-rules.md
```
If it exists, apply it as the project-specific coverage checklist. If it does not exist, continue with the generic workflow below.
## Workflow
1. Gather focused context:
```bash
git diff HEAD --stat
git diff HEAD --name-only
git log --oneline -10
rg --files docs
```
If the user gives a topic, focus on that topic. Otherwise infer the doc topic from changed files. Avoid reading large full diffs by default; inspect focused files and symbols:
```bash
git diff HEAD -- <path>
rg -n "class |def |function |export |router|@router|interface |type " <path>
```
2. Decide scope:
- Prefer updating an existing relevant doc over creating a duplicate.
- Use one document for one coherent topic.
- Split documents only when changes cross meaningful domains.
- Keep filenames lowercase and hyphenated.
3. Write the doc:
- Explain background/problem, design decisions, constraints, and operational impact.
- Keep code snippets short and directly relevant.
- List related files only when they help future maintainers navigate.
- Use the repository’s existing language, heading style, and naming conventions.
4. Verify:
- Read the completed doc once for clarity and stale statements.
- Verify important referenced paths exist with `test -e` or `rg --files`.
- Run repository-specific doc checks from `docs/documentation-coverage-rules.md` when present.
- For Markdown links, check that user-facing titles are readable and not raw filenames unless the repository rules allow it.
## Hard Constraints
- Do not leave placeholder docs or copied source text pretending to be documentation.
- Do not duplicate bilingual files byte-for-byte.
- Do not reference PR numbers, issue numbers, or the current conversation unless explicitly requested.
- Do not write changelog-style lists without the reasoning, constraints, and tradeoffs behind the change.
- Keep docs concise enough to maintain.
## Recommended Output
After editing, summarize:
```md
Updated:
- path/to/doc.md — what changed
Verified:
- checks that passed
- checks that could not be run, if any
```
+103
View File
@@ -0,0 +1,103 @@
---
name: goal-driven
description: Run a goal-driven execution loop for very large, long-horizon, rigorously verifiable tasks. Use when the user explicitly wants the lidangzzz/goal-driven method, a master-agent plus worker-agent style workflow, or a persistent loop that keeps working until concrete success criteria are satisfied.
---
# Goal-Driven
Use this skill when the user wants a strict goal-driven workflow for a hard task with:
- one clear end goal
- explicit success criteria
- repeated verification against those criteria
- continued execution until the criteria are actually met
This skill is adapted from `lidangzzz/goal-driven`, but trimmed for local skill use to avoid bloating context.
## When To Use
Use it for tasks like:
- compilers, interpreters, theorem-like proof work, deep refactors
- long-running system design or implementation work
- problems that are expensive and complex, but still objectively testable
Do not use it for:
- vague brainstorming without a success condition
- short one-shot edits
- tasks where "done" cannot be evaluated in a meaningful way
## Core Model
The workflow has two roles:
1. Master role
Defines the goal, defines the success criteria, audits progress, and decides whether the work is actually complete.
2. Worker role
Keeps advancing the task toward the goal. If a result is partial, stalled, or unverifiable, the worker continues.
In Codex, only use actual subagents when the user explicitly asks for delegation or subagent work and the platform supports it. Otherwise emulate the same loop locally: keep working, checkpointing, and re-verifying until the criteria are satisfied.
## Workflow
1. Normalize the task into two blocks:
- `Goal`
- `Criteria for success`
2. Make the criteria concrete and testable.
Good criteria usually include:
- required outputs
- required validations or tests
- edge cases or coverage thresholds
- what evidence proves completion
3. Break the work into milestones that can each produce evidence.
4. Execute the next milestone.
If subagents are explicitly allowed, the master may delegate bounded worker tasks.
If not, do the work locally but keep the master/worker mindset.
5. Whenever work pauses, stalls, or appears complete, audit against the criteria directly.
Check artifacts, tests, logs, diffs, metrics, or other real evidence.
6. If the criteria are not met, continue with a specific delta:
- what is still missing
- what evidence failed
- what the next worker pass must improve
7. Stop only when the criteria are met, or when the user explicitly stops the process.
## Operating Rules
- Prefer objective checks over self-reported completion.
- Prefer deterministic tool evidence over long model summaries: use `rg`, `git diff --stat`, targeted `git diff -- <path>`, tests, builds, linters, `curl`, or database queries when they can prove a criterion.
- Do not paste large command output into the conversation; summarize the evidence and keep raw output in tool calls.
- Do not confuse progress with completion.
- If the worker says "done", verify it.
- If verification fails, continue from the gap instead of restarting blindly.
- Keep the goal stable unless the user changes it.
- Tighten fuzzy criteria before sinking large amounts of effort.
## Recommended Response Shape
When starting a goal-driven task, structure the kickoff like this:
```md
Goal
- ...
Criteria for success
- ...
Current plan
1. ...
2. ...
3. ...
Verification
- What evidence will prove completion
```
For a reusable prompt template, read [references/prompt-template.md](references/prompt-template.md).
@@ -0,0 +1,7 @@
interface:
display_name: "Goal-Driven"
short_description: "Drive complex work until explicit success criteria are met."
default_prompt: "Use $goal-driven to turn this task into a concrete goal, explicit success criteria, and a verification-driven execution loop."
policy:
allow_implicit_invocation: true
+38
View File
@@ -0,0 +1,38 @@
# Goal-Driven Prompt Template
Use this when you want a reusable kickoff prompt for a master/worker execution loop.
```md
# Goal-Driven System
Goal: [[[[[DEFINE THE FINAL GOAL HERE]]]]]
Criteria for success: [[[[[DEFINE THE SUCCESS CRITERIA HERE]]]]]
You are the master agent.
Your job is to:
1. Keep the goal and criteria fixed.
2. Start worker execution toward the goal.
3. Audit any claimed progress against the criteria.
4. If the criteria are not met, continue the work with a precise next delta.
5. Stop only when the criteria are satisfied or the user explicitly stops the process.
Worker requirements:
1. Break the task into subproblems.
2. Keep producing concrete progress toward the goal.
3. Report evidence, not just claims.
4. Continue until the criteria are satisfied.
Master audit loop:
1. Check whether the worker is still making progress.
2. If the worker stalls or claims completion, verify against the criteria.
3. If verification fails, resume work from the remaining gap.
4. Repeat until the criteria are met.
```
## Notes
- Stronger criteria produce better results than stronger rhetoric.
- Prefer measurable checks such as tests, parity checks, generated artifacts, benchmarks, or reviewable outputs.
- If the environment does not support subagents, emulate the same loop locally.
+173
View File
@@ -0,0 +1,173 @@
---
name: release
description: Use when the user asks to release, bump version, update changelog/version files, or commit/push a repository release for the Planet repo. Determines version bump type from changes, updates all required version-bearing files, updates changelog and version-history, runs minimal validation, then commits, tags, and pushes.
---
# Release Workflow
Use this skill for release-oriented work in this repository.
## When To Use
- The user asks to `发版`
- The user asks to bump a version
- The user asks to update `CHANGELOG`, `version-history`, or version files as part of a release
- The user asks to commit/push a release or a publishable bugfix/feature bundle
Do not use this skill for ordinary commits that are not being released.
## Versioning Rules
- `feature` -> bump minor and reset patch to `0` (`x.y.z` → `x.(y+1).0`; for example `0.41.2` → `0.42.0`)
- `bugfix` -> bump `+0.0.1`
- `docs`, `maintenance`, and `refactor` do not bump by default unless the user explicitly wants a release
When intent is mixed, prefer the user's stated release intent.
## Required Files
Use `git rev-parse --show-toplevel` to get the repo root. All paths are relative to it:
- `VERSION`
- `frontend/package.json` (`"version"` field)
- `pyproject.toml` (`version =` field)
- `uv.lock` (**never edit manually** — regenerate by running `uv lock`)
- `docs/CHANGELOG.md`
- `docs/version-history.md`
## Token-Saving Rule
Release work should be driven by deterministic CLI evidence. Prefer compact commands and targeted file reads:
```bash
git status --short
git diff --stat HEAD
git diff --name-only HEAD
rg -n "version|^## |^Released:|current" VERSION frontend/package.json pyproject.toml docs/CHANGELOG.md docs/version-history.md
```
Do not inspect full diffs unless deciding whether changed code belongs in the release.
## Workflow
### Step 1 — Environment check
```bash
git branch --show-current # must be on dev
git status --short # check for unrelated uncommitted changes
cat VERSION # read current version
```
If not on `dev`, stop and tell the user. Do not proceed.
If unrelated uncommitted changes exist, list them and ask the user whether to include them or stash first.
### Step 2 — Determine release type and next version
- If the user provided an explicit type (`feature` / `bugfix`), use it
- Otherwise infer from `git diff --stat HEAD`, `git diff --name-only HEAD`, focused diffs for changed code, and recent `git log`
- Compute the next version:
- `feature`: increment minor and reset patch to `0` (e.g. `0.41.2` → `0.42.0`)
- `bugfix`: increment patch only (e.g. `0.26.2` → `0.26.3`)
- **Show the release plan before making any changes:**
```
Release plan:
Type: bugfix
Version: 0.26.2 → 0.26.3
Branch: dev
Will update: VERSION, frontend/package.json, pyproject.toml, uv.lock, CHANGELOG.md, version-history.md
```
### Step 3 — Update version files
Update in order (use Edit for precise replacement, never rewrite whole files):
1. `VERSION` — replace entire content with new version string
2. `frontend/package.json` — replace `"version": "x.x.x"` line
3. `pyproject.toml` — replace `version = "x.x.x"` line
4. Run `uv lock` at repo root to regenerate `uv.lock`
### Step 4 — Update CHANGELOG.md
Insert a new entry at the top of the file:
```markdown
## x.x.x
Released: YYYY-MM-DD
### Highlights
- ...
### Added / Fixed / Improved
- ... (high-signal items only, max 5)
---
```
Get today's date with `date +%Y-%m-%d`.
### Step 5 — Update docs/version-history.md
- Update the "current dev version" field in the file header
- Insert a new row at the top of the timeline table: `| vx.x.x | YYYY-MM-DD | one-line summary |`
### Step 6 — Validate
Run the smallest relevant validation for the changes in scope:
- Python files changed: list changed Python files with `git diff --name-only HEAD -- '*.py'`, then run `python3 -m py_compile <changed_files>`
- Frontend files changed: list changed frontend files with `git diff --name-only HEAD -- frontend`, then run the project-standard check if available; otherwise skip and say so
- Version consistency: confirm VERSION, package.json, pyproject.toml, and uv.lock all show the same version
```bash
cat VERSION
rg -n "\"version\":|^version =|version = " frontend/package.json pyproject.toml uv.lock
```
### Step 7 — Pre-commit preview
Show what will be committed:
```bash
git diff --stat HEAD
```
Confirm all required files are present and no unexpected files (debug files, `.env`, etc.) are included.
### Step 8 — Commit, tag, and push
```bash
git add VERSION frontend/package.json pyproject.toml uv.lock docs/CHANGELOG.md docs/version-history.md
# also stage any code changes included in this release
git add <code_files>
git commit -m "release: bump version to x.x.x"
git tag vx.x.x
git push origin dev
git push origin vx.x.x
```
Commit message format is fixed: `release: bump version to x.x.x`
### Step 9 — Completion summary
```
✓ Version bumped: 0.26.2 → 0.26.3
✓ CHANGELOG updated
✓ version-history updated
✓ uv.lock regenerated
✓ Validation passed
✓ commit: release: bump version to 0.26.3
✓ tag: v0.26.3
✓ Pushed to origin/dev
```
## Notes
- `uv.lock` must only be updated by running `uv lock`, never manually
- The release commit should include only version files + the code for this release — no unrelated changes
- If `uv` is unavailable in the environment, say so explicitly and remind the user to run it manually
+13
View File
@@ -0,0 +1,13 @@
**
!pyproject.toml
!uv.lock
!aiprovider/
!aiprovider/**
aiprovider/.env
aiprovider/.env.*
!aiprovider/.env.example
**/__pycache__/
**/*.pyc
**/*.pyo
+5
View File
@@ -145,3 +145,8 @@ docs/.venv/
*.temp
tmp/
temp/
# ----------------------
# Runtime Data
# ----------------------
data/ai/bgp-briefs/
+178 -1
View File
@@ -102,6 +102,13 @@
| Axios | HTTP 客户端 |
| Socket.io-client | WebSocket 客户端 |
| ECharts | 统计图表 |
| Bun | 前端包管理与脚本运行 |
前端工程统一使用 Bun:
- 安装依赖使用 `bun install`
- 运行脚本使用 `bun run <script>`
- 不使用 `npm`、`pnpm`、`yarn`
### 虚幻引擎客户端
@@ -205,10 +212,163 @@
./planet.sh health
```
前端命令约定:
```bash
cd frontend
bun install
bun run dev
bun run build
```
不要使用 `npm run ...`,避免在 WSL/Windows 混合环境里触发 `cmd.exe` 路径兼容问题。
## API 文档
启动服务后访问: `http://localhost:8000/docs`
## WSL / Windows 局域网访问
如果服务运行在 WSL 中,而你希望:
- Windows 本机浏览器访问开发服务
- 同一局域网内的手机或其他电脑访问开发服务
推荐按下面顺序排查和配置。
端口占用、`iphlpsvc` / portproxy、摄像头和依赖问题的集中排障入口见 [常见问题](/home/ray/dev/linkong/planet/docs/technical/zh/faq.md)。
### 1. 在 WSL 中启动服务
```bash
./planet.sh start --allow-lan
```
这会让前端监听 `0.0.0.0:3000`,后端监听 `0.0.0.0:8000`。
### 2. 先确认 WSL 内部服务正常
在 WSL 中执行:
```bash
curl http://localhost:3000
curl http://localhost:8000/health
ss -ltnp | grep -E ':3000|:8000'
```
预期:
- `3000` 返回前端 HTML
- `8000/health` 返回健康检查 JSON
- `ss` 中能看到 `0.0.0.0:3000` 和 `0.0.0.0:8000`
如果这一步不通,先不要继续做 Windows 转发。
### 3. 在 Windows 本机验证 localhost 直通
在 Windows PowerShell 中执行:
```powershell
curl http://localhost:3000
curl http://localhost:8000/health
```
在常见的 WSL2 开发环境下,Windows 通常可以直接通过 `localhost` 访问 WSL 中的服务。
### 4. 如果需要让局域网设备访问,再做 Windows 端口转发
注意:下面的命令必须在“以管理员身份运行”的 PowerShell 中执行。
先把 Windows 对外网卡上的 `3000` / `8000` 转发到 Windows 本机 `127.0.0.1`:
```powershell
netsh interface portproxy delete v4tov4 listenaddress=0.0.0.0 listenport=3000
netsh interface portproxy delete v4tov4 listenaddress=0.0.0.0 listenport=8000
netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000
netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000
```
再放行 Windows 防火墙:
```powershell
New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000
New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000
```
检查转发规则是否生效:
```powershell
netsh interface portproxy show all
```
预期能看到:
- `0.0.0.0:3000 -> 127.0.0.1:3000`
- `0.0.0.0:8000 -> 127.0.0.1:8000`
### 5. 查 Windows 局域网 IP,并让其他设备访问
在 Windows PowerShell 中执行:
```powershell
ipconfig
```
找到当前联网网卡的 IPv4 地址,例如 `192.168.8.228`。
局域网其他设备可访问:
- `http://<Windows局域网IP>:3000/earth`
- `http://<Windows局域网IP>:3000/admin`
例如:
- `http://192.168.8.228:3000/earth`
### 6. 常见现象与判断
- WSL 中 `curl localhost:3000` 能通,但 Windows 访问 `WSL 的局域网 IP:3000` 不通:这是正常现象之一,优先验证 Windows 的 `localhost:3000`
- Windows `localhost:3000` 能通,但局域网设备访问 `Windows 局域网 IP:3000` 不通:通常缺少 `portproxy` 或防火墙放行
- `whoami /groups` 中 `S-1-5-32-544` 显示 `deny only`:说明当前 PowerShell 不是提权管理员窗口
### 7. 本项目一次性验证顺序
建议固定按这个顺序验证:
1. WSL 中执行 `curl http://localhost:3000`
2. WSL 中执行 `curl http://localhost:8000/health`
3. Windows 中执行 `curl http://localhost:3000`
4. Windows 中执行 `curl http://localhost:8000/health`
5. 管理员 PowerShell 配置 `portproxy` 和防火墙
6. 用手机或其他电脑访问 `http://<Windows局域网IP>:3000/earth`
## 启动容错参数
`planet.sh` 现在为依赖安装、数据库、AI Provider 启动加入了有限次重试,并会在数据库与 `aiprovider` 启动后额外等待 Docker healthcheck。
可通过环境变量临时调整:
```bash
# 例: 放宽 AI Provider 与数据库在网络抖动下的自愈次数
AI_PROVIDER_START_MAX_RETRIES=5 \
AI_PROVIDER_RETRY_INTERVAL=10 \
DATABASE_START_MAX_RETRIES=5 \
DATABASE_RETRY_INTERVAL=10 \
./planet.sh restart
```
常用参数:
- `DEPENDENCY_INSTALL_MAX_RETRIES` / `DEPENDENCY_INSTALL_RETRY_INTERVAL`: 控制 `uv sync`、`bun install` 的重试次数与间隔,默认 `3` 次、`5` 秒
- `DATABASE_START_MAX_RETRIES` / `DATABASE_RETRY_INTERVAL`: 控制 `postgres`、`redis` 的启动/重启与健康检查自愈,默认 `3` 次、`5` 秒
- `AI_PROVIDER_START_MAX_RETRIES` / `AI_PROVIDER_RETRY_INTERVAL`: 控制 `aiprovider` 的构建/启动与容器重启自愈,默认 `3` 次、`5` 秒
- `BACKEND_MAX_RETRIES`: 控制后端进程启动重试次数,默认 `3`
- `FRONTEND_MAX_RETRIES`: 控制前端 dev server 启动重试次数,默认 `3`
- `BACKEND_HEALTH_CHECK_ATTEMPTS` / `BACKEND_HEALTH_CHECK_INTERVAL`: 控制后端 HTTP 健康检查等待次数与间隔,默认 `10` 次、`2` 秒
- `FRONTEND_HEALTH_CHECK_ATTEMPTS` / `FRONTEND_HEALTH_CHECK_INTERVAL`: 控制前端 HTTP 可访问检查等待次数与间隔,默认 `10` 次、`2` 秒
- `AI_PROVIDER_HEALTH_CHECK_ATTEMPTS` / `AI_PROVIDER_HEALTH_CHECK_INTERVAL`: 控制 `aiprovider` HTTP 健康检查等待次数与间隔,默认 `10` 次、`2` 秒
## AI 接口预留
项目现在采用“两层”设计:
@@ -284,8 +444,25 @@ AI_PROVIDER_SERVICE_TOKEN=change_me
详细文档:
- [docs/aiprovider.md](/home/ray/dev/linkong/planet/docs/aiprovider.md)
- [docs/technical/agents-aiprovider.md](/home/ray/dev/linkong/planet/docs/technical/agents-aiprovider.md)
- [aiprovider/README.md](/home/ray/dev/linkong/planet/aiprovider/README.md)
- [docs/technical/frontend-layout-guidelines.md](/home/ray/dev/linkong/planet/docs/technical/frontend-layout-guidelines.md)
- [docs/plans/frontend-ai-playground-development-plan.md](/home/ray/dev/linkong/planet/docs/plans/frontend-ai-playground-development-plan.md)
- [docs/plans/agents-situational-awareness-foundation-plan.md](/home/ray/dev/linkong/planet/docs/plans/agents-situational-awareness-foundation-plan.md)
## 前端页面布局规范
管理后台页面默认遵循“单屏工作区”原则:
- 页头、摘要区、主工作区应在一屏内形成稳定结构
- 主表格 / 主图表 / 主分析区应占据页面主要可视空间
- 模块内容超出时优先在卡片、表格、标签页内部滚动
- 不依赖整页纵向撑开来容纳主要工作区
当前推荐参考实现:
- [frontend/src/pages/BGP/BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx)
- [docs/technical/frontend-layout-guidelines.md](/home/ray/dev/linkong/planet/docs/technical/frontend-layout-guidelines.md)
## License
+26
View File
@@ -13,7 +13,33 @@
- [x] 接入 `IPtoASN / IPtoCountry` 作为 prefix-centric geography 的主数据源
- [x] 接入 `OpenGeoFeed` 作为 prefix geography 的高质量覆盖/override 数据源
- [x] 把 RIR delegated 设计成 prefix geography 的 fallback,而不是主来源
- [ ] 为 `aiprovider` 建立 `provider -> api adapter -> compat policy` 的配置中心,优先落成 `json` 或 `yaml` 文件,运行时按 `provider/model` 读取兼容设置,而不是把专项兼容继续散落在 Python 分支里
- [ ] 为市面上主流 AI 服务补专项兼容配置并固化到配置文件中,至少覆盖 `OpenAI / Anthropic / MiniMax / Ollama / Moonshot / DeepSeek / Qwen / GLM / Gemini / OpenRouter / vLLM / LM Studio / One API`
- [ ] 在兼容配置中补齐可声明项:`api adapter`、`base_url pattern`、`auth header`、`thinking default`、`reasoning block mapping`、`stream path`、`tool-call capability`、`multimodal capability`、`provider-specific request patch`
- [ ] 接入 `inetnum` / `inet6num` whois 作为比 RIR 更细粒度的后备层
- [x] 在 activity layer 之后继续补 `route leak` 和 `path instability / flap` detector
- [ ] 对 [frontend/public/earth/js/bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) 做按职责拆分的小重构,拆成 data / markers / overlays / animation,降低后续维护复杂度
- [ ] 可选优化(非必做):将 BGP incident/collector 标点改为 HTML marker(参考 worldmonitor 的 `htmlElementsData` 思路),实现近乎固定屏幕尺寸与更高密度可点击性
- [ ] 保持 Earth 当前这批纯个人偏好设置继续走本地持久化:`旋转模式`、HUD 面板显示/隐藏、`地形透明度` 暂不升级到后端系统设置,避免把设备级偏好过早做成全局配置
- [ ] 如果后续明确需要“账号级同步 Earth 偏好”,再单独设计 `Earth user preferences`:优先按用户维度而不是全局系统设置保存,并规划 `localStorage -> backend` 的平滑迁移策略
- [ ] 为 Planet / Earth 补一个可用的日志查看系统:先明确前后端/AI Provider/采集任务的日志入口、最近日志聚合、筛选与 tail 能力,再决定是先做脚本级统一入口还是控制台内置日志面板
- [ ] 重写控制台 UI,逐步抛弃 Ant Design,建立自有组件体系,并统一采用 `tabler.io` / Tabler Icons 作为控制台主图标库
- [ ] 把 Earth 态势新闻源从 [earth_news.py](/home/ray/dev/linkong/planet/backend/app/services/earth_news.py) 的硬编码列表抽成可配置目录,优先保持当前“实时聚合”链路不变,只先解决新闻源不可配置的问题
- [ ] 为 Earth 态势新闻设计后续采集器化方案:明确新闻数据模型、去重策略、区域映射、过期清理和 Earth/AI 复用方式,再决定何时把新闻从实时抓取升级成正式 collector
- [ ] AIS v3.1:修复船只聚合完整性,`/geo/vessels` 合并 raw observation 聚合结果与 legacy `vessel_position + vessel_static` 最新结果,确保 BarentsWatch-only 船只不会因为 AISStream 子集存在而消失,并增加 raw/legacy/final unique MMSI 诊断统计
- [ ] AIS v3.2:把 AISStream 从收满 `max_messages` 后结束的批采集改成长连接 streaming service,持续写入 raw observations,通过内部 `/ws` 的 `vessels` channel 推送新船、位置和航向增量,Earth 前端按 MMSI upsert marker
- [ ] AIS v3.3:修正 AISStream 采集页面状态语义,使用 connecting/streaming/reconnecting/stopped 与 indeterminate 状态,展示运行时长、消息数、unique MMSI、message rate、最近消息和错误,不再用一次性 REST 进度条表示长连接
- [ ] AIS v3.4:修复船只身份字段和名称聚合,MMSI/IMO/callsign 按字符串显示且不带千分位符;查询并列出所有仍以 MMSI 号码或 `MMSI <number>` 作为船名的记录,标注来源、最近观测、message types 和缺失原因,并把这批 fallback-name 船只纳入名称聚合修复集合
- [ ] Earth Live Sync:建立统一态势实时同步链路,新增 `earth_summary` WS channel,任意采集器成功后广播轻量 summary invalidation,前端收到后重新拉 `/api/v1/visualization/geo/summary` 并更新 HUD;同时为 BGP 增加 `bgp` WS channel,使 BGP incidents/anomalies/collectors 在不刷新页面时也能 upsert 图层;卫星采集完成后触发 summary 刷新,必要时按 TLE 版本重新 hydrate 卫星数据
- [ ] AIS v4:开放船只多源聚合策略配置,支持 source priority、字段级规则、freshness 窗口和高级保护开关;保存时校验未知字段、非法模式和危险动态字段锁定,并在聚合接口返回命中的配置版本
- [ ] AIS v5:实现船舶资料 enrichment 与冲突治理,按 `mmsi + imo + name + callsign` 异步补充船型细分、AIS 大类、旗国、尺寸、建造年份、运营方和图片缓存;详情面板展示缓存资料和字段来源,不在实时 AIS 请求链路现场抓第三方页面
- [ ] 为 Earth 地球表面增加一层与基础纹理对齐的材质/纹理 overlay,并在同层叠加国界轮廓参考线;要求国界线与底图稳定对齐,且 hover 到国家轮廓时能高亮当前国家,便于校准地表和增强交互
- [ ] 把 Earth 新闻接入通用巡航队列:按新闻发生地和时间排序生成巡航目标,巡航聚焦到新闻事件时显示对应新闻卡片,并保持实现边界为“通用巡航层 + 新闻业务适配层”,不要再把新闻逻辑直接耦合回 `main.js` 状态机
- [ ] 为未知位置的算力中心建立分层坐标补全链路:优先 `精确坐标 > 站点/园区命中 > 城市 > 州/省 > 国家内主要算力城市 > 国家质心`,并把每次回退的 `confidence / reason / precision` 明确写进统一 GeoJSON
- [ ] 为算力中心补一份可维护的本地位置注册表,例如 `canonical_name / aliases / operator / country / region / city / lat / lon / confidence / source_note`,避免把地点知识长期硬编码在 `visualization.py`
- [ ] 增强 `epoch_ai_gpu` 和相关算力采集器的源页面解析:即使公开 API 不给坐标,也继续尝试从详情页、HTML、内嵌 JSON、schema.org、OpenGraph、脚本变量和 PDF/新闻稿链接里抽地点线索
- [ ] 为未知位置算力中心增加外部富化策略评估:可选接入公开知识源或搜索兜底,只抓“站点名/园区名/城市名”级别线索,不直接抓经纬度结论,并把结果作为候选证据而不是真值
- [ ] 为算力中心建立 `operator / cluster name / facility alias` 归一化层,先解决 `xAI / Colossus / Memphis`、`OpenAI / Stargate`、`CoreWeave`、`Lambda`、`Crusoe` 这类同一对象多种写法导致的地点匹配失败
- [ ] 为估算位置增加更细的视觉和产品表达:除了问号角标,还要支持 tooltip/详情中的“估算依据”“精度级别”“最后核验时间”,并允许在设置中单独开关“仅看精确位置”
- [ ] 为国家级估算点设计更合理的落点策略:优先落在“该国主要算力/数据中心城市候选集”而不是几何质心,必要时同国多节点做稳定散列分配,避免大量节点堆在荒漠或海上
- [ ] 为未知位置算力中心建立人工校验工作流:支持导出待核验清单、记录人工确认结果,并把人工确认反哺到位置注册表,逐步减少问号点比例
+1 -1
View File
@@ -1 +1 @@
0.23.0
0.50.0
+44 -13
View File
@@ -6,29 +6,60 @@ AI_TIMEOUT_SECONDS=60
AI_HTTP_RETRY_ATTEMPTS=2
AI_ANALYSIS_SYSTEM_PROMPT=你是态势感知分析助手。请基于输入的上下文、观测与约束,输出结构化、克制、可执行的分析。
# Select one provider mode:
# - openai_compatible
# - claude_compatible
# Provider identity. Recommended values:
# - minimax
# - openai
# - ollama
AI_PROVIDER=ollama
# Compatibility aliases still accepted:
# - openai_compatible
# - anthropic_compatible
# - claude_compatible
AI_PROVIDER=minimax
# Request adapter style, following OpenClaw's API-seam pattern:
# - auto
# - openai-completions
# - anthropic-messages
# - ollama-generate
AI_PROVIDER_API=anthropic-messages
# Common model selection
AI_MODEL=qwen2.5:7b
AI_MODEL=MiniMax-M2.7
# MiniMax CN Anthropic-compatible example
AI_BASE_URL=https://api.minimaxi.com/anthropic
AI_API_KEY=sk-cp-change-me
AI_MAX_TOKENS=1200
AI_ANTHROPIC_VERSION=2023-06-01
# Optional provider-specific keys used by Settings fallback before AI_API_KEY
# MINIMAX_API_KEY=sk-cp-change-me
# OPENAI_API_KEY=sk-change-me
# ANTHROPIC_API_KEY=sk-ant-change-me
# DEEPSEEK_API_KEY=sk-change-me
# DASHSCOPE_API_KEY=sk-change-me
# MOONSHOT_API_KEY=sk-change-me
# OPENROUTER_API_KEY=sk-or-change-me
# OpenAI-compatible example (vLLM / LM Studio / One API / local gateway)
# AI_PROVIDER=openai_compatible
# AI_PROVIDER=openai
# AI_PROVIDER_API=openai-completions
# AI_BASE_URL=http://127.0.0.1:8001/v1
# AI_API_KEY=local-key
# AI_MODEL=your-local-model
# Claude-compatible example (Anthropic / MiniMax / Claude-compatible gateway)
# AI_PROVIDER=claude_compatible
# AI_BASE_URL=http://127.0.0.1:8002
# Anthropic-compatible example (Claude-compatible gateway)
# AI_PROVIDER=anthropic
# AI_PROVIDER_API=anthropic-messages
# AI_BASE_URL=http://127.0.0.1:8002/anthropic
# AI_API_KEY=local-key
# AI_MODEL=your-model
# AI_MAX_TOKENS=1200
# AI_ANTHROPIC_VERSION=2023-06-01
# Ollama native example
AI_BASE_URL=http://127.0.0.1:11434
AI_API_KEY=
AI_MAX_TOKENS=1200
AI_ANTHROPIC_VERSION=2023-06-01
# AI_PROVIDER=ollama
# AI_PROVIDER_API=ollama-generate
# AI_BASE_URL=http://127.0.0.1:11434
# AI_API_KEY=
# AI_MODEL=qwen2.5:7b
+11 -4
View File
@@ -1,6 +1,12 @@
FROM python:3.14-slim
# syntax=docker/dockerfile:1.7
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
ARG PYTHON_IMAGE=python:3.14-slim
ARG UV_IMAGE=ghcr.io/astral-sh/uv:latest
FROM ${UV_IMAGE} AS uv
FROM ${PYTHON_IMAGE}
COPY --from=uv /uv /uvx /bin/
WORKDIR /app
@@ -14,9 +20,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
COPY pyproject.toml uv.lock /app/
RUN uv sync --frozen --no-dev
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-dev
COPY . /app
COPY aiprovider /app/aiprovider
EXPOSE 8010
+30 -16
View File
@@ -4,21 +4,31 @@
完整使用说明见:
- [docs/aiprovider.md](/home/ray/dev/linkong/planet/docs/aiprovider.md)
- [docs/agents/aiprovider.md](/home/ray/dev/linkong/planet/docs/agents/aiprovider.md)
当前支持:
- `AI_PROVIDER=openai`
- `AI_PROVIDER=openai_compatible`
- `AI_PROVIDER=anthropic`
- `AI_PROVIDER=anthropic_compatible`
- `AI_PROVIDER=claude_compatible`
- `AI_PROVIDER=ollama`
- provider identity:
- `AI_PROVIDER=openai`
- `AI_PROVIDER=anthropic`
- `AI_PROVIDER=minimax`
- `AI_PROVIDER=ollama`
- request adapter:
- `AI_PROVIDER_API=openai-completions`
- `AI_PROVIDER_API=anthropic-messages`
- `AI_PROVIDER_API=ollama-generate`
兼容别名仍然保留:
- `openai_compatible`
- `anthropic_compatible`
- `claude_compatible`
典型配置:
```env
AI_PROVIDER=openai_compatible
AI_PROVIDER=openai
AI_PROVIDER_API=openai-completions
AI_BASE_URL=https://api.openai.com/v1
AI_API_KEY=your_api_key
AI_MODEL=gpt-4o-mini
@@ -26,13 +36,14 @@ AI_TIMEOUT_SECONDS=60
AI_PROVIDER_SERVICE_TOKEN=change_me
```
Claude 兼容供应商示例:
MiniMax 中国大陆节点示例:
```env
AI_PROVIDER=claude_compatible
AI_BASE_URL=https://your-claude-compatible-endpoint.example.com
AI_API_KEY=your_api_key
AI_MODEL=your-claude-compatible-model
AI_PROVIDER=minimax
AI_PROVIDER_API=anthropic-messages
AI_BASE_URL=https://api.minimaxi.com/anthropic
AI_API_KEY=sk-cp-xxxxx
AI_MODEL=MiniMax-M2.7
AI_TIMEOUT_SECONDS=60
AI_MAX_TOKENS=1200
AI_ANTHROPIC_VERSION=2023-06-01
@@ -43,12 +54,15 @@ AI_PROVIDER_SERVICE_TOKEN=change_me
- Anthropic 官方 Claude API
- Claude 兼容网关
- MiniMax 等提供 Claude/Anthropic 风格消息接口的服务
- MiniMax 等提供 Anthropic Messages 风格接口的服务
这套命名方式参考了 OpenClaw 的接入模式: provider 负责标识供应商, `AI_PROVIDER_API` 负责标识协议适配层, 避免把“供应商”和“协议”绑死在一起。
Ollama 原生示例:
```env
AI_PROVIDER=ollama
AI_PROVIDER_API=ollama-generate
AI_BASE_URL=http://127.0.0.1:11434
AI_API_KEY=
AI_MODEL=qwen2.5:7b
@@ -58,8 +72,8 @@ AI_PROVIDER_SERVICE_TOKEN=change_me
本地模型接入建议:
- `vLLM`、`LM Studio`、`One API`:优先使用 `openai_compatible`
- `MiniMax`、Claude 兼容网关:使用 `claude_compatible`
- `vLLM`、`LM Studio`、`One API`:`AI_PROVIDER=openai` + `AI_PROVIDER_API=openai-completions`
- `MiniMax`、Claude 兼容网关:`AI_PROVIDER=minimax|anthropic` + `AI_PROVIDER_API=anthropic-messages`
- `Ollama`:可直接使用 `ollama`
启动模板:
+1
View File
@@ -9,6 +9,7 @@ class Settings(BaseSettings):
SERVICE_VERSION: str = "0.1.0"
AI_PROVIDER: str = "disabled"
AI_PROVIDER_API: str = "auto"
AI_BASE_URL: str = "https://api.openai.com/v1"
AI_API_KEY: str = ""
AI_MODEL: str = ""
+20 -2
View File
@@ -37,8 +37,26 @@ def verify_service_token(x_provider_token: str | None = Header(default=None)) ->
)
def get_provider_service() -> ProviderService:
return ProviderService()
def get_provider_service(
x_ai_provider: str | None = Header(default=None),
x_ai_provider_api: str | None = Header(default=None),
x_ai_base_url: str | None = Header(default=None),
x_ai_api_key: str | None = Header(default=None),
x_ai_model: str | None = Header(default=None),
x_ai_max_tokens: str | None = Header(default=None),
x_ai_anthropic_version: str | None = Header(default=None),
) -> ProviderService:
overrides = {
"provider": x_ai_provider,
"provider_api": x_ai_provider_api,
"base_url": x_ai_base_url,
"api_key": x_ai_api_key,
"model": x_ai_model,
"anthropic_version": x_ai_anthropic_version,
}
if x_ai_max_tokens:
overrides["max_tokens"] = x_ai_max_tokens
return ProviderService({key: value for key, value in overrides.items() if value not in (None, "")})
@app.get("/health")
+151 -16
View File
@@ -8,6 +8,7 @@ from fastapi import HTTPException, status
from aiprovider.config import settings
from aiprovider.schemas import (
AIContentBlock,
AIProviderStatusResponse,
SituationalAnalysisRequest,
SituationalAnalysisResponse,
@@ -18,23 +19,58 @@ def _normalize_provider(value: str) -> str:
return (value or "disabled").strip().lower()
def _normalize_provider_api(value: str) -> str:
return (value or "auto").strip().lower().replace("_", "-")
def _resolve_provider_api(provider: str, configured_api: str) -> str:
if configured_api and configured_api != "auto":
return configured_api
if provider in {"openai", "openai-compatible", "openai_compatible"}:
return "openai-completions"
if provider in {
"anthropic",
"anthropic-compatible",
"anthropic_compatible",
"claude-compatible",
"claude_compatible",
"minimax",
"kimi-coding",
"moonshot-anthropic",
}:
return "anthropic-messages"
if provider == "ollama":
return "ollama-generate"
return "disabled"
class ProviderService:
def __init__(self) -> None:
self.provider = _normalize_provider(settings.AI_PROVIDER)
self.base_url = settings.AI_BASE_URL.rstrip("/")
self.api_key = settings.AI_API_KEY
self.default_model = settings.AI_MODEL
def __init__(self, overrides: dict[str, Any] | None = None) -> None:
overrides = overrides or {}
self.provider = _normalize_provider(overrides.get("provider") or settings.AI_PROVIDER)
self.provider_api = _resolve_provider_api(
self.provider,
_normalize_provider_api(overrides.get("provider_api") or settings.AI_PROVIDER_API),
)
self.base_url = str(overrides.get("base_url") or settings.AI_BASE_URL).rstrip("/")
self.api_key = str(overrides.get("api_key") or settings.AI_API_KEY)
self.default_model = str(overrides.get("model") or settings.AI_MODEL)
self.timeout = settings.AI_TIMEOUT_SECONDS
self.http_retry_attempts = max(settings.AI_HTTP_RETRY_ATTEMPTS, 1)
self.max_tokens = settings.AI_MAX_TOKENS
self.anthropic_version = settings.AI_ANTHROPIC_VERSION
self.max_tokens = int(overrides.get("max_tokens") or settings.AI_MAX_TOKENS)
self.anthropic_version = str(
overrides.get("anthropic_version") or settings.AI_ANTHROPIC_VERSION
)
self.system_prompt = settings.AI_ANALYSIS_SYSTEM_PROMPT
def get_status(self) -> AIProviderStatusResponse:
enabled = self.provider != "disabled"
configured = enabled and bool(self.base_url and self.api_key and self.default_model)
has_credentials = bool(self.api_key) if self._requires_api_key() else True
configured = enabled and bool(self.base_url and has_credentials and self.default_model)
return AIProviderStatusResponse(
provider=self.provider,
api=self.provider_api if enabled else None,
enabled=enabled,
configured=configured,
model=self.default_model or None,
@@ -49,7 +85,8 @@ class ProviderService:
)
model = payload.preferred_model or self.default_model
if not self.base_url or not self.api_key or not model:
has_credentials = bool(self.api_key) if self._requires_api_key() else True
if not self.base_url or not has_credentials or not model:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="AI provider is not fully configured. Check AI_BASE_URL, AI_API_KEY, and AI_MODEL.",
@@ -57,28 +94,40 @@ class ProviderService:
prompt = self._build_prompt(payload)
if self.provider in {"openai", "openai_compatible"}:
if self.provider_api == "openai-completions":
data = await self._request_openai_compatible(model, prompt)
content = self._extract_openai_content(data)
elif self.provider in {"anthropic", "anthropic_compatible", "claude_compatible"}:
data = await self._request_anthropic_compatible(model, prompt)
content_blocks = self._extract_openai_blocks(data)
elif self.provider_api == "anthropic-messages":
data = await self._request_anthropic_messages(model, prompt, payload.thinking)
content = self._extract_anthropic_content(data)
elif self.provider == "ollama":
content_blocks = self._extract_anthropic_blocks(data)
elif self.provider_api == "ollama-generate":
data = await self._request_ollama(model, prompt)
content = self._extract_ollama_content(data)
content_blocks = self._extract_ollama_blocks(data)
else:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unsupported AI provider: {self.provider}",
detail=f"Unsupported AI provider API: {self.provider_api}",
)
text_blocks = [block.text for block in content_blocks if block.text]
thinking_blocks = [block.thinking for block in content_blocks if block.thinking]
return SituationalAnalysisResponse(
provider=self.provider,
model=model,
content=content,
content_blocks=content_blocks,
text_blocks=text_blocks,
thinking_blocks=thinking_blocks,
raw_response=data,
)
def _requires_api_key(self) -> bool:
return self.provider_api != "ollama-generate"
def _build_prompt(self, payload: SituationalAnalysisRequest) -> str:
sections = [
f"任务标题:\n{payload.title}",
@@ -113,7 +162,12 @@ class ProviderService:
request_body=request_body,
)
async def _request_anthropic_compatible(self, model: str, prompt: str) -> dict[str, Any]:
async def _request_anthropic_messages(
self,
model: str,
prompt: str,
thinking: dict[str, Any] | None = None,
) -> dict[str, Any]:
request_body = {
"model": model,
"system": self.system_prompt,
@@ -131,8 +185,15 @@ class ProviderService:
"max_tokens": self.max_tokens,
"temperature": 0.2,
}
resolved_thinking = self._resolve_anthropic_thinking(thinking)
if resolved_thinking:
request_body["thinking"] = resolved_thinking
if self.provider == "minimax" and self.base_url.endswith("/anthropic"):
path = "/v1/messages"
else:
path = "/messages"
return await self._post(
path="/messages",
path=path,
headers={
"x-api-key": self.api_key,
"anthropic-version": self.anthropic_version,
@@ -141,6 +202,25 @@ class ProviderService:
request_body=request_body,
)
def _resolve_anthropic_thinking(self, thinking: dict[str, Any] | None) -> dict[str, Any] | None:
if thinking:
return thinking
# OpenClaw treats MiniMax's Anthropic-compatible path specially:
# disable thinking by default unless the caller explicitly opts in.
if self.provider == "minimax":
return {"type": "disabled"}
return None
async def _request_anthropic_compatible(
self,
model: str,
prompt: str,
thinking: dict[str, Any] | None = None,
) -> dict[str, Any]:
return await self._request_anthropic_messages(model, prompt, thinking)
async def _request_ollama(self, model: str, prompt: str) -> dict[str, Any]:
request_body = {
"model": model,
@@ -218,6 +298,30 @@ class ProviderService:
)
return ""
def _extract_openai_blocks(self, payload: dict[str, Any]) -> list[AIContentBlock]:
choices = payload.get("choices") or []
if not choices:
return []
message = choices[0].get("message") or {}
content = message.get("content")
if isinstance(content, str):
return [AIContentBlock(type="text", text=content)]
if not isinstance(content, list):
return []
blocks: list[AIContentBlock] = []
for item in content:
if not isinstance(item, dict):
continue
blocks.append(
AIContentBlock(
type=str(item.get("type", "text")),
text=item.get("text") if isinstance(item.get("text"), str) else None,
metadata={k: v for k, v in item.items() if k not in {"type", "text"}},
)
)
return blocks
def _extract_anthropic_content(self, payload: dict[str, Any]) -> str:
content = payload.get("content")
if isinstance(content, str):
@@ -233,8 +337,39 @@ class ProviderService:
fragments.append(item["text"])
return "".join(fragments)
def _extract_anthropic_blocks(self, payload: dict[str, Any]) -> list[AIContentBlock]:
content = payload.get("content")
if not isinstance(content, list):
return []
blocks: list[AIContentBlock] = []
for item in content:
if not isinstance(item, dict):
continue
blocks.append(
AIContentBlock(
type=str(item.get("type", "unknown")),
text=item.get("text") if isinstance(item.get("text"), str) else None,
thinking=item.get("thinking") if isinstance(item.get("thinking"), str) else None,
signature=item.get("signature") if isinstance(item.get("signature"), str) else None,
metadata={
k: v
for k, v in item.items()
if k not in {"type", "text", "thinking", "signature"}
},
)
)
return blocks
def _extract_ollama_content(self, payload: dict[str, Any]) -> str:
response = payload.get("response")
if isinstance(response, str):
return response
return ""
def _extract_ollama_blocks(self, payload: dict[str, Any]) -> list[AIContentBlock]:
response = payload.get("response")
if isinstance(response, str) and response:
return [AIContentBlock(type="text", text=response)]
return []
+13
View File
@@ -3,6 +3,14 @@ from typing import Any
from pydantic import BaseModel, Field
class AIContentBlock(BaseModel):
type: str
text: str | None = None
thinking: str | None = None
signature: str | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class SituationalAnalysisRequest(BaseModel):
title: str = Field(..., min_length=1, max_length=200)
objective: str = Field(..., min_length=1, max_length=1000)
@@ -10,17 +18,22 @@ class SituationalAnalysisRequest(BaseModel):
observations: list[str] = Field(default_factory=list)
constraints: list[str] = Field(default_factory=list)
preferred_model: str | None = Field(default=None, max_length=200)
thinking: dict[str, Any] | None = None
class SituationalAnalysisResponse(BaseModel):
provider: str
model: str
content: str
content_blocks: list[AIContentBlock] = Field(default_factory=list)
text_blocks: list[str] = Field(default_factory=list)
thinking_blocks: list[str] = Field(default_factory=list)
raw_response: dict[str, Any] = Field(default_factory=dict)
class AIProviderStatusResponse(BaseModel):
provider: str
api: str | None = None
enabled: bool
configured: bool
model: str | None = None
+6 -2
View File
@@ -1,6 +1,10 @@
FROM python:3.14-slim
ARG PYTHON_IMAGE=python:3.14-slim
ARG UV_IMAGE=ghcr.io/astral-sh/uv:latest
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
FROM ${UV_IMAGE} AS uv
FROM ${PYTHON_IMAGE}
COPY --from=uv /uv /uvx /bin/
WORKDIR /app
+12
View File
@@ -5,6 +5,7 @@ from app.api.v1 import (
users,
datasource_config,
datasources,
docs,
tasks,
dashboard,
websocket,
@@ -12,8 +13,11 @@ from app.api.v1 import (
settings,
collected_data,
visualization,
vessel_aggregation,
bgp,
news,
system_control,
tv,
)
api_router = APIRouter()
@@ -26,10 +30,18 @@ api_router.include_router(
)
api_router.include_router(datasources.router, prefix="/datasources", tags=["datasources"])
api_router.include_router(collected_data.router, prefix="/collected", tags=["collected-data"])
api_router.include_router(docs.router, prefix="/docs", tags=["docs"])
api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboard"])
api_router.include_router(alerts.router, prefix="/alerts", tags=["alerts"])
api_router.include_router(settings.router, prefix="/settings", tags=["settings"])
api_router.include_router(system_control.router, prefix="/system", tags=["system"])
api_router.include_router(visualization.router, prefix="/visualization", tags=["visualization"])
api_router.include_router(
vessel_aggregation.router,
prefix="/vessel-aggregation",
tags=["vessel-aggregation"],
)
api_router.include_router(bgp.router, prefix="/bgp", tags=["bgp"])
api_router.include_router(tv.router, prefix="/tv", tags=["tv"])
api_router.include_router(news.router, prefix="/news", tags=["news"])
+243 -1
View File
@@ -1,15 +1,52 @@
from uuid import uuid4
from fastapi import APIRouter, Depends, Request, Response
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.user import User
from app.schemas.ai import (
AIProviderStatusResponse,
AlertBriefRequest,
AlertBriefResponse,
BGPBriefRequest,
BGPBriefRecordResponse,
BGPBriefRecordSummary,
PlaygroundMessageActionResponse,
PlaygroundMessageCreateRequest,
PlaygroundMessageEditRequest,
PlaygroundMessageResendRequest,
PlaygroundMessageStopRequest,
PlaygroundSessionResponse,
PlaygroundSessionUpsertRequest,
PlaygroundThreadResponse,
SituationalAlertBriefRequest,
SituationalAlertBriefResponse,
SituationalAnalysisRequest,
SituationalAnalysisResponse,
)
from app.services.alert_ai_brief import build_alert_brief_request
from app.services.ai_client import AIProviderClient, get_ai_provider_client
from app.services.bgp_ai_brief import build_bgp_brief_request
from app.services.bgp_ai_brief_store import (
get_bgp_brief_record,
get_latest_bgp_brief_record,
list_bgp_brief_records,
save_bgp_brief_record,
)
from app.services.playground_session_store import (
get_playground_session,
upsert_playground_session,
)
from app.services.playground_chat_service import (
create_turn,
edit_user_message,
get_thread,
resend_turn,
stop_message,
)
from app.services.situational_alert_ai_brief import build_situational_alert_brief_request
router = APIRouter()
@@ -37,3 +74,208 @@ async def analyze_situational_awareness(
request_id = request.headers.get("X-Request-ID") or str(uuid4())
response.headers["X-Request-ID"] = request_id
return await provider_client.analyze(payload, request_id=request_id)
@router.get("/playground/thread", response_model=PlaygroundThreadResponse | None)
async def get_playground_thread(
session_key: str = "default",
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await get_thread(
db,
user_id=current_user.id,
session_key=session_key,
)
@router.get("/playground/session", response_model=PlaygroundSessionResponse | None)
async def get_saved_playground_session(
session_key: str = "default",
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await get_playground_session(
db,
user_id=current_user.id,
session_key=session_key,
)
@router.put("/playground/session", response_model=PlaygroundSessionResponse)
async def save_playground_session(
payload: PlaygroundSessionUpsertRequest,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await upsert_playground_session(
db,
user_id=current_user.id,
payload=payload,
)
@router.post("/playground/messages", response_model=PlaygroundMessageActionResponse)
async def create_playground_message(
payload: PlaygroundMessageCreateRequest,
current_user: User = Depends(get_current_user),
provider_client: AIProviderClient = Depends(get_ai_provider_client),
db: AsyncSession = Depends(get_db),
):
return await create_turn(
db,
user_id=current_user.id,
payload=payload,
provider_client=provider_client,
)
@router.post("/playground/messages/stop", response_model=PlaygroundMessageActionResponse)
async def stop_playground_message(
payload: PlaygroundMessageStopRequest,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await stop_message(
db,
user_id=current_user.id,
payload=payload,
)
@router.post("/playground/messages/resend", response_model=PlaygroundMessageActionResponse)
async def resend_playground_message(
payload: PlaygroundMessageResendRequest,
current_user: User = Depends(get_current_user),
provider_client: AIProviderClient = Depends(get_ai_provider_client),
db: AsyncSession = Depends(get_db),
):
return await resend_turn(
db,
user_id=current_user.id,
payload=payload,
provider_client=provider_client,
)
@router.post("/playground/messages/edit", response_model=PlaygroundMessageActionResponse)
async def edit_playground_message(
payload: PlaygroundMessageEditRequest,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await edit_user_message(
db,
user_id=current_user.id,
payload=payload,
)
@router.get("/bgp/briefs", response_model=list[BGPBriefRecordSummary])
async def list_saved_bgp_briefs(
current_user: User = Depends(get_current_user),
):
return list_bgp_brief_records()
@router.get("/bgp/briefs/latest", response_model=BGPBriefRecordResponse | None)
async def get_latest_saved_bgp_brief(
current_user: User = Depends(get_current_user),
):
return get_latest_bgp_brief_record()
@router.get("/bgp/briefs/{brief_id}", response_model=BGPBriefRecordResponse)
async def get_saved_bgp_brief(
brief_id: str,
current_user: User = Depends(get_current_user),
):
record = get_bgp_brief_record(brief_id)
if record is None:
raise HTTPException(status_code=404, detail="BGP brief not found")
return record
@router.post("/bgp/brief", response_model=BGPBriefRecordResponse)
async def analyze_bgp_brief(
payload: BGPBriefRequest,
request: Request,
response: Response,
current_user: User = Depends(get_current_user),
provider_client: AIProviderClient = Depends(get_ai_provider_client),
db: AsyncSession = Depends(get_db),
):
request_id = request.headers.get("X-Request-ID") or str(uuid4())
response.headers["X-Request-ID"] = request_id
brief_request, facts, context = await build_bgp_brief_request(
db,
incident_limit=payload.incident_limit,
anomaly_limit=payload.anomaly_limit,
collector_limit=payload.collector_limit,
)
brief_request.preferred_model = payload.preferred_model
brief_request.thinking = payload.thinking
analysis = await provider_client.analyze(brief_request, request_id=request_id)
return save_bgp_brief_record(
analysis,
request_id=request_id,
facts=facts,
context=context,
)
@router.post("/alerts/brief", response_model=AlertBriefResponse)
async def analyze_alert_brief(
payload: AlertBriefRequest,
request: Request,
response: Response,
current_user: User = Depends(get_current_user),
provider_client: AIProviderClient = Depends(get_ai_provider_client),
db: AsyncSession = Depends(get_db),
):
request_id = request.headers.get("X-Request-ID") or str(uuid4())
response.headers["X-Request-ID"] = request_id
brief_request, facts, context = await build_alert_brief_request(
db,
alert_limit=payload.alert_limit,
)
brief_request.preferred_model = payload.preferred_model
brief_request.thinking = payload.thinking
analysis = await provider_client.analyze(brief_request, request_id=request_id)
return AlertBriefResponse(
**analysis.model_dump(),
title=brief_request.title,
objective=brief_request.objective,
facts=facts,
context=context,
)
@router.post("/situational-alerts/brief", response_model=SituationalAlertBriefResponse)
async def analyze_situational_alert_brief(
payload: SituationalAlertBriefRequest,
request: Request,
response: Response,
current_user: User = Depends(get_current_user),
provider_client: AIProviderClient = Depends(get_ai_provider_client),
db: AsyncSession = Depends(get_db),
):
request_id = request.headers.get("X-Request-ID") or str(uuid4())
response.headers["X-Request-ID"] = request_id
brief_request, facts, context = await build_situational_alert_brief_request(db)
brief_request.preferred_model = payload.preferred_model
brief_request.thinking = payload.thinking
analysis = await provider_client.analyze(brief_request, request_id=request_id)
return SituationalAlertBriefResponse(
**analysis.model_dump(),
title=brief_request.title,
objective=brief_request.objective,
facts=facts,
context=context,
)
+42 -22
View File
@@ -1,7 +1,7 @@
from datetime import UTC, datetime
from typing import Optional
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select, func, case
from sqlalchemy.ext.asyncio import AsyncSession
@@ -9,6 +9,7 @@ from app.db.session import get_db
from app.models.user import User
from app.core.security import get_current_user
from app.models.alert import Alert, AlertSeverity, AlertStatus
from app.schemas.alert import AlertResolutionRequest
router = APIRouter()
@@ -77,7 +78,7 @@ async def acknowledge_alert(
@router.post("/{alert_id}/resolve")
async def resolve_alert(
alert_id: int,
resolution: str,
payload: AlertResolutionRequest,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
@@ -85,12 +86,12 @@ async def resolve_alert(
alert = result.scalar_one_or_none()
if not alert:
return {"error": "Alert not found"}
raise HTTPException(status_code=404, detail="Alert not found")
alert.status = AlertStatus.RESOLVED
alert.resolved_by = current_user.id
alert.resolved_at = datetime.now(UTC)
alert.resolution_notes = resolution
alert.resolution_notes = payload.resolution
await db.commit()
return {"message": "Alert resolved", "alert": alert.to_dict()}
@@ -101,25 +102,44 @@ async def get_alert_stats(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
critical_query = select(func.count(Alert.id)).where(
Alert.severity == AlertSeverity.CRITICAL,
Alert.status == AlertStatus.ACTIVE,
result = await db.execute(
select(
func.sum(
case(
(
(Alert.severity == AlertSeverity.CRITICAL)
& (Alert.status == AlertStatus.ACTIVE),
1,
),
else_=0,
)
).label("critical"),
func.sum(
case(
(
(Alert.severity == AlertSeverity.WARNING)
& (Alert.status == AlertStatus.ACTIVE),
1,
),
else_=0,
)
).label("warning"),
func.sum(
case(
(
(Alert.severity == AlertSeverity.INFO)
& (Alert.status == AlertStatus.ACTIVE),
1,
),
else_=0,
)
).label("info"),
)
)
warning_query = select(func.count(Alert.id)).where(
Alert.severity == AlertSeverity.WARNING,
Alert.status == AlertStatus.ACTIVE,
)
info_query = select(func.count(Alert.id)).where(
Alert.severity == AlertSeverity.INFO,
Alert.status == AlertStatus.ACTIVE,
)
critical_result = await db.execute(critical_query)
warning_result = await db.execute(warning_query)
info_result = await db.execute(info_query)
row = result.one()
return {
"critical": critical_result.scalar() or 0,
"warning": warning_result.scalar() or 0,
"info": info_result.scalar() or 0,
"critical": row.critical or 0,
"warning": row.warning or 0,
"info": row.info or 0,
}
+5 -1
View File
@@ -28,7 +28,7 @@ async def login(
):
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active FROM users WHERE username = :username"
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE username = :username"
),
{"username": form_data.username},
)
@@ -46,6 +46,7 @@ async def login(
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
if not verify_password(form_data.password, user.password_hash):
raise HTTPException(
@@ -73,6 +74,7 @@ async def login(
"id": user.id,
"username": user.username,
"role": user.role,
"gatekeeper_groups": user.gatekeeper_groups or [],
},
}
@@ -95,6 +97,7 @@ async def refresh_token(
"id": current_user.id,
"username": current_user.username,
"role": current_user.role,
"gatekeeper_groups": current_user.gatekeeper_groups or [],
},
}
@@ -111,6 +114,7 @@ async def get_me(current_user: User = Depends(get_current_user)):
"username": current_user.username,
"email": current_user.email,
"role": current_user.role,
"gatekeeper_groups": current_user.gatekeeper_groups or [],
"is_active": current_user.is_active,
"created_at": current_user.created_at,
}
+358 -129
View File
@@ -5,13 +5,22 @@ from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from pydantic import BaseModel
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.models.user import User
from app.services.bgp_collector_locations import (
build_bgp_collector_location_query,
collect_bgp_collector_location_candidates,
get_bgp_collector_location_dict,
)
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.ai_client import get_ai_provider_client
from app.services.location.llm_fallback import collect_llm_location_fallback_candidate
router = APIRouter()
@@ -22,16 +31,161 @@ def _parse_dt(value: Optional[str]) -> Optional[datetime]:
if not value:
return None
return datetime.fromisoformat(value.replace("Z", "+00:00"))
def _event_filters(
*,
prefix: Optional[str],
origin_asn: Optional[int],
peer_asn: Optional[int],
collector: Optional[str],
event_type: Optional[str],
source: Optional[str],
time_from: Optional[datetime],
time_to: Optional[datetime],
):
filters = [BGPObservation.source.in_(BGP_SOURCES)]
if source:
filters.append(BGPObservation.source == source)
if prefix:
filters.append(BGPObservation.prefix == prefix)
if origin_asn is not None:
filters.append(BGPObservation.origin_asn == origin_asn)
if peer_asn is not None:
filters.append(BGPObservation.peer_asn == peer_asn)
if collector:
filters.append(BGPObservation.collector == collector)
if event_type:
filters.append(BGPObservation.event_type == event_type)
if time_from:
filters.append(BGPObservation.observed_at >= time_from)
if time_to:
filters.append(BGPObservation.observed_at <= time_to)
return filters
def _matches_time(value: Optional[datetime], time_from: Optional[datetime], time_to: Optional[datetime]) -> bool:
if value is None:
return False
if time_from and value < time_from:
return False
if time_to and value > time_to:
return False
return True
def _anomaly_filters(
*,
severity: Optional[str],
anomaly_type: Optional[str],
status: Optional[str],
prefix: Optional[str],
origin_asn: Optional[int],
time_from: Optional[datetime],
time_to: Optional[datetime],
):
filters = []
if severity:
filters.append(BGPAnomaly.severity == severity)
if anomaly_type:
filters.append(BGPAnomaly.anomaly_type == anomaly_type)
if status:
filters.append(BGPAnomaly.status == status)
if prefix:
filters.append(BGPAnomaly.prefix == prefix)
if origin_asn is not None:
filters.append(BGPAnomaly.origin_asn == origin_asn)
if time_from:
filters.append(BGPAnomaly.created_at >= time_from)
if time_to:
filters.append(BGPAnomaly.created_at <= time_to)
return filters
def _incident_filters(
*,
severity: Optional[str],
incident_type: Optional[str],
status: Optional[str],
):
filters = []
if severity:
filters.append(BGPIncident.severity == severity)
if incident_type:
filters.append(BGPIncident.incident_type == incident_type)
if status:
filters.append(BGPIncident.status == status)
return filters
async def _build_event_summary_payload(db: AsyncSession) -> dict:
base_filters = [BGPObservation.source.in_(BGP_SOURCES)]
total_result = await db.execute(
select(func.count(BGPObservation.id)).where(*base_filters)
)
collectors_result = await db.execute(
select(func.count(func.distinct(BGPObservation.collector))).where(
*base_filters, BGPObservation.collector.isnot(None)
)
)
prefixes_result = await db.execute(
select(func.count(func.distinct(BGPObservation.prefix))).where(
*base_filters, BGPObservation.prefix.isnot(None)
)
)
type_result = await db.execute(
select(BGPObservation.event_type, func.count(BGPObservation.id))
.where(*base_filters)
.group_by(BGPObservation.event_type)
)
return {
"total": total_result.scalar() or 0,
"collector_count": collectors_result.scalar() or 0,
"prefix_count": prefixes_result.scalar() or 0,
"by_type": {row[0]: row[1] for row in type_result.fetchall()},
}
async def _build_anomaly_summary_payload(db: AsyncSession) -> dict:
total_result = await db.execute(select(func.count(BGPAnomaly.id)))
type_result = await db.execute(
select(BGPAnomaly.anomaly_type, func.count(BGPAnomaly.id))
.group_by(BGPAnomaly.anomaly_type)
.order_by(func.count(BGPAnomaly.id).desc())
)
severity_result = await db.execute(
select(BGPAnomaly.severity, func.count(BGPAnomaly.id))
.group_by(BGPAnomaly.severity)
.order_by(func.count(BGPAnomaly.id).desc())
)
status_result = await db.execute(
select(BGPAnomaly.status, func.count(BGPAnomaly.id))
.group_by(BGPAnomaly.status)
.order_by(func.count(BGPAnomaly.id).desc())
)
return {
"total": total_result.scalar() or 0,
"by_type": {row[0]: row[1] for row in type_result.fetchall()},
"by_severity": {row[0]: row[1] for row in severity_result.fetchall()},
"by_status": {row[0]: row[1] for row in status_result.fetchall()},
}
async def _build_incident_summary_payload(db: AsyncSession) -> dict:
total_result = await db.execute(select(func.count(BGPIncident.id)))
type_result = await db.execute(
select(BGPIncident.incident_type, func.count(BGPIncident.id))
.group_by(BGPIncident.incident_type)
.order_by(func.count(BGPIncident.id).desc())
)
severity_result = await db.execute(
select(BGPIncident.severity, func.count(BGPIncident.id))
.group_by(BGPIncident.severity)
.order_by(func.count(BGPIncident.id).desc())
)
status_result = await db.execute(
select(BGPIncident.status, func.count(BGPIncident.id))
.group_by(BGPIncident.status)
.order_by(func.count(BGPIncident.id).desc())
)
return {
"total": total_result.scalar() or 0,
"by_type": {row[0]: row[1] for row in type_result.fetchall()},
"by_severity": {row[0]: row[1] for row in severity_result.fetchall()},
"by_status": {row[0]: row[1] for row in status_result.fetchall()},
}
@router.get("/events")
@@ -49,41 +203,36 @@ async def list_bgp_events(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
stmt = (
select(BGPObservation)
.where(BGPObservation.source.in_(BGP_SOURCES))
.order_by(BGPObservation.observed_at.desc(), BGPObservation.id.desc())
)
if source:
stmt = stmt.where(BGPObservation.source == source)
result = await db.execute(stmt)
records = result.scalars().all()
dt_from = _parse_dt(time_from)
dt_to = _parse_dt(time_to)
filtered = []
for record in records:
if prefix and record.prefix != prefix:
continue
if origin_asn is not None and record.origin_asn != origin_asn:
continue
if peer_asn is not None and record.peer_asn != peer_asn:
continue
if collector and record.collector != collector:
continue
if event_type and record.event_type != event_type:
continue
if (dt_from or dt_to) and not _matches_time(record.observed_at, dt_from, dt_to):
continue
filtered.append(record)
filters = _event_filters(
prefix=prefix,
origin_asn=origin_asn,
peer_asn=peer_asn,
collector=collector,
event_type=event_type,
source=source,
time_from=dt_from,
time_to=dt_to,
)
offset = (page - 1) * page_size
count_result = await db.execute(
select(func.count(BGPObservation.id)).where(*filters)
)
data_result = await db.execute(
select(BGPObservation)
.where(*filters)
.order_by(BGPObservation.observed_at.desc(), BGPObservation.id.desc())
.offset(offset)
.limit(page_size)
)
records = data_result.scalars().all()
return {
"total": len(filtered),
"total": count_result.scalar() or 0,
"page": page,
"page_size": page_size,
"data": [record.to_dict() for record in filtered[offset : offset + page_size]],
"data": [record.to_dict() for record in records],
}
@@ -92,21 +241,7 @@ async def get_bgp_event_summary(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
result = await db.execute(select(BGPObservation).where(BGPObservation.source.in_(BGP_SOURCES)))
records = result.scalars().all()
collectors = sorted({record.collector for record in records if record.collector})
prefixes = sorted({record.prefix for record in records if record.prefix})
by_type: dict[str, int] = {}
for record in records:
by_type[record.event_type] = by_type.get(record.event_type, 0) + 1
return {
"total": len(records),
"collector_count": len(collectors),
"prefix_count": len(prefixes),
"by_type": by_type,
}
return await _build_event_summary_payload(db)
@router.get("/collectors")
@@ -138,6 +273,133 @@ async def get_bgp_collector_summary(
}
class CollectBGPCollectorLocationRequest(BaseModel):
city: Optional[str] = None
country: Optional[str] = None
site: Optional[str] = None
operator: Optional[str] = None
@router.post("/collectors/{collector_id}/collect-location")
async def collect_bgp_collector_location(
collector_id: str,
payload: CollectBGPCollectorLocationRequest,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Run the shared location pipeline for a BGP route collector.
Mirrors ``POST /api/v1/visualization/compute-centers/{source_id}/collect-location``.
Returns ranked candidates from source coordinates and Nominatim queries
built around the collector's stored context (IXP / city / country). Stored
collector locations provide context only; they are not emitted as
candidates.
"""
if not collector_id or not collector_id.strip():
raise HTTPException(status_code=400, detail="collector_id is required")
legacy = get_bgp_collector_location_dict(collector_id) or {}
site = payload.site or legacy.get("matched_location_name")
city = payload.city or legacy.get("city")
country = payload.country or legacy.get("country")
operator = payload.operator or "RIPE NCC"
candidates, attempted_queries = collect_bgp_collector_location_candidates(
collector=collector_id,
site=site,
city=city,
country=country,
operator=operator,
)
llm_failure_reason = None
if not candidates:
query = build_bgp_collector_location_query(
collector=collector_id,
site=site,
city=city,
country=country,
operator=operator,
)
try:
provider_client = await get_ai_provider_client(db)
llm_result = await collect_llm_location_fallback_candidate(
provider_client=provider_client,
query=query,
entity_type="bgp_collector",
attempted_queries=attempted_queries,
)
except Exception as exc:
llm_result = None
llm_failure_reason = f"LLM location factcheck unavailable: {exc}"
attempted_queries = [
*attempted_queries,
f"llm_factcheck:bgp_collector:{collector_id or 'unknown'}",
]
if llm_result is not None:
attempted_queries = [*attempted_queries, *llm_result.attempted_queries]
candidates = llm_result.candidates
llm_failure_reason = llm_result.failure_reason
context = {
"collector": collector_id,
"site": site,
"city": city,
"country": country,
"operator": operator,
}
if not candidates:
return {
"collector_id": collector_id,
"name": collector_id,
"success": False,
"failure_reason": (
"No source coordinates or online geocoding result reached"
" city-level precision for this collector."
),
"candidates": [],
"attempted_queries": list(attempted_queries),
"llm_failure_reason": llm_failure_reason,
"context": context,
}
return {
"collector_id": collector_id,
"name": collector_id,
"success": True,
"candidates": [candidate.to_dict() for candidate in candidates],
"best_candidate": candidates[0].to_dict(),
"attempted_queries": list(attempted_queries),
"context": context,
}
@router.get("/overview/summary")
async def get_bgp_overview_summary(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
event_summary = await _build_event_summary_payload(db)
anomaly_summary = await _build_anomaly_summary_payload(db)
incident_summary = await _build_incident_summary_payload(db)
collectors = await build_bgp_collector_coverage(db, source_filter=BGP_SOURCES)
active_collectors = [item for item in collectors if item["observation_count"] > 0]
return {
"incidentSummary": incident_summary,
"anomalySummary": anomaly_summary,
"eventSummary": event_summary,
"collectorSummary": {
"total": len(collectors),
"active_collectors": len(active_collectors),
"observed_prefixes": sum(item["prefix_count"] for item in active_collectors),
"observed_origins": sum(item["origin_asn_count"] for item in active_collectors),
"recent_24h_events": sum(item["recent_24h_observation_count"] for item in active_collectors),
"recent_7d_events": sum(item["recent_7d_observation_count"] for item in active_collectors),
},
}
@router.get("/events/{event_id}")
async def get_bgp_event(
event_id: int,
@@ -164,31 +426,35 @@ async def list_bgp_anomalies(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
stmt = select(BGPAnomaly).order_by(BGPAnomaly.created_at.desc(), BGPAnomaly.id.desc())
if severity:
stmt = stmt.where(BGPAnomaly.severity == severity)
if anomaly_type:
stmt = stmt.where(BGPAnomaly.anomaly_type == anomaly_type)
if status:
stmt = stmt.where(BGPAnomaly.status == status)
if prefix:
stmt = stmt.where(BGPAnomaly.prefix == prefix)
if origin_asn is not None:
stmt = stmt.where(BGPAnomaly.origin_asn == origin_asn)
result = await db.execute(stmt)
records = result.scalars().all()
dt_from = _parse_dt(time_from)
dt_to = _parse_dt(time_to)
if dt_from or dt_to:
records = [record for record in records if _matches_time(record.created_at, dt_from, dt_to)]
filters = _anomaly_filters(
severity=severity,
anomaly_type=anomaly_type,
status=status,
prefix=prefix,
origin_asn=origin_asn,
time_from=dt_from,
time_to=dt_to,
)
offset = (page - 1) * page_size
total_result = await db.execute(
select(func.count(BGPAnomaly.id)).where(*filters)
)
data_result = await db.execute(
select(BGPAnomaly)
.where(*filters)
.order_by(BGPAnomaly.created_at.desc(), BGPAnomaly.id.desc())
.offset(offset)
.limit(page_size)
)
records = data_result.scalars().all()
return {
"total": len(records),
"total": total_result.scalar() or 0,
"page": page,
"page_size": page_size,
"data": [record.to_dict() for record in records[offset : offset + page_size]],
"data": [record.to_dict() for record in records],
}
@@ -197,29 +463,7 @@ async def get_bgp_anomaly_summary(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
total_result = await db.execute(select(func.count(BGPAnomaly.id)))
type_result = await db.execute(
select(BGPAnomaly.anomaly_type, func.count(BGPAnomaly.id))
.group_by(BGPAnomaly.anomaly_type)
.order_by(func.count(BGPAnomaly.id).desc())
)
severity_result = await db.execute(
select(BGPAnomaly.severity, func.count(BGPAnomaly.id))
.group_by(BGPAnomaly.severity)
.order_by(func.count(BGPAnomaly.id).desc())
)
status_result = await db.execute(
select(BGPAnomaly.status, func.count(BGPAnomaly.id))
.group_by(BGPAnomaly.status)
.order_by(func.count(BGPAnomaly.id).desc())
)
return {
"total": total_result.scalar() or 0,
"by_type": {row[0]: row[1] for row in type_result.fetchall()},
"by_severity": {row[0]: row[1] for row in severity_result.fetchall()},
"by_status": {row[0]: row[1] for row in status_result.fetchall()},
}
return await _build_anomaly_summary_payload(db)
@router.get("/anomalies/{anomaly_id}")
@@ -244,22 +488,29 @@ async def list_bgp_incidents(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
stmt = select(BGPIncident).order_by(BGPIncident.created_at.desc(), BGPIncident.id.desc())
if severity:
stmt = stmt.where(BGPIncident.severity == severity)
if incident_type:
stmt = stmt.where(BGPIncident.incident_type == incident_type)
if status:
stmt = stmt.where(BGPIncident.status == status)
result = await db.execute(stmt)
records = result.scalars().all()
filters = _incident_filters(
severity=severity,
incident_type=incident_type,
status=status,
)
offset = (page - 1) * page_size
total_result = await db.execute(
select(func.count(BGPIncident.id)).where(*filters)
)
data_result = await db.execute(
select(BGPIncident)
.where(*filters)
.order_by(BGPIncident.created_at.desc(), BGPIncident.id.desc())
.offset(offset)
.limit(page_size)
)
records = data_result.scalars().all()
return {
"total": len(records),
"total": total_result.scalar() or 0,
"page": page,
"page_size": page_size,
"data": [record.to_dict() for record in records[offset : offset + page_size]],
"data": [record.to_dict() for record in records],
}
@@ -268,29 +519,7 @@ async def get_bgp_incident_summary(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
total_result = await db.execute(select(func.count(BGPIncident.id)))
type_result = await db.execute(
select(BGPIncident.incident_type, func.count(BGPIncident.id))
.group_by(BGPIncident.incident_type)
.order_by(func.count(BGPIncident.id).desc())
)
severity_result = await db.execute(
select(BGPIncident.severity, func.count(BGPIncident.id))
.group_by(BGPIncident.severity)
.order_by(func.count(BGPIncident.id).desc())
)
status_result = await db.execute(
select(BGPIncident.status, func.count(BGPIncident.id))
.group_by(BGPIncident.status)
.order_by(func.count(BGPIncident.id).desc())
)
return {
"total": total_result.scalar() or 0,
"by_type": {row[0]: row[1] for row in type_result.fetchall()},
"by_severity": {row[0]: row[1] for row in severity_result.fetchall()},
"by_status": {row[0]: row[1] for row in status_result.fetchall()},
}
return await _build_incident_summary_payload(db)
@router.get("/incidents/{incident_id}")
+57 -38
View File
@@ -2,7 +2,7 @@
from datetime import UTC, datetime, timedelta
from fastapi import APIRouter, Depends
from sqlalchemy import select, func, text
from sqlalchemy import case, select, func, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.session import get_db
@@ -118,58 +118,77 @@ async def get_stats(
built_in_count = len(COLLECTOR_INFO)
built_in_active = built_in_count # Built-in are always "active" for counting purposes
# Count custom configs from database
result = await db.execute(select(func.count(DataSourceConfig.id)))
custom_count = result.scalar() or 0
result = await db.execute(
select(func.count(DataSourceConfig.id)).where(DataSourceConfig.is_active == True)
select(
func.count(DataSourceConfig.id).label("custom_count"),
func.sum(
case((DataSourceConfig.is_active == True, 1), else_=0)
).label("custom_active"),
)
)
custom_active = result.scalar() or 0
datasource_stats = result.one()
custom_count = datasource_stats.custom_count or 0
custom_active = datasource_stats.custom_active or 0
# Total datasources
total_datasources = built_in_count + custom_count
active_datasources = built_in_active + custom_active
# Tasks today (from database)
result = await db.execute(
select(func.count(CollectionTask.id)).where(CollectionTask.started_at >= today_start)
)
tasks_today = result.scalar() or 0
result = await db.execute(
select(func.count(CollectionTask.id)).where(
CollectionTask.status == "success",
CollectionTask.started_at >= today_start,
select(
func.count(CollectionTask.id).label("tasks_today"),
func.sum(
case(
(CollectionTask.status == "success", 1),
else_=0,
)
).label("success_tasks"),
)
.where(CollectionTask.started_at >= today_start)
)
success_tasks = result.scalar() or 0
task_stats = result.one()
tasks_today = task_stats.tasks_today or 0
success_tasks = task_stats.success_tasks or 0
success_rate = (success_tasks / tasks_today * 100) if tasks_today > 0 else 0
# Alerts
result = await db.execute(
select(func.count(Alert.id)).where(
Alert.severity == AlertSeverity.CRITICAL,
Alert.status == "active",
select(
func.sum(
case(
(
(Alert.severity == AlertSeverity.CRITICAL)
& (Alert.status == "active"),
1,
),
else_=0,
)
).label("critical_alerts"),
func.sum(
case(
(
(Alert.severity == AlertSeverity.WARNING)
& (Alert.status == "active"),
1,
),
else_=0,
)
).label("warning_alerts"),
func.sum(
case(
(
(Alert.severity == AlertSeverity.INFO)
& (Alert.status == "active"),
1,
),
else_=0,
)
).label("info_alerts"),
)
)
critical_alerts = result.scalar() or 0
result = await db.execute(
select(func.count(Alert.id)).where(
Alert.severity == AlertSeverity.WARNING,
Alert.status == "active",
)
)
warning_alerts = result.scalar() or 0
result = await db.execute(
select(func.count(Alert.id)).where(
Alert.severity == AlertSeverity.INFO,
Alert.status == "active",
)
)
info_alerts = result.scalar() or 0
alert_stats = result.one()
critical_alerts = alert_stats.critical_alerts or 0
warning_alerts = alert_stats.warning_alerts or 0
info_alerts = alert_stats.info_alerts or 0
response = {
"total_datasources": total_datasources,
+695 -31
View File
@@ -1,20 +1,52 @@
"""DataSourceConfig API for user-defined data sources"""
from typing import Optional
from typing import Any, Optional
from datetime import datetime
import base64
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy import select, func
import json
import re
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import delete, select, func
from sqlalchemy.ext.asyncio import AsyncSession
from pydantic import BaseModel, Field
import httpx
from app.core.target_schema_registry import get_target_schema, list_target_schemas
from app.core.datasource_defaults import DEFAULT_DATASOURCES
from app.db.session import get_db
from app.models.user import User
from app.models.datasource_config import DataSourceConfig
from app.models.datasource_mapping import DataSourceMappingTemplate
from app.models.collected_data import CollectedData
from app.models.vessel import AISRawObservation, AISSourceHealth
from app.core.security import get_current_user
from app.core.cache import cache
from app.core.time import to_iso8601_utc
from app.schemas.ai import SituationalAnalysisRequest
from app.services.ai_client import AIProviderClient, get_ai_provider_client
from app.services.datasource_mapping import (
MappingError,
build_heuristic_mapping,
execute_mapping,
redact_for_llm,
stable_payload_hash,
)
from app.services.custom_datasource_runtime import (
CustomDatasourceRuntimeError,
fetch_rest_payload,
get_custom_stream_status,
run_mapped_rest_config,
run_mapped_websocket_config,
start_custom_stream,
stop_custom_stream,
test_websocket_config,
)
from app.services.datasource_connectivity import (
get_builtin_connection_status,
save_connectivity_success,
strip_connectivity_validation,
test_builtin_connectivity,
)
router = APIRouter()
@@ -22,7 +54,7 @@ router = APIRouter()
class DataSourceConfigCreate(BaseModel):
name: str = Field(..., min_length=1, max_length=100)
description: Optional[str] = None
source_type: str = Field(..., description="http, api, database")
source_type: str = Field(..., description="rest, websocket, http, api, database")
endpoint: str = Field(..., max_length=500)
auth_type: str = Field(default="none", description="none, bearer, api_key, basic")
auth_config: dict = Field(default={})
@@ -59,6 +91,70 @@ class DataSourceConfigResponse(BaseModel):
from_attributes = True
def _is_builtin_config_name(name: str | None) -> bool:
return bool(name and name in DEFAULT_DATASOURCES)
async def _ensure_builtin_connection_verified(
db: AsyncSession,
config_data: DataSourceConfigCreate,
) -> None:
if not _is_builtin_config_name(config_data.name):
return
status_result = await get_builtin_connection_status(
db,
config_data.name,
config_data.endpoint,
config_data.auth_type,
config_data.headers,
config_data.config,
)
if not status_result.get("connected"):
raise HTTPException(
status_code=400,
detail=status_result.get("message") or "请先完成连接验证,再保存内置采集器配置。",
)
class CustomSampleRequest(BaseModel):
datasource_config_id: Optional[int] = None
config: Optional[DataSourceConfigCreate] = None
limit_bytes: int = Field(default=200000, ge=1000, le=1000000)
class MappingProposeRequest(BaseModel):
sample_payload: Any
target_schema: str
use_ai: bool = True
class MappingPreviewRequest(BaseModel):
sample_payload: Any
target_schema: str
mapping_json: dict
limit: int = Field(default=20, ge=1, le=100)
class MappingTemplateCreate(BaseModel):
datasource_config_id: int
target_schema: str
mapping_json: dict
sample_payload: Any | None = None
sample_payload_hash: Optional[str] = None
validation_status: str = Field(default="draft", pattern="^(draft|valid|invalid)$")
is_active: bool = False
class MappingTemplateUpdate(BaseModel):
target_schema: Optional[str] = None
mapping_json: Optional[dict] = None
sample_payload: Any | None = None
sample_payload_hash: Optional[str] = None
validation_status: Optional[str] = Field(default=None, pattern="^(draft|valid|invalid)$")
is_active: Optional[bool] = None
async def test_endpoint(
endpoint: str,
auth_type: str,
@@ -96,6 +192,136 @@ async def test_endpoint(
}
def _build_request_headers(auth_type: str, auth_config: dict, headers: dict) -> dict[str, str]:
request_headers = {str(key): str(value) for key, value in (headers or {}).items()}
auth_type = str(auth_type or "none").lower()
auth_config = auth_config or {}
if auth_type == "bearer" and auth_config.get("token"):
request_headers["Authorization"] = f"Bearer {auth_config['token']}"
elif auth_type == "api_key" and auth_config.get("api_key"):
location = str(auth_config.get("in") or auth_config.get("location") or "header").lower()
if location != "query":
key_name = auth_config.get("key_name", "X-API-Key")
request_headers[str(key_name)] = str(auth_config["api_key"])
elif auth_type == "basic":
username = auth_config.get("username", "")
password = auth_config.get("password", "")
credentials = f"{username}:{password}"
encoded = base64.b64encode(credentials.encode()).decode()
request_headers["Authorization"] = f"Basic {encoded}"
return request_headers
def _build_query_params(auth_type: str, auth_config: dict, config: dict) -> dict[str, Any]:
params = {}
candidate = (config or {}).get("params") or (config or {}).get("query_params")
if isinstance(candidate, dict):
params.update(candidate)
auth_type = str(auth_type or "none").lower()
auth_config = auth_config or {}
if auth_type == "api_key" and auth_config.get("api_key"):
location = str(auth_config.get("in") or auth_config.get("location") or "header").lower()
if location == "query":
key_name = auth_config.get("key_name") or auth_config.get("param_name") or "api_key"
params[str(key_name)] = auth_config["api_key"]
return params
async def fetch_custom_sample_from_config(config: DataSourceConfig, limit_bytes: int) -> Any:
if str(config.source_type or "").lower() in {"websocket", "ws"}:
raise HTTPException(status_code=400, detail="WebSocket sources must use connection test or run-mapped stream.")
request_config = config.config or {}
method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
if method not in {"GET", "POST"}:
raise HTTPException(status_code=400, detail="Only GET and POST sample requests are supported.")
headers = _build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {})
params = _build_query_params(config.auth_type, config.auth_config or {}, request_config)
timeout = float(request_config.get("timeout", 30))
json_body = request_config.get("json_body")
if json_body is None and str(request_config.get("body_type") or "").lower() in {"json", ""}:
candidate = request_config.get("body")
if isinstance(candidate, (dict, list)):
json_body = candidate
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
response = await client.request(
method,
config.endpoint,
headers=headers,
params=params or None,
json=json_body,
)
response.raise_for_status()
content = response.content[:limit_bytes]
if "application/json" in response.headers.get("content-type", ""):
return json.loads(content.decode(response.encoding or "utf-8"))
return {"text": content.decode(response.encoding or "utf-8", errors="replace")}
def _parse_mapping_from_ai_text(content: str) -> dict[str, Any] | None:
if not content:
return None
candidates = [content]
fenced = re.findall(r"```(?:json)?\s*(\{.*?\})\s*```", content, flags=re.DOTALL)
candidates = fenced + candidates
for candidate in candidates:
try:
parsed = json.loads(candidate)
except json.JSONDecodeError:
continue
if isinstance(parsed, dict) and isinstance(parsed.get("fields"), dict):
return parsed
return None
async def _get_config_for_sample(
payload: CustomSampleRequest,
db: AsyncSession,
) -> DataSourceConfig:
if payload.datasource_config_id is not None:
result = await db.execute(
select(DataSourceConfig).where(DataSourceConfig.id == payload.datasource_config_id)
)
config = result.scalar_one_or_none()
if not config:
raise HTTPException(status_code=404, detail="Configuration not found")
return config
if payload.config is None:
raise HTTPException(status_code=400, detail="datasource_config_id or config is required")
config_data = payload.config
return DataSourceConfig(
name=config_data.name,
description=config_data.description,
source_type=config_data.source_type,
endpoint=config_data.endpoint,
auth_type=config_data.auth_type,
auth_config=config_data.auth_config,
headers=config_data.headers,
config=config_data.config,
)
def serialize_mapping_template(template: DataSourceMappingTemplate) -> dict[str, Any]:
return {
"id": template.id,
"datasource_config_id": template.datasource_config_id,
"target_schema": template.target_schema,
"mapping_json": template.mapping_json,
"sample_payload_hash": template.sample_payload_hash,
"validation_status": template.validation_status,
"version": template.version,
"is_active": template.is_active,
"created_at": to_iso8601_utc(template.created_at),
"updated_at": to_iso8601_utc(template.updated_at),
}
@router.get("/configs")
async def list_configs(
active_only: bool = False,
@@ -105,7 +331,7 @@ async def list_configs(
"""List all user-defined data source configurations"""
query = select(DataSourceConfig)
if active_only:
query = query.where(DataSourceConfig.is_active == True)
query = query.where(DataSourceConfig.is_active)
query = query.order_by(DataSourceConfig.created_at.desc())
result = await db.execute(query)
@@ -132,6 +358,52 @@ async def list_configs(
}
@router.get("/configs/all")
async def list_all_datasources(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""List all data sources: YAML defaults + DB overrides"""
from app.core.data_sources import COLLECTOR_URL_KEYS, get_data_sources_config
config = get_data_sources_config()
db_query = await db.execute(select(DataSourceConfig))
db_configs = {c.name: c for c in db_query.scalars().all()}
result = []
for name, yaml_key in COLLECTOR_URL_KEYS.items():
yaml_url = config.get_yaml_url(name)
db_config = db_configs.get(name)
result.append(
{
"name": name,
"default_url": yaml_url,
"endpoint": db_config.endpoint if db_config else yaml_url,
"is_overridden": db_config is not None and db_config.endpoint != yaml_url
if yaml_url
else db_config is not None,
"is_active": db_config.is_active if db_config else True,
"source_type": db_config.source_type if db_config else "http",
"auth_type": db_config.auth_type if db_config else "none",
"auth_configured": {
"api_key": bool((db_config.auth_config or {}).get("api_key"))
if db_config
else False,
},
"headers": db_config.headers if db_config else {},
"config": strip_connectivity_validation(db_config.config if db_config else {}),
"config_id": db_config.id if db_config else None,
"description": db_config.description
if db_config
else f"Data source from YAML: {yaml_key}",
}
)
return {"total": len(result), "data": result}
@router.get("/configs/{config_id}")
async def get_config(
config_id: int,
@@ -176,7 +448,7 @@ async def create_config(
auth_type=config_data.auth_type,
auth_config=config_data.auth_config,
headers=config_data.headers,
config=config_data.config,
config=strip_connectivity_validation(config_data.config),
)
db.add(config)
@@ -208,6 +480,10 @@ async def update_config(
update_data = config_data.model_dump(exclude_unset=True)
for field, value in update_data.items():
if field == "config":
value = strip_connectivity_validation(value)
if field == "auth_config" and value == {} and (config.auth_config or {}):
continue
setattr(config, field, value)
await db.commit()
@@ -225,6 +501,8 @@ async def update_config(
@router.delete("/configs/{config_id}")
async def delete_config(
config_id: int,
delete_mappings: bool = Query(False),
delete_source_data: bool = Query(False),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
@@ -235,12 +513,59 @@ async def delete_config(
if not config:
raise HTTPException(status_code=404, detail="Configuration not found")
deleted_mappings = 0
deleted_records = {
"collected_data": 0,
"ais_raw_observations": 0,
"ais_source_health": 0,
}
if delete_source_data:
collected_result = await db.execute(
delete(CollectedData).where(CollectedData.source == config.name)
)
raw_result = await db.execute(
delete(AISRawObservation).where(AISRawObservation.source == config.name)
)
health_result = await db.execute(
delete(AISSourceHealth).where(AISSourceHealth.source == config.name)
)
deleted_records = {
"collected_data": collected_result.rowcount or 0,
"ais_raw_observations": raw_result.rowcount or 0,
"ais_source_health": health_result.rowcount or 0,
}
if delete_mappings or delete_source_data:
mapping_result = await db.execute(
delete(DataSourceMappingTemplate).where(
DataSourceMappingTemplate.datasource_config_id == config_id
)
)
deleted_mappings = mapping_result.rowcount or 0
await db.delete(config)
await db.commit()
cache.delete_pattern("datasource_configs:*")
return {"message": "Configuration deleted successfully"}
if delete_source_data and (config.config or {}).get("target_schema") == "vessel_ais":
from app.core.websocket.broadcaster import broadcaster
await broadcaster.broadcast_custom(
"vessels",
{
"action": "reload",
"source": config.name,
"reason": "custom_source_deleted",
},
)
return {
"message": "Configuration deleted successfully",
"deleted_mappings": deleted_mappings,
"deleted_records": deleted_records,
}
@router.post("/configs/{config_id}/test")
@@ -257,6 +582,8 @@ async def test_config(
raise HTTPException(status_code=404, detail="Configuration not found")
try:
if str(config.source_type or "").lower() in {"websocket", "ws"}:
return await test_websocket_config(config)
result = await test_endpoint(
endpoint=config.endpoint,
auth_type=config.auth_type,
@@ -287,6 +614,18 @@ async def test_new_config(
):
"""Test a new data source configuration without saving"""
try:
if str(config_data.source_type or "").lower() in {"websocket", "ws"}:
config = DataSourceConfig(
name=config_data.name,
description=config_data.description,
source_type=config_data.source_type,
endpoint=config_data.endpoint,
auth_type=config_data.auth_type,
auth_config=config_data.auth_config,
headers=config_data.headers,
config=config_data.config,
)
return await test_websocket_config(config)
result = await test_endpoint(
endpoint=config_data.endpoint,
auth_type=config_data.auth_type,
@@ -310,38 +649,363 @@ async def test_new_config(
}
@router.get("/configs/all")
async def list_all_datasources(
@router.post("/configs/builtin/connection-status")
async def get_builtin_config_connection_status(
config_data: DataSourceConfigCreate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""List all data sources: YAML defaults + DB overrides"""
from app.core.data_sources import COLLECTOR_URL_KEYS, get_data_sources_config
if not _is_builtin_config_name(config_data.name):
raise HTTPException(status_code=400, detail="Only built-in datasource configs are supported.")
config = get_data_sources_config()
return await get_builtin_connection_status(
db,
config_data.name,
config_data.endpoint,
config_data.auth_type,
config_data.headers,
config_data.config,
)
db_query = await db.execute(select(DataSourceConfig))
db_configs = {c.name: c for c in db_query.scalars().all()}
result = []
for name, yaml_key in COLLECTOR_URL_KEYS.items():
yaml_url = config.get_yaml_url(name)
db_config = db_configs.get(name)
@router.post("/configs/builtin/connect")
async def connect_builtin_config(
config_data: DataSourceConfigCreate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
if not _is_builtin_config_name(config_data.name):
raise HTTPException(status_code=400, detail="Only built-in datasource configs are supported.")
result.append(
result = await test_builtin_connectivity(
config_data.name,
config_data.endpoint,
config_data.auth_type,
config_data.headers,
config_data.config,
db,
config_data.auth_config,
)
if result.get("success") and result.get("checksum"):
validation = await save_connectivity_success(
db,
config_data.name,
result["checksum"],
result,
connected_by="connection_button",
)
await db.commit()
return {
**result,
"connected": True,
"validation": validation,
}
return {
**result,
"connected": False,
}
@router.post("/custom/sample")
async def fetch_custom_sample(
payload: CustomSampleRequest,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Fetch a sample payload for a saved or draft custom data source."""
config = await _get_config_for_sample(payload, db)
try:
sample = await fetch_custom_sample_from_config(config, payload.limit_bytes)
except httpx.HTTPStatusError as exc:
raise HTTPException(
status_code=exc.response.status_code,
detail=f"Sample request failed: HTTP {exc.response.status_code}",
) from exc
except httpx.HTTPError as exc:
raise HTTPException(status_code=502, detail=f"Sample request failed: {exc}") from exc
return {
"success": True,
"sample_payload": sample,
"sample_payload_hash": stable_payload_hash(sample),
"redacted_preview": redact_for_llm(sample),
}
@router.get("/target-schemas")
async def get_datasource_target_schemas(
current_user: User = Depends(get_current_user),
):
"""List target schemas available for custom datasource mapping."""
return {"data": list_target_schemas()}
@router.post("/mappings/propose")
async def propose_datasource_mapping(
payload: MappingProposeRequest,
current_user: User = Depends(get_current_user),
ai_client: AIProviderClient = Depends(get_ai_provider_client),
):
"""Generate a mapping draft for a sample payload and target schema."""
schema = get_target_schema(payload.target_schema)
redacted_sample = redact_for_llm(payload.sample_payload)
fallback_mapping = build_heuristic_mapping(redacted_sample, payload.target_schema)
ai_error: str | None = None
mapping = fallback_mapping
generated_by = "heuristic"
if payload.use_ai:
try:
response = await ai_client.analyze(
SituationalAnalysisRequest(
title=f"Generate datasource mapping for {schema.key}",
objective=(
"Return only JSON for a deterministic mapping DSL. "
"The JSON must contain source.items_path and fields. "
"Do not include prose or code."
),
context={
"target_schema": schema.to_dict(),
"sample_payload": redacted_sample,
"mapping_dsl_example": fallback_mapping,
},
observations=[
"Use JSONPath-like paths beginning with $.",
"Never generate executable code.",
"Use field types from the target schema.",
],
constraints=[
"Return a single JSON object.",
"Do not include credentials or secrets.",
"Mark uncertain optional fields with default null.",
],
)
)
parsed = _parse_mapping_from_ai_text(response.content)
if parsed:
mapping = parsed
generated_by = "ai_provider"
else:
ai_error = "AI provider did not return a valid mapping JSON object."
except HTTPException as exc:
ai_error = str(exc.detail)
mapping.setdefault("meta", {})
if isinstance(mapping["meta"], dict):
mapping["meta"].update(
{
"name": name,
"default_url": yaml_url,
"endpoint": db_config.endpoint if db_config else yaml_url,
"is_overridden": db_config is not None and db_config.endpoint != yaml_url
if yaml_url
else db_config is not None,
"is_active": db_config.is_active if db_config else True,
"source_type": db_config.source_type if db_config else "http",
"description": db_config.description
if db_config
else f"Data source from YAML: {yaml_key}",
"generated_by": generated_by,
"requires_review": True,
"ai_error": ai_error,
}
)
return {"total": len(result), "data": result}
return {
"target_schema": schema.to_dict(),
"mapping_json": mapping,
"sample_payload_hash": stable_payload_hash(payload.sample_payload),
"redacted_sample_payload": redacted_sample,
}
@router.post("/mappings/preview")
async def preview_datasource_mapping(
payload: MappingPreviewRequest,
current_user: User = Depends(get_current_user),
):
"""Preview deterministic mapping output for a sample payload."""
try:
preview = execute_mapping(
payload.sample_payload,
payload.mapping_json,
payload.target_schema,
limit=payload.limit,
)
except (MappingError, ValueError) as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return {
"success": preview["failed_count"] == 0,
"preview": preview,
"sample_payload_hash": stable_payload_hash(payload.sample_payload),
}
@router.get("/mappings")
async def list_datasource_mappings(
datasource_config_id: Optional[int] = None,
active_only: bool = False,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""List saved mapping templates."""
query = select(DataSourceMappingTemplate).order_by(
DataSourceMappingTemplate.datasource_config_id,
DataSourceMappingTemplate.version.desc(),
)
if datasource_config_id is not None:
query = query.where(DataSourceMappingTemplate.datasource_config_id == datasource_config_id)
if active_only:
query = query.where(DataSourceMappingTemplate.is_active.is_(True))
result = await db.execute(query)
mappings = result.scalars().all()
return {"total": len(mappings), "data": [serialize_mapping_template(item) for item in mappings]}
@router.post("/mappings")
async def create_datasource_mapping(
payload: MappingTemplateCreate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Save a mapping template for a datasource config."""
get_target_schema(payload.target_schema)
datasource = await db.get(DataSourceConfig, payload.datasource_config_id)
if not datasource:
raise HTTPException(status_code=404, detail="Configuration not found")
if payload.sample_payload is not None:
try:
execute_mapping(payload.sample_payload, payload.mapping_json, payload.target_schema, limit=100)
except (MappingError, ValueError) as exc:
raise HTTPException(status_code=400, detail=f"Mapping validation failed: {exc}") from exc
result = await db.execute(
select(func.max(DataSourceMappingTemplate.version)).where(
DataSourceMappingTemplate.datasource_config_id == payload.datasource_config_id,
DataSourceMappingTemplate.target_schema == payload.target_schema,
)
)
next_version = int(result.scalar() or 0) + 1
if payload.is_active:
await db.execute(
DataSourceMappingTemplate.__table__.update()
.where(DataSourceMappingTemplate.datasource_config_id == payload.datasource_config_id)
.values(is_active=False)
)
template = DataSourceMappingTemplate(
datasource_config_id=payload.datasource_config_id,
target_schema=payload.target_schema,
mapping_json=payload.mapping_json,
sample_payload_hash=payload.sample_payload_hash
or (stable_payload_hash(payload.sample_payload) if payload.sample_payload is not None else None),
validation_status=payload.validation_status,
version=next_version,
is_active=payload.is_active,
)
db.add(template)
await db.commit()
await db.refresh(template)
return {"message": "Mapping template saved successfully", "data": serialize_mapping_template(template)}
@router.put("/mappings/{mapping_id}")
async def update_datasource_mapping(
mapping_id: int,
payload: MappingTemplateUpdate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Update a mapping template in place."""
template = await db.get(DataSourceMappingTemplate, mapping_id)
if not template:
raise HTTPException(status_code=404, detail="Mapping template not found")
target_schema = payload.target_schema or template.target_schema
mapping_json = payload.mapping_json or template.mapping_json
get_target_schema(target_schema)
if payload.sample_payload is not None:
try:
execute_mapping(payload.sample_payload, mapping_json, target_schema, limit=100)
except (MappingError, ValueError) as exc:
raise HTTPException(status_code=400, detail=f"Mapping validation failed: {exc}") from exc
if payload.is_active is True:
await db.execute(
DataSourceMappingTemplate.__table__.update()
.where(DataSourceMappingTemplate.datasource_config_id == template.datasource_config_id)
.where(DataSourceMappingTemplate.id != template.id)
.values(is_active=False)
)
template.target_schema = target_schema
template.mapping_json = mapping_json
if payload.sample_payload_hash is not None:
template.sample_payload_hash = payload.sample_payload_hash
elif payload.sample_payload is not None:
template.sample_payload_hash = stable_payload_hash(payload.sample_payload)
if payload.validation_status is not None:
template.validation_status = payload.validation_status
if payload.is_active is not None:
template.is_active = payload.is_active
await db.commit()
await db.refresh(template)
return {"message": "Mapping template updated successfully", "data": serialize_mapping_template(template)}
@router.post("/{config_id}/run-mapped")
async def run_mapped_datasource(
config_id: int,
background: bool = Query(False, description="For WebSocket sources, start a background stream task."),
debug_max_messages: int | None = Query(None, ge=1),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Run a saved custom datasource through its active deterministic mapping."""
datasource = await db.get(DataSourceConfig, config_id)
if not datasource:
raise HTTPException(status_code=404, detail="Configuration not found")
try:
if str(datasource.source_type or "").lower() in {"websocket", "ws"}:
if background and debug_max_messages is None:
started = start_custom_stream(config_id)
if not started:
raise HTTPException(status_code=409, detail="Custom WebSocket source is already running")
return {
"status": "started",
"datasource_config_id": config_id,
"stream": get_custom_stream_status(config_id),
}
return await run_mapped_websocket_config(
db,
datasource,
debug_max_messages=debug_max_messages,
)
return await run_mapped_rest_config(db, datasource)
except httpx.HTTPStatusError as exc:
raise HTTPException(
status_code=exc.response.status_code,
detail=f"Datasource request failed: HTTP {exc.response.status_code}",
) from exc
except httpx.HTTPError as exc:
raise HTTPException(status_code=502, detail=f"Datasource request failed: {exc}") from exc
except (CustomDatasourceRuntimeError, MappingError, ValueError) as exc:
raise HTTPException(status_code=400, detail=f"Mapping failed: {exc}") from exc
@router.post("/{config_id}/stop-mapped")
async def stop_mapped_datasource(
config_id: int,
current_user: User = Depends(get_current_user),
):
stopped = await stop_custom_stream(config_id)
return {
"status": "stopped" if stopped else "not_running",
"datasource_config_id": config_id,
"stream": get_custom_stream_status(config_id),
}
@router.get("/{config_id}/stream-status")
async def get_mapped_stream_status(
config_id: int,
current_user: User = Depends(get_current_user),
):
return get_custom_stream_status(config_id)
+348 -39
View File
@@ -3,18 +3,26 @@ from datetime import datetime, timedelta, timezone
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import func, select
from sqlalchemy import func, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.time import to_iso8601_utc
from app.core.security import get_current_user
from app.core.data_sources import get_data_sources_config
from app.core.datasource_defaults import DEFAULT_DATASOURCES
from app.db.session import get_db
from app.models.collected_data import CollectedData
from app.models.data_snapshot import DataSnapshot
from app.models.datasource import DataSource
from app.models.datasource_config import DataSourceConfig
from app.models.task import CollectionTask
from app.models.user import User
from app.services.scheduler import get_latest_task_id_for_datasource, run_collector_now, sync_datasource_job
from app.services.scheduler import (
cancel_running_collector_now,
get_latest_task_id_for_datasource,
run_collector_now,
sync_datasource_job,
)
router = APIRouter()
STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90
@@ -28,12 +36,131 @@ def format_frequency_label(minutes: int) -> str:
return f"{minutes}m"
def datasource_metadata(source: str) -> dict:
info = DEFAULT_DATASOURCES.get(source, {})
return {
"display_name": info.get("display_name") or info.get("name") or source,
"is_free": bool(info.get("is_free", True)),
"requires_credentials": bool(info.get("requires_credentials", False)),
"credential_provider": info.get("credential_provider"),
"credential_status": info.get("credential_status", "none"),
}
def is_due_for_collection(datasource: DataSource, now: datetime) -> bool:
if datasource.last_run_at is None:
return True
return datasource.last_run_at + timedelta(minutes=datasource.frequency_minutes) <= now
def _task_rank_column(order_column):
return func.row_number().over(
partition_by=CollectionTask.datasource_id,
order_by=(order_column.desc().nullslast(), CollectionTask.id.desc()),
).label("row_num")
async def _load_latest_running_tasks(
db: AsyncSession,
datasource_ids: list[int],
) -> dict[int, CollectionTask]:
if not datasource_ids:
return {}
ranked_tasks = (
select(
CollectionTask.id.label("task_id"),
_task_rank_column(CollectionTask.started_at),
)
.where(CollectionTask.datasource_id.in_(datasource_ids))
.where(CollectionTask.status == "running")
.subquery()
)
result = await db.execute(
select(CollectionTask)
.join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id)
.where(ranked_tasks.c.row_num == 1)
)
return {task.datasource_id: task for task in result.scalars().all()}
async def _load_latest_task_ids(
db: AsyncSession,
datasource_ids: list[int],
) -> dict[int, int]:
if not datasource_ids:
return {}
ranked_tasks = (
select(
CollectionTask.id.label("task_id"),
CollectionTask.datasource_id.label("datasource_id"),
func.row_number().over(
partition_by=CollectionTask.datasource_id,
order_by=CollectionTask.id.desc(),
).label("row_num"),
)
.where(CollectionTask.datasource_id.in_(datasource_ids))
.subquery()
)
result = await db.execute(
select(ranked_tasks.c.datasource_id, ranked_tasks.c.task_id)
.where(ranked_tasks.c.row_num == 1)
)
return {datasource_id: task_id for datasource_id, task_id in result.all()}
async def _load_datasource_endpoint_overrides(
db: AsyncSession,
sources: list[str],
) -> dict[str, str]:
if not sources:
return {}
result = await db.execute(
select(DataSourceConfig.name, DataSourceConfig.endpoint)
.where(DataSourceConfig.name.in_(sources))
.where(DataSourceConfig.is_active.is_(True))
.where(DataSourceConfig.endpoint.isnot(None))
)
return {
name: endpoint
for name, endpoint in result.all()
if endpoint
}
async def _load_datasource_list_context(
db: AsyncSession,
datasources: list[DataSource],
) -> tuple[dict[int, CollectionTask], dict[str, str]]:
datasource_ids = [datasource.id for datasource in datasources]
sources = [datasource.source for datasource in datasources]
running_tasks = await _load_latest_running_tasks(db, datasource_ids)
datasource_by_id = {datasource.id: datasource for datasource in datasources}
now = datetime.now(timezone.utc)
stale_datasource_ids: list[int] = []
for datasource_id, task in running_tasks.items():
started_at = task.started_at
if started_at is None:
continue
if started_at.tzinfo is None:
started_at = started_at.replace(tzinfo=timezone.utc)
if now - started_at > timedelta(minutes=STALE_RUNNING_TASK_TIMEOUT_MINUTES):
datasource = datasource_by_id.get(datasource_id)
if datasource is not None:
await fail_and_rollback_stale_running_task(db, datasource, task)
stale_datasource_ids.append(datasource_id)
if stale_datasource_ids:
running_tasks = await _load_latest_running_tasks(db, datasource_ids)
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
return running_tasks, endpoint_overrides
async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
datasource = None
try:
@@ -52,18 +179,6 @@ async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[Da
return result.scalar_one_or_none()
async def get_last_completed_task(db: AsyncSession, datasource_id: int) -> Optional[CollectionTask]:
result = await db.execute(
select(CollectionTask)
.where(CollectionTask.datasource_id == datasource_id)
.where(CollectionTask.completed_at.isnot(None))
.where(CollectionTask.status.in_(("success", "failed", "cancelled")))
.order_by(CollectionTask.completed_at.desc())
.limit(1)
)
return result.scalar_one_or_none()
async def get_running_task(db: AsyncSession, datasource_id: int) -> Optional[CollectionTask]:
result = await db.execute(
select(CollectionTask)
@@ -87,17 +202,152 @@ async def get_running_task(db: AsyncSession, datasource_id: int) -> Optional[Col
if now - started_at <= timedelta(minutes=STALE_RUNNING_TASK_TIMEOUT_MINUTES):
return task
existing_error = (task.error_message or "").strip()
datasource = await db.get(DataSource, datasource_id)
if datasource is not None:
await fail_and_rollback_stale_running_task(db, datasource, task)
else:
existing_error = (task.error_message or "").strip()
stale_reason = (
f"Marked failed automatically after stale running timeout "
f"({STALE_RUNNING_TASK_TIMEOUT_MINUTES}m)"
)
task.status = "failed"
task.phase = "failed"
task.completed_at = now
task.error_message = f"{existing_error}\n{stale_reason}".strip() if existing_error else stale_reason
await db.commit()
return None
async def rollback_orphaned_running_task(
db: AsyncSession,
datasource: DataSource,
running_task: CollectionTask,
) -> None:
snapshot_result = await db.execute(
select(DataSnapshot)
.where(
DataSnapshot.datasource_id == datasource.id,
DataSnapshot.task_id == running_task.id,
)
.order_by(DataSnapshot.id.desc())
.limit(1)
)
snapshot = snapshot_result.scalar_one_or_none()
await db.execute(CollectedData.__table__.delete().where(CollectedData.task_id == running_task.id))
await db.execute(
text(
"""
UPDATE collected_data
SET is_current = FALSE
WHERE source = :source
"""
),
{"source": datasource.source},
)
if snapshot is not None:
snapshot.status = "cancelled"
snapshot.is_current = False
snapshot.completed_at = datetime.now(timezone.utc)
summary = dict(snapshot.summary or {})
summary["rollback"] = True
summary["rollback_reason"] = "orphaned_running_task_after_backend_restart"
snapshot.summary = summary
if snapshot.parent_snapshot_id is not None:
parent_snapshot = await db.get(DataSnapshot, snapshot.parent_snapshot_id)
if parent_snapshot:
parent_snapshot.is_current = True
await db.execute(
text(
"""
UPDATE collected_data
SET is_current = TRUE
WHERE snapshot_id = :snapshot_id
"""
),
{"snapshot_id": snapshot.parent_snapshot_id},
)
running_task.status = "cancelled"
running_task.phase = "cancelled"
running_task.completed_at = datetime.now(timezone.utc)
existing_error = (running_task.error_message or "").strip()
cancel_reason = "Cancelled after backend restart because the running task handle was lost; incomplete writes rolled back"
running_task.error_message = f"{existing_error}\n{cancel_reason}".strip() if existing_error else cancel_reason
datasource.last_status = "cancelled"
datasource.last_run_at = datetime.now(timezone.utc)
await db.commit()
async def fail_and_rollback_stale_running_task(
db: AsyncSession,
datasource: DataSource,
running_task: CollectionTask,
) -> None:
snapshot_result = await db.execute(
select(DataSnapshot)
.where(
DataSnapshot.datasource_id == datasource.id,
DataSnapshot.task_id == running_task.id,
)
.order_by(DataSnapshot.id.desc())
.limit(1)
)
snapshot = snapshot_result.scalar_one_or_none()
await db.execute(CollectedData.__table__.delete().where(CollectedData.task_id == running_task.id))
await db.execute(
text(
"""
UPDATE collected_data
SET is_current = FALSE
WHERE source = :source
"""
),
{"source": datasource.source},
)
if snapshot is not None:
snapshot.status = "failed"
snapshot.is_current = False
snapshot.completed_at = datetime.now(timezone.utc)
summary = dict(snapshot.summary or {})
summary["rollback"] = True
summary["rollback_reason"] = "stale_running_task_timeout"
snapshot.summary = summary
if snapshot.parent_snapshot_id is not None:
parent_snapshot = await db.get(DataSnapshot, snapshot.parent_snapshot_id)
if parent_snapshot:
parent_snapshot.is_current = True
await db.execute(
text(
"""
UPDATE collected_data
SET is_current = TRUE
WHERE snapshot_id = :snapshot_id
"""
),
{"snapshot_id": snapshot.parent_snapshot_id},
)
existing_error = (running_task.error_message or "").strip()
stale_reason = (
f"Marked failed automatically after stale running timeout "
f"({STALE_RUNNING_TASK_TIMEOUT_MINUTES}m)"
f"({STALE_RUNNING_TASK_TIMEOUT_MINUTES}m); incomplete writes rolled back"
)
task.status = "failed"
task.phase = "failed"
task.completed_at = now
task.error_message = f"{existing_error}\n{stale_reason}".strip() if existing_error else stale_reason
running_task.status = "failed"
running_task.phase = "failed"
running_task.completed_at = datetime.now(timezone.utc)
running_task.error_message = f"{existing_error}\n{stale_reason}".strip() if existing_error else stale_reason
datasource.last_status = "failed"
datasource.last_run_at = datetime.now(timezone.utc)
await db.commit()
return None
@router.get("")
@@ -121,23 +371,19 @@ async def list_datasources(
collector_list = []
config = get_data_sources_config()
running_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
for datasource in datasources:
running_task = await get_running_task(db, datasource.id)
last_task = await get_last_completed_task(db, datasource.id)
endpoint = await config.get_url(datasource.source, db)
data_count_result = await db.execute(
select(func.count(CollectedData.id)).where(CollectedData.source == datasource.source)
)
data_count = data_count_result.scalar() or 0
last_run_at = datasource.last_run_at or (last_task.completed_at if last_task else None)
last_run = to_iso8601_utc(last_run_at)
last_status = datasource.last_status or (last_task.status if last_task else None)
running_task = running_tasks.get(datasource.id)
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
last_run_at = datasource.last_run_at
last_status = datasource.last_status
collector_list.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
**datasource_metadata(datasource.source),
"module": datasource.module,
"priority": datasource.priority,
"frequency": format_frequency_label(datasource.frequency_minutes),
@@ -145,15 +391,18 @@ async def list_datasources(
"is_active": datasource.is_active,
"collector_class": datasource.collector_class,
"endpoint": endpoint,
"last_run": last_run,
"last_run": to_iso8601_utc(last_run_at),
"last_run_at": to_iso8601_utc(last_run_at),
"last_status": last_status,
"last_records_processed": last_task.records_processed if last_task else None,
"data_count": data_count,
"is_running": running_task is not None,
"task_id": running_task.id if running_task else None,
"progress": running_task.progress if running_task else None,
"phase": running_task.phase if running_task else None,
"phase_progress": running_task.phase_progress if running_task else None,
"phase_message": running_task.phase_message if running_task else None,
"phase_current": running_task.phase_current if running_task else None,
"phase_total": running_task.phase_total if running_task else None,
"phase_unit": running_task.phase_unit if running_task else None,
"records_processed": running_task.records_processed if running_task else None,
"total_records": running_task.total_records if running_task else None,
}
@@ -189,9 +438,13 @@ async def trigger_all_datasources(
skipped_sources: list[dict] = []
failed_sources: list[dict] = []
now = datetime.now(timezone.utc)
running_tasks = await _load_latest_running_tasks(
db,
[datasource.id for datasource in datasources],
)
for datasource in datasources:
running_task = await get_running_task(db, datasource.id)
running_task = running_tasks.get(datasource.id)
if running_task is not None:
skipped_sources.append(
{
@@ -219,7 +472,7 @@ async def trigger_all_datasources(
)
continue
previous_task_ids[datasource.id] = await get_latest_task_id_for_datasource(datasource.id)
previous_task_ids[datasource.id] = None
success = run_collector_now(datasource.source)
if not success:
failed_sources.append(
@@ -241,13 +494,24 @@ async def trigger_all_datasources(
}
)
latest_task_ids = await _load_latest_task_ids(
db,
[datasource.id for datasource in datasources],
)
for datasource_id in previous_task_ids:
previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id)
for _ in range(20):
await asyncio.sleep(0.1)
pending = [item for item in triggered_sources if item["task_id"] is None]
if not pending:
break
latest_task_ids = await _load_latest_task_ids(
db,
[item["id"] for item in pending],
)
for item in pending:
task_id = await get_latest_task_id_for_datasource(item["id"])
task_id = latest_task_ids.get(item["id"])
if task_id is not None and task_id != previous_task_ids.get(item["id"]):
item["task_id"] = task_id
@@ -277,6 +541,7 @@ async def get_datasource(
return {
"id": datasource.id,
"name": datasource.name,
**datasource_metadata(datasource.source),
"module": datasource.module,
"priority": datasource.priority,
"frequency": format_frequency_label(datasource.frequency_minutes),
@@ -346,6 +611,7 @@ async def get_datasource_stats(
@router.post("/{source_id}/trigger")
async def trigger_datasource(
source_id: str,
force: bool = Query(False),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
@@ -356,6 +622,31 @@ async def trigger_datasource(
if not datasource.is_active:
raise HTTPException(status_code=400, detail="Data source is disabled")
running_task = await get_running_task(db, datasource.id)
if running_task is not None and not force:
raise HTTPException(
status_code=409,
detail={
"reason": "running_task_in_progress",
"message": "当前采集任务尚未完成,重新触发会丢失本次未完成进度。是否强制重新采集?",
"task_id": running_task.id,
"phase": running_task.phase,
"phase_progress": running_task.phase_progress,
"phase_message": running_task.phase_message,
"phase_current": running_task.phase_current,
"phase_total": running_task.phase_total,
"phase_unit": running_task.phase_unit,
"progress": running_task.progress,
"records_processed": running_task.records_processed,
"total_records": running_task.total_records,
},
)
if running_task is not None and force:
cancelled = await cancel_running_collector_now(datasource.source)
if not cancelled:
await rollback_orphaned_running_task(db, datasource, running_task)
previous_task_id = await get_latest_task_id_for_datasource(datasource.id)
success = run_collector_now(datasource.source)
if not success:
@@ -375,6 +666,7 @@ async def trigger_datasource(
"source_id": datasource.id,
"task_id": task_id,
"collector_name": datasource.source,
"force": force,
"message": f"Collector '{datasource.source}' has been triggered",
}
@@ -412,6 +704,7 @@ async def clear_datasource_data(
async def get_task_status(
source_id: str,
task_id: Optional[int] = None,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
datasource = await get_datasource_record(db, source_id)
@@ -426,13 +719,29 @@ async def get_task_status(
task = await get_running_task(db, datasource.id)
if not task:
return {"is_running": False, "task_id": None, "progress": None, "phase": None, "status": "idle"}
return {
"is_running": False,
"task_id": None,
"progress": None,
"phase": None,
"phase_progress": None,
"phase_message": None,
"phase_current": None,
"phase_total": None,
"phase_unit": None,
"status": "idle",
}
return {
"is_running": task.status == "running",
"task_id": task.id,
"progress": task.progress,
"phase": task.phase,
"phase_progress": task.phase_progress,
"phase_message": task.phase_message,
"phase_current": task.phase_current,
"phase_total": task.phase_total,
"phase_unit": task.phase_unit,
"records_processed": task.records_processed,
"total_records": task.total_records,
"status": task.status,
+102
View File
@@ -0,0 +1,102 @@
"""Authenticated documentation APIs."""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from sqlalchemy import text
from app.core.security import decode_token
from app.db.session import async_session_factory
from app.models.user import User
from app.services.docs_gatekeeper import (
DOCS_BY_SLUG,
VALID_DOCS_LANGS,
can_read_doc,
catalog_for_user,
doc_path_for,
title_for,
)
router = APIRouter()
optional_bearer = HTTPBearer(auto_error=False)
async def get_optional_current_user(
credentials: HTTPAuthorizationCredentials | None = Depends(optional_bearer),
) -> User | None:
if credentials is None:
return None
payload = decode_token(credentials.credentials)
if payload is None or payload.get("type") != "access" or payload.get("sub") is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid token",
)
async with async_session_factory() as db:
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": int(payload["sub"])},
)
row = result.fetchone()
if row is None or not row[5]:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="User not found or inactive",
)
user = User()
user.id = row[0]
user.username = row[1]
user.email = row[2]
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
return user
@router.get("/catalog")
async def get_docs_catalog(current_user: User | None = Depends(get_optional_current_user)):
return {
"items": catalog_for_user(current_user),
"authenticated": current_user is not None,
}
@router.get("/{lang}/{slug}")
async def get_doc_content(
lang: str,
slug: str,
current_user: User | None = Depends(get_optional_current_user),
):
if lang not in VALID_DOCS_LANGS:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
entry = DOCS_BY_SLUG.get(slug)
if entry is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
path = doc_path_for(entry, lang)
if not path.exists():
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
if not can_read_doc(entry, current_user):
if current_user is None:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authentication required")
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Insufficient Docs permissions")
return {
"slug": entry.slug,
"filename": entry.filename,
"lang": lang,
"title": title_for(entry, lang),
"group": entry.group,
"order": entry.order,
"access": entry.access,
"markdown": path.read_text(encoding="utf-8"),
}
+13
View File
@@ -0,0 +1,13 @@
from fastapi import APIRouter, Query
from app.services.earth_news import get_earth_news_payload
router = APIRouter()
@router.get("/earth-feed")
async def get_earth_feed(
lat: float | None = Query(None, description="Current Earth view center latitude"),
lon: float | None = Query(None, description="Current Earth view center longitude"),
):
return await get_earth_news_payload(lat=lat, lon=lon)
+711 -9
View File
@@ -1,18 +1,51 @@
from copy import deepcopy
from datetime import UTC, datetime
from pathlib import Path
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel, EmailStr, Field
from dotenv import dotenv_values
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.security import get_current_user
from app.core.time import to_iso8601_utc
from app.core.config import settings as app_settings
from app.core.data_sources import get_data_sources_config
from app.core.datasource_defaults import DEFAULT_DATASOURCES
from app.db.session import get_db
from app.models.datasource import DataSource
from app.models.datasource_config import DataSourceConfig
from app.models.system_setting import SystemSetting
from app.models.user import User
from app.models.vessel import AISSourceHealth
from app.schemas.ai import SituationalAnalysisRequest
from app.services.barentswatch import (
BarentsWatchConfig,
check_barentswatch_config,
check_barentswatch_connectivity,
get_barentswatch_datasource_record,
resolve_barentswatch_config,
)
from app.services.credential_guides import (
generate_credential_guide,
get_credential_guide,
reset_credential_guide,
)
from app.services.datasource_connectivity import (
build_builtin_connectivity_checksum,
save_connectivity_success,
)
from app.services.ai_client import AIProviderClient, get_ai_provider_client
from app.services.llm_provider_catalog import (
FALLBACK_LLM_PROVIDER_PRESETS,
get_fallback_llm_provider_preset,
list_fallback_llm_provider_presets,
refresh_llm_provider_preset,
)
from app.services.scheduler import sync_datasource_job
from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings
router = APIRouter()
@@ -36,6 +69,17 @@ DEFAULT_SETTINGS = {
"max_login_attempts": 5,
"password_policy": "medium",
},
"tv": DEFAULT_TV_SETTINGS,
"external_integrations": {
"ai_provider": {
"service_url": "",
"service_token": "",
"default_provider": "minimax",
"providers": {},
"timeout_seconds": 60,
"retry_attempts": 2,
}
},
}
@@ -67,8 +111,63 @@ class CollectorSettingsUpdate(BaseModel):
frequency_minutes: int = Field(default=60, ge=1, le=10080)
class TVStreamSourceUpdate(BaseModel):
id: str = Field(min_length=1, max_length=100)
name: str = Field(min_length=1, max_length=200)
provider: str = Field(default="Unknown", max_length=100)
region: str = Field(default="Global", max_length=100)
language: str = Field(default="und", max_length=32)
source_type: str = Field(default="iframe", pattern="^(iframe|hls|video|external|youtube)$")
embed_url: str = ""
stream_url: str = ""
homepage_url: str = ""
poster_url: str = ""
youtube_video_id: str = ""
youtube_channel: str = ""
is_enabled: bool = True
is_fallback: bool = False
sort_order: int = Field(default=10, ge=0, le=9999)
collector_source: Optional[str] = None
notes: str = ""
class TVSettingsUpdate(BaseModel):
default_source_id: str = Field(default=DEFAULT_TV_SETTINGS["default_source_id"], min_length=1)
auto_fallback: bool = True
sources: list[TVStreamSourceUpdate] = Field(default_factory=list)
class AIProviderIntegrationUpdate(BaseModel):
service_url: str = ""
service_token: Optional[str] = None
default_provider: Optional[str] = None
provider: str = Field(default="minimax", max_length=80)
provider_api: str = Field(default="anthropic-messages", max_length=80)
base_url: str = Field(default="", max_length=500)
model: str = Field(default="", max_length=200)
api_key: Optional[str] = None
max_tokens: int = Field(default=1200, ge=1, le=200000)
anthropic_version: str = Field(default="2023-06-01", max_length=40)
timeout_seconds: int = Field(default=60, ge=5, le=600)
retry_attempts: int = Field(default=2, ge=1, le=10)
clear_service_token: bool = False
clear_api_key: bool = False
class BarentsWatchIntegrationUpdate(BaseModel):
endpoint: str = ""
client_id: str = ""
client_secret: Optional[str] = None
clear_client_secret: bool = False
class ExternalIntegrationsUpdate(BaseModel):
ai_provider: AIProviderIntegrationUpdate
barentswatch: BarentsWatchIntegrationUpdate
def merge_with_defaults(category: str, payload: Optional[dict]) -> dict:
merged = DEFAULT_SETTINGS[category].copy()
merged = deepcopy(DEFAULT_SETTINGS[category])
if payload:
merged.update(payload)
return merged
@@ -79,6 +178,26 @@ async def get_setting_record(db: AsyncSession, category: str) -> Optional[System
return result.scalar_one_or_none()
async def get_setting_payloads(db: AsyncSession, categories: list[str]) -> dict[str, dict]:
if not categories:
return {}
result = await db.execute(
select(SystemSetting).where(SystemSetting.category.in_(categories))
)
records_by_category = {
record.category: record
for record in result.scalars().all()
}
return {
category: merge_with_defaults(
category,
records_by_category.get(category).payload if records_by_category.get(category) else None,
)
for category in categories
}
async def get_setting_payload(db: AsyncSession, category: str) -> dict:
record = await get_setting_record(db, category)
return merge_with_defaults(category, record.payload if record else None)
@@ -97,6 +216,341 @@ async def save_setting_payload(db: AsyncSession, category: str, payload: dict) -
return merge_with_defaults(category, record.payload)
AI_PROVIDER_ENV_FILE = Path(__file__).resolve().parents[4] / "aiprovider" / ".env"
def _mask_secret(value: Optional[str], source: str = "") -> dict:
if not value:
return {"configured": False, "preview": "", "source": source}
text = str(value)
if "-" in text:
prefix = text.split("-", 1)[0] + "-"
preview = prefix + ("*" * max(len(text) - len(prefix), 1))
else:
preview = "*" * len(text)
return {"configured": True, "preview": preview, "source": source}
def _normalize_provider_id(provider: Optional[str]) -> str:
return (provider or "minimax").strip().lower() or "minimax"
def _get_provider_preset(provider: str) -> dict:
try:
return get_fallback_llm_provider_preset(provider)
except ValueError:
return {
"provider": provider,
"provider_api": "openai-completions",
"base_url": "",
"model": "",
"models": [],
"api_key_env": "",
}
def _read_ai_provider_env_file() -> dict[str, str]:
if not AI_PROVIDER_ENV_FILE.exists():
return {}
return {
key: str(value)
for key, value in dotenv_values(AI_PROVIDER_ENV_FILE).items()
if value is not None
}
def _resolve_env_secret(*names: str) -> tuple[str, str]:
env_file_values = _read_ai_provider_env_file()
for name in names:
if not name:
continue
value = env_file_values.get(name)
if value:
return value, "env_file"
return "", ""
def _provider_defaults(provider: str) -> dict:
preset = _get_provider_preset(provider)
return {
"provider": provider,
"provider_api": preset.get("provider_api") or "openai-completions",
"base_url": preset.get("base_url") or "",
"model": preset.get("model") or "",
"api_key": "",
"max_tokens": (
1200 if preset.get("provider_api") == "anthropic-messages" else 4096
),
"anthropic_version": "2023-06-01",
}
def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict:
raw = dict(ai_payload or {})
default_provider = _normalize_provider_id(raw.get("default_provider") or raw.get("provider"))
providers = {
_normalize_provider_id(provider): dict(config or {})
for provider, config in (raw.get("providers") or {}).items()
if provider
}
legacy_fields = {
key: raw.get(key)
for key in (
"provider_api",
"base_url",
"model",
"api_key",
"max_tokens",
"anthropic_version",
)
if raw.get(key) not in (None, "")
}
if legacy_fields:
providers[default_provider] = {
**providers.get(default_provider, {}),
**legacy_fields,
}
normalized_providers: dict[str, dict] = {}
for provider, config in providers.items():
provider_id = _normalize_provider_id(provider)
normalized_providers[provider_id] = {
**_provider_defaults(provider_id),
**dict(config or {}),
"provider": provider_id,
}
if default_provider not in normalized_providers:
normalized_providers[default_provider] = _provider_defaults(default_provider)
return {
"service_url": raw.get("service_url") or "",
"service_token": raw.get("service_token") or "",
"default_provider": default_provider,
"providers": normalized_providers,
"timeout_seconds": int(raw.get("timeout_seconds") or 60),
"retry_attempts": int(raw.get("retry_attempts") or 2),
}
def _resolve_provider_api_key(provider: str, provider_config: dict) -> tuple[str, str]:
saved_key = provider_config.get("api_key") or ""
if saved_key:
return str(saved_key), "runtime"
preset = _get_provider_preset(provider)
api_key_env = preset.get("api_key_env") or ""
return _resolve_env_secret(api_key_env, "AI_API_KEY")
def _resolve_service_token(ai_payload: dict) -> tuple[str, str]:
saved_token = ai_payload.get("service_token") or ""
if saved_token:
return str(saved_token), "runtime"
token, source = _resolve_env_secret("AI_PROVIDER_SERVICE_TOKEN")
if token:
return token, source
if app_settings.AI_PROVIDER_SERVICE_TOKEN:
return app_settings.AI_PROVIDER_SERVICE_TOKEN, "backend_env"
return "", ""
def _is_secret_placeholder(value: Optional[str], current_preview: str = "") -> bool:
if value in (None, ""):
return True
text = str(value).strip()
if not text:
return True
return text == current_preview or text.startswith("••••") or "*" in text
def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrationUpdate) -> dict:
current_ai = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
provider_id = _normalize_provider_id(update.default_provider or update.provider)
current_providers = {
provider: dict(config or {})
for provider, config in current_ai.get("providers", {}).items()
}
current_provider = current_providers.get(provider_id) or _provider_defaults(provider_id)
current_api_key, current_api_key_source = _resolve_provider_api_key(provider_id, current_provider)
current_api_key_preview = _mask_secret(current_api_key, current_api_key_source)["preview"]
provider_payload = {
**_provider_defaults(provider_id),
**current_provider,
"provider": provider_id,
"provider_api": update.provider_api.strip()
or current_provider.get("provider_api")
or "anthropic-messages",
"base_url": update.base_url.strip(),
"model": update.model.strip(),
"max_tokens": update.max_tokens,
"anthropic_version": update.anthropic_version.strip() or "2023-06-01",
}
if not _is_secret_placeholder(update.api_key, current_api_key_preview):
provider_payload["api_key"] = str(update.api_key).strip()
elif current_provider.get("api_key"):
provider_payload["api_key"] = current_provider.get("api_key") or ""
else:
provider_payload["api_key"] = ""
current_providers[provider_id] = provider_payload
current_service_token, current_service_source = _resolve_service_token(current_ai)
current_service_preview = _mask_secret(current_service_token, current_service_source)["preview"]
ai_payload = {
"service_url": update.service_url.strip()
or app_settings.AI_PROVIDER_SERVICE_URL,
"service_token": current_ai.get("service_token") or "",
"default_provider": provider_id,
"providers": current_providers,
"timeout_seconds": update.timeout_seconds,
"retry_attempts": update.retry_attempts,
}
if not _is_secret_placeholder(update.service_token, current_service_preview):
ai_payload["service_token"] = str(update.service_token).strip()
return ai_payload
def _runtime_config_from_ai_payload(ai_payload: dict) -> dict:
normalized_ai = _normalize_ai_provider_payload(ai_payload)
default_provider = normalized_ai["default_provider"]
provider_config = (
normalized_ai["providers"].get(default_provider) or _provider_defaults(default_provider)
)
api_key, _api_key_source = _resolve_provider_api_key(default_provider, provider_config)
return {
"service_url": normalized_ai.get("service_url") or app_settings.AI_PROVIDER_SERVICE_URL,
"service_token": _resolve_service_token(normalized_ai)[0],
"timeout_seconds": int(
normalized_ai.get("timeout_seconds") or app_settings.AI_PROVIDER_TIMEOUT_SECONDS
),
"retry_attempts": int(
normalized_ai.get("retry_attempts") or app_settings.AI_PROVIDER_RETRY_ATTEMPTS
),
"llm_config": {
"provider": default_provider,
"provider_api": provider_config.get("provider_api") or "anthropic-messages",
"base_url": provider_config.get("base_url") or "",
"model": provider_config.get("model") or "",
"api_key": api_key,
"max_tokens": int(provider_config.get("max_tokens") or 1200),
"anthropic_version": provider_config.get("anthropic_version") or "2023-06-01",
},
}
async def get_runtime_ai_provider_config(db: AsyncSession) -> dict:
runtime_record = await get_setting_record(db, "external_integrations")
payload = merge_with_defaults(
"external_integrations",
runtime_record.payload if runtime_record else None,
)
return _runtime_config_from_ai_payload(payload.get("ai_provider") or {})
async def get_barentswatch_config_record(db: AsyncSession) -> Optional[DataSourceConfig]:
return await get_barentswatch_datasource_record(db)
async def serialize_external_integrations(db: AsyncSession) -> dict:
ai_config = await get_runtime_ai_provider_config(db)
runtime_setting = await get_setting_record(db, "external_integrations")
raw_payload = merge_with_defaults(
"external_integrations",
runtime_setting.payload if runtime_setting else None,
)
normalized_ai = _normalize_ai_provider_payload(raw_payload.get("ai_provider") or {})
default_provider = normalized_ai["default_provider"]
providers_payload: dict[str, dict] = {}
for provider in sorted({
*FALLBACK_LLM_PROVIDER_PRESETS.keys(),
*normalized_ai["providers"].keys(),
default_provider,
}):
provider_id = _normalize_provider_id(provider)
provider_config = normalized_ai["providers"].get(provider_id) or _provider_defaults(provider_id)
api_key, api_key_source = _resolve_provider_api_key(provider_id, provider_config)
providers_payload[provider_id] = {
"provider": provider_id,
"provider_api": provider_config.get("provider_api") or "openai-completions",
"base_url": provider_config.get("base_url") or "",
"model": provider_config.get("model") or "",
"api_key": _mask_secret(api_key, api_key_source),
"max_tokens": int(provider_config.get("max_tokens") or 1200),
"anthropic_version": provider_config.get("anthropic_version") or "2023-06-01",
"source": "runtime" if provider_config.get("api_key") else (api_key_source or "preset"),
}
display_llm_config = providers_payload.get(default_provider) or _provider_defaults(default_provider)
barentswatch_record = await get_barentswatch_config_record(db)
barentswatch_auth = barentswatch_record.auth_config if barentswatch_record else {}
barentswatch_auth = barentswatch_auth or {}
resolved_barentswatch = await resolve_barentswatch_config(db)
return {
"ai_provider": {
"service_url": ai_config["service_url"],
"service_token": _mask_secret(*_resolve_service_token(normalized_ai)),
"default_provider": default_provider,
"provider": default_provider,
"provider_api": display_llm_config.get("provider_api") or "anthropic-messages",
"base_url": display_llm_config.get("base_url") or "https://api.minimaxi.com/anthropic",
"model": display_llm_config.get("model") or "MiniMax-M2.7",
"api_key": display_llm_config.get("api_key") or _mask_secret(None),
"providers": providers_payload,
"max_tokens": int(display_llm_config.get("max_tokens") or 1200),
"anthropic_version": display_llm_config.get("anthropic_version") or "2023-06-01",
"timeout_seconds": ai_config["timeout_seconds"],
"retry_attempts": ai_config["retry_attempts"],
"source": "runtime" if runtime_setting else "env",
},
"barentswatch": {
"endpoint": resolved_barentswatch.endpoint,
"client_id": barentswatch_auth.get("client_id") or resolved_barentswatch.client_id,
"client_secret": _mask_secret(
barentswatch_auth.get("client_secret") or resolved_barentswatch.client_secret
),
"source": resolved_barentswatch.credential_source,
},
}
async def save_external_integrations_payload(
db: AsyncSession,
update: ExternalIntegrationsUpdate,
) -> dict:
current_payload = await get_setting_payload(db, "external_integrations")
ai_payload = _build_ai_provider_payload(current_payload, update.ai_provider)
await save_setting_payload(db, "external_integrations", {"ai_provider": ai_payload})
default_endpoint = get_data_sources_config().get_yaml_url("barentswatch_vessels")
barentswatch_record = await get_barentswatch_config_record(db)
if barentswatch_record is None:
barentswatch_record = DataSourceConfig(
name="barentswatch_vessels",
description="BarentsWatch Live AIS credentials",
source_type="api",
endpoint=update.barentswatch.endpoint.strip() or default_endpoint,
auth_type="oauth_client",
auth_config={},
headers={},
config={},
is_active=True,
)
db.add(barentswatch_record)
current_auth = dict(barentswatch_record.auth_config or {})
if update.barentswatch.clear_client_secret:
current_auth.pop("client_secret", None)
elif update.barentswatch.client_secret not in (None, ""):
current_auth["client_secret"] = update.barentswatch.client_secret
current_auth["client_id"] = update.barentswatch.client_id.strip()
barentswatch_record.endpoint = update.barentswatch.endpoint.strip() or default_endpoint
barentswatch_record.auth_type = "oauth_client"
barentswatch_record.auth_config = current_auth
await db.commit()
return await serialize_external_integrations(db)
def format_frequency_label(minutes: int) -> str:
if minutes % 1440 == 0:
return f"{minutes // 1440}d"
@@ -105,10 +559,17 @@ def format_frequency_label(minutes: int) -> str:
return f"{minutes}m"
def serialize_collector(datasource: DataSource) -> dict:
async def get_ais_source_health_by_source(db: AsyncSession) -> dict[str, dict]:
result = await db.execute(select(AISSourceHealth))
return {item.source: item.to_dict() for item in result.scalars().all()}
def serialize_collector(datasource: DataSource, ais_health_by_source: dict[str, dict] | None = None) -> dict:
defaults = DEFAULT_DATASOURCES.get(datasource.source, {})
return {
"id": datasource.id,
"name": datasource.name,
"display_name": defaults.get("display_name") or datasource.name,
"source": datasource.source,
"module": datasource.module,
"priority": datasource.priority,
@@ -118,6 +579,11 @@ def serialize_collector(datasource: DataSource) -> dict:
"last_run_at": to_iso8601_utc(datasource.last_run_at),
"last_status": datasource.last_status,
"next_run_at": to_iso8601_utc(datasource.next_run_at),
"is_free": bool(defaults.get("is_free", True)),
"requires_credentials": bool(defaults.get("requires_credentials", False)),
"credential_provider": defaults.get("credential_provider"),
"credential_status": defaults.get("credential_status", "none"),
"ais_health": (ais_health_by_source or {}).get(datasource.source),
}
@@ -175,6 +641,233 @@ async def update_security_settings(
return {"status": "updated", "security": payload}
@router.get("/tv")
async def get_tv_settings(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return {"tv": await get_tv_settings_payload(db)}
@router.put("/tv")
async def update_tv_settings(
settings: TVSettingsUpdate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
payload = normalize_tv_settings(settings.model_dump())
saved = await save_setting_payload(db, "tv", payload)
return {"status": "updated", "tv": normalize_tv_settings(saved)}
@router.get("/integrations")
async def get_external_integrations(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return {"integrations": await serialize_external_integrations(db)}
@router.get("/integrations/barentswatch/connectivity")
async def get_barentswatch_connectivity(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await check_barentswatch_connectivity(db)
@router.post("/integrations/barentswatch/connect")
async def connect_barentswatch_integration(
payload: BarentsWatchIntegrationUpdate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
current = await resolve_barentswatch_config(db)
config = BarentsWatchConfig(
endpoint=payload.endpoint.strip() or current.endpoint,
client_id=payload.client_id.strip() or current.client_id,
client_secret=(
""
if payload.clear_client_secret
else payload.client_secret or current.client_secret
),
credential_source="draft",
endpoint_source="draft",
)
result = await check_barentswatch_config(config)
if result.get("success"):
checksum, _context = await build_builtin_connectivity_checksum(
"barentswatch_vessels",
config.endpoint,
"none",
{},
{},
db,
credential_override={
"client_id": config.client_id,
"client_secret": config.client_secret,
},
)
validation = await save_connectivity_success(
db,
"barentswatch_vessels",
checksum,
result,
connected_by="connection_button",
)
await db.commit()
return {**result, "connected": True, "validation": validation}
return {**result, "connected": False}
@router.post("/integrations/ai-provider/connect")
async def connect_ai_provider_integration(
payload: AIProviderIntegrationUpdate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
current_payload = await get_setting_payload(db, "external_integrations")
draft_ai_payload = _build_ai_provider_payload(current_payload, payload)
runtime_config = _runtime_config_from_ai_payload(draft_ai_payload)
client = AIProviderClient(
service_url=runtime_config["service_url"],
service_token=runtime_config["service_token"],
timeout=runtime_config["timeout_seconds"],
retry_attempts=runtime_config["retry_attempts"],
llm_config=runtime_config.get("llm_config") or {},
)
try:
status_result = await client.get_status()
if not status_result.configured:
return {
"success": False,
"connected": False,
"message": "AI Provider 可访问,但当前 provider/model/key 未完整配置。",
"status": status_result.model_dump(),
}
analysis_result = await client.analyze(
SituationalAnalysisRequest(
title="连接测试",
objective="请用一句话回复连接可用。",
observations=["这是配置中心发起的 LLM 连接测试。"],
constraints=["回复尽量简短。"],
)
)
await save_setting_payload(db, "external_integrations", {"ai_provider": draft_ai_payload})
return {
"success": True,
"connected": True,
"message": "AI Provider 连接成功,已保存为全局默认配置。",
"status": status_result.model_dump(),
"provider": analysis_result.provider,
"model": analysis_result.model,
"integrations": await serialize_external_integrations(db),
}
except HTTPException as exc:
return {
"success": False,
"connected": False,
"message": str(exc.detail),
}
except Exception as exc:
return {
"success": False,
"connected": False,
"message": f"AI Provider 连接测试失败: {exc}",
}
@router.get("/integrations/ai-provider/secrets")
async def reveal_ai_provider_secrets(
provider: str = Query(default=""),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
current_payload = await get_setting_payload(db, "external_integrations")
ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
provider_id = _normalize_provider_id(provider or ai_payload["default_provider"])
provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id)
api_key, api_key_source = _resolve_provider_api_key(provider_id, provider_config)
service_token, service_token_source = _resolve_service_token(ai_payload)
return {
"provider": provider_id,
"api_key": api_key,
"api_key_source": api_key_source,
"service_token": service_token,
"service_token_source": service_token_source,
}
@router.get("/credential-guides/{provider}")
async def read_credential_guide(
provider: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
return {"guide": await get_credential_guide(db, provider)}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@router.post("/credential-guides/{provider}/generate")
async def generate_provider_credential_guide(
provider: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
ai_client: AIProviderClient = Depends(get_ai_provider_client),
):
try:
return {"guide": await generate_credential_guide(db, provider, ai_client)}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@router.post("/credential-guides/{provider}/reset")
async def reset_provider_credential_guide(
provider: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
return {"guide": await reset_credential_guide(db, provider)}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@router.get("/integrations/ai-provider/presets")
async def get_ai_provider_presets(
current_user: User = Depends(get_current_user),
):
return {"data": list_fallback_llm_provider_presets()}
@router.post("/integrations/ai-provider/presets/{provider}/refresh")
async def refresh_ai_provider_preset(
provider: str,
current_user: User = Depends(get_current_user),
):
try:
return {"data": await refresh_llm_provider_preset(provider)}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except Exception as exc:
fallback = get_fallback_llm_provider_preset(provider)
fallback["refresh_error"] = str(exc)
return {"data": fallback}
@router.put("/integrations")
async def update_external_integrations(
payload: ExternalIntegrationsUpdate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
saved = await save_external_integrations_payload(db, payload)
return {"status": "updated", "integrations": saved}
@router.get("/collectors")
async def get_collector_settings(
current_user: User = Depends(get_current_user),
@@ -182,7 +875,8 @@ async def get_collector_settings(
):
result = await db.execute(select(DataSource).order_by(DataSource.module, DataSource.id))
datasources = result.scalars().all()
return {"collectors": [serialize_collector(datasource) for datasource in datasources]}
ais_health_by_source = await get_ais_source_health_by_source(db)
return {"collectors": [serialize_collector(datasource, ais_health_by_source) for datasource in datasources]}
@router.put("/collectors/{datasource_id}")
@@ -202,7 +896,8 @@ async def update_collector_settings(
await db.commit()
await db.refresh(datasource)
await sync_datasource_job(datasource.id)
return {"status": "updated", "collector": serialize_collector(datasource)}
ais_health_by_source = await get_ais_source_health_by_source(db)
return {"status": "updated", "collector": serialize_collector(datasource, ais_health_by_source)}
@router.get("")
@@ -212,10 +907,17 @@ async def get_all_settings(
):
result = await db.execute(select(DataSource).order_by(DataSource.module, DataSource.id))
datasources = result.scalars().all()
setting_payloads = await get_setting_payloads(
db,
["system", "notifications", "security"],
)
ais_health_by_source = await get_ais_source_health_by_source(db)
return {
"system": await get_setting_payload(db, "system"),
"notifications": await get_setting_payload(db, "notifications"),
"security": await get_setting_payload(db, "security"),
"collectors": [serialize_collector(datasource) for datasource in datasources],
"system": setting_payloads["system"],
"notifications": setting_payloads["notifications"],
"security": setting_payloads["security"],
"tv": await get_tv_settings_payload(db),
"integrations": await serialize_external_integrations(db),
"collectors": [serialize_collector(datasource, ais_health_by_source) for datasource in datasources],
"generated_at": to_iso8601_utc(datetime.now(UTC)),
}
+188 -1
View File
@@ -4,12 +4,15 @@ import os
import subprocess
import sys
from fastapi import APIRouter, Depends, HTTPException, status
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from pydantic import BaseModel
from app.core.config import ROOT_DIR
from app.core.security import get_current_user
from app.models.user import User
from app.services.persistent_logs import record_audit_log, record_system_log
from app.services.system_control import (
build_task_id,
clear_active_task_id,
@@ -23,6 +26,15 @@ from app.services.system_control import (
set_active_task_id,
upsert_task_state,
)
from app.services.system_logs import (
DEFAULT_LOG_LINE_LIMIT,
MAX_LOG_LINE_LIMIT,
SUPPORTED_LOG_LEVELS,
append_buffer_log,
list_log_sources,
normalize_log_level,
read_log_snapshot,
)
router = APIRouter()
@@ -47,6 +59,59 @@ class RestartTaskLogsResponse(BaseModel):
lines: list[str]
class SystemLogSourceSummary(BaseModel):
source_id: str
name: str
kind: str
location: str
description: str
category: str
status: str
class SystemLogSourcesResponse(BaseModel):
items: list[SystemLogSourceSummary]
class SystemLogDailyMarker(BaseModel):
date_token: str
total: int
dominant_level: str
class SystemLogSnapshotResponse(BaseModel):
source_id: str
name: str
kind: str
location: str
description: str
category: str
status: str
level: str
selected_levels: list[str] = []
search_query: str = ""
available_levels: list[str]
daily_markers: list[SystemLogDailyMarker] = []
line_limit: int
line_count: int
lines: list[str]
class EarthClientLogEventCreate(BaseModel):
level: str = "error"
message: str
category: str | None = None
url: str | None = None
module: str | None = None
detail: str | None = None
class EarthClientLogEventResponse(BaseModel):
accepted: bool
source_id: str
level: str
def ensure_super_admin(current_user: User) -> None:
if not require_super_admin(current_user.role):
raise HTTPException(
@@ -55,9 +120,22 @@ def ensure_super_admin(current_user: User) -> None:
)
def validate_log_date(raw_value: str | None, field_name: str) -> str | None:
if raw_value in {None, ""}:
return None
try:
return datetime.strptime(raw_value, "%Y-%m-%d").date().isoformat()
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"{field_name} must be in YYYY-MM-DD format",
) from exc
@router.post("/restart-tasks", response_model=RestartTaskResponse)
async def create_restart_task(
payload: RestartTaskCreate,
request: Request,
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
@@ -133,11 +211,31 @@ async def create_restart_task(
requested_by=requested_by,
)
clear_active_task_id(task_id)
await record_audit_log(
action="system.restart_task.requested",
actor_id=current_user.id,
actor_name=current_user.username,
target_type="restart_task",
target_id=task_id,
result="failed",
ip=request.client.host if request.client else None,
details={"action": payload.action, "message": task_state["message"]},
)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=task_state["message"],
) from exc
await record_audit_log(
action="system.restart_task.requested",
actor_id=current_user.id,
actor_name=current_user.username,
target_type="restart_task",
target_id=task_id,
result="accepted",
ip=request.client.host if request.client else None,
details={"action": payload.action},
)
return task_state
@@ -165,3 +263,92 @@ async def get_restart_task_logs(
if task is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Restart task not found")
return {"task_id": task_id, "lines": get_task_logs(task_id)}
@router.get("/logs/sources", response_model=SystemLogSourcesResponse)
async def get_system_log_sources(
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
return {"items": list_log_sources()}
@router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse)
async def get_system_log_snapshot(
source_id: str,
limit: int = DEFAULT_LOG_LINE_LIMIT,
level: str = "all",
levels: str | None = Query(None, description="Comma-separated log levels"),
start_date: str | None = Query(None, description="Filter logs from this date (YYYY-MM-DD)"),
end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"),
search: str | None = Query(None, description="Case-insensitive substring search"),
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
if limit < 1 or limit > MAX_LOG_LINE_LIMIT:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"limit must be between 1 and {MAX_LOG_LINE_LIMIT}",
)
if str(level).strip().lower() not in SUPPORTED_LOG_LEVELS and normalize_log_level(level) == "all" and str(level).strip().lower() not in {"", "all"}:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unsupported log level")
if levels:
for raw_level in str(levels).split(","):
normalized_level = str(raw_level).strip().lower()
if not normalized_level:
continue
if normalized_level not in SUPPORTED_LOG_LEVELS and normalize_log_level(normalized_level) == "all":
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unsupported log level")
normalized_start_date = validate_log_date(start_date, "start_date")
normalized_end_date = validate_log_date(end_date, "end_date")
if normalized_start_date and normalized_end_date and normalized_start_date > normalized_end_date:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="start_date must be earlier than or equal to end_date")
snapshot = read_log_snapshot(
source_id,
limit,
level=level,
levels=levels,
start_date=normalized_start_date,
end_date=normalized_end_date,
search=search,
)
if snapshot is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Log source not found")
return snapshot
@router.post("/logs/earth-client", response_model=EarthClientLogEventResponse)
async def ingest_earth_client_log(
payload: EarthClientLogEventCreate,
request: Request,
):
normalized_level = normalize_log_level(payload.level)
append_buffer_log(
"earth-client",
level=normalized_level,
message=payload.message,
context={
"category": payload.category or "",
"url": payload.url or "",
"module": payload.module or "",
"detail": payload.detail or "",
},
)
await record_system_log(
source="earth-client",
service="earth",
module=payload.module or "earth-client",
event="earth.client.runtime_log",
level=normalized_level,
message=payload.message,
category=payload.category or "client-runtime",
context={
"url": payload.url or "",
"detail": payload.detail or "",
"module": payload.module or "",
"client_ip": request.client.host if request.client else "",
},
)
return {"accepted": True, "source_id": "earth-client", "level": normalized_level}
+22 -2
View File
@@ -27,7 +27,9 @@ async def list_tasks(
offset = (page - 1) * page_size
query = """
SELECT ct.id, ct.datasource_id, ds.name as datasource_name, ct.status,
ct.started_at, ct.completed_at, ct.records_processed, ct.error_message
ct.started_at, ct.completed_at, ct.records_processed, ct.error_message,
ct.phase, ct.phase_progress, ct.phase_message, ct.phase_current,
ct.phase_total, ct.phase_unit, ct.total_records, ct.progress
FROM collection_tasks ct
JOIN data_sources ds ON ct.datasource_id = ds.id
WHERE 1=1
@@ -66,6 +68,14 @@ async def list_tasks(
"completed_at": to_iso8601_utc(t[5]),
"records_processed": t[6],
"error_message": t[7],
"phase": t[8],
"phase_progress": t[9],
"phase_message": t[10],
"phase_current": t[11],
"phase_total": t[12],
"phase_unit": t[13],
"total_records": t[14],
"progress": t[15],
}
for t in tasks
],
@@ -81,7 +91,9 @@ async def get_task(
result = await db.execute(
text("""
SELECT ct.id, ct.datasource_id, ds.name as datasource_name, ct.status,
ct.started_at, ct.completed_at, ct.records_processed, ct.error_message
ct.started_at, ct.completed_at, ct.records_processed, ct.error_message,
ct.phase, ct.phase_progress, ct.phase_message, ct.phase_current,
ct.phase_total, ct.phase_unit, ct.total_records, ct.progress
FROM collection_tasks ct
JOIN data_sources ds ON ct.datasource_id = ds.id
WHERE ct.id = :id
@@ -105,6 +117,14 @@ async def get_task(
"completed_at": to_iso8601_utc(task[5]),
"records_processed": task[6],
"error_message": task[7],
"phase": task[8],
"phase_progress": task[9],
"phase_message": task[10],
"phase_current": task[11],
"phase_total": task[12],
"phase_unit": task[13],
"total_records": task[14],
"progress": task[15],
}
+70
View File
@@ -0,0 +1,70 @@
from urllib.parse import quote, urljoin
import httpx
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import Response
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.session import get_db
from app.services.tv_streams import get_public_tv_payload, is_allowed_tv_proxy_url
router = APIRouter()
@router.get("/streams")
async def list_public_tv_streams(
db: AsyncSession = Depends(get_db),
):
return await get_public_tv_payload(db)
@router.get("/proxy")
async def proxy_tv_stream(
url: str = Query(..., description="Upstream TV stream or manifest URL"),
db: AsyncSession = Depends(get_db),
):
payload = await get_public_tv_payload(db)
if not is_allowed_tv_proxy_url(url, payload.get("sources", [])):
raise HTTPException(status_code=403, detail="TV proxy target is not allowed")
try:
async with httpx.AsyncClient(follow_redirects=True, timeout=20.0) as client:
upstream = await client.get(
url,
headers={
"User-Agent": "Mozilla/5.0",
"Referer": "https://tv.cctv.com/live/cctv4/",
},
)
upstream.raise_for_status()
except httpx.HTTPError as exc:
raise HTTPException(status_code=502, detail=f"Failed to fetch TV stream: {exc}") from exc
content_type = upstream.headers.get("content-type", "application/octet-stream")
raw_content = upstream.content
response_url = str(upstream.url)
is_manifest = (
response_url.endswith(".m3u8")
or "mpegurl" in content_type.lower()
or raw_content.lstrip().startswith(b"#EXTM3U")
)
headers = {"Cache-Control": "no-store"}
if is_manifest:
manifest_text = upstream.text
rewritten_lines: list[str] = []
for line in manifest_text.splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("#"):
rewritten_lines.append(line)
continue
absolute_url = urljoin(response_url, stripped)
rewritten_lines.append(f"/api/v1/tv/proxy?url={quote(absolute_url, safe='')}")
return Response(
content="\n".join(rewritten_lines),
media_type="application/vnd.apple.mpegurl",
headers=headers,
)
return Response(content=raw_content, media_type=content_type, headers=headers)
+33 -5
View File
@@ -1,3 +1,4 @@
import json
from typing import List
from fastapi import APIRouter, Depends, HTTPException, status
@@ -7,10 +8,12 @@ from sqlalchemy import text
from app.core.security import get_current_user, get_password_hash
from app.db.session import get_db
from app.models.user import User
from app.schemas.user import UserCreate, UserResponse, UserUpdate
from app.schemas.user import UserCreate, UserUpdate
router = APIRouter()
VALID_GATEKEEPER_GROUPS = {"docs_user", "docs_developer", "docs_admin"}
def check_permission(current_user: User, required_roles: List[str]) -> bool:
user_role_value = (
@@ -52,7 +55,7 @@ async def list_users(
offset = (page - 1) * page_size
query = text(
f"SELECT id, username, email, role, is_active, last_login_at, created_at FROM users WHERE {where_sql} ORDER BY created_at DESC LIMIT {page_size} OFFSET {offset}"
f"SELECT id, username, email, role, is_active, last_login_at, created_at, gatekeeper_groups FROM users WHERE {where_sql} ORDER BY created_at DESC LIMIT {page_size} OFFSET {offset}"
)
count_query = text(f"SELECT COUNT(*) FROM users WHERE {where_sql}")
@@ -75,6 +78,7 @@ async def list_users(
"is_active": u[4],
"last_login_at": u[5],
"created_at": u[6],
"gatekeeper_groups": u[7] or [],
}
for u in users
],
@@ -95,7 +99,7 @@ async def get_user(
result = await db.execute(
text(
"SELECT id, username, email, role, is_active, last_login_at, created_at FROM users WHERE id = :id"
"SELECT id, username, email, role, is_active, last_login_at, created_at, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": user_id},
)
@@ -114,6 +118,7 @@ async def get_user(
"is_active": user[4],
"last_login_at": user[5],
"created_at": user[6],
"gatekeeper_groups": user[7] or [],
}
@@ -128,6 +133,12 @@ async def create_user(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can create users",
)
invalid_groups = sorted(set(user_data.gatekeeper_groups) - VALID_GATEKEEPER_GROUPS)
if invalid_groups:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unsupported Gatekeeper groups: {', '.join(invalid_groups)}",
)
result = await db.execute(
text("SELECT id FROM users WHERE username = :username OR email = :email"),
@@ -142,13 +153,14 @@ async def create_user(
hashed_password = get_password_hash(user_data.password)
await db.execute(
text("""INSERT INTO users (username, email, password_hash, role, is_active, created_at, updated_at)
VALUES (:username, :email, :password_hash, :role, :is_active, NOW(), NOW())"""),
text("""INSERT INTO users (username, email, password_hash, role, gatekeeper_groups, is_active, created_at, updated_at)
VALUES (:username, :email, :password_hash, :role, CAST(:gatekeeper_groups AS jsonb), :is_active, NOW(), NOW())"""),
{
"username": user_data.username,
"email": user_data.email,
"password_hash": hashed_password,
"role": user_data.role,
"gatekeeper_groups": json.dumps(user_data.gatekeeper_groups),
"is_active": True,
},
)
@@ -172,6 +184,7 @@ async def create_user(
"username": user_data.username,
"email": user_data.email,
"role": user_data.role,
"gatekeeper_groups": user_data.gatekeeper_groups,
"is_active": True,
}
@@ -194,6 +207,18 @@ async def update_user(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can change user role",
)
if not check_permission(current_user, ["super_admin"]) and user_data.gatekeeper_groups is not None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can change Gatekeeper groups",
)
if user_data.gatekeeper_groups is not None:
invalid_groups = sorted(set(user_data.gatekeeper_groups) - VALID_GATEKEEPER_GROUPS)
if invalid_groups:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unsupported Gatekeeper groups: {', '.join(invalid_groups)}",
)
result = await db.execute(
text("SELECT id FROM users WHERE id = :id"),
@@ -213,6 +238,9 @@ async def update_user(
if user_data.role is not None:
update_fields.append("role = :role")
params["role"] = user_data.role
if user_data.gatekeeper_groups is not None:
update_fields.append("gatekeeper_groups = CAST(:gatekeeper_groups AS jsonb)")
params["gatekeeper_groups"] = json.dumps(user_data.gatekeeper_groups)
if user_data.is_active is not None:
update_fields.append("is_active = :is_active")
params["is_active"] = user_data.is_active
+132
View File
@@ -0,0 +1,132 @@
"""v4 strategy + v5 conflict-promotion + enrichment APIs for vessel_ais."""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.user import User
from app.models.vessel import AISConflictRecord
from app.services.vessel_aggregation_strategy import (
StrategyValidationError,
load_strategy,
reset_strategy,
save_strategy,
)
from app.services.vessel_enrichment import (
get_vessel_enrichment_bundle,
upsert_vessel_media_enrichment,
upsert_vessel_profile_enrichment,
)
router = APIRouter()
@router.get("/strategy")
async def get_aggregation_strategy(db: AsyncSession = Depends(get_db)):
return await load_strategy(db)
@router.put("/strategy")
async def put_aggregation_strategy(
payload: dict[str, Any],
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
return await save_strategy(db, payload)
except StrategyValidationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.delete("/strategy")
async def reset_aggregation_strategy(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await reset_strategy(db)
@router.post("/conflicts/{mmsi}/{field}/promote-to-rule")
async def promote_conflict_to_rule(
mmsi: int,
field: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Lift the current conflict resolution into a persistent strategy rule."""
result = await db.execute(
select(AISConflictRecord)
.where(AISConflictRecord.target_schema == "vessel_ais")
.where(AISConflictRecord.entity_key == str(mmsi))
.where(AISConflictRecord.field == field)
.order_by(AISConflictRecord.updated_at.desc(), AISConflictRecord.id.desc())
.limit(1)
)
record = result.scalar_one_or_none()
if record is None or not record.selected_source:
raise HTTPException(status_code=404, detail="Conflict record with selected_source not found")
strategy = await load_strategy(db)
vessel_ais = dict(strategy.get("vessel_ais") or {})
field_rules = dict(vessel_ais.get("field_rules") or {})
field_rules[field] = {"mode": "source_priority", "source_priority": [record.selected_source]}
vessel_ais["field_rules"] = field_rules
incoming = {"version": int(strategy.get("version") or 0), "vessel_ais": vessel_ais}
try:
return await save_strategy(db, incoming)
except StrategyValidationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.delete("/conflicts/{mmsi}/{field}/promote-to-rule")
async def revert_conflict_rule(
mmsi: int,
field: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
strategy = await load_strategy(db)
vessel_ais = dict(strategy.get("vessel_ais") or {})
field_rules = dict(vessel_ais.get("field_rules") or {})
if field in field_rules:
del field_rules[field]
vessel_ais["field_rules"] = field_rules
incoming = {"version": int(strategy.get("version") or 0), "vessel_ais": vessel_ais}
try:
return await save_strategy(db, incoming)
except StrategyValidationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.get("/enrichment/{mmsi}")
async def get_vessel_enrichment(mmsi: int, db: AsyncSession = Depends(get_db)):
return await get_vessel_enrichment_bundle(db, mmsi)
@router.put("/enrichment/{mmsi}/profile")
async def put_vessel_profile_enrichment(
mmsi: int,
payload: dict[str, Any],
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await upsert_vessel_profile_enrichment(db, mmsi=mmsi, payload=payload)
@router.put("/enrichment/{mmsi}/media")
async def put_vessel_media_enrichment(
mmsi: int,
payload: dict[str, Any],
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await upsert_vessel_media_enrichment(db, mmsi=mmsi, payload=payload)
File diff suppressed because it is too large. Load diff
+47 -18
View File
@@ -2,7 +2,6 @@
import asyncio
import json
import logging
from datetime import UTC, datetime
from typing import Optional
@@ -10,10 +9,11 @@ from fastapi import APIRouter, WebSocket, WebSocketDisconnect, Query
from jose import jwt, JWTError
from app.core.config import settings
from app.core.logging import get_logger
from app.core.time import to_iso8601_utc
from app.core.websocket.manager import manager
logger = logging.getLogger(__name__)
logger = get_logger(__name__, service="api")
router = APIRouter()
@@ -22,28 +22,52 @@ async def authenticate_token(token: str) -> Optional[dict]:
try:
payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM])
if payload.get("type") != "access":
logger.warning(f"WebSocket auth failed: wrong token type")
logger.warning_event(
"WebSocket auth failed: wrong token type",
event="auth.websocket.invalid_token_type",
)
return None
return payload
except JWTError as e:
logger.warning(f"WebSocket auth failed: {e}")
logger.warning_event(
"WebSocket auth failed",
event="auth.websocket.decode_failed",
context={"error": str(e)},
)
return None
@router.websocket("/ws")
async def websocket_endpoint(
websocket: WebSocket,
token: str = Query(...),
token: str | None = Query(None),
):
"""WebSocket endpoint for real-time data"""
logger.info(f"WebSocket connection attempt with token: {token[:20]}...")
payload = await authenticate_token(token)
if payload is None:
logger.warning("WebSocket authentication failed, closing connection")
logger.info_event(
"WebSocket connection attempt",
event="auth.websocket.connection_attempt",
context={"token_preview": f"{token[:8]}..." if token else "anonymous"},
)
payload = await authenticate_token(token) if token else None
if token and payload is None:
logger.warning_event(
"WebSocket authentication failed, closing connection",
event="auth.websocket.connection_rejected",
)
await websocket.close(code=4001)
return
user_id = str(payload.get("sub"))
is_anonymous = payload is None
user_id = str(payload.get("sub")) if payload else f"anonymous:{id(websocket)}"
supported_channels = ["vessels"] if is_anonymous else [
"gpu_clusters",
"submarine_cables",
"ixp_nodes",
"alerts",
"dashboard",
"datasource_tasks",
"vessels",
]
await manager.connect(websocket, user_id)
try:
@@ -54,14 +78,7 @@ async def websocket_endpoint(
"connection_id": f"conn_{user_id}",
"server_version": settings.VERSION,
"heartbeat_interval": 30,
"supported_channels": [
"gpu_clusters",
"submarine_cables",
"ixp_nodes",
"alerts",
"dashboard",
"datasource_tasks",
],
"supported_channels": supported_channels,
},
}
)
@@ -79,12 +96,24 @@ async def websocket_endpoint(
)
elif data.get("type") == "subscribe":
channels = data.get("data", {}).get("channels", [])
if is_anonymous:
channels = [channel for channel in channels if channel in supported_channels]
manager.subscribe(websocket, channels)
await websocket.send_json(
{
"type": "subscription_confirmed",
"data": {"action": "subscribe", "channels": channels},
}
)
elif data.get("type") == "unsubscribe":
channels = data.get("data", {}).get("channels", [])
manager.unsubscribe(websocket, channels)
await websocket.send_json(
{
"type": "subscription_confirmed",
"data": {"action": "unsubscribe", "channels": channels},
}
)
elif data.get("type") == "control_frame":
await websocket.send_json(
{"type": "control_acknowledged", "data": {"received": True}}
+6 -6
View File
@@ -1,15 +1,15 @@
"""Redis caching service"""
import json
import logging
from datetime import timedelta
from typing import Optional, Any
import redis
from app.core.config import settings
from app.core.logging import get_logger
logger = logging.getLogger(__name__)
logger = get_logger(__name__)
# Lazy Redis client initialization
@@ -47,7 +47,7 @@ class CacheService:
return json.loads(value)
return None
except Exception as e:
logger.warning(f"Cache get error: {e}")
logger.warning_event("Cache get error", event="cache.get.failed", context={"error": str(e)})
return None
def set(
@@ -61,7 +61,7 @@ class CacheService:
serialized = json.dumps(value, default=str)
return self.client.setex(key, expire_seconds, serialized)
except Exception as e:
logger.warning(f"Cache set error: {e}")
logger.warning_event("Cache set error", event="cache.set.failed", context={"error": str(e)})
return False
def delete(self, key: str) -> bool:
@@ -69,7 +69,7 @@ class CacheService:
try:
return self.client.delete(key) > 0
except Exception as e:
logger.warning(f"Cache delete error: {e}")
logger.warning_event("Cache delete error", event="cache.delete.failed", context={"error": str(e)})
return False
def delete_pattern(self, pattern: str) -> int:
@@ -80,7 +80,7 @@ class CacheService:
return self.client.delete(*keys)
return 0
except Exception as e:
logger.warning(f"Cache delete_pattern error: {e}")
logger.warning_event("Cache delete_pattern error", event="cache.delete_pattern.failed", context={"error": str(e)})
return 0
def get_or_set(
+49 -2
View File
@@ -4,8 +4,8 @@ from typing import Any, Dict, Optional
FIELD_ALIASES = {
"country": ("country",),
"city": ("city",),
"latitude": ("latitude",),
"longitude": ("longitude",),
"latitude": ("latitude", "lat"),
"longitude": ("longitude", "lon", "lng"),
"value": ("value",),
"unit": ("unit",),
"cores": ("cores",),
@@ -14,6 +14,28 @@ FIELD_ALIASES = {
"power": ("power",),
}
NESTED_FIELD_ALIASES = {
"latitude": (
("location", "latitude"),
("location", "lat"),
("geo", "latitude"),
("geo", "lat"),
("coordinates", "latitude"),
("coordinates", "lat"),
),
"longitude": (
("location", "longitude"),
("location", "lon"),
("location", "lng"),
("geo", "longitude"),
("geo", "lon"),
("geo", "lng"),
("coordinates", "longitude"),
("coordinates", "lon"),
("coordinates", "lng"),
),
}
def get_metadata_field(metadata: Optional[Dict[str, Any]], field: str, fallback: Any = None) -> Any:
if isinstance(metadata, dict):
@@ -21,9 +43,34 @@ def get_metadata_field(metadata: Optional[Dict[str, Any]], field: str, fallback:
value = metadata.get(key)
if value not in (None, ""):
return value
for path in NESTED_FIELD_ALIASES.get(field, ()):
current: Any = metadata
for key in path:
if not isinstance(current, dict):
current = None
break
current = current.get(key)
if current not in (None, ""):
return current
if field in {"latitude", "longitude"}:
value = _get_coordinate_sequence_value(metadata, field)
if value not in (None, ""):
return value
return fallback
def _get_coordinate_sequence_value(metadata: Dict[str, Any], field: str) -> Any:
for key in ("coordinates", "coord", "coords"):
value = metadata.get(key)
if not isinstance(value, (list, tuple)) or len(value) < 2:
continue
# GeoJSON uses [longitude, latitude]. Most raw collector tuples in this
# codebase use explicit field names, so only sequence aliases are treated
# as GeoJSON-shaped to avoid guessing.
return value[1] if field == "latitude" else value[0]
return None
def build_dynamic_metadata(
metadata: Optional[Dict[str, Any]],
*,
+15 -7
View File
@@ -1,7 +1,6 @@
import os
import yaml
from functools import lru_cache
from typing import Optional
COLLECTOR_URL_KEYS = {
@@ -11,6 +10,7 @@ COLLECTOR_URL_KEYS = {
"fao_landing_points": "fao.landing_point_url",
"telegeography_cables": "telegeography.cable_url",
"telegeography_landing": "telegeography.landing_point_url",
"telegeography_systems": "telegeography.cable_url",
"huggingface_models": "huggingface.models_url",
"huggingface_datasets": "huggingface.datasets_url",
"huggingface_spaces": "huggingface.spaces_url",
@@ -23,11 +23,15 @@ COLLECTOR_URL_KEYS = {
"top500": "top500.url",
"epoch_ai_gpu": "epoch_ai.gpu_clusters_url",
"spacetrack_tle": "spacetrack.tle_query_url",
"celestrak_tle": "celestrak.base_url",
"ris_live_bgp": "ris_live.url",
"bgpstream_bgp": "bgpstream.url",
"iptoasn_prefix_geo": "iptoasn.combined_url",
"opengeofeed_prefix_geo": "opengeofeed.public_csv_url",
"nro_delegated_prefix_geo": "nro.delegated_stats_url",
"news_live_streams": "news_live_streams.channels_url",
"barentswatch_vessels": "barentswatch_vessels.url",
"aisstream_vessels": "aisstream_vessels.url",
}
@@ -41,18 +45,22 @@ class DataSourcesConfig:
with open(config_path, "r") as f:
self._yaml_config = yaml.safe_load(f) or {}
def get_yaml_url(self, collector_name: str) -> str:
key = COLLECTOR_URL_KEYS.get(collector_name, "")
def get_yaml_value(self, key: str):
if not key:
return ""
return None
parts = key.split(".")
value = self._yaml_config
for part in parts:
if isinstance(value, dict):
value = value.get(part, "")
value = value.get(part)
else:
return ""
return None
return value
def get_yaml_url(self, collector_name: str) -> str:
key = COLLECTOR_URL_KEYS.get(collector_name, "")
value = self.get_yaml_value(key)
return value if isinstance(value, str) else ""
async def get_url(self, collector_name: str, db) -> str:
@@ -66,7 +74,7 @@ class DataSourcesConfig:
from app.models.datasource_config import DataSourceConfig
query = select(DataSourceConfig).where(
DataSourceConfig.name == collector_name, DataSourceConfig.is_active == True
DataSourceConfig.name == collector_name, DataSourceConfig.is_active
)
result = await db.execute(query)
db_config = result.scalar_one_or_none()
+50
View File
@@ -2,53 +2,103 @@
# All external data source URLs should be configured here
arcgis:
# ArcGIS 海缆 GeoJSON 查询接口
cable_url: "https://services.arcgis.com/6DIQcwlPy8knb6sg/ArcGIS/rest/services/SubmarineCables/FeatureServer/2/query"
# ArcGIS 登陆点 GeoJSON 查询接口
landing_point_url: "https://services.arcgis.com/6DIQcwlPy8knb6sg/ArcGIS/rest/services/SubmarineCables/FeatureServer/1/query"
# ArcGIS 海缆与登陆点关联关系查询接口
cable_landing_relation_url: "https://services.arcgis.com/6DIQcwlPy8knb6sg/ArcGIS/rest/services/SubmarineCables/FeatureServer/3/query"
fao:
# FAO 登陆点 CSV 下载地址
landing_point_url: "https://data.apps.fao.org/catalog/dataset/1b75ff21-92f2-4b96-9b7b-98e8aa65ad5d/resource/b6071077-d1d4-4e97-aa00-42e902847c87/download/landing-point-geo.csv"
telegeography:
# TeleGeography 海缆/系统主数据源,当前使用 GitHub 镜像 JSON
cable_url: "https://raw.githubusercontent.com/lintaojlu/submarine_cable_information/main/cable.json"
# TeleGeography 登陆点主数据源,当前使用 GitHub 镜像 JSON
landing_point_url: "https://raw.githubusercontent.com/lintaojlu/submarine_cable_information/main/landing_point.json"
# TeleGeography 历史 API 存档,用于 cable collector 的 fallback
archived_cable_url: "https://web.archive.org/web/2024/https://www.submarinecablemap.com/api/v3/cable"
# TeleGeography 官网页面,用于 cable collector 的最终 HTML 抓取 fallback
live_map_url: "https://www.submarinecablemap.com"
huggingface:
# Hugging Face 模型目录 API
models_url: "https://huggingface.co/api/models"
# Hugging Face 数据集目录 API
datasets_url: "https://huggingface.co/api/datasets"
# Hugging Face Spaces 目录 API
spaces_url: "https://huggingface.co/api/spaces"
cloudflare:
# Cloudflare Radar 设备类型摘要接口
radar_device_url: "https://api.cloudflare.com/client/v4/radar/http/summary/device_type"
# Cloudflare Radar 请求量时间序列接口
radar_traffic_url: "https://api.cloudflare.com/client/v4/radar/http/timeseries/requests"
# Cloudflare Radar 热点地理位置接口
radar_top_locations_url: "https://api.cloudflare.com/client/v4/radar/http/top/locations"
peeringdb:
# PeeringDB IXP API
ixp_url: "https://www.peeringdb.com/api/ix"
# PeeringDB Network API
network_url: "https://www.peeringdb.com/api/net"
# PeeringDB Facility API
facility_url: "https://www.peeringdb.com/api/fac"
top500:
# TOP500 榜单页面,用于主表抓取
url: "https://top500.org/lists/top500/list/2025/11/"
# TOP500 站点根地址,用于拼详情页链接
base_url: "https://top500.org"
epoch_ai:
# Epoch AI GPU Cluster 页面
gpu_clusters_url: "https://epoch.ai/data/gpu-clusters"
spacetrack:
# Space-Track 站点根地址,用于首页访问和登录地址推导
base_url: "https://www.space-track.org"
# Space-Track TLE 主查询接口
tle_query_url: "https://www.space-track.org/basicspacedata/query/class/gp/orderby/EPOCH%20desc/limit/1000/format/json"
celestrak:
# CelesTrak TLE 基础接口,collector 会在其后拼接 GROUP / FORMAT 参数
base_url: "https://celestrak.org/NORAD/elements/gp.php"
ris_live:
# RIPE RIS Live 流式订阅地址
url: "https://ris-live.ripe.net/v1/stream/?format=json&client=planet-ris-live"
bgpstream:
# CAIDA BGPStream Broker API
url: "https://broker.bgpstream.caida.org/v2"
iptoasn:
# IPtoASN prefix geography 合并数据下载地址
combined_url: "https://iptoasn.com/data/ip2asn-combined.tsv.gz"
opengeofeed:
# OpenGeoFeed 公共 geofeed CSV
public_csv_url: "https://opengeofeed.org/feed/public.csv"
nro:
# NRO delegated stats 下载地址
delegated_stats_url: "https://ftp.ripe.net/pub/stats/ripencc/nro-stats/latest/nro-delegated-stats"
news_live_streams:
# IPTV-org 频道元数据 JSON
channels_url: "https://iptv-org.github.io/api/channels.json"
# IPTV-org 频道播放流 JSON
streams_url: "https://iptv-org.github.io/api/streams.json"
# IPTV-org 台标 JSON
logos_url: "https://iptv-org.github.io/api/logos.json"
barentswatch_vessels:
# BarentsWatch Live AIS latest combined endpoint. Requires an AIS bearer token.
url: "https://live.ais.barentswatch.no/v1/latest/combined"
aisstream_vessels:
# AISStream realtime WebSocket endpoint. Requires an AISStream API key.
url: "wss://stream.aisstream.io/v0/stream"
+102
View File
@@ -4,156 +4,258 @@ DEFAULT_DATASOURCES = {
"top500": {
"id": 1,
"name": "TOP500 Supercomputers",
"display_name": "TOP500 超算榜单",
"module": "L1",
"priority": "P0",
"frequency_minutes": 240,
"is_free": True,
"requires_credentials": False,
},
"epoch_ai_gpu": {
"id": 2,
"name": "Epoch AI GPU Clusters",
"display_name": "Epoch AI GPU 集群",
"module": "L1",
"priority": "P0",
"frequency_minutes": 360,
"is_free": True,
"requires_credentials": False,
},
"huggingface_models": {
"id": 3,
"name": "HuggingFace Models",
"display_name": "Hugging Face 模型",
"module": "L2",
"priority": "P1",
"frequency_minutes": 720,
"is_free": True,
"requires_credentials": False,
},
"huggingface_datasets": {
"id": 4,
"name": "HuggingFace Datasets",
"display_name": "Hugging Face 数据集",
"module": "L2",
"priority": "P1",
"frequency_minutes": 720,
"is_free": True,
"requires_credentials": False,
},
"huggingface_spaces": {
"id": 5,
"name": "HuggingFace Spaces",
"display_name": "Hugging Face Spaces",
"module": "L2",
"priority": "P2",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"peeringdb_ixp": {
"id": 6,
"name": "PeeringDB IXP",
"display_name": "PeeringDB 交换中心",
"module": "L2",
"priority": "P1",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"peeringdb_network": {
"id": 7,
"name": "PeeringDB Networks",
"display_name": "PeeringDB 网络",
"module": "L2",
"priority": "P2",
"frequency_minutes": 2880,
"is_free": True,
"requires_credentials": False,
},
"peeringdb_facility": {
"id": 8,
"name": "PeeringDB Facilities",
"display_name": "PeeringDB 设施",
"module": "L2",
"priority": "P2",
"frequency_minutes": 2880,
"is_free": True,
"requires_credentials": False,
},
"telegeography_cables": {
"id": 9,
"name": "Submarine Cables",
"display_name": "海底光缆",
"module": "L2",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"telegeography_landing": {
"id": 10,
"name": "Cable Landing Points",
"display_name": "光缆登陆点",
"module": "L2",
"priority": "P2",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"telegeography_systems": {
"id": 11,
"name": "Cable Systems",
"display_name": "光缆系统",
"module": "L2",
"priority": "P2",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"arcgis_cables": {
"id": 15,
"name": "ArcGIS Submarine Cables",
"display_name": "ArcGIS 海底光缆",
"module": "L2",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"arcgis_landing_points": {
"id": 16,
"name": "ArcGIS Landing Points",
"display_name": "ArcGIS 登陆点",
"module": "L2",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"arcgis_cable_landing_relation": {
"id": 17,
"name": "ArcGIS Cable-Landing Relations",
"display_name": "ArcGIS 光缆登陆关系",
"module": "L2",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"fao_landing_points": {
"id": 18,
"name": "FAO Landing Points",
"display_name": "FAO 登陆点",
"module": "L2",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"spacetrack_tle": {
"id": 19,
"name": "Space-Track TLE",
"display_name": "Space-Track 轨道根数",
"module": "L3",
"priority": "P2",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": True,
"credential_provider": "spacetrack",
"credential_status": "planned",
},
"celestrak_tle": {
"id": 20,
"name": "CelesTrak TLE",
"display_name": "CelesTrak 轨道根数",
"module": "L3",
"priority": "P2",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"ris_live_bgp": {
"id": 21,
"name": "RIPE RIS Live BGP",
"display_name": "RIPE RIS 实时 BGP",
"module": "L3",
"priority": "P1",
"frequency_minutes": 15,
"is_free": True,
"requires_credentials": False,
},
"bgpstream_bgp": {
"id": 22,
"name": "CAIDA BGPStream Backfill",
"display_name": "CAIDA BGPStream 回填",
"module": "L3",
"priority": "P1",
"frequency_minutes": 360,
"is_free": True,
"requires_credentials": False,
},
"iptoasn_prefix_geo": {
"id": 23,
"name": "IPtoASN Prefix Geography",
"display_name": "IPtoASN 前缀地理",
"module": "L3",
"priority": "P1",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"opengeofeed_prefix_geo": {
"id": 24,
"name": "OpenGeoFeed Prefix Geography",
"display_name": "OpenGeoFeed 前缀地理",
"module": "L3",
"priority": "P1",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"nro_delegated_prefix_geo": {
"id": 25,
"name": "NRO Delegated Prefix Geography",
"display_name": "NRO 分配前缀地理",
"module": "L3",
"priority": "P1",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"news_live_streams": {
"id": 26,
"name": "News Live Streams",
"display_name": "新闻直播源",
"module": "L4",
"priority": "P2",
"frequency_minutes": 720,
"is_free": True,
"requires_credentials": False,
},
"barentswatch_vessels": {
"id": 27,
"name": "BarentsWatch AIS Vessels",
"display_name": "BarentsWatch AIS 船舶",
"module": "L4",
"priority": "P1",
"frequency_minutes": 1,
"is_free": True,
"requires_credentials": True,
"credential_provider": "barentswatch",
"credential_status": "supported",
},
"aisstream_vessels": {
"id": 28,
"name": "AISStream Vessels",
"display_name": "AISStream 实时船舶",
"module": "L4",
"priority": "P1",
"frequency_minutes": 1,
"is_free": True,
"requires_credentials": True,
"credential_provider": "aisstream",
"credential_status": "supported",
},
}
+161
View File
@@ -0,0 +1,161 @@
from __future__ import annotations
import json
import logging
import os
import re
from collections.abc import Mapping, Sequence
from typing import Any
from app.core.request_context import get_request_id
DEFAULT_SERVICE = "backend"
DEFAULT_EVENT = "app.log"
DEFAULT_LOG_LEVEL = os.getenv("PLANET_LOG_LEVEL", "INFO").upper()
REDACTED = "[REDACTED]"
SENSITIVE_FIELD_NAMES = {
"access_token",
"api_key",
"authorization",
"cookie",
"password",
"refresh_token",
"secret",
"token",
}
SENSITIVE_TEXT_PATTERNS = (
re.compile(r"(?i)(authorization\s*[:=]\s*)(.+)"),
re.compile(r"(?i)(bearer\s+)([A-Za-z0-9._\-]+)"),
re.compile(r"(?i)(token\s*[:=]\s*)(.+)"),
re.compile(r"(?i)(password\s*[:=]\s*)(.+)"),
re.compile(r"(?i)(cookie\s*[:=]\s*)(.+)"),
)
def sanitize_log_value(value: Any) -> Any:
if isinstance(value, Mapping):
return {
str(key): (REDACTED if str(key).lower() in SENSITIVE_FIELD_NAMES else sanitize_log_value(item))
for key, item in value.items()
}
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
return [sanitize_log_value(item) for item in value]
if isinstance(value, str):
sanitized = value
for pattern in SENSITIVE_TEXT_PATTERNS:
sanitized = pattern.sub(lambda match: f"{match.group(1)}{REDACTED}", sanitized)
return sanitized
return value
def _normalize_context(context: Any) -> dict[str, Any]:
if context is None:
return {}
if isinstance(context, Mapping):
sanitized = sanitize_log_value(context)
return {str(key): value for key, value in sanitized.items()}
return {"value": sanitize_log_value(context)}
class PlanetContextFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
record.request_id = getattr(record, "request_id", None) or get_request_id() or "-"
record.service = getattr(record, "service", None) or DEFAULT_SERVICE
record.event = getattr(record, "event", None) or DEFAULT_EVENT
record.context = _normalize_context(getattr(record, "context", None))
record.message = sanitize_log_value(record.getMessage())
return True
class PlanetFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
timestamp = self.formatTime(record, self.datefmt)
level = record.levelname
service = getattr(record, "service", DEFAULT_SERVICE)
module_name = record.name
event = getattr(record, "event", DEFAULT_EVENT)
request_id = getattr(record, "request_id", "-")
message = sanitize_log_value(record.getMessage())
context = _normalize_context(getattr(record, "context", None))
context_suffix = ""
if context:
context_suffix = f" context={json.dumps(context, ensure_ascii=False, sort_keys=True)}"
rendered = (
f"{timestamp} {level} service={service} module={module_name} "
f"event={event} request_id={request_id} message={message}{context_suffix}"
)
if record.exc_info:
rendered = f"{rendered}\n{self.formatException(record.exc_info)}"
return rendered
class PlanetLoggerAdapter(logging.LoggerAdapter):
def process(self, msg: Any, kwargs: dict[str, Any]) -> tuple[Any, dict[str, Any]]:
extra = dict(self.extra)
extra.update(kwargs.get("extra", {}))
if "context" in extra:
extra["context"] = _normalize_context(extra.get("context"))
kwargs["extra"] = extra
return sanitize_log_value(msg), kwargs
def log_event(
self,
level: int,
message: str,
*,
event: str,
context: Mapping[str, Any] | None = None,
**extra: Any,
) -> None:
self.log(level, message, extra={"event": event, "context": context or {}, **extra})
def debug_event(self, message: str, *, event: str, context: Mapping[str, Any] | None = None, **extra: Any) -> None:
self.log_event(logging.DEBUG, message, event=event, context=context, **extra)
def info_event(self, message: str, *, event: str, context: Mapping[str, Any] | None = None, **extra: Any) -> None:
self.log_event(logging.INFO, message, event=event, context=context, **extra)
def warning_event(self, message: str, *, event: str, context: Mapping[str, Any] | None = None, **extra: Any) -> None:
self.log_event(logging.WARNING, message, event=event, context=context, **extra)
def error_event(self, message: str, *, event: str, context: Mapping[str, Any] | None = None, **extra: Any) -> None:
self.log_event(logging.ERROR, message, event=event, context=context, **extra)
def exception_event(
self,
message: str,
*,
event: str,
context: Mapping[str, Any] | None = None,
**extra: Any,
) -> None:
self.error(message, exc_info=True, extra={"event": event, "context": context or {}, **extra})
def get_logger(name: str, *, service: str = DEFAULT_SERVICE) -> PlanetLoggerAdapter:
return PlanetLoggerAdapter(logging.getLogger(name), {"service": service})
def configure_logging(level: str | None = None) -> None:
root_logger = logging.getLogger()
if getattr(configure_logging, "_configured", False):
if level:
root_logger.setLevel(level.upper())
return
handler = logging.StreamHandler()
handler.setFormatter(PlanetFormatter(datefmt="%Y-%m-%d %H:%M:%S"))
handler.addFilter(PlanetContextFilter())
root_logger.handlers.clear()
root_logger.addHandler(handler)
root_logger.setLevel((level or DEFAULT_LOG_LEVEL).upper())
for logger_name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
target_logger = logging.getLogger(logger_name)
target_logger.handlers.clear()
target_logger.propagate = True
logging.captureWarnings(True)
configure_logging._configured = True
+14
View File
@@ -0,0 +1,14 @@
from __future__ import annotations
from contextvars import ContextVar
request_id_context: ContextVar[str | None] = ContextVar("request_id", default=None)
def set_request_id(request_id: str | None) -> None:
request_id_context.set(request_id)
def get_request_id() -> str | None:
return request_id_context.get()
+4 -2
View File
@@ -105,7 +105,7 @@ async def get_current_user(
)
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active FROM users WHERE id = :id"
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": int(user_id)},
)
@@ -122,6 +122,7 @@ async def get_current_user(
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
return user
@@ -144,7 +145,7 @@ async def get_current_user_refresh(
)
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active FROM users WHERE id = :id"
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": int(user_id)},
)
@@ -161,6 +162,7 @@ async def get_current_user_refresh(
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
return user
+157
View File
@@ -0,0 +1,157 @@
"""Registry of target schemas supported by mapped custom data sources."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field, ValidationError, field_validator
class VesselAISRecord(BaseModel):
mmsi: int = Field(ge=100000000, le=999999999)
lat: float = Field(ge=-90, le=90)
lon: float = Field(ge=-180, le=180)
sog: float | None = None
cog: float | None = Field(default=None, ge=0, le=360)
heading: int | None = Field(default=None, ge=0, le=511)
nav_status: int | None = None
name: str | None = None
callsign: str | None = None
vessel_type: str | int | None = None
vessel_type_name: str | None = None
received_at: datetime | None = None
class GeoPointRecord(BaseModel):
lat: float = Field(ge=-90, le=90)
lon: float = Field(ge=-180, le=180)
name: str | None = None
type: str | None = None
source_id: str | None = None
observed_at: datetime | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class GenericRecord(BaseModel):
data: dict[str, Any] = Field(default_factory=dict)
source_id: str | None = None
observed_at: datetime | None = None
@field_validator("data")
@classmethod
def require_payload(cls, value: dict[str, Any]) -> dict[str, Any]:
if not value:
raise ValueError("generic_records requires a non-empty data object")
return value
@dataclass(frozen=True)
class TargetField:
name: str
type: str
required: bool = False
description: str = ""
example: Any = None
def to_dict(self) -> dict[str, Any]:
return {
"name": self.name,
"type": self.type,
"required": self.required,
"description": self.description,
"example": self.example,
}
@dataclass(frozen=True)
class TargetSchema:
key: str
label: str
description: str
fields: tuple[TargetField, ...]
model: type[BaseModel]
destination: str
def to_dict(self) -> dict[str, Any]:
return {
"key": self.key,
"label": self.label,
"description": self.description,
"destination": self.destination,
"fields": [field.to_dict() for field in self.fields],
}
def validate_record(self, record: dict[str, Any]) -> tuple[dict[str, Any] | None, list[str]]:
try:
return self.model.model_validate(record).model_dump(mode="json"), []
except ValidationError as exc:
return None, [
".".join(str(part) for part in error["loc"]) + f": {error['msg']}"
for error in exc.errors()
]
TARGET_SCHEMAS: dict[str, TargetSchema] = {
"vessel_ais": TargetSchema(
key="vessel_ais",
label="船舶 AIS",
description="船只位置、航速、航向、MMSI 等 AIS 数据。",
destination="vessel_position",
model=VesselAISRecord,
fields=(
TargetField("mmsi", "integer", True, "MMSI 九位船舶标识", 257123000),
TargetField("lat", "float", True, "纬度", 59.91),
TargetField("lon", "float", True, "经度", 10.75),
TargetField("sog", "float", False, "对地航速,单位节", 12.4),
TargetField("cog", "float", False, "对地航向,0-360 度", 184.5),
TargetField("heading", "integer", False, "船首向,0-511", 186),
TargetField("nav_status", "integer", False, "导航状态码", 0),
TargetField("name", "string", False, "船名", "OSLO EXPRESS"),
TargetField("callsign", "string", False, "呼号", "LAAB"),
TargetField("vessel_type", "string", False, "船型代码", 70),
TargetField("vessel_type_name", "string", False, "船型名称", "Cargo"),
TargetField("received_at", "datetime", False, "数据接收时间", "2026-04-28T00:00:00Z"),
),
),
"geo_points": TargetSchema(
key="geo_points",
label="通用地理点",
description="带经纬度的通用实体或事件点位。",
destination="generic_geo_points",
model=GeoPointRecord,
fields=(
TargetField("lat", "float", True, "纬度", 1.3),
TargetField("lon", "float", True, "经度", 103.8),
TargetField("name", "string", False, "点位名称", "Singapore"),
TargetField("type", "string", False, "点位类型", "datacenter"),
TargetField("source_id", "string", False, "来源侧 ID", "sg-1"),
TargetField("observed_at", "datetime", False, "观测时间", "2026-04-28T00:00:00Z"),
TargetField("metadata", "object", False, "扩展字段", {"provider": "example"}),
),
),
"generic_records": TargetSchema(
key="generic_records",
label="通用结构化记录",
description="未知结构数据沉淀,不直接进入 Earth 图层。",
destination="collected_data",
model=GenericRecord,
fields=(
TargetField("data", "object", True, "结构化记录主体", {"raw": "value"}),
TargetField("source_id", "string", False, "来源侧 ID", "record-1"),
TargetField("observed_at", "datetime", False, "观测时间", "2026-04-28T00:00:00Z"),
),
),
}
def list_target_schemas() -> list[dict[str, Any]]:
return [schema.to_dict() for schema in TARGET_SCHEMAS.values()]
def get_target_schema(key: str) -> TargetSchema:
try:
return TARGET_SCHEMAS[key]
except KeyError as exc:
raise ValueError(f"Unsupported target schema: {key}") from exc
+1 -1
View File
@@ -75,7 +75,7 @@ class DataBroadcaster:
"timestamp": to_iso8601_utc(datetime.now(UTC)),
"payload": data,
},
channel=channel if channel in manager.active_connections else "all",
channel=channel,
)
async def broadcast_datasource_task_update(self, data: Dict[str, Any]):
+42 -4
View File
@@ -1,9 +1,6 @@
"""WebSocket Connection Manager"""
import json
import asyncio
from typing import Dict, Set, Optional
from datetime import datetime
from fastapi import WebSocket
import redis.asyncio as redis
@@ -15,6 +12,8 @@ class ConnectionManager:
def __init__(self):
self.active_connections: Dict[str, Set[WebSocket]] = {} # user_id -> connections
self.channel_subscriptions: Dict[str, Set[WebSocket]] = {}
self.websocket_channels: Dict[WebSocket, Set[str]] = {}
self.redis_client: Optional[redis.Redis] = None
async def connect(self, websocket: WebSocket, user_id: str):
@@ -40,6 +39,39 @@ class ConnectionManager:
self.active_connections[user_id].discard(websocket)
if not self.active_connections[user_id]:
del self.active_connections[user_id]
self.unsubscribe_all(websocket)
def subscribe(self, websocket: WebSocket, channels: list[str]):
normalized_channels = {
str(channel).strip()
for channel in channels
if str(channel).strip()
}
if not normalized_channels:
return
socket_channels = self.websocket_channels.setdefault(websocket, set())
for channel in normalized_channels:
self.channel_subscriptions.setdefault(channel, set()).add(websocket)
socket_channels.add(channel)
def unsubscribe(self, websocket: WebSocket, channels: list[str]):
for channel in {str(channel).strip() for channel in channels if str(channel).strip()}:
subscribers = self.channel_subscriptions.get(channel)
if subscribers is not None:
subscribers.discard(websocket)
if not subscribers:
del self.channel_subscriptions[channel]
socket_channels = self.websocket_channels.get(websocket)
if socket_channels is not None:
socket_channels.discard(channel)
if not socket_channels:
del self.websocket_channels[websocket]
def unsubscribe_all(self, websocket: WebSocket):
channels = list(self.websocket_channels.get(websocket, set()))
if channels:
self.unsubscribe(websocket, channels)
async def send_personal_message(self, message: dict, user_id: str):
if user_id in self.active_connections:
@@ -54,13 +86,19 @@ class ConnectionManager:
for user_id in self.active_connections:
await self.send_personal_message(message, user_id)
else:
await self.send_personal_message(message, channel)
for connection in list(self.channel_subscriptions.get(channel, set())):
try:
await connection.send_json(message)
except Exception:
self.unsubscribe_all(connection)
async def close_all(self):
for user_id in self.active_connections:
for connection in self.active_connections[user_id]:
await connection.close()
self.active_connections.clear()
self.channel_subscriptions.clear()
self.websocket_channels.clear()
manager = ConnectionManager()
@@ -0,0 +1,328 @@
{
"_comment": "Seed payload for the bgp_collector_locations DB table. Coordinates were migrated from the legacy RIPE_RIS_COLLECTOR_COORDS table and default to city-center; seeded rows are unverified and should be upgraded in the database with source evidence when known.",
"locations": [
{
"canonical_name": "RIPE RIS rrc00",
"aliases": ["rrc00", "RIPE RIS rrc00", "AMS-IX"],
"operator": "RIPE NCC",
"site": "AMS-IX",
"city": "Amsterdam",
"country": "Netherlands",
"latitude": 52.3676,
"longitude": 4.9041,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc01",
"aliases": ["rrc01", "RIPE RIS rrc01", "LINX"],
"operator": "RIPE NCC",
"site": "LINX",
"city": "London",
"country": "United Kingdom",
"latitude": 51.5072,
"longitude": -0.1276,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc03",
"aliases": ["rrc03", "RIPE RIS rrc03", "AMS-IX"],
"operator": "RIPE NCC",
"site": "AMS-IX",
"city": "Amsterdam",
"country": "Netherlands",
"latitude": 52.3676,
"longitude": 4.9041,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc04",
"aliases": ["rrc04", "RIPE RIS rrc04", "CIXP", "CERN Internet Exchange Point"],
"operator": "RIPE NCC",
"site": "CIXP",
"city": "Geneva",
"country": "Switzerland",
"latitude": 46.2044,
"longitude": 6.1432,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc05",
"aliases": ["rrc05", "RIPE RIS rrc05", "VIX", "Vienna Internet Exchange"],
"operator": "RIPE NCC",
"site": "VIX",
"city": "Vienna",
"country": "Austria",
"latitude": 48.2082,
"longitude": 16.3738,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc06",
"aliases": ["rrc06", "RIPE RIS rrc06", "JPIX", "Otemachi"],
"operator": "RIPE NCC",
"site": "JPIX",
"city": "Otemachi",
"country": "Japan",
"latitude": 35.686,
"longitude": 139.7671,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc07",
"aliases": ["rrc07", "RIPE RIS rrc07", "Netnod", "Netnod Stockholm"],
"operator": "RIPE NCC",
"site": "Netnod Stockholm",
"city": "Stockholm",
"country": "Sweden",
"latitude": 59.3293,
"longitude": 18.0686,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc10",
"aliases": ["rrc10", "RIPE RIS rrc10", "MIX", "Milan Internet Exchange"],
"operator": "RIPE NCC",
"site": "MIX",
"city": "Milan",
"country": "Italy",
"latitude": 45.4642,
"longitude": 9.19,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc11",
"aliases": ["rrc11", "RIPE RIS rrc11", "NYIIX", "New York International Internet Exchange"],
"operator": "RIPE NCC",
"site": "NYIIX",
"city": "New York",
"country": "United States",
"latitude": 40.7128,
"longitude": -74.006,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc12",
"aliases": ["rrc12", "RIPE RIS rrc12", "DE-CIX", "DE-CIX Frankfurt"],
"operator": "RIPE NCC",
"site": "DE-CIX Frankfurt",
"city": "Frankfurt",
"country": "Germany",
"latitude": 50.1109,
"longitude": 8.6821,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc13",
"aliases": ["rrc13", "RIPE RIS rrc13", "MSK-IX"],
"operator": "RIPE NCC",
"site": "MSK-IX",
"city": "Moscow",
"country": "Russia",
"latitude": 55.7558,
"longitude": 37.6173,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc14",
"aliases": ["rrc14", "RIPE RIS rrc14", "PAIX", "Palo Alto Internet Exchange"],
"operator": "RIPE NCC",
"site": "PAIX",
"city": "Palo Alto",
"country": "United States",
"latitude": 37.4419,
"longitude": -122.143,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc15",
"aliases": ["rrc15", "RIPE RIS rrc15", "PTT.br Sao Paulo", "PTTMetro Sao Paulo"],
"operator": "RIPE NCC",
"site": "PTT.br",
"city": "Sao Paulo",
"country": "Brazil",
"latitude": -23.5558,
"longitude": -46.6396,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc16",
"aliases": ["rrc16", "RIPE RIS rrc16", "Equinix Miami", "NOTA Miami"],
"operator": "RIPE NCC",
"site": "Equinix Miami",
"city": "Miami",
"country": "United States",
"latitude": 25.7617,
"longitude": -80.1918,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc18",
"aliases": ["rrc18", "RIPE RIS rrc18", "CATNIX"],
"operator": "RIPE NCC",
"site": "CATNIX",
"city": "Barcelona",
"country": "Spain",
"latitude": 41.3874,
"longitude": 2.1686,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc19",
"aliases": ["rrc19", "RIPE RIS rrc19", "NAPAfrica", "JINX", "NAPAfrica Johannesburg"],
"operator": "RIPE NCC",
"site": "NAPAfrica Johannesburg",
"city": "Johannesburg",
"country": "South Africa",
"latitude": -26.2041,
"longitude": 28.0473,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc20",
"aliases": ["rrc20", "RIPE RIS rrc20", "SwissIX"],
"operator": "RIPE NCC",
"site": "SwissIX",
"city": "Zurich",
"country": "Switzerland",
"latitude": 47.3769,
"longitude": 8.5417,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc21",
"aliases": ["rrc21", "RIPE RIS rrc21", "France-IX Paris"],
"operator": "RIPE NCC",
"site": "France-IX Paris",
"city": "Paris",
"country": "France",
"latitude": 48.8566,
"longitude": 2.3522,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc22",
"aliases": ["rrc22", "RIPE RIS rrc22", "InterLAN Bucharest"],
"operator": "RIPE NCC",
"site": "InterLAN Bucharest",
"city": "Bucharest",
"country": "Romania",
"latitude": 44.4268,
"longitude": 26.1025,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc23",
"aliases": ["rrc23", "RIPE RIS rrc23", "Equinix Singapore"],
"operator": "RIPE NCC",
"site": "Equinix Singapore",
"city": "Singapore",
"country": "Singapore",
"latitude": 1.3521,
"longitude": 103.8198,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc24",
"aliases": ["rrc24", "RIPE RIS rrc24", "LACNIC Montevideo"],
"operator": "RIPE NCC",
"site": "LACNIC Montevideo",
"city": "Montevideo",
"country": "Uruguay",
"latitude": -34.9011,
"longitude": -56.1645,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc25",
"aliases": ["rrc25", "RIPE RIS rrc25", "AMS-IX"],
"operator": "RIPE NCC",
"site": "AMS-IX",
"city": "Amsterdam",
"country": "Netherlands",
"latitude": 52.3676,
"longitude": 4.9041,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc26",
"aliases": ["rrc26", "RIPE RIS rrc26", "UAE-IX"],
"operator": "RIPE NCC",
"site": "UAE-IX",
"city": "Dubai",
"country": "United Arab Emirates",
"latitude": 25.2048,
"longitude": 55.2708,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
}
],
"city_fallbacks": []
}
+79 -1
View File
@@ -5,10 +5,22 @@ from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sess
from sqlalchemy.orm import declarative_base
from app.core.config import settings
from app.core.logging import get_logger
logger = get_logger(__name__)
DB_POOL_CONFIG = {
"pool_pre_ping": True,
"pool_recycle": 1800,
"pool_size": 10,
"max_overflow": 20,
"pool_timeout": 30,
}
engine = create_async_engine(
settings.DATABASE_URL,
echo=settings.DEBUG if hasattr(settings, "DEBUG") else False,
**DB_POOL_CONFIG,
)
async_session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
@@ -91,13 +103,41 @@ async def init_db():
import app.models.datasource_config # noqa: F401
import app.models.alert # noqa: F401
import app.models.bgp_anomaly # noqa: F401
import app.models.bgp_collector_location # noqa: F401
import app.models.bgp_incident # noqa: F401
import app.models.bgp_observation # noqa: F401
import app.models.collected_data # noqa: F401
import app.models.compute_center_location # noqa: F401
import app.models.system_setting # noqa: F401
import app.models.playground_session # noqa: F401
import app.models.playground_message # noqa: F401
import app.models.system_log # noqa: F401
import app.models.vessel # noqa: F401
import app.models.vessel_enrichment # noqa: F401
import app.models.datasource_mapping # noqa: F401
logger.warning_event(
"Database pool settings active",
event="database.pool.initialized",
context={
"pool_pre_ping": DB_POOL_CONFIG["pool_pre_ping"],
"pool_recycle": DB_POOL_CONFIG["pool_recycle"],
"pool_size": DB_POOL_CONFIG["pool_size"],
"max_overflow": DB_POOL_CONFIG["max_overflow"],
"pool_timeout": DB_POOL_CONFIG["pool_timeout"],
},
)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
await conn.execute(
text(
"""
ALTER TABLE users
ADD COLUMN IF NOT EXISTS gatekeeper_groups JSONB DEFAULT '[]'::jsonb
"""
)
)
await conn.execute(
text(
"""
@@ -117,7 +157,12 @@ async def init_db():
text(
"""
ALTER TABLE collection_tasks
ADD COLUMN IF NOT EXISTS phase VARCHAR(30) DEFAULT 'queued'
ADD COLUMN IF NOT EXISTS phase VARCHAR(30) DEFAULT 'queued',
ADD COLUMN IF NOT EXISTS phase_progress DOUBLE PRECISION,
ADD COLUMN IF NOT EXISTS phase_message VARCHAR(255),
ADD COLUMN IF NOT EXISTS phase_current BIGINT,
ADD COLUMN IF NOT EXISTS phase_total BIGINT,
ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30)
"""
)
)
@@ -129,6 +174,30 @@ async def init_db():
"""
)
)
await conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS idx_collected_data_source_current_id
ON collected_data (source, is_current, id)
"""
)
)
await conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS idx_collected_data_source_task_id
ON collected_data (source, task_id, id)
"""
)
)
await conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS idx_ais_raw_schema_observed_entity
ON ais_raw_observations (target_schema, observed_at, entity_key)
"""
)
)
await conn.execute(
text(
"""
@@ -149,5 +218,14 @@ async def init_db():
)
async with async_session_factory() as session:
from app.services.bgp_collector_locations import (
seed_default_bgp_collector_locations,
)
from app.services.compute_center_locations import (
seed_compute_center_locations_from_source_coords,
)
await seed_default_bgp_collector_locations(session)
await seed_compute_center_locations_from_source_coords(session)
await seed_default_datasources(session)
await ensure_default_admin_user(session)
+19
View File
@@ -1,4 +1,5 @@
from contextlib import asynccontextmanager
from uuid import uuid4
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
@@ -7,6 +8,8 @@ from starlette.middleware.base import BaseHTTPMiddleware
from app.api.main import api_router
from app.api.v1 import websocket
from app.core.config import settings
from app.core.logging import configure_logging
from app.core.request_context import set_request_id
from app.core.websocket.broadcaster import broadcaster
from app.db.session import init_db
from app.services.scheduler import (
@@ -17,6 +20,9 @@ from app.services.scheduler import (
)
configure_logging()
class WebSocketCORSMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request, call_next):
if request.url.path.startswith("/ws") and request.method == "GET":
@@ -28,6 +34,18 @@ class WebSocketCORSMiddleware(BaseHTTPMiddleware):
return await call_next(request)
class RequestContextMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request, call_next):
request_id = request.headers.get("X-Request-ID") or uuid4().hex
set_request_id(request_id)
try:
response = await call_next(request)
finally:
set_request_id(None)
response.headers["X-Request-ID"] = request_id
return response
@asynccontextmanager
async def lifespan(app: FastAPI):
await init_db()
@@ -58,6 +76,7 @@ app.add_middleware(
allow_headers=["*"],
)
app.add_middleware(RequestContextMiddleware)
app.add_middleware(WebSocketCORSMiddleware)
app.include_router(api_router, prefix="/api/v1")
+19
View File
@@ -6,9 +6,16 @@ from app.models.datasource import DataSource
from app.models.datasource_config import DataSourceConfig
from app.models.alert import Alert, AlertSeverity, AlertStatus
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_collector_location import BGPCollectorLocation
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.models.compute_center_location import ComputeCenterLocationRecord
from app.models.system_setting import SystemSetting
from app.models.playground_session import PlaygroundSession
from app.models.playground_message import PlaygroundMessage
from app.models.system_log import SystemLog, AuditLog
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth, VesselPosition, VesselStatic
from app.models.datasource_mapping import DataSourceMappingTemplate
__all__ = [
"User",
@@ -22,6 +29,18 @@ __all__ = [
"AlertSeverity",
"AlertStatus",
"BGPAnomaly",
"BGPCollectorLocation",
"BGPIncident",
"BGPObservation",
"ComputeCenterLocationRecord",
"SystemLog",
"AuditLog",
"PlaygroundSession",
"PlaygroundMessage",
"VesselPosition",
"VesselStatic",
"AISRawObservation",
"AISConflictRecord",
"AISSourceHealth",
"DataSourceMappingTemplate",
]
@@ -0,0 +1,52 @@
"""Stored BGP route-collector locations."""
from sqlalchemy import Boolean, Column, DateTime, Float, Integer, JSON, String, Text
from sqlalchemy.sql import func
from app.core.time import to_iso8601_utc
from app.db.session import Base
class BGPCollectorLocation(Base):
"""Current known location for a BGP route collector."""
__tablename__ = "bgp_collector_locations"
id = Column(Integer, primary_key=True, autoincrement=True)
collector_id = Column(String(100), nullable=False, unique=True, index=True)
operator = Column(String(255), nullable=True)
site = Column(String(255), nullable=True)
city = Column(String(255), nullable=True)
country = Column(String(255), nullable=True)
latitude = Column(Float, nullable=True)
longitude = Column(Float, nullable=True)
precision = Column(String(30), nullable=False, default="city")
confidence = Column(Float, nullable=True)
source = Column(String(80), nullable=False, default="legacy_seed", index=True)
source_url = Column(String(500), nullable=True)
source_note = Column(Text, nullable=True)
raw_payload = Column(JSON, nullable=False, default=dict)
needs_confirmation = Column(Boolean, nullable=False, default=True, index=True)
verification_status = Column(String(30), nullable=False, default="unverified", index=True)
verified_at = Column(DateTime(timezone=True), nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
def to_location_dict(self) -> dict:
return {
"city": self.city,
"country": self.country,
"latitude": self.latitude,
"longitude": self.longitude,
"precision": self.precision,
"source": self.source,
"needs_confirmation": self.needs_confirmation,
"matched_location_name": self.site or self.collector_id,
"verified_at": to_iso8601_utc(self.verified_at),
"confidence": self.confidence,
"operator": self.operator,
"site": self.site,
"verification_status": self.verification_status,
"source_note": self.source_note,
"source_url": self.source_url,
}
+2
View File
@@ -48,6 +48,8 @@ class CollectedData(Base):
# Indexes for common queries
__table_args__ = (
Index("idx_collected_data_source_collected", "source", "collected_at"),
Index("idx_collected_data_source_current_id", "source", "is_current", "id"),
Index("idx_collected_data_source_task_id", "source", "task_id", "id"),
Index("idx_collected_data_source_type", "source", "data_type"),
Index("idx_collected_data_source_source_id", "source", "source_id"),
)
@@ -0,0 +1,60 @@
"""Stored compute-center locations."""
from sqlalchemy import Boolean, Column, DateTime, Float, Integer, JSON, String, Text, UniqueConstraint
from sqlalchemy.sql import func
from app.core.time import to_iso8601_utc
from app.db.session import Base
class ComputeCenterLocationRecord(Base):
"""Current known location for a compute-center record."""
__tablename__ = "compute_center_locations"
__table_args__ = (
UniqueConstraint("source", "source_id", name="uq_compute_center_location_source_id"),
)
id = Column(Integer, primary_key=True, autoincrement=True)
source = Column(String(100), nullable=False, index=True)
source_id = Column(String(255), nullable=False, index=True)
name = Column(String(500), nullable=True)
operator = Column(String(255), nullable=True)
site = Column(String(255), nullable=True)
city = Column(String(255), nullable=True)
country = Column(String(255), nullable=True)
latitude = Column(Float, nullable=True)
longitude = Column(Float, nullable=True)
precision = Column(String(30), nullable=False, default="city")
confidence = Column(Float, nullable=True)
location_source = Column(String(80), nullable=False, default="stored_compute_center_location", index=True)
source_url = Column(String(500), nullable=True)
source_note = Column(Text, nullable=True)
raw_payload = Column(JSON, nullable=False, default=dict)
needs_confirmation = Column(Boolean, nullable=False, default=False, index=True)
verification_status = Column(String(30), nullable=False, default="verified", index=True)
verified_at = Column(DateTime(timezone=True), nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
def to_location_dict(self) -> dict:
return {
"source": self.source,
"source_id": self.source_id,
"name": self.name,
"operator": self.operator,
"site": self.site,
"city": self.city,
"country": self.country,
"latitude": self.latitude,
"longitude": self.longitude,
"precision": self.precision,
"confidence": self.confidence,
"location_source": self.location_source,
"source_url": self.source_url,
"source_note": self.source_note,
"raw_payload": self.raw_payload or {},
"needs_confirmation": self.needs_confirmation,
"verification_status": self.verification_status,
"verified_at": to_iso8601_utc(self.verified_at),
}
+32
View File
@@ -0,0 +1,32 @@
"""Mapping templates for user-defined data source payloads."""
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, JSON, String
from sqlalchemy.sql import func
from app.db.session import Base
class DataSourceMappingTemplate(Base):
__tablename__ = "datasource_mapping_templates"
id = Column(Integer, primary_key=True, autoincrement=True)
datasource_config_id = Column(
Integer,
ForeignKey("datasource_configs.id"),
nullable=False,
index=True,
)
target_schema = Column(String(80), nullable=False, index=True)
mapping_json = Column(JSON, nullable=False, default={})
sample_payload_hash = Column(String(64), nullable=True)
validation_status = Column(String(30), nullable=False, default="draft")
version = Column(Integer, nullable=False, default=1)
is_active = Column(Boolean, nullable=False, default=False, index=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
def __repr__(self):
return (
f"<DataSourceMappingTemplate {self.id}: "
f"{self.datasource_config_id}/{self.target_schema}/v{self.version}>"
)
+40
View File
@@ -0,0 +1,40 @@
from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Integer, String, Text
from sqlalchemy.sql import func
from app.db.session import Base
class PlaygroundMessage(Base):
__tablename__ = "playground_messages"
id = Column(Integer, primary_key=True, autoincrement=True)
public_id = Column(String(64), unique=True, index=True, nullable=False)
session_id = Column(Integer, ForeignKey("playground_sessions.id", ondelete="CASCADE"), nullable=False, index=True)
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
parent_message_id = Column(Integer, ForeignKey("playground_messages.id", ondelete="SET NULL"), nullable=True)
role = Column(String(20), nullable=False)
kind = Column(String(20), nullable=False, default="message")
status = Column(String(20), nullable=False, default="done")
title = Column(String(255), nullable=True)
content = Column(Text, nullable=False, default="")
thinking_content = Column(Text, nullable=False, default="")
meta = Column(JSON, nullable=False, default=list)
provider = Column(String(100), nullable=True)
model = Column(String(200), nullable=True)
request_id = Column(String(100), nullable=True)
raw_response = Column(JSON, nullable=False, default=dict)
content_blocks = Column(JSON, nullable=False, default=list)
text_blocks = Column(JSON, nullable=False, default=list)
thinking_blocks = Column(JSON, nullable=False, default=list)
sort_order = Column(Integer, nullable=False, default=0, index=True)
is_visible = Column(Boolean, nullable=False, default=True)
created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
updated_at = Column(
DateTime(timezone=True),
server_default=func.now(),
onupdate=func.now(),
nullable=False,
)
def __repr__(self):
return f"<PlaygroundMessage public_id={self.public_id} role={self.role} status={self.status}>"
+27
View File
@@ -0,0 +1,27 @@
from sqlalchemy import JSON, Column, DateTime, ForeignKey, Integer, String, UniqueConstraint
from sqlalchemy.sql import func
from app.db.session import Base
class PlaygroundSession(Base):
__tablename__ = "playground_sessions"
__table_args__ = (
UniqueConstraint("user_id", "session_key", name="uq_playground_sessions_user_session_key"),
)
id = Column(Integer, primary_key=True, autoincrement=True)
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
session_key = Column(String(100), nullable=False, default="default")
title = Column(String(200), nullable=False, default="Playground 会话")
state = Column(JSON, nullable=False, default={})
created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
updated_at = Column(
DateTime(timezone=True),
server_default=func.now(),
onupdate=func.now(),
nullable=False,
)
def __repr__(self):
return f"<PlaygroundSession user_id={self.user_id} session_key={self.session_key}>"
+40
View File
@@ -0,0 +1,40 @@
from sqlalchemy import JSON, Column, DateTime, Integer, String, Text
from sqlalchemy.sql import func
from app.db.session import Base
class SystemLog(Base):
__tablename__ = "system_logs"
id = Column(Integer, primary_key=True, autoincrement=True)
occurred_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
source = Column(String(50), nullable=False, index=True)
service = Column(String(50), nullable=True)
module = Column(String(120), nullable=True)
event = Column(String(160), nullable=True, index=True)
level = Column(String(20), nullable=False, index=True)
message = Column(Text, nullable=False)
request_id = Column(String(64), nullable=True, index=True)
trace_id = Column(String(64), nullable=True)
user_id = Column(Integer, nullable=True, index=True)
category = Column(String(80), nullable=True, index=True)
context = Column(JSON, nullable=False, default=dict)
created_at = Column(DateTime(timezone=True), server_default=func.now())
class AuditLog(Base):
__tablename__ = "audit_logs"
id = Column(Integer, primary_key=True, autoincrement=True)
occurred_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
actor_id = Column(Integer, nullable=True, index=True)
actor_name = Column(String(255), nullable=True)
action = Column(String(120), nullable=False, index=True)
target_type = Column(String(80), nullable=True)
target_id = Column(String(120), nullable=True)
result = Column(String(40), nullable=True, index=True)
request_id = Column(String(64), nullable=True, index=True)
ip = Column(String(64), nullable=True)
details = Column(JSON, nullable=False, default=dict)
created_at = Column(DateTime(timezone=True), server_default=func.now())
+6 -1
View File
@@ -1,6 +1,6 @@
"""Collection Task model"""
from sqlalchemy import Column, DateTime, Integer, String, Text, Float
from sqlalchemy import BigInteger, Column, DateTime, Integer, String, Text, Float
from sqlalchemy.sql import func
from app.db.session import Base
@@ -13,6 +13,11 @@ class CollectionTask(Base):
datasource_id = Column(Integer, nullable=False, index=True)
status = Column(String(20), nullable=False) # pending, running, success, failed, cancelled
phase = Column(String(30), default="queued")
phase_progress = Column(Float)
phase_message = Column(String(255))
phase_current = Column(BigInteger)
phase_total = Column(BigInteger)
phase_unit = Column(String(30))
started_at = Column(DateTime(timezone=True))
completed_at = Column(DateTime(timezone=True))
records_processed = Column(Integer, default=0)
+2 -1
View File
@@ -1,4 +1,4 @@
from sqlalchemy import Boolean, Column, Integer, String, DateTime
from sqlalchemy import Boolean, Column, DateTime, Integer, JSON, String
from sqlalchemy.sql import func
from app.db.session import Base
@@ -12,6 +12,7 @@ class User(Base):
email = Column(String(255), unique=True, index=True, nullable=False)
password_hash = Column(String(255), nullable=False)
role = Column(String(20), default="viewer")
gatekeeper_groups = Column(JSON, default=list)
is_active = Column(Boolean, default=True)
last_login_at = Column(DateTime(timezone=True))
created_at = Column(DateTime(timezone=True), server_default=func.now())
+186
View File
@@ -0,0 +1,186 @@
"""Vessel AIS models for live maritime tracking."""
from sqlalchemy import BigInteger, Column, DateTime, Float, Index, Integer, JSON, SmallInteger, String
from sqlalchemy.sql import func
from app.core.time import to_iso8601_utc
from app.db.session import Base
class VesselStatic(Base):
"""Slow-changing vessel identity and dimensions."""
__tablename__ = "vessel_static"
mmsi = Column(BigInteger, primary_key=True)
name = Column(String(128), nullable=True)
callsign = Column(String(16), nullable=True)
vessel_type = Column(SmallInteger, nullable=True, index=True)
vessel_type_name = Column(String(64), nullable=True, index=True)
flag = Column(String(4), nullable=True, index=True)
length = Column(Float, nullable=True)
width = Column(Float, nullable=True)
draught = Column(Float, nullable=True)
imo = Column(BigInteger, nullable=True)
updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
def to_dict(self) -> dict:
return {
"mmsi": self.mmsi,
"name": self.name,
"callsign": self.callsign,
"vessel_type": self.vessel_type,
"vessel_type_name": self.vessel_type_name,
"flag": self.flag,
"length": self.length,
"width": self.width,
"draught": self.draught,
"imo": self.imo,
"updated_at": to_iso8601_utc(self.updated_at),
}
class VesselPosition(Base):
"""Append-only AIS positions retained for short history windows."""
__tablename__ = "vessel_position"
id = Column(Integer, primary_key=True, autoincrement=True)
mmsi = Column(BigInteger, nullable=False, index=True)
lat = Column(Float, nullable=False)
lon = Column(Float, nullable=False)
sog = Column(Float, nullable=True)
cog = Column(Float, nullable=True)
heading = Column(SmallInteger, nullable=True)
nav_status = Column(SmallInteger, nullable=True, index=True)
received_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
__table_args__ = (
Index("idx_vessel_pos_mmsi_time", "mmsi", "received_at"),
Index("idx_vessel_pos_time", "received_at"),
Index("idx_vessel_pos_lat_lon", "lat", "lon"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"mmsi": self.mmsi,
"lat": self.lat,
"lon": self.lon,
"sog": self.sog,
"cog": self.cog,
"heading": self.heading,
"nav_status": self.nav_status,
"received_at": to_iso8601_utc(self.received_at),
}
class AISRawObservation(Base):
"""Source-level AIS fact before aggregation and conflict resolution."""
__tablename__ = "ais_raw_observations"
id = Column(Integer, primary_key=True, autoincrement=True)
target_schema = Column(String(64), nullable=False, default="vessel_ais", index=True)
source = Column(String(100), nullable=False, index=True)
entity_key = Column(String(64), nullable=False, index=True)
delivery_mode = Column(String(32), nullable=False, index=True)
transport = Column(String(32), nullable=False, index=True)
message_type = Column(String(64), nullable=True, index=True)
source_message_id = Column(String(128), nullable=True, index=True)
observation_hash = Column(String(64), nullable=False, unique=True, index=True)
observed_at = Column(DateTime(timezone=True), nullable=False, index=True)
collected_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
normalized_payload = Column(JSON, default=dict)
raw_payload = Column(JSON, default=dict)
quality_flags = Column(JSON, default=list)
__table_args__ = (
Index("idx_ais_raw_entity_observed", "target_schema", "entity_key", "observed_at"),
Index("idx_ais_raw_schema_observed_entity", "target_schema", "observed_at", "entity_key"),
Index("idx_ais_raw_source_entity", "source", "entity_key"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"target_schema": self.target_schema,
"source": self.source,
"entity_key": self.entity_key,
"delivery_mode": self.delivery_mode,
"transport": self.transport,
"message_type": self.message_type,
"source_message_id": self.source_message_id,
"observation_hash": self.observation_hash,
"observed_at": to_iso8601_utc(self.observed_at),
"collected_at": to_iso8601_utc(self.collected_at),
"normalized_payload": self.normalized_payload or {},
"raw_payload": self.raw_payload or {},
"quality_flags": self.quality_flags or [],
}
class AISConflictRecord(Base):
"""Recorded field-level disagreement between AIS sources."""
__tablename__ = "ais_conflict_records"
id = Column(Integer, primary_key=True, autoincrement=True)
target_schema = Column(String(64), nullable=False, default="vessel_ais", index=True)
entity_key = Column(String(64), nullable=False, index=True)
field = Column(String(64), nullable=False, index=True)
candidates = Column(JSON, default=dict)
selected_source = Column(String(100), nullable=True, index=True)
selected_value = Column(JSON, nullable=True)
selected_reason = Column(String(64), nullable=True, index=True)
resolved_by = Column(String(32), nullable=False, default="system", index=True)
status = Column(String(32), nullable=False, default="open", index=True)
created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
__table_args__ = (
Index("idx_ais_conflict_entity_field", "target_schema", "entity_key", "field"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"target_schema": self.target_schema,
"entity_key": self.entity_key,
"field": self.field,
"candidates": self.candidates or {},
"selected_source": self.selected_source,
"selected_value": self.selected_value,
"selected_reason": self.selected_reason,
"resolved_by": self.resolved_by,
"status": self.status,
"created_at": to_iso8601_utc(self.created_at),
"updated_at": to_iso8601_utc(self.updated_at),
}
class AISSourceHealth(Base):
"""Runtime health signal for an AIS collector source."""
__tablename__ = "ais_source_health"
source = Column(String(100), primary_key=True)
connection_state = Column(String(32), nullable=False, default="disconnected", index=True)
last_seen_at = Column(DateTime(timezone=True), nullable=True, index=True)
last_success_at = Column(DateTime(timezone=True), nullable=True, index=True)
last_error = Column(String(500), nullable=True)
message_rate = Column(Float, nullable=True)
lag_seconds = Column(Float, nullable=True)
updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
def to_dict(self) -> dict:
return {
"source": self.source,
"connection_state": self.connection_state,
"last_seen_at": to_iso8601_utc(self.last_seen_at),
"last_success_at": to_iso8601_utc(self.last_success_at),
"last_error": self.last_error,
"message_rate": self.message_rate,
"lag_seconds": self.lag_seconds,
"updated_at": to_iso8601_utc(self.updated_at),
}
+63
View File
@@ -0,0 +1,63 @@
"""Vessel enrichment cache tables (v5).
Profile and media enrichment are stored separately so cache TTLs can differ
and so the conflict-resolution + display layers can read either independently.
"""
from sqlalchemy import BigInteger, Column, DateTime, Float, JSON, String
from sqlalchemy.sql import func
from app.core.time import to_iso8601_utc
from app.db.session import Base
class VesselProfileEnrichment(Base):
"""Cached static vessel profile (type, flag, dimensions, operator, etc.)."""
__tablename__ = "vessel_profile_enrichment"
mmsi = Column(BigInteger, primary_key=True)
source = Column(String(100), nullable=False, default="system")
payload = Column(JSON, nullable=False, default=dict)
fetched_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
expires_at = Column(DateTime(timezone=True), nullable=True)
confidence = Column(Float, nullable=True)
reference_url = Column(String(500), nullable=True)
updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now())
def to_dict(self) -> dict:
return {
"mmsi": self.mmsi,
"source": self.source,
"payload": self.payload or {},
"fetched_at": to_iso8601_utc(self.fetched_at),
"expires_at": to_iso8601_utc(self.expires_at),
"confidence": self.confidence,
"reference_url": self.reference_url,
}
class VesselMediaEnrichment(Base):
"""Cached vessel imagery / external detail references."""
__tablename__ = "vessel_media_enrichment"
mmsi = Column(BigInteger, primary_key=True)
source = Column(String(100), nullable=False, default="system")
payload = Column(JSON, nullable=False, default=dict)
fetched_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
expires_at = Column(DateTime(timezone=True), nullable=True)
confidence = Column(Float, nullable=True)
reference_url = Column(String(500), nullable=True)
updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now())
def to_dict(self) -> dict:
return {
"mmsi": self.mmsi,
"source": self.source,
"payload": self.payload or {},
"fetched_at": to_iso8601_utc(self.fetched_at),
"expires_at": to_iso8601_utc(self.expires_at),
"confidence": self.confidence,
"reference_url": self.reference_url,
}
+148
View File
@@ -3,6 +3,14 @@ from typing import Any
from pydantic import BaseModel, Field
class AIContentBlock(BaseModel):
type: str
text: str | None = None
thinking: str | None = None
signature: str | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class SituationalAnalysisRequest(BaseModel):
title: str = Field(..., min_length=1, max_length=200)
objective: str = Field(..., min_length=1, max_length=1000)
@@ -10,18 +18,158 @@ class SituationalAnalysisRequest(BaseModel):
observations: list[str] = Field(default_factory=list)
constraints: list[str] = Field(default_factory=list)
preferred_model: str | None = Field(default=None, max_length=200)
thinking: dict[str, Any] | None = None
class BGPBriefRequest(BaseModel):
incident_limit: int = Field(default=5, ge=1, le=10)
anomaly_limit: int = Field(default=6, ge=1, le=12)
collector_limit: int = Field(default=5, ge=1, le=10)
preferred_model: str | None = Field(default=None, max_length=200)
thinking: dict[str, Any] | None = None
class AlertBriefRequest(BaseModel):
alert_limit: int = Field(default=8, ge=1, le=20)
preferred_model: str | None = Field(default=None, max_length=200)
thinking: dict[str, Any] | None = None
class SituationalAlertBriefRequest(BaseModel):
preferred_model: str | None = Field(default=None, max_length=200)
thinking: dict[str, Any] | None = None
class SituationalAnalysisResponse(BaseModel):
provider: str
model: str
content: str
content_blocks: list[AIContentBlock] = Field(default_factory=list)
text_blocks: list[str] = Field(default_factory=list)
thinking_blocks: list[str] = Field(default_factory=list)
raw_response: dict[str, Any] = Field(default_factory=dict)
class BGPBriefRecordSummary(BaseModel):
id: str
title: str
provider: str
model: str
request_id: str | None = None
generated_at: str
class BGPBriefRecordResponse(BGPBriefRecordSummary):
content_markdown: str
facts: list[str] = Field(default_factory=list)
context: dict[str, Any] = Field(default_factory=dict)
class AlertBriefResponse(SituationalAnalysisResponse):
title: str
objective: str
facts: list[str] = Field(default_factory=list)
context: dict[str, Any] = Field(default_factory=dict)
class SituationalAlertBriefResponse(SituationalAnalysisResponse):
title: str
objective: str
facts: list[str] = Field(default_factory=list)
context: dict[str, Any] = Field(default_factory=dict)
class AIProviderStatusResponse(BaseModel):
provider: str
api: str | None = None
enabled: bool
configured: bool
model: str | None = None
base_url: str | None = None
class PlaygroundSessionState(BaseModel):
messages: list[dict[str, Any]] = Field(default_factory=list)
selectedPresetKey: str = Field(default="bgp-brief", max_length=100)
title: str = Field(default="", max_length=200)
objective: str = Field(default="", max_length=1000)
constraints: str = Field(default="")
inputValue: str = Field(default="")
analysis: dict[str, Any] | None = None
latestAnalysisMessageId: str | None = Field(default=None, max_length=200)
analysisMeta: dict[str, Any] = Field(default_factory=dict)
helpExpanded: bool = True
class PlaygroundSessionUpsertRequest(BaseModel):
session_key: str = Field(default="default", min_length=1, max_length=100)
title: str | None = Field(default=None, max_length=200)
state: PlaygroundSessionState
class PlaygroundMessageRecord(BaseModel):
id: str
role: str
kind: str = "message"
status: str = "done"
title: str | None = None
content: str = ""
thinking_content: str = ""
meta: list[str] = Field(default_factory=list)
markdown: bool = True
provider: str | None = None
model: str | None = None
request_id: str | None = None
raw_response: dict[str, Any] = Field(default_factory=dict)
content_blocks: list[dict[str, Any]] = Field(default_factory=list)
text_blocks: list[str] = Field(default_factory=list)
thinking_blocks: list[str] = Field(default_factory=list)
parent_message_id: str | None = None
created_at: str
updated_at: str
class PlaygroundSessionResponse(BaseModel):
id: str
session_key: str
title: str
state: PlaygroundSessionState
created_at: str
updated_at: str
class PlaygroundThreadResponse(BaseModel):
session: PlaygroundSessionResponse
messages: list[PlaygroundMessageRecord] = Field(default_factory=list)
class PlaygroundMessageCreateRequest(BaseModel):
session_key: str = Field(default="default", min_length=1, max_length=100)
title: str = Field(..., min_length=1, max_length=200)
objective: str = Field(..., min_length=1, max_length=1000)
constraints: str = Field(default="")
input: str = Field(..., min_length=1)
selected_preset_key: str = Field(default="bgp-brief", max_length=100)
help_expanded: bool = True
class PlaygroundMessageActionResponse(BaseModel):
session: PlaygroundSessionResponse
messages: list[PlaygroundMessageRecord] = Field(default_factory=list)
active_message_id: str | None = None
class PlaygroundMessageStopRequest(BaseModel):
session_key: str = Field(default="default", min_length=1, max_length=100)
message_id: str = Field(..., min_length=1, max_length=64)
class PlaygroundMessageResendRequest(BaseModel):
session_key: str = Field(default="default", min_length=1, max_length=100)
user_message_id: str = Field(..., min_length=1, max_length=64)
class PlaygroundMessageEditRequest(BaseModel):
session_key: str = Field(default="default", min_length=1, max_length=100)
user_message_id: str = Field(..., min_length=1, max_length=64)
content: str = Field(..., min_length=1)
+5
View File
@@ -0,0 +1,5 @@
from pydantic import BaseModel, Field
class AlertResolutionRequest(BaseModel):
resolution: str = Field(..., min_length=1, max_length=1000)
+4
View File
@@ -12,17 +12,20 @@ class UserBase(BaseModel):
class UserCreate(UserBase):
password: str = Field(..., min_length=8)
role: str = "viewer"
gatekeeper_groups: list[str] = Field(default_factory=list)
class UserUpdate(BaseModel):
email: Optional[EmailStr] = None
role: Optional[str] = None
gatekeeper_groups: Optional[list[str]] = None
is_active: Optional[bool] = None
class UserInDB(UserBase):
id: int
role: str
gatekeeper_groups: list[str] = Field(default_factory=list)
is_active: bool
last_login_at: Optional[datetime]
created_at: datetime
@@ -34,6 +37,7 @@ class UserInDB(UserBase):
class UserResponse(UserBase):
id: int
role: str
gatekeeper_groups: list[str] = Field(default_factory=list)
is_active: bool
created_at: datetime
+48 -8
View File
@@ -3,9 +3,11 @@ from __future__ import annotations
import asyncio
import httpx
from fastapi import HTTPException, status
from fastapi import Depends, HTTPException, status
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import settings
from app.db.session import get_db
from app.schemas.ai import (
AIProviderStatusResponse,
SituationalAnalysisRequest,
@@ -14,11 +16,27 @@ from app.schemas.ai import (
class AIProviderClient:
def __init__(self) -> None:
self.service_url = settings.AI_PROVIDER_SERVICE_URL.rstrip("/")
self.service_token = settings.AI_PROVIDER_SERVICE_TOKEN
self.timeout = settings.AI_PROVIDER_TIMEOUT_SECONDS
self.retry_attempts = max(settings.AI_PROVIDER_RETRY_ATTEMPTS, 1)
def __init__(
self,
*,
service_url: str | None = None,
service_token: str | None = None,
timeout: int | None = None,
retry_attempts: int | None = None,
llm_config: dict | None = None,
) -> None:
self.service_url = (
service_url if service_url is not None else settings.AI_PROVIDER_SERVICE_URL
).rstrip("/")
self.service_token = (
service_token if service_token is not None else settings.AI_PROVIDER_SERVICE_TOKEN
)
self.timeout = timeout if timeout is not None else settings.AI_PROVIDER_TIMEOUT_SECONDS
self.retry_attempts = max(
retry_attempts if retry_attempts is not None else settings.AI_PROVIDER_RETRY_ATTEMPTS,
1,
)
self.llm_config = llm_config or {}
def _headers(self, request_id: str | None = None) -> dict[str, str]:
headers = {"Content-Type": "application/json"}
@@ -26,6 +44,19 @@ class AIProviderClient:
headers["X-Provider-Token"] = self.service_token
if request_id:
headers["X-Request-ID"] = request_id
llm_header_map = {
"provider": "X-AI-Provider",
"provider_api": "X-AI-Provider-API",
"base_url": "X-AI-Base-URL",
"api_key": "X-AI-API-Key",
"model": "X-AI-Model",
"max_tokens": "X-AI-Max-Tokens",
"anthropic_version": "X-AI-Anthropic-Version",
}
for key, header_name in llm_header_map.items():
value = self.llm_config.get(key)
if value not in (None, ""):
headers[header_name] = str(value)
return headers
async def get_status(self, request_id: str | None = None) -> AIProviderStatusResponse:
@@ -105,5 +136,14 @@ class AIProviderClient:
)
def get_ai_provider_client() -> AIProviderClient:
return AIProviderClient()
async def get_ai_provider_client(db: AsyncSession = Depends(get_db)) -> AIProviderClient:
from app.api.v1.settings import get_runtime_ai_provider_config
runtime_config = await get_runtime_ai_provider_config(db)
return AIProviderClient(
service_url=runtime_config["service_url"],
service_token=runtime_config["service_token"],
timeout=runtime_config["timeout_seconds"],
retry_attempts=runtime_config["retry_attempts"],
llm_config=runtime_config.get("llm_config") or {},
)
+103
View File
@@ -0,0 +1,103 @@
from __future__ import annotations
from collections import Counter
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.alert import Alert, AlertSeverity, AlertStatus
from app.schemas.ai import AlertBriefRequest, SituationalAnalysisRequest
def _format_counter(counter: Counter[str], empty_text: str = "无") -> str:
if not counter:
return empty_text
return ",".join(f"{key} {value}" for key, value in counter.items())
async def build_alert_brief_request(
db: AsyncSession,
*,
alert_limit: int = 8,
) -> tuple[SituationalAnalysisRequest, list[str], dict[str, Any]]:
recent_alerts_result = await db.execute(
select(Alert)
.order_by(Alert.created_at.desc(), Alert.id.desc())
.limit(max(alert_limit, 1))
)
total_result = await db.execute(select(func.count(Alert.id)))
active_result = await db.execute(select(func.count(Alert.id)).where(Alert.status == AlertStatus.ACTIVE))
acknowledged_result = await db.execute(
select(func.count(Alert.id)).where(Alert.status == AlertStatus.ACKNOWLEDGED)
)
resolved_result = await db.execute(select(func.count(Alert.id)).where(Alert.status == AlertStatus.RESOLVED))
recent_alerts = recent_alerts_result.scalars().all()
total_alerts = total_result.scalar() or 0
active_alerts = active_result.scalar() or 0
acknowledged_alerts = acknowledged_result.scalar() or 0
resolved_alerts = resolved_result.scalar() or 0
severity_counts = Counter((item.severity.value if item.severity else "unknown") for item in recent_alerts)
status_counts = Counter((item.status.value if item.status else "unknown") for item in recent_alerts)
datasource_counts = Counter((item.datasource_name or "未命名数据源") for item in recent_alerts)
active_datasource_counts = Counter(
(item.datasource_name or "未命名数据源")
for item in recent_alerts
if item.status == AlertStatus.ACTIVE
)
facts = [
f"告警总量 {total_alerts} 条,其中 active {active_alerts} 条、acknowledged {acknowledged_alerts} 条、resolved {resolved_alerts} 条。",
f"最近告警严重度分布:{_format_counter(severity_counts)}。",
f"最近告警状态分布:{_format_counter(status_counts)}。",
f"最近告警数据源分布:{_format_counter(Counter(dict(datasource_counts.most_common(6))))}。",
]
if active_datasource_counts:
facts.append(
"当前待处理告警主要集中在:"
+ _format_counter(Counter(dict(active_datasource_counts.most_common(5))))
+ "。"
)
if recent_alerts:
facts.append(
"最近告警摘录:"
+ ";".join(
[
f"{item.datasource_name or '未命名数据源'} / {item.severity.value if item.severity else '-'} / {item.status.value if item.status else '-'} / {item.message or '-'}"
for item in recent_alerts[:6]
]
)
)
context = {
"source": "alerts",
"total_alerts": total_alerts,
"active_alerts": active_alerts,
"acknowledged_alerts": acknowledged_alerts,
"resolved_alerts": resolved_alerts,
"severity_distribution": dict(severity_counts),
"status_distribution": dict(status_counts),
"top_datasources": dict(datasource_counts.most_common(6)),
"top_active_datasources": dict(active_datasource_counts.most_common(5)),
}
return (
SituationalAnalysisRequest(
title="告警态势 AI 简报",
objective="基于当前告警总量、严重度、状态、数据源分布与最近告警摘录,生成一份面向值班人员的简明告警态势简报,突出待处理风险、告警集中点和优先动作。",
observations=facts,
constraints=[
"明确区分事实、推断与建议。",
"优先指出仍处于 active 状态且高严重度的告警簇。",
"不要把 acknowledged 或 resolved 告警误判成当前仍在扩大。",
"如果证据不足,请明确指出缺失的上下文。",
],
context=context,
),
facts,
context,
)
+209
View File
@@ -0,0 +1,209 @@
"""BarentsWatch AIS credential resolution and connectivity checks."""
from __future__ import annotations
import os
import shlex
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import httpx
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.data_sources import get_data_sources_config
from app.models.datasource_config import DataSourceConfig
BARENTSWATCH_LATEST_URL = "https://live.ais.barentswatch.no/v1/latest/combined"
BARENTSWATCH_TOKEN_URL = "https://id.barentswatch.no/connect/token"
BARENTSWATCH_DATASOURCE_NAME = "barentswatch_vessels"
@dataclass(frozen=True)
class BarentsWatchConfig:
endpoint: str
client_id: str
client_secret: str
credential_source: str
endpoint_source: str
def _read_zshrc_env(path: Path | None = None) -> dict[str, str]:
zshrc_path = path or Path.home() / ".zshrc"
if not zshrc_path.exists():
return {}
values: dict[str, str] = {}
for raw_line in zshrc_path.read_text(encoding="utf-8", errors="ignore").splitlines():
line = raw_line.strip()
if not line or line.startswith("#"):
continue
if line.startswith("export "):
line = line[len("export ") :].strip()
if "=" not in line:
continue
key, value = line.split("=", 1)
key = key.strip()
if not key or not key.replace("_", "").isalnum() or not key[0].isalpha():
continue
try:
parsed = shlex.split(value, comments=True, posix=True)
except ValueError:
parsed = [value.strip().strip("'\"")]
if parsed:
values[key] = parsed[0]
return values
def _first_env_value(zshrc_env: dict[str, str], *keys: str) -> tuple[str, str]:
for key in keys:
value = os.getenv(key)
if value:
return value, "environment"
for key in keys:
value = zshrc_env.get(key)
if value:
return value, "~/.zshrc"
return "", ""
async def get_barentswatch_datasource_record(db: AsyncSession) -> DataSourceConfig | None:
result = await db.execute(
select(DataSourceConfig)
.where(DataSourceConfig.name == BARENTSWATCH_DATASOURCE_NAME)
.where(DataSourceConfig.is_active.is_(True))
)
return result.scalar_one_or_none()
async def resolve_barentswatch_config(db: AsyncSession | None = None) -> BarentsWatchConfig:
record = await get_barentswatch_datasource_record(db) if db else None
auth_config = dict(record.auth_config or {}) if record else {}
config = dict(record.config or {}) if record else {}
zshrc_env = _read_zshrc_env()
env_client_id, env_source = _first_env_value(
zshrc_env,
"BARENTSWATCH_CLIENT_ID",
"BARRENTSWATCH_CLIENT_ID",
)
env_client_secret, secret_env_source = _first_env_value(
zshrc_env,
"BARENTSWATCH_CLIENT_SECRET",
"BARRENTSWATCH_CLIENT_SECRET",
)
client_id = auth_config.get("client_id") or config.get("client_id") or env_client_id
client_secret = (
auth_config.get("client_secret") or config.get("client_secret") or env_client_secret
)
credential_source = ""
if auth_config.get("client_id") or auth_config.get("client_secret"):
credential_source = "datasource_config"
elif config.get("client_id") or config.get("client_secret"):
credential_source = "datasource_runtime_config"
elif env_source or secret_env_source:
credential_source = env_source or secret_env_source
yaml_endpoint = get_data_sources_config().get_yaml_url(BARENTSWATCH_DATASOURCE_NAME)
endpoint = record.endpoint if record and record.endpoint else yaml_endpoint
return BarentsWatchConfig(
endpoint=endpoint or BARENTSWATCH_LATEST_URL,
client_id=str(client_id or ""),
client_secret=str(client_secret or ""),
credential_source=credential_source or "missing",
endpoint_source="datasource_config" if record and record.endpoint else "default",
)
async def fetch_barentswatch_access_token(
client: httpx.AsyncClient,
config: BarentsWatchConfig,
) -> str | None:
if not config.client_id or not config.client_secret:
return None
response = await client.post(
BARENTSWATCH_TOKEN_URL,
data={
"client_id": config.client_id,
"client_secret": config.client_secret,
"scope": "ais",
"grant_type": "client_credentials",
},
headers={"Content-Type": "application/x-www-form-urlencoded"},
)
response.raise_for_status()
payload = response.json()
token = payload.get("access_token")
return str(token) if token else None
async def check_barentswatch_connectivity(db: AsyncSession) -> dict[str, Any]:
config = await resolve_barentswatch_config(db)
return await check_barentswatch_config(config)
async def check_barentswatch_config(config: BarentsWatchConfig) -> dict[str, Any]:
if not config.client_id or not config.client_secret:
return {
"success": False,
"stage": "credentials",
"message": "未找到 BarentsWatch client id/client secret,请先配置采集器凭证。",
"endpoint": config.endpoint,
"credential_source": config.credential_source,
"settings_tab": "collector_credentials",
}
try:
async with httpx.AsyncClient(timeout=20.0) as client:
token = await fetch_barentswatch_access_token(client, config)
if not token:
return {
"success": False,
"stage": "token",
"message": "BarentsWatch token 响应中没有 access_token,请检查凭证。",
"endpoint": config.endpoint,
"credential_source": config.credential_source,
"settings_tab": "collector_credentials",
}
async with client.stream(
"GET",
config.endpoint,
headers={"Authorization": f"Bearer {token}"},
) as response:
response.raise_for_status()
return {
"success": True,
"stage": "endpoint",
"message": "BarentsWatch AIS token 和数据接口均可连通。",
"endpoint": config.endpoint,
"credential_source": config.credential_source,
"endpoint_source": config.endpoint_source,
}
except httpx.HTTPStatusError as exc:
status_code = exc.response.status_code
stage = "token" if str(exc.request.url) == BARENTSWATCH_TOKEN_URL else "endpoint"
return {
"success": False,
"stage": stage,
"message": f"BarentsWatch {stage} 请求返回 HTTP {status_code},请检查凭证或接口地址。",
"endpoint": config.endpoint,
"credential_source": config.credential_source,
"settings_tab": "collector_credentials",
}
except httpx.HTTPError as exc:
return {
"success": False,
"stage": "network",
"message": f"BarentsWatch 链路检查失败:{exc.__class__.__name__}",
"endpoint": config.endpoint,
"credential_source": config.credential_source,
"settings_tab": "collector_credentials",
}
+259
View File
@@ -0,0 +1,259 @@
from __future__ import annotations
from collections import Counter
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.v1.bgp import BGP_SOURCES
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.schemas.ai import SituationalAnalysisRequest
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.bgp_enrichment import lookup_prefix_geography
def _format_counter(counter: dict[str, int], empty_text: str = "无") -> str:
if not counter:
return empty_text
return ",".join(f"{key} {value}" for key, value in counter.items())
def _severity_rank(value: str | None) -> int:
order = {
"critical": 0,
"high": 1,
"medium": 2,
"low": 3,
"info": 4,
}
return order.get((value or "").lower(), 99)
def _normalize_geo_key(country: str | None, city: str | None) -> str:
if city and country:
return f"{city}, {country}"
return city or country or "未知区域"
def _top_counter_items(counter: Counter[str], limit: int = 5) -> dict[str, int]:
return {name: count for name, count in counter.most_common(limit) if name}
def _collect_incident_regions(incidents: list[BGPIncident]) -> Counter[str]:
counter: Counter[str] = Counter()
for item in incidents:
for region in item.affected_regions or []:
if not isinstance(region, dict):
continue
counter[_normalize_geo_key(region.get("country"), region.get("city"))] += 1
return counter
def _collect_collector_regions(collectors: list[dict[str, Any]]) -> Counter[str]:
counter: Counter[str] = Counter()
for item in collectors:
counter[_normalize_geo_key(item.get("country"), item.get("city"))] += int(item.get("recent_24h_observation_count") or 0)
return counter
def _format_geo_evidence(prefix_geographies: dict[str, dict[str, Any]], limit: int = 6) -> str:
if not prefix_geographies:
return "没有命中 prefix geography 证据。"
rows = []
for prefix, item in list(prefix_geographies.items())[:limit]:
region = _normalize_geo_key(item.get("country"), item.get("city"))
source = item.get("source") or item.get("geography_mode") or "unknown"
as_hint = item.get("asn")
as_name = item.get("as_name")
as_text = ""
if as_hint:
as_text = f" / ASN AS{as_hint}"
if as_name:
as_text += f" ({as_name})"
rows.append(f"{prefix} -> {region} / 来源 {source}{as_text}")
return ";".join(rows)
async def build_bgp_brief_request(
db: AsyncSession,
*,
incident_limit: int = 5,
anomaly_limit: int = 6,
collector_limit: int = 5,
) -> tuple[SituationalAnalysisRequest, list[str], dict[str, int | str | dict[str, int]]]:
incidents_result = await db.execute(
select(BGPIncident)
.order_by(BGPIncident.created_at.desc(), BGPIncident.id.desc())
.limit(max(incident_limit, 1))
)
anomalies_result = await db.execute(
select(BGPAnomaly)
.order_by(BGPAnomaly.created_at.desc(), BGPAnomaly.id.desc())
.limit(max(anomaly_limit, 1))
)
observations_result = await db.execute(
select(BGPObservation).where(BGPObservation.source.in_(BGP_SOURCES))
)
incident_count_result = await db.execute(select(func.count(BGPIncident.id)))
anomaly_count_result = await db.execute(select(func.count(BGPAnomaly.id)))
incidents = incidents_result.scalars().all()
anomalies = anomalies_result.scalars().all()
observations = observations_result.scalars().all()
collectors = await build_bgp_collector_coverage(db, source_filter=BGP_SOURCES)
total_incidents = incident_count_result.scalar() or 0
total_anomalies = anomaly_count_result.scalar() or 0
total_observations = len(observations)
active_collectors = [item for item in collectors if item["observation_count"] > 0]
incident_status_counts = Counter((item.status or "unknown") for item in incidents)
incident_severity_counts = Counter((item.severity or "unknown") for item in incidents)
incident_type_counts = Counter((item.incident_type or "unknown") for item in incidents)
anomaly_type_counts = Counter((item.anomaly_type or "unknown") for item in anomalies)
event_type_counts = Counter((item.event_type or "unknown") for item in observations)
incident_region_counts = _collect_incident_regions(incidents)
top_collectors = sorted(
active_collectors,
key=lambda item: (
-int(item["recent_24h_observation_count"]),
-int(item["observation_count"]),
str(item["collector"]),
),
)[: max(collector_limit, 1)]
collector_region_counts = _collect_collector_regions(top_collectors)
prefix_candidates = sorted(
{
prefix
for item in incidents
for prefix in (item.affected_prefixes or [])
if prefix
}
| {item.prefix for item in anomalies if item.prefix}
)
prefix_geographies = await lookup_prefix_geography(db, prefix_candidates) if prefix_candidates else {}
geography_region_counts = Counter(
_normalize_geo_key(item.get("country"), item.get("city"))
for item in prefix_geographies.values()
if item.get("country") or item.get("city")
)
hotspot_region_counts = geography_region_counts + incident_region_counts
collector_bias_regions = [
region
for region, count in collector_region_counts.most_common(3)
if count > hotspot_region_counts.get(region, 0)
]
observations_lines: list[str] = [
f"当前共有 {total_incidents} 起 BGP incidents、{total_anomalies} 条 anomalies、{total_observations} 条原始观测事件。",
f"活跃观测站 {len(active_collectors)} 个;近 24 小时事件数合计 {sum(int(item['recent_24h_observation_count']) for item in active_collectors)}。",
f"最近 incidents 严重度分布:{_format_counter(dict(sorted(incident_severity_counts.items(), key=lambda item: _severity_rank(item[0]))))}。",
f"最近 incidents 状态分布:{_format_counter(dict(incident_status_counts))}。",
f"最近 incidents 类型分布:{_format_counter(dict(incident_type_counts.most_common(5)))}。",
f"最近 anomalies 类型分布:{_format_counter(dict(anomaly_type_counts.most_common(6)))}。",
f"观测事件类型分布:{_format_counter(dict(event_type_counts.most_common(6)))}。",
]
if hotspot_region_counts:
observations_lines.append(
"区域热点事实层:"
+ _format_counter(_top_counter_items(hotspot_region_counts, limit=5), empty_text="无明显区域聚集")
+ "。"
)
if prefix_geographies:
observations_lines.append("Prefix geography 证据:" + _format_geo_evidence(prefix_geographies))
if collector_bias_regions:
observations_lines.append(
"观测偏差提示:重点观测站最近 24h 活跃度更集中在 "
+ "、".join(collector_bias_regions)
+ ",这些区域的事件升温结论需要结合 prefix geography 与 affected regions 交叉验证。"
)
elif top_collectors:
observations_lines.append(
"观测偏差提示:当前未发现明显高于区域热点事实层的单一观测站集中区域,但仍需区分 collector coverage 与真实区域风险。"
)
if incidents:
observations_lines.append(
"最近 incident 摘要:" + ";".join(
[
f"{item.incident_type} / {item.severity} / {item.status}"
f" / 前缀 {', '.join(item.affected_prefixes[:2]) if item.affected_prefixes else '-'}"
f" / 观测站 {len(item.affected_collectors or [])} 个"
for item in incidents
]
)
)
if anomalies:
observations_lines.append(
"最近 anomaly 摘要:" + ";".join(
[
f"{item.anomaly_type} / {item.severity}"
f" / 前缀 {item.prefix or '-'}"
f" / ASN {item.new_origin_asn or item.origin_asn or '-'}"
for item in anomalies
]
)
)
if top_collectors:
observations_lines.append(
"重点观测站:" + ";".join(
[
f"{item['collector']} ({', '.join([part for part in [item.get('city'), item.get('country')] if part]) or '未知位置'})"
f" / 近24h {item['recent_24h_observation_count']} 条"
f" / 前缀 {item['prefix_count']} 个"
for item in top_collectors
]
)
)
context = {
"source": "bgp-overview",
"incident_total": total_incidents,
"anomaly_total": total_anomalies,
"observation_total": total_observations,
"active_collectors": len(active_collectors),
"top_incident_types": dict(incident_type_counts.most_common(5)),
"top_anomaly_types": dict(anomaly_type_counts.most_common(6)),
"top_event_types": dict(event_type_counts.most_common(6)),
"region_hotspots": _top_counter_items(hotspot_region_counts, limit=6),
"incident_regions": _top_counter_items(incident_region_counts, limit=6),
"collector_bias_regions": collector_bias_regions,
"prefix_geography_sources": dict(
Counter(str(item.get("source") or "unknown") for item in prefix_geographies.values()).most_common(5)
),
"prefix_geography_sample": {
prefix: {
"country": item.get("country"),
"city": item.get("city"),
"source": item.get("source"),
"asn": item.get("asn"),
"as_name": item.get("as_name"),
}
for prefix, item in list(prefix_geographies.items())[:8]
},
}
return SituationalAnalysisRequest(
title="BGP 态势 AI 简报",
objective="基于当前 BGP incidents、anomalies、原始观测事件、观测站覆盖与 prefix geography 证据,生成一份面向操作员的简明态势简报,突出区域热点、观测偏差、当前风险、证据和优先动作。",
observations=observations_lines,
constraints=[
"明确区分事实、推断与建议。",
"优先指出需要立即关注的高严重度 incident 或异常模式。",
"需要单独指出哪些区域结论来自 prefix geography / affected regions,哪些可能受 collector coverage 偏差影响。",
"结论应服务值班排障,不要写成泛泛的模型演示文案。",
"如果证据不足,要明确指出缺失数据。",
],
context=context,
), observations_lines, context
+160
View File
@@ -0,0 +1,160 @@
from __future__ import annotations
import json
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from uuid import uuid4
from app.core.config import ROOT_DIR
from app.schemas.ai import BGPBriefRecordResponse, BGPBriefRecordSummary, SituationalAnalysisResponse
_BRIEF_STORAGE_DIR = ROOT_DIR / "data" / "ai" / "bgp-briefs"
_METADATA_PREFIX = "<!-- planet-bgp-brief-meta "
_METADATA_SUFFIX = " -->"
_BRIEF_TITLE = "BGP AI 简报"
@dataclass(slots=True)
class _StoredBrief:
id: str
title: str
provider: str
model: str
request_id: str | None
generated_at: str
content_markdown: str
facts: list[str]
context: dict[str, Any]
path: Path
def _ensure_storage_dir() -> Path:
_BRIEF_STORAGE_DIR.mkdir(parents=True, exist_ok=True)
return _BRIEF_STORAGE_DIR
def _build_metadata_line(metadata: dict[str, Any]) -> str:
return f"{_METADATA_PREFIX}{json.dumps(metadata, ensure_ascii=False)}{_METADATA_SUFFIX}"
def _parse_brief_file(path: Path) -> _StoredBrief | None:
try:
raw_text = path.read_text(encoding="utf-8")
except OSError:
return None
first_line, separator, remainder = raw_text.partition("\n")
if not separator or not first_line.startswith(_METADATA_PREFIX) or not first_line.endswith(_METADATA_SUFFIX):
return None
metadata_payload = first_line[len(_METADATA_PREFIX) : -len(_METADATA_SUFFIX)]
try:
metadata = json.loads(metadata_payload)
except json.JSONDecodeError:
return None
return _StoredBrief(
id=str(metadata.get("id") or path.stem),
title=str(metadata.get("title") or _BRIEF_TITLE),
provider=str(metadata.get("provider") or "-"),
model=str(metadata.get("model") or "-"),
request_id=metadata.get("request_id"),
generated_at=str(metadata.get("generated_at") or datetime.fromtimestamp(path.stat().st_mtime, UTC).isoformat()),
content_markdown=remainder.lstrip("\n"),
facts=list(metadata.get("facts") or []),
context=dict(metadata.get("context") or {}),
path=path,
)
def list_bgp_brief_records(limit: int = 50) -> list[BGPBriefRecordSummary]:
storage_dir = _ensure_storage_dir()
records: list[_StoredBrief] = []
for path in storage_dir.glob("*.md"):
parsed = _parse_brief_file(path)
if parsed is not None:
records.append(parsed)
records.sort(key=lambda item: item.generated_at, reverse=True)
return [
BGPBriefRecordSummary(
id=item.id,
title=item.title,
provider=item.provider,
model=item.model,
request_id=item.request_id,
generated_at=item.generated_at,
)
for item in records[: max(limit, 1)]
]
def get_bgp_brief_record(brief_id: str) -> BGPBriefRecordResponse | None:
path = _ensure_storage_dir() / f"{brief_id}.md"
parsed = _parse_brief_file(path)
if parsed is None:
return None
return BGPBriefRecordResponse(
id=parsed.id,
title=parsed.title,
provider=parsed.provider,
model=parsed.model,
request_id=parsed.request_id,
generated_at=parsed.generated_at,
content_markdown=parsed.content_markdown,
facts=parsed.facts,
context=parsed.context,
)
def get_latest_bgp_brief_record() -> BGPBriefRecordResponse | None:
summaries = list_bgp_brief_records(limit=1)
if not summaries:
return None
return get_bgp_brief_record(summaries[0].id)
def save_bgp_brief_record(
analysis: SituationalAnalysisResponse,
*,
request_id: str | None,
facts: list[str] | None = None,
context: dict[str, Any] | None = None,
generated_at: datetime | None = None,
) -> BGPBriefRecordResponse:
created_at = generated_at or datetime.now(UTC)
brief_id = f"{created_at.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}"
path = _ensure_storage_dir() / f"{brief_id}.md"
metadata = {
"id": brief_id,
"title": _BRIEF_TITLE,
"provider": analysis.provider,
"model": analysis.model,
"request_id": request_id,
"generated_at": created_at.isoformat(),
"facts": facts or [],
"context": context or {},
}
markdown_text = f"{_build_metadata_line(metadata)}\n\n{analysis.content.rstrip()}\n"
path.write_text(markdown_text, encoding="utf-8")
return BGPBriefRecordResponse(
id=brief_id,
title=_BRIEF_TITLE,
provider=analysis.provider,
model=analysis.model,
request_id=request_id,
generated_at=created_at.isoformat(),
content_markdown=analysis.content,
facts=facts or [],
context=context or {},
)
@@ -0,0 +1,324 @@
"""BGP route-collector location resolver.
Collector positions are stored in the ``bgp_collector_locations`` database
table. The old JSON registry is now only a seed payload used during database
initialization, not a runtime resolver or candidate source.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Iterator
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.bgp_collector_location import BGPCollectorLocation
from app.services.location import (
LocationCandidate,
LocationPipeline,
LocationQuery,
NominatimResolver,
ResolutionResult,
ResolverOutput,
SourceCoordinatesResolver,
build_default_nominatim_geocoder,
coerce_str,
normalize_text,
)
SEED_PATH = (
Path(__file__).resolve().parents[1]
/ "data"
/ "seeds"
/ "ripe_ris_collector_locations_seed.json"
)
# ── Geocoder (kept at module level for monkeypatching + cache_clear) ──
_geocode_online = build_default_nominatim_geocoder()
# ── In-process compatibility cache ──────────────────────────────────
RIPE_RIS_COLLECTOR_COORDS: dict[str, dict[str, Any]] = {}
def _collector_record_to_dict(record: BGPCollectorLocation) -> dict[str, Any]:
return record.to_location_dict()
def set_bgp_collector_location_cache(
locations: dict[str, dict[str, Any]],
) -> None:
"""Replace the legacy compatibility cache in-place."""
RIPE_RIS_COLLECTOR_COORDS.clear()
RIPE_RIS_COLLECTOR_COORDS.update(
{coerce_str(key): dict(value) for key, value in locations.items()}
)
async def refresh_bgp_collector_location_cache(
session: AsyncSession,
) -> dict[str, dict[str, Any]]:
result = await session.execute(select(BGPCollectorLocation))
records = result.scalars().all()
cache = {
record.collector_id: _collector_record_to_dict(record)
for record in records
if record.collector_id
}
set_bgp_collector_location_cache(cache)
return cache
def _load_seed_payload() -> dict[str, Any]:
with SEED_PATH.open("r", encoding="utf-8") as handle:
return json.load(handle)
def _seed_entry_to_record_kwargs(entry: dict[str, Any], collector_id: str) -> dict[str, Any]:
return {
"collector_id": collector_id,
"operator": entry.get("operator") or "RIPE NCC",
"site": entry.get("site"),
"city": entry.get("city"),
"country": entry.get("country"),
"latitude": entry.get("latitude"),
"longitude": entry.get("longitude"),
"precision": entry.get("precision") or "city",
"confidence": entry.get("confidence"),
"source": "legacy_seed",
"source_url": None,
"source_note": entry.get("source_note")
or "Seeded from legacy RIPE RIS collector coordinates",
"raw_payload": entry,
"needs_confirmation": True,
"verification_status": "unverified",
"verified_at": None,
}
async def seed_default_bgp_collector_locations(session: AsyncSession) -> None:
"""Seed default RIPE RIS collector locations without overwriting users."""
payload = _load_seed_payload()
for entry in payload.get("locations", []):
aliases = entry.get("aliases") or []
collector_ids = [
coerce_str(alias)
for alias in aliases
if coerce_str(alias).startswith("rrc")
]
if not collector_ids:
continue
collector_id = collector_ids[0]
existing = await session.scalar(
select(BGPCollectorLocation).where(
BGPCollectorLocation.collector_id == collector_id
)
)
if existing:
continue
session.add(
BGPCollectorLocation(
**_seed_entry_to_record_kwargs(entry, collector_id)
)
)
await session.commit()
await refresh_bgp_collector_location_cache(session)
def get_bgp_collector_location_dict(collector_name: str) -> dict[str, Any]:
"""Return the current cached collector location dict, or ``{}`` if unknown."""
return dict(RIPE_RIS_COLLECTOR_COORDS.get(coerce_str(collector_name), {}))
def iter_known_collector_names() -> Iterator[str]:
"""Yield every collector technical name (rrcXX) known in the cache."""
return iter(sorted(RIPE_RIS_COLLECTOR_COORDS.keys()))
# ── Pipeline construction ──────────────────────────────────────────
class StoredCollectorLocationResolver:
"""Resolve a collector through the DB-backed compatibility cache."""
name = "stored_collector_location"
def resolve(self, query: LocationQuery) -> ResolverOutput:
collector = coerce_str(query.name)
if not collector:
for alias in query.aliases:
collector = coerce_str(alias)
if collector:
break
if not collector:
return ResolverOutput()
location = get_bgp_collector_location_dict(collector)
if not location:
return ResolverOutput()
latitude = location.get("latitude")
longitude = location.get("longitude")
if latitude in (None, 0.0) or longitude in (None, 0.0):
return ResolverOutput()
return ResolverOutput(
candidates=(
LocationCandidate(
latitude=float(latitude),
longitude=float(longitude),
display_name=location.get("matched_location_name") or collector,
precision=location.get("precision") or "city",
confidence=float(location.get("confidence") or 0.85),
query=f"stored_collector_location::{collector}",
source=location.get("source") or self.name,
source_note=location.get("source_note"),
matched_fields=("collector",),
needs_confirmation=bool(location.get("needs_confirmation")),
city=location.get("city"),
region=None,
country=location.get("country"),
matched_location_name=(
location.get("matched_location_name") or collector
),
location_verified_at=location.get("verified_at"),
suggested_registry_entry=None,
),
)
)
def _bgp_collector_query_plan(
query: LocationQuery,
) -> list[tuple[str, tuple[str, ...]]]:
"""Build the Nominatim query plan for a BGP collector."""
extra = query.extra or {}
site = str(extra.get("site") or "")
operator = str(extra.get("operator") or "")
city = query.city or ""
country = query.country or ""
plan: list[tuple[str, tuple[str, ...]]] = []
def add(parts: list[tuple[str, str]]) -> None:
non_empty = [(field, value) for field, value in parts if value]
if not non_empty:
return
seen: set[str] = set()
cleaned: list[str] = []
fields: list[str] = []
for field, value in non_empty:
key = normalize_text(value)
if not key or key in seen:
continue
seen.add(key)
cleaned.append(value)
fields.append(field)
if not cleaned:
return
composed = ", ".join(cleaned)
if not any(composed == existing for existing, _ in plan):
plan.append((composed, tuple(fields)))
add([("site", site), ("city", city), ("country", country)])
add([("site", site), ("country", country)])
add([("operator", operator), ("city", city), ("country", country)])
add([("city", city), ("country", country)])
return plan
BGP_COLLECTOR_PIPELINE = LocationPipeline(
[
SourceCoordinatesResolver(),
StoredCollectorLocationResolver(),
],
failure_reason=(
"Could not resolve BGP collector to renderable coordinates from"
" source coordinates or stored collector location."
),
)
BGP_COLLECTOR_COLLECTION_PIPELINE = LocationPipeline(
[
SourceCoordinatesResolver(),
NominatimResolver(
query_plan_builder=_bgp_collector_query_plan,
# Late-binding so tests can monkeypatch ``_geocode_online``.
geocoder=lambda q: _geocode_online(q),
),
],
failure_reason=(
"Could not resolve BGP collector to renderable coordinates from"
" source coordinates or online geocoding."
),
)
# ── Public API ─────────────────────────────────────────────────────
def resolve_bgp_collector_location(
collector_name: str,
*,
city: str | None = None,
country: str | None = None,
site: str | None = None,
operator: str | None = None,
) -> ResolutionResult:
"""Resolve a BGP collector to its best-known stored location."""
stored = get_bgp_collector_location_dict(collector_name)
name = coerce_str(collector_name) or None
query = LocationQuery(
name=name,
aliases=tuple(filter(None, (collector_name,))),
city=coerce_str(city or stored.get("city")) or None,
country=coerce_str(country or stored.get("country")) or None,
extra={
"site": coerce_str(site or stored.get("site")),
"operator": coerce_str(operator or stored.get("operator")) or "RIPE NCC",
},
)
return BGP_COLLECTOR_PIPELINE.resolve_best(query)
def collect_bgp_collector_location_candidates(
*,
collector: str | None = None,
city: str | None = None,
country: str | None = None,
site: str | None = None,
operator: str | None = None,
) -> tuple[list[LocationCandidate], list[str]]:
query = build_bgp_collector_location_query(
collector=collector,
city=city,
country=country,
site=site,
operator=operator,
)
return BGP_COLLECTOR_COLLECTION_PIPELINE.collect_candidates(query)
def build_bgp_collector_location_query(
*,
collector: str | None = None,
city: str | None = None,
country: str | None = None,
site: str | None = None,
operator: str | None = None,
) -> LocationQuery:
stored = get_bgp_collector_location_dict(collector or "")
name = coerce_str(collector) or None
return LocationQuery(
name=name,
aliases=tuple(filter(None, (collector,))),
city=coerce_str(city or stored.get("city")) or None,
country=coerce_str(country or stored.get("country")) or None,
extra={
"site": coerce_str(site or stored.get("site")),
"operator": coerce_str(operator or stored.get("operator")) or "RIPE NCC",
"collector": coerce_str(collector),
},
)
+161 -126
View File
@@ -6,7 +6,7 @@ from collections import defaultdict
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import select
from sqlalchemy import case, distinct, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.time import to_iso8601_utc
@@ -14,6 +14,16 @@ from app.models.bgp_observation import BGPObservation
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
def _collector_base_filters(source_filter: tuple[str, ...] | None) -> list[Any]:
filters: list[Any] = [
BGPObservation.collector.isnot(None),
func.length(func.btrim(BGPObservation.collector)) > 0,
]
if source_filter:
filters.append(BGPObservation.source.in_(source_filter))
return filters
async def build_bgp_collector_coverage(
db: AsyncSession,
*,
@@ -24,88 +34,148 @@ async def build_bgp_collector_coverage(
recent_24h_threshold = now - timedelta(hours=24)
recent_7d_threshold = now - timedelta(days=7)
stmt = select(BGPObservation).order_by(BGPObservation.observed_at.desc(), BGPObservation.id.desc())
if source_filter:
stmt = stmt.where(BGPObservation.source.in_(source_filter))
filters = _collector_base_filters(source_filter)
country_expr = func.nullif(BGPObservation.collector_geo["country"].as_string(), "")
city_expr = func.nullif(BGPObservation.collector_geo["city"].as_string(), "")
result = await db.execute(stmt)
records = list(result.scalars().all())
aggregate_stmt = (
select(
BGPObservation.collector.label("collector"),
func.count(BGPObservation.id).label("observation_count"),
func.count(distinct(BGPObservation.prefix)).label("prefix_count"),
func.count(distinct(BGPObservation.origin_asn)).label("origin_asn_count"),
func.count(distinct(BGPObservation.peer_asn)).label("peer_asn_count"),
func.sum(case((BGPObservation.observed_at >= recent_15m_threshold, 1), else_=0)).label("recent_15m_observation_count"),
func.sum(case((BGPObservation.observed_at >= recent_24h_threshold, 1), else_=0)).label("recent_24h_observation_count"),
func.sum(case((BGPObservation.observed_at >= recent_7d_threshold, 1), else_=0)).label("recent_7d_observation_count"),
func.count(distinct(case((BGPObservation.observed_at >= recent_15m_threshold, BGPObservation.prefix), else_=None))).label("recent_15m_prefix_count"),
func.count(distinct(case((BGPObservation.observed_at >= recent_24h_threshold, BGPObservation.prefix), else_=None))).label("recent_24h_prefix_count"),
func.count(distinct(case((BGPObservation.observed_at >= recent_7d_threshold, BGPObservation.prefix), else_=None))).label("recent_7d_prefix_count"),
func.max(BGPObservation.observed_at).label("latest_observed_at"),
)
.where(*filters)
.group_by(BGPObservation.collector)
)
aggregate_rows = (await db.execute(aggregate_stmt)).all()
latest_subquery = (
select(
BGPObservation.collector.label("collector"),
BGPObservation.event_type.label("latest_event_type"),
country_expr.label("country"),
city_expr.label("city"),
func.row_number()
.over(
partition_by=BGPObservation.collector,
order_by=(BGPObservation.observed_at.desc(), BGPObservation.id.desc()),
)
.label("rn"),
)
.where(*filters)
.subquery()
)
latest_rows = (
await db.execute(
select(
latest_subquery.c.collector,
latest_subquery.c.latest_event_type,
latest_subquery.c.country,
latest_subquery.c.city,
).where(latest_subquery.c.rn == 1)
)
).all()
event_counts_subquery = (
select(
BGPObservation.collector.label("collector"),
BGPObservation.event_type.label("event_type"),
func.count(BGPObservation.id).label("count"),
func.row_number()
.over(
partition_by=BGPObservation.collector,
order_by=(func.count(BGPObservation.id).desc(), BGPObservation.event_type.asc()),
)
.label("rn"),
)
.where(*filters)
.group_by(BGPObservation.collector, BGPObservation.event_type)
.subquery()
)
top_event_rows = (
await db.execute(
select(
event_counts_subquery.c.collector,
event_counts_subquery.c.event_type,
event_counts_subquery.c.count,
).where(event_counts_subquery.c.rn <= 3)
)
).all()
scope_rows = (
await db.execute(
select(
BGPObservation.collector.label("collector"),
country_expr.label("country"),
city_expr.label("city"),
)
.where(*filters)
.distinct()
)
).all()
latest_by_collector = {
row.collector: {
"latest_event_type": row.latest_event_type,
"country": row.country,
"city": row.city,
}
for row in latest_rows
}
scope_by_collector: dict[str, dict[str, set[str]]] = defaultdict(lambda: {"countries": set(), "cities": set()})
for row in scope_rows:
if row.country:
scope_by_collector[row.collector]["countries"].add(row.country)
if row.city:
scope_by_collector[row.collector]["cities"].add(row.city)
top_events_by_collector: dict[str, list[dict[str, Any]]] = defaultdict(list)
for row in top_event_rows:
top_events_by_collector[row.collector].append(
{"event_type": row.event_type, "count": row.count}
)
by_collector: dict[str, dict[str, Any]] = {}
for record in records:
collector = str(record.collector or "").strip()
if not collector:
continue
for row in aggregate_rows:
collector = row.collector
latest = latest_by_collector.get(collector, {})
fallback_location = RIPE_RIS_COLLECTOR_COORDS.get(collector, {})
scope = scope_by_collector.get(collector, {"countries": set(), "cities": set()})
coverage = by_collector.get(collector)
if coverage is None:
location = record.collector_geo or RIPE_RIS_COLLECTOR_COORDS.get(collector, {})
coverage = {
"collector": collector,
"city": location.get("city"),
"country": location.get("country"),
"latitude": location.get("latitude"),
"longitude": location.get("longitude"),
"observation_count": 0,
"prefixes": set(),
"origin_asns": set(),
"peer_asns": set(),
"event_types": defaultdict(int),
"countries": set(),
"cities": set(),
"recent_15m_observation_count": 0,
"recent_24h_observation_count": 0,
"recent_7d_observation_count": 0,
"recent_15m_prefixes": set(),
"recent_24h_prefixes": set(),
"recent_7d_prefixes": set(),
"latest_observed_at": None,
"latest_event_type": None,
}
by_collector[collector] = coverage
coverage["observation_count"] += 1
if record.prefix:
coverage["prefixes"].add(record.prefix)
if record.origin_asn is not None:
coverage["origin_asns"].add(record.origin_asn)
if record.peer_asn is not None:
coverage["peer_asns"].add(record.peer_asn)
if record.event_type:
coverage["event_types"][record.event_type] += 1
observed_at = record.observed_at
if observed_at is not None:
aware_observed_at = (
observed_at.astimezone(UTC)
if observed_at.tzinfo
else observed_at.replace(tzinfo=UTC)
)
if aware_observed_at >= recent_15m_threshold:
coverage["recent_15m_observation_count"] += 1
if record.prefix:
coverage["recent_15m_prefixes"].add(record.prefix)
if aware_observed_at >= recent_24h_threshold:
coverage["recent_24h_observation_count"] += 1
if record.prefix:
coverage["recent_24h_prefixes"].add(record.prefix)
if aware_observed_at >= recent_7d_threshold:
coverage["recent_7d_observation_count"] += 1
if record.prefix:
coverage["recent_7d_prefixes"].add(record.prefix)
geo = record.collector_geo or {}
if geo.get("country"):
coverage["countries"].add(geo["country"])
if geo.get("city"):
coverage["cities"].add(geo["city"])
current_latest = coverage["latest_observed_at"]
if current_latest is None or (
record.observed_at is not None and record.observed_at > current_latest
):
coverage["latest_observed_at"] = record.observed_at
coverage["latest_event_type"] = record.event_type
by_collector[collector] = {
"collector": collector,
"city": latest.get("city") or fallback_location.get("city"),
"country": latest.get("country") or fallback_location.get("country"),
"latitude": fallback_location.get("latitude"),
"longitude": fallback_location.get("longitude"),
"observation_count": row.observation_count or 0,
"prefix_count": row.prefix_count or 0,
"origin_asn_count": row.origin_asn_count or 0,
"peer_asn_count": row.peer_asn_count or 0,
"recent_15m_observation_count": row.recent_15m_observation_count or 0,
"recent_24h_observation_count": row.recent_24h_observation_count or 0,
"recent_7d_observation_count": row.recent_7d_observation_count or 0,
"recent_15m_prefix_count": row.recent_15m_prefix_count or 0,
"recent_24h_prefix_count": row.recent_24h_prefix_count or 0,
"recent_7d_prefix_count": row.recent_7d_prefix_count or 0,
"top_event_types": top_events_by_collector.get(collector, []),
"latest_observed_at": to_iso8601_utc(row.latest_observed_at),
"latest_event_type": latest.get("latest_event_type"),
"baseline_scope": {
"countries": sorted(scope["countries"]),
"cities": sorted(scope["cities"]),
},
}
for collector, location in RIPE_RIS_COLLECTOR_COORDS.items():
if collector in by_collector:
@@ -117,57 +187,22 @@ async def build_bgp_collector_coverage(
"latitude": location.get("latitude"),
"longitude": location.get("longitude"),
"observation_count": 0,
"prefixes": set(),
"origin_asns": set(),
"peer_asns": set(),
"event_types": defaultdict(int),
"countries": {location.get("country")} if location.get("country") else set(),
"cities": {location.get("city")} if location.get("city") else set(),
"prefix_count": 0,
"origin_asn_count": 0,
"peer_asn_count": 0,
"recent_15m_observation_count": 0,
"recent_24h_observation_count": 0,
"recent_7d_observation_count": 0,
"recent_15m_prefixes": set(),
"recent_24h_prefixes": set(),
"recent_7d_prefixes": set(),
"recent_15m_prefix_count": 0,
"recent_24h_prefix_count": 0,
"recent_7d_prefix_count": 0,
"top_event_types": [],
"latest_observed_at": None,
"latest_event_type": None,
"baseline_scope": {
"countries": [location["country"]] if location.get("country") else [],
"cities": [location["city"]] if location.get("city") else [],
},
}
results: list[dict[str, Any]] = []
for collector in sorted(by_collector.keys()):
item = by_collector[collector]
top_event_types = sorted(
item["event_types"].items(),
key=lambda pair: (-pair[1], pair[0]),
)
results.append(
{
"collector": item["collector"],
"city": item["city"],
"country": item["country"],
"latitude": item["latitude"],
"longitude": item["longitude"],
"observation_count": item["observation_count"],
"prefix_count": len(item["prefixes"]),
"origin_asn_count": len(item["origin_asns"]),
"peer_asn_count": len(item["peer_asns"]),
"recent_15m_observation_count": item["recent_15m_observation_count"],
"recent_24h_observation_count": item["recent_24h_observation_count"],
"recent_7d_observation_count": item["recent_7d_observation_count"],
"recent_15m_prefix_count": len(item["recent_15m_prefixes"]),
"recent_24h_prefix_count": len(item["recent_24h_prefixes"]),
"recent_7d_prefix_count": len(item["recent_7d_prefixes"]),
"top_event_types": [
{"event_type": event_type, "count": count}
for event_type, count in top_event_types[:3]
],
"latest_observed_at": to_iso8601_utc(item["latest_observed_at"]),
"latest_event_type": item["latest_event_type"],
"baseline_scope": {
"countries": sorted(country for country in item["countries"] if country),
"cities": sorted(city for city in item["cities"] if city),
},
}
)
return results
return [by_collector[collector] for collector in sorted(by_collector.keys())]
+37 -13
View File
@@ -7,7 +7,7 @@ from collections import defaultdict
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select, text
from sqlalchemy import Integer, cast, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.countries import get_country_centroid, normalize_country
@@ -231,6 +231,13 @@ async def _lookup_prefix_geography(
return results
async def lookup_prefix_geography(
db: AsyncSession,
prefix_values: list[str],
) -> dict[str, dict[str, Any]]:
return await _lookup_prefix_geography(db, prefix_values)
async def enrich_bgp_events_for_batch(
db: AsyncSession,
*,
@@ -261,29 +268,40 @@ async def enrich_bgp_events_for_batch(
historical_prefix_baseline: dict[str, dict[str, Any]] = {}
if prefix_values:
previous_result = await db.execute(
select(BGPObservation).where(
select(
BGPObservation.prefix,
BGPObservation.origin_asn,
BGPObservation.collector,
BGPObservation.collector_geo,
).where(
BGPObservation.source == source,
BGPObservation.prefix.in_(prefix_values),
)
)
by_prefix: defaultdict[str, list[BGPObservation]] = defaultdict(list)
for observation in previous_result.scalars().all():
if observation.prefix:
by_prefix[observation.prefix].append(observation)
by_prefix: defaultdict[str, list[dict[str, Any]]] = defaultdict(list)
for prefix, origin_asn, collector, collector_geo in previous_result.all():
if prefix:
by_prefix[str(prefix)].append(
{
"origin_asn": origin_asn,
"collector": collector,
"collector_geo": collector_geo or {},
}
)
for prefix, observations in by_prefix.items():
unique_origins = sorted(
{
observation.origin_asn
observation["origin_asn"]
for observation in observations
if observation.origin_asn is not None
if observation["origin_asn"] is not None
}
)
unique_collectors = sorted(
{
observation.collector
observation["collector"]
for observation in observations
if observation.collector
if observation["collector"]
}
)
historical_prefix_baseline[prefix] = {
@@ -292,9 +310,9 @@ async def enrich_bgp_events_for_batch(
"historical_observation_count": len(observations),
"historical_regions": _compact_locations(
[
observation.collector_geo or {}
observation["collector_geo"] or {}
for observation in observations
if observation.collector_geo
if observation["collector_geo"]
]
),
}
@@ -303,7 +321,13 @@ async def enrich_bgp_events_for_batch(
prefix_geographies = await _lookup_prefix_geography(db, prefix_values) if prefix_values else {}
if origin_asns:
peeringdb_result = await db.execute(
select(CollectedData).where(CollectedData.source == "peeringdb_network")
select(CollectedData)
.where(CollectedData.source == "peeringdb_network")
.where(CollectedData.is_current.is_(True))
.where(
cast(CollectedData.extra_data["asn"].as_string(), Integer).in_(origin_asns),
)
.order_by(CollectedData.id.desc())
)
for record in peeringdb_result.scalars().all():
metadata = record.extra_data or {}
+155
View File
@@ -0,0 +1,155 @@
"""BGP event location resolver.
A BGP event (announcement / withdrawal / RIB entry) is geographically tied to
the route collector that observed it. This module defines the pipeline that
turns an event payload into renderable coordinates.
Current resolver chain:
SourceCoordinates → event payload itself carries lat/lon (rare; some
enriched feeds do).
InheritFromCollector → look up the owning collector via
:func:`resolve_bgp_collector_location`.
Future plug-ins (no consumer changes required, just append to the list):
ASNFacilityResolver — origin/peer ASN → peeringdb facility.
PrefixGeoResolver — prefix → IP range geo lookup (iptoasn / opengeofeed).
"""
from __future__ import annotations
from typing import Any
from app.services.bgp_collector_locations import (
get_bgp_collector_location_dict,
)
from app.services.location import (
InheritFromAnotherEntityResolver,
LocationCandidate,
LocationPipeline,
LocationQuery,
ResolutionResult,
SourceCoordinatesResolver,
coerce_str,
)
def _inherit_from_owning_collector(
query: LocationQuery,
) -> LocationCandidate | None:
"""Look up the event's owning collector by exact name in the DB-backed cache."""
extra = query.extra or {}
collector_name = coerce_str(extra.get("collector"))
if not collector_name:
return None
legacy = get_bgp_collector_location_dict(collector_name)
if not legacy:
return None
latitude = legacy.get("latitude")
longitude = legacy.get("longitude")
if latitude in (None, 0.0) or longitude in (None, 0.0):
return None
return LocationCandidate(
latitude=float(latitude),
longitude=float(longitude),
display_name=legacy.get("matched_location_name") or collector_name,
precision=legacy.get("precision") or "city",
confidence=float(legacy.get("confidence") or 0.85),
query=f"inherit_from_collector::{collector_name}",
source="inherited_from_collector",
source_note=(
f"Inherited from owning collector {collector_name}"
),
matched_fields=("collector",),
needs_confirmation=bool(legacy.get("needs_confirmation")),
city=legacy.get("city"),
region=None,
country=legacy.get("country"),
matched_location_name=legacy.get("matched_location_name"),
location_verified_at=legacy.get("verified_at"),
suggested_registry_entry=None,
)
BGP_EVENT_PIPELINE = LocationPipeline(
[
SourceCoordinatesResolver(),
InheritFromAnotherEntityResolver(
source_lookup=_inherit_from_owning_collector,
name="inherited_from_collector",
),
# Plug new resolvers (peeringdb / ASN facility / prefix-geo) here.
],
failure_reason=(
"Could not resolve BGP event coordinates: no source coords, owning"
" collector unknown, and no fallback resolver matched."
),
)
def resolve_bgp_event_location(
*,
collector: str,
source_latitude: float | None = None,
source_longitude: float | None = None,
site: str | None = None,
operator: str | None = None,
peer_asn: int | None = None,
origin_asn: int | None = None,
prefix: str | None = None,
) -> ResolutionResult:
"""Resolve a BGP event to its renderable coordinates.
The ``peer_asn`` / ``origin_asn`` / ``prefix`` arguments are accepted
today so future resolvers (ASN→facility, prefix→geo) can consume them
without callers needing to change.
"""
query = LocationQuery(
name=collector or None,
aliases=tuple(filter(None, (collector,))),
source_latitude=source_latitude,
source_longitude=source_longitude,
extra={
"collector": collector or "",
"site": coerce_str(site),
"operator": coerce_str(operator),
"peer_asn": peer_asn,
"origin_asn": origin_asn,
"prefix": coerce_str(prefix),
},
)
return BGP_EVENT_PIPELINE.resolve_best(query)
def resolve_bgp_event_geo_dict(
collector: str,
*,
source_latitude: float | None = None,
source_longitude: float | None = None,
) -> dict[str, Any]:
"""Convenience wrapper returning the legacy ``collector_geo`` dict shape.
Preserves ``city``/``country``/``latitude``/``longitude`` keys (consumed
by existing detectors / enrichment / DB serialization) and adds
``precision``/``source``/``needs_confirmation`` for richer downstream use.
"""
result = resolve_bgp_event_location(
collector=collector,
source_latitude=source_latitude,
source_longitude=source_longitude,
)
candidate = result.location
if candidate is None:
return {}
return {
"city": candidate.city,
"country": candidate.country,
"latitude": candidate.latitude,
"longitude": candidate.longitude,
"precision": candidate.precision,
"source": candidate.source,
"needs_confirmation": candidate.needs_confirmation,
"matched_location_name": candidate.matched_location_name,
"confidence": candidate.confidence,
}
+32 -20
View File
@@ -48,14 +48,36 @@ def _collector_regions_from_anomaly(anomaly: BGPAnomaly) -> list[dict]:
return collected
def _dedupe_collected_records(records: list[CollectedData]) -> list[CollectedData]:
latest_by_key: dict[str, CollectedData] = {}
for record in records:
dedupe_key = str(record.source_id or record.entity_key or record.name or record.id)
existing = latest_by_key.get(dedupe_key)
if existing is None or (record.id or 0) > (existing.id or 0):
latest_by_key[dedupe_key] = record
return list(latest_by_key.values())
async def _load_current_infrastructure_records(
db: AsyncSession,
) -> tuple[list[CollectedData], list[CollectedData], list[CollectedData]]:
result = await db.execute(
select(CollectedData)
.where(
CollectedData.source.in_(
(
"arcgis_landing_points",
"arcgis_cable_landing_relation",
"arcgis_cables",
)
)
)
.where(CollectedData.is_current.is_(True))
.order_by(CollectedData.source.asc(), CollectedData.id.desc())
)
grouped_records = {
"arcgis_landing_points": [],
"arcgis_cable_landing_relation": [],
"arcgis_cables": [],
}
for record in result.scalars().all():
grouped_records.setdefault(record.source, []).append(record)
return (
grouped_records["arcgis_landing_points"],
grouped_records["arcgis_cable_landing_relation"],
grouped_records["arcgis_cables"],
)
async def infer_related_infrastructure(
@@ -75,19 +97,9 @@ async def infer_related_infrastructure(
if not valid_regions:
return {"related_cables": [], "related_ixps": []}
landing_result = await db.execute(
select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
landing_records, relation_records, cable_records = await _load_current_infrastructure_records(
db,
)
relation_result = await db.execute(
select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
)
cable_result = await db.execute(
select(CollectedData).where(CollectedData.source == "arcgis_cables")
)
landing_records = _dedupe_collected_records(list(landing_result.scalars().all()))
relation_records = _dedupe_collected_records(list(relation_result.scalars().all()))
cable_records = _dedupe_collected_records(list(cable_result.scalars().all()))
city_to_cable_ids: dict[int, list[int]] = {}
for relation in relation_records:
@@ -35,6 +35,9 @@ from app.services.collectors.bgpstream import BGPStreamBackfillCollector
from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
from app.services.collectors.opengeofeed import OpenGeoFeedPrefixGeoCollector
from app.services.collectors.nro_delegated import NRODelegatedPrefixGeoCollector
from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
from app.services.collectors.aisstream import AISStreamCollector
from app.services.collectors.vessel_ais import VesselAISCollector
collector_registry.register(TOP500Collector())
collector_registry.register(EpochAIGPUCollector())
@@ -61,3 +64,42 @@ collector_registry.register(BGPStreamBackfillCollector())
collector_registry.register(IPtoASNPrefixGeoCollector())
collector_registry.register(OpenGeoFeedPrefixGeoCollector())
collector_registry.register(NRODelegatedPrefixGeoCollector())
collector_registry.register(NewsLiveStreamsCollector())
collector_registry.register(VesselAISCollector())
collector_registry.register(AISStreamCollector())
__all__ = [
"BaseCollector",
"HTTPCollector",
"IntervalCollector",
"collector_registry",
"CollectorRegistry",
"TOP500Collector",
"EpochAIGPUCollector",
"HuggingFaceModelCollector",
"HuggingFaceDatasetCollector",
"HuggingFaceSpacesCollector",
"PeeringDBIXPCollector",
"PeeringDBNetworkCollector",
"PeeringDBFacilityCollector",
"TeleGeographyCableCollector",
"TeleGeographyLandingPointCollector",
"TeleGeographyCableSystemCollector",
"CloudflareRadarDeviceCollector",
"CloudflareRadarTrafficCollector",
"CloudflareRadarTopASCollector",
"ArcGISCableCollector",
"FAOLandingPointCollector",
"ArcGISLandingPointCollector",
"ArcGISCableLandingRelationCollector",
"SpaceTrackTLECollector",
"CelesTrakTLECollector",
"RISLiveCollector",
"BGPStreamBackfillCollector",
"IPtoASNPrefixGeoCollector",
"OpenGeoFeedPrefixGeoCollector",
"NRODelegatedPrefixGeoCollector",
"NewsLiveStreamsCollector",
"VesselAISCollector",
"AISStreamCollector",
]
@@ -0,0 +1,491 @@
"""AISStream WebSocket collector for realtime vessel AIS observations."""
from datetime import UTC, datetime
import asyncio
import json
import os
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.data_sources import get_data_sources_config
from app.core.time import to_iso8601_utc
from app.core.websocket.broadcaster import broadcaster
from app.models.datasource_config import DataSourceConfig
from app.models.task import CollectionTask
from app.services.collectors.base import BaseCollector
from app.services.vessel_ais_aggregation import (
AISSTREAM_DELIVERY_MODE,
AISSTREAM_TRANSPORT,
record_vessel_ais_observation,
update_ais_source_health,
)
from app.services.vessel_types import normalize_vessel_type_name
DEFAULT_AISSTREAM_URL = "wss://stream.aisstream.io/v0/stream"
DEFAULT_BOUNDING_BOXES = [[[-90, -180], [90, 180]]]
DEFAULT_MESSAGE_TYPES = ["PositionReport", "ShipStaticData"]
class AISStreamCollector(BaseCollector):
"""Collect AISStream WebSocket messages into the raw AIS observation layer."""
name = "aisstream_vessels"
priority = "P1"
module = "L4"
frequency_hours = 1
data_type = "vessel_ais"
fail_on_empty = False
async def _load_datasource_config(self) -> DataSourceConfig | None:
if self._db_session is None:
return None
result = await self._db_session.execute(
select(DataSourceConfig)
.where(DataSourceConfig.name == self.name)
.where(DataSourceConfig.is_active.is_(True))
)
return result.scalar_one_or_none()
async def _get_effective_config(self) -> dict[str, Any]:
datasource_config = await self._load_datasource_config()
config = dict(datasource_config.config or {}) if datasource_config else {}
auth_config = dict(datasource_config.auth_config or {}) if datasource_config else {}
endpoint = (
(datasource_config.endpoint if datasource_config else None)
or self._resolved_url
or get_data_sources_config().get_yaml_url(self.name)
or DEFAULT_AISSTREAM_URL
)
api_key = (
auth_config.get("api_key")
or config.get("api_key")
or os.getenv("AISSTREAM_API_KEY")
)
return {
"endpoint": endpoint,
"api_key": api_key,
"bounding_boxes": config.get("bounding_boxes") or DEFAULT_BOUNDING_BOXES,
"message_types": config.get("message_types") or DEFAULT_MESSAGE_TYPES,
"max_messages": int(config.get("max_messages") or 500),
"streaming_enabled": config.get("streaming_enabled", True) is not False,
"streaming_commit_interval": int(config.get("streaming_commit_interval") or 1),
"streaming_max_messages": int(config.get("streaming_max_messages") or 0),
"reconnect_delay_seconds": float(config.get("reconnect_delay_seconds") or 5),
"receive_timeout_seconds": float(config.get("receive_timeout_seconds") or 30),
}
def _build_subscription(self, config: dict[str, Any]) -> dict[str, Any]:
return {
"APIKey": config["api_key"],
"BoundingBoxes": config["bounding_boxes"],
"FilterMessageTypes": config["message_types"],
}
async def fetch(self) -> list[dict[str, Any]]:
config = await self._get_effective_config()
if not config["api_key"]:
raise RuntimeError("AISStream API key is not configured")
try:
import websockets
except ImportError as exc:
raise RuntimeError("Python package 'websockets' is required for AISStream") from exc
subscription = self._build_subscription(config)
messages: list[dict[str, Any]] = []
try:
async with websockets.connect(config["endpoint"]) as websocket:
await websocket.send(json.dumps(subscription))
while len(messages) < config["max_messages"]:
try:
raw_message = await asyncio.wait_for(
websocket.recv(),
timeout=config["receive_timeout_seconds"],
)
except TimeoutError:
break
payload = json.loads(raw_message)
if isinstance(payload, dict):
messages.append(payload)
except Exception as exc:
if self._db_session is not None:
await update_ais_source_health(
self._db_session,
source=self.name,
connection_state="disconnected",
last_error=f"{exc.__class__.__name__}: {exc}",
)
await self._db_session.commit()
raise
return messages
async def run(self, db: AsyncSession) -> dict[str, Any]:
"""Run AISStream as a long-lived streaming collector by default."""
config = await self._get_effective_config()
if not config.get("streaming_enabled", True):
return await super().run(db)
if not config["api_key"]:
return {"status": "failed", "error": "AISStream API key is not configured"}
from app.services.collectors.registry import collector_registry
if not collector_registry.is_active(self.name):
return {"status": "skipped", "reason": "Collector is disabled"}
try:
import websockets
except ImportError as exc:
return {"status": "failed", "error": "Python package 'websockets' is required for AISStream"}
start_time = datetime.now(UTC)
task = CollectionTask(
datasource_id=getattr(self, "_datasource_id", 1),
status="running",
phase="connecting",
phase_message="正在连接 AISStream 实时流",
phase_unit="messages",
started_at=start_time,
)
db.add(task)
await db.commit()
self._current_task = task
self._db_session = db
self._last_broadcast_progress = None
await self.resolve_url(db)
await self._publish_task_update(force=True)
records_added = 0
messages_seen = 0
unique_mmsi: set[str] = set()
reconnect_delay = config["reconnect_delay_seconds"]
try:
while True:
config = await self._get_effective_config()
subscription = self._build_subscription(config)
try:
await update_ais_source_health(
db,
source=self.name,
connection_state="connecting",
)
await self.set_phase("connecting", message="正在连接 AISStream 实时流")
await db.commit()
async with websockets.connect(config["endpoint"]) as websocket:
await websocket.send(json.dumps(subscription))
await update_ais_source_health(
db,
source=self.name,
connection_state="connected",
last_success_at=datetime.now(UTC),
)
await self.set_phase(
"streaming",
message="正在接收 AISStream 实时消息",
reset_progress=False,
)
await db.commit()
while True:
try:
raw_message = await asyncio.wait_for(
websocket.recv(),
timeout=config["receive_timeout_seconds"],
)
except TimeoutError:
await update_ais_source_health(
db,
source=self.name,
connection_state="connected",
last_success_at=datetime.now(UTC),
)
await db.commit()
continue
payload = json.loads(raw_message)
if not isinstance(payload, dict):
continue
messages_seen += 1
record = self._normalize_message(payload)
if not record:
continue
unique_mmsi.add(str(record["mmsi"]))
created = await self._save_stream_record(db, record)
if created:
records_added += 1
task.records_processed = messages_seen
task.total_records = None
task.progress = None
task.phase = "streaming"
task.phase_message = "正在接收 AISStream 实时消息"
task.phase_current = messages_seen
task.phase_total = None
task.phase_unit = "messages"
await self._publish_task_update(force=True)
if config["streaming_max_messages"] and messages_seen >= config["streaming_max_messages"]:
task.status = "success"
task.phase = "stopped"
task.phase_message = "AISStream 测试流已停止"
task.completed_at = datetime.now(UTC)
await db.commit()
await self._publish_task_update(force=True)
return {
"status": "success",
"task_id": task.id,
"records_processed": records_added,
"messages_seen": messages_seen,
"unique_mmsi": len(unique_mmsi),
"execution_time_seconds": (datetime.now(UTC) - start_time).total_seconds(),
}
except asyncio.CancelledError:
raise
except Exception as exc:
await update_ais_source_health(
db,
source=self.name,
connection_state="reconnecting",
last_error=f"{exc.__class__.__name__}: {exc}",
)
task.phase = "reconnecting"
task.phase_message = "AISStream 连接中断,正在重连"
task.error_message = f"{exc.__class__.__name__}: {exc}"
await db.commit()
await self._publish_task_update(force=True)
await asyncio.sleep(reconnect_delay)
except asyncio.CancelledError:
task.status = "cancelled"
task.phase = "stopped"
task.phase_message = "AISStream 实时流已停止"
task.completed_at = datetime.now(UTC)
await update_ais_source_health(
db,
source=self.name,
connection_state="disconnected",
last_error=None,
)
await db.commit()
await self._publish_task_update(force=True)
raise
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
records = []
for item in raw_data:
record = self._normalize_message(item)
if record:
records.append(record)
return records
async def _save_data(
self,
db: AsyncSession,
data: list[dict[str, Any]],
task_id: int | None = None,
snapshot_id: int | None = None,
) -> int:
now = datetime.now(UTC)
records_added = 0
latest_observed_at = now
for index, item in enumerate(data):
observed_at = item.get("received_at") or now
observation = await record_vessel_ais_observation(
db,
source=self.name,
normalized_payload=item,
raw_payload=item.get("_raw_payload") or item,
delivery_mode=AISSTREAM_DELIVERY_MODE,
transport=AISSTREAM_TRANSPORT,
message_type=item.get("_message_type") or "PositionReport",
source_message_id=item.get("_source_message_id"),
observed_at=observed_at,
collected_at=now,
)
if observation is not None:
records_added += 1
if isinstance(observed_at, datetime) and observed_at > latest_observed_at:
latest_observed_at = observed_at
if (index + 1) % 1000 == 0:
await self.update_progress(index + 1, commit=True)
await update_ais_source_health(
db,
source=self.name,
connection_state="connected",
observed_count=len(data),
last_seen_at=latest_observed_at,
last_success_at=now if data else None,
lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
)
await db.commit()
await self.update_progress(records_added, force=True)
return records_added
async def _save_stream_record(self, db: AsyncSession, item: dict[str, Any]) -> bool:
now = datetime.now(UTC)
observed_at = item.get("received_at") or now
observation = await record_vessel_ais_observation(
db,
source=self.name,
normalized_payload=item,
raw_payload=item.get("_raw_payload") or item,
delivery_mode=AISSTREAM_DELIVERY_MODE,
transport=AISSTREAM_TRANSPORT,
message_type=item.get("_message_type") or "PositionReport",
source_message_id=item.get("_source_message_id"),
observed_at=observed_at,
collected_at=now,
)
await update_ais_source_health(
db,
source=self.name,
connection_state="connected",
observed_count=1,
last_seen_at=observed_at if isinstance(observed_at, datetime) else now,
last_success_at=now,
lag_seconds=max((now - observed_at).total_seconds(), 0) if isinstance(observed_at, datetime) else None,
)
await db.commit()
await self._broadcast_vessel_delta(item, created=observation is not None)
return observation is not None
async def _broadcast_vessel_delta(self, item: dict[str, Any], *, created: bool) -> None:
await broadcaster.broadcast_custom(
"vessels",
{
"action": "upsert",
"source": self.name,
"created": created,
"vessels": [
{
"mmsi": item.get("mmsi"),
"mmsi_display": str(item.get("mmsi")) if item.get("mmsi") is not None else None,
"name": item.get("name"),
"lat": item.get("lat"),
"lon": item.get("lon"),
"sog": item.get("sog"),
"cog": item.get("cog"),
"heading": item.get("heading"),
"nav_status": item.get("nav_status"),
"vessel_type": item.get("vessel_type"),
"vessel_type_name": item.get("vessel_type_name"),
"received_at": to_iso8601_utc(item.get("received_at")),
}
],
},
)
def _normalize_message(self, item: dict[str, Any]) -> dict[str, Any] | None:
message_type = str(item.get("MessageType") or item.get("message_type") or "")
metadata = item.get("MetaData") if isinstance(item.get("MetaData"), dict) else {}
message = item.get("Message") if isinstance(item.get("Message"), dict) else {}
body = message.get(message_type) if isinstance(message.get(message_type), dict) else message
if not isinstance(body, dict):
body = {}
mmsi = _as_int(_pick(metadata, "MMSI", "mmsi") or _pick(body, "MMSI", "mmsi"))
if mmsi is None:
return None
received_at = _parse_datetime(
_pick(metadata, "time_utc", "Time_UTC", "timestamp")
or _pick(body, "Timestamp", "timestamp", "time")
)
ship_name = _clean_text(
_pick(body, "Name", "ShipName", "name")
or _pick(metadata, "ShipName", "ship_name", "name")
)
record: dict[str, Any] = {
"mmsi": mmsi,
"received_at": received_at,
"_message_type": message_type or None,
"_source_message_id": item.get("MessageID") or item.get("message_id"),
"_raw_payload": item,
}
lat = _as_float(_pick(body, "Latitude", "lat", "latitude"))
lon = _as_float(_pick(body, "Longitude", "lon", "lng", "longitude"))
if lat is not None and lon is not None:
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
return None
record.update(
{
"lat": lat,
"lon": lon,
"sog": _as_float(_pick(body, "Sog", "SOG", "speedOverGround")),
"cog": _as_float(_pick(body, "Cog", "COG", "courseOverGround")),
"heading": _as_int(_pick(body, "TrueHeading", "Heading", "heading")),
"nav_status": _as_int(_pick(body, "NavigationalStatus", "nav_status")),
}
)
vessel_type = _as_int(_pick(body, "Type", "ShipType", "vessel_type"))
record.update(
{
"name": ship_name,
"callsign": _pick(body, "CallSign", "callsign"),
"imo": _as_int(_pick(body, "ImoNumber", "IMO", "imo")),
"vessel_type": vessel_type,
"vessel_type_name": _pick(body, "TypeName", "ShipTypeName", "vessel_type_name")
or normalize_vessel_type_name(vessel_type),
"length": _as_float(_pick(body, "DimensionToBow", "Length", "length")),
"width": _as_float(_pick(body, "DimensionToPort", "Width", "width")),
}
)
return record
def _pick(item: dict[str, Any], *keys: str) -> Any:
for key in keys:
if key in item and item[key] not in (None, ""):
return item[key]
return None
def _clean_text(value: Any) -> str | None:
if value in (None, ""):
return None
text = str(value).strip()
return text or None
def _as_float(value: Any) -> float | None:
try:
if value in (None, ""):
return None
return float(value)
except (TypeError, ValueError):
return None
def _as_int(value: Any) -> int | None:
try:
if value in (None, ""):
return None
return int(float(value))
except (TypeError, ValueError):
return None
def _parse_datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if not value:
return None
if isinstance(value, (int, float)):
timestamp = float(value)
if timestamp > 10_000_000_000:
timestamp /= 1000
return datetime.fromtimestamp(timestamp, UTC)
if isinstance(value, str):
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
except ValueError:
return None
return None
+151 -25
View File
@@ -1,5 +1,6 @@
"""Base collector class for all data sources"""
import asyncio
from abc import ABC, abstractmethod
from typing import Dict, List, Any, Optional
from datetime import UTC, datetime
@@ -53,6 +54,11 @@ class BaseCollector(ABC):
"task_id": self._current_task.id,
"status": self._current_task.status,
"phase": self._current_task.phase,
"phase_progress": self._current_task.phase_progress,
"phase_message": self._current_task.phase_message,
"phase_current": self._current_task.phase_current,
"phase_total": self._current_task.phase_total,
"phase_unit": self._current_task.phase_unit,
"progress": progress,
"records_processed": self._current_task.records_processed,
"total_records": self._current_task.total_records,
@@ -79,12 +85,52 @@ class BaseCollector(ABC):
await self._publish_task_update(force=force)
async def set_phase(self, phase: str):
async def set_phase(self, phase: str, *, message: str | None = None, reset_progress: bool = True):
if self._current_task and self._db_session:
self._current_task.phase = phase
self._current_task.phase_message = message
if reset_progress:
self._current_task.phase_progress = None
self._current_task.phase_current = None
self._current_task.phase_total = None
self._current_task.phase_unit = None
await self._db_session.commit()
await self._publish_task_update(force=True)
async def update_phase_progress(
self,
*,
current: int | None = None,
total: int | None = None,
unit: str | None = None,
message: str | None = None,
progress: float | None = None,
commit: bool = False,
force: bool = False,
):
"""Update progress for the current phase without changing task totals."""
if not self._current_task or not self._db_session:
return
if progress is None and current is not None and total and total > 0:
progress = (current / total) * 100
if progress is not None:
self._current_task.phase_progress = max(0.0, min(float(progress), 100.0))
if current is not None:
self._current_task.phase_current = max(0, int(current))
if total is not None:
self._current_task.phase_total = max(0, int(total))
if unit is not None:
self._current_task.phase_unit = unit
if message is not None:
self._current_task.phase_message = message
if commit:
await self._db_session.commit()
await self._publish_task_update(force=force)
@abstractmethod
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch raw data from source"""
@@ -166,6 +212,59 @@ class BaseCollector(ABC):
await db.commit()
return snapshot.id
async def _rollback_incomplete_run(
self,
db: AsyncSession,
*,
task_id: int,
snapshot_id: Optional[int],
reason: str,
) -> None:
from app.models.collected_data import CollectedData
from app.models.data_snapshot import DataSnapshot
await db.execute(CollectedData.__table__.delete().where(CollectedData.task_id == task_id))
parent_snapshot_id: Optional[int] = None
if snapshot_id is not None:
snapshot = await db.get(DataSnapshot, snapshot_id)
if snapshot:
parent_snapshot_id = snapshot.parent_snapshot_id
snapshot.status = "cancelled"
snapshot.is_current = False
snapshot.completed_at = datetime.now(UTC)
summary = dict(snapshot.summary or {})
summary["rollback"] = True
summary["rollback_reason"] = reason
snapshot.summary = summary
await db.execute(
text(
"""
UPDATE collected_data
SET is_current = FALSE
WHERE source = :source
"""
),
{"source": self.name},
)
if parent_snapshot_id is not None:
parent_snapshot = await db.get(DataSnapshot, parent_snapshot_id)
if parent_snapshot:
parent_snapshot.is_current = True
await db.execute(
text(
"""
UPDATE collected_data
SET is_current = TRUE
WHERE snapshot_id = :snapshot_id
"""
),
{"snapshot_id": parent_snapshot_id},
)
async def run(self, db: AsyncSession) -> Dict[str, Any]:
"""Full pipeline: fetch -> transform -> save"""
from app.services.collectors.registry import collector_registry
@@ -197,7 +296,7 @@ class BaseCollector(ABC):
await self._publish_task_update(force=True)
try:
await self.set_phase("fetching")
await self.set_phase("fetching", message="正在拉取原始数据")
raw_data = await self.fetch()
task.total_records = len(raw_data)
await db.commit()
@@ -206,15 +305,20 @@ class BaseCollector(ABC):
if self.fail_on_empty and not raw_data:
raise RuntimeError(f"Collector {self.name} returned no data")
await self.set_phase("transforming")
await self.set_phase("transforming", message="正在转换采集数据")
data = self.transform(raw_data)
snapshot_id = await self._create_snapshot(db, task_id, data, start_time)
await self.set_phase("saving")
await self.set_phase("saving", message="正在保存采集数据")
records_count = await self._save_data(db, data, task_id=task_id, snapshot_id=snapshot_id)
task.status = "success"
task.phase = "completed"
task.phase_progress = 100.0
task.phase_message = "采集完成"
task.phase_current = records_count
task.phase_total = records_count
task.phase_unit = "records"
task.records_processed = records_count
task.progress = 100.0
task.completed_at = datetime.now(UTC)
@@ -227,9 +331,28 @@ class BaseCollector(ABC):
"records_processed": records_count,
"execution_time_seconds": (datetime.now(UTC) - start_time).total_seconds(),
}
except asyncio.CancelledError:
await db.rollback()
task.status = "cancelled"
task.phase = "cancelled"
task.phase_message = "采集已取消"
task.error_message = "Collection cancelled by operator and rolled back"
task.completed_at = datetime.now(UTC)
if snapshot_id is not None:
await self._rollback_incomplete_run(
db,
task_id=task_id,
snapshot_id=snapshot_id,
reason="cancelled_by_operator",
)
await db.commit()
await self._publish_task_update(force=True)
raise
except Exception as e:
await db.rollback()
task.status = "failed"
task.phase = "failed"
task.phase_message = str(e)
task.error_message = str(e)
task.completed_at = datetime.now(UTC)
if snapshot_id is not None:
@@ -276,20 +399,34 @@ class BaseCollector(ABC):
updated_count = 0
unchanged_count = 0
seen_entity_keys: set[str] = set()
previous_current_keys: set[str] = set()
progress_commit_interval = 1000
previous_current_result = await db.execute(
select(CollectedData.entity_key).where(
select(CollectedData)
.where(
CollectedData.source == self.name,
CollectedData.is_current == True,
)
.order_by(CollectedData.entity_key.asc(), CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
)
previous_current_keys = {row[0] for row in previous_current_result.fetchall() if row[0]}
previous_current_records = previous_current_result.scalars().all()
previous_current_keys = {record.entity_key for record in previous_current_records if record.entity_key}
previous_current_map: dict[str, CollectedData] = {}
stale_previous_records: list[CollectedData] = []
for existing_record in previous_current_records:
entity_key = existing_record.entity_key
if not entity_key:
continue
if entity_key not in previous_current_map:
previous_current_map[entity_key] = existing_record
continue
stale_previous_records.append(existing_record)
for stale_record in stale_previous_records:
stale_record.is_current = False
for i, item in enumerate(data):
print(
f"DEBUG: Saving item {i}: name={item.get('name')}, metadata={item.get('metadata', 'NOT FOUND')}"
)
raw_metadata = item.get("metadata", {})
extra_data = build_dynamic_metadata(
raw_metadata,
@@ -318,20 +455,9 @@ class BaseCollector(ABC):
previous_record = None
if entity_key and entity_key not in seen_entity_keys:
result = await db.execute(
select(CollectedData)
.where(
CollectedData.source == self.name,
CollectedData.entity_key == entity_key,
CollectedData.is_current == True,
)
.order_by(CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
)
previous_records = result.scalars().all()
if previous_records:
previous_record = previous_records[0]
for old_record in previous_records:
old_record.is_current = False
previous_record = previous_current_map.get(entity_key)
if previous_record is not None:
previous_record.is_current = False
record = CollectedData(
snapshot_id=snapshot_id,
@@ -375,7 +501,7 @@ class BaseCollector(ABC):
seen_entity_keys.add(entity_key)
records_added += 1
if i % 100 == 0:
if (i + 1) % progress_commit_interval == 0:
await self.update_progress(i + 1, commit=True)
if snapshot_id is not None:
+29 -27
View File
@@ -13,6 +13,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_observation import BGPObservation
from app.models.collected_data import CollectedData
from app.services.bgp_collector_locations import (
RIPE_RIS_COLLECTOR_COORDS,
get_bgp_collector_location_dict,
)
from app.services.bgp_event_locations import resolve_bgp_event_geo_dict
from app.services.bgp_incidents import create_bgp_incidents_for_anomalies
from app.services.bgp_detectors import (
detect_mass_withdrawal_anomalies,
@@ -23,32 +28,17 @@ from app.services.bgp_detectors import (
)
from app.services.bgp_enrichment import enrich_bgp_events_for_batch, extract_bgp_network_fields
RIPE_RIS_COLLECTOR_COORDS: dict[str, dict[str, Any]] = {
"rrc00": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041},
"rrc01": {"city": "London", "country": "United Kingdom", "latitude": 51.5072, "longitude": -0.1276},
"rrc03": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041},
"rrc04": {"city": "Geneva", "country": "Switzerland", "latitude": 46.2044, "longitude": 6.1432},
"rrc05": {"city": "Vienna", "country": "Austria", "latitude": 48.2082, "longitude": 16.3738},
"rrc06": {"city": "Otemachi", "country": "Japan", "latitude": 35.686, "longitude": 139.7671},
"rrc07": {"city": "Stockholm", "country": "Sweden", "latitude": 59.3293, "longitude": 18.0686},
"rrc10": {"city": "Milan", "country": "Italy", "latitude": 45.4642, "longitude": 9.19},
"rrc11": {"city": "New York", "country": "United States", "latitude": 40.7128, "longitude": -74.006},
"rrc12": {"city": "Frankfurt", "country": "Germany", "latitude": 50.1109, "longitude": 8.6821},
"rrc13": {"city": "Moscow", "country": "Russia", "latitude": 55.7558, "longitude": 37.6173},
"rrc14": {"city": "Palo Alto", "country": "United States", "latitude": 37.4419, "longitude": -122.143},
"rrc15": {"city": "Sao Paulo", "country": "Brazil", "latitude": -23.5558, "longitude": -46.6396},
"rrc16": {"city": "Miami", "country": "United States", "latitude": 25.7617, "longitude": -80.1918},
"rrc18": {"city": "Barcelona", "country": "Spain", "latitude": 41.3874, "longitude": 2.1686},
"rrc19": {"city": "Johannesburg", "country": "South Africa", "latitude": -26.2041, "longitude": 28.0473},
"rrc20": {"city": "Zurich", "country": "Switzerland", "latitude": 47.3769, "longitude": 8.5417},
"rrc21": {"city": "Paris", "country": "France", "latitude": 48.8566, "longitude": 2.3522},
"rrc22": {"city": "Bucharest", "country": "Romania", "latitude": 44.4268, "longitude": 26.1025},
"rrc23": {"city": "Singapore", "country": "Singapore", "latitude": 1.3521, "longitude": 103.8198},
"rrc24": {"city": "Montevideo", "country": "Uruguay", "latitude": -34.9011, "longitude": -56.1645},
"rrc25": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041},
"rrc26": {"city": "Dubai", "country": "United Arab Emirates", "latitude": 25.2048, "longitude": 55.2708},
}
# Re-exported for backward compatibility with anything that imports
# ``RIPE_RIS_COLLECTOR_COORDS`` from this module. New code should call
# ``app.services.bgp_collector_locations.get_bgp_collector_location_dict()``
# or ``resolve_bgp_collector_location()`` instead — those use the DB-backed
# collector-location cache.
__all__ = [
"RIPE_RIS_COLLECTOR_COORDS",
"normalize_bgp_event",
"save_bgp_observations_for_batch",
"create_bgp_anomalies_for_batch",
]
def _safe_int(value: Any) -> int | None:
@@ -131,7 +121,19 @@ def normalize_bgp_event(payload: dict[str, Any], *, project: str) -> dict[str, A
)
source_id = hashlib.sha1(source_material.encode("utf-8")).hexdigest()[:24]
collector_location = RIPE_RIS_COLLECTOR_COORDS.get(collector, {})
# Routes through the BGP event pipeline: source coords (if any) →
# collector inheritance. Returned dict keeps the legacy
# {city, country, latitude, longitude} keys plus richer
# {precision, source, needs_confirmation, matched_location_name, confidence}.
collector_location = resolve_bgp_event_geo_dict(
collector,
source_latitude=payload.get("latitude"),
source_longitude=payload.get("longitude"),
)
# Empty result (unknown collector & no source coords) — keep the
# downstream-expected dict shape so detectors / serializers don't crash.
if not collector_location:
collector_location = get_bgp_collector_location_dict(collector)
network_fields = extract_bgp_network_fields(prefix)
metadata = {
"project": project,
+6 -2
View File
@@ -21,7 +21,7 @@ class CelesTrakTLECollector(BaseCollector):
@property
def base_url(self) -> str:
return "https://celestrak.org/NORAD/elements/gp.php"
return self._resolved_url or ""
async def fetch(self) -> List[Dict[str, Any]]:
satellite_groups = [
@@ -40,12 +40,15 @@ class CelesTrakTLECollector(BaseCollector):
async with httpx.AsyncClient(timeout=120.0) as client:
for group in satellite_groups:
try:
url = f"https://celestrak.org/NORAD/elements/gp.php?GROUP={group}&FORMAT=json"
url = f"{self.base_url}?GROUP={group}&FORMAT=json"
response = await client.get(url)
if response.status_code == 200:
data = response.json()
if isinstance(data, list):
for item in data:
if isinstance(item, dict):
item["_celestrak_group"] = group
all_satellites.extend(data)
print(f"CelesTrak: Fetched {len(data)} satellites from group '{group}'")
except Exception as e:
@@ -78,6 +81,7 @@ class CelesTrakTLECollector(BaseCollector):
"name": item.get("OBJECT_NAME", "Unknown"),
"reference_date": item.get("EPOCH", ""),
"metadata": {
"constellation_group": item.get("_celestrak_group"),
"norad_cat_id": item.get("NORAD_CAT_ID"),
"international_designator": item.get("OBJECT_ID"),
"epoch": item.get("EPOCH"),
@@ -39,6 +39,16 @@ class CloudflareRadarDeviceCollector(HTTPCollector):
if CLOUDFLARE_API_TOKEN:
self.headers["Authorization"] = f"Bearer {CLOUDFLARE_API_TOKEN}"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Cloudflare Radar device type response"""
data = []
@@ -87,6 +97,16 @@ class CloudflareRadarTrafficCollector(HTTPCollector):
if CLOUDFLARE_API_TOKEN:
self.headers["Authorization"] = f"Bearer {CLOUDFLARE_API_TOKEN}"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Cloudflare Radar traffic timeseries response"""
data = []
@@ -135,6 +155,16 @@ class CloudflareRadarTopASCollector(HTTPCollector):
if CLOUDFLARE_API_TOKEN:
self.headers["Authorization"] = f"Bearer {CLOUDFLARE_API_TOKEN}"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Cloudflare Radar top locations response"""
data = []
+1 -1
View File
@@ -23,7 +23,7 @@ class EpochAIGPUCollector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch Epoch AI GPU clusters data from webpage"""
url = "https://epoch.ai/data/gpu-clusters"
url = self._resolved_url or ""
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(url)
@@ -18,11 +18,9 @@ class FAOLandingPointCollector(BaseCollector):
frequency_hours = 168
data_type = "landing_point"
csv_url = "https://data.apps.fao.org/catalog/dataset/1b75ff21-92f2-4b96-9b7b-98e8aa65ad5d/resource/b6071077-d1d4-4e97-aa00-42e902847c87/download/landing-point-geo.csv"
async def fetch(self) -> List[Dict[str, Any]]:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.csv_url)
response = await client.get(self._resolved_url or "")
response.raise_for_status()
return self.parse_csv(response.text)
@@ -21,6 +21,18 @@ class HuggingFaceModelCollector(HTTPCollector):
data_type = "model"
base_url = "https://huggingface.co/api/models"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
from httpx import AsyncClient
async with AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Hugging Face models API response"""
data = []
@@ -63,6 +75,18 @@ class HuggingFaceDatasetCollector(HTTPCollector):
data_type = "dataset"
base_url = "https://huggingface.co/api/datasets"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
from httpx import AsyncClient
async with AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Hugging Face datasets API response"""
data = []
@@ -104,6 +128,18 @@ class HuggingFaceSpacesCollector(HTTPCollector):
data_type = "space"
base_url = "https://huggingface.co/api/spaces"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
from httpx import AsyncClient
async with AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Hugging Face Spaces API response"""
data = []
+19 -1
View File
@@ -108,6 +108,11 @@ class IPtoASNPrefixGeoCollector(BaseCollector):
self._current_task.total_records = total_expected
self._current_task.records_processed = 0
self._current_task.progress = 0.0
self._current_task.phase_progress = 0.0
self._current_task.phase_message = "正在下载 IPtoASN 数据"
self._current_task.phase_current = 0
self._current_task.phase_total = total_expected
self._current_task.phase_unit = "bytes"
await self._db_session.commit()
await self._publish_task_update(force=True)
@@ -135,7 +140,14 @@ class IPtoASNPrefixGeoCollector(BaseCollector):
return
last_emit["value"] = aggregated
last_emit["t"] = now
await self.update_progress(min(aggregated, total_expected), commit=True)
current = min(aggregated, total_expected)
await self.update_phase_progress(
current=current,
total=total_expected,
unit="bytes",
message="正在下载 IPtoASN 数据",
)
await self.update_progress(current, commit=True)
batches = await asyncio.gather(
*(
@@ -148,6 +160,12 @@ class IPtoASNPrefixGeoCollector(BaseCollector):
)
)
if total_expected > 0:
await self.update_phase_progress(
current=total_expected,
total=total_expected,
unit="bytes",
message="IPtoASN 数据下载完成",
)
await self.update_progress(total_expected, commit=True, force=True)
rows: list[dict[str, Any]] = []
@@ -0,0 +1,570 @@
from __future__ import annotations
import asyncio
import base64
from datetime import UTC, datetime
from typing import Any
from urllib.parse import urlparse
import httpx
from sqlalchemy import select
from app.core.data_sources import get_data_sources_config
from app.models.datasource_config import DataSourceConfig
from app.services.collectors.base import BaseCollector
class NewsLiveStreamsCollector(BaseCollector):
"""Collect normalized news live-stream sources from a JSON endpoint."""
name = "news_live_streams"
priority = "P2"
module = "L4"
frequency_hours = 12
data_type = "news_live_stream"
fail_on_empty = False
DEFAULT_TIMEOUT = 45.0
DEFAULT_HEADERS = {
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/json",
}
RESPONSE_CANDIDATE_KEYS = ("sources", "streams", "channels", "items", "results", "data")
DEFAULT_ADAPTER = "iptv_org"
DEFAULT_IPTV_ORG_STREAMS_URL = "https://iptv-org.github.io/api/streams.json"
DEFAULT_IPTV_ORG_LOGOS_URL = "https://iptv-org.github.io/api/logos.json"
DEFAULT_IPTV_ORG_NEWS_CATEGORIES = ("news", "business", "weather")
DEFAULT_IPTV_ORG_EXCLUDE_CATEGORIES = ("music", "sports", "kids", "entertainment")
DEFAULT_IPTV_ORG_MAX_SOURCES = 120
async def fetch(self) -> list[dict[str, Any]]:
request_url = (self._resolved_url or "").strip()
if not request_url:
return []
datasource_config = await self._load_datasource_config()
effective_config = self._get_effective_config(datasource_config)
adapter = str(effective_config.get("adapter") or "").strip().lower()
if adapter == "iptv_org":
return await self._fetch_iptv_org(request_url, effective_config)
request_headers = self._build_request_headers(datasource_config)
request_config = self._get_request_config(datasource_config)
request_params = self._build_request_params(datasource_config)
request_json = self._build_request_json_body(datasource_config)
request_data = self._build_request_form_body(datasource_config)
timeout = self._get_timeout(datasource_config)
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
response = await client.request(
request_config["method"],
request_url,
headers=request_headers,
params=request_params or None,
json=request_json,
data=request_data,
)
response.raise_for_status()
return self.parse_response(
response.json(),
response_path=request_config["response_path"],
)
async def _load_datasource_config(self) -> DataSourceConfig | None:
if not self._db_session:
return None
result = await self._db_session.execute(
select(DataSourceConfig)
.where(DataSourceConfig.name == self.name)
.where(DataSourceConfig.is_active.is_(True))
.limit(1)
)
return result.scalar_one_or_none()
def _get_effective_config(self, datasource_config: DataSourceConfig | None) -> dict[str, Any]:
payload = dict(datasource_config.config or {}) if datasource_config else {}
if payload:
return payload
yaml_config = get_data_sources_config()
return {
"adapter": self.DEFAULT_ADAPTER,
"streams_url": yaml_config.get_yaml_value("news_live_streams.streams_url")
or self.DEFAULT_IPTV_ORG_STREAMS_URL,
"logos_url": yaml_config.get_yaml_value("news_live_streams.logos_url")
or self.DEFAULT_IPTV_ORG_LOGOS_URL,
"news_categories": list(self.DEFAULT_IPTV_ORG_NEWS_CATEGORIES),
"exclude_categories": list(self.DEFAULT_IPTV_ORG_EXCLUDE_CATEGORIES),
"max_sources": self.DEFAULT_IPTV_ORG_MAX_SOURCES,
}
def _get_request_config(self, datasource_config: DataSourceConfig | None) -> dict[str, Any]:
payload = self._get_effective_config(datasource_config)
raw_method = payload.get("method") or payload.get("request_method") or "GET"
method = str(raw_method).strip().upper() or "GET"
if method not in {"GET", "POST"}:
method = "GET"
response_path = payload.get("response_path") or payload.get("payload_path") or payload.get("items_path")
if isinstance(response_path, str):
response_path = response_path.strip()
else:
response_path = None
return {
"method": method,
"response_path": response_path or None,
}
def _get_timeout(self, datasource_config: DataSourceConfig | None) -> float:
payload = self._get_effective_config(datasource_config)
try:
return float(payload.get("timeout", self.DEFAULT_TIMEOUT))
except (TypeError, ValueError):
return self.DEFAULT_TIMEOUT
def _build_request_headers(self, datasource_config: DataSourceConfig | None) -> dict[str, str]:
headers = dict(self.DEFAULT_HEADERS)
if datasource_config:
headers.update(self._normalize_headers(datasource_config.headers))
headers.update(self._build_auth_headers(datasource_config))
return headers
def _build_request_params(self, datasource_config: DataSourceConfig | None) -> dict[str, Any]:
params: dict[str, Any] = {}
if not datasource_config:
return params
payload = datasource_config.config or {}
candidate = payload.get("params") or payload.get("query_params")
if isinstance(candidate, dict):
params.update(candidate)
if datasource_config.auth_type == "api_key":
auth_config = datasource_config.auth_config or {}
if str(auth_config.get("in") or auth_config.get("location") or "header").lower() == "query":
api_key = auth_config.get("api_key")
key_name = auth_config.get("key_name") or auth_config.get("param_name") or "api_key"
if api_key and key_name:
params[str(key_name)] = api_key
return params
def _build_request_json_body(self, datasource_config: DataSourceConfig | None) -> Any:
if not datasource_config:
return None
payload = datasource_config.config or {}
body = payload.get("json_body")
if body is None and str(payload.get("body_type") or "").lower() in {"json", ""}:
candidate = payload.get("body")
if isinstance(candidate, (dict, list)):
body = candidate
return body
def _build_request_form_body(self, datasource_config: DataSourceConfig | None) -> Any:
if not datasource_config:
return None
payload = datasource_config.config or {}
form_body = payload.get("form_body")
if form_body is not None:
return form_body
if str(payload.get("body_type") or "").lower() == "form":
candidate = payload.get("body")
if isinstance(candidate, dict):
return candidate
return None
def _normalize_headers(self, headers: Any) -> dict[str, str]:
if not isinstance(headers, dict):
return {}
normalized: dict[str, str] = {}
for key, value in headers.items():
header_name = str(key).strip()
if not header_name or value is None:
continue
normalized[header_name] = str(value)
return normalized
def _build_auth_headers(self, datasource_config: DataSourceConfig | None) -> dict[str, str]:
if not datasource_config:
return {}
auth_type = str(datasource_config.auth_type or "none").lower()
auth_config = datasource_config.auth_config or {}
if auth_type == "bearer" and auth_config.get("token"):
return {"Authorization": f"Bearer {auth_config['token']}"}
if auth_type == "api_key" and auth_config.get("api_key"):
location = str(auth_config.get("in") or auth_config.get("location") or "header").lower()
if location == "query":
return {}
key_name = auth_config.get("key_name") or "X-API-Key"
return {str(key_name): str(auth_config["api_key"])}
if auth_type == "basic":
username = str(auth_config.get("username") or "")
password = str(auth_config.get("password") or "")
encoded = base64.b64encode(f"{username}:{password}".encode()).decode()
return {"Authorization": f"Basic {encoded}"}
return {}
def _extract_candidates(self, response: Any, response_path: str | None) -> list[Any]:
if response_path:
extracted = self._extract_from_path(response, response_path)
if isinstance(extracted, list):
return extracted
if isinstance(extracted, dict):
for key in self.RESPONSE_CANDIDATE_KEYS:
nested = extracted.get(key)
if isinstance(nested, list):
return nested
return [extracted]
if isinstance(response, dict):
for key in self.RESPONSE_CANDIDATE_KEYS:
nested = response.get(key)
if isinstance(nested, list):
return nested
return []
if isinstance(response, list):
return response
return []
def _extract_from_path(self, payload: Any, path: str) -> Any:
current = payload
for segment in (part.strip() for part in path.split(".") if part.strip()):
if isinstance(current, dict):
current = current.get(segment)
continue
if isinstance(current, list):
try:
current = current[int(segment)]
except (TypeError, ValueError, IndexError):
return None
continue
return None
return current
def _infer_source_type(self, item: dict[str, Any]) -> str:
explicit = str(item.get("source_type") or item.get("type") or "").strip().lower()
if explicit in {"iframe", "hls", "video", "external", "youtube"}:
return explicit
youtube_video_id = self._clean_text(
item.get("youtube_video_id")
or item.get("video_id")
or item.get("youtubeVideoId")
)
youtube_channel = self._clean_text(item.get("youtube_channel") or item.get("channel_handle"))
embed_url = self._clean_url(item.get("embed_url") or item.get("embed") or item.get("page_url"))
stream_url = self._clean_url(item.get("stream_url") or item.get("stream") or item.get("playback_url") or item.get("hls_url"))
homepage_url = self._clean_url(item.get("homepage_url") or item.get("source_url") or item.get("website"))
if youtube_video_id or youtube_channel:
return "youtube"
if stream_url.endswith(".m3u8"):
return "hls"
if stream_url:
return "video"
if embed_url:
parsed = urlparse(embed_url)
if "youtube.com" in (parsed.netloc or "") or "youtu.be" in (parsed.netloc or ""):
return "youtube"
return "iframe"
if homepage_url:
return "external"
return "iframe"
def _parse_enabled(self, item: dict[str, Any]) -> bool:
if "is_enabled" in item:
return self._to_bool(item.get("is_enabled"), default=True)
if "enabled" in item:
return self._to_bool(item.get("enabled"), default=True)
if "active" in item:
return self._to_bool(item.get("active"), default=True)
if "status" in item:
status = str(item.get("status") or "").strip().lower()
if status in {"disabled", "inactive", "offline"}:
return False
if status in {"enabled", "active", "online", "live"}:
return True
return True
def _to_bool(self, value: Any, *, default: bool) -> bool:
if isinstance(value, bool):
return value
if value in (None, ""):
return default
if isinstance(value, str):
lowered = value.strip().lower()
if lowered in {"1", "true", "yes", "on", "enabled", "active", "online", "live"}:
return True
if lowered in {"0", "false", "no", "off", "disabled", "inactive", "offline"}:
return False
return bool(value)
def _clean_text(self, value: Any) -> str:
if value is None:
return ""
return str(value).strip()
def _clean_url(self, value: Any) -> str:
text = self._clean_text(value)
if not text:
return ""
parsed = urlparse(text)
if parsed.scheme and parsed.scheme not in {"http", "https"}:
return ""
if parsed.scheme and not parsed.netloc:
return ""
return text
async def _fetch_iptv_org(self, channels_url: str, collector_config: dict[str, Any]) -> list[dict[str, Any]]:
streams_url = self._clean_url(collector_config.get("streams_url")) or self.DEFAULT_IPTV_ORG_STREAMS_URL
logos_url = self._clean_url(collector_config.get("logos_url")) or self.DEFAULT_IPTV_ORG_LOGOS_URL
news_categories = {
self._clean_text(value).lower()
for value in (collector_config.get("news_categories") or self.DEFAULT_IPTV_ORG_NEWS_CATEGORIES)
if self._clean_text(value)
}
exclude_categories = {
self._clean_text(value).lower()
for value in (collector_config.get("exclude_categories") or self.DEFAULT_IPTV_ORG_EXCLUDE_CATEGORIES)
if self._clean_text(value)
}
try:
max_sources = int(collector_config.get("max_sources", self.DEFAULT_IPTV_ORG_MAX_SOURCES))
except (TypeError, ValueError):
max_sources = self.DEFAULT_IPTV_ORG_MAX_SOURCES
timeout = self.DEFAULT_TIMEOUT
try:
timeout = float(collector_config.get("timeout", self.DEFAULT_TIMEOUT))
except (TypeError, ValueError):
timeout = self.DEFAULT_TIMEOUT
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
channels_payload, streams_payload, logos_payload = await self._gather_iptv_org_payloads(
client,
channels_url,
streams_url,
logos_url,
)
channels = channels_payload if isinstance(channels_payload, list) else []
streams = streams_payload if isinstance(streams_payload, list) else []
logos = logos_payload if isinstance(logos_payload, list) else []
logo_by_channel = {
self._clean_text(item.get("channel")): self._clean_url(item.get("url"))
for item in logos
if isinstance(item, dict) and self._clean_text(item.get("channel")) and self._clean_url(item.get("url"))
}
streams_by_channel: dict[str, list[dict[str, Any]]] = {}
for stream in streams:
if not isinstance(stream, dict):
continue
channel_id = self._clean_text(stream.get("channel"))
if not channel_id:
continue
streams_by_channel.setdefault(channel_id, []).append(stream)
normalized: list[dict[str, Any]] = []
for channel in channels:
if not isinstance(channel, dict):
continue
categories = [
self._clean_text(value).lower()
for value in (channel.get("categories") or [])
if self._clean_text(value)
]
if news_categories and not any(category in news_categories for category in categories):
continue
if exclude_categories and any(category in exclude_categories for category in categories):
continue
if channel.get("is_nsfw") is True:
continue
if channel.get("closed"):
continue
channel_id = self._clean_text(channel.get("id"))
if not channel_id:
continue
stream = self._pick_iptv_org_stream(streams_by_channel.get(channel_id) or [])
if not stream:
continue
stream_url = self._clean_url(stream.get("url"))
if not stream_url:
continue
name = self._clean_text(channel.get("name")) or channel_id
notes_parts = [
f"Imported from IPTV-org catalog ({channel_id})",
f"Categories: {', '.join(categories)}" if categories else "",
f"Quality: {self._clean_text(stream.get('quality'))}" if self._clean_text(stream.get("quality")) else "",
]
metadata = {
"provider": self._clean_text(channel.get("network")) or "IPTV-org",
"region": self._clean_text(channel.get("country")) or "Global",
"language": "und",
"source_type": "hls" if stream_url.endswith(".m3u8") else "video",
"embed_url": "",
"stream_url": stream_url,
"homepage_url": self._clean_url(channel.get("website")),
"poster_url": logo_by_channel.get(channel_id, ""),
"youtube_video_id": "",
"youtube_channel": "",
"sort_order": 400 + len(normalized),
"notes": "; ".join(part for part in notes_parts if part),
"is_enabled": True,
"collector_adapter": "iptv_org",
"channel_id": channel_id,
"categories": categories,
"quality": self._clean_text(stream.get("quality")),
"stream_label": self._clean_text(stream.get("label") or stream.get("title")),
"stream_referrer": self._clean_text(stream.get("referrer")),
"stream_user_agent": self._clean_text(stream.get("user_agent")),
}
normalized.append(
{
"source_id": channel_id,
"name": name,
"description": metadata["notes"],
"metadata": metadata,
"reference_date": datetime.now(UTC).isoformat(),
}
)
if len(normalized) >= max_sources:
break
return normalized
async def _gather_iptv_org_payloads(
self,
client: httpx.AsyncClient,
channels_url: str,
streams_url: str,
logos_url: str,
) -> tuple[Any, Any, Any]:
headers = dict(self.DEFAULT_HEADERS)
channels_payload, streams_payload, logos_payload = await asyncio.gather(
client.get(channels_url, headers=headers),
client.get(streams_url, headers=headers),
client.get(logos_url, headers=headers),
)
channels_payload.raise_for_status()
streams_payload.raise_for_status()
logos_payload.raise_for_status()
return channels_payload.json(), streams_payload.json(), logos_payload.json()
def _pick_iptv_org_stream(self, streams: list[dict[str, Any]]) -> dict[str, Any] | None:
if not streams:
return None
def score(stream: dict[str, Any]) -> tuple[int, int]:
url = self._clean_url(stream.get("url"))
quality = self._clean_text(stream.get("quality")).lower()
quality_score = 0
if quality.endswith("p"):
try:
quality_score = int(quality[:-1])
except ValueError:
quality_score = 0
stream_score = 1000 if url.endswith(".m3u8") else 0
return stream_score, quality_score
sorted_streams = sorted(streams, key=score, reverse=True)
return sorted_streams[0]
def parse_response(self, response: Any, *, response_path: str | None = None) -> list[dict[str, Any]]:
candidates = self._extract_candidates(response, response_path)
normalized: list[dict[str, Any]] = []
for index, item in enumerate(candidates):
if not isinstance(item, dict):
continue
stream_id = (
item.get("id")
or item.get("source_id")
or item.get("slug")
or item.get("channel_id")
or item.get("code")
or f"news-live-{index + 1}"
)
name = self._clean_text(
item.get("name")
or item.get("title")
or item.get("channel")
or item.get("display_name")
or f"News Live {index + 1}"
)
if not name:
continue
source_type = self._infer_source_type(item)
stream_url = self._clean_url(
item.get("stream_url")
or item.get("stream")
or item.get("playback_url")
or item.get("hls_url")
or item.get("m3u8_url")
)
embed_url = self._clean_url(
item.get("embed_url")
or item.get("embed")
or item.get("page_url")
or (item.get("url") if source_type == "iframe" else "")
)
homepage_url = self._clean_url(
item.get("homepage_url")
or item.get("source_url")
or item.get("website")
or item.get("url")
)
metadata = {
"provider": self._clean_text(item.get("provider") or item.get("publisher") or item.get("network")) or "Collector",
"region": self._clean_text(item.get("region") or item.get("country") or item.get("market")) or "Global",
"language": self._clean_text(item.get("language") or item.get("lang") or item.get("locale")) or "und",
"source_type": source_type,
"embed_url": embed_url,
"stream_url": stream_url,
"homepage_url": homepage_url,
"poster_url": self._clean_url(item.get("poster_url") or item.get("thumbnail_url") or item.get("logo_url")),
"youtube_video_id": self._clean_text(
item.get("youtube_video_id")
or item.get("video_id")
or item.get("youtubeVideoId")
),
"youtube_channel": self._clean_text(
item.get("youtube_channel")
or item.get("channel_handle")
or item.get("youtubeChannel")
),
"sort_order": item.get("sort_order", 200 + index),
"notes": self._clean_text(item.get("notes") or item.get("description") or item.get("summary")),
"is_enabled": self._parse_enabled(item),
}
normalized.append(
{
"source_id": str(stream_id),
"name": name,
"description": metadata["notes"],
"metadata": metadata,
"reference_date": item.get("reference_date") or datetime.now(UTC).isoformat(),
}
)
return normalized
@@ -39,12 +39,23 @@ class NRODelegatedPrefixGeoCollector(BaseCollector):
self._current_task.total_records = total_expected
self._current_task.records_processed = 0
self._current_task.progress = 0.0
self._current_task.phase_progress = 0.0
self._current_task.phase_message = "正在下载 NRO delegated 数据"
self._current_task.phase_current = 0
self._current_task.phase_total = total_expected
self._current_task.phase_unit = "bytes"
await self._db_session.commit()
await self._publish_task_update(force=True)
async def on_progress(downloaded: int, total: int | None) -> None:
if not total or total <= 0:
return
await self.update_phase_progress(
current=min(downloaded, total),
total=total,
unit="bytes",
message="正在下载 NRO delegated 数据",
)
await self.update_progress(min(downloaded, total), commit=True)
body_path = await self._downloader.download_file(
@@ -40,12 +40,23 @@ class OpenGeoFeedPrefixGeoCollector(BaseCollector):
self._current_task.total_records = total_expected
self._current_task.records_processed = 0
self._current_task.progress = 0.0
self._current_task.phase_progress = 0.0
self._current_task.phase_message = "正在下载 OpenGeoFeed 数据"
self._current_task.phase_current = 0
self._current_task.phase_total = total_expected
self._current_task.phase_unit = "bytes"
await self._db_session.commit()
await self._publish_task_update(force=True)
async def on_progress(downloaded: int, total: int | None) -> None:
if not total or total <= 0:
return
await self.update_phase_progress(
current=min(downloaded, total),
total=total,
unit="bytes",
message="正在下载 OpenGeoFeed 数据",
)
await self.update_progress(min(downloaded, total), commit=True)
body_path = await self._downloader.download_file(
+25 -10
View File
@@ -16,6 +16,7 @@ from typing import Dict, Any, List
from datetime import UTC, datetime
import httpx
from urllib.parse import urlencode
from app.services.collectors.base import HTTPCollector
@@ -38,9 +39,13 @@ class PeeringDBIXPCollector(HTTPCollector):
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/json",
}
# API key is added to URL as query parameter
if PEERINGDB_API_KEY:
self.base_url = f"{self.base_url}?key={PEERINGDB_API_KEY}"
@property
def request_url(self) -> str:
base = self._resolved_url or self.base_url
if not PEERINGDB_API_KEY:
return base
separator = "&" if "?" in base else "?"
return f"{base}{separator}{urlencode({'key': PEERINGDB_API_KEY})}"
async def fetch_with_retry(
self, max_retries: int = 3, base_delay: float = 2.0
@@ -51,7 +56,7 @@ class PeeringDBIXPCollector(HTTPCollector):
for attempt in range(max_retries):
try:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.base_url, headers=self.headers)
response = await client.get(self.request_url, headers=self.headers)
if response.status_code == 429:
# Rate limited - wait and retry with exponential backoff
@@ -141,8 +146,13 @@ class PeeringDBNetworkCollector(HTTPCollector):
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/json",
}
if PEERINGDB_API_KEY:
self.base_url = f"{self.base_url}?key={PEERINGDB_API_KEY}"
@property
def request_url(self) -> str:
base = self._resolved_url or self.base_url
if not PEERINGDB_API_KEY:
return base
separator = "&" if "?" in base else "?"
return f"{base}{separator}{urlencode({'key': PEERINGDB_API_KEY})}"
async def fetch_with_retry(
self, max_retries: int = 3, base_delay: float = 2.0
@@ -153,7 +163,7 @@ class PeeringDBNetworkCollector(HTTPCollector):
for attempt in range(max_retries):
try:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.base_url, headers=self.headers)
response = await client.get(self.request_url, headers=self.headers)
if response.status_code == 429:
delay = base_delay * (2**attempt)
@@ -244,8 +254,13 @@ class PeeringDBFacilityCollector(HTTPCollector):
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/json",
}
if PEERINGDB_API_KEY:
self.base_url = f"{self.base_url}?key={PEERINGDB_API_KEY}"
@property
def request_url(self) -> str:
base = self._resolved_url or self.base_url
if not PEERINGDB_API_KEY:
return base
separator = "&" if "?" in base else "?"
return f"{base}{separator}{urlencode({'key': PEERINGDB_API_KEY})}"
async def fetch_with_retry(
self, max_retries: int = 3, base_delay: float = 2.0
@@ -256,7 +271,7 @@ class PeeringDBFacilityCollector(HTTPCollector):
for attempt in range(max_retries):
try:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.base_url, headers=self.headers)
response = await client.get(self.request_url, headers=self.headers)
if response.status_code == 429:
delay = base_delay * (2**attempt)
+1 -1
View File
@@ -33,7 +33,7 @@ class RISLiveCollector(BaseCollector):
def _fetch_via_stream(self) -> list[dict[str, Any]]:
events: list[dict[str, Any]] = []
stream_url = "https://ris-live.ripe.net/v1/stream/?format=json&client=planet-ris-live"
stream_url = self._resolved_url or ""
subscribe = json.dumps(
{
"host": "rrc00",
+30 -21
View File
@@ -7,6 +7,7 @@ API documentation: https://www.space-track.org/documentation
import json
from typing import Dict, Any, List
import httpx
from urllib.parse import urlparse
from app.services.collectors.base import BaseCollector
from app.core.data_sources import get_data_sources_config
@@ -21,12 +22,30 @@ class SpaceTrackTLECollector(BaseCollector):
data_type = "satellite_tle"
@property
def base_url(self) -> str:
def query_url(self) -> str:
config = get_data_sources_config()
if self._resolved_url:
return self._resolved_url
return config.get_yaml_url("spacetrack_tle")
@property
def site_root(self) -> str:
config = get_data_sources_config()
configured_root = config.get_yaml_value("spacetrack.base_url")
if isinstance(configured_root, str) and configured_root:
return configured_root.rstrip("/")
parsed = urlparse(self.query_url)
return f"{parsed.scheme}://{parsed.netloc}".rstrip("/")
@property
def login_url(self) -> str:
return f"{self.site_root}/ajaxauth/login"
@property
def probe_url(self) -> str:
return f"{self.site_root}/basicspacedata/query/class/gp/NORAD_CAT_ID/25544/format/json"
async def fetch(self) -> List[Dict[str, Any]]:
from app.core.config import settings
@@ -47,13 +66,13 @@ class SpaceTrackTLECollector(BaseCollector):
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "application/json, text/html, */*",
"Accept-Language": "en-US,en;q=0.9",
"Referer": "https://www.space-track.org/",
"Referer": f"{self.site_root}/",
},
) as client:
await client.get("https://www.space-track.org/")
await client.get(f"{self.site_root}/")
login_response = await client.post(
"https://www.space-track.org/ajaxauth/login",
self.login_url,
data={
"identity": username,
"password": password,
@@ -69,7 +88,7 @@ class SpaceTrackTLECollector(BaseCollector):
timeout=120.0,
follow_redirects=True,
) as alt_client:
await alt_client.get("https://www.space-track.org/")
await alt_client.get(f"{self.site_root}/")
form_data = {
"username": username,
@@ -77,7 +96,7 @@ class SpaceTrackTLECollector(BaseCollector):
"query": "class/gp/NORAD_CAT_ID/25544/format/json",
}
alt_login = await alt_client.post(
"https://www.space-track.org/ajaxauth/login",
self.login_url,
data={
"identity": username,
"password": password,
@@ -86,9 +105,7 @@ class SpaceTrackTLECollector(BaseCollector):
print(f"SPACETRACK: Alt login status: {alt_login.status_code}")
if alt_login.status_code == 200:
tle_response = await alt_client.get(
"https://www.space-track.org/basicspacedata/query/class/gp/NORAD_CAT_ID/25544/format/json"
)
tle_response = await alt_client.get(self.probe_url)
if tle_response.status_code == 200:
data = tle_response.json()
print(f"SPACETRACK: Received {len(data)} records via alt method")
@@ -98,9 +115,7 @@ class SpaceTrackTLECollector(BaseCollector):
print(f"SPACETRACK: Login failed, using sample data")
return self._get_sample_data()
tle_response = await client.get(
"https://www.space-track.org/basicspacedata/query/class/gp/NORAD_CAT_ID/25544/format/json"
)
tle_response = await client.get(self.probe_url)
print(f"SPACETRACK: TLE query status: {tle_response.status_code}")
if tle_response.status_code != 200:
@@ -127,11 +142,11 @@ class SpaceTrackTLECollector(BaseCollector):
},
) as client:
# First, visit the main page to get any cookies
await client.get("https://www.space-track.org/")
await client.get(f"{self.site_root}/")
# Login to get session cookie
login_response = await client.post(
"https://www.space-track.org/ajaxauth/login",
self.login_url,
data={
"identity": username,
"password": password,
@@ -146,13 +161,7 @@ class SpaceTrackTLECollector(BaseCollector):
return self._get_sample_data()
# Query for TLE data (get first 1000 satellites)
tle_response = await client.get(
"https://www.space-track.org/basicspacedata/query"
"/class/gp"
"/orderby/EPOCH%20desc"
"/limit/1000"
"/format/json"
)
tle_response = await client.get(self.query_url)
print(f"SPACETRACK: TLE query status: {tle_response.status_code}")
if tle_response.status_code != 200:
@@ -11,6 +11,7 @@ from datetime import UTC, datetime
from bs4 import BeautifulSoup
import httpx
from app.core.data_sources import get_data_sources_config
from app.services.collectors.base import BaseCollector
@@ -24,15 +25,17 @@ class TeleGeographyCableCollector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch submarine cable data from Wayback Machine"""
config = get_data_sources_config()
# Try multiple data sources
sources = [
# Wayback Machine archive of TeleGeography
"https://web.archive.org/web/2024/https://www.submarinecablemap.com/api/v3/cable",
# Alternative: Try scraping the page
"https://www.submarinecablemap.com",
self._resolved_url or "",
str(config.get_yaml_value("telegeography.archived_cable_url") or ""),
str(config.get_yaml_value("telegeography.live_map_url") or ""),
]
for url in sources:
if not url:
continue
try:
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
response = await client.get(url)
@@ -161,7 +164,7 @@ class TeleGeographyLandingPointCollector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch landing point data from GitHub mirror"""
url = "https://raw.githubusercontent.com/lintaojlu/submarine_cable_information/main/landing_point.json"
url = self._resolved_url or ""
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(url)
@@ -225,7 +228,7 @@ class TeleGeographyCableSystemCollector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch cable system data"""
url = "https://raw.githubusercontent.com/lintaojlu/submarine_cable_information/main/cable.json"
url = self._resolved_url or ""
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(url)
+5 -2
View File
@@ -10,6 +10,7 @@ from typing import Dict, Any, List
from bs4 import BeautifulSoup
import httpx
from app.core.data_sources import get_data_sources_config
from app.services.collectors.base import BaseCollector
@@ -22,7 +23,7 @@ class TOP500Collector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch TOP500 list data and enrich each row with detail-page metadata."""
url = "https://top500.org/lists/top500/list/2025/11/"
url = self._resolved_url or ""
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
response = await client.get(url)
@@ -48,11 +49,13 @@ class TOP500Collector(BaseCollector):
return await asyncio.gather(*(enrich(entry) for entry in entries))
def _extract_system_fields(self, system_cell) -> Dict[str, str]:
config = get_data_sources_config()
top500_base_url = config.get_yaml_value("top500.base_url") or "https://top500.org"
link = system_cell.find("a")
system_name = link.get_text(" ", strip=True) if link else system_cell.get_text(" ", strip=True)
detail_url = ""
if link and link.get("href"):
detail_url = f"https://top500.org{link.get('href')}"
detail_url = f"{str(top500_base_url).rstrip('/')}{link.get('href')}"
manufacturer = ""
if link and link.next_sibling:
@@ -0,0 +1,279 @@
"""BarentsWatch AIS collector for vessel tracking."""
from datetime import UTC, datetime
from typing import Any
import httpx
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.time import to_iso8601_utc
from app.core.websocket.broadcaster import broadcaster
from app.services.barentswatch import (
BARENTSWATCH_LATEST_URL,
fetch_barentswatch_access_token,
resolve_barentswatch_config,
)
from app.services.collectors.base import BaseCollector
from app.services.vessel_ais_aggregation import (
BARENTSWATCH_DELIVERY_MODE,
BARENTSWATCH_TRANSPORT,
record_vessel_ais_observation,
update_ais_source_health,
)
from app.services.vessel_types import normalize_vessel_type_name
class VesselAISCollector(BaseCollector):
"""Collect latest AIS positions and append them to vessel tables."""
name = "barentswatch_vessels"
priority = "P1"
module = "L4"
frequency_hours = 1
data_type = "vessel_ais"
@property
def base_url(self) -> str:
return self._resolved_url or BARENTSWATCH_LATEST_URL
async def _get_access_token(self, client: httpx.AsyncClient) -> str | None:
config = await resolve_barentswatch_config(self._db_session)
return await fetch_barentswatch_access_token(client, config)
async def fetch(self) -> list[dict[str, Any]]:
async with httpx.AsyncClient(timeout=60.0) as client:
headers: dict[str, str] = {}
token = await self._get_access_token(client)
if token:
headers["Authorization"] = f"Bearer {token}"
response = await client.get(self.base_url, headers=headers)
if response.status_code == 401 and not token:
return self._get_sample_data()
response.raise_for_status()
payload = response.json()
if isinstance(payload, list):
return [item for item in payload if isinstance(item, dict)]
if isinstance(payload, dict):
for key in ("features", "data", "items", "vessels"):
value = payload.get(key)
if isinstance(value, list):
if key == "features":
return [
{
**(item.get("properties") or {}),
"geometry": item.get("geometry"),
}
for item in value
if isinstance(item, dict)
]
return [item for item in value if isinstance(item, dict)]
return self._get_sample_data()
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
transformed = []
for item in raw_data:
record = self._normalize_record(item)
if record:
transformed.append(record)
return transformed
async def _save_data(
self,
db: AsyncSession,
data: list[dict[str, Any]],
task_id: int | None = None,
snapshot_id: int | None = None,
) -> int:
now = datetime.now(UTC)
records_added = 0
for index, item in enumerate(data):
observed_at = item.get("received_at") or now
await record_vessel_ais_observation(
db,
source=self.name,
normalized_payload=item,
raw_payload=item,
delivery_mode=BARENTSWATCH_DELIVERY_MODE,
transport=BARENTSWATCH_TRANSPORT,
observed_at=observed_at,
collected_at=now,
)
records_added += 1
if (index + 1) % 1000 == 0:
await self.update_progress(index + 1, commit=True)
latest_observed_at = max(
(item.get("received_at") for item in data if item.get("received_at")),
default=now,
)
await update_ais_source_health(
db,
source=self.name,
connection_state="connected",
observed_count=len(data),
last_seen_at=latest_observed_at,
last_success_at=now if data else None,
lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
)
await db.commit()
await self._broadcast_vessel_snapshot(data)
await self.update_progress(records_added, force=True)
return records_added
async def _broadcast_vessel_snapshot(self, data: list[dict[str, Any]]) -> None:
"""Push REST collector updates through the same realtime vessel channel."""
if not data:
return
batch_size = 500
for offset in range(0, len(data), batch_size):
batch = data[offset : offset + batch_size]
await broadcaster.broadcast_custom(
"vessels",
{
"action": "upsert",
"source": self.name,
"created": True,
"vessels": [
{
"mmsi": item.get("mmsi"),
"mmsi_display": str(item.get("mmsi")) if item.get("mmsi") is not None else None,
"name": item.get("name"),
"callsign": item.get("callsign"),
"lat": item.get("lat"),
"lon": item.get("lon"),
"sog": item.get("sog"),
"cog": item.get("cog"),
"heading": item.get("heading"),
"nav_status": item.get("nav_status"),
"vessel_type": item.get("vessel_type"),
"vessel_type_name": item.get("vessel_type_name"),
"received_at": to_iso8601_utc(item.get("received_at")),
}
for item in batch
],
},
)
def _normalize_record(self, item: dict[str, Any]) -> dict[str, Any] | None:
mmsi = _as_int(_pick(item, "mmsi", "MMSI", "Mmsi"))
lat = _as_float(_pick(item, "lat", "latitude", "Latitude"))
lon = _as_float(_pick(item, "lon", "lng", "longitude", "Longitude"))
geometry = item.get("geometry")
coordinates = geometry.get("coordinates") if isinstance(geometry, dict) else None
if (lat is None or lon is None) and isinstance(coordinates, list) and len(coordinates) >= 2:
lon = _as_float(coordinates[0])
lat = _as_float(coordinates[1])
if mmsi is None or lat is None or lon is None:
return None
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
return None
vessel_type = _as_int(_pick(item, "vessel_type", "shipType", "ship_type", "ShipType"))
vessel_type_name = (
_pick(item, "vessel_type_name", "shipTypeName", "ship_type_name", "VesselTypeName")
or normalize_vessel_type_name(vessel_type)
)
received_at = _parse_datetime(_pick(item, "received_at", "timestamp", "time", "msgtime"))
return {
"mmsi": mmsi,
"name": _pick(item, "name", "shipName", "ship_name", "Name"),
"callsign": _pick(item, "callsign", "callSign", "CallSign"),
"vessel_type": vessel_type,
"vessel_type_name": vessel_type_name,
"flag": _pick(item, "flag", "country", "Flag"),
"length": _as_float(_pick(item, "length", "shipLength", "Length")),
"width": _as_float(_pick(item, "width", "shipWidth", "Width")),
"draught": _as_float(_pick(item, "draught", "draft", "Draught")),
"imo": _as_int(_pick(item, "imo", "IMO", "imoNumber")),
"lat": lat,
"lon": lon,
"sog": _as_float(_pick(item, "sog", "speedOverGround", "SOG")),
"cog": _as_float(_pick(item, "cog", "courseOverGround", "COG")),
"heading": _as_int(_pick(item, "heading", "trueHeading", "Heading")),
"nav_status": _as_int(_pick(item, "nav_status", "navStatus", "NavigationalStatus")),
"received_at": received_at,
}
def _get_sample_data(self) -> list[dict[str, Any]]:
return [
{
"mmsi": 257123000,
"name": "OSLO TRADER",
"lat": 59.91,
"lon": 10.73,
"sog": 12.4,
"cog": 214,
"heading": 215,
"nav_status": 0,
"vessel_type": 70,
"vessel_type_name": "Cargo",
"flag": "NO",
"length": 185,
},
{
"mmsi": 257456000,
"name": "NORDIC FJORD",
"lat": 60.39,
"lon": 5.32,
"sog": 0.2,
"cog": 82,
"heading": 80,
"nav_status": 1,
"vessel_type": 60,
"vessel_type_name": "Passenger",
"flag": "NO",
"length": 126,
},
]
def _pick(item: dict[str, Any], *keys: str) -> Any:
for key in keys:
if key in item and item[key] not in (None, ""):
return item[key]
return None
def _as_float(value: Any) -> float | None:
try:
if value in (None, ""):
return None
return float(value)
except (TypeError, ValueError):
return None
def _as_int(value: Any) -> int | None:
try:
if value in (None, ""):
return None
return int(float(value))
except (TypeError, ValueError):
return None
def _parse_datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if not value:
return None
if isinstance(value, (int, float)):
timestamp = float(value)
if timestamp > 10_000_000_000:
timestamp /= 1000
return datetime.fromtimestamp(timestamp, UTC)
if isinstance(value, str):
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
except ValueError:
return None
return None
Loaded 100 of 404 files, more files were not shown because too many files have changed in this diff. Show more