diff --git a/.claude/commands/cleanup.md b/.claude/commands/cleanup.md index 68c17323..97e0d0b1 100644 --- a/.claude/commands/cleanup.md +++ b/.claude/commands/cleanup.md @@ -12,6 +12,19 @@ allowed-tools: ["Read", "Edit", "Bash", "Grep", "Glob"] 若 `$ARGUMENTS` 非空，则只检查指定文件/目录；否则检查所有未提交修改（`git diff HEAD`）。 +## 节省上下文规则 + +优先用确定性的 CLI 检查缩小范围，不要一上来把完整文件或大 diff 读入上下文： + +```bash +git diff --name-only HEAD +git diff --unified=0 HEAD -- +git diff --check +rg -n "TODO|FIXME|console\.log|debugger|print\(" +``` + +只有 focused diff 不足以安全判断或修改时，才读取完整文件。 + ## 审查清单 按优先级检查以下问题（只报告在本次 diff 中**新增或修改**的代码里存在的问题）： @@ -59,8 +72,13 @@ git diff HEAD --name-only ### Step 2 — 逐文件阅读并分析 -- 用 Read 工具读取完整文件（不只读 diff） -- 对照审查清单，记录每个问题：文件名、行号、问题类型、建议修复方式 +先从 focused diff 开始： + +```bash +git diff --unified=0 HEAD -- +``` + +用 `rg`、`git diff --check`、编译器或 linter 输出确认确定性问题。只有需要上下文时才用 Read 读取完整文件。对照审查清单，记录每个问题：文件名、行号、问题类型、建议修复方式。 ### Step 3 — 报告问题清单 @@ -95,6 +113,7 @@ git diff HEAD --name-only - 只改在审查清单中发现的问题，不做额外优化 - 每次 Edit 只修改确实有问题的行，保持 diff 最小 - 改完后用 `grep` 验证旧的坏代码已消失 +- 优先做精确补丁；只有仓库已有对应格式化流程时，才运行格式化工具 ### Step 5 — 输出总结 diff --git a/.claude/commands/docs.md b/.claude/commands/docs.md new file mode 100644 index 00000000..e67c2b97 --- /dev/null +++ b/.claude/commands/docs.md @@ -0,0 +1,93 @@ +--- +description: Create or update repository documentation from current code changes +argument-hint: Optional: topic to document, or leave empty to infer from git diff +allowed-tools: ["Read", "Edit", "Write", "Bash", "Glob", "Grep"] +--- + +# /docs — Documentation Workflow + +## Goal + +Create or update documentation that explains why a change exists, how it behaves, and what maintainers need to know. Keep this command generic. Repository-specific coverage rules live in the repository and must be loaded separately. + +## Repository Rules + +Before deciding scope, check whether the repository has a documentation rules file: + +```bash +test -f docs/documentation-coverage-rules.md && sed -n '1,240p' docs/documentation-coverage-rules.md +``` + +If it exists, apply it as the project-specific coverage checklist. If it does not exist, continue with the generic workflow below. + +## Workflow + +### Step 1 — Understand The Change + +```bash +git diff HEAD --stat +git diff HEAD --name-only +git log --oneline -10 +rg --files docs +``` + +If `$ARGUMENTS` specifies a topic, focus on that topic. Otherwise infer the documentation topic from the changed files. Do not read the full repository diff by default; inspect focused files only: + +```bash +git diff HEAD -- +rg -n "class |def |function |export |router|@router|interface |type " +``` + +### Step 2 — Decide Scope + +- Prefer updating an existing relevant document over creating a duplicate. +- Use one document for one coherent topic. +- Split documents only when the change crosses meaningful domains. +- Keep filenames lowercase and hyphenated. +- Apply the repository-specific rules file before writing. + +For ambiguous or large documentation changes, briefly state the intended doc plan before editing. For clear small changes, proceed directly. + +### Step 3 — Write + +Explain: + +- Background/problem: what was wrong or missing before. +- Core design decisions and rationale. +- Operational or user-facing impact. +- Relevant code paths, only when useful for future maintainers. + +Style: + +- Follow the repository’s existing language and heading conventions. +- Use fenced code blocks with language tags. +- Prefer tables for comparisons or parameter lists. +- Keep snippets concise and relevant. + +### Step 4 — Verify + +- Read the completed docs once for clarity and stale statements. +- Verify referenced paths exist with `test -e` or `rg --files`. +- Run applicable checks from `docs/documentation-coverage-rules.md`. +- Check Markdown links use readable user-facing titles unless repository rules allow otherwise. + +### Step 5 — Report + +Summarize changed docs and verification: + +```md +Updated: +- path/to/doc.md — what changed + +Verified: +- checks that passed +- checks that could not be run, if any +``` + +## Hard Constraints + +- Do not leave placeholder docs. +- Do not duplicate bilingual files byte-for-byte. +- Do not reference PR numbers, issue numbers, or the current conversation unless explicitly requested. +- Do not write changelog-style lists without the reasoning and tradeoffs behind the change. +- Keep docs maintainable and concise. diff --git a/.claude/commands/goal-driven.md b/.claude/commands/goal-driven.md new file mode 100644 index 00000000..4757a94c --- /dev/null +++ b/.claude/commands/goal-driven.md @@ -0,0 +1,93 @@ +--- +description: 用 goal-driven 方法推动一个复杂任务持续执行，直到明确成功标准被满足 +argument-hint: 建议填写任务目标；若同时给出成功标准更好 +allowed-tools: ["Read", "Edit", "Bash", "Grep", "Glob"] +--- + +# /goal-driven — 目标驱动执行模式 + +使用 `lidangzzz/goal-driven` 的核心思想来推进复杂任务：先固定目标与成功标准，再持续执行和反复验收，直到标准真正满足。 + +适用场景： + +- 长周期实现任务 +- 高复杂度工程任务 +- 可被明确验收的研究、实现、迁移、验证类工作 + +不适用场景： + +- 纯脑暴 +- 无法定义成功标准的模糊任务 +- 很小的一次性修改 + +## 输入要求 + +若 `$ARGUMENTS` 只包含目标，没有成功标准，先补全一版可执行的成功标准再开始。 + +启动时先输出： + +```md +Goal +- ... + +Criteria for success +- ... + +Plan +1. ... +2. ... +3. ... + +Verification +- ... +``` + +## 执行规则 + +1. 先把任务固化为两个核心块： + - `Goal` + - `Criteria for success` + +2. 成功标准必须尽量客观，可验证，可落地。 + 优先写成： + - 需要交付什么 + - 需要通过哪些测试或验证 + - 如何判断结果真的完成 + +3. 进入持续执行循环： + - 完成一个阶段 + - 检查当前结果是否满足成功标准 + - 若未满足，明确剩余差距并继续推进 + +4. 任何“完成了”“差不多了”“已实现”之类的结论，都必须经过验证，不能直接接受。 + +5. 如果验证失败： + - 明确指出哪条成功标准没满足 + - 继续工作，不要把阶段性进展误判为完成 + +6. 只有在以下情况之一才能停止： + - 成功标准已满足 + - 用户明确要求停止 + +## 执行风格 + +- 重证据，轻口头判断 +- 优先使用确定性工具证据：`rg`、`git diff --stat`、`git diff -- `、测试、构建、lint、`curl`、数据库查询等能直接证明成功标准的方式 +- 不把大段命令输出粘进回复；保留在工具调用里，回复只总结关键证据 +- 重验收，轻自我感觉 +- 优先用测试、日志、产物、对比结果来证明完成 +- 对长期任务保持“未达标就继续”的节奏 + +## 简版模板 + +```md +Goal: [[[[[在此填写最终目标]]]]] + +Criteria for success: [[[[[在此填写成功标准]]]]] + +循环执行： +1. 推进任务 +2. 检查是否满足成功标准 +3. 若未满足，继续工作 +4. 直到满足标准或用户明确停止 +``` diff --git a/.claude/commands/release.md b/.claude/commands/release.md index 3708ccd2..43ff3813 100644 --- a/.claude/commands/release.md +++ b/.claude/commands/release.md @@ -28,6 +28,19 @@ allowed-tools: ["Read", "Edit", "Bash", "Glob", "Grep"] - `docs/CHANGELOG.md` - `docs/version-history.md` +## 节省上下文规则 + +发版判断应以确定性 CLI 证据为主，优先使用紧凑命令和定点读取： + +```bash +git status --short +git diff --stat HEAD +git diff --name-only HEAD +rg -n "version|^## |^Released:|当前开发版本|current" VERSION frontend/package.json pyproject.toml docs/CHANGELOG.md docs/version-history.md +``` + +除非需要判断某个代码变更是否属于本次发版，否则不要读取完整 diff。 + ## 执行步骤 ### Step 1 — 环境检查 @@ -45,7 +58,7 @@ cat VERSION # 读取当前版本 ### Step 2 — 确定发版类型与新版本号 - 若 `$ARGUMENTS` 提供了明确类型（`feature` / `bugfix`），直接使用 -- 否则根据当前 `git diff HEAD` 和 `git log` 推断 +- 否则根据 `git diff --stat HEAD`、`git diff --name-only HEAD`、必要的 focused diff 和 `git log` 推断 - 计算新版本号（例：`0.26.2` → bugfix → `0.26.3`） - **先输出发版计划供用户确认**： @@ -91,12 +104,13 @@ cat VERSION # 读取当前版本 针对本次变更范围做最小验证： -- Python 文件有修改：`python3 -m py_compile ` -- Frontend 文件有修改：运行项目标准检查（若无则跳过并说明） +- Python 文件有修改：先用 `git diff --name-only HEAD -- '*.py'` 列出，再运行 `python3 -m py_compile ` +- Frontend 文件有修改：先用 `git diff --name-only HEAD -- frontend` 判断范围，再运行项目标准检查（若无则跳过并说明） - 版本号一致性检查：用 grep 确认 VERSION、package.json、pyproject.toml 中的版本号完全一致 ```bash -grep -h "version" VERSION frontend/package.json pyproject.toml +cat VERSION +rg -n "\"version\":|^version =|version = " frontend/package.json pyproject.toml uv.lock ``` ### Step 7 — 提交前预览 diff --git a/.codex/config.toml b/.codex/config.toml new file mode 100644 index 00000000..ca1b2693 --- /dev/null +++ b/.codex/config.toml @@ -0,0 +1,3 @@ +approval_policy = "never" + +sandbox_mode = "danger-full-access" diff --git a/.codex/skills/cleanup/SKILL.md b/.codex/skills/cleanup/SKILL.md index 07359c6c..778c16e8 100644 --- a/.codex/skills/cleanup/SKILL.md +++ b/.codex/skills/cleanup/SKILL.md @@ -21,6 +21,19 @@ If the user specifies a file or directory, check only that. Otherwise check all Only report issues present in **newly added or modified** lines of this diff — do not audit unchanged code. +## Token-Saving Rule + +Prefer deterministic CLI checks before reading files into model context: + +```bash +git diff --name-only HEAD +git diff --unified=0 HEAD -- +git diff --check +rg -n "TODO|FIXME|console\.log|debugger|print\(" +``` + +Read full files only when the focused diff does not provide enough surrounding context to make a safe edit. + ## Checklist ### 1. Duplicate Logic @@ -64,7 +77,13 @@ Filter to the user-specified path if one was provided. ### Step 2 — Read and analyze each file -Read the full file (not just the diff) with the Read tool. For each file, record every issue found: filename, line number, category, and suggested fix. +Start with focused diffs: + +```bash +git diff --unified=0 HEAD -- +``` + +Use `rg`, `git diff --check`, and compiler/linter output for deterministic findings. Read the full file only for files that need surrounding context. For each issue found, record filename, line number, category, and suggested fix. ### Step 3 — Report findings before touching anything @@ -99,6 +118,7 @@ Principles: - Only fix issues identified in the checklist — no extra improvements - Keep each Edit as small as possible - After fixing, verify the old bad pattern is gone with grep +- Prefer `apply_patch` for targeted edits; use formatters only when the repository already uses them for the touched file type ### Step 5 — Summary diff --git a/.codex/skills/docs/SKILL.md b/.codex/skills/docs/SKILL.md new file mode 100644 index 00000000..9c399c85 --- /dev/null +++ b/.codex/skills/docs/SKILL.md @@ -0,0 +1,82 @@ +--- +name: docs +description: Create or update repository documentation from current code changes. Use when the user asks to write docs, update docs, summarize implementation changes into docs, or check documentation coverage. Load repository-specific coverage rules from docs/documentation-coverage-rules.md when present. +--- + +# Docs + +Use this skill when the task is documentation work: creating, updating, checking, or summarizing docs for code or behavior changes. + +## Goal + +Write documentation that explains why a change exists, how it behaves, and what maintainers need to know. Keep the skill generic; repository-specific rules belong in the repository, not in this skill. + +## Repository Rules + +Before deciding scope, check whether the repository has a documentation rules file: + +```bash +test -f docs/documentation-coverage-rules.md && sed -n '1,240p' docs/documentation-coverage-rules.md +``` + +If it exists, apply it as the project-specific coverage checklist. If it does not exist, continue with the generic workflow below. + +## Workflow + +1. Gather focused context: + +```bash +git diff HEAD --stat +git diff HEAD --name-only +git log --oneline -10 +rg --files docs +``` + +If the user gives a topic, focus on that topic. Otherwise infer the doc topic from changed files. Avoid reading large full diffs by default; inspect focused files and symbols: + +```bash +git diff HEAD -- +rg -n "class |def |function |export |router|@router|interface |type " +``` + +2. Decide scope: + +- Prefer updating an existing relevant doc over creating a duplicate. +- Use one document for one coherent topic. +- Split documents only when changes cross meaningful domains. +- Keep filenames lowercase and hyphenated. + +3. Write the doc: + +- Explain background/problem, design decisions, constraints, and operational impact. +- Keep code snippets short and directly relevant. +- List related files only when they help future maintainers navigate. +- Use the repository’s existing language, heading style, and naming conventions. + +4. Verify: + +- Read the completed doc once for clarity and stale statements. +- Verify important referenced paths exist with `test -e` or `rg --files`. +- Run repository-specific doc checks from `docs/documentation-coverage-rules.md` when present. +- For Markdown links, check that user-facing titles are readable and not raw filenames unless the repository rules allow it. + +## Hard Constraints + +- Do not leave placeholder docs or copied source text pretending to be documentation. +- Do not duplicate bilingual files byte-for-byte. +- Do not reference PR numbers, issue numbers, or the current conversation unless explicitly requested. +- Do not write changelog-style lists without the reasoning, constraints, and tradeoffs behind the change. +- Keep docs concise enough to maintain. + +## Recommended Output + +After editing, summarize: + +```md +Updated: +- path/to/doc.md — what changed + +Verified: +- checks that passed +- checks that could not be run, if any +``` diff --git a/.codex/skills/goal-driven/SKILL.md b/.codex/skills/goal-driven/SKILL.md new file mode 100755 index 00000000..4f410deb --- /dev/null +++ b/.codex/skills/goal-driven/SKILL.md @@ -0,0 +1,103 @@ +--- +name: goal-driven +description: Run a goal-driven execution loop for very large, long-horizon, rigorously verifiable tasks. Use when the user explicitly wants the lidangzzz/goal-driven method, a master-agent plus worker-agent style workflow, or a persistent loop that keeps working until concrete success criteria are satisfied. +--- + +# Goal-Driven + +Use this skill when the user wants a strict goal-driven workflow for a hard task with: + +- one clear end goal +- explicit success criteria +- repeated verification against those criteria +- continued execution until the criteria are actually met + +This skill is adapted from `lidangzzz/goal-driven`, but trimmed for local skill use to avoid bloating context. + +## When To Use + +Use it for tasks like: + +- compilers, interpreters, theorem-like proof work, deep refactors +- long-running system design or implementation work +- problems that are expensive and complex, but still objectively testable + +Do not use it for: + +- vague brainstorming without a success condition +- short one-shot edits +- tasks where "done" cannot be evaluated in a meaningful way + +## Core Model + +The workflow has two roles: + +1. Master role + Defines the goal, defines the success criteria, audits progress, and decides whether the work is actually complete. + +2. Worker role + Keeps advancing the task toward the goal. If a result is partial, stalled, or unverifiable, the worker continues. + +In Codex, only use actual subagents when the user explicitly asks for delegation or subagent work and the platform supports it. Otherwise emulate the same loop locally: keep working, checkpointing, and re-verifying until the criteria are satisfied. + +## Workflow + +1. Normalize the task into two blocks: + - `Goal` + - `Criteria for success` + +2. Make the criteria concrete and testable. + Good criteria usually include: + - required outputs + - required validations or tests + - edge cases or coverage thresholds + - what evidence proves completion + +3. Break the work into milestones that can each produce evidence. + +4. Execute the next milestone. + If subagents are explicitly allowed, the master may delegate bounded worker tasks. + If not, do the work locally but keep the master/worker mindset. + +5. Whenever work pauses, stalls, or appears complete, audit against the criteria directly. + Check artifacts, tests, logs, diffs, metrics, or other real evidence. + +6. If the criteria are not met, continue with a specific delta: + - what is still missing + - what evidence failed + - what the next worker pass must improve + +7. Stop only when the criteria are met, or when the user explicitly stops the process. + +## Operating Rules + +- Prefer objective checks over self-reported completion. +- Prefer deterministic tool evidence over long model summaries: use `rg`, `git diff --stat`, targeted `git diff -- `, tests, builds, linters, `curl`, or database queries when they can prove a criterion. +- Do not paste large command output into the conversation; summarize the evidence and keep raw output in tool calls. +- Do not confuse progress with completion. +- If the worker says "done", verify it. +- If verification fails, continue from the gap instead of restarting blindly. +- Keep the goal stable unless the user changes it. +- Tighten fuzzy criteria before sinking large amounts of effort. + +## Recommended Response Shape + +When starting a goal-driven task, structure the kickoff like this: + +```md +Goal +- ... + +Criteria for success +- ... + +Current plan +1. ... +2. ... +3. ... + +Verification +- What evidence will prove completion +``` + +For a reusable prompt template, read [references/prompt-template.md](references/prompt-template.md). diff --git a/.codex/skills/goal-driven/agents/openai.yaml b/.codex/skills/goal-driven/agents/openai.yaml new file mode 100644 index 00000000..68bc76e1 --- /dev/null +++ b/.codex/skills/goal-driven/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "Goal-Driven" + short_description: "Drive complex work until explicit success criteria are met." + default_prompt: "Use $goal-driven to turn this task into a concrete goal, explicit success criteria, and a verification-driven execution loop." + +policy: + allow_implicit_invocation: true diff --git a/.codex/skills/goal-driven/references/prompt-template.md b/.codex/skills/goal-driven/references/prompt-template.md new file mode 100755 index 00000000..3d860519 --- /dev/null +++ b/.codex/skills/goal-driven/references/prompt-template.md @@ -0,0 +1,38 @@ +# Goal-Driven Prompt Template + +Use this when you want a reusable kickoff prompt for a master/worker execution loop. + +```md +# Goal-Driven System + +Goal: [[[[[DEFINE THE FINAL GOAL HERE]]]]] + +Criteria for success: [[[[[DEFINE THE SUCCESS CRITERIA HERE]]]]] + +You are the master agent. + +Your job is to: +1. Keep the goal and criteria fixed. +2. Start worker execution toward the goal. +3. Audit any claimed progress against the criteria. +4. If the criteria are not met, continue the work with a precise next delta. +5. Stop only when the criteria are satisfied or the user explicitly stops the process. + +Worker requirements: +1. Break the task into subproblems. +2. Keep producing concrete progress toward the goal. +3. Report evidence, not just claims. +4. Continue until the criteria are satisfied. + +Master audit loop: +1. Check whether the worker is still making progress. +2. If the worker stalls or claims completion, verify against the criteria. +3. If verification fails, resume work from the remaining gap. +4. Repeat until the criteria are met. +``` + +## Notes + +- Stronger criteria produce better results than stronger rhetoric. +- Prefer measurable checks such as tests, parity checks, generated artifacts, benchmarks, or reviewable outputs. +- If the environment does not support subagents, emulate the same loop locally. diff --git a/.codex/skills/release/SKILL.md b/.codex/skills/release/SKILL.md index cd02b3fd..e5eff680 100644 --- a/.codex/skills/release/SKILL.md +++ b/.codex/skills/release/SKILL.md @@ -18,7 +18,7 @@ Do not use this skill for ordinary commits that are not being released. ## Versioning Rules -- `feature` -> bump `+0.1.0` +- `feature` -> bump minor and reset patch to `0` (`x.y.z` → `x.(y+1).0`; for example `0.41.2` → `0.42.0`) - `bugfix` -> bump `+0.0.1` - `docs`, `maintenance`, and `refactor` do not bump by default unless the user explicitly wants a release @@ -35,6 +35,19 @@ Use `git rev-parse --show-toplevel` to get the repo root. All paths are relative - `docs/CHANGELOG.md` - `docs/version-history.md` +## Token-Saving Rule + +Release work should be driven by deterministic CLI evidence. Prefer compact commands and targeted file reads: + +```bash +git status --short +git diff --stat HEAD +git diff --name-only HEAD +rg -n "version|^## |^Released:|current" VERSION frontend/package.json pyproject.toml docs/CHANGELOG.md docs/version-history.md +``` + +Do not inspect full diffs unless deciding whether changed code belongs in the release. + ## Workflow ### Step 1 — Environment check @@ -52,8 +65,10 @@ If unrelated uncommitted changes exist, list them and ask the user whether to in ### Step 2 — Determine release type and next version - If the user provided an explicit type (`feature` / `bugfix`), use it -- Otherwise infer from `git diff HEAD` and recent `git log` -- Compute the next version (e.g. `0.26.2` → bugfix → `0.26.3`) +- Otherwise infer from `git diff --stat HEAD`, `git diff --name-only HEAD`, focused diffs for changed code, and recent `git log` +- Compute the next version: + - `feature`: increment minor and reset patch to `0` (e.g. `0.41.2` → `0.42.0`) + - `bugfix`: increment patch only (e.g. `0.26.2` → `0.26.3`) - **Show the release plan before making any changes:** ``` @@ -104,12 +119,13 @@ Get today's date with `date +%Y-%m-%d`. Run the smallest relevant validation for the changes in scope: -- Python files changed: `python3 -m py_compile ` -- Frontend files changed: run the project-standard check if available; otherwise skip and say so +- Python files changed: list changed Python files with `git diff --name-only HEAD -- '*.py'`, then run `python3 -m py_compile ` +- Frontend files changed: list changed frontend files with `git diff --name-only HEAD -- frontend`, then run the project-standard check if available; otherwise skip and say so - Version consistency: confirm VERSION, package.json, pyproject.toml, and uv.lock all show the same version ```bash -grep -h "version" VERSION frontend/package.json pyproject.toml +cat VERSION +rg -n "\"version\":|^version =|version = " frontend/package.json pyproject.toml uv.lock ``` ### Step 7 — Pre-commit preview diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..d683496a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +** + +!pyproject.toml +!uv.lock +!aiprovider/ +!aiprovider/** + +aiprovider/.env +aiprovider/.env.* +!aiprovider/.env.example +**/__pycache__/ +**/*.pyc +**/*.pyo diff --git a/README.md b/README.md index a91acf37..141629a4 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,120 @@ bun run build 启动服务后访问: `http://localhost:8000/docs` +## WSL / Windows 局域网访问 + +如果服务运行在 WSL 中，而你希望： + +- Windows 本机浏览器访问开发服务 +- 同一局域网内的手机或其他电脑访问开发服务 + +推荐按下面顺序排查和配置。 + +### 1. 在 WSL 中启动服务 + +```bash +./planet.sh start --allow-lan +``` + +这会让前端监听 `0.0.0.0:3000`，后端监听 `0.0.0.0:8000`。 + +### 2. 先确认 WSL 内部服务正常 + +在 WSL 中执行： + +```bash +curl http://localhost:3000 +curl http://localhost:8000/health +ss -ltnp | grep -E ':3000|:8000' +``` + +预期： + +- `3000` 返回前端 HTML +- `8000/health` 返回健康检查 JSON +- `ss` 中能看到 `0.0.0.0:3000` 和 `0.0.0.0:8000` + +如果这一步不通，先不要继续做 Windows 转发。 + +### 3. 在 Windows 本机验证 localhost 直通 + +在 Windows PowerShell 中执行： + +```powershell +curl http://localhost:3000 +curl http://localhost:8000/health +``` + +在常见的 WSL2 开发环境下，Windows 通常可以直接通过 `localhost` 访问 WSL 中的服务。 + +### 4. 如果需要让局域网设备访问，再做 Windows 端口转发 + +注意：下面的命令必须在“以管理员身份运行”的 PowerShell 中执行。 + +先把 Windows 对外网卡上的 `3000` / `8000` 转发到 Windows 本机 `127.0.0.1`： + +```powershell +netsh interface portproxy delete v4tov4 listenaddress=0.0.0.0 listenport=3000 +netsh interface portproxy delete v4tov4 listenaddress=0.0.0.0 listenport=8000 + +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 +``` + +再放行 Windows 防火墙： + +```powershell +New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 +New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 +``` + +检查转发规则是否生效： + +```powershell +netsh interface portproxy show all +``` + +预期能看到： + +- `0.0.0.0:3000 -> 127.0.0.1:3000` +- `0.0.0.0:8000 -> 127.0.0.1:8000` + +### 5. 查 Windows 局域网 IP，并让其他设备访问 + +在 Windows PowerShell 中执行： + +```powershell +ipconfig +``` + +找到当前联网网卡的 IPv4 地址，例如 `192.168.8.228`。 + +局域网其他设备可访问： + +- `http://:3000/earth` +- `http://:3000/admin` + +例如： + +- `http://192.168.8.228:3000/earth` + +### 6. 常见现象与判断 + +- WSL 中 `curl localhost:3000` 能通，但 Windows 访问 `WSL 的局域网 IP:3000` 不通：这是正常现象之一，优先验证 Windows 的 `localhost:3000` +- Windows `localhost:3000` 能通，但局域网设备访问 `Windows 局域网 IP:3000` 不通：通常缺少 `portproxy` 或防火墙放行 +- `whoami /groups` 中 `S-1-5-32-544` 显示 `deny only`：说明当前 PowerShell 不是提权管理员窗口 + +### 7. 本项目一次性验证顺序 + +建议固定按这个顺序验证： + +1. WSL 中执行 `curl http://localhost:3000` +2. WSL 中执行 `curl http://localhost:8000/health` +3. Windows 中执行 `curl http://localhost:3000` +4. Windows 中执行 `curl http://localhost:8000/health` +5. 管理员 PowerShell 配置 `portproxy` 和防火墙 +6. 用手机或其他电脑访问 `http://:3000/earth` + ## 启动容错参数 `planet.sh` 现在为依赖安装、数据库、AI Provider 启动加入了有限次重试，并会在数据库与 `aiprovider` 启动后额外等待 Docker healthcheck。 diff --git a/TODO.md b/TODO.md index 92a3a977..717e8953 100644 --- a/TODO.md +++ b/TODO.md @@ -22,5 +22,24 @@ - [ ] 可选优化（非必做）：将 BGP incident/collector 标点改为 HTML marker（参考 worldmonitor 的 `htmlElementsData` 思路），实现近乎固定屏幕尺寸与更高密度可点击性 - [ ] 保持 Earth 当前这批纯个人偏好设置继续走本地持久化：`旋转模式`、HUD 面板显示/隐藏、`地形透明度` 暂不升级到后端系统设置，避免把设备级偏好过早做成全局配置 - [ ] 如果后续明确需要“账号级同步 Earth 偏好”，再单独设计 `Earth user preferences`：优先按用户维度而不是全局系统设置保存，并规划 `localStorage -> backend` 的平滑迁移策略 +- [ ] 为 Planet / Earth 补一个可用的日志查看系统：先明确前后端/AI Provider/采集任务的日志入口、最近日志聚合、筛选与 tail 能力，再决定是先做脚本级统一入口还是控制台内置日志面板 +- [ ] 重写控制台 UI，逐步抛弃 Ant Design，建立自有组件体系，并统一采用 `tabler.io` / Tabler Icons 作为控制台主图标库 - [ ] 把 Earth 态势新闻源从 [earth_news.py](/home/ray/dev/linkong/planet/backend/app/services/earth_news.py) 的硬编码列表抽成可配置目录，优先保持当前“实时聚合”链路不变，只先解决新闻源不可配置的问题 - [ ] 为 Earth 态势新闻设计后续采集器化方案：明确新闻数据模型、去重策略、区域映射、过期清理和 Earth/AI 复用方式，再决定何时把新闻从实时抓取升级成正式 collector +- [ ] AIS v3.1：修复船只聚合完整性，`/geo/vessels` 合并 raw observation 聚合结果与 legacy `vessel_position + vessel_static` 最新结果，确保 BarentsWatch-only 船只不会因为 AISStream 子集存在而消失，并增加 raw/legacy/final unique MMSI 诊断统计 +- [ ] AIS v3.2：把 AISStream 从收满 `max_messages` 后结束的批采集改成长连接 streaming service，持续写入 raw observations，通过内部 `/ws` 的 `vessels` channel 推送新船、位置和航向增量，Earth 前端按 MMSI upsert marker +- [ ] AIS v3.3：修正 AISStream 采集页面状态语义，使用 connecting/streaming/reconnecting/stopped 与 indeterminate 状态，展示运行时长、消息数、unique MMSI、message rate、最近消息和错误，不再用一次性 REST 进度条表示长连接 +- [ ] AIS v3.4：修复船只身份字段和名称聚合，MMSI/IMO/callsign 按字符串显示且不带千分位符；查询并列出所有仍以 MMSI 号码或 `MMSI ` 作为船名的记录，标注来源、最近观测、message types 和缺失原因，并把这批 fallback-name 船只纳入名称聚合修复集合 +- [ ] Earth Live Sync：建立统一态势实时同步链路，新增 `earth_summary` WS channel，任意采集器成功后广播轻量 summary invalidation，前端收到后重新拉 `/api/v1/visualization/geo/summary` 并更新 HUD；同时为 BGP 增加 `bgp` WS channel，使 BGP incidents/anomalies/collectors 在不刷新页面时也能 upsert 图层；卫星采集完成后触发 summary 刷新，必要时按 TLE 版本重新 hydrate 卫星数据 +- [ ] AIS v4：开放船只多源聚合策略配置，支持 source priority、字段级规则、freshness 窗口和高级保护开关；保存时校验未知字段、非法模式和危险动态字段锁定，并在聚合接口返回命中的配置版本 +- [ ] AIS v5：实现船舶资料 enrichment 与冲突治理，按 `mmsi + imo + name + callsign` 异步补充船型细分、AIS 大类、旗国、尺寸、建造年份、运营方和图片缓存；详情面板展示缓存资料和字段来源，不在实时 AIS 请求链路现场抓第三方页面 +- [ ] 为 Earth 地球表面增加一层与基础纹理对齐的材质/纹理 overlay，并在同层叠加国界轮廓参考线；要求国界线与底图稳定对齐，且 hover 到国家轮廓时能高亮当前国家，便于校准地表和增强交互 +- [ ] 把 Earth 新闻接入通用巡航队列：按新闻发生地和时间排序生成巡航目标，巡航聚焦到新闻事件时显示对应新闻卡片，并保持实现边界为“通用巡航层 + 新闻业务适配层”，不要再把新闻逻辑直接耦合回 `main.js` 状态机 +- [ ] 为未知位置的算力中心建立分层坐标补全链路：优先 `精确坐标 > 站点/园区命中 > 城市 > 州/省 > 国家内主要算力城市 > 国家质心`，并把每次回退的 `confidence / reason / precision` 明确写进统一 GeoJSON +- [ ] 为算力中心补一份可维护的本地位置注册表，例如 `canonical_name / aliases / operator / country / region / city / lat / lon / confidence / source_note`，避免把地点知识长期硬编码在 `visualization.py` +- [ ] 增强 `epoch_ai_gpu` 和相关算力采集器的源页面解析：即使公开 API 不给坐标，也继续尝试从详情页、HTML、内嵌 JSON、schema.org、OpenGraph、脚本变量和 PDF/新闻稿链接里抽地点线索 +- [ ] 为未知位置算力中心增加外部富化策略评估：可选接入公开知识源或搜索兜底，只抓“站点名/园区名/城市名”级别线索，不直接抓经纬度结论，并把结果作为候选证据而不是真值 +- [ ] 为算力中心建立 `operator / cluster name / facility alias` 归一化层，先解决 `xAI / Colossus / Memphis`、`OpenAI / Stargate`、`CoreWeave`、`Lambda`、`Crusoe` 这类同一对象多种写法导致的地点匹配失败 +- [ ] 为估算位置增加更细的视觉和产品表达：除了问号角标，还要支持 tooltip/详情中的“估算依据”“精度级别”“最后核验时间”，并允许在设置中单独开关“仅看精确位置” +- [ ] 为国家级估算点设计更合理的落点策略：优先落在“该国主要算力/数据中心城市候选集”而不是几何质心，必要时同国多节点做稳定散列分配，避免大量节点堆在荒漠或海上 +- [ ] 为未知位置算力中心建立人工校验工作流：支持导出待核验清单、记录人工确认结果，并把人工确认反哺到位置注册表，逐步减少问号点比例 diff --git a/VERSION b/VERSION index be386c9e..5c4503b7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.33.0 +0.49.0 diff --git a/aiprovider/Dockerfile b/aiprovider/Dockerfile index c598f6be..b3651008 100644 --- a/aiprovider/Dockerfile +++ b/aiprovider/Dockerfile @@ -1,6 +1,12 @@ -FROM python:3.14-slim +# syntax=docker/dockerfile:1.7 -COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +ARG PYTHON_IMAGE=python:3.14-slim +ARG UV_IMAGE=ghcr.io/astral-sh/uv:latest + +FROM ${UV_IMAGE} AS uv +FROM ${PYTHON_IMAGE} + +COPY --from=uv /uv /uvx /bin/ WORKDIR /app @@ -14,9 +20,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* COPY pyproject.toml uv.lock /app/ -RUN uv sync --frozen --no-dev +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev -COPY . /app +COPY aiprovider /app/aiprovider EXPOSE 8010 diff --git a/aiprovider/main.py b/aiprovider/main.py index afcc71af..b35670b9 100644 --- a/aiprovider/main.py +++ b/aiprovider/main.py @@ -37,8 +37,26 @@ def verify_service_token(x_provider_token: str | None = Header(default=None)) -> ) -def get_provider_service() -> ProviderService: - return ProviderService() +def get_provider_service( + x_ai_provider: str | None = Header(default=None), + x_ai_provider_api: str | None = Header(default=None), + x_ai_base_url: str | None = Header(default=None), + x_ai_api_key: str | None = Header(default=None), + x_ai_model: str | None = Header(default=None), + x_ai_max_tokens: str | None = Header(default=None), + x_ai_anthropic_version: str | None = Header(default=None), +) -> ProviderService: + overrides = { + "provider": x_ai_provider, + "provider_api": x_ai_provider_api, + "base_url": x_ai_base_url, + "api_key": x_ai_api_key, + "model": x_ai_model, + "anthropic_version": x_ai_anthropic_version, + } + if x_ai_max_tokens: + overrides["max_tokens"] = x_ai_max_tokens + return ProviderService({key: value for key, value in overrides.items() if value not in (None, "")}) @app.get("/health") diff --git a/aiprovider/provider_service.py b/aiprovider/provider_service.py index 282506da..19f75925 100644 --- a/aiprovider/provider_service.py +++ b/aiprovider/provider_service.py @@ -46,19 +46,22 @@ def _resolve_provider_api(provider: str, configured_api: str) -> str: class ProviderService: - def __init__(self) -> None: - self.provider = _normalize_provider(settings.AI_PROVIDER) + def __init__(self, overrides: dict[str, Any] | None = None) -> None: + overrides = overrides or {} + self.provider = _normalize_provider(overrides.get("provider") or settings.AI_PROVIDER) self.provider_api = _resolve_provider_api( self.provider, - _normalize_provider_api(settings.AI_PROVIDER_API), + _normalize_provider_api(overrides.get("provider_api") or settings.AI_PROVIDER_API), ) - self.base_url = settings.AI_BASE_URL.rstrip("/") - self.api_key = settings.AI_API_KEY - self.default_model = settings.AI_MODEL + self.base_url = str(overrides.get("base_url") or settings.AI_BASE_URL).rstrip("/") + self.api_key = str(overrides.get("api_key") or settings.AI_API_KEY) + self.default_model = str(overrides.get("model") or settings.AI_MODEL) self.timeout = settings.AI_TIMEOUT_SECONDS self.http_retry_attempts = max(settings.AI_HTTP_RETRY_ATTEMPTS, 1) - self.max_tokens = settings.AI_MAX_TOKENS - self.anthropic_version = settings.AI_ANTHROPIC_VERSION + self.max_tokens = int(overrides.get("max_tokens") or settings.AI_MAX_TOKENS) + self.anthropic_version = str( + overrides.get("anthropic_version") or settings.AI_ANTHROPIC_VERSION + ) self.system_prompt = settings.AI_ANALYSIS_SYSTEM_PROMPT def get_status(self) -> AIProviderStatusResponse: diff --git a/backend/Dockerfile b/backend/Dockerfile index 88bd01a6..3a190666 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,6 +1,10 @@ -FROM python:3.14-slim +ARG PYTHON_IMAGE=python:3.14-slim +ARG UV_IMAGE=ghcr.io/astral-sh/uv:latest -COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +FROM ${UV_IMAGE} AS uv +FROM ${PYTHON_IMAGE} + +COPY --from=uv /uv /uvx /bin/ WORKDIR /app diff --git a/backend/app/api/main.py b/backend/app/api/main.py index 54f4f4b6..26fb73b4 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -5,6 +5,7 @@ from app.api.v1 import ( users, datasource_config, datasources, + docs, tasks, dashboard, websocket, @@ -12,6 +13,7 @@ from app.api.v1 import ( settings, collected_data, visualization, + vessel_aggregation, bgp, news, system_control, @@ -28,12 +30,18 @@ api_router.include_router( ) api_router.include_router(datasources.router, prefix="/datasources", tags=["datasources"]) api_router.include_router(collected_data.router, prefix="/collected", tags=["collected-data"]) +api_router.include_router(docs.router, prefix="/docs", tags=["docs"]) api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"]) api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboard"]) api_router.include_router(alerts.router, prefix="/alerts", tags=["alerts"]) api_router.include_router(settings.router, prefix="/settings", tags=["settings"]) api_router.include_router(system_control.router, prefix="/system", tags=["system"]) api_router.include_router(visualization.router, prefix="/visualization", tags=["visualization"]) +api_router.include_router( + vessel_aggregation.router, + prefix="/vessel-aggregation", + tags=["vessel-aggregation"], +) api_router.include_router(bgp.router, prefix="/bgp", tags=["bgp"]) api_router.include_router(tv.router, prefix="/tv", tags=["tv"]) api_router.include_router(news.router, prefix="/news", tags=["news"]) diff --git a/backend/app/api/v1/auth.py b/backend/app/api/v1/auth.py index 804b244c..379e4853 100644 --- a/backend/app/api/v1/auth.py +++ b/backend/app/api/v1/auth.py @@ -28,7 +28,7 @@ async def login( ): result = await db.execute( text( - "SELECT id, username, email, password_hash, role, is_active FROM users WHERE username = :username" + "SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE username = :username" ), {"username": form_data.username}, ) @@ -46,6 +46,7 @@ async def login( user.password_hash = row[3] user.role = row[4] user.is_active = row[5] + user.gatekeeper_groups = row[6] or [] if not verify_password(form_data.password, user.password_hash): raise HTTPException( @@ -73,6 +74,7 @@ async def login( "id": user.id, "username": user.username, "role": user.role, + "gatekeeper_groups": user.gatekeeper_groups or [], }, } @@ -95,6 +97,7 @@ async def refresh_token( "id": current_user.id, "username": current_user.username, "role": current_user.role, + "gatekeeper_groups": current_user.gatekeeper_groups or [], }, } @@ -111,6 +114,7 @@ async def get_me(current_user: User = Depends(get_current_user)): "username": current_user.username, "email": current_user.email, "role": current_user.role, + "gatekeeper_groups": current_user.gatekeeper_groups or [], "is_active": current_user.is_active, "created_at": current_user.created_at, } diff --git a/backend/app/api/v1/bgp.py b/backend/app/api/v1/bgp.py index ff32dd83..fea3d59b 100644 --- a/backend/app/api/v1/bgp.py +++ b/backend/app/api/v1/bgp.py @@ -5,12 +5,18 @@ from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession +from pydantic import BaseModel + from app.core.security import get_current_user from app.db.session import get_db from app.models.bgp_anomaly import BGPAnomaly from app.models.bgp_incident import BGPIncident from app.models.bgp_observation import BGPObservation from app.models.user import User +from app.services.bgp_collector_locations import ( + collect_bgp_collector_location_candidates, + get_bgp_collector_location_dict, +) from app.services.bgp_collectors import build_bgp_collector_coverage router = APIRouter() @@ -264,6 +270,77 @@ async def get_bgp_collector_summary( } +class CollectBGPCollectorLocationRequest(BaseModel): + city: Optional[str] = None + country: Optional[str] = None + site: Optional[str] = None + operator: Optional[str] = None + + +@router.post("/collectors/{collector_id}/collect-location") +async def collect_bgp_collector_location( + collector_id: str, + payload: CollectBGPCollectorLocationRequest, + current_user: User = Depends(get_current_user), +): + """Run the shared location pipeline for a BGP route collector. + + Mirrors ``POST /api/v1/visualization/compute-centers/{source_id}/collect-location``. + Returns ranked candidates from source coordinates and Nominatim queries + built around the collector's stored context (IXP / city / country). Stored + collector locations provide context only; they are not emitted as + candidates. + """ + if not collector_id or not collector_id.strip(): + raise HTTPException(status_code=400, detail="collector_id is required") + + legacy = get_bgp_collector_location_dict(collector_id) or {} + site = payload.site or legacy.get("matched_location_name") + city = payload.city or legacy.get("city") + country = payload.country or legacy.get("country") + operator = payload.operator or "RIPE NCC" + + candidates, attempted_queries = collect_bgp_collector_location_candidates( + collector=collector_id, + site=site, + city=city, + country=country, + operator=operator, + ) + + context = { + "collector": collector_id, + "site": site, + "city": city, + "country": country, + "operator": operator, + } + + if not candidates: + return { + "collector_id": collector_id, + "name": collector_id, + "success": False, + "failure_reason": ( + "No source coordinates or online geocoding result reached" + " city-level precision for this collector." + ), + "candidates": [], + "attempted_queries": list(attempted_queries), + "context": context, + } + + return { + "collector_id": collector_id, + "name": collector_id, + "success": True, + "candidates": [candidate.to_dict() for candidate in candidates], + "best_candidate": candidates[0].to_dict(), + "attempted_queries": list(attempted_queries), + "context": context, + } + + @router.get("/overview/summary") async def get_bgp_overview_summary( current_user: User = Depends(get_current_user), diff --git a/backend/app/api/v1/datasource_config.py b/backend/app/api/v1/datasource_config.py index edb72286..f41146ec 100644 --- a/backend/app/api/v1/datasource_config.py +++ b/backend/app/api/v1/datasource_config.py @@ -1,20 +1,52 @@ """DataSourceConfig API for user-defined data sources""" -from typing import Optional +from typing import Any, Optional from datetime import datetime import base64 -from fastapi import APIRouter, Depends, HTTPException, status -from sqlalchemy import select, func +import json +import re +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import delete, select, func from sqlalchemy.ext.asyncio import AsyncSession from pydantic import BaseModel, Field import httpx +from app.core.target_schema_registry import get_target_schema, list_target_schemas +from app.core.datasource_defaults import DEFAULT_DATASOURCES from app.db.session import get_db from app.models.user import User from app.models.datasource_config import DataSourceConfig +from app.models.datasource_mapping import DataSourceMappingTemplate +from app.models.collected_data import CollectedData +from app.models.vessel import AISRawObservation, AISSourceHealth from app.core.security import get_current_user from app.core.cache import cache from app.core.time import to_iso8601_utc +from app.schemas.ai import SituationalAnalysisRequest +from app.services.ai_client import AIProviderClient, get_ai_provider_client +from app.services.datasource_mapping import ( + MappingError, + build_heuristic_mapping, + execute_mapping, + redact_for_llm, + stable_payload_hash, +) +from app.services.custom_datasource_runtime import ( + CustomDatasourceRuntimeError, + fetch_rest_payload, + get_custom_stream_status, + run_mapped_rest_config, + run_mapped_websocket_config, + start_custom_stream, + stop_custom_stream, + test_websocket_config, +) +from app.services.datasource_connectivity import ( + get_builtin_connection_status, + save_connectivity_success, + strip_connectivity_validation, + test_builtin_connectivity, +) router = APIRouter() @@ -22,7 +54,7 @@ router = APIRouter() class DataSourceConfigCreate(BaseModel): name: str = Field(..., min_length=1, max_length=100) description: Optional[str] = None - source_type: str = Field(..., description="http, api, database") + source_type: str = Field(..., description="rest, websocket, http, api, database") endpoint: str = Field(..., max_length=500) auth_type: str = Field(default="none", description="none, bearer, api_key, basic") auth_config: dict = Field(default={}) @@ -59,6 +91,70 @@ class DataSourceConfigResponse(BaseModel): from_attributes = True +def _is_builtin_config_name(name: str | None) -> bool: + return bool(name and name in DEFAULT_DATASOURCES) + + +async def _ensure_builtin_connection_verified( + db: AsyncSession, + config_data: DataSourceConfigCreate, +) -> None: + if not _is_builtin_config_name(config_data.name): + return + + status_result = await get_builtin_connection_status( + db, + config_data.name, + config_data.endpoint, + config_data.auth_type, + config_data.headers, + config_data.config, + ) + if not status_result.get("connected"): + raise HTTPException( + status_code=400, + detail=status_result.get("message") or "请先完成连接验证，再保存内置采集器配置。", + ) + + +class CustomSampleRequest(BaseModel): + datasource_config_id: Optional[int] = None + config: Optional[DataSourceConfigCreate] = None + limit_bytes: int = Field(default=200000, ge=1000, le=1000000) + + +class MappingProposeRequest(BaseModel): + sample_payload: Any + target_schema: str + use_ai: bool = True + + +class MappingPreviewRequest(BaseModel): + sample_payload: Any + target_schema: str + mapping_json: dict + limit: int = Field(default=20, ge=1, le=100) + + +class MappingTemplateCreate(BaseModel): + datasource_config_id: int + target_schema: str + mapping_json: dict + sample_payload: Any | None = None + sample_payload_hash: Optional[str] = None + validation_status: str = Field(default="draft", pattern="^(draft|valid|invalid)$") + is_active: bool = False + + +class MappingTemplateUpdate(BaseModel): + target_schema: Optional[str] = None + mapping_json: Optional[dict] = None + sample_payload: Any | None = None + sample_payload_hash: Optional[str] = None + validation_status: Optional[str] = Field(default=None, pattern="^(draft|valid|invalid)$") + is_active: Optional[bool] = None + + async def test_endpoint( endpoint: str, auth_type: str, @@ -96,6 +192,136 @@ async def test_endpoint( } +def _build_request_headers(auth_type: str, auth_config: dict, headers: dict) -> dict[str, str]: + request_headers = {str(key): str(value) for key, value in (headers or {}).items()} + auth_type = str(auth_type or "none").lower() + auth_config = auth_config or {} + + if auth_type == "bearer" and auth_config.get("token"): + request_headers["Authorization"] = f"Bearer {auth_config['token']}" + elif auth_type == "api_key" and auth_config.get("api_key"): + location = str(auth_config.get("in") or auth_config.get("location") or "header").lower() + if location != "query": + key_name = auth_config.get("key_name", "X-API-Key") + request_headers[str(key_name)] = str(auth_config["api_key"]) + elif auth_type == "basic": + username = auth_config.get("username", "") + password = auth_config.get("password", "") + credentials = f"{username}:{password}" + encoded = base64.b64encode(credentials.encode()).decode() + request_headers["Authorization"] = f"Basic {encoded}" + return request_headers + + +def _build_query_params(auth_type: str, auth_config: dict, config: dict) -> dict[str, Any]: + params = {} + candidate = (config or {}).get("params") or (config or {}).get("query_params") + if isinstance(candidate, dict): + params.update(candidate) + + auth_type = str(auth_type or "none").lower() + auth_config = auth_config or {} + if auth_type == "api_key" and auth_config.get("api_key"): + location = str(auth_config.get("in") or auth_config.get("location") or "header").lower() + if location == "query": + key_name = auth_config.get("key_name") or auth_config.get("param_name") or "api_key" + params[str(key_name)] = auth_config["api_key"] + return params + + +async def fetch_custom_sample_from_config(config: DataSourceConfig, limit_bytes: int) -> Any: + if str(config.source_type or "").lower() in {"websocket", "ws"}: + raise HTTPException(status_code=400, detail="WebSocket sources must use connection test or run-mapped stream.") + request_config = config.config or {} + method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper() + if method not in {"GET", "POST"}: + raise HTTPException(status_code=400, detail="Only GET and POST sample requests are supported.") + + headers = _build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {}) + params = _build_query_params(config.auth_type, config.auth_config or {}, request_config) + timeout = float(request_config.get("timeout", 30)) + json_body = request_config.get("json_body") + if json_body is None and str(request_config.get("body_type") or "").lower() in {"json", ""}: + candidate = request_config.get("body") + if isinstance(candidate, (dict, list)): + json_body = candidate + + async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: + response = await client.request( + method, + config.endpoint, + headers=headers, + params=params or None, + json=json_body, + ) + response.raise_for_status() + content = response.content[:limit_bytes] + if "application/json" in response.headers.get("content-type", ""): + return json.loads(content.decode(response.encoding or "utf-8")) + return {"text": content.decode(response.encoding or "utf-8", errors="replace")} + + +def _parse_mapping_from_ai_text(content: str) -> dict[str, Any] | None: + if not content: + return None + + candidates = [content] + fenced = re.findall(r"```(?:json)?\s*(\{.*?\})\s*```", content, flags=re.DOTALL) + candidates = fenced + candidates + for candidate in candidates: + try: + parsed = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict) and isinstance(parsed.get("fields"), dict): + return parsed + return None + + +async def _get_config_for_sample( + payload: CustomSampleRequest, + db: AsyncSession, +) -> DataSourceConfig: + if payload.datasource_config_id is not None: + result = await db.execute( + select(DataSourceConfig).where(DataSourceConfig.id == payload.datasource_config_id) + ) + config = result.scalar_one_or_none() + if not config: + raise HTTPException(status_code=404, detail="Configuration not found") + return config + + if payload.config is None: + raise HTTPException(status_code=400, detail="datasource_config_id or config is required") + + config_data = payload.config + return DataSourceConfig( + name=config_data.name, + description=config_data.description, + source_type=config_data.source_type, + endpoint=config_data.endpoint, + auth_type=config_data.auth_type, + auth_config=config_data.auth_config, + headers=config_data.headers, + config=config_data.config, + ) + + +def serialize_mapping_template(template: DataSourceMappingTemplate) -> dict[str, Any]: + return { + "id": template.id, + "datasource_config_id": template.datasource_config_id, + "target_schema": template.target_schema, + "mapping_json": template.mapping_json, + "sample_payload_hash": template.sample_payload_hash, + "validation_status": template.validation_status, + "version": template.version, + "is_active": template.is_active, + "created_at": to_iso8601_utc(template.created_at), + "updated_at": to_iso8601_utc(template.updated_at), + } + + @router.get("/configs") async def list_configs( active_only: bool = False, @@ -105,7 +331,7 @@ async def list_configs( """List all user-defined data source configurations""" query = select(DataSourceConfig) if active_only: - query = query.where(DataSourceConfig.is_active == True) + query = query.where(DataSourceConfig.is_active) query = query.order_by(DataSourceConfig.created_at.desc()) result = await db.execute(query) @@ -132,6 +358,52 @@ async def list_configs( } +@router.get("/configs/all") +async def list_all_datasources( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """List all data sources: YAML defaults + DB overrides""" + from app.core.data_sources import COLLECTOR_URL_KEYS, get_data_sources_config + + config = get_data_sources_config() + + db_query = await db.execute(select(DataSourceConfig)) + db_configs = {c.name: c for c in db_query.scalars().all()} + + result = [] + for name, yaml_key in COLLECTOR_URL_KEYS.items(): + yaml_url = config.get_yaml_url(name) + db_config = db_configs.get(name) + + result.append( + { + "name": name, + "default_url": yaml_url, + "endpoint": db_config.endpoint if db_config else yaml_url, + "is_overridden": db_config is not None and db_config.endpoint != yaml_url + if yaml_url + else db_config is not None, + "is_active": db_config.is_active if db_config else True, + "source_type": db_config.source_type if db_config else "http", + "auth_type": db_config.auth_type if db_config else "none", + "auth_configured": { + "api_key": bool((db_config.auth_config or {}).get("api_key")) + if db_config + else False, + }, + "headers": db_config.headers if db_config else {}, + "config": strip_connectivity_validation(db_config.config if db_config else {}), + "config_id": db_config.id if db_config else None, + "description": db_config.description + if db_config + else f"Data source from YAML: {yaml_key}", + } + ) + + return {"total": len(result), "data": result} + + @router.get("/configs/{config_id}") async def get_config( config_id: int, @@ -176,7 +448,7 @@ async def create_config( auth_type=config_data.auth_type, auth_config=config_data.auth_config, headers=config_data.headers, - config=config_data.config, + config=strip_connectivity_validation(config_data.config), ) db.add(config) @@ -208,6 +480,10 @@ async def update_config( update_data = config_data.model_dump(exclude_unset=True) for field, value in update_data.items(): + if field == "config": + value = strip_connectivity_validation(value) + if field == "auth_config" and value == {} and (config.auth_config or {}): + continue setattr(config, field, value) await db.commit() @@ -225,6 +501,8 @@ async def update_config( @router.delete("/configs/{config_id}") async def delete_config( config_id: int, + delete_mappings: bool = Query(False), + delete_source_data: bool = Query(False), current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db), ): @@ -235,12 +513,59 @@ async def delete_config( if not config: raise HTTPException(status_code=404, detail="Configuration not found") + deleted_mappings = 0 + deleted_records = { + "collected_data": 0, + "ais_raw_observations": 0, + "ais_source_health": 0, + } + + if delete_source_data: + collected_result = await db.execute( + delete(CollectedData).where(CollectedData.source == config.name) + ) + raw_result = await db.execute( + delete(AISRawObservation).where(AISRawObservation.source == config.name) + ) + health_result = await db.execute( + delete(AISSourceHealth).where(AISSourceHealth.source == config.name) + ) + deleted_records = { + "collected_data": collected_result.rowcount or 0, + "ais_raw_observations": raw_result.rowcount or 0, + "ais_source_health": health_result.rowcount or 0, + } + + if delete_mappings or delete_source_data: + mapping_result = await db.execute( + delete(DataSourceMappingTemplate).where( + DataSourceMappingTemplate.datasource_config_id == config_id + ) + ) + deleted_mappings = mapping_result.rowcount or 0 + await db.delete(config) await db.commit() cache.delete_pattern("datasource_configs:*") - return {"message": "Configuration deleted successfully"} + if delete_source_data and (config.config or {}).get("target_schema") == "vessel_ais": + from app.core.websocket.broadcaster import broadcaster + + await broadcaster.broadcast_custom( + "vessels", + { + "action": "reload", + "source": config.name, + "reason": "custom_source_deleted", + }, + ) + + return { + "message": "Configuration deleted successfully", + "deleted_mappings": deleted_mappings, + "deleted_records": deleted_records, + } @router.post("/configs/{config_id}/test") @@ -257,6 +582,8 @@ async def test_config( raise HTTPException(status_code=404, detail="Configuration not found") try: + if str(config.source_type or "").lower() in {"websocket", "ws"}: + return await test_websocket_config(config) result = await test_endpoint( endpoint=config.endpoint, auth_type=config.auth_type, @@ -287,6 +614,18 @@ async def test_new_config( ): """Test a new data source configuration without saving""" try: + if str(config_data.source_type or "").lower() in {"websocket", "ws"}: + config = DataSourceConfig( + name=config_data.name, + description=config_data.description, + source_type=config_data.source_type, + endpoint=config_data.endpoint, + auth_type=config_data.auth_type, + auth_config=config_data.auth_config, + headers=config_data.headers, + config=config_data.config, + ) + return await test_websocket_config(config) result = await test_endpoint( endpoint=config_data.endpoint, auth_type=config_data.auth_type, @@ -310,38 +649,363 @@ async def test_new_config( } -@router.get("/configs/all") -async def list_all_datasources( +@router.post("/configs/builtin/connection-status") +async def get_builtin_config_connection_status( + config_data: DataSourceConfigCreate, current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db), ): - """List all data sources: YAML defaults + DB overrides""" - from app.core.data_sources import COLLECTOR_URL_KEYS, get_data_sources_config + if not _is_builtin_config_name(config_data.name): + raise HTTPException(status_code=400, detail="Only built-in datasource configs are supported.") - config = get_data_sources_config() + return await get_builtin_connection_status( + db, + config_data.name, + config_data.endpoint, + config_data.auth_type, + config_data.headers, + config_data.config, + ) - db_query = await db.execute(select(DataSourceConfig)) - db_configs = {c.name: c for c in db_query.scalars().all()} - result = [] - for name, yaml_key in COLLECTOR_URL_KEYS.items(): - yaml_url = config.get_yaml_url(name) - db_config = db_configs.get(name) +@router.post("/configs/builtin/connect") +async def connect_builtin_config( + config_data: DataSourceConfigCreate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + if not _is_builtin_config_name(config_data.name): + raise HTTPException(status_code=400, detail="Only built-in datasource configs are supported.") - result.append( + result = await test_builtin_connectivity( + config_data.name, + config_data.endpoint, + config_data.auth_type, + config_data.headers, + config_data.config, + db, + config_data.auth_config, + ) + if result.get("success") and result.get("checksum"): + validation = await save_connectivity_success( + db, + config_data.name, + result["checksum"], + result, + connected_by="connection_button", + ) + await db.commit() + return { + **result, + "connected": True, + "validation": validation, + } + + return { + **result, + "connected": False, + } + + +@router.post("/custom/sample") +async def fetch_custom_sample( + payload: CustomSampleRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Fetch a sample payload for a saved or draft custom data source.""" + config = await _get_config_for_sample(payload, db) + try: + sample = await fetch_custom_sample_from_config(config, payload.limit_bytes) + except httpx.HTTPStatusError as exc: + raise HTTPException( + status_code=exc.response.status_code, + detail=f"Sample request failed: HTTP {exc.response.status_code}", + ) from exc + except httpx.HTTPError as exc: + raise HTTPException(status_code=502, detail=f"Sample request failed: {exc}") from exc + + return { + "success": True, + "sample_payload": sample, + "sample_payload_hash": stable_payload_hash(sample), + "redacted_preview": redact_for_llm(sample), + } + + +@router.get("/target-schemas") +async def get_datasource_target_schemas( + current_user: User = Depends(get_current_user), +): + """List target schemas available for custom datasource mapping.""" + return {"data": list_target_schemas()} + + +@router.post("/mappings/propose") +async def propose_datasource_mapping( + payload: MappingProposeRequest, + current_user: User = Depends(get_current_user), + ai_client: AIProviderClient = Depends(get_ai_provider_client), +): + """Generate a mapping draft for a sample payload and target schema.""" + schema = get_target_schema(payload.target_schema) + redacted_sample = redact_for_llm(payload.sample_payload) + fallback_mapping = build_heuristic_mapping(redacted_sample, payload.target_schema) + + ai_error: str | None = None + mapping = fallback_mapping + generated_by = "heuristic" + if payload.use_ai: + try: + response = await ai_client.analyze( + SituationalAnalysisRequest( + title=f"Generate datasource mapping for {schema.key}", + objective=( + "Return only JSON for a deterministic mapping DSL. " + "The JSON must contain source.items_path and fields. " + "Do not include prose or code." + ), + context={ + "target_schema": schema.to_dict(), + "sample_payload": redacted_sample, + "mapping_dsl_example": fallback_mapping, + }, + observations=[ + "Use JSONPath-like paths beginning with $.", + "Never generate executable code.", + "Use field types from the target schema.", + ], + constraints=[ + "Return a single JSON object.", + "Do not include credentials or secrets.", + "Mark uncertain optional fields with default null.", + ], + ) + ) + parsed = _parse_mapping_from_ai_text(response.content) + if parsed: + mapping = parsed + generated_by = "ai_provider" + else: + ai_error = "AI provider did not return a valid mapping JSON object." + except HTTPException as exc: + ai_error = str(exc.detail) + + mapping.setdefault("meta", {}) + if isinstance(mapping["meta"], dict): + mapping["meta"].update( { - "name": name, - "default_url": yaml_url, - "endpoint": db_config.endpoint if db_config else yaml_url, - "is_overridden": db_config is not None and db_config.endpoint != yaml_url - if yaml_url - else db_config is not None, - "is_active": db_config.is_active if db_config else True, - "source_type": db_config.source_type if db_config else "http", - "description": db_config.description - if db_config - else f"Data source from YAML: {yaml_key}", + "generated_by": generated_by, + "requires_review": True, + "ai_error": ai_error, } ) - return {"total": len(result), "data": result} + return { + "target_schema": schema.to_dict(), + "mapping_json": mapping, + "sample_payload_hash": stable_payload_hash(payload.sample_payload), + "redacted_sample_payload": redacted_sample, + } + + +@router.post("/mappings/preview") +async def preview_datasource_mapping( + payload: MappingPreviewRequest, + current_user: User = Depends(get_current_user), +): + """Preview deterministic mapping output for a sample payload.""" + try: + preview = execute_mapping( + payload.sample_payload, + payload.mapping_json, + payload.target_schema, + limit=payload.limit, + ) + except (MappingError, ValueError) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + return { + "success": preview["failed_count"] == 0, + "preview": preview, + "sample_payload_hash": stable_payload_hash(payload.sample_payload), + } + + +@router.get("/mappings") +async def list_datasource_mappings( + datasource_config_id: Optional[int] = None, + active_only: bool = False, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """List saved mapping templates.""" + query = select(DataSourceMappingTemplate).order_by( + DataSourceMappingTemplate.datasource_config_id, + DataSourceMappingTemplate.version.desc(), + ) + if datasource_config_id is not None: + query = query.where(DataSourceMappingTemplate.datasource_config_id == datasource_config_id) + if active_only: + query = query.where(DataSourceMappingTemplate.is_active.is_(True)) + + result = await db.execute(query) + mappings = result.scalars().all() + return {"total": len(mappings), "data": [serialize_mapping_template(item) for item in mappings]} + + +@router.post("/mappings") +async def create_datasource_mapping( + payload: MappingTemplateCreate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Save a mapping template for a datasource config.""" + get_target_schema(payload.target_schema) + datasource = await db.get(DataSourceConfig, payload.datasource_config_id) + if not datasource: + raise HTTPException(status_code=404, detail="Configuration not found") + + if payload.sample_payload is not None: + try: + execute_mapping(payload.sample_payload, payload.mapping_json, payload.target_schema, limit=100) + except (MappingError, ValueError) as exc: + raise HTTPException(status_code=400, detail=f"Mapping validation failed: {exc}") from exc + + result = await db.execute( + select(func.max(DataSourceMappingTemplate.version)).where( + DataSourceMappingTemplate.datasource_config_id == payload.datasource_config_id, + DataSourceMappingTemplate.target_schema == payload.target_schema, + ) + ) + next_version = int(result.scalar() or 0) + 1 + + if payload.is_active: + await db.execute( + DataSourceMappingTemplate.__table__.update() + .where(DataSourceMappingTemplate.datasource_config_id == payload.datasource_config_id) + .values(is_active=False) + ) + + template = DataSourceMappingTemplate( + datasource_config_id=payload.datasource_config_id, + target_schema=payload.target_schema, + mapping_json=payload.mapping_json, + sample_payload_hash=payload.sample_payload_hash + or (stable_payload_hash(payload.sample_payload) if payload.sample_payload is not None else None), + validation_status=payload.validation_status, + version=next_version, + is_active=payload.is_active, + ) + db.add(template) + await db.commit() + await db.refresh(template) + return {"message": "Mapping template saved successfully", "data": serialize_mapping_template(template)} + + +@router.put("/mappings/{mapping_id}") +async def update_datasource_mapping( + mapping_id: int, + payload: MappingTemplateUpdate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Update a mapping template in place.""" + template = await db.get(DataSourceMappingTemplate, mapping_id) + if not template: + raise HTTPException(status_code=404, detail="Mapping template not found") + + target_schema = payload.target_schema or template.target_schema + mapping_json = payload.mapping_json or template.mapping_json + get_target_schema(target_schema) + if payload.sample_payload is not None: + try: + execute_mapping(payload.sample_payload, mapping_json, target_schema, limit=100) + except (MappingError, ValueError) as exc: + raise HTTPException(status_code=400, detail=f"Mapping validation failed: {exc}") from exc + + if payload.is_active is True: + await db.execute( + DataSourceMappingTemplate.__table__.update() + .where(DataSourceMappingTemplate.datasource_config_id == template.datasource_config_id) + .where(DataSourceMappingTemplate.id != template.id) + .values(is_active=False) + ) + + template.target_schema = target_schema + template.mapping_json = mapping_json + if payload.sample_payload_hash is not None: + template.sample_payload_hash = payload.sample_payload_hash + elif payload.sample_payload is not None: + template.sample_payload_hash = stable_payload_hash(payload.sample_payload) + if payload.validation_status is not None: + template.validation_status = payload.validation_status + if payload.is_active is not None: + template.is_active = payload.is_active + + await db.commit() + await db.refresh(template) + return {"message": "Mapping template updated successfully", "data": serialize_mapping_template(template)} + + +@router.post("/{config_id}/run-mapped") +async def run_mapped_datasource( + config_id: int, + background: bool = Query(False, description="For WebSocket sources, start a background stream task."), + debug_max_messages: int | None = Query(None, ge=1), + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Run a saved custom datasource through its active deterministic mapping.""" + datasource = await db.get(DataSourceConfig, config_id) + if not datasource: + raise HTTPException(status_code=404, detail="Configuration not found") + + try: + if str(datasource.source_type or "").lower() in {"websocket", "ws"}: + if background and debug_max_messages is None: + started = start_custom_stream(config_id) + if not started: + raise HTTPException(status_code=409, detail="Custom WebSocket source is already running") + return { + "status": "started", + "datasource_config_id": config_id, + "stream": get_custom_stream_status(config_id), + } + return await run_mapped_websocket_config( + db, + datasource, + debug_max_messages=debug_max_messages, + ) + + return await run_mapped_rest_config(db, datasource) + except httpx.HTTPStatusError as exc: + raise HTTPException( + status_code=exc.response.status_code, + detail=f"Datasource request failed: HTTP {exc.response.status_code}", + ) from exc + except httpx.HTTPError as exc: + raise HTTPException(status_code=502, detail=f"Datasource request failed: {exc}") from exc + except (CustomDatasourceRuntimeError, MappingError, ValueError) as exc: + raise HTTPException(status_code=400, detail=f"Mapping failed: {exc}") from exc + + +@router.post("/{config_id}/stop-mapped") +async def stop_mapped_datasource( + config_id: int, + current_user: User = Depends(get_current_user), +): + stopped = await stop_custom_stream(config_id) + return { + "status": "stopped" if stopped else "not_running", + "datasource_config_id": config_id, + "stream": get_custom_stream_status(config_id), + } + + +@router.get("/{config_id}/stream-status") +async def get_mapped_stream_status( + config_id: int, + current_user: User = Depends(get_current_user), +): + return get_custom_stream_status(config_id) diff --git a/backend/app/api/v1/datasources.py b/backend/app/api/v1/datasources.py index fedcad0c..16d0239d 100644 --- a/backend/app/api/v1/datasources.py +++ b/backend/app/api/v1/datasources.py @@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.time import to_iso8601_utc from app.core.security import get_current_user from app.core.data_sources import get_data_sources_config +from app.core.datasource_defaults import DEFAULT_DATASOURCES from app.db.session import get_db from app.models.collected_data import CollectedData from app.models.data_snapshot import DataSnapshot @@ -35,6 +36,17 @@ def format_frequency_label(minutes: int) -> str: return f"{minutes}m" +def datasource_metadata(source: str) -> dict: + info = DEFAULT_DATASOURCES.get(source, {}) + return { + "display_name": info.get("display_name") or info.get("name") or source, + "is_free": bool(info.get("is_free", True)), + "requires_credentials": bool(info.get("requires_credentials", False)), + "credential_provider": info.get("credential_provider"), + "credential_status": info.get("credential_status", "none"), + } + + def is_due_for_collection(datasource: DataSource, now: datetime) -> bool: if datasource.last_run_at is None: return True @@ -72,31 +84,6 @@ async def _load_latest_running_tasks( return {task.datasource_id: task for task in result.scalars().all()} -async def _load_latest_completed_tasks( - db: AsyncSession, - datasource_ids: list[int], -) -> dict[int, CollectionTask]: - if not datasource_ids: - return {} - - ranked_tasks = ( - select( - CollectionTask.id.label("task_id"), - _task_rank_column(CollectionTask.completed_at), - ) - .where(CollectionTask.datasource_id.in_(datasource_ids)) - .where(CollectionTask.completed_at.isnot(None)) - .where(CollectionTask.status.in_(("success", "failed", "cancelled"))) - .subquery() - ) - result = await db.execute( - select(CollectionTask) - .join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id) - .where(ranked_tasks.c.row_num == 1) - ) - return {task.datasource_id: task for task in result.scalars().all()} - - async def _load_latest_task_ids( db: AsyncSession, datasource_ids: list[int], @@ -123,21 +110,6 @@ async def _load_latest_task_ids( return {datasource_id: task_id for datasource_id, task_id in result.all()} -async def _load_datasource_data_counts( - db: AsyncSession, - sources: list[str], -) -> dict[str, int]: - if not sources: - return {} - - result = await db.execute( - select(CollectedData.source, func.count(CollectedData.id)) - .where(CollectedData.source.in_(sources)) - .group_by(CollectedData.source) - ) - return {source: count for source, count in result.all()} - - async def _load_datasource_endpoint_overrides( db: AsyncSession, sources: list[str], @@ -161,7 +133,7 @@ async def _load_datasource_endpoint_overrides( async def _load_datasource_list_context( db: AsyncSession, datasources: list[DataSource], -) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, int], dict[str, str]]: +) -> tuple[dict[int, CollectionTask], dict[str, str]]: datasource_ids = [datasource.id for datasource in datasources] sources = [datasource.source for datasource in datasources] @@ -185,10 +157,8 @@ async def _load_datasource_list_context( if stale_datasource_ids: running_tasks = await _load_latest_running_tasks(db, datasource_ids) - completed_tasks = await _load_latest_completed_tasks(db, datasource_ids) - data_counts = await _load_datasource_data_counts(db, sources) endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources) - return running_tasks, completed_tasks, data_counts, endpoint_overrides + return running_tasks, endpoint_overrides async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]: @@ -401,27 +371,19 @@ async def list_datasources( collector_list = [] config = get_data_sources_config() - running_tasks, completed_tasks, data_counts, endpoint_overrides = await _load_datasource_list_context( - db, - datasources, - ) + running_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources) for datasource in datasources: running_task = running_tasks.get(datasource.id) - last_task = completed_tasks.get(datasource.id) - endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url( - datasource.source, - ) - data_count = data_counts.get(datasource.source, 0) - - last_run_at = datasource.last_run_at or (last_task.completed_at if last_task else None) - last_run = to_iso8601_utc(last_run_at) - last_status = datasource.last_status or (last_task.status if last_task else None) + endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source) + last_run_at = datasource.last_run_at + last_status = datasource.last_status collector_list.append( { "id": datasource.id, "source": datasource.source, "name": datasource.name, + **datasource_metadata(datasource.source), "module": datasource.module, "priority": datasource.priority, "frequency": format_frequency_label(datasource.frequency_minutes), @@ -429,15 +391,18 @@ async def list_datasources( "is_active": datasource.is_active, "collector_class": datasource.collector_class, "endpoint": endpoint, - "last_run": last_run, + "last_run": to_iso8601_utc(last_run_at), "last_run_at": to_iso8601_utc(last_run_at), "last_status": last_status, - "last_records_processed": last_task.records_processed if last_task else None, - "data_count": data_count, "is_running": running_task is not None, "task_id": running_task.id if running_task else None, "progress": running_task.progress if running_task else None, "phase": running_task.phase if running_task else None, + "phase_progress": running_task.phase_progress if running_task else None, + "phase_message": running_task.phase_message if running_task else None, + "phase_current": running_task.phase_current if running_task else None, + "phase_total": running_task.phase_total if running_task else None, + "phase_unit": running_task.phase_unit if running_task else None, "records_processed": running_task.records_processed if running_task else None, "total_records": running_task.total_records if running_task else None, } @@ -576,6 +541,7 @@ async def get_datasource( return { "id": datasource.id, "name": datasource.name, + **datasource_metadata(datasource.source), "module": datasource.module, "priority": datasource.priority, "frequency": format_frequency_label(datasource.frequency_minutes), @@ -665,6 +631,11 @@ async def trigger_datasource( "message": "当前采集任务尚未完成，重新触发会丢失本次未完成进度。是否强制重新采集？", "task_id": running_task.id, "phase": running_task.phase, + "phase_progress": running_task.phase_progress, + "phase_message": running_task.phase_message, + "phase_current": running_task.phase_current, + "phase_total": running_task.phase_total, + "phase_unit": running_task.phase_unit, "progress": running_task.progress, "records_processed": running_task.records_processed, "total_records": running_task.total_records, @@ -748,13 +719,29 @@ async def get_task_status( task = await get_running_task(db, datasource.id) if not task: - return {"is_running": False, "task_id": None, "progress": None, "phase": None, "status": "idle"} + return { + "is_running": False, + "task_id": None, + "progress": None, + "phase": None, + "phase_progress": None, + "phase_message": None, + "phase_current": None, + "phase_total": None, + "phase_unit": None, + "status": "idle", + } return { "is_running": task.status == "running", "task_id": task.id, "progress": task.progress, "phase": task.phase, + "phase_progress": task.phase_progress, + "phase_message": task.phase_message, + "phase_current": task.phase_current, + "phase_total": task.phase_total, + "phase_unit": task.phase_unit, "records_processed": task.records_processed, "total_records": task.total_records, "status": task.status, diff --git a/backend/app/api/v1/docs.py b/backend/app/api/v1/docs.py new file mode 100644 index 00000000..e26dea21 --- /dev/null +++ b/backend/app/api/v1/docs.py @@ -0,0 +1,102 @@ +"""Authenticated documentation APIs.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from sqlalchemy import text + +from app.core.security import decode_token +from app.db.session import async_session_factory +from app.models.user import User +from app.services.docs_gatekeeper import ( + DOCS_BY_SLUG, + VALID_DOCS_LANGS, + can_read_doc, + catalog_for_user, + doc_path_for, + title_for, +) + +router = APIRouter() +optional_bearer = HTTPBearer(auto_error=False) + + +async def get_optional_current_user( + credentials: HTTPAuthorizationCredentials | None = Depends(optional_bearer), +) -> User | None: + if credentials is None: + return None + + payload = decode_token(credentials.credentials) + if payload is None or payload.get("type") != "access" or payload.get("sub") is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token", + ) + + async with async_session_factory() as db: + result = await db.execute( + text( + "SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id" + ), + {"id": int(payload["sub"])}, + ) + row = result.fetchone() + if row is None or not row[5]: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="User not found or inactive", + ) + + user = User() + user.id = row[0] + user.username = row[1] + user.email = row[2] + user.password_hash = row[3] + user.role = row[4] + user.is_active = row[5] + user.gatekeeper_groups = row[6] or [] + return user + + +@router.get("/catalog") +async def get_docs_catalog(current_user: User | None = Depends(get_optional_current_user)): + return { + "items": catalog_for_user(current_user), + "authenticated": current_user is not None, + } + + +@router.get("/{lang}/{slug}") +async def get_doc_content( + lang: str, + slug: str, + current_user: User | None = Depends(get_optional_current_user), +): + if lang not in VALID_DOCS_LANGS: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found") + + entry = DOCS_BY_SLUG.get(slug) + if entry is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found") + + path = doc_path_for(entry, lang) + if not path.exists(): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found") + + if not can_read_doc(entry, current_user): + if current_user is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authentication required") + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Insufficient Docs permissions") + + return { + "slug": entry.slug, + "filename": entry.filename, + "lang": lang, + "title": title_for(entry, lang), + "group": entry.group, + "order": entry.order, + "access": entry.access, + "markdown": path.read_text(encoding="utf-8"), + } diff --git a/backend/app/api/v1/settings.py b/backend/app/api/v1/settings.py index ff580a20..700a6312 100644 --- a/backend/app/api/v1/settings.py +++ b/backend/app/api/v1/settings.py @@ -9,10 +9,37 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.security import get_current_user from app.core.time import to_iso8601_utc +from app.core.config import settings as app_settings +from app.core.data_sources import get_data_sources_config +from app.core.datasource_defaults import DEFAULT_DATASOURCES from app.db.session import get_db from app.models.datasource import DataSource +from app.models.datasource_config import DataSourceConfig from app.models.system_setting import SystemSetting from app.models.user import User +from app.models.vessel import AISSourceHealth +from app.services.barentswatch import ( + BarentsWatchConfig, + check_barentswatch_config, + check_barentswatch_connectivity, + get_barentswatch_datasource_record, + resolve_barentswatch_config, +) +from app.services.credential_guides import ( + generate_credential_guide, + get_credential_guide, + reset_credential_guide, +) +from app.services.datasource_connectivity import ( + build_builtin_connectivity_checksum, + save_connectivity_success, +) +from app.services.ai_client import AIProviderClient, get_ai_provider_client +from app.services.llm_provider_catalog import ( + get_fallback_llm_provider_preset, + list_fallback_llm_provider_presets, + refresh_llm_provider_preset, +) from app.services.scheduler import sync_datasource_job from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings @@ -39,6 +66,21 @@ DEFAULT_SETTINGS = { "password_policy": "medium", }, "tv": DEFAULT_TV_SETTINGS, + "external_integrations": { + "ai_provider": { + "service_url": "", + "service_token": "", + "provider": "minimax", + "provider_api": "anthropic-messages", + "base_url": "https://api.minimaxi.com/anthropic", + "model": "MiniMax-M2.7", + "api_key": "", + "max_tokens": 1200, + "anthropic_version": "2023-06-01", + "timeout_seconds": 60, + "retry_attempts": 2, + } + }, } @@ -96,6 +138,34 @@ class TVSettingsUpdate(BaseModel): sources: list[TVStreamSourceUpdate] = Field(default_factory=list) +class AIProviderIntegrationUpdate(BaseModel): + service_url: str = "" + service_token: Optional[str] = None + provider: str = Field(default="minimax", max_length=80) + provider_api: str = Field(default="anthropic-messages", max_length=80) + base_url: str = Field(default="", max_length=500) + model: str = Field(default="", max_length=200) + api_key: Optional[str] = None + max_tokens: int = Field(default=1200, ge=1, le=200000) + anthropic_version: str = Field(default="2023-06-01", max_length=40) + timeout_seconds: int = Field(default=60, ge=5, le=600) + retry_attempts: int = Field(default=2, ge=1, le=10) + clear_service_token: bool = False + clear_api_key: bool = False + + +class BarentsWatchIntegrationUpdate(BaseModel): + endpoint: str = "" + client_id: str = "" + client_secret: Optional[str] = None + clear_client_secret: bool = False + + +class ExternalIntegrationsUpdate(BaseModel): + ai_provider: AIProviderIntegrationUpdate + barentswatch: BarentsWatchIntegrationUpdate + + def merge_with_defaults(category: str, payload: Optional[dict]) -> dict: merged = deepcopy(DEFAULT_SETTINGS[category]) if payload: @@ -146,6 +216,151 @@ async def save_setting_payload(db: AsyncSession, category: str, payload: dict) - return merge_with_defaults(category, record.payload) +def _mask_secret(value: Optional[str]) -> dict: + if not value: + return {"configured": False, "preview": ""} + text = str(value) + if "-" in text: + prefix = text.split("-", 1)[0] + "-" + preview = prefix + ("*" * max(len(text) - len(prefix), 1)) + else: + prefix_len = min(4, len(text)) + preview = text[:prefix_len] + ("*" * max(len(text) - prefix_len, 1)) + return {"configured": True, "preview": preview} + + +async def get_runtime_ai_provider_config(db: AsyncSession) -> dict: + runtime_record = await get_setting_record(db, "external_integrations") + payload = merge_with_defaults( + "external_integrations", + runtime_record.payload if runtime_record else None, + ) + ai_payload = payload.get("ai_provider") or {} + has_runtime_llm_config = bool( + runtime_record + and isinstance(runtime_record.payload, dict) + and isinstance(runtime_record.payload.get("ai_provider"), dict) + ) + return { + "service_url": ai_payload.get("service_url") or app_settings.AI_PROVIDER_SERVICE_URL, + "service_token": ai_payload.get("service_token") or app_settings.AI_PROVIDER_SERVICE_TOKEN, + "timeout_seconds": int( + ai_payload.get("timeout_seconds") or app_settings.AI_PROVIDER_TIMEOUT_SECONDS + ), + "retry_attempts": int( + ai_payload.get("retry_attempts") or app_settings.AI_PROVIDER_RETRY_ATTEMPTS + ), + "llm_config": { + "provider": ai_payload.get("provider") or "minimax", + "provider_api": ai_payload.get("provider_api") or "anthropic-messages", + "base_url": ai_payload.get("base_url") or "https://api.minimaxi.com/anthropic", + "model": ai_payload.get("model") or "MiniMax-M2.7", + "api_key": ai_payload.get("api_key") or "", + "max_tokens": int(ai_payload.get("max_tokens") or 1200), + "anthropic_version": ai_payload.get("anthropic_version") or "2023-06-01", + } if has_runtime_llm_config else {}, + } + + +async def get_barentswatch_config_record(db: AsyncSession) -> Optional[DataSourceConfig]: + return await get_barentswatch_datasource_record(db) + + +async def serialize_external_integrations(db: AsyncSession) -> dict: + ai_config = await get_runtime_ai_provider_config(db) + runtime_setting = await get_setting_record(db, "external_integrations") + display_llm_config = ai_config["llm_config"] or DEFAULT_SETTINGS["external_integrations"]["ai_provider"] + barentswatch_record = await get_barentswatch_config_record(db) + barentswatch_auth = barentswatch_record.auth_config if barentswatch_record else {} + barentswatch_auth = barentswatch_auth or {} + resolved_barentswatch = await resolve_barentswatch_config(db) + return { + "ai_provider": { + "service_url": ai_config["service_url"], + "service_token": _mask_secret(ai_config["service_token"]), + "provider": display_llm_config.get("provider") or "minimax", + "provider_api": display_llm_config.get("provider_api") or "anthropic-messages", + "base_url": display_llm_config.get("base_url") or "https://api.minimaxi.com/anthropic", + "model": display_llm_config.get("model") or "MiniMax-M2.7", + "api_key": _mask_secret(display_llm_config.get("api_key")), + "max_tokens": int(display_llm_config.get("max_tokens") or 1200), + "anthropic_version": display_llm_config.get("anthropic_version") or "2023-06-01", + "timeout_seconds": ai_config["timeout_seconds"], + "retry_attempts": ai_config["retry_attempts"], + "source": "runtime" if runtime_setting else "env", + }, + "barentswatch": { + "endpoint": resolved_barentswatch.endpoint, + "client_id": barentswatch_auth.get("client_id") or resolved_barentswatch.client_id, + "client_secret": _mask_secret( + barentswatch_auth.get("client_secret") or resolved_barentswatch.client_secret + ), + "source": resolved_barentswatch.credential_source, + }, + } + + +async def save_external_integrations_payload( + db: AsyncSession, + update: ExternalIntegrationsUpdate, +) -> dict: + current_payload = await get_setting_payload(db, "external_integrations") + current_ai = current_payload.get("ai_provider") or {} + ai_payload = { + "service_url": update.ai_provider.service_url.strip() + or app_settings.AI_PROVIDER_SERVICE_URL, + "service_token": current_ai.get("service_token") or "", + "provider": update.ai_provider.provider.strip() or "minimax", + "provider_api": update.ai_provider.provider_api.strip() or "anthropic-messages", + "base_url": update.ai_provider.base_url.strip(), + "model": update.ai_provider.model.strip(), + "api_key": current_ai.get("api_key") or "", + "max_tokens": update.ai_provider.max_tokens, + "anthropic_version": update.ai_provider.anthropic_version.strip() or "2023-06-01", + "timeout_seconds": update.ai_provider.timeout_seconds, + "retry_attempts": update.ai_provider.retry_attempts, + } + if update.ai_provider.clear_service_token: + ai_payload["service_token"] = "" + elif update.ai_provider.service_token not in (None, ""): + ai_payload["service_token"] = update.ai_provider.service_token + if update.ai_provider.clear_api_key: + ai_payload["api_key"] = "" + elif update.ai_provider.api_key not in (None, ""): + ai_payload["api_key"] = update.ai_provider.api_key + + await save_setting_payload(db, "external_integrations", {"ai_provider": ai_payload}) + + default_endpoint = get_data_sources_config().get_yaml_url("barentswatch_vessels") + barentswatch_record = await get_barentswatch_config_record(db) + if barentswatch_record is None: + barentswatch_record = DataSourceConfig( + name="barentswatch_vessels", + description="BarentsWatch Live AIS credentials", + source_type="api", + endpoint=update.barentswatch.endpoint.strip() or default_endpoint, + auth_type="oauth_client", + auth_config={}, + headers={}, + config={}, + is_active=True, + ) + db.add(barentswatch_record) + + current_auth = dict(barentswatch_record.auth_config or {}) + if update.barentswatch.clear_client_secret: + current_auth.pop("client_secret", None) + elif update.barentswatch.client_secret not in (None, ""): + current_auth["client_secret"] = update.barentswatch.client_secret + current_auth["client_id"] = update.barentswatch.client_id.strip() + barentswatch_record.endpoint = update.barentswatch.endpoint.strip() or default_endpoint + barentswatch_record.auth_type = "oauth_client" + barentswatch_record.auth_config = current_auth + await db.commit() + + return await serialize_external_integrations(db) + + def format_frequency_label(minutes: int) -> str: if minutes % 1440 == 0: return f"{minutes // 1440}d" @@ -154,10 +369,17 @@ def format_frequency_label(minutes: int) -> str: return f"{minutes}m" -def serialize_collector(datasource: DataSource) -> dict: +async def get_ais_source_health_by_source(db: AsyncSession) -> dict[str, dict]: + result = await db.execute(select(AISSourceHealth)) + return {item.source: item.to_dict() for item in result.scalars().all()} + + +def serialize_collector(datasource: DataSource, ais_health_by_source: dict[str, dict] | None = None) -> dict: + defaults = DEFAULT_DATASOURCES.get(datasource.source, {}) return { "id": datasource.id, "name": datasource.name, + "display_name": defaults.get("display_name") or datasource.name, "source": datasource.source, "module": datasource.module, "priority": datasource.priority, @@ -167,6 +389,11 @@ def serialize_collector(datasource: DataSource) -> dict: "last_run_at": to_iso8601_utc(datasource.last_run_at), "last_status": datasource.last_status, "next_run_at": to_iso8601_utc(datasource.next_run_at), + "is_free": bool(defaults.get("is_free", True)), + "requires_credentials": bool(defaults.get("requires_credentials", False)), + "credential_provider": defaults.get("credential_provider"), + "credential_status": defaults.get("credential_status", "none"), + "ais_health": (ais_health_by_source or {}).get(datasource.source), } @@ -243,6 +470,135 @@ async def update_tv_settings( return {"status": "updated", "tv": normalize_tv_settings(saved)} +@router.get("/integrations") +async def get_external_integrations( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return {"integrations": await serialize_external_integrations(db)} + + +@router.get("/integrations/barentswatch/connectivity") +async def get_barentswatch_connectivity( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return await check_barentswatch_connectivity(db) + + +@router.post("/integrations/barentswatch/connect") +async def connect_barentswatch_integration( + payload: BarentsWatchIntegrationUpdate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + current = await resolve_barentswatch_config(db) + config = BarentsWatchConfig( + endpoint=payload.endpoint.strip() or current.endpoint, + client_id=payload.client_id.strip() or current.client_id, + client_secret=( + "" + if payload.clear_client_secret + else payload.client_secret or current.client_secret + ), + credential_source="draft", + endpoint_source="draft", + ) + result = await check_barentswatch_config(config) + if result.get("success"): + checksum, _context = await build_builtin_connectivity_checksum( + "barentswatch_vessels", + config.endpoint, + "none", + {}, + {}, + db, + credential_override={ + "client_id": config.client_id, + "client_secret": config.client_secret, + }, + ) + validation = await save_connectivity_success( + db, + "barentswatch_vessels", + checksum, + result, + connected_by="connection_button", + ) + await db.commit() + return {**result, "connected": True, "validation": validation} + return {**result, "connected": False} + + +@router.get("/credential-guides/{provider}") +async def read_credential_guide( + provider: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + try: + return {"guide": await get_credential_guide(db, provider)} + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.post("/credential-guides/{provider}/generate") +async def generate_provider_credential_guide( + provider: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), + ai_client: AIProviderClient = Depends(get_ai_provider_client), +): + try: + return {"guide": await generate_credential_guide(db, provider, ai_client)} + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.post("/credential-guides/{provider}/reset") +async def reset_provider_credential_guide( + provider: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + try: + return {"guide": await reset_credential_guide(db, provider)} + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/integrations/ai-provider/presets") +async def get_ai_provider_presets( + current_user: User = Depends(get_current_user), +): + return {"data": list_fallback_llm_provider_presets()} + + +@router.post("/integrations/ai-provider/presets/{provider}/refresh") +async def refresh_ai_provider_preset( + provider: str, + current_user: User = Depends(get_current_user), +): + try: + return {"data": await refresh_llm_provider_preset(provider)} + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except Exception as exc: + fallback = get_fallback_llm_provider_preset(provider) + fallback["refresh_error"] = str(exc) + return {"data": fallback} + + +@router.put("/integrations") +async def update_external_integrations( + payload: ExternalIntegrationsUpdate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + saved = await save_external_integrations_payload(db, payload) + return {"status": "updated", "integrations": saved} + + @router.get("/collectors") async def get_collector_settings( current_user: User = Depends(get_current_user), @@ -250,7 +606,8 @@ async def get_collector_settings( ): result = await db.execute(select(DataSource).order_by(DataSource.module, DataSource.id)) datasources = result.scalars().all() - return {"collectors": [serialize_collector(datasource) for datasource in datasources]} + ais_health_by_source = await get_ais_source_health_by_source(db) + return {"collectors": [serialize_collector(datasource, ais_health_by_source) for datasource in datasources]} @router.put("/collectors/{datasource_id}") @@ -270,7 +627,8 @@ async def update_collector_settings( await db.commit() await db.refresh(datasource) await sync_datasource_job(datasource.id) - return {"status": "updated", "collector": serialize_collector(datasource)} + ais_health_by_source = await get_ais_source_health_by_source(db) + return {"status": "updated", "collector": serialize_collector(datasource, ais_health_by_source)} @router.get("") @@ -284,11 +642,13 @@ async def get_all_settings( db, ["system", "notifications", "security"], ) + ais_health_by_source = await get_ais_source_health_by_source(db) return { "system": setting_payloads["system"], "notifications": setting_payloads["notifications"], "security": setting_payloads["security"], "tv": await get_tv_settings_payload(db), - "collectors": [serialize_collector(datasource) for datasource in datasources], + "integrations": await serialize_external_integrations(db), + "collectors": [serialize_collector(datasource, ais_health_by_source) for datasource in datasources], "generated_at": to_iso8601_utc(datetime.now(UTC)), } diff --git a/backend/app/api/v1/system_control.py b/backend/app/api/v1/system_control.py index 21c81208..254fe353 100644 --- a/backend/app/api/v1/system_control.py +++ b/backend/app/api/v1/system_control.py @@ -4,12 +4,15 @@ import os import subprocess import sys -from fastapi import APIRouter, Depends, HTTPException, status +from datetime import datetime + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from pydantic import BaseModel from app.core.config import ROOT_DIR from app.core.security import get_current_user from app.models.user import User +from app.services.persistent_logs import record_audit_log, record_system_log from app.services.system_control import ( build_task_id, clear_active_task_id, @@ -23,6 +26,15 @@ from app.services.system_control import ( set_active_task_id, upsert_task_state, ) +from app.services.system_logs import ( + DEFAULT_LOG_LINE_LIMIT, + MAX_LOG_LINE_LIMIT, + SUPPORTED_LOG_LEVELS, + append_buffer_log, + list_log_sources, + normalize_log_level, + read_log_snapshot, +) router = APIRouter() @@ -47,6 +59,59 @@ class RestartTaskLogsResponse(BaseModel): lines: list[str] +class SystemLogSourceSummary(BaseModel): + source_id: str + name: str + kind: str + location: str + description: str + category: str + status: str + + +class SystemLogSourcesResponse(BaseModel): + items: list[SystemLogSourceSummary] + + +class SystemLogDailyMarker(BaseModel): + date_token: str + total: int + dominant_level: str + + +class SystemLogSnapshotResponse(BaseModel): + source_id: str + name: str + kind: str + location: str + description: str + category: str + status: str + level: str + selected_levels: list[str] = [] + search_query: str = "" + available_levels: list[str] + daily_markers: list[SystemLogDailyMarker] = [] + line_limit: int + line_count: int + lines: list[str] + + +class EarthClientLogEventCreate(BaseModel): + level: str = "error" + message: str + category: str | None = None + url: str | None = None + module: str | None = None + detail: str | None = None + + +class EarthClientLogEventResponse(BaseModel): + accepted: bool + source_id: str + level: str + + def ensure_super_admin(current_user: User) -> None: if not require_super_admin(current_user.role): raise HTTPException( @@ -55,9 +120,22 @@ def ensure_super_admin(current_user: User) -> None: ) +def validate_log_date(raw_value: str | None, field_name: str) -> str | None: + if raw_value in {None, ""}: + return None + try: + return datetime.strptime(raw_value, "%Y-%m-%d").date().isoformat() + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"{field_name} must be in YYYY-MM-DD format", + ) from exc + + @router.post("/restart-tasks", response_model=RestartTaskResponse) async def create_restart_task( payload: RestartTaskCreate, + request: Request, current_user: User = Depends(get_current_user), ): ensure_super_admin(current_user) @@ -133,11 +211,31 @@ async def create_restart_task( requested_by=requested_by, ) clear_active_task_id(task_id) + await record_audit_log( + action="system.restart_task.requested", + actor_id=current_user.id, + actor_name=current_user.username, + target_type="restart_task", + target_id=task_id, + result="failed", + ip=request.client.host if request.client else None, + details={"action": payload.action, "message": task_state["message"]}, + ) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=task_state["message"], ) from exc + await record_audit_log( + action="system.restart_task.requested", + actor_id=current_user.id, + actor_name=current_user.username, + target_type="restart_task", + target_id=task_id, + result="accepted", + ip=request.client.host if request.client else None, + details={"action": payload.action}, + ) return task_state @@ -165,3 +263,92 @@ async def get_restart_task_logs( if task is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Restart task not found") return {"task_id": task_id, "lines": get_task_logs(task_id)} + + +@router.get("/logs/sources", response_model=SystemLogSourcesResponse) +async def get_system_log_sources( + current_user: User = Depends(get_current_user), +): + ensure_super_admin(current_user) + return {"items": list_log_sources()} + + +@router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse) +async def get_system_log_snapshot( + source_id: str, + limit: int = DEFAULT_LOG_LINE_LIMIT, + level: str = "all", + levels: str | None = Query(None, description="Comma-separated log levels"), + start_date: str | None = Query(None, description="Filter logs from this date (YYYY-MM-DD)"), + end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"), + search: str | None = Query(None, description="Case-insensitive substring search"), + current_user: User = Depends(get_current_user), +): + ensure_super_admin(current_user) + + if limit < 1 or limit > MAX_LOG_LINE_LIMIT: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"limit must be between 1 and {MAX_LOG_LINE_LIMIT}", + ) + if str(level).strip().lower() not in SUPPORTED_LOG_LEVELS and normalize_log_level(level) == "all" and str(level).strip().lower() not in {"", "all"}: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unsupported log level") + if levels: + for raw_level in str(levels).split(","): + normalized_level = str(raw_level).strip().lower() + if not normalized_level: + continue + if normalized_level not in SUPPORTED_LOG_LEVELS and normalize_log_level(normalized_level) == "all": + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unsupported log level") + normalized_start_date = validate_log_date(start_date, "start_date") + normalized_end_date = validate_log_date(end_date, "end_date") + if normalized_start_date and normalized_end_date and normalized_start_date > normalized_end_date: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="start_date must be earlier than or equal to end_date") + + snapshot = read_log_snapshot( + source_id, + limit, + level=level, + levels=levels, + start_date=normalized_start_date, + end_date=normalized_end_date, + search=search, + ) + if snapshot is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Log source not found") + return snapshot + + +@router.post("/logs/earth-client", response_model=EarthClientLogEventResponse) +async def ingest_earth_client_log( + payload: EarthClientLogEventCreate, + request: Request, +): + normalized_level = normalize_log_level(payload.level) + append_buffer_log( + "earth-client", + level=normalized_level, + message=payload.message, + context={ + "category": payload.category or "", + "url": payload.url or "", + "module": payload.module or "", + "detail": payload.detail or "", + }, + ) + await record_system_log( + source="earth-client", + service="earth", + module=payload.module or "earth-client", + event="earth.client.runtime_log", + level=normalized_level, + message=payload.message, + category=payload.category or "client-runtime", + context={ + "url": payload.url or "", + "detail": payload.detail or "", + "module": payload.module or "", + "client_ip": request.client.host if request.client else "", + }, + ) + return {"accepted": True, "source_id": "earth-client", "level": normalized_level} diff --git a/backend/app/api/v1/tasks.py b/backend/app/api/v1/tasks.py index 8dfb7af5..2ad29304 100644 --- a/backend/app/api/v1/tasks.py +++ b/backend/app/api/v1/tasks.py @@ -27,7 +27,9 @@ async def list_tasks( offset = (page - 1) * page_size query = """ SELECT ct.id, ct.datasource_id, ds.name as datasource_name, ct.status, - ct.started_at, ct.completed_at, ct.records_processed, ct.error_message + ct.started_at, ct.completed_at, ct.records_processed, ct.error_message, + ct.phase, ct.phase_progress, ct.phase_message, ct.phase_current, + ct.phase_total, ct.phase_unit, ct.total_records, ct.progress FROM collection_tasks ct JOIN data_sources ds ON ct.datasource_id = ds.id WHERE 1=1 @@ -66,6 +68,14 @@ async def list_tasks( "completed_at": to_iso8601_utc(t[5]), "records_processed": t[6], "error_message": t[7], + "phase": t[8], + "phase_progress": t[9], + "phase_message": t[10], + "phase_current": t[11], + "phase_total": t[12], + "phase_unit": t[13], + "total_records": t[14], + "progress": t[15], } for t in tasks ], @@ -81,7 +91,9 @@ async def get_task( result = await db.execute( text(""" SELECT ct.id, ct.datasource_id, ds.name as datasource_name, ct.status, - ct.started_at, ct.completed_at, ct.records_processed, ct.error_message + ct.started_at, ct.completed_at, ct.records_processed, ct.error_message, + ct.phase, ct.phase_progress, ct.phase_message, ct.phase_current, + ct.phase_total, ct.phase_unit, ct.total_records, ct.progress FROM collection_tasks ct JOIN data_sources ds ON ct.datasource_id = ds.id WHERE ct.id = :id @@ -105,6 +117,14 @@ async def get_task( "completed_at": to_iso8601_utc(task[5]), "records_processed": task[6], "error_message": task[7], + "phase": task[8], + "phase_progress": task[9], + "phase_message": task[10], + "phase_current": task[11], + "phase_total": task[12], + "phase_unit": task[13], + "total_records": task[14], + "progress": task[15], } diff --git a/backend/app/api/v1/users.py b/backend/app/api/v1/users.py index bc547f65..fdcfca95 100644 --- a/backend/app/api/v1/users.py +++ b/backend/app/api/v1/users.py @@ -1,3 +1,4 @@ +import json from typing import List from fastapi import APIRouter, Depends, HTTPException, status @@ -7,10 +8,12 @@ from sqlalchemy import text from app.core.security import get_current_user, get_password_hash from app.db.session import get_db from app.models.user import User -from app.schemas.user import UserCreate, UserResponse, UserUpdate +from app.schemas.user import UserCreate, UserUpdate router = APIRouter() +VALID_GATEKEEPER_GROUPS = {"docs_user", "docs_developer", "docs_admin"} + def check_permission(current_user: User, required_roles: List[str]) -> bool: user_role_value = ( @@ -52,7 +55,7 @@ async def list_users( offset = (page - 1) * page_size query = text( - f"SELECT id, username, email, role, is_active, last_login_at, created_at FROM users WHERE {where_sql} ORDER BY created_at DESC LIMIT {page_size} OFFSET {offset}" + f"SELECT id, username, email, role, is_active, last_login_at, created_at, gatekeeper_groups FROM users WHERE {where_sql} ORDER BY created_at DESC LIMIT {page_size} OFFSET {offset}" ) count_query = text(f"SELECT COUNT(*) FROM users WHERE {where_sql}") @@ -75,6 +78,7 @@ async def list_users( "is_active": u[4], "last_login_at": u[5], "created_at": u[6], + "gatekeeper_groups": u[7] or [], } for u in users ], @@ -95,7 +99,7 @@ async def get_user( result = await db.execute( text( - "SELECT id, username, email, role, is_active, last_login_at, created_at FROM users WHERE id = :id" + "SELECT id, username, email, role, is_active, last_login_at, created_at, gatekeeper_groups FROM users WHERE id = :id" ), {"id": user_id}, ) @@ -114,6 +118,7 @@ async def get_user( "is_active": user[4], "last_login_at": user[5], "created_at": user[6], + "gatekeeper_groups": user[7] or [], } @@ -128,6 +133,12 @@ async def create_user( status_code=status.HTTP_403_FORBIDDEN, detail="Only super_admin can create users", ) + invalid_groups = sorted(set(user_data.gatekeeper_groups) - VALID_GATEKEEPER_GROUPS) + if invalid_groups: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported Gatekeeper groups: {', '.join(invalid_groups)}", + ) result = await db.execute( text("SELECT id FROM users WHERE username = :username OR email = :email"), @@ -142,13 +153,14 @@ async def create_user( hashed_password = get_password_hash(user_data.password) await db.execute( - text("""INSERT INTO users (username, email, password_hash, role, is_active, created_at, updated_at) - VALUES (:username, :email, :password_hash, :role, :is_active, NOW(), NOW())"""), + text("""INSERT INTO users (username, email, password_hash, role, gatekeeper_groups, is_active, created_at, updated_at) + VALUES (:username, :email, :password_hash, :role, CAST(:gatekeeper_groups AS jsonb), :is_active, NOW(), NOW())"""), { "username": user_data.username, "email": user_data.email, "password_hash": hashed_password, "role": user_data.role, + "gatekeeper_groups": json.dumps(user_data.gatekeeper_groups), "is_active": True, }, ) @@ -172,6 +184,7 @@ async def create_user( "username": user_data.username, "email": user_data.email, "role": user_data.role, + "gatekeeper_groups": user_data.gatekeeper_groups, "is_active": True, } @@ -194,6 +207,18 @@ async def update_user( status_code=status.HTTP_403_FORBIDDEN, detail="Only super_admin can change user role", ) + if not check_permission(current_user, ["super_admin"]) and user_data.gatekeeper_groups is not None: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Only super_admin can change Gatekeeper groups", + ) + if user_data.gatekeeper_groups is not None: + invalid_groups = sorted(set(user_data.gatekeeper_groups) - VALID_GATEKEEPER_GROUPS) + if invalid_groups: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported Gatekeeper groups: {', '.join(invalid_groups)}", + ) result = await db.execute( text("SELECT id FROM users WHERE id = :id"), @@ -213,6 +238,9 @@ async def update_user( if user_data.role is not None: update_fields.append("role = :role") params["role"] = user_data.role + if user_data.gatekeeper_groups is not None: + update_fields.append("gatekeeper_groups = CAST(:gatekeeper_groups AS jsonb)") + params["gatekeeper_groups"] = json.dumps(user_data.gatekeeper_groups) if user_data.is_active is not None: update_fields.append("is_active = :is_active") params["is_active"] = user_data.is_active diff --git a/backend/app/api/v1/vessel_aggregation.py b/backend/app/api/v1/vessel_aggregation.py new file mode 100644 index 00000000..0f685f3f --- /dev/null +++ b/backend/app/api/v1/vessel_aggregation.py @@ -0,0 +1,132 @@ +"""v4 strategy + v5 conflict-promotion + enrichment APIs for vessel_ais.""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.security import get_current_user +from app.db.session import get_db +from app.models.user import User +from app.models.vessel import AISConflictRecord +from app.services.vessel_aggregation_strategy import ( + StrategyValidationError, + load_strategy, + reset_strategy, + save_strategy, +) +from app.services.vessel_enrichment import ( + get_vessel_enrichment_bundle, + upsert_vessel_media_enrichment, + upsert_vessel_profile_enrichment, +) + +router = APIRouter() + + +@router.get("/strategy") +async def get_aggregation_strategy(db: AsyncSession = Depends(get_db)): + return await load_strategy(db) + + +@router.put("/strategy") +async def put_aggregation_strategy( + payload: dict[str, Any], + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + try: + return await save_strategy(db, payload) + except StrategyValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.delete("/strategy") +async def reset_aggregation_strategy( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return await reset_strategy(db) + + +@router.post("/conflicts/{mmsi}/{field}/promote-to-rule") +async def promote_conflict_to_rule( + mmsi: int, + field: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Lift the current conflict resolution into a persistent strategy rule.""" + + result = await db.execute( + select(AISConflictRecord) + .where(AISConflictRecord.target_schema == "vessel_ais") + .where(AISConflictRecord.entity_key == str(mmsi)) + .where(AISConflictRecord.field == field) + .order_by(AISConflictRecord.updated_at.desc(), AISConflictRecord.id.desc()) + .limit(1) + ) + record = result.scalar_one_or_none() + if record is None or not record.selected_source: + raise HTTPException(status_code=404, detail="Conflict record with selected_source not found") + + strategy = await load_strategy(db) + vessel_ais = dict(strategy.get("vessel_ais") or {}) + field_rules = dict(vessel_ais.get("field_rules") or {}) + field_rules[field] = {"mode": "source_priority", "source_priority": [record.selected_source]} + vessel_ais["field_rules"] = field_rules + + incoming = {"version": int(strategy.get("version") or 0), "vessel_ais": vessel_ais} + try: + return await save_strategy(db, incoming) + except StrategyValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.delete("/conflicts/{mmsi}/{field}/promote-to-rule") +async def revert_conflict_rule( + mmsi: int, + field: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + strategy = await load_strategy(db) + vessel_ais = dict(strategy.get("vessel_ais") or {}) + field_rules = dict(vessel_ais.get("field_rules") or {}) + if field in field_rules: + del field_rules[field] + vessel_ais["field_rules"] = field_rules + + incoming = {"version": int(strategy.get("version") or 0), "vessel_ais": vessel_ais} + try: + return await save_strategy(db, incoming) + except StrategyValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/enrichment/{mmsi}") +async def get_vessel_enrichment(mmsi: int, db: AsyncSession = Depends(get_db)): + return await get_vessel_enrichment_bundle(db, mmsi) + + +@router.put("/enrichment/{mmsi}/profile") +async def put_vessel_profile_enrichment( + mmsi: int, + payload: dict[str, Any], + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return await upsert_vessel_profile_enrichment(db, mmsi=mmsi, payload=payload) + + +@router.put("/enrichment/{mmsi}/media") +async def put_vessel_media_enrichment( + mmsi: int, + payload: dict[str, Any], + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return await upsert_vessel_media_enrichment(db, mmsi=mmsi, payload=payload) diff --git a/backend/app/api/v1/visualization.py b/backend/app/api/v1/visualization.py index 23487b65..5eb07822 100644 --- a/backend/app/api/v1/visualization.py +++ b/backend/app/api/v1/visualization.py @@ -4,10 +4,15 @@ Unified API for all visualization data sources. Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium. """ -from datetime import UTC, datetime +import asyncio +import base64 +from collections import OrderedDict +from datetime import UTC, datetime, timedelta import math +import re import httpx from fastapi import APIRouter, HTTPException, Depends, Query, Response +from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, func from typing import List, Dict, Any, Optional @@ -18,15 +23,52 @@ from app.core.time import to_iso8601_utc from app.db.session import get_db from app.models.bgp_anomaly import BGPAnomaly from app.models.bgp_incident import BGPIncident +from app.models.bgp_observation import BGPObservation from app.models.collected_data import CollectedData +from app.models.vessel import AISSourceHealth, VesselPosition, VesselStatic from app.services.bgp_collectors import build_bgp_collector_coverage from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance +from app.services.compute_center_locations import ( + RENDERABLE_PRECISIONS, + ResolutionDiagnostic, + collect_location_candidates, + refresh_compute_center_location_cache, + resolve_compute_center_location_full, + upsert_compute_center_location, +) from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS +from app.services.persistent_logs import record_system_log +from app.services.vessel_ais_aggregation import ( + build_field_conflict_candidates, + count_unique_raw_vessel_mmsi, + get_aggregated_vessel, + get_aggregated_vessel_track, + get_aggregated_vessels, + get_vessel_conflict_records, + get_vessel_raw_observations, +) +from app.core.logging import get_logger router = APIRouter() +logger = get_logger(__name__, service="api") TERRAIN_TILE_URL_TEMPLATE = ( "https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png" ) +TERRAIN_TILE_CACHE_MAX_ITEMS = 512 +TERRAIN_TILE_BATCH_MAX_ITEMS = 128 +TERRAIN_TILE_BATCH_CONCURRENCY = 16 +_terrain_tile_cache: OrderedDict[tuple[int, int, int], tuple[bytes, str, dict[str, str]]] = OrderedDict() +VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE) + + +class TerrariumTileRequest(BaseModel): + z: int = Field(ge=0, le=14) + x: int = Field(ge=0) + y: int = Field(ge=0) + + +class TerrariumTileBatchRequest(BaseModel): + tiles: List[TerrariumTileRequest] = Field(min_length=1, max_length=TERRAIN_TILE_BATCH_MAX_ITEMS) # ============== Converter Functions ============== @@ -180,6 +222,12 @@ def convert_satellite_to_geojson(records: List[CollectedData]) -> Dict[str, Any] mean_motion=metadata.get("mean_motion"), ) + constellation_group = _normalize_satellite_constellation_group( + metadata.get("constellation_group"), + record.name, + ) + footprint_policy = _get_satellite_footprint_policy(constellation_group) + features.append( { "type": "Feature", @@ -189,6 +237,8 @@ def convert_satellite_to_geojson(records: List[CollectedData]) -> Dict[str, Any] "id": record.id, "norad_cat_id": norad_id, "name": record.name, + "constellation_group": constellation_group, + "footprint_policy": footprint_policy, "international_designator": metadata.get("international_designator"), "epoch": metadata.get("epoch"), "inclination": metadata.get("inclination"), @@ -209,6 +259,31 @@ def convert_satellite_to_geojson(records: List[CollectedData]) -> Dict[str, Any] return {"type": "FeatureCollection", "features": features} +def _normalize_satellite_constellation_group( + raw_group: Any, + name: Optional[str], +) -> Optional[str]: + normalized_group = str(raw_group or "").strip().lower() + if normalized_group: + return normalized_group + + normalized_name = str(name or "").strip().upper() + if normalized_name.startswith("STARLINK"): + return "starlink" + if normalized_name.startswith("IRIDIUM"): + return "iridium-next" + + return None + + +def _get_satellite_footprint_policy(constellation_group: Optional[str]) -> str: + if constellation_group == "starlink": + return "starlink_ground_footprint" + if constellation_group == "iridium-next": + return "iridium_coverage_ring" + return "none" + + def _current_collected_data_stmt(source: str): return ( select(CollectedData) @@ -235,6 +310,120 @@ async def _load_current_collected_data( return list(result.scalars().all()) +async def _latest_task_id_for_source( + db: AsyncSession, + source: str, + *, + exclude_unknown_name: bool = False, +) -> int | None: + stmt = ( + select( + CollectedData.task_id, + func.max(CollectedData.collected_at).label("latest_collected_at"), + func.max(CollectedData.id).label("latest_id"), + ) + .where(CollectedData.source == source) + .where(CollectedData.task_id.isnot(None)) + .group_by(CollectedData.task_id) + .order_by(func.max(CollectedData.collected_at).desc(), func.max(CollectedData.id).desc()) + .limit(1) + ) + if exclude_unknown_name: + stmt = stmt.where(CollectedData.name != "Unknown") + + result = await db.execute(stmt) + row = result.first() + return int(row.task_id) if row and row.task_id is not None else None + + +async def _load_current_or_latest_task_data( + db: AsyncSession, + source: str, + *, + exclude_unknown_name: bool = False, + limit: Optional[int] = None, +) -> List[CollectedData]: + records = await _load_current_collected_data( + db, + source, + exclude_unknown_name=exclude_unknown_name, + limit=limit, + ) + if records: + return records + + latest_task_id = await _latest_task_id_for_source( + db, + source, + exclude_unknown_name=exclude_unknown_name, + ) + if latest_task_id is None: + return [] + + stmt = ( + select(CollectedData) + .where(CollectedData.source == source) + .where(CollectedData.task_id == latest_task_id) + .order_by(CollectedData.id.desc()) + ) + if exclude_unknown_name: + stmt = stmt.where(CollectedData.name != "Unknown") + if limit is not None: + stmt = stmt.limit(limit) + + result = await db.execute(stmt) + return list(result.scalars().all()) + + +async def _count_current_or_latest_task_data( + db: AsyncSession, + source: str, + *, + exclude_unknown_name: bool = False, +) -> int: + current_stmt = ( + select(func.count(CollectedData.id)) + .where(CollectedData.source == source) + .where(CollectedData.is_current.is_(True)) + ) + if exclude_unknown_name: + current_stmt = current_stmt.where(CollectedData.name != "Unknown") + + current_result = await db.execute(current_stmt) + current_scalar = current_result.scalar() + if current_scalar is None and hasattr(current_result, "scalars"): + current_rows = current_result.scalars().all() + current_count = sum( + 1 + for row in current_rows + if getattr(row, "source", None) == source + and (not exclude_unknown_name or getattr(row, "name", None) != "Unknown") + ) + else: + current_count = int(current_scalar or 0) + if current_count > 0: + return current_count + + latest_task_id = await _latest_task_id_for_source( + db, + source, + exclude_unknown_name=exclude_unknown_name, + ) + if latest_task_id is None: + return 0 + + latest_stmt = ( + select(func.count(CollectedData.id)) + .where(CollectedData.source == source) + .where(CollectedData.task_id == latest_task_id) + ) + if exclude_unknown_name: + latest_stmt = latest_stmt.where(CollectedData.name != "Unknown") + + latest_result = await db.execute(latest_stmt) + return int(latest_result.scalar() or 0) + + async def _load_current_collected_data_by_sources( db: AsyncSession, sources: List[str], @@ -363,6 +552,407 @@ def convert_gpu_cluster_to_geojson(records: List[CollectedData]) -> Dict[str, An return {"type": "FeatureCollection", "features": features} +def _parse_float(value: Any) -> Optional[float]: + try: + if value in (None, ""): + return None + return float(value) + except (TypeError, ValueError): + return None + + +def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str) -> str: + if capacity_value is None: + return "unknown" + + unit = str(capacity_unit or "").strip().lower() + if unit in {"pflop/s", "pflops", "pflop"}: + normalized_tflops = capacity_value * 1000 + elif unit in {"gflop/s", "gflops", "gflop"}: + normalized_tflops = capacity_value + else: + normalized_tflops = capacity_value + + if normalized_tflops >= 1_000_000: + return "exascale" + if normalized_tflops >= 100_000: + return "ultra" + if normalized_tflops >= 10_000: + return "large" + if normalized_tflops > 0: + return "regional" + return "unknown" + + +def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str, Any]: + """Convert compute infrastructure records into a unified GeoJSON layer. + + Records that cannot be resolved to at least city-level precision are NOT + silently dropped: they are returned in ``unresolved`` so the UI can offer + the click-to-collect coordinate flow. The features list never contains + ``[0, 0]`` placeholders or country/region/unknown precision points. + """ + features: List[Dict[str, Any]] = [] + unresolved: List[Dict[str, Any]] = [] + + for record in records: + metadata = record.extra_data or {} + result = resolve_compute_center_location_full(record, metadata) + site_type = ( + "supercomputer" + if record.source == "top500" or record.data_type == "supercomputer" + else "gpu_cluster" + ) + + if not result.is_resolved: + diagnostic = result.diagnostic or ResolutionDiagnostic( + failure_reason="Unknown resolver failure", + attempted_queries=(), + record_id=getattr(record, "id", None), + source=getattr(record, "source", None), + source_id=getattr(record, "source_id", None), + name=getattr(record, "name", None), + ) + unresolved.append({ + **diagnostic.to_dict(), + "site_type": site_type, + }) + continue + + location = result.location + if location is None or not location.is_renderable: + # Defensive: should not happen because is_resolved guards this. + continue + + location_props = location.to_geojson_properties() + latitude = location.latitude + longitude = location.longitude + + if site_type == "supercomputer": + capacity_value = _parse_float(get_record_field(record, "rmax")) + capacity_unit = "GFlops" + else: + capacity_value = _parse_float(get_record_field(record, "value")) + capacity_unit = str(get_record_field(record, "unit") or "TFlop/s") + + vendor = ( + metadata.get("manufacturer") + or metadata.get("vendor") + or metadata.get("gpu_type") + ) + operator = ( + metadata.get("organization") + or metadata.get("operator") + or metadata.get("owner") + ) + rank = metadata.get("rank") + if rank in (None, "") and site_type == "supercomputer": + rank = get_record_field(record, "rank") + + updated_at = to_iso8601_utc(record.reference_date or record.collected_at) + + features.append( + { + "type": "Feature", + "id": record.id, + "geometry": { + "type": "Point", + "coordinates": [longitude, latitude], + }, + "properties": { + "id": record.id, + "source_id": record.source_id, + "name": record.name, + "site_type": site_type, + "country": get_record_field(record, "country") or location.country, + "city": get_record_field(record, "city") or location.city, + "region": location.region, + "latitude": latitude, + "longitude": longitude, + "operator": operator, + "vendor": vendor, + "capacity_value": capacity_value, + "capacity_unit": capacity_unit, + "capacity_band": _normalize_capacity_band(capacity_value, capacity_unit), + "rank": rank, + "gpu_count": metadata.get("gpu_count"), + "gpu_type": metadata.get("gpu_type"), + "cores": get_record_field(record, "cores"), + "power": get_record_field(record, "power"), + "source": record.source, + "updated_at": updated_at, + "status": "observed", + **location_props, + "data_type": "compute_center", + "metadata": metadata, + }, + } + ) + + return {"type": "FeatureCollection", "features": features, "unresolved": unresolved} + + +VESSEL_TYPE_FILTERS = { + "cargo": lambda props: str(props.get("vessel_type_name", "")).lower() == "cargo" + or 70 <= int(props.get("vessel_type") or -1) <= 79, + "tanker": lambda props: str(props.get("vessel_type_name", "")).lower() == "tanker" + or 80 <= int(props.get("vessel_type") or -1) <= 89, + "passenger": lambda props: str(props.get("vessel_type_name", "")).lower() == "passenger" + or 60 <= int(props.get("vessel_type") or -1) <= 69, + "fishing": lambda props: str(props.get("vessel_type_name", "")).lower() == "fishing" + or int(props.get("vessel_type") or -1) == 30, + "military": lambda props: str(props.get("vessel_type_name", "")).lower() == "military" + or int(props.get("vessel_type") or -1) == 35, + "other": lambda props: str(props.get("vessel_type_name", "")).lower() + not in {"cargo", "tanker", "passenger", "fishing", "military"}, +} + + +def convert_vessels_to_geojson(rows: List[Any]) -> Dict[str, Any]: + features = [] + seen_mmsi: set[int] = set() + for position, static in rows: + if position.lat is None or position.lon is None: + continue + if position.mmsi in seen_mmsi: + continue + seen_mmsi.add(position.mmsi) + props = { + "mmsi": position.mmsi, + "mmsi_display": str(position.mmsi), + "name": getattr(static, "name", None) or f"MMSI {position.mmsi}", + "name_is_fallback": _is_vessel_name_fallback(getattr(static, "name", None), position.mmsi), + "callsign": getattr(static, "callsign", None), + "imo": getattr(static, "imo", None), + "imo_display": str(getattr(static, "imo")) if getattr(static, "imo", None) else None, + "vessel_type": getattr(static, "vessel_type", None), + "vessel_type_name": getattr(static, "vessel_type_name", None) or "Other", + "flag": getattr(static, "flag", None), + "length": getattr(static, "length", None), + "width": getattr(static, "width", None), + "draught": getattr(static, "draught", None), + "sog": position.sog, + "cog": position.cog, + "heading": position.heading, + "nav_status": position.nav_status, + "received_at": to_iso8601_utc(position.received_at), + "data_type": "vessel", + } + features.append( + { + "type": "Feature", + "id": position.mmsi, + "geometry": { + "type": "Point", + "coordinates": [position.lon, position.lat], + }, + "properties": props, + } + ) + + return {"type": "FeatureCollection", "features": features} + + +def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict[str, Any]: + features = [] + for vessel in vessels: + if vessel.get("lat") is None or vessel.get("lon") is None: + continue + source_summary = {} + for source, summary in (vessel.get("source_summary") or {}).items(): + source_summary[source] = { + **summary, + "latest_observed_at": to_iso8601_utc(summary.get("latest_observed_at")), + } + props = { + "mmsi": vessel["mmsi"], + "mmsi_display": str(vessel["mmsi"]), + "name": vessel.get("name") or f"MMSI {vessel['mmsi']}", + "name_is_fallback": _is_vessel_name_fallback(vessel.get("name"), vessel["mmsi"]), + "callsign": vessel.get("callsign"), + "imo": vessel.get("imo"), + "imo_display": str(vessel.get("imo")) if vessel.get("imo") else None, + "vessel_type": vessel.get("vessel_type"), + "vessel_type_name": vessel.get("vessel_type_name") or "Other", + "flag": vessel.get("flag"), + "length": vessel.get("length"), + "width": vessel.get("width"), + "draught": vessel.get("draught"), + "sog": vessel.get("sog"), + "cog": vessel.get("cog"), + "heading": vessel.get("heading"), + "nav_status": vessel.get("nav_status"), + "received_at": to_iso8601_utc(vessel.get("received_at")), + "field_sources": vessel.get("field_sources") or {}, + "selected_reasons": vessel.get("selected_reasons") or {}, + "source_summary": source_summary, + "quality_flags": vessel.get("quality_flags") or [], + "conflict_count": vessel.get("conflict_count", 0), + "aggregation_strategy_version": vessel.get("aggregation_strategy_version", 0), + "data_type": "vessel", + } + features.append( + { + "type": "Feature", + "id": vessel["mmsi"], + "geometry": { + "type": "Point", + "coordinates": [vessel["lon"], vessel["lat"]], + }, + "properties": props, + } + ) + return {"type": "FeatureCollection", "features": features} + + +def _parse_bbox(value: Optional[str]) -> tuple[float, float, float, float] | None: + if not value: + return None + parts = [part.strip() for part in value.split(",")] + if len(parts) != 4: + raise HTTPException(status_code=400, detail="bbox must be lon_min,lat_min,lon_max,lat_max") + try: + lon_min, lat_min, lon_max, lat_max = [float(part) for part in parts] + except ValueError as exc: + raise HTTPException(status_code=400, detail="bbox values must be numbers") from exc + if lat_min > lat_max: + lat_min, lat_max = lat_max, lat_min + if lon_min > lon_max: + lon_min, lon_max = lon_max, lon_min + return lon_min, lat_min, lon_max, lat_max + + +def _is_vessel_name_fallback(name: Any, mmsi: Any) -> bool: + text = str(name or "").strip() + mmsi_text = str(mmsi or "").strip() + if not text: + return True + if mmsi_text and text == mmsi_text: + return True + return bool(VESSEL_NAME_FALLBACK_PATTERN.match(text)) + + +def _requested_vessel_types(value: Optional[str]) -> set[str]: + return { + item.strip().lower() + for item in (value or "").split(",") + if item.strip() + } + + +def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bool: + if not requested_types: + return True + for requested_type in requested_types: + predicate = VESSEL_TYPE_FILTERS.get(requested_type) + if predicate and predicate(props): + return True + return False + + +def _feature_mmsi_key(feature: dict[str, Any]) -> str | None: + props = feature.get("properties", {}) + mmsi = props.get("mmsi") or feature.get("id") + if mmsi in (None, ""): + return None + return str(mmsi) + + +def _feature_in_bbox(feature: dict[str, Any], bbox: tuple[float, float, float, float] | None) -> bool: + if bbox is None: + return True + coordinates = feature.get("geometry", {}).get("coordinates") or [] + if len(coordinates) < 2: + return False + try: + lon = float(coordinates[0]) + lat = float(coordinates[1]) + except (TypeError, ValueError): + return False + lon_min, lat_min, lon_max, lat_max = bbox + return lon_min <= lon <= lon_max and lat_min <= lat <= lat_max + + +def _filter_vessel_features( + features: list[dict[str, Any]], + *, + bbox: tuple[float, float, float, float] | None, + requested_types: set[str], +) -> list[dict[str, Any]]: + return [ + feature + for feature in features + if _feature_in_bbox(feature, bbox) + and _matches_vessel_type(feature.get("properties", {}), requested_types) + ] + + +def _merge_vessel_features( + raw_features: list[dict[str, Any]], + legacy_features: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Prefer aggregated raw observations as the canonical source of truth. + + Legacy `vessel_position` rows only fill MMSIs that the unified pipeline does + not yet know about, so a vessel never appears twice when both BarentsWatch + and AISStream observe it. Once the legacy table drains, this branch becomes + a no-op. + """ + + merged: list[dict[str, Any]] = [] + seen: set[str] = set() + raw_keys: set[str] = set() + legacy_keys: set[str] = set() + + for feature in raw_features: + key = _feature_mmsi_key(feature) + if key is None or key in seen: + continue + seen.add(key) + raw_keys.add(key) + merged.append(feature) + + legacy_added = 0 + for feature in legacy_features: + key = _feature_mmsi_key(feature) + if key is None: + continue + legacy_keys.add(key) + if key in seen: + continue + seen.add(key) + legacy_added += 1 + merged.append(feature) + + return merged, { + "raw_unique_mmsi": len(raw_keys), + "legacy_unique_mmsi": len(legacy_keys), + "legacy_backfilled_mmsi": legacy_added, + "final_unique_mmsi": len(seen), + } + + +def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]: + by_type: dict[str, int] = {} + underway = 0 + anchored_or_moored = 0 + for feature in features: + props = feature.get("properties", {}) + vessel_type = str(props.get("vessel_type_name") or "Other") + by_type[vessel_type] = by_type.get(vessel_type, 0) + 1 + nav_status = props.get("nav_status") + if nav_status in (1, 5): + anchored_or_moored += 1 + else: + underway += 1 + return { + "total": len(features), + "by_type": by_type, + "underway": underway, + "anchored_or_moored": anchored_or_moored, + } + + def convert_bgp_anomalies_to_geojson( records: List[BGPAnomaly], geography_hints: Optional[Dict[str, Dict[str, Any]]] = None, @@ -780,6 +1370,21 @@ async def get_cables_geojson(db: AsyncSession = Depends(get_db)): except HTTPException: raise except Exception as e: + logger.exception_event( + "Failed to build cables GeoJSON response", + event="visualization.cables.load_failed", + context={"error": str(e)}, + ) + await record_system_log( + source="backend", + service="api", + module=__name__, + event="visualization.cables.load_failed", + level="error", + message="Failed to build cables GeoJSON response", + category="visualization", + context={"error": str(e)}, + ) raise HTTPException(status_code=500, detail=f"Internal error: {str(e)}") @@ -816,24 +1421,33 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)): except HTTPException: raise except Exception as e: + logger.exception_event( + "Failed to build landing points GeoJSON response", + event="visualization.landing_points.load_failed", + context={"error": str(e)}, + ) + await record_system_log( + source="backend", + service="api", + module=__name__, + event="visualization.landing_points.load_failed", + level="error", + message="Failed to build landing points GeoJSON response", + category="visualization", + context={"error": str(e)}, + ) raise HTTPException(status_code=500, detail=f"Internal error: {str(e)}") @router.get("/terrain/terrarium/{z}/{x}/{y}.png") async def get_terrarium_tile(z: int, x: int, y: int): """Proxy Terrarium elevation tiles through the backend to avoid browser CORS issues.""" - if z < 0 or x < 0 or y < 0: + if not _is_valid_terrain_tile(z, x, y): raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates") - url = TERRAIN_TILE_URL_TEMPLATE.format(z=z, x=x, y=y) - try: - async with httpx.AsyncClient( - timeout=20.0, - follow_redirects=True, - ) as client: - upstream = await client.get(url) - upstream.raise_for_status() + async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client: + content, content_type, headers = await _fetch_terrain_tile(client, z, x, y) except httpx.HTTPStatusError as exc: raise HTTPException( status_code=exc.response.status_code, @@ -845,22 +1459,140 @@ async def get_terrarium_tile(z: int, x: int, y: int): detail=f"Terrain tile fetch failed: {exc}", ) from exc + return Response( + content=content, + media_type=content_type, + headers=headers, + ) + + +def _is_valid_terrain_tile(z: int, x: int, y: int) -> bool: + if z < 0 or x < 0 or y < 0: + return False + max_tile = 2 ** z + return x < max_tile and y < max_tile + + +def _get_cached_terrain_tile(z: int, x: int, y: int) -> tuple[bytes, str, dict[str, str]] | None: + key = (z, x, y) + cached = _terrain_tile_cache.get(key) + if cached is None: + return None + _terrain_tile_cache.move_to_end(key) + content, content_type, headers = cached + return content, content_type, dict(headers) + + +def _cache_terrain_tile( + z: int, + x: int, + y: int, + content: bytes, + content_type: str, + headers: dict[str, str], +) -> None: + key = (z, x, y) + _terrain_tile_cache[key] = (content, content_type, dict(headers)) + _terrain_tile_cache.move_to_end(key) + while len(_terrain_tile_cache) > TERRAIN_TILE_CACHE_MAX_ITEMS: + _terrain_tile_cache.popitem(last=False) + + +async def _fetch_terrain_tile( + client: httpx.AsyncClient, + z: int, + x: int, + y: int, +) -> tuple[bytes, str, dict[str, str]]: + cached = _get_cached_terrain_tile(z, x, y) + if cached is not None: + return cached + + url = TERRAIN_TILE_URL_TEMPLATE.format(z=z, x=x, y=y) + upstream = await client.get(url) + upstream.raise_for_status() + cache_control = upstream.headers.get("cache-control") or "public, max-age=86400" - etag = upstream.headers.get("etag") - last_modified = upstream.headers.get("last-modified") headers = { "Cache-Control": cache_control, } + etag = upstream.headers.get("etag") + last_modified = upstream.headers.get("last-modified") if etag: headers["ETag"] = etag if last_modified: headers["Last-Modified"] = last_modified - return Response( - content=upstream.content, - media_type=upstream.headers.get("content-type", "image/png"), - headers=headers, - ) + content_type = upstream.headers.get("content-type", "image/png") + content = upstream.content + _cache_terrain_tile(z, x, y, content, content_type, headers) + return content, content_type, dict(headers) + + +@router.post("/terrain/terrarium/batch") +async def get_terrarium_tile_batch(payload: TerrariumTileBatchRequest): + """Fetch Terrarium elevation tiles in batches so the browser avoids many tiny requests.""" + unique_tiles: list[TerrariumTileRequest] = [] + seen: set[tuple[int, int, int]] = set() + for tile in payload.tiles: + key = (tile.z, tile.x, tile.y) + if key in seen: + continue + seen.add(key) + if not _is_valid_terrain_tile(tile.z, tile.x, tile.y): + raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates") + unique_tiles.append(tile) + + semaphore = asyncio.Semaphore(TERRAIN_TILE_BATCH_CONCURRENCY) + results: list[dict[str, Any]] = [] + errors: list[dict[str, Any]] = [] + + async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client: + async def fetch_one(tile: TerrariumTileRequest) -> None: + async with semaphore: + try: + content, content_type, _headers = await _fetch_terrain_tile( + client, + tile.z, + tile.x, + tile.y, + ) + results.append( + { + "z": tile.z, + "x": tile.x, + "y": tile.y, + "content_type": content_type, + "data": base64.b64encode(content).decode("ascii"), + }, + ) + except httpx.HTTPStatusError as exc: + errors.append( + { + "z": tile.z, + "x": tile.x, + "y": tile.y, + "status_code": exc.response.status_code, + "message": f"upstream error: {exc.response.status_code}", + }, + ) + except httpx.HTTPError as exc: + errors.append( + { + "z": tile.z, + "x": tile.x, + "y": tile.y, + "status_code": 502, + "message": str(exc), + }, + ) + + await asyncio.gather(*(fetch_one(tile) for tile in unique_tiles)) + + return { + "tiles": results, + "errors": errors, + } @router.get("/geo/all") @@ -912,7 +1644,7 @@ async def get_satellites_geojson( db: AsyncSession = Depends(get_db), ): """获取卫星 TLE GeoJSON 数据""" - records = await _load_current_collected_data( + records = await _load_current_or_latest_task_data( db, "celestrak_tle", exclude_unknown_name=True, @@ -975,6 +1707,571 @@ async def get_gpu_clusters_geojson( } +@router.get("/geo/compute-centers") +async def get_compute_centers_geojson( + limit: int = Query(200, ge=1, le=1000), + db: AsyncSession = Depends(get_db), +): + """获取统一算力中心 GeoJSON 数据""" + records_by_source = await _load_current_collected_data_by_sources( + db, + ["top500", "epoch_ai_gpu"], + ) + records = _filter_known_records( + records_by_source.get("top500", []) + records_by_source.get("epoch_ai_gpu", []), + ) + if limit is not None: + records = records[:limit] + + if not records: + return { + "type": "FeatureCollection", + "features": [], + "unresolved": [], + "count": 0, + "stats": { + "total": 0, + "supercomputers": 0, + "gpu_clusters": 0, + "unresolved": 0, + }, + } + + await refresh_compute_center_location_cache(db) + geojson = convert_compute_centers_to_geojson(records) + features = geojson.get("features", []) + unresolved = geojson.get("unresolved", []) + # Belt-and-suspenders: ensure no Feature ever sneaks through without + # city-or-better precision and finite, non-zero coordinates. + sanitized_features: List[Dict[str, Any]] = [] + for feature in features: + coords = feature.get("geometry", {}).get("coordinates") or [] + precision = feature.get("properties", {}).get("location_precision") + if precision not in RENDERABLE_PRECISIONS: + unresolved.append({ + "failure_reason": f"Rejected non-renderable precision '{precision}'", + "record_id": feature.get("id"), + "source_id": feature.get("properties", {}).get("source_id"), + "name": feature.get("properties", {}).get("name"), + }) + continue + if ( + len(coords) != 2 + or coords[0] in (None, 0, 0.0) + or coords[1] in (None, 0, 0.0) + ): + unresolved.append({ + "failure_reason": "Rejected feature with [0,0] or invalid coordinates", + "record_id": feature.get("id"), + "source_id": feature.get("properties", {}).get("source_id"), + "name": feature.get("properties", {}).get("name"), + }) + continue + sanitized_features.append(feature) + return { + "type": "FeatureCollection", + "features": sanitized_features, + "unresolved": unresolved, + "count": len(sanitized_features), + "stats": { + "total": len(sanitized_features), + "supercomputers": sum( + 1 for feature in sanitized_features + if feature.get("properties", {}).get("site_type") == "supercomputer" + ), + "gpu_clusters": sum( + 1 for feature in sanitized_features + if feature.get("properties", {}).get("site_type") == "gpu_cluster" + ), + "unresolved": len(unresolved), + }, + } + + +class CollectComputeCenterLocationRequest(BaseModel): + name: Optional[str] = None + source: Optional[str] = None + operator: Optional[str] = None + site: Optional[str] = None + organization: Optional[str] = None + city: Optional[str] = None + country: Optional[str] = None + record_id: Optional[int] = Field(default=None, alias="id") + + model_config = {"populate_by_name": True} + + +class SaveComputeCenterLocationRequest(BaseModel): + source: Optional[str] = None + name: Optional[str] = None + operator: Optional[str] = None + site: Optional[str] = None + city: Optional[str] = None + country: Optional[str] = None + latitude: float + longitude: float + precision: str = "city" + confidence: Optional[float] = None + location_source: Optional[str] = None + source_url: Optional[str] = None + source_note: Optional[str] = None + raw_payload: Dict[str, Any] = Field(default_factory=dict) + needs_confirmation: bool = False + verification_status: Optional[str] = None + + model_config = {"populate_by_name": True} + + +@router.post("/compute-centers/{source_id}/collect-location") +async def collect_compute_center_location( + source_id: str, + payload: CollectComputeCenterLocationRequest, + db: AsyncSession = Depends(get_db), +): + """Run the full multi-query location collection pipeline for a record. + + The endpoint accepts the source_id of a compute center plus contextual + fields (name/operator/site/city/country/...) and returns ranked candidate + locations from source coordinates, open organization lookups, and online + geocoding combinations. The caller never has to type coordinates by hand: + if any candidate is accepted it can be applied directly. If no candidate + can reach city-level precision the response includes an explicit + ``failure_reason`` and the list of attempted queries. + """ + if not source_id or not source_id.strip(): + raise HTTPException(status_code=400, detail="source_id is required") + + record = await _load_compute_center_record(db, source_id) + name = payload.name or (record.name if record else None) + metadata = (record.extra_data or {}) if record else {} + + operator = payload.operator or metadata.get("operator") or metadata.get("organization") or metadata.get("owner") + site = payload.site or metadata.get("site") + organization = payload.organization or metadata.get("organization") + city = payload.city or get_record_field(record, "city") if record else payload.city + country = payload.country or (get_record_field(record, "country") if record else None) + source = payload.source or (record.source if record else None) + record_id = payload.record_id or (record.id if record else None) + + candidates, attempted_queries = collect_location_candidates( + name=name, + source=source, + source_id=source_id, + operator=operator, + site=site, + organization=organization, + city=city, + country=country, + record_id=record_id, + ) + + if not candidates: + return { + "source_id": source_id, + "record_id": record_id, + "name": name, + "success": False, + "failure_reason": ( + "No source coordinates, organization lookup, or online geocoding" + " result reached city-level precision." + ), + "candidates": [], + "attempted_queries": list(attempted_queries), + "context": { + "name": name, + "operator": operator, + "site": site, + "city": city, + "country": country, + }, + } + + return { + "source_id": source_id, + "record_id": record_id, + "name": name, + "success": True, + "candidates": [candidate.to_dict() for candidate in candidates], + "best_candidate": candidates[0].to_dict(), + "attempted_queries": list(attempted_queries), + "context": { + "name": name, + "operator": operator, + "site": site, + "city": city, + "country": country, + }, + } + + +@router.post("/compute-centers/{source_id}/location") +async def save_compute_center_location( + source_id: str, + payload: SaveComputeCenterLocationRequest, + db: AsyncSession = Depends(get_db), +): + """Persist the user-selected compute-center location candidate.""" + if not source_id or not source_id.strip(): + raise HTTPException(status_code=400, detail="source_id is required") + if payload.latitude in (0.0, None) or payload.longitude in (0.0, None): + raise HTTPException(status_code=400, detail="latitude/longitude are required") + if payload.precision not in RENDERABLE_PRECISIONS: + raise HTTPException(status_code=400, detail="precision must be precise, site, or city") + + record = await _load_compute_center_record(db, source_id) + metadata = (record.extra_data or {}) if record else {} + record_source = payload.source or (record.source if record else None) + if not record_source: + raise HTTPException(status_code=400, detail="source is required for unknown compute center") + + operator = ( + payload.operator + or metadata.get("operator") + or metadata.get("organization") + or metadata.get("owner") + or metadata.get("manufacturer") + ) + site = payload.site or metadata.get("site") or metadata.get("organization") + saved = await upsert_compute_center_location( + db, + source=record_source, + source_id=source_id, + name=payload.name or (record.name if record else None), + operator=operator, + site=site, + city=payload.city or (get_record_field(record, "city") if record else None), + country=payload.country or (get_record_field(record, "country") if record else None), + latitude=payload.latitude, + longitude=payload.longitude, + precision=payload.precision, + confidence=payload.confidence, + location_source=payload.location_source or "manual_selection", + source_url=payload.source_url, + source_note=payload.source_note, + raw_payload=payload.raw_payload, + needs_confirmation=payload.needs_confirmation, + verification_status=payload.verification_status + or ("unverified" if payload.needs_confirmation else "verified"), + ) + + return { + "success": True, + "source": saved.source, + "source_id": saved.source_id, + "location": saved.to_location_dict(), + } + + +async def _load_compute_center_record(db: AsyncSession, source_id: str) -> CollectedData | None: + stmt = ( + select(CollectedData) + .where(CollectedData.source_id == source_id) + .where(CollectedData.source.in_(["top500", "epoch_ai_gpu"])) + .order_by(CollectedData.is_current.desc(), CollectedData.id.desc()) + .limit(1) + ) + result = await db.execute(stmt) + return result.scalars().first() + + +@router.get("/geo/vessels") +async def get_vessels_geojson( + bbox: Optional[str] = Query( + None, + description="Viewport bbox as lon_min,lat_min,lon_max,lat_max", + ), + type: Optional[str] = Query( + None, + description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other", + ), + limit: Optional[int] = Query( + None, + ge=0, + description="Maximum vessel features to return. Omit or pass 0 for no limit.", + ), + db: AsyncSession = Depends(get_db), +): + """Return latest vessel positions as GeoJSON points.""" + parsed_bbox = _parse_bbox(bbox) + requested_types = _requested_vessel_types(type) + merged_features, diagnostics = await _load_merged_vessel_features(db) + features = _filter_vessel_features( + merged_features, + bbox=parsed_bbox, + requested_types=requested_types, + ) + if limit and limit > 0: + features = features[:limit] + return { + "type": "FeatureCollection", + "features": features, + "count": len(features), + "stats": _build_vessel_stats(features), + "diagnostics": { + **diagnostics, + "filtered_count": len(features), + }, + } + + +async def _load_merged_vessel_features(db: AsyncSession) -> tuple[list[dict[str, Any]], dict[str, Any]]: + aggregated_vessels = await get_aggregated_vessels(db) + raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels) + + latest_times = ( + select( + VesselPosition.mmsi.label("mmsi"), + func.max(VesselPosition.received_at).label("received_at"), + ) + .group_by(VesselPosition.mmsi) + .subquery() + ) + stmt = ( + select(VesselPosition, VesselStatic) + .join( + latest_times, + (VesselPosition.mmsi == latest_times.c.mmsi) + & (VesselPosition.received_at == latest_times.c.received_at), + ) + .outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi) + .order_by(VesselPosition.received_at.desc()) + ) + + result = await db.execute(stmt) + rows = list(result.all()) + legacy_geojson = convert_vessels_to_geojson(rows) + merged_features, diagnostics = _merge_vessel_features( + raw_geojson.get("features", []), + legacy_geojson.get("features", []), + ) + return merged_features, { + **diagnostics, + "raw_feature_count": len(raw_geojson.get("features", [])), + "legacy_feature_count": len(legacy_geojson.get("features", [])), + } + + +@router.get("/vessels/custom-supplements") +async def get_vessel_custom_supplements(db: AsyncSession = Depends(get_db)): + """Group custom vessel_ais sources by their declared merge target for diagnostics.""" + + from app.models.datasource_config import DataSourceConfig + + result = await db.execute( + select(DataSourceConfig.name, DataSourceConfig.config, DataSourceConfig.is_active) + .where(DataSourceConfig.config["target_schema"].as_string() == "vessel_ais") + ) + grouped: dict[str, dict[str, Any]] = {} + for name, config, is_active in result.all(): + config = config or {} + merge_target = str(config.get("merge_target_source") or "barentswatch_vessels") + bucket = grouped.setdefault(merge_target, {"merge_target": merge_target, "sources": []}) + bucket["sources"].append({"name": name, "is_active": bool(is_active)}) + return {"groups": list(grouped.values())} + + +@router.get("/vessels/name-fallbacks") +async def get_vessel_name_fallbacks( + limit: int = Query(500, ge=0, description="Maximum fallback-name vessels to return. 0 means no limit."), + db: AsyncSession = Depends(get_db), +): + """Return vessels whose display name still falls back to MMSI.""" + aggregated_vessels = await get_aggregated_vessels(db) + raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels) + + latest_times = ( + select( + VesselPosition.mmsi.label("mmsi"), + func.max(VesselPosition.received_at).label("received_at"), + ) + .group_by(VesselPosition.mmsi) + .subquery() + ) + result = await db.execute( + select(VesselPosition, VesselStatic) + .join( + latest_times, + (VesselPosition.mmsi == latest_times.c.mmsi) + & (VesselPosition.received_at == latest_times.c.received_at), + ) + .outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi) + .order_by(VesselPosition.received_at.desc()) + ) + legacy_geojson = convert_vessels_to_geojson(list(result.all())) + features, diagnostics = _merge_vessel_features( + raw_geojson.get("features", []), + legacy_geojson.get("features", []), + ) + + fallback_items = [] + for feature in features: + props = feature.get("properties", {}) + mmsi = props.get("mmsi") + name = props.get("name") + if not _is_vessel_name_fallback(name, mmsi): + continue + source_summary = props.get("source_summary") or {} + fallback_items.append( + { + "mmsi": str(mmsi), + "display_name": name or f"MMSI {mmsi}", + "reason": "missing_real_name", + "received_at": props.get("received_at"), + "sources": sorted(source_summary.keys()), + "source_summary": source_summary, + "message_types": sorted( + { + message_type + for summary in source_summary.values() + for message_type in (summary.get("message_types") or []) + } + ), + "field_sources": props.get("field_sources") or {}, + } + ) + + if limit and limit > 0: + fallback_items = fallback_items[:limit] + return { + "count": len(fallback_items), + "items": fallback_items, + "diagnostics": diagnostics, + } + + +@router.get("/vessels/{mmsi}") +async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)): + from app.services.vessel_enrichment import get_vessel_enrichment_bundle + + aggregated = await get_aggregated_vessel(db, mmsi) + enrichment = await get_vessel_enrichment_bundle(db, mmsi) + if aggregated is not None: + return { + **aggregated, + "received_at": to_iso8601_utc(aggregated.get("received_at")), + "latitude": aggregated["lat"], + "longitude": aggregated["lon"], + "enrichment": enrichment, + } + + latest_position_stmt = ( + select(VesselPosition) + .where(VesselPosition.mmsi == mmsi) + .order_by(VesselPosition.received_at.desc()) + .limit(1) + ) + static = await db.get(VesselStatic, mmsi) + result = await db.execute(latest_position_stmt) + position = result.scalar_one_or_none() + if position is None: + raise HTTPException(status_code=404, detail="Vessel not found") + geojson = convert_vessels_to_geojson([(position, static)]) + return { + **(geojson["features"][0]["properties"]), + "latitude": position.lat, + "longitude": position.lon, + "enrichment": enrichment, + } + + +@router.get("/vessels/{mmsi}/track") +async def get_vessel_track( + mmsi: int, + hours: int = Query(6, ge=1, le=24), + db: AsyncSession = Depends(get_db), +): + cutoff = datetime.now(UTC) - timedelta(hours=hours) + aggregated_points = await get_aggregated_vessel_track(db, mmsi, cutoff=cutoff) + if aggregated_points: + return { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "LineString", + "coordinates": [[point["lon"], point["lat"]] for point in aggregated_points], + }, + "properties": { + "mmsi": mmsi, + "hours": hours, + "point_count": len(aggregated_points), + "start_at": to_iso8601_utc(aggregated_points[0]["observed_at"]), + "end_at": to_iso8601_utc(aggregated_points[-1]["observed_at"]), + "point_sources": [point["source"] for point in aggregated_points], + }, + } + ], + "count": 1, + } + + result = await db.execute( + select(VesselPosition) + .where(VesselPosition.mmsi == mmsi) + .where(VesselPosition.received_at >= cutoff) + .order_by(VesselPosition.received_at.asc()) + ) + positions = list(result.scalars().all()) + if not positions: + return { + "type": "FeatureCollection", + "features": [], + "count": 0, + } + + return { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "LineString", + "coordinates": [[position.lon, position.lat] for position in positions], + }, + "properties": { + "mmsi": mmsi, + "hours": hours, + "point_count": len(positions), + "start_at": to_iso8601_utc(positions[0].received_at), + "end_at": to_iso8601_utc(positions[-1].received_at), + }, + } + ], + "count": 1, + } + + +@router.get("/vessels/{mmsi}/observations") +async def get_vessel_observations( + mmsi: int, + limit: int = Query(100, ge=1, le=500), + db: AsyncSession = Depends(get_db), +): + """Return raw AIS observations for debugging source-level collector facts.""" + + observations = await get_vessel_raw_observations(db, mmsi, limit=limit) + return { + "mmsi": mmsi, + "count": len(observations), + "observations": [item.to_dict() for item in observations], + "conflict_candidates": build_field_conflict_candidates(observations), + } + + +@router.get("/vessels/{mmsi}/conflicts") +async def get_vessel_conflicts(mmsi: int, db: AsyncSession = Depends(get_db)): + """Return recorded AIS conflicts plus current raw-observation candidates.""" + + records = await get_vessel_conflict_records(db, mmsi) + observations = await get_vessel_raw_observations(db, mmsi, limit=500) + return { + "mmsi": mmsi, + "count": len(records), + "conflicts": [item.to_dict() for item in records], + "candidates": build_field_conflict_candidates(observations), + } + + @router.get("/geo/bgp-anomalies") async def get_bgp_anomalies_geojson( severity: Optional[str] = Query(None), @@ -1030,6 +2327,82 @@ async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db)): return {**geojson, "count": len(geojson.get("features", []))} +@router.get("/geo/summary") +async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)): + """Return lightweight Earth HUD counts without loading layer GeoJSON payloads.""" + cable_count = await _count_current_or_latest_task_data(db, "arcgis_cables") + landing_point_count = await _count_current_or_latest_task_data(db, "arcgis_landing_points") + satellite_count = await _count_current_or_latest_task_data( + db, + "celestrak_tle", + exclude_unknown_name=True, + ) + supercomputer_count = await _count_current_or_latest_task_data(db, "top500") + gpu_cluster_count = await _count_current_or_latest_task_data(db, "epoch_ai_gpu") + compute_center_count = supercomputer_count + gpu_cluster_count + + active_incident_result = await db.execute( + select(func.count(BGPIncident.id)).where(BGPIncident.status == "active"), + ) + active_anomaly_result = await db.execute( + select(func.count(BGPAnomaly.id)).where(BGPAnomaly.status == "active"), + ) + active_incident_count = int(active_incident_result.scalar() or 0) + active_anomaly_count = int(active_anomaly_result.scalar() or 0) + bgp_collector_result = await db.execute( + select(func.count(func.distinct(BGPObservation.collector))) + .where(BGPObservation.collector.isnot(None)) + .where(func.length(func.btrim(BGPObservation.collector)) > 0) + .where(BGPObservation.source.in_(("ris_live_bgp", "bgpstream_bgp"))) + ) + bgp_collector_scalar = bgp_collector_result.scalar() + if bgp_collector_scalar is None: + bgp_collectors = await build_bgp_collector_coverage( + db, + source_filter=("ris_live_bgp", "bgpstream_bgp"), + ) + bgp_collector_count = len( + [item for item in bgp_collectors if item.get("collector")] + ) + else: + bgp_collector_count = int(bgp_collector_scalar or 0) + raw_unique_window_hours = 24 + raw_unique_mmsi = await count_unique_raw_vessel_mmsi( + db, + observed_since=datetime.now(UTC) - timedelta(hours=raw_unique_window_hours), + ) + legacy_unique_result = await db.execute( + select(func.count(func.distinct(VesselPosition.mmsi))) + ) + legacy_unique_mmsi = int(legacy_unique_result.scalar() or 0) + vessel_count = max(raw_unique_mmsi, legacy_unique_mmsi) + aisstream_health = await db.get(AISSourceHealth, "aisstream_vessels") + + return { + "generated_at": to_iso8601_utc(datetime.now(UTC)), + "stats": { + "cable_count": cable_count, + "landing_point_count": landing_point_count, + "satellite_count": satellite_count, + "compute_center_count": compute_center_count, + "vessel_count": vessel_count, + "vessel_raw_unique_mmsi": raw_unique_mmsi, + "vessel_raw_unique_window_hours": raw_unique_window_hours, + "vessel_legacy_unique_mmsi": legacy_unique_mmsi, + "aisstream_connection_state": aisstream_health.connection_state if aisstream_health else None, + "aisstream_last_seen_at": to_iso8601_utc(aisstream_health.last_seen_at) if aisstream_health else None, + "aisstream_message_rate": aisstream_health.message_rate if aisstream_health else None, + "aisstream_lag_seconds": aisstream_health.lag_seconds if aisstream_health else None, + "supercomputer_count": supercomputer_count, + "gpu_cluster_count": gpu_cluster_count, + "bgp_event_count": active_incident_count or active_anomaly_count, + "bgp_incident_count": active_incident_count, + "bgp_anomaly_count": active_anomaly_count, + "bgp_collector_count": bgp_collector_count, + }, + } + + @router.get("/all") async def get_all_visualization_data(db: AsyncSession = Depends(get_db)): """获取所有可视化数据的统一端点 diff --git a/backend/app/api/v1/websocket.py b/backend/app/api/v1/websocket.py index 23ccb3fe..b4cbc903 100644 --- a/backend/app/api/v1/websocket.py +++ b/backend/app/api/v1/websocket.py @@ -2,7 +2,6 @@ import asyncio import json -import logging from datetime import UTC, datetime from typing import Optional @@ -10,10 +9,11 @@ from fastapi import APIRouter, WebSocket, WebSocketDisconnect, Query from jose import jwt, JWTError from app.core.config import settings +from app.core.logging import get_logger from app.core.time import to_iso8601_utc from app.core.websocket.manager import manager -logger = logging.getLogger(__name__) +logger = get_logger(__name__, service="api") router = APIRouter() @@ -22,28 +22,52 @@ async def authenticate_token(token: str) -> Optional[dict]: try: payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM]) if payload.get("type") != "access": - logger.warning(f"WebSocket auth failed: wrong token type") + logger.warning_event( + "WebSocket auth failed: wrong token type", + event="auth.websocket.invalid_token_type", + ) return None return payload except JWTError as e: - logger.warning(f"WebSocket auth failed: {e}") + logger.warning_event( + "WebSocket auth failed", + event="auth.websocket.decode_failed", + context={"error": str(e)}, + ) return None @router.websocket("/ws") async def websocket_endpoint( websocket: WebSocket, - token: str = Query(...), + token: str | None = Query(None), ): """WebSocket endpoint for real-time data""" - logger.info(f"WebSocket connection attempt with token: {token[:20]}...") - payload = await authenticate_token(token) - if payload is None: - logger.warning("WebSocket authentication failed, closing connection") + logger.info_event( + "WebSocket connection attempt", + event="auth.websocket.connection_attempt", + context={"token_preview": f"{token[:8]}..." if token else "anonymous"}, + ) + payload = await authenticate_token(token) if token else None + if token and payload is None: + logger.warning_event( + "WebSocket authentication failed, closing connection", + event="auth.websocket.connection_rejected", + ) await websocket.close(code=4001) return - user_id = str(payload.get("sub")) + is_anonymous = payload is None + user_id = str(payload.get("sub")) if payload else f"anonymous:{id(websocket)}" + supported_channels = ["vessels"] if is_anonymous else [ + "gpu_clusters", + "submarine_cables", + "ixp_nodes", + "alerts", + "dashboard", + "datasource_tasks", + "vessels", + ] await manager.connect(websocket, user_id) try: @@ -54,14 +78,7 @@ async def websocket_endpoint( "connection_id": f"conn_{user_id}", "server_version": settings.VERSION, "heartbeat_interval": 30, - "supported_channels": [ - "gpu_clusters", - "submarine_cables", - "ixp_nodes", - "alerts", - "dashboard", - "datasource_tasks", - ], + "supported_channels": supported_channels, }, } ) @@ -79,12 +96,24 @@ async def websocket_endpoint( ) elif data.get("type") == "subscribe": channels = data.get("data", {}).get("channels", []) + if is_anonymous: + channels = [channel for channel in channels if channel in supported_channels] + manager.subscribe(websocket, channels) await websocket.send_json( { "type": "subscription_confirmed", "data": {"action": "subscribe", "channels": channels}, } ) + elif data.get("type") == "unsubscribe": + channels = data.get("data", {}).get("channels", []) + manager.unsubscribe(websocket, channels) + await websocket.send_json( + { + "type": "subscription_confirmed", + "data": {"action": "unsubscribe", "channels": channels}, + } + ) elif data.get("type") == "control_frame": await websocket.send_json( {"type": "control_acknowledged", "data": {"received": True}} diff --git a/backend/app/core/cache.py b/backend/app/core/cache.py index d3c250a4..b4885022 100644 --- a/backend/app/core/cache.py +++ b/backend/app/core/cache.py @@ -1,15 +1,15 @@ """Redis caching service""" import json -import logging from datetime import timedelta from typing import Optional, Any import redis from app.core.config import settings +from app.core.logging import get_logger -logger = logging.getLogger(__name__) +logger = get_logger(__name__) # Lazy Redis client initialization @@ -47,7 +47,7 @@ class CacheService: return json.loads(value) return None except Exception as e: - logger.warning(f"Cache get error: {e}") + logger.warning_event("Cache get error", event="cache.get.failed", context={"error": str(e)}) return None def set( @@ -61,7 +61,7 @@ class CacheService: serialized = json.dumps(value, default=str) return self.client.setex(key, expire_seconds, serialized) except Exception as e: - logger.warning(f"Cache set error: {e}") + logger.warning_event("Cache set error", event="cache.set.failed", context={"error": str(e)}) return False def delete(self, key: str) -> bool: @@ -69,7 +69,7 @@ class CacheService: try: return self.client.delete(key) > 0 except Exception as e: - logger.warning(f"Cache delete error: {e}") + logger.warning_event("Cache delete error", event="cache.delete.failed", context={"error": str(e)}) return False def delete_pattern(self, pattern: str) -> int: @@ -80,7 +80,7 @@ class CacheService: return self.client.delete(*keys) return 0 except Exception as e: - logger.warning(f"Cache delete_pattern error: {e}") + logger.warning_event("Cache delete_pattern error", event="cache.delete_pattern.failed", context={"error": str(e)}) return 0 def get_or_set( diff --git a/backend/app/core/collected_data_fields.py b/backend/app/core/collected_data_fields.py index 5574605c..04c7bcb0 100644 --- a/backend/app/core/collected_data_fields.py +++ b/backend/app/core/collected_data_fields.py @@ -4,8 +4,8 @@ from typing import Any, Dict, Optional FIELD_ALIASES = { "country": ("country",), "city": ("city",), - "latitude": ("latitude",), - "longitude": ("longitude",), + "latitude": ("latitude", "lat"), + "longitude": ("longitude", "lon", "lng"), "value": ("value",), "unit": ("unit",), "cores": ("cores",), @@ -14,6 +14,28 @@ FIELD_ALIASES = { "power": ("power",), } +NESTED_FIELD_ALIASES = { + "latitude": ( + ("location", "latitude"), + ("location", "lat"), + ("geo", "latitude"), + ("geo", "lat"), + ("coordinates", "latitude"), + ("coordinates", "lat"), + ), + "longitude": ( + ("location", "longitude"), + ("location", "lon"), + ("location", "lng"), + ("geo", "longitude"), + ("geo", "lon"), + ("geo", "lng"), + ("coordinates", "longitude"), + ("coordinates", "lon"), + ("coordinates", "lng"), + ), +} + def get_metadata_field(metadata: Optional[Dict[str, Any]], field: str, fallback: Any = None) -> Any: if isinstance(metadata, dict): @@ -21,9 +43,34 @@ def get_metadata_field(metadata: Optional[Dict[str, Any]], field: str, fallback: value = metadata.get(key) if value not in (None, ""): return value + for path in NESTED_FIELD_ALIASES.get(field, ()): + current: Any = metadata + for key in path: + if not isinstance(current, dict): + current = None + break + current = current.get(key) + if current not in (None, ""): + return current + if field in {"latitude", "longitude"}: + value = _get_coordinate_sequence_value(metadata, field) + if value not in (None, ""): + return value return fallback +def _get_coordinate_sequence_value(metadata: Dict[str, Any], field: str) -> Any: + for key in ("coordinates", "coord", "coords"): + value = metadata.get(key) + if not isinstance(value, (list, tuple)) or len(value) < 2: + continue + # GeoJSON uses [longitude, latitude]. Most raw collector tuples in this + # codebase use explicit field names, so only sequence aliases are treated + # as GeoJSON-shaped to avoid guessing. + return value[1] if field == "latitude" else value[0] + return None + + def build_dynamic_metadata( metadata: Optional[Dict[str, Any]], *, diff --git a/backend/app/core/data_sources.py b/backend/app/core/data_sources.py index 8a2a7669..c3b746bb 100644 --- a/backend/app/core/data_sources.py +++ b/backend/app/core/data_sources.py @@ -1,7 +1,6 @@ import os import yaml from functools import lru_cache -from typing import Optional COLLECTOR_URL_KEYS = { @@ -31,6 +30,8 @@ COLLECTOR_URL_KEYS = { "opengeofeed_prefix_geo": "opengeofeed.public_csv_url", "nro_delegated_prefix_geo": "nro.delegated_stats_url", "news_live_streams": "news_live_streams.channels_url", + "barentswatch_vessels": "barentswatch_vessels.url", + "aisstream_vessels": "aisstream_vessels.url", } @@ -73,7 +74,7 @@ class DataSourcesConfig: from app.models.datasource_config import DataSourceConfig query = select(DataSourceConfig).where( - DataSourceConfig.name == collector_name, DataSourceConfig.is_active == True + DataSourceConfig.name == collector_name, DataSourceConfig.is_active ) result = await db.execute(query) db_config = result.scalar_one_or_none() diff --git a/backend/app/core/data_sources.yaml b/backend/app/core/data_sources.yaml index 17e5658e..513a533c 100644 --- a/backend/app/core/data_sources.yaml +++ b/backend/app/core/data_sources.yaml @@ -94,3 +94,11 @@ news_live_streams: streams_url: "https://iptv-org.github.io/api/streams.json" # IPTV-org 台标 JSON logos_url: "https://iptv-org.github.io/api/logos.json" + +barentswatch_vessels: + # BarentsWatch Live AIS latest combined endpoint. Requires an AIS bearer token. + url: "https://live.ais.barentswatch.no/v1/latest/combined" + +aisstream_vessels: + # AISStream realtime WebSocket endpoint. Requires an AISStream API key. + url: "wss://stream.aisstream.io/v0/stream" diff --git a/backend/app/core/datasource_defaults.py b/backend/app/core/datasource_defaults.py index b1fb18f7..be5adbf0 100644 --- a/backend/app/core/datasource_defaults.py +++ b/backend/app/core/datasource_defaults.py @@ -4,163 +4,258 @@ DEFAULT_DATASOURCES = { "top500": { "id": 1, "name": "TOP500 Supercomputers", + "display_name": "TOP500 超算榜单", "module": "L1", "priority": "P0", "frequency_minutes": 240, + "is_free": True, + "requires_credentials": False, }, "epoch_ai_gpu": { "id": 2, "name": "Epoch AI GPU Clusters", + "display_name": "Epoch AI GPU 集群", "module": "L1", "priority": "P0", "frequency_minutes": 360, + "is_free": True, + "requires_credentials": False, }, "huggingface_models": { "id": 3, "name": "HuggingFace Models", + "display_name": "Hugging Face 模型", "module": "L2", "priority": "P1", "frequency_minutes": 720, + "is_free": True, + "requires_credentials": False, }, "huggingface_datasets": { "id": 4, "name": "HuggingFace Datasets", + "display_name": "Hugging Face 数据集", "module": "L2", "priority": "P1", "frequency_minutes": 720, + "is_free": True, + "requires_credentials": False, }, "huggingface_spaces": { "id": 5, "name": "HuggingFace Spaces", + "display_name": "Hugging Face Spaces", "module": "L2", "priority": "P2", "frequency_minutes": 1440, + "is_free": True, + "requires_credentials": False, }, "peeringdb_ixp": { "id": 6, "name": "PeeringDB IXP", + "display_name": "PeeringDB 交换中心", "module": "L2", "priority": "P1", "frequency_minutes": 1440, + "is_free": True, + "requires_credentials": False, }, "peeringdb_network": { "id": 7, "name": "PeeringDB Networks", + "display_name": "PeeringDB 网络", "module": "L2", "priority": "P2", "frequency_minutes": 2880, + "is_free": True, + "requires_credentials": False, }, "peeringdb_facility": { "id": 8, "name": "PeeringDB Facilities", + "display_name": "PeeringDB 设施", "module": "L2", "priority": "P2", "frequency_minutes": 2880, + "is_free": True, + "requires_credentials": False, }, "telegeography_cables": { "id": 9, "name": "Submarine Cables", + "display_name": "海底光缆", "module": "L2", "priority": "P1", "frequency_minutes": 10080, + "is_free": True, + "requires_credentials": False, }, "telegeography_landing": { "id": 10, "name": "Cable Landing Points", + "display_name": "光缆登陆点", "module": "L2", "priority": "P2", "frequency_minutes": 10080, + "is_free": True, + "requires_credentials": False, }, "telegeography_systems": { "id": 11, "name": "Cable Systems", + "display_name": "光缆系统", "module": "L2", "priority": "P2", "frequency_minutes": 10080, + "is_free": True, + "requires_credentials": False, }, "arcgis_cables": { "id": 15, "name": "ArcGIS Submarine Cables", + "display_name": "ArcGIS 海底光缆", "module": "L2", "priority": "P1", "frequency_minutes": 10080, + "is_free": True, + "requires_credentials": False, }, "arcgis_landing_points": { "id": 16, "name": "ArcGIS Landing Points", + "display_name": "ArcGIS 登陆点", "module": "L2", "priority": "P1", "frequency_minutes": 10080, + "is_free": True, + "requires_credentials": False, }, "arcgis_cable_landing_relation": { "id": 17, "name": "ArcGIS Cable-Landing Relations", + "display_name": "ArcGIS 光缆登陆关系", "module": "L2", "priority": "P1", "frequency_minutes": 10080, + "is_free": True, + "requires_credentials": False, }, "fao_landing_points": { "id": 18, "name": "FAO Landing Points", + "display_name": "FAO 登陆点", "module": "L2", "priority": "P1", "frequency_minutes": 10080, + "is_free": True, + "requires_credentials": False, }, "spacetrack_tle": { "id": 19, "name": "Space-Track TLE", + "display_name": "Space-Track 轨道根数", "module": "L3", "priority": "P2", "frequency_minutes": 1440, + "is_free": True, + "requires_credentials": True, + "credential_provider": "spacetrack", + "credential_status": "planned", }, "celestrak_tle": { "id": 20, "name": "CelesTrak TLE", + "display_name": "CelesTrak 轨道根数", "module": "L3", "priority": "P2", "frequency_minutes": 1440, + "is_free": True, + "requires_credentials": False, }, "ris_live_bgp": { "id": 21, "name": "RIPE RIS Live BGP", + "display_name": "RIPE RIS 实时 BGP", "module": "L3", "priority": "P1", "frequency_minutes": 15, + "is_free": True, + "requires_credentials": False, }, "bgpstream_bgp": { "id": 22, "name": "CAIDA BGPStream Backfill", + "display_name": "CAIDA BGPStream 回填", "module": "L3", "priority": "P1", "frequency_minutes": 360, + "is_free": True, + "requires_credentials": False, }, "iptoasn_prefix_geo": { "id": 23, "name": "IPtoASN Prefix Geography", + "display_name": "IPtoASN 前缀地理", "module": "L3", "priority": "P1", "frequency_minutes": 1440, + "is_free": True, + "requires_credentials": False, }, "opengeofeed_prefix_geo": { "id": 24, "name": "OpenGeoFeed Prefix Geography", + "display_name": "OpenGeoFeed 前缀地理", "module": "L3", "priority": "P1", "frequency_minutes": 1440, + "is_free": True, + "requires_credentials": False, }, "nro_delegated_prefix_geo": { "id": 25, "name": "NRO Delegated Prefix Geography", + "display_name": "NRO 分配前缀地理", "module": "L3", "priority": "P1", "frequency_minutes": 1440, + "is_free": True, + "requires_credentials": False, }, "news_live_streams": { "id": 26, "name": "News Live Streams", + "display_name": "新闻直播源", "module": "L4", "priority": "P2", "frequency_minutes": 720, + "is_free": True, + "requires_credentials": False, + }, + "barentswatch_vessels": { + "id": 27, + "name": "BarentsWatch AIS Vessels", + "display_name": "BarentsWatch AIS 船舶", + "module": "L4", + "priority": "P1", + "frequency_minutes": 1, + "is_free": True, + "requires_credentials": True, + "credential_provider": "barentswatch", + "credential_status": "supported", + }, + "aisstream_vessels": { + "id": 28, + "name": "AISStream Vessels", + "display_name": "AISStream 实时船舶", + "module": "L4", + "priority": "P1", + "frequency_minutes": 1, + "is_free": True, + "requires_credentials": True, + "credential_provider": "aisstream", + "credential_status": "supported", }, } diff --git a/backend/app/core/logging.py b/backend/app/core/logging.py new file mode 100644 index 00000000..82adf89e --- /dev/null +++ b/backend/app/core/logging.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import json +import logging +import os +import re + +from collections.abc import Mapping, Sequence +from typing import Any + +from app.core.request_context import get_request_id + +DEFAULT_SERVICE = "backend" +DEFAULT_EVENT = "app.log" +DEFAULT_LOG_LEVEL = os.getenv("PLANET_LOG_LEVEL", "INFO").upper() +REDACTED = "[REDACTED]" +SENSITIVE_FIELD_NAMES = { + "access_token", + "api_key", + "authorization", + "cookie", + "password", + "refresh_token", + "secret", + "token", +} +SENSITIVE_TEXT_PATTERNS = ( + re.compile(r"(?i)(authorization\s*[:=]\s*)(.+)"), + re.compile(r"(?i)(bearer\s+)([A-Za-z0-9._\-]+)"), + re.compile(r"(?i)(token\s*[:=]\s*)(.+)"), + re.compile(r"(?i)(password\s*[:=]\s*)(.+)"), + re.compile(r"(?i)(cookie\s*[:=]\s*)(.+)"), +) + + +def sanitize_log_value(value: Any) -> Any: + if isinstance(value, Mapping): + return { + str(key): (REDACTED if str(key).lower() in SENSITIVE_FIELD_NAMES else sanitize_log_value(item)) + for key, item in value.items() + } + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [sanitize_log_value(item) for item in value] + if isinstance(value, str): + sanitized = value + for pattern in SENSITIVE_TEXT_PATTERNS: + sanitized = pattern.sub(lambda match: f"{match.group(1)}{REDACTED}", sanitized) + return sanitized + return value + + +def _normalize_context(context: Any) -> dict[str, Any]: + if context is None: + return {} + if isinstance(context, Mapping): + sanitized = sanitize_log_value(context) + return {str(key): value for key, value in sanitized.items()} + return {"value": sanitize_log_value(context)} + + +class PlanetContextFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + record.request_id = getattr(record, "request_id", None) or get_request_id() or "-" + record.service = getattr(record, "service", None) or DEFAULT_SERVICE + record.event = getattr(record, "event", None) or DEFAULT_EVENT + record.context = _normalize_context(getattr(record, "context", None)) + record.message = sanitize_log_value(record.getMessage()) + return True + + +class PlanetFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + timestamp = self.formatTime(record, self.datefmt) + level = record.levelname + service = getattr(record, "service", DEFAULT_SERVICE) + module_name = record.name + event = getattr(record, "event", DEFAULT_EVENT) + request_id = getattr(record, "request_id", "-") + message = sanitize_log_value(record.getMessage()) + context = _normalize_context(getattr(record, "context", None)) + context_suffix = "" + if context: + context_suffix = f" context={json.dumps(context, ensure_ascii=False, sort_keys=True)}" + rendered = ( + f"{timestamp} {level} service={service} module={module_name} " + f"event={event} request_id={request_id} message={message}{context_suffix}" + ) + if record.exc_info: + rendered = f"{rendered}\n{self.formatException(record.exc_info)}" + return rendered + + +class PlanetLoggerAdapter(logging.LoggerAdapter): + def process(self, msg: Any, kwargs: dict[str, Any]) -> tuple[Any, dict[str, Any]]: + extra = dict(self.extra) + extra.update(kwargs.get("extra", {})) + if "context" in extra: + extra["context"] = _normalize_context(extra.get("context")) + kwargs["extra"] = extra + return sanitize_log_value(msg), kwargs + + def log_event( + self, + level: int, + message: str, + *, + event: str, + context: Mapping[str, Any] | None = None, + **extra: Any, + ) -> None: + self.log(level, message, extra={"event": event, "context": context or {}, **extra}) + + def debug_event(self, message: str, *, event: str, context: Mapping[str, Any] | None = None, **extra: Any) -> None: + self.log_event(logging.DEBUG, message, event=event, context=context, **extra) + + def info_event(self, message: str, *, event: str, context: Mapping[str, Any] | None = None, **extra: Any) -> None: + self.log_event(logging.INFO, message, event=event, context=context, **extra) + + def warning_event(self, message: str, *, event: str, context: Mapping[str, Any] | None = None, **extra: Any) -> None: + self.log_event(logging.WARNING, message, event=event, context=context, **extra) + + def error_event(self, message: str, *, event: str, context: Mapping[str, Any] | None = None, **extra: Any) -> None: + self.log_event(logging.ERROR, message, event=event, context=context, **extra) + + def exception_event( + self, + message: str, + *, + event: str, + context: Mapping[str, Any] | None = None, + **extra: Any, + ) -> None: + self.error(message, exc_info=True, extra={"event": event, "context": context or {}, **extra}) + + +def get_logger(name: str, *, service: str = DEFAULT_SERVICE) -> PlanetLoggerAdapter: + return PlanetLoggerAdapter(logging.getLogger(name), {"service": service}) + + +def configure_logging(level: str | None = None) -> None: + root_logger = logging.getLogger() + if getattr(configure_logging, "_configured", False): + if level: + root_logger.setLevel(level.upper()) + return + + handler = logging.StreamHandler() + handler.setFormatter(PlanetFormatter(datefmt="%Y-%m-%d %H:%M:%S")) + handler.addFilter(PlanetContextFilter()) + + root_logger.handlers.clear() + root_logger.addHandler(handler) + root_logger.setLevel((level or DEFAULT_LOG_LEVEL).upper()) + + for logger_name in ("uvicorn", "uvicorn.error", "uvicorn.access"): + target_logger = logging.getLogger(logger_name) + target_logger.handlers.clear() + target_logger.propagate = True + + logging.captureWarnings(True) + configure_logging._configured = True diff --git a/backend/app/core/request_context.py b/backend/app/core/request_context.py new file mode 100644 index 00000000..de30d925 --- /dev/null +++ b/backend/app/core/request_context.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from contextvars import ContextVar + + +request_id_context: ContextVar[str | None] = ContextVar("request_id", default=None) + + +def set_request_id(request_id: str | None) -> None: + request_id_context.set(request_id) + + +def get_request_id() -> str | None: + return request_id_context.get() diff --git a/backend/app/core/security.py b/backend/app/core/security.py index 4b0bdcc5..4812424f 100644 --- a/backend/app/core/security.py +++ b/backend/app/core/security.py @@ -105,7 +105,7 @@ async def get_current_user( ) result = await db.execute( text( - "SELECT id, username, email, password_hash, role, is_active FROM users WHERE id = :id" + "SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id" ), {"id": int(user_id)}, ) @@ -122,6 +122,7 @@ async def get_current_user( user.password_hash = row[3] user.role = row[4] user.is_active = row[5] + user.gatekeeper_groups = row[6] or [] return user @@ -144,7 +145,7 @@ async def get_current_user_refresh( ) result = await db.execute( text( - "SELECT id, username, email, password_hash, role, is_active FROM users WHERE id = :id" + "SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id" ), {"id": int(user_id)}, ) @@ -161,6 +162,7 @@ async def get_current_user_refresh( user.password_hash = row[3] user.role = row[4] user.is_active = row[5] + user.gatekeeper_groups = row[6] or [] return user diff --git a/backend/app/core/target_schema_registry.py b/backend/app/core/target_schema_registry.py new file mode 100644 index 00000000..61ade066 --- /dev/null +++ b/backend/app/core/target_schema_registry.py @@ -0,0 +1,157 @@ +"""Registry of target schemas supported by mapped custom data sources.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field, ValidationError, field_validator + + +class VesselAISRecord(BaseModel): + mmsi: int = Field(ge=100000000, le=999999999) + lat: float = Field(ge=-90, le=90) + lon: float = Field(ge=-180, le=180) + sog: float | None = None + cog: float | None = Field(default=None, ge=0, le=360) + heading: int | None = Field(default=None, ge=0, le=511) + nav_status: int | None = None + name: str | None = None + callsign: str | None = None + vessel_type: str | int | None = None + vessel_type_name: str | None = None + received_at: datetime | None = None + + +class GeoPointRecord(BaseModel): + lat: float = Field(ge=-90, le=90) + lon: float = Field(ge=-180, le=180) + name: str | None = None + type: str | None = None + source_id: str | None = None + observed_at: datetime | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class GenericRecord(BaseModel): + data: dict[str, Any] = Field(default_factory=dict) + source_id: str | None = None + observed_at: datetime | None = None + + @field_validator("data") + @classmethod + def require_payload(cls, value: dict[str, Any]) -> dict[str, Any]: + if not value: + raise ValueError("generic_records requires a non-empty data object") + return value + + +@dataclass(frozen=True) +class TargetField: + name: str + type: str + required: bool = False + description: str = "" + example: Any = None + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "type": self.type, + "required": self.required, + "description": self.description, + "example": self.example, + } + + +@dataclass(frozen=True) +class TargetSchema: + key: str + label: str + description: str + fields: tuple[TargetField, ...] + model: type[BaseModel] + destination: str + + def to_dict(self) -> dict[str, Any]: + return { + "key": self.key, + "label": self.label, + "description": self.description, + "destination": self.destination, + "fields": [field.to_dict() for field in self.fields], + } + + def validate_record(self, record: dict[str, Any]) -> tuple[dict[str, Any] | None, list[str]]: + try: + return self.model.model_validate(record).model_dump(mode="json"), [] + except ValidationError as exc: + return None, [ + ".".join(str(part) for part in error["loc"]) + f": {error['msg']}" + for error in exc.errors() + ] + + +TARGET_SCHEMAS: dict[str, TargetSchema] = { + "vessel_ais": TargetSchema( + key="vessel_ais", + label="船舶 AIS", + description="船只位置、航速、航向、MMSI 等 AIS 数据。", + destination="vessel_position", + model=VesselAISRecord, + fields=( + TargetField("mmsi", "integer", True, "MMSI 九位船舶标识", 257123000), + TargetField("lat", "float", True, "纬度", 59.91), + TargetField("lon", "float", True, "经度", 10.75), + TargetField("sog", "float", False, "对地航速，单位节", 12.4), + TargetField("cog", "float", False, "对地航向，0-360 度", 184.5), + TargetField("heading", "integer", False, "船首向，0-511", 186), + TargetField("nav_status", "integer", False, "导航状态码", 0), + TargetField("name", "string", False, "船名", "OSLO EXPRESS"), + TargetField("callsign", "string", False, "呼号", "LAAB"), + TargetField("vessel_type", "string", False, "船型代码", 70), + TargetField("vessel_type_name", "string", False, "船型名称", "Cargo"), + TargetField("received_at", "datetime", False, "数据接收时间", "2026-04-28T00:00:00Z"), + ), + ), + "geo_points": TargetSchema( + key="geo_points", + label="通用地理点", + description="带经纬度的通用实体或事件点位。", + destination="generic_geo_points", + model=GeoPointRecord, + fields=( + TargetField("lat", "float", True, "纬度", 1.3), + TargetField("lon", "float", True, "经度", 103.8), + TargetField("name", "string", False, "点位名称", "Singapore"), + TargetField("type", "string", False, "点位类型", "datacenter"), + TargetField("source_id", "string", False, "来源侧 ID", "sg-1"), + TargetField("observed_at", "datetime", False, "观测时间", "2026-04-28T00:00:00Z"), + TargetField("metadata", "object", False, "扩展字段", {"provider": "example"}), + ), + ), + "generic_records": TargetSchema( + key="generic_records", + label="通用结构化记录", + description="未知结构数据沉淀，不直接进入 Earth 图层。", + destination="collected_data", + model=GenericRecord, + fields=( + TargetField("data", "object", True, "结构化记录主体", {"raw": "value"}), + TargetField("source_id", "string", False, "来源侧 ID", "record-1"), + TargetField("observed_at", "datetime", False, "观测时间", "2026-04-28T00:00:00Z"), + ), + ), +} + + +def list_target_schemas() -> list[dict[str, Any]]: + return [schema.to_dict() for schema in TARGET_SCHEMAS.values()] + + +def get_target_schema(key: str) -> TargetSchema: + try: + return TARGET_SCHEMAS[key] + except KeyError as exc: + raise ValueError(f"Unsupported target schema: {key}") from exc diff --git a/backend/app/core/websocket/broadcaster.py b/backend/app/core/websocket/broadcaster.py index 0fbc2523..563c23de 100644 --- a/backend/app/core/websocket/broadcaster.py +++ b/backend/app/core/websocket/broadcaster.py @@ -75,7 +75,7 @@ class DataBroadcaster: "timestamp": to_iso8601_utc(datetime.now(UTC)), "payload": data, }, - channel=channel if channel in manager.active_connections else "all", + channel=channel, ) async def broadcast_datasource_task_update(self, data: Dict[str, Any]): diff --git a/backend/app/core/websocket/manager.py b/backend/app/core/websocket/manager.py index a4994119..57bad1ad 100644 --- a/backend/app/core/websocket/manager.py +++ b/backend/app/core/websocket/manager.py @@ -1,9 +1,6 @@ """WebSocket Connection Manager""" -import json -import asyncio from typing import Dict, Set, Optional -from datetime import datetime from fastapi import WebSocket import redis.asyncio as redis @@ -15,6 +12,8 @@ class ConnectionManager: def __init__(self): self.active_connections: Dict[str, Set[WebSocket]] = {} # user_id -> connections + self.channel_subscriptions: Dict[str, Set[WebSocket]] = {} + self.websocket_channels: Dict[WebSocket, Set[str]] = {} self.redis_client: Optional[redis.Redis] = None async def connect(self, websocket: WebSocket, user_id: str): @@ -40,6 +39,39 @@ class ConnectionManager: self.active_connections[user_id].discard(websocket) if not self.active_connections[user_id]: del self.active_connections[user_id] + self.unsubscribe_all(websocket) + + def subscribe(self, websocket: WebSocket, channels: list[str]): + normalized_channels = { + str(channel).strip() + for channel in channels + if str(channel).strip() + } + if not normalized_channels: + return + + socket_channels = self.websocket_channels.setdefault(websocket, set()) + for channel in normalized_channels: + self.channel_subscriptions.setdefault(channel, set()).add(websocket) + socket_channels.add(channel) + + def unsubscribe(self, websocket: WebSocket, channels: list[str]): + for channel in {str(channel).strip() for channel in channels if str(channel).strip()}: + subscribers = self.channel_subscriptions.get(channel) + if subscribers is not None: + subscribers.discard(websocket) + if not subscribers: + del self.channel_subscriptions[channel] + socket_channels = self.websocket_channels.get(websocket) + if socket_channels is not None: + socket_channels.discard(channel) + if not socket_channels: + del self.websocket_channels[websocket] + + def unsubscribe_all(self, websocket: WebSocket): + channels = list(self.websocket_channels.get(websocket, set())) + if channels: + self.unsubscribe(websocket, channels) async def send_personal_message(self, message: dict, user_id: str): if user_id in self.active_connections: @@ -54,13 +86,19 @@ class ConnectionManager: for user_id in self.active_connections: await self.send_personal_message(message, user_id) else: - await self.send_personal_message(message, channel) + for connection in list(self.channel_subscriptions.get(channel, set())): + try: + await connection.send_json(message) + except Exception: + self.unsubscribe_all(connection) async def close_all(self): for user_id in self.active_connections: for connection in self.active_connections[user_id]: await connection.close() self.active_connections.clear() + self.channel_subscriptions.clear() + self.websocket_channels.clear() manager = ConnectionManager() diff --git a/backend/app/data/seeds/ripe_ris_collector_locations_seed.json b/backend/app/data/seeds/ripe_ris_collector_locations_seed.json new file mode 100644 index 00000000..9689c869 --- /dev/null +++ b/backend/app/data/seeds/ripe_ris_collector_locations_seed.json @@ -0,0 +1,328 @@ +{ + "_comment": "Seed payload for the bgp_collector_locations DB table. Coordinates were migrated from the legacy RIPE_RIS_COLLECTOR_COORDS table and default to city-center; seeded rows are unverified and should be upgraded in the database with source evidence when known.", + "locations": [ + { + "canonical_name": "RIPE RIS rrc00", + "aliases": ["rrc00", "RIPE RIS rrc00", "AMS-IX"], + "operator": "RIPE NCC", + "site": "AMS-IX", + "city": "Amsterdam", + "country": "Netherlands", + "latitude": 52.3676, + "longitude": 4.9041, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc01", + "aliases": ["rrc01", "RIPE RIS rrc01", "LINX"], + "operator": "RIPE NCC", + "site": "LINX", + "city": "London", + "country": "United Kingdom", + "latitude": 51.5072, + "longitude": -0.1276, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc03", + "aliases": ["rrc03", "RIPE RIS rrc03", "AMS-IX"], + "operator": "RIPE NCC", + "site": "AMS-IX", + "city": "Amsterdam", + "country": "Netherlands", + "latitude": 52.3676, + "longitude": 4.9041, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc04", + "aliases": ["rrc04", "RIPE RIS rrc04", "CIXP", "CERN Internet Exchange Point"], + "operator": "RIPE NCC", + "site": "CIXP", + "city": "Geneva", + "country": "Switzerland", + "latitude": 46.2044, + "longitude": 6.1432, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc05", + "aliases": ["rrc05", "RIPE RIS rrc05", "VIX", "Vienna Internet Exchange"], + "operator": "RIPE NCC", + "site": "VIX", + "city": "Vienna", + "country": "Austria", + "latitude": 48.2082, + "longitude": 16.3738, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc06", + "aliases": ["rrc06", "RIPE RIS rrc06", "JPIX", "Otemachi"], + "operator": "RIPE NCC", + "site": "JPIX", + "city": "Otemachi", + "country": "Japan", + "latitude": 35.686, + "longitude": 139.7671, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc07", + "aliases": ["rrc07", "RIPE RIS rrc07", "Netnod", "Netnod Stockholm"], + "operator": "RIPE NCC", + "site": "Netnod Stockholm", + "city": "Stockholm", + "country": "Sweden", + "latitude": 59.3293, + "longitude": 18.0686, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc10", + "aliases": ["rrc10", "RIPE RIS rrc10", "MIX", "Milan Internet Exchange"], + "operator": "RIPE NCC", + "site": "MIX", + "city": "Milan", + "country": "Italy", + "latitude": 45.4642, + "longitude": 9.19, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc11", + "aliases": ["rrc11", "RIPE RIS rrc11", "NYIIX", "New York International Internet Exchange"], + "operator": "RIPE NCC", + "site": "NYIIX", + "city": "New York", + "country": "United States", + "latitude": 40.7128, + "longitude": -74.006, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc12", + "aliases": ["rrc12", "RIPE RIS rrc12", "DE-CIX", "DE-CIX Frankfurt"], + "operator": "RIPE NCC", + "site": "DE-CIX Frankfurt", + "city": "Frankfurt", + "country": "Germany", + "latitude": 50.1109, + "longitude": 8.6821, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc13", + "aliases": ["rrc13", "RIPE RIS rrc13", "MSK-IX"], + "operator": "RIPE NCC", + "site": "MSK-IX", + "city": "Moscow", + "country": "Russia", + "latitude": 55.7558, + "longitude": 37.6173, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc14", + "aliases": ["rrc14", "RIPE RIS rrc14", "PAIX", "Palo Alto Internet Exchange"], + "operator": "RIPE NCC", + "site": "PAIX", + "city": "Palo Alto", + "country": "United States", + "latitude": 37.4419, + "longitude": -122.143, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc15", + "aliases": ["rrc15", "RIPE RIS rrc15", "PTT.br Sao Paulo", "PTTMetro Sao Paulo"], + "operator": "RIPE NCC", + "site": "PTT.br", + "city": "Sao Paulo", + "country": "Brazil", + "latitude": -23.5558, + "longitude": -46.6396, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc16", + "aliases": ["rrc16", "RIPE RIS rrc16", "Equinix Miami", "NOTA Miami"], + "operator": "RIPE NCC", + "site": "Equinix Miami", + "city": "Miami", + "country": "United States", + "latitude": 25.7617, + "longitude": -80.1918, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc18", + "aliases": ["rrc18", "RIPE RIS rrc18", "CATNIX"], + "operator": "RIPE NCC", + "site": "CATNIX", + "city": "Barcelona", + "country": "Spain", + "latitude": 41.3874, + "longitude": 2.1686, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc19", + "aliases": ["rrc19", "RIPE RIS rrc19", "NAPAfrica", "JINX", "NAPAfrica Johannesburg"], + "operator": "RIPE NCC", + "site": "NAPAfrica Johannesburg", + "city": "Johannesburg", + "country": "South Africa", + "latitude": -26.2041, + "longitude": 28.0473, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc20", + "aliases": ["rrc20", "RIPE RIS rrc20", "SwissIX"], + "operator": "RIPE NCC", + "site": "SwissIX", + "city": "Zurich", + "country": "Switzerland", + "latitude": 47.3769, + "longitude": 8.5417, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc21", + "aliases": ["rrc21", "RIPE RIS rrc21", "France-IX Paris"], + "operator": "RIPE NCC", + "site": "France-IX Paris", + "city": "Paris", + "country": "France", + "latitude": 48.8566, + "longitude": 2.3522, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc22", + "aliases": ["rrc22", "RIPE RIS rrc22", "InterLAN Bucharest"], + "operator": "RIPE NCC", + "site": "InterLAN Bucharest", + "city": "Bucharest", + "country": "Romania", + "latitude": 44.4268, + "longitude": 26.1025, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc23", + "aliases": ["rrc23", "RIPE RIS rrc23", "Equinix Singapore"], + "operator": "RIPE NCC", + "site": "Equinix Singapore", + "city": "Singapore", + "country": "Singapore", + "latitude": 1.3521, + "longitude": 103.8198, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc24", + "aliases": ["rrc24", "RIPE RIS rrc24", "LACNIC Montevideo"], + "operator": "RIPE NCC", + "site": "LACNIC Montevideo", + "city": "Montevideo", + "country": "Uruguay", + "latitude": -34.9011, + "longitude": -56.1645, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc25", + "aliases": ["rrc25", "RIPE RIS rrc25", "AMS-IX"], + "operator": "RIPE NCC", + "site": "AMS-IX", + "city": "Amsterdam", + "country": "Netherlands", + "latitude": 52.3676, + "longitude": 4.9041, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + }, + { + "canonical_name": "RIPE RIS rrc26", + "aliases": ["rrc26", "RIPE RIS rrc26", "UAE-IX"], + "operator": "RIPE NCC", + "site": "UAE-IX", + "city": "Dubai", + "country": "United Arab Emirates", + "latitude": 25.2048, + "longitude": 55.2708, + "precision": "city", + "confidence": 0.85, + "source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table", + "verified_at": null + } + ], + "city_fallbacks": [] +} diff --git a/backend/app/db/session.py b/backend/app/db/session.py index 4aed0860..4f0e0404 100644 --- a/backend/app/db/session.py +++ b/backend/app/db/session.py @@ -5,10 +5,22 @@ from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sess from sqlalchemy.orm import declarative_base from app.core.config import settings +from app.core.logging import get_logger + +logger = get_logger(__name__) + +DB_POOL_CONFIG = { + "pool_pre_ping": True, + "pool_recycle": 1800, + "pool_size": 10, + "max_overflow": 20, + "pool_timeout": 30, +} engine = create_async_engine( settings.DATABASE_URL, echo=settings.DEBUG if hasattr(settings, "DEBUG") else False, + **DB_POOL_CONFIG, ) async_session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) @@ -91,15 +103,41 @@ async def init_db(): import app.models.datasource_config # noqa: F401 import app.models.alert # noqa: F401 import app.models.bgp_anomaly # noqa: F401 + import app.models.bgp_collector_location # noqa: F401 import app.models.bgp_incident # noqa: F401 import app.models.bgp_observation # noqa: F401 import app.models.collected_data # noqa: F401 + import app.models.compute_center_location # noqa: F401 import app.models.system_setting # noqa: F401 import app.models.playground_session # noqa: F401 import app.models.playground_message # noqa: F401 + import app.models.system_log # noqa: F401 + import app.models.vessel # noqa: F401 + import app.models.vessel_enrichment # noqa: F401 + import app.models.datasource_mapping # noqa: F401 + + logger.warning_event( + "Database pool settings active", + event="database.pool.initialized", + context={ + "pool_pre_ping": DB_POOL_CONFIG["pool_pre_ping"], + "pool_recycle": DB_POOL_CONFIG["pool_recycle"], + "pool_size": DB_POOL_CONFIG["pool_size"], + "max_overflow": DB_POOL_CONFIG["max_overflow"], + "pool_timeout": DB_POOL_CONFIG["pool_timeout"], + }, + ) async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) + await conn.execute( + text( + """ + ALTER TABLE users + ADD COLUMN IF NOT EXISTS gatekeeper_groups JSONB DEFAULT '[]'::jsonb + """ + ) + ) await conn.execute( text( """ @@ -119,7 +157,12 @@ async def init_db(): text( """ ALTER TABLE collection_tasks - ADD COLUMN IF NOT EXISTS phase VARCHAR(30) DEFAULT 'queued' + ADD COLUMN IF NOT EXISTS phase VARCHAR(30) DEFAULT 'queued', + ADD COLUMN IF NOT EXISTS phase_progress DOUBLE PRECISION, + ADD COLUMN IF NOT EXISTS phase_message VARCHAR(255), + ADD COLUMN IF NOT EXISTS phase_current BIGINT, + ADD COLUMN IF NOT EXISTS phase_total BIGINT, + ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30) """ ) ) @@ -131,6 +174,30 @@ async def init_db(): """ ) ) + await conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_collected_data_source_current_id + ON collected_data (source, is_current, id) + """ + ) + ) + await conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_collected_data_source_task_id + ON collected_data (source, task_id, id) + """ + ) + ) + await conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_ais_raw_schema_observed_entity + ON ais_raw_observations (target_schema, observed_at, entity_key) + """ + ) + ) await conn.execute( text( """ @@ -151,5 +218,14 @@ async def init_db(): ) async with async_session_factory() as session: + from app.services.bgp_collector_locations import ( + seed_default_bgp_collector_locations, + ) + from app.services.compute_center_locations import ( + seed_compute_center_locations_from_source_coords, + ) + + await seed_default_bgp_collector_locations(session) + await seed_compute_center_locations_from_source_coords(session) await seed_default_datasources(session) await ensure_default_admin_user(session) diff --git a/backend/app/main.py b/backend/app/main.py index a65b77b1..ae5ac8cf 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,4 +1,5 @@ from contextlib import asynccontextmanager +from uuid import uuid4 from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -7,6 +8,8 @@ from starlette.middleware.base import BaseHTTPMiddleware from app.api.main import api_router from app.api.v1 import websocket from app.core.config import settings +from app.core.logging import configure_logging +from app.core.request_context import set_request_id from app.core.websocket.broadcaster import broadcaster from app.db.session import init_db from app.services.scheduler import ( @@ -17,6 +20,9 @@ from app.services.scheduler import ( ) +configure_logging() + + class WebSocketCORSMiddleware(BaseHTTPMiddleware): async def dispatch(self, request, call_next): if request.url.path.startswith("/ws") and request.method == "GET": @@ -28,6 +34,18 @@ class WebSocketCORSMiddleware(BaseHTTPMiddleware): return await call_next(request) +class RequestContextMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request, call_next): + request_id = request.headers.get("X-Request-ID") or uuid4().hex + set_request_id(request_id) + try: + response = await call_next(request) + finally: + set_request_id(None) + response.headers["X-Request-ID"] = request_id + return response + + @asynccontextmanager async def lifespan(app: FastAPI): await init_db() @@ -58,6 +76,7 @@ app.add_middleware( allow_headers=["*"], ) +app.add_middleware(RequestContextMiddleware) app.add_middleware(WebSocketCORSMiddleware) app.include_router(api_router, prefix="/api/v1") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index ebc4d6ca..4e9bc11c 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -6,11 +6,16 @@ from app.models.datasource import DataSource from app.models.datasource_config import DataSourceConfig from app.models.alert import Alert, AlertSeverity, AlertStatus from app.models.bgp_anomaly import BGPAnomaly +from app.models.bgp_collector_location import BGPCollectorLocation from app.models.bgp_incident import BGPIncident from app.models.bgp_observation import BGPObservation +from app.models.compute_center_location import ComputeCenterLocationRecord from app.models.system_setting import SystemSetting from app.models.playground_session import PlaygroundSession from app.models.playground_message import PlaygroundMessage +from app.models.system_log import SystemLog, AuditLog +from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth, VesselPosition, VesselStatic +from app.models.datasource_mapping import DataSourceMappingTemplate __all__ = [ "User", @@ -24,6 +29,18 @@ __all__ = [ "AlertSeverity", "AlertStatus", "BGPAnomaly", + "BGPCollectorLocation", "BGPIncident", "BGPObservation", + "ComputeCenterLocationRecord", + "SystemLog", + "AuditLog", + "PlaygroundSession", + "PlaygroundMessage", + "VesselPosition", + "VesselStatic", + "AISRawObservation", + "AISConflictRecord", + "AISSourceHealth", + "DataSourceMappingTemplate", ] diff --git a/backend/app/models/bgp_collector_location.py b/backend/app/models/bgp_collector_location.py new file mode 100644 index 00000000..e9b5ce76 --- /dev/null +++ b/backend/app/models/bgp_collector_location.py @@ -0,0 +1,52 @@ +"""Stored BGP route-collector locations.""" + +from sqlalchemy import Boolean, Column, DateTime, Float, Integer, JSON, String, Text +from sqlalchemy.sql import func + +from app.core.time import to_iso8601_utc +from app.db.session import Base + + +class BGPCollectorLocation(Base): + """Current known location for a BGP route collector.""" + + __tablename__ = "bgp_collector_locations" + + id = Column(Integer, primary_key=True, autoincrement=True) + collector_id = Column(String(100), nullable=False, unique=True, index=True) + operator = Column(String(255), nullable=True) + site = Column(String(255), nullable=True) + city = Column(String(255), nullable=True) + country = Column(String(255), nullable=True) + latitude = Column(Float, nullable=True) + longitude = Column(Float, nullable=True) + precision = Column(String(30), nullable=False, default="city") + confidence = Column(Float, nullable=True) + source = Column(String(80), nullable=False, default="legacy_seed", index=True) + source_url = Column(String(500), nullable=True) + source_note = Column(Text, nullable=True) + raw_payload = Column(JSON, nullable=False, default=dict) + needs_confirmation = Column(Boolean, nullable=False, default=True, index=True) + verification_status = Column(String(30), nullable=False, default="unverified", index=True) + verified_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + def to_location_dict(self) -> dict: + return { + "city": self.city, + "country": self.country, + "latitude": self.latitude, + "longitude": self.longitude, + "precision": self.precision, + "source": self.source, + "needs_confirmation": self.needs_confirmation, + "matched_location_name": self.site or self.collector_id, + "verified_at": to_iso8601_utc(self.verified_at), + "confidence": self.confidence, + "operator": self.operator, + "site": self.site, + "verification_status": self.verification_status, + "source_note": self.source_note, + "source_url": self.source_url, + } diff --git a/backend/app/models/collected_data.py b/backend/app/models/collected_data.py index 438db389..5bb8f195 100644 --- a/backend/app/models/collected_data.py +++ b/backend/app/models/collected_data.py @@ -48,6 +48,8 @@ class CollectedData(Base): # Indexes for common queries __table_args__ = ( Index("idx_collected_data_source_collected", "source", "collected_at"), + Index("idx_collected_data_source_current_id", "source", "is_current", "id"), + Index("idx_collected_data_source_task_id", "source", "task_id", "id"), Index("idx_collected_data_source_type", "source", "data_type"), Index("idx_collected_data_source_source_id", "source", "source_id"), ) diff --git a/backend/app/models/compute_center_location.py b/backend/app/models/compute_center_location.py new file mode 100644 index 00000000..5466e7cb --- /dev/null +++ b/backend/app/models/compute_center_location.py @@ -0,0 +1,60 @@ +"""Stored compute-center locations.""" + +from sqlalchemy import Boolean, Column, DateTime, Float, Integer, JSON, String, Text, UniqueConstraint +from sqlalchemy.sql import func + +from app.core.time import to_iso8601_utc +from app.db.session import Base + + +class ComputeCenterLocationRecord(Base): + """Current known location for a compute-center record.""" + + __tablename__ = "compute_center_locations" + __table_args__ = ( + UniqueConstraint("source", "source_id", name="uq_compute_center_location_source_id"), + ) + + id = Column(Integer, primary_key=True, autoincrement=True) + source = Column(String(100), nullable=False, index=True) + source_id = Column(String(255), nullable=False, index=True) + name = Column(String(500), nullable=True) + operator = Column(String(255), nullable=True) + site = Column(String(255), nullable=True) + city = Column(String(255), nullable=True) + country = Column(String(255), nullable=True) + latitude = Column(Float, nullable=True) + longitude = Column(Float, nullable=True) + precision = Column(String(30), nullable=False, default="city") + confidence = Column(Float, nullable=True) + location_source = Column(String(80), nullable=False, default="stored_compute_center_location", index=True) + source_url = Column(String(500), nullable=True) + source_note = Column(Text, nullable=True) + raw_payload = Column(JSON, nullable=False, default=dict) + needs_confirmation = Column(Boolean, nullable=False, default=False, index=True) + verification_status = Column(String(30), nullable=False, default="verified", index=True) + verified_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + def to_location_dict(self) -> dict: + return { + "source": self.source, + "source_id": self.source_id, + "name": self.name, + "operator": self.operator, + "site": self.site, + "city": self.city, + "country": self.country, + "latitude": self.latitude, + "longitude": self.longitude, + "precision": self.precision, + "confidence": self.confidence, + "location_source": self.location_source, + "source_url": self.source_url, + "source_note": self.source_note, + "raw_payload": self.raw_payload or {}, + "needs_confirmation": self.needs_confirmation, + "verification_status": self.verification_status, + "verified_at": to_iso8601_utc(self.verified_at), + } diff --git a/backend/app/models/datasource_mapping.py b/backend/app/models/datasource_mapping.py new file mode 100644 index 00000000..eee2c269 --- /dev/null +++ b/backend/app/models/datasource_mapping.py @@ -0,0 +1,32 @@ +"""Mapping templates for user-defined data source payloads.""" + +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, JSON, String +from sqlalchemy.sql import func + +from app.db.session import Base + + +class DataSourceMappingTemplate(Base): + __tablename__ = "datasource_mapping_templates" + + id = Column(Integer, primary_key=True, autoincrement=True) + datasource_config_id = Column( + Integer, + ForeignKey("datasource_configs.id"), + nullable=False, + index=True, + ) + target_schema = Column(String(80), nullable=False, index=True) + mapping_json = Column(JSON, nullable=False, default={}) + sample_payload_hash = Column(String(64), nullable=True) + validation_status = Column(String(30), nullable=False, default="draft") + version = Column(Integer, nullable=False, default=1) + is_active = Column(Boolean, nullable=False, default=False, index=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + def __repr__(self): + return ( + f"" + ) diff --git a/backend/app/models/system_log.py b/backend/app/models/system_log.py new file mode 100644 index 00000000..5024c9b4 --- /dev/null +++ b/backend/app/models/system_log.py @@ -0,0 +1,40 @@ +from sqlalchemy import JSON, Column, DateTime, Integer, String, Text +from sqlalchemy.sql import func + +from app.db.session import Base + + +class SystemLog(Base): + __tablename__ = "system_logs" + + id = Column(Integer, primary_key=True, autoincrement=True) + occurred_at = Column(DateTime(timezone=True), server_default=func.now(), index=True) + source = Column(String(50), nullable=False, index=True) + service = Column(String(50), nullable=True) + module = Column(String(120), nullable=True) + event = Column(String(160), nullable=True, index=True) + level = Column(String(20), nullable=False, index=True) + message = Column(Text, nullable=False) + request_id = Column(String(64), nullable=True, index=True) + trace_id = Column(String(64), nullable=True) + user_id = Column(Integer, nullable=True, index=True) + category = Column(String(80), nullable=True, index=True) + context = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class AuditLog(Base): + __tablename__ = "audit_logs" + + id = Column(Integer, primary_key=True, autoincrement=True) + occurred_at = Column(DateTime(timezone=True), server_default=func.now(), index=True) + actor_id = Column(Integer, nullable=True, index=True) + actor_name = Column(String(255), nullable=True) + action = Column(String(120), nullable=False, index=True) + target_type = Column(String(80), nullable=True) + target_id = Column(String(120), nullable=True) + result = Column(String(40), nullable=True, index=True) + request_id = Column(String(64), nullable=True, index=True) + ip = Column(String(64), nullable=True) + details = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime(timezone=True), server_default=func.now()) diff --git a/backend/app/models/task.py b/backend/app/models/task.py index 12d858c2..0e29c299 100644 --- a/backend/app/models/task.py +++ b/backend/app/models/task.py @@ -1,6 +1,6 @@ """Collection Task model""" -from sqlalchemy import Column, DateTime, Integer, String, Text, Float +from sqlalchemy import BigInteger, Column, DateTime, Integer, String, Text, Float from sqlalchemy.sql import func from app.db.session import Base @@ -13,6 +13,11 @@ class CollectionTask(Base): datasource_id = Column(Integer, nullable=False, index=True) status = Column(String(20), nullable=False) # pending, running, success, failed, cancelled phase = Column(String(30), default="queued") + phase_progress = Column(Float) + phase_message = Column(String(255)) + phase_current = Column(BigInteger) + phase_total = Column(BigInteger) + phase_unit = Column(String(30)) started_at = Column(DateTime(timezone=True)) completed_at = Column(DateTime(timezone=True)) records_processed = Column(Integer, default=0) diff --git a/backend/app/models/user.py b/backend/app/models/user.py index 5e588539..b16f4c49 100644 --- a/backend/app/models/user.py +++ b/backend/app/models/user.py @@ -1,4 +1,4 @@ -from sqlalchemy import Boolean, Column, Integer, String, DateTime +from sqlalchemy import Boolean, Column, DateTime, Integer, JSON, String from sqlalchemy.sql import func from app.db.session import Base @@ -12,6 +12,7 @@ class User(Base): email = Column(String(255), unique=True, index=True, nullable=False) password_hash = Column(String(255), nullable=False) role = Column(String(20), default="viewer") + gatekeeper_groups = Column(JSON, default=list) is_active = Column(Boolean, default=True) last_login_at = Column(DateTime(timezone=True)) created_at = Column(DateTime(timezone=True), server_default=func.now()) diff --git a/backend/app/models/vessel.py b/backend/app/models/vessel.py new file mode 100644 index 00000000..35e93b1c --- /dev/null +++ b/backend/app/models/vessel.py @@ -0,0 +1,186 @@ +"""Vessel AIS models for live maritime tracking.""" + +from sqlalchemy import BigInteger, Column, DateTime, Float, Index, Integer, JSON, SmallInteger, String +from sqlalchemy.sql import func + +from app.core.time import to_iso8601_utc +from app.db.session import Base + + +class VesselStatic(Base): + """Slow-changing vessel identity and dimensions.""" + + __tablename__ = "vessel_static" + + mmsi = Column(BigInteger, primary_key=True) + name = Column(String(128), nullable=True) + callsign = Column(String(16), nullable=True) + vessel_type = Column(SmallInteger, nullable=True, index=True) + vessel_type_name = Column(String(64), nullable=True, index=True) + flag = Column(String(4), nullable=True, index=True) + length = Column(Float, nullable=True) + width = Column(Float, nullable=True) + draught = Column(Float, nullable=True) + imo = Column(BigInteger, nullable=True) + updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + def to_dict(self) -> dict: + return { + "mmsi": self.mmsi, + "name": self.name, + "callsign": self.callsign, + "vessel_type": self.vessel_type, + "vessel_type_name": self.vessel_type_name, + "flag": self.flag, + "length": self.length, + "width": self.width, + "draught": self.draught, + "imo": self.imo, + "updated_at": to_iso8601_utc(self.updated_at), + } + + +class VesselPosition(Base): + """Append-only AIS positions retained for short history windows.""" + + __tablename__ = "vessel_position" + + id = Column(Integer, primary_key=True, autoincrement=True) + mmsi = Column(BigInteger, nullable=False, index=True) + lat = Column(Float, nullable=False) + lon = Column(Float, nullable=False) + sog = Column(Float, nullable=True) + cog = Column(Float, nullable=True) + heading = Column(SmallInteger, nullable=True) + nav_status = Column(SmallInteger, nullable=True, index=True) + received_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True) + + __table_args__ = ( + Index("idx_vessel_pos_mmsi_time", "mmsi", "received_at"), + Index("idx_vessel_pos_time", "received_at"), + Index("idx_vessel_pos_lat_lon", "lat", "lon"), + ) + + def to_dict(self) -> dict: + return { + "id": self.id, + "mmsi": self.mmsi, + "lat": self.lat, + "lon": self.lon, + "sog": self.sog, + "cog": self.cog, + "heading": self.heading, + "nav_status": self.nav_status, + "received_at": to_iso8601_utc(self.received_at), + } + + +class AISRawObservation(Base): + """Source-level AIS fact before aggregation and conflict resolution.""" + + __tablename__ = "ais_raw_observations" + + id = Column(Integer, primary_key=True, autoincrement=True) + target_schema = Column(String(64), nullable=False, default="vessel_ais", index=True) + source = Column(String(100), nullable=False, index=True) + entity_key = Column(String(64), nullable=False, index=True) + delivery_mode = Column(String(32), nullable=False, index=True) + transport = Column(String(32), nullable=False, index=True) + message_type = Column(String(64), nullable=True, index=True) + source_message_id = Column(String(128), nullable=True, index=True) + observation_hash = Column(String(64), nullable=False, unique=True, index=True) + observed_at = Column(DateTime(timezone=True), nullable=False, index=True) + collected_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True) + normalized_payload = Column(JSON, default=dict) + raw_payload = Column(JSON, default=dict) + quality_flags = Column(JSON, default=list) + + __table_args__ = ( + Index("idx_ais_raw_entity_observed", "target_schema", "entity_key", "observed_at"), + Index("idx_ais_raw_schema_observed_entity", "target_schema", "observed_at", "entity_key"), + Index("idx_ais_raw_source_entity", "source", "entity_key"), + ) + + def to_dict(self) -> dict: + return { + "id": self.id, + "target_schema": self.target_schema, + "source": self.source, + "entity_key": self.entity_key, + "delivery_mode": self.delivery_mode, + "transport": self.transport, + "message_type": self.message_type, + "source_message_id": self.source_message_id, + "observation_hash": self.observation_hash, + "observed_at": to_iso8601_utc(self.observed_at), + "collected_at": to_iso8601_utc(self.collected_at), + "normalized_payload": self.normalized_payload or {}, + "raw_payload": self.raw_payload or {}, + "quality_flags": self.quality_flags or [], + } + + +class AISConflictRecord(Base): + """Recorded field-level disagreement between AIS sources.""" + + __tablename__ = "ais_conflict_records" + + id = Column(Integer, primary_key=True, autoincrement=True) + target_schema = Column(String(64), nullable=False, default="vessel_ais", index=True) + entity_key = Column(String(64), nullable=False, index=True) + field = Column(String(64), nullable=False, index=True) + candidates = Column(JSON, default=dict) + selected_source = Column(String(100), nullable=True, index=True) + selected_value = Column(JSON, nullable=True) + selected_reason = Column(String(64), nullable=True, index=True) + resolved_by = Column(String(32), nullable=False, default="system", index=True) + status = Column(String(32), nullable=False, default="open", index=True) + created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True) + updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + __table_args__ = ( + Index("idx_ais_conflict_entity_field", "target_schema", "entity_key", "field"), + ) + + def to_dict(self) -> dict: + return { + "id": self.id, + "target_schema": self.target_schema, + "entity_key": self.entity_key, + "field": self.field, + "candidates": self.candidates or {}, + "selected_source": self.selected_source, + "selected_value": self.selected_value, + "selected_reason": self.selected_reason, + "resolved_by": self.resolved_by, + "status": self.status, + "created_at": to_iso8601_utc(self.created_at), + "updated_at": to_iso8601_utc(self.updated_at), + } + + +class AISSourceHealth(Base): + """Runtime health signal for an AIS collector source.""" + + __tablename__ = "ais_source_health" + + source = Column(String(100), primary_key=True) + connection_state = Column(String(32), nullable=False, default="disconnected", index=True) + last_seen_at = Column(DateTime(timezone=True), nullable=True, index=True) + last_success_at = Column(DateTime(timezone=True), nullable=True, index=True) + last_error = Column(String(500), nullable=True) + message_rate = Column(Float, nullable=True) + lag_seconds = Column(Float, nullable=True) + updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True) + + def to_dict(self) -> dict: + return { + "source": self.source, + "connection_state": self.connection_state, + "last_seen_at": to_iso8601_utc(self.last_seen_at), + "last_success_at": to_iso8601_utc(self.last_success_at), + "last_error": self.last_error, + "message_rate": self.message_rate, + "lag_seconds": self.lag_seconds, + "updated_at": to_iso8601_utc(self.updated_at), + } diff --git a/backend/app/models/vessel_enrichment.py b/backend/app/models/vessel_enrichment.py new file mode 100644 index 00000000..fc73208d --- /dev/null +++ b/backend/app/models/vessel_enrichment.py @@ -0,0 +1,63 @@ +"""Vessel enrichment cache tables (v5). + +Profile and media enrichment are stored separately so cache TTLs can differ +and so the conflict-resolution + display layers can read either independently. +""" + +from sqlalchemy import BigInteger, Column, DateTime, Float, JSON, String +from sqlalchemy.sql import func + +from app.core.time import to_iso8601_utc +from app.db.session import Base + + +class VesselProfileEnrichment(Base): + """Cached static vessel profile (type, flag, dimensions, operator, etc.).""" + + __tablename__ = "vessel_profile_enrichment" + + mmsi = Column(BigInteger, primary_key=True) + source = Column(String(100), nullable=False, default="system") + payload = Column(JSON, nullable=False, default=dict) + fetched_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + expires_at = Column(DateTime(timezone=True), nullable=True) + confidence = Column(Float, nullable=True) + reference_url = Column(String(500), nullable=True) + updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()) + + def to_dict(self) -> dict: + return { + "mmsi": self.mmsi, + "source": self.source, + "payload": self.payload or {}, + "fetched_at": to_iso8601_utc(self.fetched_at), + "expires_at": to_iso8601_utc(self.expires_at), + "confidence": self.confidence, + "reference_url": self.reference_url, + } + + +class VesselMediaEnrichment(Base): + """Cached vessel imagery / external detail references.""" + + __tablename__ = "vessel_media_enrichment" + + mmsi = Column(BigInteger, primary_key=True) + source = Column(String(100), nullable=False, default="system") + payload = Column(JSON, nullable=False, default=dict) + fetched_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + expires_at = Column(DateTime(timezone=True), nullable=True) + confidence = Column(Float, nullable=True) + reference_url = Column(String(500), nullable=True) + updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()) + + def to_dict(self) -> dict: + return { + "mmsi": self.mmsi, + "source": self.source, + "payload": self.payload or {}, + "fetched_at": to_iso8601_utc(self.fetched_at), + "expires_at": to_iso8601_utc(self.expires_at), + "confidence": self.confidence, + "reference_url": self.reference_url, + } diff --git a/backend/app/schemas/user.py b/backend/app/schemas/user.py index 13742b4a..9148d561 100644 --- a/backend/app/schemas/user.py +++ b/backend/app/schemas/user.py @@ -12,17 +12,20 @@ class UserBase(BaseModel): class UserCreate(UserBase): password: str = Field(..., min_length=8) role: str = "viewer" + gatekeeper_groups: list[str] = Field(default_factory=list) class UserUpdate(BaseModel): email: Optional[EmailStr] = None role: Optional[str] = None + gatekeeper_groups: Optional[list[str]] = None is_active: Optional[bool] = None class UserInDB(UserBase): id: int role: str + gatekeeper_groups: list[str] = Field(default_factory=list) is_active: bool last_login_at: Optional[datetime] created_at: datetime @@ -34,6 +37,7 @@ class UserInDB(UserBase): class UserResponse(UserBase): id: int role: str + gatekeeper_groups: list[str] = Field(default_factory=list) is_active: bool created_at: datetime diff --git a/backend/app/services/ai_client.py b/backend/app/services/ai_client.py index 916a02d1..ed32bb09 100644 --- a/backend/app/services/ai_client.py +++ b/backend/app/services/ai_client.py @@ -3,9 +3,11 @@ from __future__ import annotations import asyncio import httpx -from fastapi import HTTPException, status +from fastapi import Depends, HTTPException, status +from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import settings +from app.db.session import get_db from app.schemas.ai import ( AIProviderStatusResponse, SituationalAnalysisRequest, @@ -14,11 +16,27 @@ from app.schemas.ai import ( class AIProviderClient: - def __init__(self) -> None: - self.service_url = settings.AI_PROVIDER_SERVICE_URL.rstrip("/") - self.service_token = settings.AI_PROVIDER_SERVICE_TOKEN - self.timeout = settings.AI_PROVIDER_TIMEOUT_SECONDS - self.retry_attempts = max(settings.AI_PROVIDER_RETRY_ATTEMPTS, 1) + def __init__( + self, + *, + service_url: str | None = None, + service_token: str | None = None, + timeout: int | None = None, + retry_attempts: int | None = None, + llm_config: dict | None = None, + ) -> None: + self.service_url = ( + service_url if service_url is not None else settings.AI_PROVIDER_SERVICE_URL + ).rstrip("/") + self.service_token = ( + service_token if service_token is not None else settings.AI_PROVIDER_SERVICE_TOKEN + ) + self.timeout = timeout if timeout is not None else settings.AI_PROVIDER_TIMEOUT_SECONDS + self.retry_attempts = max( + retry_attempts if retry_attempts is not None else settings.AI_PROVIDER_RETRY_ATTEMPTS, + 1, + ) + self.llm_config = llm_config or {} def _headers(self, request_id: str | None = None) -> dict[str, str]: headers = {"Content-Type": "application/json"} @@ -26,6 +44,19 @@ class AIProviderClient: headers["X-Provider-Token"] = self.service_token if request_id: headers["X-Request-ID"] = request_id + llm_header_map = { + "provider": "X-AI-Provider", + "provider_api": "X-AI-Provider-API", + "base_url": "X-AI-Base-URL", + "api_key": "X-AI-API-Key", + "model": "X-AI-Model", + "max_tokens": "X-AI-Max-Tokens", + "anthropic_version": "X-AI-Anthropic-Version", + } + for key, header_name in llm_header_map.items(): + value = self.llm_config.get(key) + if value not in (None, ""): + headers[header_name] = str(value) return headers async def get_status(self, request_id: str | None = None) -> AIProviderStatusResponse: @@ -105,5 +136,14 @@ class AIProviderClient: ) -def get_ai_provider_client() -> AIProviderClient: - return AIProviderClient() +async def get_ai_provider_client(db: AsyncSession = Depends(get_db)) -> AIProviderClient: + from app.api.v1.settings import get_runtime_ai_provider_config + + runtime_config = await get_runtime_ai_provider_config(db) + return AIProviderClient( + service_url=runtime_config["service_url"], + service_token=runtime_config["service_token"], + timeout=runtime_config["timeout_seconds"], + retry_attempts=runtime_config["retry_attempts"], + llm_config=runtime_config.get("llm_config") or {}, + ) diff --git a/backend/app/services/barentswatch.py b/backend/app/services/barentswatch.py new file mode 100644 index 00000000..13515bed --- /dev/null +++ b/backend/app/services/barentswatch.py @@ -0,0 +1,209 @@ +"""BarentsWatch AIS credential resolution and connectivity checks.""" + +from __future__ import annotations + +import os +import shlex +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import httpx +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.data_sources import get_data_sources_config +from app.models.datasource_config import DataSourceConfig + + +BARENTSWATCH_LATEST_URL = "https://live.ais.barentswatch.no/v1/latest/combined" +BARENTSWATCH_TOKEN_URL = "https://id.barentswatch.no/connect/token" +BARENTSWATCH_DATASOURCE_NAME = "barentswatch_vessels" + + +@dataclass(frozen=True) +class BarentsWatchConfig: + endpoint: str + client_id: str + client_secret: str + credential_source: str + endpoint_source: str + + +def _read_zshrc_env(path: Path | None = None) -> dict[str, str]: + zshrc_path = path or Path.home() / ".zshrc" + if not zshrc_path.exists(): + return {} + + values: dict[str, str] = {} + for raw_line in zshrc_path.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[len("export ") :].strip() + if "=" not in line: + continue + + key, value = line.split("=", 1) + key = key.strip() + if not key or not key.replace("_", "").isalnum() or not key[0].isalpha(): + continue + + try: + parsed = shlex.split(value, comments=True, posix=True) + except ValueError: + parsed = [value.strip().strip("'\"")] + if parsed: + values[key] = parsed[0] + return values + + +def _first_env_value(zshrc_env: dict[str, str], *keys: str) -> tuple[str, str]: + for key in keys: + value = os.getenv(key) + if value: + return value, "environment" + for key in keys: + value = zshrc_env.get(key) + if value: + return value, "~/.zshrc" + return "", "" + + +async def get_barentswatch_datasource_record(db: AsyncSession) -> DataSourceConfig | None: + result = await db.execute( + select(DataSourceConfig) + .where(DataSourceConfig.name == BARENTSWATCH_DATASOURCE_NAME) + .where(DataSourceConfig.is_active.is_(True)) + ) + return result.scalar_one_or_none() + + +async def resolve_barentswatch_config(db: AsyncSession | None = None) -> BarentsWatchConfig: + record = await get_barentswatch_datasource_record(db) if db else None + auth_config = dict(record.auth_config or {}) if record else {} + config = dict(record.config or {}) if record else {} + zshrc_env = _read_zshrc_env() + + env_client_id, env_source = _first_env_value( + zshrc_env, + "BARENTSWATCH_CLIENT_ID", + "BARRENTSWATCH_CLIENT_ID", + ) + env_client_secret, secret_env_source = _first_env_value( + zshrc_env, + "BARENTSWATCH_CLIENT_SECRET", + "BARRENTSWATCH_CLIENT_SECRET", + ) + client_id = auth_config.get("client_id") or config.get("client_id") or env_client_id + client_secret = ( + auth_config.get("client_secret") or config.get("client_secret") or env_client_secret + ) + + credential_source = "" + if auth_config.get("client_id") or auth_config.get("client_secret"): + credential_source = "datasource_config" + elif config.get("client_id") or config.get("client_secret"): + credential_source = "datasource_runtime_config" + elif env_source or secret_env_source: + credential_source = env_source or secret_env_source + + yaml_endpoint = get_data_sources_config().get_yaml_url(BARENTSWATCH_DATASOURCE_NAME) + endpoint = record.endpoint if record and record.endpoint else yaml_endpoint + return BarentsWatchConfig( + endpoint=endpoint or BARENTSWATCH_LATEST_URL, + client_id=str(client_id or ""), + client_secret=str(client_secret or ""), + credential_source=credential_source or "missing", + endpoint_source="datasource_config" if record and record.endpoint else "default", + ) + + +async def fetch_barentswatch_access_token( + client: httpx.AsyncClient, + config: BarentsWatchConfig, +) -> str | None: + if not config.client_id or not config.client_secret: + return None + + response = await client.post( + BARENTSWATCH_TOKEN_URL, + data={ + "client_id": config.client_id, + "client_secret": config.client_secret, + "scope": "ais", + "grant_type": "client_credentials", + }, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + response.raise_for_status() + payload = response.json() + token = payload.get("access_token") + return str(token) if token else None + + +async def check_barentswatch_connectivity(db: AsyncSession) -> dict[str, Any]: + config = await resolve_barentswatch_config(db) + return await check_barentswatch_config(config) + + +async def check_barentswatch_config(config: BarentsWatchConfig) -> dict[str, Any]: + if not config.client_id or not config.client_secret: + return { + "success": False, + "stage": "credentials", + "message": "未找到 BarentsWatch client id/client secret，请先配置采集器凭证。", + "endpoint": config.endpoint, + "credential_source": config.credential_source, + "settings_tab": "collector_credentials", + } + + try: + async with httpx.AsyncClient(timeout=20.0) as client: + token = await fetch_barentswatch_access_token(client, config) + if not token: + return { + "success": False, + "stage": "token", + "message": "BarentsWatch token 响应中没有 access_token，请检查凭证。", + "endpoint": config.endpoint, + "credential_source": config.credential_source, + "settings_tab": "collector_credentials", + } + + async with client.stream( + "GET", + config.endpoint, + headers={"Authorization": f"Bearer {token}"}, + ) as response: + response.raise_for_status() + + return { + "success": True, + "stage": "endpoint", + "message": "BarentsWatch AIS token 和数据接口均可连通。", + "endpoint": config.endpoint, + "credential_source": config.credential_source, + "endpoint_source": config.endpoint_source, + } + except httpx.HTTPStatusError as exc: + status_code = exc.response.status_code + stage = "token" if str(exc.request.url) == BARENTSWATCH_TOKEN_URL else "endpoint" + return { + "success": False, + "stage": stage, + "message": f"BarentsWatch {stage} 请求返回 HTTP {status_code}，请检查凭证或接口地址。", + "endpoint": config.endpoint, + "credential_source": config.credential_source, + "settings_tab": "collector_credentials", + } + except httpx.HTTPError as exc: + return { + "success": False, + "stage": "network", + "message": f"BarentsWatch 链路检查失败：{exc.__class__.__name__}", + "endpoint": config.endpoint, + "credential_source": config.credential_source, + "settings_tab": "collector_credentials", + } diff --git a/backend/app/services/bgp_collector_locations.py b/backend/app/services/bgp_collector_locations.py new file mode 100644 index 00000000..5798514a --- /dev/null +++ b/backend/app/services/bgp_collector_locations.py @@ -0,0 +1,306 @@ +"""BGP route-collector location resolver. + +Collector positions are stored in the ``bgp_collector_locations`` database +table. The old JSON registry is now only a seed payload used during database +initialization, not a runtime resolver or candidate source. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Iterator + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.bgp_collector_location import BGPCollectorLocation +from app.services.location import ( + LocationCandidate, + LocationPipeline, + LocationQuery, + NominatimResolver, + ResolutionResult, + ResolverOutput, + SourceCoordinatesResolver, + build_default_nominatim_geocoder, + coerce_str, + normalize_text, +) + +SEED_PATH = ( + Path(__file__).resolve().parents[1] + / "data" + / "seeds" + / "ripe_ris_collector_locations_seed.json" +) + +# ── Geocoder (kept at module level for monkeypatching + cache_clear) ── + +_geocode_online = build_default_nominatim_geocoder() + + +# ── In-process compatibility cache ────────────────────────────────── + + +RIPE_RIS_COLLECTOR_COORDS: dict[str, dict[str, Any]] = {} + + +def _collector_record_to_dict(record: BGPCollectorLocation) -> dict[str, Any]: + return record.to_location_dict() + + +def set_bgp_collector_location_cache( + locations: dict[str, dict[str, Any]], +) -> None: + """Replace the legacy compatibility cache in-place.""" + RIPE_RIS_COLLECTOR_COORDS.clear() + RIPE_RIS_COLLECTOR_COORDS.update( + {coerce_str(key): dict(value) for key, value in locations.items()} + ) + + +async def refresh_bgp_collector_location_cache( + session: AsyncSession, +) -> dict[str, dict[str, Any]]: + result = await session.execute(select(BGPCollectorLocation)) + records = result.scalars().all() + cache = { + record.collector_id: _collector_record_to_dict(record) + for record in records + if record.collector_id + } + set_bgp_collector_location_cache(cache) + return cache + + +def _load_seed_payload() -> dict[str, Any]: + with SEED_PATH.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def _seed_entry_to_record_kwargs(entry: dict[str, Any], collector_id: str) -> dict[str, Any]: + return { + "collector_id": collector_id, + "operator": entry.get("operator") or "RIPE NCC", + "site": entry.get("site"), + "city": entry.get("city"), + "country": entry.get("country"), + "latitude": entry.get("latitude"), + "longitude": entry.get("longitude"), + "precision": entry.get("precision") or "city", + "confidence": entry.get("confidence"), + "source": "legacy_seed", + "source_url": None, + "source_note": entry.get("source_note") + or "Seeded from legacy RIPE RIS collector coordinates", + "raw_payload": entry, + "needs_confirmation": True, + "verification_status": "unverified", + "verified_at": None, + } + + +async def seed_default_bgp_collector_locations(session: AsyncSession) -> None: + """Seed default RIPE RIS collector locations without overwriting users.""" + payload = _load_seed_payload() + for entry in payload.get("locations", []): + aliases = entry.get("aliases") or [] + collector_ids = [ + coerce_str(alias) + for alias in aliases + if coerce_str(alias).startswith("rrc") + ] + if not collector_ids: + continue + collector_id = collector_ids[0] + existing = await session.scalar( + select(BGPCollectorLocation).where( + BGPCollectorLocation.collector_id == collector_id + ) + ) + if existing: + continue + session.add( + BGPCollectorLocation( + **_seed_entry_to_record_kwargs(entry, collector_id) + ) + ) + await session.commit() + await refresh_bgp_collector_location_cache(session) + + +def get_bgp_collector_location_dict(collector_name: str) -> dict[str, Any]: + """Return the current cached collector location dict, or ``{}`` if unknown.""" + return dict(RIPE_RIS_COLLECTOR_COORDS.get(coerce_str(collector_name), {})) + + +def iter_known_collector_names() -> Iterator[str]: + """Yield every collector technical name (rrcXX) known in the cache.""" + return iter(sorted(RIPE_RIS_COLLECTOR_COORDS.keys())) + + +# ── Pipeline construction ────────────────────────────────────────── + + +class StoredCollectorLocationResolver: + """Resolve a collector through the DB-backed compatibility cache.""" + + name = "stored_collector_location" + + def resolve(self, query: LocationQuery) -> ResolverOutput: + collector = coerce_str(query.name) + if not collector: + for alias in query.aliases: + collector = coerce_str(alias) + if collector: + break + if not collector: + return ResolverOutput() + location = get_bgp_collector_location_dict(collector) + if not location: + return ResolverOutput() + latitude = location.get("latitude") + longitude = location.get("longitude") + if latitude in (None, 0.0) or longitude in (None, 0.0): + return ResolverOutput() + return ResolverOutput( + candidates=( + LocationCandidate( + latitude=float(latitude), + longitude=float(longitude), + display_name=location.get("matched_location_name") or collector, + precision=location.get("precision") or "city", + confidence=float(location.get("confidence") or 0.85), + query=f"stored_collector_location::{collector}", + source=location.get("source") or self.name, + source_note=location.get("source_note"), + matched_fields=("collector",), + needs_confirmation=bool(location.get("needs_confirmation")), + city=location.get("city"), + region=None, + country=location.get("country"), + matched_location_name=( + location.get("matched_location_name") or collector + ), + location_verified_at=location.get("verified_at"), + suggested_registry_entry=None, + ), + ) + ) + + +def _bgp_collector_query_plan( + query: LocationQuery, +) -> list[tuple[str, tuple[str, ...]]]: + """Build the Nominatim query plan for a BGP collector.""" + extra = query.extra or {} + site = str(extra.get("site") or "") + operator = str(extra.get("operator") or "") + city = query.city or "" + country = query.country or "" + + plan: list[tuple[str, tuple[str, ...]]] = [] + + def add(parts: list[tuple[str, str]]) -> None: + non_empty = [(field, value) for field, value in parts if value] + if not non_empty: + return + seen: set[str] = set() + cleaned: list[str] = [] + fields: list[str] = [] + for field, value in non_empty: + key = normalize_text(value) + if not key or key in seen: + continue + seen.add(key) + cleaned.append(value) + fields.append(field) + if not cleaned: + return + composed = ", ".join(cleaned) + if not any(composed == existing for existing, _ in plan): + plan.append((composed, tuple(fields))) + + add([("site", site), ("city", city), ("country", country)]) + add([("site", site), ("country", country)]) + add([("operator", operator), ("city", city), ("country", country)]) + add([("city", city), ("country", country)]) + return plan + + +BGP_COLLECTOR_PIPELINE = LocationPipeline( + [ + SourceCoordinatesResolver(), + StoredCollectorLocationResolver(), + ], + failure_reason=( + "Could not resolve BGP collector to renderable coordinates from" + " source coordinates or stored collector location." + ), +) + +BGP_COLLECTOR_COLLECTION_PIPELINE = LocationPipeline( + [ + SourceCoordinatesResolver(), + NominatimResolver( + query_plan_builder=_bgp_collector_query_plan, + # Late-binding so tests can monkeypatch ``_geocode_online``. + geocoder=lambda q: _geocode_online(q), + ), + ], + failure_reason=( + "Could not resolve BGP collector to renderable coordinates from" + " source coordinates or online geocoding." + ), +) + + +# ── Public API ───────────────────────────────────────────────────── + + +def resolve_bgp_collector_location( + collector_name: str, + *, + city: str | None = None, + country: str | None = None, + site: str | None = None, + operator: str | None = None, +) -> ResolutionResult: + """Resolve a BGP collector to its best-known stored location.""" + stored = get_bgp_collector_location_dict(collector_name) + name = coerce_str(collector_name) or None + query = LocationQuery( + name=name, + aliases=tuple(filter(None, (collector_name,))), + city=coerce_str(city or stored.get("city")) or None, + country=coerce_str(country or stored.get("country")) or None, + extra={ + "site": coerce_str(site or stored.get("site")), + "operator": coerce_str(operator or stored.get("operator")) or "RIPE NCC", + }, + ) + return BGP_COLLECTOR_PIPELINE.resolve_best(query) + + +def collect_bgp_collector_location_candidates( + *, + collector: str | None = None, + city: str | None = None, + country: str | None = None, + site: str | None = None, + operator: str | None = None, +) -> tuple[list[LocationCandidate], list[str]]: + stored = get_bgp_collector_location_dict(collector or "") + name = coerce_str(collector) or None + query = LocationQuery( + name=name, + aliases=tuple(filter(None, (collector,))), + city=coerce_str(city or stored.get("city")) or None, + country=coerce_str(country or stored.get("country")) or None, + extra={ + "site": coerce_str(site or stored.get("site")), + "operator": coerce_str(operator or stored.get("operator")) or "RIPE NCC", + }, + ) + return BGP_COLLECTOR_COLLECTION_PIPELINE.collect_candidates(query) diff --git a/backend/app/services/bgp_event_locations.py b/backend/app/services/bgp_event_locations.py new file mode 100644 index 00000000..a647f258 --- /dev/null +++ b/backend/app/services/bgp_event_locations.py @@ -0,0 +1,155 @@ +"""BGP event location resolver. + +A BGP event (announcement / withdrawal / RIB entry) is geographically tied to +the route collector that observed it. This module defines the pipeline that +turns an event payload into renderable coordinates. + +Current resolver chain: + + SourceCoordinates → event payload itself carries lat/lon (rare; some + enriched feeds do). + InheritFromCollector → look up the owning collector via + :func:`resolve_bgp_collector_location`. + +Future plug-ins (no consumer changes required, just append to the list): + + ASNFacilityResolver — origin/peer ASN → peeringdb facility. + PrefixGeoResolver — prefix → IP range geo lookup (iptoasn / opengeofeed). +""" + +from __future__ import annotations + +from typing import Any + +from app.services.bgp_collector_locations import ( + get_bgp_collector_location_dict, +) +from app.services.location import ( + InheritFromAnotherEntityResolver, + LocationCandidate, + LocationPipeline, + LocationQuery, + ResolutionResult, + SourceCoordinatesResolver, + coerce_str, +) + + +def _inherit_from_owning_collector( + query: LocationQuery, +) -> LocationCandidate | None: + """Look up the event's owning collector by exact name in the DB-backed cache.""" + extra = query.extra or {} + collector_name = coerce_str(extra.get("collector")) + if not collector_name: + return None + legacy = get_bgp_collector_location_dict(collector_name) + if not legacy: + return None + latitude = legacy.get("latitude") + longitude = legacy.get("longitude") + if latitude in (None, 0.0) or longitude in (None, 0.0): + return None + return LocationCandidate( + latitude=float(latitude), + longitude=float(longitude), + display_name=legacy.get("matched_location_name") or collector_name, + precision=legacy.get("precision") or "city", + confidence=float(legacy.get("confidence") or 0.85), + query=f"inherit_from_collector::{collector_name}", + source="inherited_from_collector", + source_note=( + f"Inherited from owning collector {collector_name}" + ), + matched_fields=("collector",), + needs_confirmation=bool(legacy.get("needs_confirmation")), + city=legacy.get("city"), + region=None, + country=legacy.get("country"), + matched_location_name=legacy.get("matched_location_name"), + location_verified_at=legacy.get("verified_at"), + suggested_registry_entry=None, + ) + + +BGP_EVENT_PIPELINE = LocationPipeline( + [ + SourceCoordinatesResolver(), + InheritFromAnotherEntityResolver( + source_lookup=_inherit_from_owning_collector, + name="inherited_from_collector", + ), + # Plug new resolvers (peeringdb / ASN facility / prefix-geo) here. + ], + failure_reason=( + "Could not resolve BGP event coordinates: no source coords, owning" + " collector unknown, and no fallback resolver matched." + ), +) + + +def resolve_bgp_event_location( + *, + collector: str, + source_latitude: float | None = None, + source_longitude: float | None = None, + site: str | None = None, + operator: str | None = None, + peer_asn: int | None = None, + origin_asn: int | None = None, + prefix: str | None = None, +) -> ResolutionResult: + """Resolve a BGP event to its renderable coordinates. + + The ``peer_asn`` / ``origin_asn`` / ``prefix`` arguments are accepted + today so future resolvers (ASN→facility, prefix→geo) can consume them + without callers needing to change. + """ + query = LocationQuery( + name=collector or None, + aliases=tuple(filter(None, (collector,))), + source_latitude=source_latitude, + source_longitude=source_longitude, + extra={ + "collector": collector or "", + "site": coerce_str(site), + "operator": coerce_str(operator), + "peer_asn": peer_asn, + "origin_asn": origin_asn, + "prefix": coerce_str(prefix), + }, + ) + return BGP_EVENT_PIPELINE.resolve_best(query) + + +def resolve_bgp_event_geo_dict( + collector: str, + *, + source_latitude: float | None = None, + source_longitude: float | None = None, +) -> dict[str, Any]: + """Convenience wrapper returning the legacy ``collector_geo`` dict shape. + + Preserves ``city``/``country``/``latitude``/``longitude`` keys (consumed + by existing detectors / enrichment / DB serialization) and adds + ``precision``/``source``/``needs_confirmation`` for richer downstream use. + """ + result = resolve_bgp_event_location( + collector=collector, + source_latitude=source_latitude, + source_longitude=source_longitude, + ) + candidate = result.location + if candidate is None: + return {} + return { + "city": candidate.city, + "country": candidate.country, + "latitude": candidate.latitude, + "longitude": candidate.longitude, + "precision": candidate.precision, + "source": candidate.source, + "needs_confirmation": candidate.needs_confirmation, + "matched_location_name": candidate.matched_location_name, + "confidence": candidate.confidence, + } diff --git a/backend/app/services/collectors/__init__.py b/backend/app/services/collectors/__init__.py index add854b0..9ddbf91b 100644 --- a/backend/app/services/collectors/__init__.py +++ b/backend/app/services/collectors/__init__.py @@ -36,6 +36,8 @@ from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector from app.services.collectors.opengeofeed import OpenGeoFeedPrefixGeoCollector from app.services.collectors.nro_delegated import NRODelegatedPrefixGeoCollector from app.services.collectors.news_live_streams import NewsLiveStreamsCollector +from app.services.collectors.aisstream import AISStreamCollector +from app.services.collectors.vessel_ais import VesselAISCollector collector_registry.register(TOP500Collector()) collector_registry.register(EpochAIGPUCollector()) @@ -63,3 +65,41 @@ collector_registry.register(IPtoASNPrefixGeoCollector()) collector_registry.register(OpenGeoFeedPrefixGeoCollector()) collector_registry.register(NRODelegatedPrefixGeoCollector()) collector_registry.register(NewsLiveStreamsCollector()) +collector_registry.register(VesselAISCollector()) +collector_registry.register(AISStreamCollector()) + +__all__ = [ + "BaseCollector", + "HTTPCollector", + "IntervalCollector", + "collector_registry", + "CollectorRegistry", + "TOP500Collector", + "EpochAIGPUCollector", + "HuggingFaceModelCollector", + "HuggingFaceDatasetCollector", + "HuggingFaceSpacesCollector", + "PeeringDBIXPCollector", + "PeeringDBNetworkCollector", + "PeeringDBFacilityCollector", + "TeleGeographyCableCollector", + "TeleGeographyLandingPointCollector", + "TeleGeographyCableSystemCollector", + "CloudflareRadarDeviceCollector", + "CloudflareRadarTrafficCollector", + "CloudflareRadarTopASCollector", + "ArcGISCableCollector", + "FAOLandingPointCollector", + "ArcGISLandingPointCollector", + "ArcGISCableLandingRelationCollector", + "SpaceTrackTLECollector", + "CelesTrakTLECollector", + "RISLiveCollector", + "BGPStreamBackfillCollector", + "IPtoASNPrefixGeoCollector", + "OpenGeoFeedPrefixGeoCollector", + "NRODelegatedPrefixGeoCollector", + "NewsLiveStreamsCollector", + "VesselAISCollector", + "AISStreamCollector", +] diff --git a/backend/app/services/collectors/aisstream.py b/backend/app/services/collectors/aisstream.py new file mode 100644 index 00000000..3bd24362 --- /dev/null +++ b/backend/app/services/collectors/aisstream.py @@ -0,0 +1,491 @@ +"""AISStream WebSocket collector for realtime vessel AIS observations.""" + +from datetime import UTC, datetime +import asyncio +import json +import os +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.data_sources import get_data_sources_config +from app.core.time import to_iso8601_utc +from app.core.websocket.broadcaster import broadcaster +from app.models.datasource_config import DataSourceConfig +from app.models.task import CollectionTask +from app.services.collectors.base import BaseCollector +from app.services.vessel_ais_aggregation import ( + AISSTREAM_DELIVERY_MODE, + AISSTREAM_TRANSPORT, + record_vessel_ais_observation, + update_ais_source_health, +) +from app.services.vessel_types import normalize_vessel_type_name + +DEFAULT_AISSTREAM_URL = "wss://stream.aisstream.io/v0/stream" +DEFAULT_BOUNDING_BOXES = [[[-90, -180], [90, 180]]] +DEFAULT_MESSAGE_TYPES = ["PositionReport", "ShipStaticData"] + + +class AISStreamCollector(BaseCollector): + """Collect AISStream WebSocket messages into the raw AIS observation layer.""" + + name = "aisstream_vessels" + priority = "P1" + module = "L4" + frequency_hours = 1 + data_type = "vessel_ais" + fail_on_empty = False + + async def _load_datasource_config(self) -> DataSourceConfig | None: + if self._db_session is None: + return None + result = await self._db_session.execute( + select(DataSourceConfig) + .where(DataSourceConfig.name == self.name) + .where(DataSourceConfig.is_active.is_(True)) + ) + return result.scalar_one_or_none() + + async def _get_effective_config(self) -> dict[str, Any]: + datasource_config = await self._load_datasource_config() + config = dict(datasource_config.config or {}) if datasource_config else {} + auth_config = dict(datasource_config.auth_config or {}) if datasource_config else {} + endpoint = ( + (datasource_config.endpoint if datasource_config else None) + or self._resolved_url + or get_data_sources_config().get_yaml_url(self.name) + or DEFAULT_AISSTREAM_URL + ) + api_key = ( + auth_config.get("api_key") + or config.get("api_key") + or os.getenv("AISSTREAM_API_KEY") + ) + return { + "endpoint": endpoint, + "api_key": api_key, + "bounding_boxes": config.get("bounding_boxes") or DEFAULT_BOUNDING_BOXES, + "message_types": config.get("message_types") or DEFAULT_MESSAGE_TYPES, + "max_messages": int(config.get("max_messages") or 500), + "streaming_enabled": config.get("streaming_enabled", True) is not False, + "streaming_commit_interval": int(config.get("streaming_commit_interval") or 1), + "streaming_max_messages": int(config.get("streaming_max_messages") or 0), + "reconnect_delay_seconds": float(config.get("reconnect_delay_seconds") or 5), + "receive_timeout_seconds": float(config.get("receive_timeout_seconds") or 30), + } + + def _build_subscription(self, config: dict[str, Any]) -> dict[str, Any]: + return { + "APIKey": config["api_key"], + "BoundingBoxes": config["bounding_boxes"], + "FilterMessageTypes": config["message_types"], + } + + async def fetch(self) -> list[dict[str, Any]]: + config = await self._get_effective_config() + if not config["api_key"]: + raise RuntimeError("AISStream API key is not configured") + + try: + import websockets + except ImportError as exc: + raise RuntimeError("Python package 'websockets' is required for AISStream") from exc + + subscription = self._build_subscription(config) + + messages: list[dict[str, Any]] = [] + try: + async with websockets.connect(config["endpoint"]) as websocket: + await websocket.send(json.dumps(subscription)) + while len(messages) < config["max_messages"]: + try: + raw_message = await asyncio.wait_for( + websocket.recv(), + timeout=config["receive_timeout_seconds"], + ) + except TimeoutError: + break + payload = json.loads(raw_message) + if isinstance(payload, dict): + messages.append(payload) + except Exception as exc: + if self._db_session is not None: + await update_ais_source_health( + self._db_session, + source=self.name, + connection_state="disconnected", + last_error=f"{exc.__class__.__name__}: {exc}", + ) + await self._db_session.commit() + raise + + return messages + + async def run(self, db: AsyncSession) -> dict[str, Any]: + """Run AISStream as a long-lived streaming collector by default.""" + config = await self._get_effective_config() + if not config.get("streaming_enabled", True): + return await super().run(db) + if not config["api_key"]: + return {"status": "failed", "error": "AISStream API key is not configured"} + + from app.services.collectors.registry import collector_registry + + if not collector_registry.is_active(self.name): + return {"status": "skipped", "reason": "Collector is disabled"} + + try: + import websockets + except ImportError as exc: + return {"status": "failed", "error": "Python package 'websockets' is required for AISStream"} + + start_time = datetime.now(UTC) + task = CollectionTask( + datasource_id=getattr(self, "_datasource_id", 1), + status="running", + phase="connecting", + phase_message="正在连接 AISStream 实时流", + phase_unit="messages", + started_at=start_time, + ) + db.add(task) + await db.commit() + self._current_task = task + self._db_session = db + self._last_broadcast_progress = None + await self.resolve_url(db) + await self._publish_task_update(force=True) + + records_added = 0 + messages_seen = 0 + unique_mmsi: set[str] = set() + reconnect_delay = config["reconnect_delay_seconds"] + + try: + while True: + config = await self._get_effective_config() + subscription = self._build_subscription(config) + try: + await update_ais_source_health( + db, + source=self.name, + connection_state="connecting", + ) + await self.set_phase("connecting", message="正在连接 AISStream 实时流") + await db.commit() + + async with websockets.connect(config["endpoint"]) as websocket: + await websocket.send(json.dumps(subscription)) + await update_ais_source_health( + db, + source=self.name, + connection_state="connected", + last_success_at=datetime.now(UTC), + ) + await self.set_phase( + "streaming", + message="正在接收 AISStream 实时消息", + reset_progress=False, + ) + await db.commit() + + while True: + try: + raw_message = await asyncio.wait_for( + websocket.recv(), + timeout=config["receive_timeout_seconds"], + ) + except TimeoutError: + await update_ais_source_health( + db, + source=self.name, + connection_state="connected", + last_success_at=datetime.now(UTC), + ) + await db.commit() + continue + + payload = json.loads(raw_message) + if not isinstance(payload, dict): + continue + messages_seen += 1 + record = self._normalize_message(payload) + if not record: + continue + unique_mmsi.add(str(record["mmsi"])) + created = await self._save_stream_record(db, record) + if created: + records_added += 1 + + task.records_processed = messages_seen + task.total_records = None + task.progress = None + task.phase = "streaming" + task.phase_message = "正在接收 AISStream 实时消息" + task.phase_current = messages_seen + task.phase_total = None + task.phase_unit = "messages" + await self._publish_task_update(force=True) + + if config["streaming_max_messages"] and messages_seen >= config["streaming_max_messages"]: + task.status = "success" + task.phase = "stopped" + task.phase_message = "AISStream 测试流已停止" + task.completed_at = datetime.now(UTC) + await db.commit() + await self._publish_task_update(force=True) + return { + "status": "success", + "task_id": task.id, + "records_processed": records_added, + "messages_seen": messages_seen, + "unique_mmsi": len(unique_mmsi), + "execution_time_seconds": (datetime.now(UTC) - start_time).total_seconds(), + } + except asyncio.CancelledError: + raise + except Exception as exc: + await update_ais_source_health( + db, + source=self.name, + connection_state="reconnecting", + last_error=f"{exc.__class__.__name__}: {exc}", + ) + task.phase = "reconnecting" + task.phase_message = "AISStream 连接中断，正在重连" + task.error_message = f"{exc.__class__.__name__}: {exc}" + await db.commit() + await self._publish_task_update(force=True) + await asyncio.sleep(reconnect_delay) + except asyncio.CancelledError: + task.status = "cancelled" + task.phase = "stopped" + task.phase_message = "AISStream 实时流已停止" + task.completed_at = datetime.now(UTC) + await update_ais_source_health( + db, + source=self.name, + connection_state="disconnected", + last_error=None, + ) + await db.commit() + await self._publish_task_update(force=True) + raise + + def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]: + records = [] + for item in raw_data: + record = self._normalize_message(item) + if record: + records.append(record) + return records + + async def _save_data( + self, + db: AsyncSession, + data: list[dict[str, Any]], + task_id: int | None = None, + snapshot_id: int | None = None, + ) -> int: + now = datetime.now(UTC) + records_added = 0 + latest_observed_at = now + for index, item in enumerate(data): + observed_at = item.get("received_at") or now + observation = await record_vessel_ais_observation( + db, + source=self.name, + normalized_payload=item, + raw_payload=item.get("_raw_payload") or item, + delivery_mode=AISSTREAM_DELIVERY_MODE, + transport=AISSTREAM_TRANSPORT, + message_type=item.get("_message_type") or "PositionReport", + source_message_id=item.get("_source_message_id"), + observed_at=observed_at, + collected_at=now, + ) + if observation is not None: + records_added += 1 + if isinstance(observed_at, datetime) and observed_at > latest_observed_at: + latest_observed_at = observed_at + if (index + 1) % 1000 == 0: + await self.update_progress(index + 1, commit=True) + + await update_ais_source_health( + db, + source=self.name, + connection_state="connected", + observed_count=len(data), + last_seen_at=latest_observed_at, + last_success_at=now if data else None, + lag_seconds=max((now - latest_observed_at).total_seconds(), 0), + ) + await db.commit() + await self.update_progress(records_added, force=True) + return records_added + + async def _save_stream_record(self, db: AsyncSession, item: dict[str, Any]) -> bool: + now = datetime.now(UTC) + observed_at = item.get("received_at") or now + observation = await record_vessel_ais_observation( + db, + source=self.name, + normalized_payload=item, + raw_payload=item.get("_raw_payload") or item, + delivery_mode=AISSTREAM_DELIVERY_MODE, + transport=AISSTREAM_TRANSPORT, + message_type=item.get("_message_type") or "PositionReport", + source_message_id=item.get("_source_message_id"), + observed_at=observed_at, + collected_at=now, + ) + await update_ais_source_health( + db, + source=self.name, + connection_state="connected", + observed_count=1, + last_seen_at=observed_at if isinstance(observed_at, datetime) else now, + last_success_at=now, + lag_seconds=max((now - observed_at).total_seconds(), 0) if isinstance(observed_at, datetime) else None, + ) + await db.commit() + await self._broadcast_vessel_delta(item, created=observation is not None) + return observation is not None + + async def _broadcast_vessel_delta(self, item: dict[str, Any], *, created: bool) -> None: + await broadcaster.broadcast_custom( + "vessels", + { + "action": "upsert", + "source": self.name, + "created": created, + "vessels": [ + { + "mmsi": item.get("mmsi"), + "mmsi_display": str(item.get("mmsi")) if item.get("mmsi") is not None else None, + "name": item.get("name"), + "lat": item.get("lat"), + "lon": item.get("lon"), + "sog": item.get("sog"), + "cog": item.get("cog"), + "heading": item.get("heading"), + "nav_status": item.get("nav_status"), + "vessel_type": item.get("vessel_type"), + "vessel_type_name": item.get("vessel_type_name"), + "received_at": to_iso8601_utc(item.get("received_at")), + } + ], + }, + ) + + def _normalize_message(self, item: dict[str, Any]) -> dict[str, Any] | None: + message_type = str(item.get("MessageType") or item.get("message_type") or "") + metadata = item.get("MetaData") if isinstance(item.get("MetaData"), dict) else {} + message = item.get("Message") if isinstance(item.get("Message"), dict) else {} + body = message.get(message_type) if isinstance(message.get(message_type), dict) else message + if not isinstance(body, dict): + body = {} + + mmsi = _as_int(_pick(metadata, "MMSI", "mmsi") or _pick(body, "MMSI", "mmsi")) + if mmsi is None: + return None + + received_at = _parse_datetime( + _pick(metadata, "time_utc", "Time_UTC", "timestamp") + or _pick(body, "Timestamp", "timestamp", "time") + ) + ship_name = _clean_text( + _pick(body, "Name", "ShipName", "name") + or _pick(metadata, "ShipName", "ship_name", "name") + ) + record: dict[str, Any] = { + "mmsi": mmsi, + "received_at": received_at, + "_message_type": message_type or None, + "_source_message_id": item.get("MessageID") or item.get("message_id"), + "_raw_payload": item, + } + + lat = _as_float(_pick(body, "Latitude", "lat", "latitude")) + lon = _as_float(_pick(body, "Longitude", "lon", "lng", "longitude")) + if lat is not None and lon is not None: + if not (-90 <= lat <= 90 and -180 <= lon <= 180): + return None + record.update( + { + "lat": lat, + "lon": lon, + "sog": _as_float(_pick(body, "Sog", "SOG", "speedOverGround")), + "cog": _as_float(_pick(body, "Cog", "COG", "courseOverGround")), + "heading": _as_int(_pick(body, "TrueHeading", "Heading", "heading")), + "nav_status": _as_int(_pick(body, "NavigationalStatus", "nav_status")), + } + ) + + vessel_type = _as_int(_pick(body, "Type", "ShipType", "vessel_type")) + record.update( + { + "name": ship_name, + "callsign": _pick(body, "CallSign", "callsign"), + "imo": _as_int(_pick(body, "ImoNumber", "IMO", "imo")), + "vessel_type": vessel_type, + "vessel_type_name": _pick(body, "TypeName", "ShipTypeName", "vessel_type_name") + or normalize_vessel_type_name(vessel_type), + "length": _as_float(_pick(body, "DimensionToBow", "Length", "length")), + "width": _as_float(_pick(body, "DimensionToPort", "Width", "width")), + } + ) + return record + + +def _pick(item: dict[str, Any], *keys: str) -> Any: + for key in keys: + if key in item and item[key] not in (None, ""): + return item[key] + return None + + +def _clean_text(value: Any) -> str | None: + if value in (None, ""): + return None + text = str(value).strip() + return text or None + + +def _as_float(value: Any) -> float | None: + try: + if value in (None, ""): + return None + return float(value) + except (TypeError, ValueError): + return None + + +def _as_int(value: Any) -> int | None: + try: + if value in (None, ""): + return None + return int(float(value)) + except (TypeError, ValueError): + return None + + +def _parse_datetime(value: Any) -> datetime | None: + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=UTC) + if not value: + return None + if isinstance(value, (int, float)): + timestamp = float(value) + if timestamp > 10_000_000_000: + timestamp /= 1000 + return datetime.fromtimestamp(timestamp, UTC) + if isinstance(value, str): + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + except ValueError: + return None + return None diff --git a/backend/app/services/collectors/base.py b/backend/app/services/collectors/base.py index dcdbe36a..f1d0285d 100644 --- a/backend/app/services/collectors/base.py +++ b/backend/app/services/collectors/base.py @@ -54,6 +54,11 @@ class BaseCollector(ABC): "task_id": self._current_task.id, "status": self._current_task.status, "phase": self._current_task.phase, + "phase_progress": self._current_task.phase_progress, + "phase_message": self._current_task.phase_message, + "phase_current": self._current_task.phase_current, + "phase_total": self._current_task.phase_total, + "phase_unit": self._current_task.phase_unit, "progress": progress, "records_processed": self._current_task.records_processed, "total_records": self._current_task.total_records, @@ -80,12 +85,52 @@ class BaseCollector(ABC): await self._publish_task_update(force=force) - async def set_phase(self, phase: str): + async def set_phase(self, phase: str, *, message: str | None = None, reset_progress: bool = True): if self._current_task and self._db_session: self._current_task.phase = phase + self._current_task.phase_message = message + if reset_progress: + self._current_task.phase_progress = None + self._current_task.phase_current = None + self._current_task.phase_total = None + self._current_task.phase_unit = None await self._db_session.commit() await self._publish_task_update(force=True) + async def update_phase_progress( + self, + *, + current: int | None = None, + total: int | None = None, + unit: str | None = None, + message: str | None = None, + progress: float | None = None, + commit: bool = False, + force: bool = False, + ): + """Update progress for the current phase without changing task totals.""" + if not self._current_task or not self._db_session: + return + + if progress is None and current is not None and total and total > 0: + progress = (current / total) * 100 + + if progress is not None: + self._current_task.phase_progress = max(0.0, min(float(progress), 100.0)) + if current is not None: + self._current_task.phase_current = max(0, int(current)) + if total is not None: + self._current_task.phase_total = max(0, int(total)) + if unit is not None: + self._current_task.phase_unit = unit + if message is not None: + self._current_task.phase_message = message + + if commit: + await self._db_session.commit() + + await self._publish_task_update(force=force) + @abstractmethod async def fetch(self) -> List[Dict[str, Any]]: """Fetch raw data from source""" @@ -251,7 +296,7 @@ class BaseCollector(ABC): await self._publish_task_update(force=True) try: - await self.set_phase("fetching") + await self.set_phase("fetching", message="正在拉取原始数据") raw_data = await self.fetch() task.total_records = len(raw_data) await db.commit() @@ -260,15 +305,20 @@ class BaseCollector(ABC): if self.fail_on_empty and not raw_data: raise RuntimeError(f"Collector {self.name} returned no data") - await self.set_phase("transforming") + await self.set_phase("transforming", message="正在转换采集数据") data = self.transform(raw_data) snapshot_id = await self._create_snapshot(db, task_id, data, start_time) - await self.set_phase("saving") + await self.set_phase("saving", message="正在保存采集数据") records_count = await self._save_data(db, data, task_id=task_id, snapshot_id=snapshot_id) task.status = "success" task.phase = "completed" + task.phase_progress = 100.0 + task.phase_message = "采集完成" + task.phase_current = records_count + task.phase_total = records_count + task.phase_unit = "records" task.records_processed = records_count task.progress = 100.0 task.completed_at = datetime.now(UTC) @@ -285,6 +335,7 @@ class BaseCollector(ABC): await db.rollback() task.status = "cancelled" task.phase = "cancelled" + task.phase_message = "采集已取消" task.error_message = "Collection cancelled by operator and rolled back" task.completed_at = datetime.now(UTC) if snapshot_id is not None: @@ -301,6 +352,7 @@ class BaseCollector(ABC): await db.rollback() task.status = "failed" task.phase = "failed" + task.phase_message = str(e) task.error_message = str(e) task.completed_at = datetime.now(UTC) if snapshot_id is not None: diff --git a/backend/app/services/collectors/bgp_common.py b/backend/app/services/collectors/bgp_common.py index ec54a73c..b4cbf413 100644 --- a/backend/app/services/collectors/bgp_common.py +++ b/backend/app/services/collectors/bgp_common.py @@ -13,6 +13,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.models.bgp_anomaly import BGPAnomaly from app.models.bgp_observation import BGPObservation from app.models.collected_data import CollectedData +from app.services.bgp_collector_locations import ( + RIPE_RIS_COLLECTOR_COORDS, + get_bgp_collector_location_dict, +) +from app.services.bgp_event_locations import resolve_bgp_event_geo_dict from app.services.bgp_incidents import create_bgp_incidents_for_anomalies from app.services.bgp_detectors import ( detect_mass_withdrawal_anomalies, @@ -23,32 +28,17 @@ from app.services.bgp_detectors import ( ) from app.services.bgp_enrichment import enrich_bgp_events_for_batch, extract_bgp_network_fields - -RIPE_RIS_COLLECTOR_COORDS: dict[str, dict[str, Any]] = { - "rrc00": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041}, - "rrc01": {"city": "London", "country": "United Kingdom", "latitude": 51.5072, "longitude": -0.1276}, - "rrc03": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041}, - "rrc04": {"city": "Geneva", "country": "Switzerland", "latitude": 46.2044, "longitude": 6.1432}, - "rrc05": {"city": "Vienna", "country": "Austria", "latitude": 48.2082, "longitude": 16.3738}, - "rrc06": {"city": "Otemachi", "country": "Japan", "latitude": 35.686, "longitude": 139.7671}, - "rrc07": {"city": "Stockholm", "country": "Sweden", "latitude": 59.3293, "longitude": 18.0686}, - "rrc10": {"city": "Milan", "country": "Italy", "latitude": 45.4642, "longitude": 9.19}, - "rrc11": {"city": "New York", "country": "United States", "latitude": 40.7128, "longitude": -74.006}, - "rrc12": {"city": "Frankfurt", "country": "Germany", "latitude": 50.1109, "longitude": 8.6821}, - "rrc13": {"city": "Moscow", "country": "Russia", "latitude": 55.7558, "longitude": 37.6173}, - "rrc14": {"city": "Palo Alto", "country": "United States", "latitude": 37.4419, "longitude": -122.143}, - "rrc15": {"city": "Sao Paulo", "country": "Brazil", "latitude": -23.5558, "longitude": -46.6396}, - "rrc16": {"city": "Miami", "country": "United States", "latitude": 25.7617, "longitude": -80.1918}, - "rrc18": {"city": "Barcelona", "country": "Spain", "latitude": 41.3874, "longitude": 2.1686}, - "rrc19": {"city": "Johannesburg", "country": "South Africa", "latitude": -26.2041, "longitude": 28.0473}, - "rrc20": {"city": "Zurich", "country": "Switzerland", "latitude": 47.3769, "longitude": 8.5417}, - "rrc21": {"city": "Paris", "country": "France", "latitude": 48.8566, "longitude": 2.3522}, - "rrc22": {"city": "Bucharest", "country": "Romania", "latitude": 44.4268, "longitude": 26.1025}, - "rrc23": {"city": "Singapore", "country": "Singapore", "latitude": 1.3521, "longitude": 103.8198}, - "rrc24": {"city": "Montevideo", "country": "Uruguay", "latitude": -34.9011, "longitude": -56.1645}, - "rrc25": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041}, - "rrc26": {"city": "Dubai", "country": "United Arab Emirates", "latitude": 25.2048, "longitude": 55.2708}, -} +# Re-exported for backward compatibility with anything that imports +# ``RIPE_RIS_COLLECTOR_COORDS`` from this module. New code should call +# ``app.services.bgp_collector_locations.get_bgp_collector_location_dict()`` +# or ``resolve_bgp_collector_location()`` instead — those use the DB-backed +# collector-location cache. +__all__ = [ + "RIPE_RIS_COLLECTOR_COORDS", + "normalize_bgp_event", + "save_bgp_observations_for_batch", + "create_bgp_anomalies_for_batch", +] def _safe_int(value: Any) -> int | None: @@ -131,7 +121,19 @@ def normalize_bgp_event(payload: dict[str, Any], *, project: str) -> dict[str, A ) source_id = hashlib.sha1(source_material.encode("utf-8")).hexdigest()[:24] - collector_location = RIPE_RIS_COLLECTOR_COORDS.get(collector, {}) + # Routes through the BGP event pipeline: source coords (if any) → + # collector inheritance. Returned dict keeps the legacy + # {city, country, latitude, longitude} keys plus richer + # {precision, source, needs_confirmation, matched_location_name, confidence}. + collector_location = resolve_bgp_event_geo_dict( + collector, + source_latitude=payload.get("latitude"), + source_longitude=payload.get("longitude"), + ) + # Empty result (unknown collector & no source coords) — keep the + # downstream-expected dict shape so detectors / serializers don't crash. + if not collector_location: + collector_location = get_bgp_collector_location_dict(collector) network_fields = extract_bgp_network_fields(prefix) metadata = { "project": project, diff --git a/backend/app/services/collectors/celestrak.py b/backend/app/services/collectors/celestrak.py index 6c82d4a2..49038b42 100644 --- a/backend/app/services/collectors/celestrak.py +++ b/backend/app/services/collectors/celestrak.py @@ -46,6 +46,9 @@ class CelesTrakTLECollector(BaseCollector): if response.status_code == 200: data = response.json() if isinstance(data, list): + for item in data: + if isinstance(item, dict): + item["_celestrak_group"] = group all_satellites.extend(data) print(f"CelesTrak: Fetched {len(data)} satellites from group '{group}'") except Exception as e: @@ -78,6 +81,7 @@ class CelesTrakTLECollector(BaseCollector): "name": item.get("OBJECT_NAME", "Unknown"), "reference_date": item.get("EPOCH", ""), "metadata": { + "constellation_group": item.get("_celestrak_group"), "norad_cat_id": item.get("NORAD_CAT_ID"), "international_designator": item.get("OBJECT_ID"), "epoch": item.get("EPOCH"), diff --git a/backend/app/services/collectors/iptoasn.py b/backend/app/services/collectors/iptoasn.py index 665aad94..24f3e17d 100644 --- a/backend/app/services/collectors/iptoasn.py +++ b/backend/app/services/collectors/iptoasn.py @@ -108,6 +108,11 @@ class IPtoASNPrefixGeoCollector(BaseCollector): self._current_task.total_records = total_expected self._current_task.records_processed = 0 self._current_task.progress = 0.0 + self._current_task.phase_progress = 0.0 + self._current_task.phase_message = "正在下载 IPtoASN 数据" + self._current_task.phase_current = 0 + self._current_task.phase_total = total_expected + self._current_task.phase_unit = "bytes" await self._db_session.commit() await self._publish_task_update(force=True) @@ -135,7 +140,14 @@ class IPtoASNPrefixGeoCollector(BaseCollector): return last_emit["value"] = aggregated last_emit["t"] = now - await self.update_progress(min(aggregated, total_expected), commit=True) + current = min(aggregated, total_expected) + await self.update_phase_progress( + current=current, + total=total_expected, + unit="bytes", + message="正在下载 IPtoASN 数据", + ) + await self.update_progress(current, commit=True) batches = await asyncio.gather( *( @@ -148,6 +160,12 @@ class IPtoASNPrefixGeoCollector(BaseCollector): ) ) if total_expected > 0: + await self.update_phase_progress( + current=total_expected, + total=total_expected, + unit="bytes", + message="IPtoASN 数据下载完成", + ) await self.update_progress(total_expected, commit=True, force=True) rows: list[dict[str, Any]] = [] diff --git a/backend/app/services/collectors/nro_delegated.py b/backend/app/services/collectors/nro_delegated.py index 52967ad1..207880ee 100644 --- a/backend/app/services/collectors/nro_delegated.py +++ b/backend/app/services/collectors/nro_delegated.py @@ -39,12 +39,23 @@ class NRODelegatedPrefixGeoCollector(BaseCollector): self._current_task.total_records = total_expected self._current_task.records_processed = 0 self._current_task.progress = 0.0 + self._current_task.phase_progress = 0.0 + self._current_task.phase_message = "正在下载 NRO delegated 数据" + self._current_task.phase_current = 0 + self._current_task.phase_total = total_expected + self._current_task.phase_unit = "bytes" await self._db_session.commit() await self._publish_task_update(force=True) async def on_progress(downloaded: int, total: int | None) -> None: if not total or total <= 0: return + await self.update_phase_progress( + current=min(downloaded, total), + total=total, + unit="bytes", + message="正在下载 NRO delegated 数据", + ) await self.update_progress(min(downloaded, total), commit=True) body_path = await self._downloader.download_file( diff --git a/backend/app/services/collectors/opengeofeed.py b/backend/app/services/collectors/opengeofeed.py index bb91f4fb..80e1c3b4 100644 --- a/backend/app/services/collectors/opengeofeed.py +++ b/backend/app/services/collectors/opengeofeed.py @@ -40,12 +40,23 @@ class OpenGeoFeedPrefixGeoCollector(BaseCollector): self._current_task.total_records = total_expected self._current_task.records_processed = 0 self._current_task.progress = 0.0 + self._current_task.phase_progress = 0.0 + self._current_task.phase_message = "正在下载 OpenGeoFeed 数据" + self._current_task.phase_current = 0 + self._current_task.phase_total = total_expected + self._current_task.phase_unit = "bytes" await self._db_session.commit() await self._publish_task_update(force=True) async def on_progress(downloaded: int, total: int | None) -> None: if not total or total <= 0: return + await self.update_phase_progress( + current=min(downloaded, total), + total=total, + unit="bytes", + message="正在下载 OpenGeoFeed 数据", + ) await self.update_progress(min(downloaded, total), commit=True) body_path = await self._downloader.download_file( diff --git a/backend/app/services/collectors/vessel_ais.py b/backend/app/services/collectors/vessel_ais.py new file mode 100644 index 00000000..810b91c2 --- /dev/null +++ b/backend/app/services/collectors/vessel_ais.py @@ -0,0 +1,279 @@ +"""BarentsWatch AIS collector for vessel tracking.""" + +from datetime import UTC, datetime +from typing import Any + +import httpx +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.time import to_iso8601_utc +from app.core.websocket.broadcaster import broadcaster +from app.services.barentswatch import ( + BARENTSWATCH_LATEST_URL, + fetch_barentswatch_access_token, + resolve_barentswatch_config, +) +from app.services.collectors.base import BaseCollector +from app.services.vessel_ais_aggregation import ( + BARENTSWATCH_DELIVERY_MODE, + BARENTSWATCH_TRANSPORT, + record_vessel_ais_observation, + update_ais_source_health, +) +from app.services.vessel_types import normalize_vessel_type_name + + +class VesselAISCollector(BaseCollector): + """Collect latest AIS positions and append them to vessel tables.""" + + name = "barentswatch_vessels" + priority = "P1" + module = "L4" + frequency_hours = 1 + data_type = "vessel_ais" + + @property + def base_url(self) -> str: + return self._resolved_url or BARENTSWATCH_LATEST_URL + + async def _get_access_token(self, client: httpx.AsyncClient) -> str | None: + config = await resolve_barentswatch_config(self._db_session) + return await fetch_barentswatch_access_token(client, config) + + async def fetch(self) -> list[dict[str, Any]]: + async with httpx.AsyncClient(timeout=60.0) as client: + headers: dict[str, str] = {} + token = await self._get_access_token(client) + if token: + headers["Authorization"] = f"Bearer {token}" + + response = await client.get(self.base_url, headers=headers) + if response.status_code == 401 and not token: + return self._get_sample_data() + response.raise_for_status() + payload = response.json() + + if isinstance(payload, list): + return [item for item in payload if isinstance(item, dict)] + if isinstance(payload, dict): + for key in ("features", "data", "items", "vessels"): + value = payload.get(key) + if isinstance(value, list): + if key == "features": + return [ + { + **(item.get("properties") or {}), + "geometry": item.get("geometry"), + } + for item in value + if isinstance(item, dict) + ] + return [item for item in value if isinstance(item, dict)] + return self._get_sample_data() + + def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]: + transformed = [] + for item in raw_data: + record = self._normalize_record(item) + if record: + transformed.append(record) + return transformed + + async def _save_data( + self, + db: AsyncSession, + data: list[dict[str, Any]], + task_id: int | None = None, + snapshot_id: int | None = None, + ) -> int: + now = datetime.now(UTC) + records_added = 0 + + for index, item in enumerate(data): + observed_at = item.get("received_at") or now + await record_vessel_ais_observation( + db, + source=self.name, + normalized_payload=item, + raw_payload=item, + delivery_mode=BARENTSWATCH_DELIVERY_MODE, + transport=BARENTSWATCH_TRANSPORT, + observed_at=observed_at, + collected_at=now, + ) + records_added += 1 + + if (index + 1) % 1000 == 0: + await self.update_progress(index + 1, commit=True) + + latest_observed_at = max( + (item.get("received_at") for item in data if item.get("received_at")), + default=now, + ) + await update_ais_source_health( + db, + source=self.name, + connection_state="connected", + observed_count=len(data), + last_seen_at=latest_observed_at, + last_success_at=now if data else None, + lag_seconds=max((now - latest_observed_at).total_seconds(), 0), + ) + await db.commit() + await self._broadcast_vessel_snapshot(data) + await self.update_progress(records_added, force=True) + return records_added + + async def _broadcast_vessel_snapshot(self, data: list[dict[str, Any]]) -> None: + """Push REST collector updates through the same realtime vessel channel.""" + if not data: + return + + batch_size = 500 + for offset in range(0, len(data), batch_size): + batch = data[offset : offset + batch_size] + await broadcaster.broadcast_custom( + "vessels", + { + "action": "upsert", + "source": self.name, + "created": True, + "vessels": [ + { + "mmsi": item.get("mmsi"), + "mmsi_display": str(item.get("mmsi")) if item.get("mmsi") is not None else None, + "name": item.get("name"), + "callsign": item.get("callsign"), + "lat": item.get("lat"), + "lon": item.get("lon"), + "sog": item.get("sog"), + "cog": item.get("cog"), + "heading": item.get("heading"), + "nav_status": item.get("nav_status"), + "vessel_type": item.get("vessel_type"), + "vessel_type_name": item.get("vessel_type_name"), + "received_at": to_iso8601_utc(item.get("received_at")), + } + for item in batch + ], + }, + ) + + def _normalize_record(self, item: dict[str, Any]) -> dict[str, Any] | None: + mmsi = _as_int(_pick(item, "mmsi", "MMSI", "Mmsi")) + lat = _as_float(_pick(item, "lat", "latitude", "Latitude")) + lon = _as_float(_pick(item, "lon", "lng", "longitude", "Longitude")) + + geometry = item.get("geometry") + coordinates = geometry.get("coordinates") if isinstance(geometry, dict) else None + if (lat is None or lon is None) and isinstance(coordinates, list) and len(coordinates) >= 2: + lon = _as_float(coordinates[0]) + lat = _as_float(coordinates[1]) + + if mmsi is None or lat is None or lon is None: + return None + if not (-90 <= lat <= 90 and -180 <= lon <= 180): + return None + + vessel_type = _as_int(_pick(item, "vessel_type", "shipType", "ship_type", "ShipType")) + vessel_type_name = ( + _pick(item, "vessel_type_name", "shipTypeName", "ship_type_name", "VesselTypeName") + or normalize_vessel_type_name(vessel_type) + ) + received_at = _parse_datetime(_pick(item, "received_at", "timestamp", "time", "msgtime")) + + return { + "mmsi": mmsi, + "name": _pick(item, "name", "shipName", "ship_name", "Name"), + "callsign": _pick(item, "callsign", "callSign", "CallSign"), + "vessel_type": vessel_type, + "vessel_type_name": vessel_type_name, + "flag": _pick(item, "flag", "country", "Flag"), + "length": _as_float(_pick(item, "length", "shipLength", "Length")), + "width": _as_float(_pick(item, "width", "shipWidth", "Width")), + "draught": _as_float(_pick(item, "draught", "draft", "Draught")), + "imo": _as_int(_pick(item, "imo", "IMO", "imoNumber")), + "lat": lat, + "lon": lon, + "sog": _as_float(_pick(item, "sog", "speedOverGround", "SOG")), + "cog": _as_float(_pick(item, "cog", "courseOverGround", "COG")), + "heading": _as_int(_pick(item, "heading", "trueHeading", "Heading")), + "nav_status": _as_int(_pick(item, "nav_status", "navStatus", "NavigationalStatus")), + "received_at": received_at, + } + + def _get_sample_data(self) -> list[dict[str, Any]]: + return [ + { + "mmsi": 257123000, + "name": "OSLO TRADER", + "lat": 59.91, + "lon": 10.73, + "sog": 12.4, + "cog": 214, + "heading": 215, + "nav_status": 0, + "vessel_type": 70, + "vessel_type_name": "Cargo", + "flag": "NO", + "length": 185, + }, + { + "mmsi": 257456000, + "name": "NORDIC FJORD", + "lat": 60.39, + "lon": 5.32, + "sog": 0.2, + "cog": 82, + "heading": 80, + "nav_status": 1, + "vessel_type": 60, + "vessel_type_name": "Passenger", + "flag": "NO", + "length": 126, + }, + ] + + +def _pick(item: dict[str, Any], *keys: str) -> Any: + for key in keys: + if key in item and item[key] not in (None, ""): + return item[key] + return None + + +def _as_float(value: Any) -> float | None: + try: + if value in (None, ""): + return None + return float(value) + except (TypeError, ValueError): + return None + + +def _as_int(value: Any) -> int | None: + try: + if value in (None, ""): + return None + return int(float(value)) + except (TypeError, ValueError): + return None + + +def _parse_datetime(value: Any) -> datetime | None: + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=UTC) + if not value: + return None + if isinstance(value, (int, float)): + timestamp = float(value) + if timestamp > 10_000_000_000: + timestamp /= 1000 + return datetime.fromtimestamp(timestamp, UTC) + if isinstance(value, str): + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + except ValueError: + return None + return None diff --git a/backend/app/services/compute_center_locations.py b/backend/app/services/compute_center_locations.py new file mode 100644 index 00000000..5bf47dbf --- /dev/null +++ b/backend/app/services/compute_center_locations.py @@ -0,0 +1,863 @@ +"""Compute-center location resolver, built on the shared location pipeline. + +This module is a thin domain wrapper that wires up +:mod:`app.services.location` for compute centers: + + SourceCoordinates + +The online Nominatim step is intentionally reserved for the user-triggered +``collect-location`` flow. The regular GeoJSON endpoint runs during Earth +startup, so it must stay local and deterministic. + +For the full design and the reason behind the abstraction (compute centers, +BGP collectors, BGP events, and future entities all share one pipeline), +see ``docs/plans/location-resolver-shared-pipeline-plan.md``. + +The ``ComputeCenterLocation`` dataclass and the public function signatures are +preserved verbatim so existing callers and tests do not need to change. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from functools import lru_cache +from typing import Any + +import httpx +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.collected_data_fields import get_record_field +from app.models.collected_data import CollectedData +from app.models.compute_center_location import ComputeCenterLocationRecord + +from app.services.location import ( + LocationCandidate, + LocationPipeline, + LocationQuery, + NominatimResolver, + ResolverOutput, + SourceCoordinatesResolver, + build_default_nominatim_geocoder, + coerce_str, + normalize_country_text, + normalize_text, + parse_float, +) + +ROR_SEARCH_URL = "https://api.ror.org/v2/organizations" +DEFAULT_ROR_USER_AGENT = "planet-earth-location-resolver/1.0" +DEFAULT_ROR_TIMEOUT_SECONDS = 8.0 +RENDERABLE_PRECISIONS: tuple[str, ...] = ("precise", "site", "city") +FORBIDDEN_PRECISIONS: tuple[str, ...] = ( + "country", + "estimated_country", + "country_major_compute_city", + "region", + "unknown", +) + +# ── Public dataclasses ────────────────────────────────────────────── + + +@dataclass(frozen=True) +class ComputeCenterLocation: + latitude: float | None + longitude: float | None + location_precision: str + geography_mode: str + is_estimated: bool + estimated_reason: str | None = None + location_confidence: float | None = None + location_source: str | None = None + location_source_note: str | None = None + location_verified_at: str | None = None + matched_location_name: str | None = None + needs_confirmation: bool = False + city: str | None = None + region: str | None = None + country: str | None = None + + @property + def is_renderable(self) -> bool: + if self.latitude in (None, 0.0) or self.longitude in (None, 0.0): + return False + return self.location_precision in RENDERABLE_PRECISIONS + + def to_geojson_properties(self) -> dict[str, Any]: + return { + "latitude": self.latitude, + "longitude": self.longitude, + "location_precision": self.location_precision, + "geography_mode": self.geography_mode, + "is_estimated": self.is_estimated, + "estimated_reason": self.estimated_reason, + "location_confidence": self.location_confidence, + "location_source": self.location_source, + "location_source_note": self.location_source_note, + "location_verified_at": self.location_verified_at, + "matched_location_name": self.matched_location_name, + "needs_confirmation": self.needs_confirmation, + } + + +@dataclass(frozen=True) +class ResolutionDiagnostic: + failure_reason: str + attempted_queries: tuple[str, ...] = () + record_id: int | None = None + source: str | None = None + source_id: str | None = None + name: str | None = None + country: str | None = None + city: str | None = None + site: str | None = None + operator: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "failure_reason": self.failure_reason, + "attempted_queries": list(self.attempted_queries), + "record_id": self.record_id, + "source": self.source, + "source_id": self.source_id, + "name": self.name, + "country": self.country, + "city": self.city, + "site": self.site, + "operator": self.operator, + } + + +@dataclass(frozen=True) +class ResolutionResult: + location: ComputeCenterLocation | None + diagnostic: ResolutionDiagnostic | None + + @property + def is_resolved(self) -> bool: + return bool(self.location and self.location.is_renderable) + + +# ── Geocoder (kept at module level so tests can monkeypatch + cache_clear) ── + +_geocode_online = build_default_nominatim_geocoder() + + +# ── Stored location cache ─────────────────────────────────────────── + + +COMPUTE_CENTER_LOCATION_CACHE: dict[str, dict[str, Any]] = {} + + +def _cache_key(source: str | None, source_id: str | None) -> str: + return f"{coerce_str(source)}:{coerce_str(source_id)}" + + +def set_compute_center_location_cache( + locations: dict[str, dict[str, Any]], +) -> None: + COMPUTE_CENTER_LOCATION_CACHE.clear() + COMPUTE_CENTER_LOCATION_CACHE.update( + {coerce_str(key): dict(value) for key, value in locations.items()} + ) + + +async def refresh_compute_center_location_cache( + session: AsyncSession, +) -> dict[str, dict[str, Any]]: + result = await session.execute(select(ComputeCenterLocationRecord)) + records = result.scalars().all() + cache = {} + for record in records: + if not hasattr(record, "to_location_dict"): + continue + if not record.source or not record.source_id: + continue + cache[_cache_key(record.source, record.source_id)] = record.to_location_dict() + set_compute_center_location_cache(cache) + return cache + + +def get_compute_center_location_dict( + source: str | None, + source_id: str | None, +) -> dict[str, Any]: + return dict(COMPUTE_CENTER_LOCATION_CACHE.get(_cache_key(source, source_id), {})) + + +# ── Pipeline construction ────────────────────────────────────────── + + +@lru_cache(maxsize=512) +def _lookup_ror_organization(query: str) -> dict[str, Any] | None: + """Lookup a research organization in ROR for user-triggered candidates.""" + if not query: + return None + response = httpx.get( + ROR_SEARCH_URL, + params={"query": query}, + headers={"User-Agent": DEFAULT_ROR_USER_AGENT}, + timeout=DEFAULT_ROR_TIMEOUT_SECONDS, + ) + response.raise_for_status() + payload = response.json() + items = payload.get("items") if isinstance(payload, dict) else None + if not isinstance(items, list) or not items: + return None + first = items[0] + if not isinstance(first, dict): + return None + organization = first.get("organization") + if isinstance(organization, dict): + return organization + return first + + +def _compute_center_ror_query_plan( + query: LocationQuery, +) -> list[tuple[str, tuple[str, ...]]]: + extra = query.extra or {} + raw_parts: list[tuple[str, str]] = [ + ("site", coerce_str(extra.get("site"))), + ("operator", coerce_str(extra.get("operator"))), + ("organization", coerce_str(extra.get("organization"))), + ] + for field, value in tuple(raw_parts): + if "/" not in value: + continue + raw_parts.extend( + (field, part.strip()) + for part in value.split("/") + if len(part.strip()) >= 3 + ) + + plan: list[tuple[str, tuple[str, ...]]] = [] + seen: set[str] = set() + for field, value in raw_parts: + key = normalize_text(value) + if not key or key in seen: + continue + seen.add(key) + plan.append((value, (field,))) + return plan + + +def _organization_label(organization: dict[str, Any], fallback: str) -> str: + names = organization.get("names") + if isinstance(names, list): + for name in names: + if not isinstance(name, dict): + continue + types = name.get("types") + if isinstance(types, list) and "ror_display" in types: + value = coerce_str(name.get("value")) + if value: + return value + for name in names: + if isinstance(name, dict): + value = coerce_str(name.get("value")) + if value: + return value + return fallback + + +class ROROrganizationResolver: + """Resolve source-provided organization/site text through the open ROR API.""" + + name = "ror_organization_registry" + + def __init__( + self, + *, + query_plan_builder=_compute_center_ror_query_plan, + lookup=lambda q: _lookup_ror_organization(q), + confidence: float = 0.68, + ) -> None: + self._query_plan_builder = query_plan_builder + self._lookup = lookup + self._confidence = confidence + + def resolve(self, query: LocationQuery): + from app.services.location import ResolverOutput + from app.services.location.text import parse_float + + attempted: list[str] = [] + candidates: list[LocationCandidate] = [] + context_country = normalize_text(normalize_country_text(query.country)) + + for ror_query, matched_fields in self._query_plan_builder(query): + attempted.append(f"ror:{ror_query}") + try: + organization = self._lookup(ror_query) + except Exception: + continue + if not isinstance(organization, dict): + continue + locations = organization.get("locations") + if not isinstance(locations, list) or not locations: + continue + location = locations[0] + if not isinstance(location, dict): + continue + details = location.get("geonames_details") + if not isinstance(details, dict): + continue + latitude = parse_float(details.get("lat")) + longitude = parse_float(details.get("lng")) + if latitude in (None, 0.0) or longitude in (None, 0.0): + continue + + country = normalize_country_text(details.get("country_name")) + if context_country and normalize_text(country) != context_country: + continue + + city = coerce_str(details.get("name")) or None + region = coerce_str(details.get("country_subdivision_name")) or None + display_name = _organization_label(organization, ror_query) + ror_id = coerce_str(organization.get("id")) + geonames_id = location.get("geonames_id") + source_note = ( + f"ROR organization match: {display_name}" + + (f" ({ror_id})" if ror_id else "") + + (f"; GeoNames {geonames_id}" if geonames_id else "") + ) + candidates.append( + LocationCandidate( + latitude=latitude, + longitude=longitude, + display_name=display_name, + precision="city", + confidence=self._confidence, + query=ror_query, + source=self.name, + source_note=source_note, + matched_fields=matched_fields, + needs_confirmation=True, + city=city, + region=region, + country=country or query.country, + matched_location_name=display_name, + location_verified_at=None, + suggested_registry_entry=None, + ) + ) + + return ResolverOutput( + candidates=tuple(candidates), + attempted_queries=tuple(attempted), + ) + + +class StoredComputeCenterLocationResolver: + """Resolve a compute center through the DB-backed current-location cache.""" + + name = "stored_compute_center_location" + + def resolve(self, query: LocationQuery) -> ResolverOutput: + extra = query.extra or {} + stored = get_compute_center_location_dict( + coerce_str(extra.get("source")), + coerce_str(extra.get("source_id")), + ) + if not stored: + return ResolverOutput() + latitude = parse_float(stored.get("latitude")) + longitude = parse_float(stored.get("longitude")) + if latitude in (None, 0.0) or longitude in (None, 0.0): + return ResolverOutput() + return ResolverOutput( + candidates=( + LocationCandidate( + latitude=latitude, + longitude=longitude, + display_name=stored.get("name") or query.name or "Compute center", + precision=stored.get("precision") or "city", + confidence=float(stored.get("confidence") or 0.85), + query=f"stored_compute_center_location::{stored.get('source')}:{stored.get('source_id')}", + source=self.name, + source_note=stored.get("source_note"), + matched_fields=("source", "source_id"), + needs_confirmation=bool(stored.get("needs_confirmation")), + city=stored.get("city") or query.city, + region=None, + country=stored.get("country") or query.country, + matched_location_name=stored.get("site") or stored.get("name") or query.name, + location_verified_at=stored.get("verified_at"), + suggested_registry_entry=None, + ), + ) + ) + + +def _short_system_name(name: Any) -> str: + """Strip vendor/system suffix from TOP500 names like ``"El Capitan - HPE Cray ..."``.""" + text = coerce_str(name) + if not text: + return "" + head = text.split(" - ", 1)[0].strip() + return head or text + + +def _record_context(record: Any, metadata: dict[str, Any]) -> dict[str, str]: + name = coerce_str(getattr(record, "name", None)) + return { + "source": coerce_str(getattr(record, "source", None)), + "source_id": coerce_str(getattr(record, "source_id", None)), + "name": name, + "name_short": _short_system_name(name), + "city": coerce_str(get_record_field(record, "city")), + "country": coerce_str(get_record_field(record, "country")), + "site": coerce_str(metadata.get("site") or metadata.get("organization")), + "operator": coerce_str( + metadata.get("operator") + or metadata.get("organization") + or metadata.get("owner") + or metadata.get("manufacturer") + ), + "organization": coerce_str(metadata.get("organization")), + } + + +def _context_to_query( + context: dict[str, str], + *, + source_lat: float | None = None, + source_lon: float | None = None, +) -> LocationQuery: + name = context.get("name") or None + name_short = context.get("name_short") or "" + aliases: tuple[str, ...] = () + if name_short and name_short != name: + aliases = (name_short,) + return LocationQuery( + name=name, + aliases=aliases, + city=context.get("city") or None, + country=context.get("country") or None, + source_latitude=source_lat, + source_longitude=source_lon, + extra={ + "source": context.get("source") or "", + "source_id": context.get("source_id") or "", + "site": context.get("site") or "", + "operator": context.get("operator") or "", + "organization": context.get("organization") or "", + }, + ) + + +def _compute_center_query_plan( + query: LocationQuery, +) -> list[tuple[str, tuple[str, ...]]]: + """Build the Nominatim query plan for a compute-center query. + + Mirrors the legacy ``_build_online_query_plan`` ordering exactly. + """ + name = query.name or "" + name_short = (query.aliases[0] if query.aliases else "") or name + extra = query.extra or {} + site = str(extra.get("site") or "") + operator = str(extra.get("operator") or "") + city = query.city or "" + country = query.country or "" + + plan: list[tuple[str, tuple[str, ...]]] = [] + + def add(parts: list[tuple[str, str]]) -> None: + non_empty = [(field, value) for field, value in parts if value] + if not non_empty: + return + seen: set[str] = set() + cleaned: list[str] = [] + fields: list[str] = [] + for field, value in non_empty: + key = normalize_text(value) + if not key or key in seen: + continue + seen.add(key) + cleaned.append(value) + fields.append(field) + if not cleaned: + return + composed = ", ".join(cleaned) + if not any(composed == existing for existing, _ in plan): + plan.append((composed, tuple(fields))) + + add([("site", site), ("country", country)]) + add([("operator", operator), ("city", city), ("country", country)]) + add([("name", name_short), ("operator", operator), ("country", country)]) + add([("name", name_short), ("site", site)]) + add([("name", name_short), ("country", country)]) + add([("name", name_short), ("city", city), ("country", country)]) + add([("city", city), ("country", country)]) + if name and name != name_short: + add([("name", name), ("country", country)]) + return plan + + +COMPUTE_CENTER_PIPELINE = LocationPipeline( + [ + SourceCoordinatesResolver(), + StoredComputeCenterLocationResolver(), + ], + failure_reason=( + "Could not resolve to city-level coordinates from source coords" + " or stored compute-center location." + ), +) + +COMPUTE_CENTER_COLLECTION_PIPELINE = LocationPipeline( + [ + SourceCoordinatesResolver(), + ROROrganizationResolver(), + NominatimResolver( + query_plan_builder=_compute_center_query_plan, + # Late-binding so test monkeypatching of ``_geocode_online`` works. + geocoder=lambda q: _geocode_online(q), + ), + ], + failure_reason=( + "Could not resolve to city-level coordinates from source coords" + ", ROR organization lookup, or online geocoding." + ), +) + + +# ── Candidate → ComputeCenterLocation conversion ─────────────────── + + +_GEOGRAPHY_MODE_BY_SOURCE = { + "source_coordinates": "source_coordinates", + "stored_compute_center_location": "stored_compute_center_location", + "ror_organization_registry": "ror_organization", + "nominatim_online_geocode": "online_geocode", +} + + +def _candidate_to_location( + candidate: LocationCandidate, + *, + context: dict[str, str], +) -> ComputeCenterLocation: + geography_mode = _GEOGRAPHY_MODE_BY_SOURCE.get(candidate.source, "online_geocode") + is_estimated = candidate.needs_confirmation or candidate.source.startswith( + "nominatim" + ) + estimated_reason: str | None + if candidate.source == "source_coordinates": + estimated_reason = None + elif candidate.source == "stored_compute_center_location": + estimated_reason = candidate.source_note + elif candidate.source == "ror_organization_registry": + fields_summary = ", ".join(candidate.matched_fields) or "organization" + estimated_reason = ( + f"Resolved by ROR organization lookup '{candidate.query}' " + f"(matched fields: {fields_summary})" + ) + elif candidate.source == "nominatim_online_geocode": + fields_summary = ", ".join(candidate.matched_fields) or "name" + estimated_reason = ( + f"Resolved by online geocoding query '{candidate.query}' " + f"(matched fields: {fields_summary})" + ) + else: + estimated_reason = candidate.source_note + + country = ( + candidate.country + or normalize_country_text(context.get("country")) + or context.get("country") + or None + ) + return ComputeCenterLocation( + latitude=candidate.latitude, + longitude=candidate.longitude, + location_precision=candidate.precision, + geography_mode=geography_mode, + is_estimated=is_estimated, + estimated_reason=estimated_reason, + location_confidence=candidate.confidence, + location_source=candidate.source, + location_source_note=candidate.source_note, + location_verified_at=candidate.location_verified_at, + matched_location_name=candidate.matched_location_name + or context.get("name") + or None, + needs_confirmation=candidate.needs_confirmation, + city=candidate.city or context.get("city") or None, + region=candidate.region, + country=country, + ) + + +def _diagnostic_for( + record: Any, + context: dict[str, str], + *, + failure_reason: str, + attempted_queries: tuple[str, ...] = (), +) -> ResolutionDiagnostic: + return ResolutionDiagnostic( + failure_reason=failure_reason, + attempted_queries=attempted_queries, + record_id=getattr(record, "id", None), + source=getattr(record, "source", None), + source_id=getattr(record, "source_id", None), + name=context.get("name") or getattr(record, "name", None), + country=context.get("country") or None, + city=context.get("city") or None, + site=context.get("site") or None, + operator=context.get("operator") or None, + ) + + +# ── Public API ───────────────────────────────────────────────────── + + +def resolve_compute_center_location( + record: Any, + metadata: dict[str, Any] | None = None, +) -> ComputeCenterLocation: + """Backwards-compatible thin wrapper returning the renderable location only. + + Records that cannot be resolved to city-level get a placeholder + :class:`ComputeCenterLocation` with ``location_precision='unknown'``. + Callers should generally prefer :func:`resolve_compute_center_location_full`. + """ + full = resolve_compute_center_location_full(record, metadata) + return full.location or ComputeCenterLocation( + latitude=None, + longitude=None, + location_precision="unknown", + geography_mode="unresolved", + is_estimated=True, + estimated_reason="No resolvable location hints", + location_confidence=0.0, + location_source="unknown", + location_source_note=( + "No source coordinates, ROR organization match, or online" + " geocoding result." + ), + matched_location_name=None, + needs_confirmation=False, + ) + + +def resolve_compute_center_location_full( + record: Any, + metadata: dict[str, Any] | None = None, + *, + allow_online: bool = False, +) -> ResolutionResult: + metadata = metadata or {} + context = _record_context(record, metadata) + + from app.services.location.text import parse_float as _parse_float + + source_lat = _parse_float(get_record_field(record, "latitude")) + source_lon = _parse_float(get_record_field(record, "longitude")) + if source_lat in (None, 0.0): + source_lat = None + if source_lon in (None, 0.0): + source_lon = None + + query = _context_to_query( + context, source_lat=source_lat, source_lon=source_lon + ) + pipeline = ( + COMPUTE_CENTER_COLLECTION_PIPELINE + if allow_online + else COMPUTE_CENTER_PIPELINE + ) + pipeline_result = pipeline.resolve_best(query) + + if pipeline_result.location and pipeline_result.location.precision in RENDERABLE_PRECISIONS: + location = _candidate_to_location(pipeline_result.location, context=context) + return ResolutionResult(location=location, diagnostic=None) + + return ResolutionResult( + location=None, + diagnostic=_diagnostic_for( + record, + context, + failure_reason=( + "Could not resolve to city-level coordinates from source coords" + ", ROR organization lookup, or online geocoding." + if allow_online + else ( + "Could not resolve to city-level coordinates from source coords" + " or stored compute-center location." + ) + ), + attempted_queries=pipeline_result.attempted_queries, + ), + ) + + +def collect_location_candidates( + *, + name: str | None = None, + source: str | None = None, + source_id: str | None = None, + operator: str | None = None, + site: str | None = None, + city: str | None = None, + country: str | None = None, + organization: str | None = None, + record_id: int | None = None, +) -> tuple[list[LocationCandidate], list[str]]: + """Run the full resolution chain and return ranked candidates with attempted queries. + + The unused ``source`` / ``source_id`` / ``record_id`` arguments are kept + for backward compatibility with the API handler that calls this function. + """ + name_value = coerce_str(name) + context: dict[str, str] = { + "source": coerce_str(source), + "source_id": coerce_str(source_id), + "name": name_value, + "name_short": _short_system_name(name_value), + "city": coerce_str(city), + "country": coerce_str(country), + "site": coerce_str(site or organization), + "operator": coerce_str(operator or organization), + "organization": coerce_str(organization), + } + query = _context_to_query(context) + return COMPUTE_CENTER_COLLECTION_PIPELINE.collect_candidates(query) + + +def _record_operator(metadata: dict[str, Any]) -> str | None: + return coerce_str( + metadata.get("operator") + or metadata.get("organization") + or metadata.get("owner") + or metadata.get("manufacturer") + ) or None + + +async def seed_compute_center_locations_from_source_coords( + session: AsyncSession, +) -> None: + """Seed stored compute-center locations only from real source coordinates.""" + stmt = ( + select(CollectedData) + .where(CollectedData.source.in_(["top500", "epoch_ai_gpu"])) + .where(CollectedData.is_current.is_(True)) + ) + result = await session.execute(stmt) + records = result.scalars().all() + changed = False + + for record in records: + source_value = coerce_str(getattr(record, "source", None)) + source_id = coerce_str(getattr(record, "source_id", None)) + if not source_value or not source_id: + continue + latitude = parse_float(get_record_field(record, "latitude")) + longitude = parse_float(get_record_field(record, "longitude")) + if latitude in (None, 0.0) or longitude in (None, 0.0): + continue + existing = await session.scalar( + select(ComputeCenterLocationRecord) + .where(ComputeCenterLocationRecord.source == source_value) + .where(ComputeCenterLocationRecord.source_id == source_id) + ) + if existing: + continue + metadata = record.extra_data or {} + session.add( + ComputeCenterLocationRecord( + source=source_value, + source_id=source_id, + name=getattr(record, "name", None), + operator=_record_operator(metadata), + site=coerce_str(metadata.get("site") or metadata.get("organization")) or None, + city=coerce_str(get_record_field(record, "city")) or None, + country=coerce_str(get_record_field(record, "country")) or None, + latitude=latitude, + longitude=longitude, + precision="precise", + confidence=1.0, + location_source="source_coordinates", + source_note="Seeded from source-provided compute-center coordinates", + raw_payload={ + "record_id": getattr(record, "id", None), + "source": source_value, + "source_id": source_id, + }, + needs_confirmation=False, + verification_status="source_provided", + verified_at=None, + ) + ) + changed = True + + if changed: + await session.commit() + await refresh_compute_center_location_cache(session) + + +async def upsert_compute_center_location( + session: AsyncSession, + *, + source: str, + source_id: str, + name: str | None = None, + operator: str | None = None, + site: str | None = None, + city: str | None = None, + country: str | None = None, + latitude: float, + longitude: float, + precision: str = "city", + confidence: float | None = None, + location_source: str = "manual_selection", + source_url: str | None = None, + source_note: str | None = None, + raw_payload: dict[str, Any] | None = None, + needs_confirmation: bool = False, + verification_status: str = "verified", +) -> ComputeCenterLocationRecord: + existing = await session.scalar( + select(ComputeCenterLocationRecord) + .where(ComputeCenterLocationRecord.source == source) + .where(ComputeCenterLocationRecord.source_id == source_id) + ) + verified_at = None if needs_confirmation else datetime.now(UTC) + values = { + "name": name, + "operator": operator, + "site": site, + "city": city, + "country": country, + "latitude": latitude, + "longitude": longitude, + "precision": precision, + "confidence": confidence, + "location_source": location_source, + "source_url": source_url, + "source_note": source_note, + "raw_payload": raw_payload or {}, + "needs_confirmation": needs_confirmation, + "verification_status": verification_status, + "verified_at": verified_at, + } + if existing: + for key, value in values.items(): + setattr(existing, key, value) + record = existing + else: + record = ComputeCenterLocationRecord( + source=source, + source_id=source_id, + **values, + ) + session.add(record) + + await session.commit() + await session.refresh(record) + await refresh_compute_center_location_cache(session) + return record diff --git a/backend/app/services/credential_guides.py b/backend/app/services/credential_guides.py new file mode 100644 index 00000000..49ba6ce1 --- /dev/null +++ b/backend/app/services/credential_guides.py @@ -0,0 +1,224 @@ +"""Credential setup guides for collector integrations.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import select + +from app.models.system_setting import SystemSetting +from app.schemas.ai import SituationalAnalysisRequest +from app.services.ai_client import AIProviderClient + + +CREDENTIAL_GUIDES_CATEGORY = "collector_credential_guides" + + +@dataclass(frozen=True) +class CredentialGuideDefault: + provider: str + title: str + prompt: str + markdown: str + + +BARENTSWATCH_DEFAULT_GUIDE = CredentialGuideDefault( + provider="barentswatch", + title="BarentsWatch AIS 凭证获取教程", + prompt=( + "请生成一份中文教程，指导开发者获取 BarentsWatch Live AIS API 的 " + "OAuth client credentials。教程要面向已经有本地开发环境的人，包含注册/登录、" + "创建 client、申请或确认 ais scope、复制 client id 和 client secret、" + "在系统设置中填写并验证连接、常见失败排查。不要编造具体页面按钮文案，" + "必须参考官方 tutorial：https://developer.barentswatch.no/docs/tutorial 。" + "必须强调 Live AIS 要选择 AIS-client / AIS - API，而不是普通 API-client。" + "如果步骤可能变化，要提醒以 BarentsWatch developer portal 当前页面为准。" + ), + markdown="""## BarentsWatch AIS 凭证获取 + +官方教程：https://developer.barentswatch.no/docs/tutorial + +1. 先打开上面的 BarentsWatch 官方 tutorial，按官方流程登录或注册开发者账号。 +2. 在 Developer access 页面选择 `AIS - API`，不要选择普通的 `BarentsWatch - API`。 +3. 在 `AIS - API` 下创建用于 Planet 的 AIS client。 +4. 创建时记下你设置的 password / client secret。 +5. 回到 My Page 复制完整 `Client ID`。它通常长得像 `your.email@example.com:client-name`。 +6. 回到 Planet 的 `设置 -> 采集器设置 -> BarentsWatch AIS`，填入 `Client ID` 和 `Client Secret`。 +7. 点击 `连接` 验证 token 和 AIS endpoint 是否可访问。 +8. 连接成功后保存凭证。 + +### 请求规则 + +- Token 地址：`https://id.barentswatch.no/connect/token` +- 请求方式：`POST` +- Content-Type：`application/x-www-form-urlencoded` +- Body 必须包含：`grant_type=client_credentials`、`client_id`、`client_secret`、`scope=ais` +- `client_id`、`client_secret`、`scope`、`grant_type` 都要放在 body，不要放在 header。 +- AIS 数据请求使用 header：`Authorization: Bearer ` + +### 常见排查 + +- `未找到凭证`：确认 `Client ID` 和 `Client Secret` 已填写，或已经写入 `~/.zshrc`。 +- `HTTP 401/403`：通常是选成了普通 `BarentsWatch - API` client、client secret 错误，或 token 请求没有使用 `scope=ais`。 +- `network` 错误：检查本机是否能访问 `id.barentswatch.no` 和 `live.ais.barentswatch.no`。 +- Endpoint 建议保持默认：`https://live.ais.barentswatch.no/v1/latest/combined`。 +""", +) + +AISSTREAM_DEFAULT_GUIDE = CredentialGuideDefault( + provider="aisstream", + title="AISStream API Key 获取教程", + prompt=( + "请生成一份中文教程，指导开发者获取 AISStream 的 API Key 并配置到 Planet。" + "教程要面向已经有本地开发环境的人，包含注册/登录 AISStream、获取 API Key、" + "理解免费额度和订阅范围、在 Planet 设置中心填写 API Key、配置 bounding boxes " + "和 message types、验证连接、常见失败排查。必须提醒用户以 AISStream 当前官网和" + "服务条款为准，不要编造具体页面按钮文案。" + ), + markdown="""## AISStream API Key 获取 + +官方入口：https://aisstream.io/ + +1. 打开 AISStream 官网，按当前页面指引注册或登录账号。 +2. 在账号/API 管理页面创建或复制你的 API Key。 +3. 先确认当前账号额度、使用条款和可订阅区域。实时 AIS 流量可能很大，不建议一开始订阅全球范围。 +4. 回到 Planet 的 `设置 -> 采集器设置 -> AISStream 实时船舶`。 +5. 在 `AISStream 凭证` 中填入 API Key。 +6. Endpoint 通常保持默认：`wss://stream.aisstream.io/v0/stream`。 +7. 按需配置 `Bounding Boxes JSON` 和 `消息类型`。 +8. 点击连接测试，确认系统能读取凭证且 WebSocket endpoint 格式有效。 +9. 保存采集器设置后再运行 `aisstream_vessels` collector。 + +### 推荐配置 + +默认消息类型： + +```json +["PositionReport", "ShipStaticData"] +``` + +默认 Bounding Boxes 示例： + +```json +[[[-90, -180], [90, 180]]] +``` + +这个示例表示全球范围。实际使用时建议先改成较小区域，降低消息量和处理压力。 + +### 请求规则 + +- Endpoint：`wss://stream.aisstream.io/v0/stream` +- 传输方式：WebSocket +- API Key 放在订阅 payload 中，不放在 HTTP header。 +- Planet 会把 AISStream 标记为 `delivery_mode = realtime_stream`、`transport = websocket`。 +- AISStream collector 只写入 AIS raw observations，不直接覆盖最终船只展示表。 + +### 常见排查 + +- `未找到凭证`：确认 API Key 已保存到采集器设置，或设置了 `AISSTREAM_API_KEY` 环境变量 / `~/.zshrc`。 +- `endpoint 必须是 ws:// 或 wss://`：AISStream 是 WebSocket 流接口，不要填普通 `https://` API 地址。 +- 采集量过大：缩小 `Bounding Boxes JSON`，减少 `message_types`，或降低单次最大消息数。 +- 没有船只数据：确认订阅区域内确实有 AIS 活动，并检查 API Key 当前额度和权限。 +- 连接中断：实时流可能受网络和上游限流影响，collector 会记录源健康状态供聚合服务回退。 +""", +) + + +DEFAULT_CREDENTIAL_GUIDES = { + BARENTSWATCH_DEFAULT_GUIDE.provider: BARENTSWATCH_DEFAULT_GUIDE, + AISSTREAM_DEFAULT_GUIDE.provider: AISSTREAM_DEFAULT_GUIDE, +} + + +async def _get_guide_store(db) -> tuple[SystemSetting | None, dict[str, Any]]: + result = await db.execute( + select(SystemSetting).where(SystemSetting.category == CREDENTIAL_GUIDES_CATEGORY) + ) + record = result.scalar_one_or_none() + payload = dict(record.payload or {}) if record and isinstance(record.payload, dict) else {} + return record, payload + + +async def get_credential_guide(db, provider: str) -> dict[str, Any]: + default = DEFAULT_CREDENTIAL_GUIDES.get(provider) + if default is None: + raise ValueError(f"Unsupported credential guide provider: {provider}") + + _record, store = await _get_guide_store(db) + custom = store.get(provider) if isinstance(store.get(provider), dict) else None + return { + "provider": provider, + "title": custom.get("title") if custom else default.title, + "markdown": custom.get("markdown") if custom else default.markdown, + "prompt": default.prompt, + "source": "ai" if custom else "default", + } + + +async def save_credential_guide(db, provider: str, title: str, markdown: str) -> dict[str, Any]: + default = DEFAULT_CREDENTIAL_GUIDES.get(provider) + if default is None: + raise ValueError(f"Unsupported credential guide provider: {provider}") + + record, store = await _get_guide_store(db) + store[provider] = { + "title": title or default.title, + "markdown": markdown, + } + if record is None: + db.add(SystemSetting(category=CREDENTIAL_GUIDES_CATEGORY, payload=store)) + else: + record.payload = store + await db.commit() + return await get_credential_guide(db, provider) + + +async def reset_credential_guide(db, provider: str) -> dict[str, Any]: + default = DEFAULT_CREDENTIAL_GUIDES.get(provider) + if default is None: + raise ValueError(f"Unsupported credential guide provider: {provider}") + + record, store = await _get_guide_store(db) + if provider in store: + store.pop(provider, None) + if record is not None: + record.payload = store + await db.commit() + return await get_credential_guide(db, provider) + + +async def generate_credential_guide( + db, + provider: str, + ai_client: AIProviderClient, +) -> dict[str, Any]: + default = DEFAULT_CREDENTIAL_GUIDES.get(provider) + if default is None: + raise ValueError(f"Unsupported credential guide provider: {provider}") + + response = await ai_client.analyze( + SituationalAnalysisRequest( + title=f"Generate credential guide for {provider}", + objective=default.prompt, + context={ + "provider": provider, + "current_default_guide": default.markdown, + "product_context": "Planet collector credential settings", + }, + observations=[ + "Use concise Chinese markdown.", + "Prefer stable concepts over brittle UI labels.", + "Include verification and troubleshooting steps.", + ], + constraints=[ + "Do not ask the user for secrets.", + "Do not include fabricated screenshots.", + "Return markdown only.", + ], + ) + ) + markdown = response.content.strip() + if not markdown: + markdown = default.markdown + return await save_credential_guide(db, provider, default.title, markdown) diff --git a/backend/app/services/custom_datasource_runtime.py b/backend/app/services/custom_datasource_runtime.py new file mode 100644 index 00000000..ebf41225 --- /dev/null +++ b/backend/app/services/custom_datasource_runtime.py @@ -0,0 +1,391 @@ +"""Runtime helpers for mapped custom data sources.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +from datetime import UTC, datetime +from typing import Any + +import httpx +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.target_schema_registry import TARGET_SCHEMAS +from app.db.session import async_session_factory +from app.models.datasource_config import DataSourceConfig +from app.models.datasource_mapping import DataSourceMappingTemplate +from app.services.datasource_mapping import ( + MappingError, + execute_mapping, + extract_path, + persist_mapped_records, +) + +DEFAULT_MAPPING_TEMPLATES: dict[str, dict[str, Any]] = { + "vessel_ais": { + "source": {"items_path": "$"}, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "name": {"path": "$.name", "type": "string", "default": None}, + "lat": {"path": "$.lat", "type": "float"}, + "lon": {"path": "$.lon", "type": "float"}, + "sog": {"path": "$.sog", "type": "float", "default": None}, + "cog": {"path": "$.cog", "type": "float", "default": None}, + "heading": {"path": "$.heading", "type": "integer", "default": None}, + "nav_status": {"path": "$.nav_status", "type": "integer", "default": None}, + "callsign": {"path": "$.callsign", "type": "string", "default": None}, + "vessel_type": {"path": "$.vessel_type", "type": "string", "default": None}, + "vessel_type_name": {"path": "$.vessel_type_name", "type": "string", "default": None}, + "received_at": {"path": "$.received_at", "type": "datetime", "default": None}, + }, + "meta": {"generated_by": "default_template", "requires_review": False}, + }, +} + +RUNNING_CUSTOM_STREAM_TASKS: dict[int, asyncio.Task[Any]] = {} + + +class CustomDatasourceRuntimeError(RuntimeError): + """Raised when a custom datasource cannot run.""" + + +def build_request_headers(auth_type: str, auth_config: dict, headers: dict) -> dict[str, str]: + request_headers = {str(key): str(value) for key, value in (headers or {}).items()} + auth_type = str(auth_type or "none").lower() + auth_config = auth_config or {} + + if auth_type == "bearer" and auth_config.get("token"): + request_headers["Authorization"] = f"Bearer {auth_config['token']}" + elif auth_type == "api_key" and auth_config.get("api_key"): + location = str(auth_config.get("in") or auth_config.get("location") or "header").lower() + if location != "query": + key_name = auth_config.get("key_name", "X-API-Key") + request_headers[str(key_name)] = str(auth_config["api_key"]) + elif auth_type == "basic": + username = auth_config.get("username", "") + password = auth_config.get("password", "") + credentials = f"{username}:{password}" + encoded = base64.b64encode(credentials.encode()).decode() + request_headers["Authorization"] = f"Basic {encoded}" + return request_headers + + +def build_query_params(auth_type: str, auth_config: dict, config: dict) -> dict[str, Any]: + params: dict[str, Any] = {} + candidate = (config or {}).get("params") or (config or {}).get("query_params") + if isinstance(candidate, dict): + params.update(candidate) + + auth_type = str(auth_type or "none").lower() + auth_config = auth_config or {} + if auth_type == "api_key" and auth_config.get("api_key"): + location = str(auth_config.get("in") or auth_config.get("location") or "header").lower() + if location == "query": + key_name = auth_config.get("key_name") or auth_config.get("param_name") or "api_key" + params[str(key_name)] = auth_config["api_key"] + return params + + +async def load_active_mapping( + db: AsyncSession, + datasource_config_id: int, +) -> DataSourceMappingTemplate: + result = await db.execute( + select(DataSourceMappingTemplate) + .where(DataSourceMappingTemplate.datasource_config_id == datasource_config_id) + .where(DataSourceMappingTemplate.is_active.is_(True)) + .order_by(DataSourceMappingTemplate.version.desc()) + .limit(1) + ) + mapping = result.scalar_one_or_none() + if mapping is not None: + return mapping + + datasource = await db.get(DataSourceConfig, datasource_config_id) + if datasource is None: + raise CustomDatasourceRuntimeError("Configuration not found") + target_schema = (datasource.config or {}).get("target_schema") + template_body = DEFAULT_MAPPING_TEMPLATES.get(str(target_schema or "")) if target_schema else None + if not template_body or target_schema not in TARGET_SCHEMAS: + raise CustomDatasourceRuntimeError( + "No active mapping template found and no default template available for this target schema" + ) + + mapping = DataSourceMappingTemplate( + datasource_config_id=datasource_config_id, + target_schema=str(target_schema), + mapping_json=template_body, + sample_payload_hash=None, + validation_status="valid", + version=1, + is_active=True, + ) + db.add(mapping) + await db.commit() + await db.refresh(mapping) + return mapping + + +async def fetch_rest_payload(config: DataSourceConfig, limit_bytes: int) -> Any: + request_config = config.config or {} + method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper() + if method not in {"GET", "POST"}: + raise CustomDatasourceRuntimeError("Only GET and POST sample requests are supported.") + + headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {}) + params = build_query_params(config.auth_type, config.auth_config or {}, request_config) + timeout = float(request_config.get("timeout", 30)) + json_body = request_config.get("json_body") + if json_body is None and str(request_config.get("body_type") or "").lower() in {"json", ""}: + candidate = request_config.get("body") + if isinstance(candidate, (dict, list)): + json_body = candidate + + async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: + response = await client.request( + method, + config.endpoint, + headers=headers, + params=params or None, + json=json_body, + ) + response.raise_for_status() + content = response.content[:limit_bytes] + if "application/json" in response.headers.get("content-type", ""): + return json.loads(content.decode(response.encoding or "utf-8")) + return {"text": content.decode(response.encoding or "utf-8", errors="replace")} + + +async def run_mapped_rest_config( + db: AsyncSession, + datasource: DataSourceConfig, +) -> dict[str, Any]: + mapping = await load_active_mapping(db, datasource.id) + sample = await fetch_rest_payload(datasource, 5_000_000) + mapped = execute_mapping(sample, mapping.mapping_json, mapping.target_schema) + if mapped["failed_count"] > 0: + return { + "status": "failed", + "datasource_config_id": datasource.id, + "mapping_id": mapping.id, + "mapping_version": mapping.version, + "target_schema": mapping.target_schema, + "mapped_count": mapped["mapped_count"], + "failed_count": mapped["failed_count"], + "errors": mapped["errors"][:20], + } + + request_config = datasource.config or {} + written_count = await persist_mapped_records( + db, + datasource_name=datasource.name, + datasource_config_id=datasource.id, + target_schema=mapping.target_schema, + records=mapped["records"], + mapping_version=mapping.version, + delivery_mode=request_config.get("delivery_mode") or "polling", + transport="http", + ) + return { + "status": "success", + "datasource_config_id": datasource.id, + "mapping_id": mapping.id, + "mapping_version": mapping.version, + "target_schema": mapping.target_schema, + "fetched_count": mapped["total_items"], + "mapped_count": mapped["mapped_count"], + "written_count": written_count, + } + + +def _items_from_ws_message(payload: Any, config: dict) -> Any: + message_path = config.get("ws_message_path") + items_path = config.get("ws_items_path") + value = extract_path(payload, message_path) if message_path else payload + return extract_path(value, items_path) if items_path else value + + +async def _connect_websocket(endpoint: str, headers: dict[str, str]): + import websockets + + try: + return await websockets.connect(endpoint, additional_headers=headers or None) + except TypeError: + return await websockets.connect(endpoint, extra_headers=headers or None) + + +async def test_websocket_config(config: DataSourceConfig) -> dict[str, Any]: + if not str(config.endpoint or "").startswith(("ws://", "wss://")): + raise CustomDatasourceRuntimeError("WebSocket datasource endpoint must start with ws:// or wss://") + + runtime_config = config.config or {} + headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {}) + receive_timeout = float(runtime_config.get("receive_timeout_seconds") or runtime_config.get("timeout") or 10) + async with await _connect_websocket(config.endpoint, headers) as websocket: + subscribe_message = runtime_config.get("ws_subscribe_message") + if isinstance(subscribe_message, (dict, list)): + await websocket.send(json.dumps(subscribe_message)) + elif isinstance(subscribe_message, str) and subscribe_message.strip(): + await websocket.send(subscribe_message) + raw_message = await asyncio.wait_for(websocket.recv(), timeout=receive_timeout) + return { + "success": True, + "message_preview": raw_message[:1000] if isinstance(raw_message, str) else str(raw_message)[:1000], + } + + +async def run_mapped_websocket_config( + db: AsyncSession, + datasource: DataSourceConfig, + *, + debug_max_messages: int | None = None, + use_config_debug_max_messages: bool = True, +) -> dict[str, Any]: + if not str(datasource.endpoint or "").startswith(("ws://", "wss://")): + raise CustomDatasourceRuntimeError("WebSocket datasource endpoint must start with ws:// or wss://") + + mapping = await load_active_mapping(db, datasource.id) + runtime_config = datasource.config or {} + max_messages = debug_max_messages + if max_messages is None and use_config_debug_max_messages: + max_messages = runtime_config.get("debug_max_messages") + max_messages = int(max_messages) if max_messages else None + receive_timeout = float(runtime_config.get("receive_timeout_seconds") or runtime_config.get("timeout") or 30) + reconnect = bool(runtime_config.get("ws_reconnect", True)) + reconnect_delay = float(runtime_config.get("reconnect_delay_seconds") or 3) + headers = build_request_headers(datasource.auth_type, datasource.auth_config or {}, datasource.headers or {}) + + messages_seen = 0 + mapped_count = 0 + failed_count = 0 + written_count = 0 + errors: list[dict[str, Any]] = [] + started_at = datetime.now(UTC) + + while True: + try: + async with await _connect_websocket(datasource.endpoint, headers) as websocket: + subscribe_message = runtime_config.get("ws_subscribe_message") + if isinstance(subscribe_message, (dict, list)): + await websocket.send(json.dumps(subscribe_message)) + elif isinstance(subscribe_message, str) and subscribe_message.strip(): + await websocket.send(subscribe_message) + + while True: + raw_message = await asyncio.wait_for(websocket.recv(), timeout=receive_timeout) + messages_seen += 1 + try: + payload = json.loads(raw_message) + except json.JSONDecodeError as exc: + failed_count += 1 + errors.append({"message": "invalid_json", "error": str(exc)}) + continue + + extracted = _items_from_ws_message(payload, runtime_config) + try: + mapped = execute_mapping(extracted, mapping.mapping_json, mapping.target_schema) + except (MappingError, ValueError) as exc: + failed_count += 1 + errors.append({"message": "mapping_failed", "error": str(exc)}) + continue + + mapped_count += mapped["mapped_count"] + failed_count += mapped["failed_count"] + if mapped["errors"]: + errors.extend(mapped["errors"][:5]) + if mapped["records"]: + written_count += await persist_mapped_records( + db, + datasource_name=datasource.name, + datasource_config_id=datasource.id, + target_schema=mapping.target_schema, + records=mapped["records"], + mapping_version=mapping.version, + delivery_mode=runtime_config.get("delivery_mode") or "realtime_stream", + transport="websocket", + ) + + if max_messages and messages_seen >= max_messages: + return { + "status": "success", + "datasource_config_id": datasource.id, + "mapping_id": mapping.id, + "mapping_version": mapping.version, + "target_schema": mapping.target_schema, + "messages_seen": messages_seen, + "mapped_count": mapped_count, + "failed_count": failed_count, + "written_count": written_count, + "errors": errors[:20], + "execution_time_seconds": (datetime.now(UTC) - started_at).total_seconds(), + } + except asyncio.CancelledError: + raise + except Exception as exc: + failed_count += 1 + errors.append({"message": "websocket_error", "error": f"{exc.__class__.__name__}: {exc}"}) + if not reconnect or max_messages: + return { + "status": "failed" if written_count == 0 else "partial", + "datasource_config_id": datasource.id, + "mapping_id": mapping.id, + "mapping_version": mapping.version, + "target_schema": mapping.target_schema, + "messages_seen": messages_seen, + "mapped_count": mapped_count, + "failed_count": failed_count, + "written_count": written_count, + "errors": errors[:20], + } + await asyncio.sleep(reconnect_delay) + + +async def run_custom_stream_by_id(config_id: int) -> dict[str, Any]: + async with async_session_factory() as db: + datasource = await db.get(DataSourceConfig, config_id) + if not datasource: + raise CustomDatasourceRuntimeError("Configuration not found") + return await run_mapped_websocket_config( + db, + datasource, + use_config_debug_max_messages=False, + ) + + +def start_custom_stream(config_id: int) -> bool: + existing = RUNNING_CUSTOM_STREAM_TASKS.get(config_id) + if existing is not None and not existing.done(): + return False + task = asyncio.create_task(run_custom_stream_by_id(config_id), name=f"custom-stream:{config_id}") + RUNNING_CUSTOM_STREAM_TASKS[config_id] = task + + def _cleanup(done_task: asyncio.Task[Any]) -> None: + if RUNNING_CUSTOM_STREAM_TASKS.get(config_id) is done_task: + RUNNING_CUSTOM_STREAM_TASKS.pop(config_id, None) + + task.add_done_callback(_cleanup) + return True + + +async def stop_custom_stream(config_id: int) -> bool: + task = RUNNING_CUSTOM_STREAM_TASKS.get(config_id) + if task is None or task.done(): + RUNNING_CUSTOM_STREAM_TASKS.pop(config_id, None) + return False + task.cancel() + try: + await task + except asyncio.CancelledError: + return True + return task.cancelled() + + +def get_custom_stream_status(config_id: int) -> dict[str, Any]: + task = RUNNING_CUSTOM_STREAM_TASKS.get(config_id) + return { + "config_id": config_id, + "running": bool(task and not task.done()), + "done": bool(task and task.done()), + } diff --git a/backend/app/services/datasource_connectivity.py b/backend/app/services/datasource_connectivity.py new file mode 100644 index 00000000..9bc89063 --- /dev/null +++ b/backend/app/services/datasource_connectivity.py @@ -0,0 +1,437 @@ +"""Connectivity validation helpers for built-in datasource overrides.""" + +from __future__ import annotations + +from datetime import UTC, datetime +import hashlib +import json +import os +from typing import Any + +import httpx +from sqlalchemy import func, select + +from app.core.data_sources import get_data_sources_config +from app.core.datasource_defaults import DEFAULT_DATASOURCES +from app.models.collected_data import CollectedData +from app.models.datasource import DataSource +from app.models.datasource_config import DataSourceConfig +from app.models.system_setting import SystemSetting +from app.services.barentswatch import ( + _read_zshrc_env, + fetch_barentswatch_access_token, + resolve_barentswatch_config, +) + + +CONNECTIVITY_VALIDATION_KEY = "connectivity_validation" +CONNECTIVITY_STORE_CATEGORY = "datasource_connectivity_validations" +SUPPORTED_CREDENTIAL_PROVIDERS = {"barentswatch", "spacetrack", "aisstream"} + + +def _sha256_json(payload: Any) -> str: + encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _resolve_spacetrack_credentials() -> tuple[str, str, str]: + zshrc_env = _read_zshrc_env() + username = os.getenv("SPACETRACK_USERNAME") or zshrc_env.get("SPACETRACK_USERNAME") or "" + password = os.getenv("SPACETRACK_PASSWORD") or zshrc_env.get("SPACETRACK_PASSWORD") or "" + source = "environment" if os.getenv("SPACETRACK_USERNAME") or os.getenv("SPACETRACK_PASSWORD") else "" + if not source and (username or password): + source = "~/.zshrc" + return username, password, source or "missing" + + +async def _resolve_aisstream_api_key( + db=None, + credential_override: dict[str, str] | None = None, +) -> tuple[str, str]: + if credential_override and credential_override.get("api_key"): + return str(credential_override["api_key"]), "draft" + + env_key = os.getenv("AISSTREAM_API_KEY") + zshrc_key = _read_zshrc_env().get("AISSTREAM_API_KEY") + if db is not None: + result = await db.execute( + select(DataSourceConfig) + .where(DataSourceConfig.name == "aisstream_vessels") + .where(DataSourceConfig.is_active.is_(True)) + ) + record = result.scalar_one_or_none() + if record: + auth_config = record.auth_config or {} + runtime_config = record.config or {} + api_key = auth_config.get("api_key") or runtime_config.get("api_key") + if api_key: + return str(api_key), "datasource_config" + + if env_key: + return env_key, "environment" + if zshrc_key: + return zshrc_key, "~/.zshrc" + return "", "missing" + + +def strip_connectivity_validation(config: dict | None) -> dict: + cleaned = dict(config or {}) + cleaned.pop(CONNECTIVITY_VALIDATION_KEY, None) + return cleaned + + +def merge_connectivity_validation(existing_config: dict | None, next_config: dict | None) -> dict: + merged = strip_connectivity_validation(next_config) + validation = (existing_config or {}).get(CONNECTIVITY_VALIDATION_KEY) + if validation: + merged[CONNECTIVITY_VALIDATION_KEY] = validation + return merged + + +def get_connectivity_validation(config: DataSourceConfig | None) -> dict | None: + validation = (config.config or {}).get(CONNECTIVITY_VALIDATION_KEY) if config else None + return validation if isinstance(validation, dict) else None + + +async def build_builtin_connectivity_checksum( + source: str, + endpoint: str, + auth_type: str, + headers: dict | None, + config: dict | None, + db=None, + credential_override: dict[str, str] | None = None, +) -> tuple[str, dict[str, Any]]: + defaults = DEFAULT_DATASOURCES.get(source, {}) + credential_provider = defaults.get("credential_provider") + credential_fingerprint = "" + credential_source = "none" + has_credentials = not defaults.get("requires_credentials", False) + + if credential_provider == "barentswatch": + if credential_override: + client_id = credential_override.get("client_id", "") + client_secret = credential_override.get("client_secret", "") + credential_source = "draft" + else: + barentswatch_config = await resolve_barentswatch_config(db) + client_id = barentswatch_config.client_id + client_secret = barentswatch_config.client_secret + credential_source = barentswatch_config.credential_source + has_credentials = bool(client_id and client_secret) + credential_fingerprint = _sha256_json( + { + "client_id": client_id, + "client_secret": client_secret, + } + ) + elif credential_provider == "spacetrack": + username, password, credential_source = _resolve_spacetrack_credentials() + has_credentials = bool(username and password) + credential_fingerprint = _sha256_json( + { + "username": username, + "password": password, + } + ) + elif credential_provider == "aisstream": + api_key, credential_source = await _resolve_aisstream_api_key(db, credential_override) + has_credentials = bool(api_key) + credential_fingerprint = _sha256_json({"api_key": api_key}) + elif defaults.get("requires_credentials"): + credential_source = str(credential_provider or "unsupported") + + checksum_payload = { + "source": source, + "endpoint": endpoint, + "auth_type": "none", + "headers": headers or {}, + "config": strip_connectivity_validation(config), + "credential_provider": credential_provider or "none", + "credential_fingerprint": credential_fingerprint, + } + return _sha256_json(checksum_payload), { + "requires_credentials": bool(defaults.get("requires_credentials", False)), + "credential_provider": credential_provider, + "credential_source": credential_source, + "has_credentials": has_credentials, + } + + +async def test_builtin_connectivity( + source: str, + endpoint: str, + auth_type: str, + headers: dict | None, + config: dict | None, + db=None, + credential_override: dict[str, str] | None = None, +) -> dict[str, Any]: + defaults = DEFAULT_DATASOURCES.get(source) + if not defaults: + return { + "success": False, + "message": "未知内置采集器，无法执行连接校验。", + } + + checksum, credential_context = await build_builtin_connectivity_checksum( + source, + endpoint, + auth_type, + headers, + config, + db, + credential_override, + ) + if credential_context["requires_credentials"] and not credential_context["has_credentials"]: + return { + "success": False, + "checksum": checksum, + "stage": "credentials", + "message": "该采集器需要凭证，请先到采集器凭证设置中配置。", + "settings_tab": "collector_credentials", + **credential_context, + } + if ( + credential_context["requires_credentials"] + and credential_context["credential_provider"] not in SUPPORTED_CREDENTIAL_PROVIDERS + ): + return { + "success": False, + "checksum": checksum, + "stage": "credentials", + "message": "该采集器的凭证链路尚未接入，暂时无法完成连接校验。", + "settings_tab": "collector_credentials", + **credential_context, + } + + request_headers = {str(key): str(value) for key, value in (headers or {}).items()} + request_config = strip_connectivity_validation(config) + timeout = float(request_config.get("timeout") or 30) + request_endpoint = endpoint + + if credential_context["credential_provider"] == "aisstream": + if not str(request_endpoint).startswith(("ws://", "wss://")): + return { + "success": False, + "checksum": checksum, + "stage": "endpoint", + "message": "AISStream endpoint 必须是 ws:// 或 wss:// WebSocket 地址。", + **credential_context, + } + return { + "success": True, + "checksum": checksum, + "stage": "credentials", + "message": "AISStream 凭证已配置，WebSocket endpoint 格式有效。", + **credential_context, + } + + try: + async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: + if credential_context["credential_provider"] == "barentswatch": + barentswatch_config = await resolve_barentswatch_config(db) + token = await fetch_barentswatch_access_token(client, barentswatch_config) + if not token: + return { + "success": False, + "checksum": checksum, + "stage": "token", + "message": "凭证可读取，但 token 响应中没有 access_token。", + "settings_tab": "collector_credentials", + **credential_context, + } + request_headers["Authorization"] = f"Bearer {token}" + elif credential_context["credential_provider"] == "spacetrack": + username, password, _source = _resolve_spacetrack_credentials() + login_url = "https://www.space-track.org/ajaxauth/login" + login_response = await client.post( + login_url, + data={ + "identity": username, + "password": password, + }, + ) + login_response.raise_for_status() + + started = datetime.now(UTC) + async with client.stream("GET", request_endpoint, headers=request_headers) as response: + response.raise_for_status() + status_code = response.status_code + elapsed_ms = (datetime.now(UTC) - started).total_seconds() * 1000 + + return { + "success": True, + "checksum": checksum, + "stage": "endpoint", + "message": "连接验证成功。", + "status_code": status_code, + "response_time_ms": elapsed_ms, + **credential_context, + } + except httpx.HTTPStatusError as exc: + return { + "success": False, + "checksum": checksum, + "stage": "endpoint", + "message": f"连接验证失败：HTTP {exc.response.status_code}", + "error": f"HTTP Error: {exc.response.status_code}", + **credential_context, + } + except httpx.HTTPError as exc: + return { + "success": False, + "checksum": checksum, + "stage": "network", + "message": f"连接验证失败：{exc.__class__.__name__}", + "error": str(exc), + **credential_context, + } + + +def make_success_validation(checksum: str, result: dict[str, Any]) -> dict[str, Any]: + return { + "checksum": checksum, + "status": "success", + "validated_at": datetime.now(UTC).isoformat(), + "status_code": result.get("status_code"), + "credential_source": result.get("credential_source"), + } + + +def is_builtin_validation_current(config: DataSourceConfig | None, checksum: str) -> bool: + validation = get_connectivity_validation(config) + return bool( + validation + and validation.get("status") == "success" + and validation.get("checksum") == checksum + ) + + +async def get_connectivity_store(db) -> dict[str, Any]: + result = await db.execute( + select(SystemSetting).where(SystemSetting.category == CONNECTIVITY_STORE_CATEGORY) + ) + record = result.scalar_one_or_none() + return dict(record.payload or {}) if record and isinstance(record.payload, dict) else {} + + +async def save_connectivity_success( + db, + source: str, + checksum: str, + result: dict[str, Any], + *, + connected_by: str, +) -> dict[str, Any]: + store = await get_connectivity_store(db) + validation = { + **make_success_validation(checksum, result), + "connected_by": connected_by, + } + store[source] = validation + + existing = await db.execute( + select(SystemSetting).where(SystemSetting.category == CONNECTIVITY_STORE_CATEGORY) + ) + record = existing.scalar_one_or_none() + if record is None: + db.add(SystemSetting(category=CONNECTIVITY_STORE_CATEGORY, payload=store)) + else: + record.payload = store + return validation + + +async def load_builtin_override_config(db, source: str) -> DataSourceConfig | None: + result = await db.execute( + select(DataSourceConfig) + .where(DataSourceConfig.name == source) + .where(DataSourceConfig.is_active.is_(True)) + ) + return result.scalar_one_or_none() + + +async def get_builtin_effective_candidate(db, source: str) -> dict[str, Any]: + override = await load_builtin_override_config(db, source) + default_endpoint = get_data_sources_config().get_yaml_url(source) + return { + "name": source, + "endpoint": (override.endpoint if override and override.endpoint else default_endpoint) or "", + "auth_type": override.auth_type if override else "none", + "headers": override.headers if override else {}, + "config": strip_connectivity_validation(override.config if override else {}), + } + + +async def has_collected_data(db, source: str) -> bool: + result = await db.execute(select(func.count(CollectedData.id)).where(CollectedData.source == source)) + if (result.scalar() or 0) > 0: + return True + + datasource_result = await db.execute(select(DataSource).where(DataSource.source == source)) + datasource = datasource_result.scalar_one_or_none() + return bool(datasource and datasource.last_status == "success") + + +async def get_builtin_connection_status( + db, + source: str, + endpoint: str, + auth_type: str, + headers: dict | None, + config: dict | None, +) -> dict[str, Any]: + checksum, credential_context = await build_builtin_connectivity_checksum( + source, + endpoint, + auth_type, + headers, + config, + db, + ) + store = await get_connectivity_store(db) + validation = store.get(source) + if isinstance(validation, dict) and validation.get("status") == "success": + if validation.get("checksum") == checksum: + return { + "connected": True, + "checksum": checksum, + "connected_by": validation.get("connected_by") or "connection_button", + "message": "当前配置已完成连接验证。", + **credential_context, + } + + effective = await get_builtin_effective_candidate(db, source) + effective_checksum, _ = await build_builtin_connectivity_checksum( + source, + effective["endpoint"], + effective["auth_type"], + effective["headers"], + effective["config"], + db, + ) + if checksum == effective_checksum and await has_collected_data(db, source): + return { + "connected": True, + "checksum": checksum, + "connected_by": "collection", + "message": "当前配置已有成功采集数据，视为已连接。", + **credential_context, + } + + if isinstance(validation, dict) and validation.get("status") == "success": + return { + "connected": False, + "checksum": checksum, + "connected_by": None, + "message": "接口地址或凭证指纹已变化，请重新点击连接验证。", + **credential_context, + } + + return { + "connected": False, + "checksum": checksum, + "connected_by": None, + "message": "当前配置尚未连接，请点击连接验证。", + **credential_context, + } diff --git a/backend/app/services/datasource_mapping.py b/backend/app/services/datasource_mapping.py new file mode 100644 index 00000000..03dbf297 --- /dev/null +++ b/backend/app/services/datasource_mapping.py @@ -0,0 +1,410 @@ +"""Deterministic mapping support for custom data sources.""" + +from __future__ import annotations + +import hashlib +import json +import re +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.target_schema_registry import TargetSchema, get_target_schema + +SECRET_KEY_PATTERN = re.compile( + r"(token|secret|password|passwd|authorization|api[_-]?key|client[_-]?secret)", + re.IGNORECASE, +) + + +class MappingError(ValueError): + """Raised when a mapping definition cannot be executed.""" + + +def stable_payload_hash(payload: Any) -> str: + encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode() + return hashlib.sha256(encoded).hexdigest() + + +def redact_for_llm(value: Any) -> Any: + if isinstance(value, dict): + redacted = {} + for key, item in value.items(): + if SECRET_KEY_PATTERN.search(str(key)): + redacted[key] = "[REDACTED]" + else: + redacted[key] = redact_for_llm(item) + return redacted + if isinstance(value, list): + return [redact_for_llm(item) for item in value[:20]] + return value + + +def extract_path(payload: Any, path: str | None) -> Any: + if not path or path == "$": + return payload + + normalized = path.strip() + if normalized.startswith("$."): + normalized = normalized[2:] + elif normalized.startswith("$"): + normalized = normalized[1:] + normalized = normalized.strip(".") + if not normalized: + return payload + + current = payload + for raw_segment in normalized.split("."): + segment = raw_segment.strip() + if not segment: + continue + + list_all = segment.endswith("[*]") + if list_all: + segment = segment[:-3] + + index = None + match = re.fullmatch(r"(.+)\[(\d+)\]", segment) + if match: + segment = match.group(1) + index = int(match.group(2)) + + if segment: + if isinstance(current, dict): + current = current.get(segment) + else: + return None + + if list_all: + return current if isinstance(current, list) else [] + + if index is not None: + if not isinstance(current, list) or index >= len(current): + return None + current = current[index] + + return current + + +def _convert_value(value: Any, target_type: str | None) -> Any: + if value is None or target_type in (None, "", "any"): + return value + + if target_type == "string": + return str(value) + if target_type == "integer": + return int(value) + if target_type == "float": + return float(value) + if target_type == "boolean": + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "y", "on"} + return bool(value) + if target_type == "datetime": + if isinstance(value, datetime): + return value + if isinstance(value, (int, float)): + return datetime.fromtimestamp(value) + if isinstance(value, str): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + return value + if target_type == "object": + if isinstance(value, dict): + return value + raise ValueError("expected object") + if target_type == "array": + if isinstance(value, list): + return value + raise ValueError("expected array") + + return value + + +def _apply_enum(value: Any, enum_map: Any) -> Any: + if not isinstance(enum_map, dict): + return value + key = str(value) + return enum_map.get(key, enum_map.get(value, value)) + + +def _map_one(item: Any, field_mapping: dict[str, Any]) -> tuple[dict[str, Any], list[str]]: + output: dict[str, Any] = {} + errors: list[str] = [] + + for field_name, rule in field_mapping.items(): + if isinstance(rule, str): + rule = {"path": rule} + if not isinstance(rule, dict): + errors.append(f"{field_name}: mapping rule must be an object or path string") + continue + + value = extract_path(item, rule.get("path")) + if value is None and "default" in rule: + value = rule.get("default") + value = _apply_enum(value, rule.get("enum")) + + try: + value = _convert_value(value, rule.get("type")) + except (TypeError, ValueError) as exc: + errors.append(f"{field_name}: failed to convert value {value!r}: {exc}") + continue + + if value is not None or rule.get("include_null", False): + output[field_name] = value + + return output, errors + + +def execute_mapping( + payload: Any, + mapping_json: dict[str, Any], + target_schema: str | TargetSchema, + *, + limit: int | None = None, +) -> dict[str, Any]: + schema = get_target_schema(target_schema) if isinstance(target_schema, str) else target_schema + source = mapping_json.get("source") or {} + fields = mapping_json.get("fields") + if not isinstance(fields, dict) or not fields: + raise MappingError("mapping_json.fields must be a non-empty object") + + items_path = source.get("items_path") or mapping_json.get("items_path") or "$" + items = extract_path(payload, items_path) + if isinstance(items, dict): + items = [items] + elif not isinstance(items, list): + items = [] + + if limit is not None: + items = items[:limit] + + mapped_records: list[dict[str, Any]] = [] + errors: list[dict[str, Any]] = [] + for index, item in enumerate(items): + mapped, mapping_errors = _map_one(item, fields) + validated, validation_errors = schema.validate_record(mapped) + all_errors = mapping_errors + validation_errors + if all_errors: + errors.append({"index": index, "errors": all_errors, "record": mapped}) + continue + if validated is not None: + mapped_records.append(validated) + + return { + "target_schema": schema.key, + "total_items": len(items), + "mapped_count": len(mapped_records), + "failed_count": len(errors), + "records": mapped_records, + "errors": errors, + } + + +def build_heuristic_mapping(sample_payload: Any, target_schema_key: str) -> dict[str, Any]: + schema = get_target_schema(target_schema_key) + items_path = "$" + sample_item = sample_payload + if isinstance(sample_payload, dict): + for key in ("data", "items", "results", "features", "vessels"): + candidate = sample_payload.get(key) + if isinstance(candidate, list) and candidate: + items_path = f"$.{key}[*]" + sample_item = candidate[0] + break + elif isinstance(sample_payload, list) and sample_payload: + items_path = "$" + sample_item = sample_payload[0] + + available = _flatten_keys(sample_item if isinstance(sample_item, dict) else {}) + fields: dict[str, Any] = {} + for field in schema.fields: + candidate = _best_field_match(field.name, available) + if candidate: + fields[field.name] = {"path": f"$.{candidate}", "type": field.type} + elif field.name == "data" and target_schema_key == "generic_records": + fields[field.name] = {"path": "$", "type": "object"} + elif not field.required: + fields[field.name] = {"path": f"$.{field.name}", "type": field.type, "default": None} + + return { + "source": {"items_path": items_path}, + "fields": fields, + "meta": { + "generated_by": "heuristic", + "requires_review": True, + }, + } + + +def _flatten_keys(payload: dict[str, Any], prefix: str = "") -> list[str]: + keys: list[str] = [] + for key, value in payload.items(): + dotted = f"{prefix}.{key}" if prefix else str(key) + keys.append(dotted) + if isinstance(value, dict): + keys.extend(_flatten_keys(value, dotted)) + return keys + + +def _best_field_match(field_name: str, candidates: list[str]) -> str | None: + aliases = { + "lat": ("lat", "latitude", "y"), + "lon": ("lon", "lng", "longitude", "x"), + "mmsi": ("mmsi",), + "sog": ("sog", "speed", "speedOverGround"), + "cog": ("cog", "course", "courseOverGround"), + "received_at": ("received_at", "timestamp", "time", "updated_at"), + "observed_at": ("observed_at", "timestamp", "time", "updated_at"), + "source_id": ("id", "source_id", "uuid"), + }.get(field_name, (field_name,)) + + lowered = {candidate.lower(): candidate for candidate in candidates} + for alias in aliases: + if alias.lower() in lowered: + return lowered[alias.lower()] + for candidate in candidates: + tail = candidate.split(".")[-1].lower() + if tail in {alias.lower() for alias in aliases}: + return candidate + return None + + +def _parse_datetime(value: Any) -> datetime | None: + if value is None: + return None + if isinstance(value, datetime): + return value + if isinstance(value, str): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + return None + + +async def persist_mapped_records( + db: AsyncSession, + *, + datasource_name: str, + datasource_config_id: int, + target_schema: str, + records: list[dict[str, Any]], + mapping_version: int, + delivery_mode: str | None = None, + transport: str | None = None, +) -> int: + """Persist validated mapped records to the destination for a target schema.""" + if target_schema == "vessel_ais": + from app.core.time import to_iso8601_utc + from app.core.websocket.broadcaster import broadcaster + from app.services.vessel_ais_aggregation import ( + record_vessel_ais_observation, + update_ais_source_health, + ) + + now = datetime.now(UTC) + latest_observed_at = now + written_count = 0 + for record in records: + observed_at = _parse_datetime(record.get("received_at")) or now + observation = await record_vessel_ais_observation( + db, + source=datasource_name, + normalized_payload=record, + raw_payload=record, + delivery_mode=delivery_mode or "polling", + transport=transport or "http", + message_type="PositionReport", + observed_at=observed_at, + collected_at=now, + ) + if observation is not None: + written_count += 1 + if observed_at > latest_observed_at: + latest_observed_at = observed_at + + await update_ais_source_health( + db, + source=datasource_name, + connection_state="connected", + observed_count=len(records), + last_seen_at=latest_observed_at, + last_success_at=now if records else None, + lag_seconds=max((now - latest_observed_at).total_seconds(), 0), + ) + await db.commit() + if records: + await broadcaster.broadcast_custom( + "vessels", + { + "action": "upsert", + "source": datasource_name, + "created": True, + "vessels": [ + { + "mmsi": record.get("mmsi"), + "mmsi_display": str(record.get("mmsi")) if record.get("mmsi") is not None else None, + "name": record.get("name"), + "callsign": record.get("callsign"), + "lat": record.get("lat"), + "lon": record.get("lon"), + "sog": record.get("sog"), + "cog": record.get("cog"), + "heading": record.get("heading"), + "nav_status": record.get("nav_status"), + "vessel_type": record.get("vessel_type"), + "vessel_type_name": record.get("vessel_type_name"), + "received_at": to_iso8601_utc(_parse_datetime(record.get("received_at"))), + } + for record in records + ], + }, + ) + return written_count + + from app.models.collected_data import CollectedData + + collected_at = datetime.now(UTC) + for index, record in enumerate(records): + if target_schema == "geo_points": + source_id = record.get("source_id") or f"{datasource_config_id}:{index}" + name = record.get("name") + metadata = { + "latitude": record.get("lat"), + "longitude": record.get("lon"), + "type": record.get("type"), + "mapping_version": mapping_version, + "target_schema": target_schema, + **(record.get("metadata") or {}), + } + reference_date = _parse_datetime(record.get("observed_at")) + else: + source_id = record.get("source_id") or f"{datasource_config_id}:{index}" + name = None + metadata = { + "data": record.get("data") or {}, + "mapping_version": mapping_version, + "target_schema": target_schema, + } + reference_date = _parse_datetime(record.get("observed_at")) + + db.add( + CollectedData( + source=datasource_name, + source_id=str(source_id), + entity_key=f"{datasource_name}:{source_id}", + data_type=target_schema, + name=name, + title=name, + extra_data=metadata, + collected_at=collected_at, + reference_date=reference_date, + is_valid=1, + is_current=True, + change_type="created", + change_summary={}, + ) + ) + + await db.commit() + return len(records) diff --git a/backend/app/services/docs_gatekeeper.py b/backend/app/services/docs_gatekeeper.py new file mode 100644 index 00000000..9cd90043 --- /dev/null +++ b/backend/app/services/docs_gatekeeper.py @@ -0,0 +1,119 @@ +"""Server-side Docs metadata and Gatekeeper authorization helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from app.models.user import User + +DocsAccess = Literal["public", "docs_user", "docs_developer", "docs_admin"] +DocsLang = Literal["zh", "en"] + +VALID_DOCS_LANGS = {"zh", "en"} +DOCS_README_FILENAME = "README.md" +DEFAULT_DOCS_SLUG = "overview" + +REPO_ROOT = Path(__file__).resolve().parents[3] +TECHNICAL_DOCS_ROOT = REPO_ROOT / "docs" / "technical" + + +@dataclass(frozen=True) +class DocsMetadata: + filename: str + slug: str + access: DocsAccess + group: str + order: int + zh_title: str + en_title: str + + +DOCS_METADATA: tuple[DocsMetadata, ...] = ( + DocsMetadata(DOCS_README_FILENAME, DEFAULT_DOCS_SLUG, "public", "Overview", 0, "技术文档", "Technical Docs"), + DocsMetadata("quickstart.md", "quickstart", "public", "Manual", 1, "快速开始", "Quickstart"), + DocsMetadata("manual.md", "manual", "public", "Manual", 2, "Planet 使用手册", "Planet Manual"), + DocsMetadata("location-pipeline-user.md", "location-pipeline-user", "public", "Manual", 3, "Earth 位置候选采集使用手册", "Earth Location Candidate Collection User Guide"), + DocsMetadata("earth-frontend-context.md", "earth-frontend-context", "docs_developer", "Earth", 10, "Earth 前端结构", "Earth Frontend Context"), + DocsMetadata("earth-layer-style-reference.md", "earth-layer-style-reference", "docs_developer", "Earth", 11, "Earth 图层样式属性索引", "Earth Layer Style Reference"), + DocsMetadata("earth-render-layer-order.md", "earth-render-layer-order", "docs_developer", "Earth", 12, "Earth 渲染图层顺序", "Earth Render Layer Order"), + DocsMetadata("earth-satellite-footprint-policy.md", "earth-satellite-footprint-policy", "docs_developer", "Earth", 13, "Earth 卫星覆盖策略", "Earth Satellite Footprint Policy"), + DocsMetadata("earth-bgp-context.md", "earth-bgp-context", "docs_developer", "Earth", 14, "BGP 态势上下文", "BGP Context"), + DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Earth", 15, "新闻直播采集格式", "News Live Streams Collector Format"), + DocsMetadata("earth-interactable-usage.md", "earth-interactable-usage", "docs_developer", "Earth", 16, "Earth 可交互图标接入", "Earth Interactable Usage"), + DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 17, "Earth 工具栏与浮层协同", "Earth Toolbar and Overlay Coordination"), + DocsMetadata("frontend-admin-frontend-context.md", "frontend-admin-frontend-context", "docs_developer", "Frontend", 20, "控制台前端结构", "Admin Frontend Context"), + DocsMetadata("frontend-layout-guidelines.md", "frontend-layout-guidelines", "docs_developer", "Frontend", 21, "前端布局指南", "Frontend Layout Guidelines"), + DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Frontend", 22, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"), + DocsMetadata("backend-collectors.md", "backend-collectors", "docs_developer", "Backend", 30, "数据采集系统", "Data Collectors"), + DocsMetadata("backend-system-service-control.md", "backend-system-service-control", "docs_admin", "Backend", 31, "系统服务控制", "System Service Control"), + DocsMetadata("datasource-collector-settings-connectivity.md", "datasource-collector-settings-connectivity", "docs_developer", "Backend", 32, "数据源、采集器设置与连接验证", "Datasource Collector Settings and Connectivity"), + DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"), + DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 34, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"), + DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"), + DocsMetadata("ops-docker-compose-buildx-upgrade.md", "ops-docker-compose-buildx-upgrade", "docs_admin", "Ops", 50, "Docker + Compose + Buildx 升级", "Docker + Compose + Buildx Upgrade"), + DocsMetadata("ops-planet-sh-startup.md", "ops-planet-sh-startup", "docs_admin", "Ops", 51, "planet.sh 启动机制", "planet.sh Startup"), +) + +DOCS_BY_SLUG = {entry.slug: entry for entry in DOCS_METADATA} + + +def get_user_gatekeeper_groups(user: User | None) -> set[str]: + if user is None: + return set() + + role = user.role.value if hasattr(user.role, "value") else str(user.role or "") + if role == "super_admin": + return {"docs_user", "docs_developer", "docs_admin"} + if role == "admin": + return {"docs_user", "docs_developer", "docs_admin"} + + groups = set() + raw_groups = user.gatekeeper_groups or [] + if isinstance(raw_groups, list): + groups.update(str(group) for group in raw_groups) + + if "docs_admin" in groups: + groups.update({"docs_developer", "docs_user"}) + if "docs_developer" in groups: + groups.add("docs_user") + return groups + + +def can_read_doc(entry: DocsMetadata, user: User | None) -> bool: + if entry.access == "public": + return True + return entry.access in get_user_gatekeeper_groups(user) + + +def doc_path_for(entry: DocsMetadata, lang: str) -> Path: + if lang not in VALID_DOCS_LANGS: + raise ValueError("Unsupported docs language") + return TECHNICAL_DOCS_ROOT / lang / entry.filename + + +def title_for(entry: DocsMetadata, lang: str) -> str: + return entry.zh_title if lang == "zh" else entry.en_title + + +def catalog_for_user(user: User | None) -> list[dict]: + items: list[dict] = [] + for entry in DOCS_METADATA: + if not can_read_doc(entry, user): + continue + for lang in sorted(VALID_DOCS_LANGS): + if not doc_path_for(entry, lang).exists(): + continue + items.append( + { + "slug": entry.slug, + "filename": entry.filename, + "lang": lang, + "title": title_for(entry, lang), + "group": entry.group, + "order": entry.order, + "access": entry.access, + } + ) + return sorted(items, key=lambda item: (item["lang"], item["order"], item["title"])) diff --git a/backend/app/services/earth_news.py b/backend/app/services/earth_news.py index 22f359a1..8de65dcb 100644 --- a/backend/app/services/earth_news.py +++ b/backend/app/services/earth_news.py @@ -30,6 +30,14 @@ class RegionProfile: accent: str +@dataclass(frozen=True) +class RegionAnchor: + region: str + label: str + latitude: float + longitude: float + + @dataclass(frozen=True) class NewsFeedSource: id: str @@ -95,6 +103,39 @@ REGION_PROFILES: dict[str, RegionProfile] = { ), } +REGION_ANCHORS: dict[str, RegionAnchor] = { + "americas": RegionAnchor( + region="americas", + label="美洲", + latitude=37.0902, + longitude=-95.7129, + ), + "europe": RegionAnchor( + region="europe", + label="欧洲", + latitude=50.1109, + longitude=8.6821, + ), + "middle-east-africa": RegionAnchor( + region="middle-east-africa", + label="中东与非洲", + latitude=25.2048, + longitude=55.2708, + ), + "asia-pacific": RegionAnchor( + region="asia-pacific", + label="亚太", + latitude=1.3521, + longitude=103.8198, + ), + "global": RegionAnchor( + region="global", + label="全球", + latitude=20.0, + longitude=0.0, + ), +} + def _google_news_feed(query: str, *, hl: str, gl: str, ceid: str) -> str: return ( @@ -213,6 +254,10 @@ def get_region_profile(region: str) -> RegionProfile: return REGION_PROFILES.get(region, REGION_PROFILES["global"]) +def get_region_anchor(region: str) -> RegionAnchor: + return REGION_ANCHORS.get(region, REGION_ANCHORS["global"]) + + def get_sources_for_region(region: str) -> list[NewsFeedSource]: return sorted( [source for source in NEWS_FEED_SOURCES if source.region in {"global", region}], @@ -342,6 +387,7 @@ def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]: def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, Any]: published_at = item.published_at + anchor = get_region_anchor(item.feed_region) return { "id": item.id, "title": item.title, @@ -352,6 +398,10 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An "region": item.feed_region, "homepage_url": item.homepage_url, "published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None, + "latitude": anchor.latitude, + "longitude": anchor.longitude, + "location_label": anchor.label, + "location_inferred": True, "is_focus_match": item.feed_region == active_region, } diff --git a/backend/app/services/llm_provider_catalog.py b/backend/app/services/llm_provider_catalog.py new file mode 100644 index 00000000..89da1a2d --- /dev/null +++ b/backend/app/services/llm_provider_catalog.py @@ -0,0 +1,150 @@ +"""LLM provider presets used by Settings and the runtime AI provider bridge.""" + +from __future__ import annotations + +from typing import Any + +import httpx + +MODELS_DEV_URL = "https://models.dev/api.json" + + +FALLBACK_LLM_PROVIDER_PRESETS: dict[str, dict[str, Any]] = { + "minimax": { + "provider": "minimax", + "label": "MiniMax", + "provider_api": "anthropic-messages", + "base_url": "https://api.minimaxi.com/anthropic", + "model": "MiniMax-M2.7", + "models": ["MiniMax-M2.7", "MiniMax-M2.7-highspeed", "MiniMax-M2.5", "MiniMax-M2"], + "api_key_env": "MINIMAX_API_KEY", + "source": "fallback", + }, + "openai": { + "provider": "openai", + "label": "OpenAI", + "provider_api": "openai-completions", + "base_url": "https://api.openai.com/v1", + "model": "gpt-5.1", + "models": ["gpt-5.1", "gpt-5.1-codex", "gpt-4.1", "gpt-4o"], + "api_key_env": "OPENAI_API_KEY", + "source": "fallback", + }, + "anthropic": { + "provider": "anthropic", + "label": "Anthropic", + "provider_api": "anthropic-messages", + "base_url": "https://api.anthropic.com/v1", + "model": "claude-sonnet-4-6", + "models": ["claude-sonnet-4-6", "claude-opus-4-5", "claude-3-5-haiku-20241022"], + "api_key_env": "ANTHROPIC_API_KEY", + "source": "fallback", + }, + "deepseek": { + "provider": "deepseek", + "label": "DeepSeek", + "provider_api": "openai-completions", + "base_url": "https://api.deepseek.com/v1", + "model": "deepseek-chat", + "models": ["deepseek-chat", "deepseek-reasoner"], + "api_key_env": "DEEPSEEK_API_KEY", + "source": "fallback", + }, + "alibaba": { + "provider": "alibaba", + "label": "Alibaba Qwen / DashScope", + "provider_api": "openai-completions", + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "model": "qwen3-max", + "models": ["qwen3-max", "qwen3.5-plus", "qwen-max", "qwen-plus"], + "api_key_env": "DASHSCOPE_API_KEY", + "source": "fallback", + }, + "moonshotai": { + "provider": "moonshotai", + "label": "Moonshot AI / Kimi", + "provider_api": "openai-completions", + "base_url": "https://api.moonshot.ai/v1", + "model": "kimi-k2.5", + "models": ["kimi-k2.5", "kimi-k2-thinking", "kimi-k2-turbo-preview"], + "api_key_env": "MOONSHOT_API_KEY", + "source": "fallback", + }, + "openrouter": { + "provider": "openrouter", + "label": "OpenRouter", + "provider_api": "openai-completions", + "base_url": "https://openrouter.ai/api/v1", + "model": "openai/gpt-5.1", + "models": ["openai/gpt-5.1", "anthropic/claude-sonnet-4.5", "qwen/qwen3-max"], + "api_key_env": "OPENROUTER_API_KEY", + "source": "fallback", + }, + "ollama": { + "provider": "ollama", + "label": "Ollama Local", + "provider_api": "ollama-generate", + "base_url": "http://127.0.0.1:11434", + "model": "qwen2.5:7b", + "models": ["qwen2.5:7b", "llama3.1:8b", "mistral:7b"], + "api_key_env": "", + "source": "fallback", + }, +} + +MODELS_DEV_PROVIDER_KEYS = { + "minimax": "minimax", + "openai": "openai", + "anthropic": "anthropic", + "deepseek": "deepseek", + "alibaba": "alibaba", + "moonshotai": "moonshotai", + "openrouter": "openrouter", +} + + +def list_fallback_llm_provider_presets() -> list[dict[str, Any]]: + return [dict(value) for value in FALLBACK_LLM_PROVIDER_PRESETS.values()] + + +def get_fallback_llm_provider_preset(provider: str) -> dict[str, Any]: + key = provider.strip().lower() + if key not in FALLBACK_LLM_PROVIDER_PRESETS: + raise ValueError(f"Unsupported LLM provider preset: {provider}") + return dict(FALLBACK_LLM_PROVIDER_PRESETS[key]) + + +async def refresh_llm_provider_preset(provider: str) -> dict[str, Any]: + fallback = get_fallback_llm_provider_preset(provider) + models_dev_key = MODELS_DEV_PROVIDER_KEYS.get(fallback["provider"]) + if not models_dev_key: + return fallback + + async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client: + response = await client.get( + MODELS_DEV_URL, + headers={"User-Agent": "Planet/1.0"}, + ) + response.raise_for_status() + catalog = response.json() + + upstream = catalog.get(models_dev_key) + if not isinstance(upstream, dict): + return fallback + + upstream_models = upstream.get("models") if isinstance(upstream.get("models"), dict) else {} + model_ids = list(upstream_models.keys())[:80] + base_url = upstream.get("api") or fallback["base_url"] + if fallback["provider"] == "deepseek" and base_url == "https://api.deepseek.com": + base_url = "https://api.deepseek.com/v1" + + refreshed = { + **fallback, + "label": upstream.get("name") or fallback["label"], + "base_url": base_url, + "model": model_ids[0] if model_ids else fallback["model"], + "models": model_ids or fallback["models"], + "api_key_env": (upstream.get("env") or [fallback["api_key_env"]])[0], + "source": MODELS_DEV_URL, + } + return refreshed diff --git a/backend/app/services/location/__init__.py b/backend/app/services/location/__init__.py new file mode 100644 index 00000000..82c0adc3 --- /dev/null +++ b/backend/app/services/location/__init__.py @@ -0,0 +1,57 @@ +"""Shared location-resolution pipeline. + +A reusable abstraction for "given a record, decide its lat/lon" — used by +compute centers, BGP collectors, BGP events, and any future entity that needs +location estimation. + +Each domain wires its own :class:`LocationPipeline` from a sequence of +:class:`LocationResolver` instances. Future algorithms (peeringdb, IXP tables, +user-confirmed coordinates, …) plug in by implementing the protocol — no +changes needed to consumers. +""" + +from .models import ( + LocationCandidate, + LocationQuery, + ResolutionDiagnostic, + ResolutionResult, + ResolverOutput, +) +from .pipeline import LocationPipeline, LocationResolver +from .resolvers.inherit import InheritFromAnotherEntityResolver +from .resolvers.nominatim import ( + NominatimResolver, + build_default_nominatim_geocoder, + interpret_geocode_result, +) +from .resolvers.registry import RegistryResolver, default_score_alias_match +from .resolvers.source_coordinates import SourceCoordinatesResolver +from .text import ( + city_key, + coerce_str, + normalize_country_text, + normalize_text, + parse_float, +) + +__all__ = [ + "LocationCandidate", + "LocationPipeline", + "LocationQuery", + "LocationResolver", + "ResolutionDiagnostic", + "ResolutionResult", + "ResolverOutput", + "InheritFromAnotherEntityResolver", + "NominatimResolver", + "RegistryResolver", + "SourceCoordinatesResolver", + "build_default_nominatim_geocoder", + "city_key", + "coerce_str", + "default_score_alias_match", + "interpret_geocode_result", + "normalize_country_text", + "normalize_text", + "parse_float", +] diff --git a/backend/app/services/location/models.py b/backend/app/services/location/models.py new file mode 100644 index 00000000..ab626009 --- /dev/null +++ b/backend/app/services/location/models.py @@ -0,0 +1,126 @@ +"""Domain-neutral data structures for the location pipeline.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + +# Renderable precision tiers, ordered from most precise to least. +RENDERABLE_PRECISIONS: tuple[str, ...] = ("precise", "site", "city") + + +@dataclass(frozen=True) +class LocationQuery: + """Domain-neutral input for the resolution pipeline. + + ``name`` and ``aliases`` are matched against registry alias indexes; + ``city`` / ``country`` / ``region`` provide geographic context for both + registry lookups and Nominatim queries; ``source_latitude`` / + ``source_longitude`` short-circuit when the record already carries + coordinates; ``extra`` carries domain-specific fields (operator, site, + organization, asn, peer_ip, …) that resolvers can opt into. + """ + + name: str | None = None + aliases: tuple[str, ...] = () + city: str | None = None + country: str | None = None + region: str | None = None + source_latitude: float | None = None + source_longitude: float | None = None + extra: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class LocationCandidate: + """A resolved location candidate produced by a resolver.""" + + latitude: float + longitude: float + display_name: str + precision: str # "precise" | "site" | "city" | (rejected: country/unknown) + confidence: float + query: str + source: str + source_note: str | None + matched_fields: tuple[str, ...] + needs_confirmation: bool + city: str | None = None + region: str | None = None + country: str | None = None + matched_location_name: str | None = None + location_verified_at: str | None = None + suggested_registry_entry: dict[str, Any] | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "latitude": self.latitude, + "longitude": self.longitude, + "display_name": self.display_name, + "precision": self.precision, + "confidence": self.confidence, + "query": self.query, + "source": self.source, + "source_note": self.source_note, + "matched_fields": list(self.matched_fields), + "needs_confirmation": self.needs_confirmation, + "city": self.city, + "region": self.region, + "country": self.country, + "matched_location_name": self.matched_location_name, + "location_verified_at": self.location_verified_at, + "suggested_registry_entry": self.suggested_registry_entry, + } + + +@dataclass(frozen=True) +class ResolverOutput: + """What a single resolver returns from one ``resolve()`` call.""" + + candidates: tuple[LocationCandidate, ...] = () + attempted_queries: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ResolutionDiagnostic: + """Why we could not resolve, plus what we tried.""" + + failure_reason: str + attempted_queries: tuple[str, ...] = () + record_id: int | None = None + source: str | None = None + source_id: str | None = None + name: str | None = None + country: str | None = None + city: str | None = None + site: str | None = None + operator: str | None = None + extra: Mapping[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "failure_reason": self.failure_reason, + "attempted_queries": list(self.attempted_queries), + "record_id": self.record_id, + "source": self.source, + "source_id": self.source_id, + "name": self.name, + "country": self.country, + "city": self.city, + "site": self.site, + "operator": self.operator, + **({"extra": dict(self.extra)} if self.extra else {}), + } + + +@dataclass(frozen=True) +class ResolutionResult: + """Pipeline output: best candidate (if any) + diagnostic on miss.""" + + location: LocationCandidate | None + diagnostic: ResolutionDiagnostic | None + attempted_queries: tuple[str, ...] = () + + @property + def is_resolved(self) -> bool: + return bool(self.location) diff --git a/backend/app/services/location/pipeline.py b/backend/app/services/location/pipeline.py new file mode 100644 index 00000000..322c46d9 --- /dev/null +++ b/backend/app/services/location/pipeline.py @@ -0,0 +1,126 @@ +"""Pipeline that runs a sequence of :class:`LocationResolver` instances.""" + +from __future__ import annotations + +from typing import Protocol, Sequence + +from .models import ( + LocationCandidate, + LocationQuery, + ResolutionDiagnostic, + ResolutionResult, + ResolverOutput, +) + + +class LocationResolver(Protocol): + """Pluggable location resolution step. + + Implementations: ``SourceCoordinatesResolver``, ``RegistryResolver``, + ``NominatimResolver``, ``InheritFromAnotherEntityResolver`` — see the + ``resolvers`` subpackage. New algorithms (peeringdb / IXP / user-confirmed + coordinates) plug in by implementing this protocol; the pipeline does not + care how candidates are produced. + """ + + name: str + + def resolve(self, query: LocationQuery) -> ResolverOutput: ... + + +def default_candidate_sort_key( + candidate: LocationCandidate, +) -> tuple[int, int, float]: + precision_rank = {"precise": 0, "site": 1, "city": 2}.get( + candidate.precision, 9 + ) + source_rank = { + "source_coordinates": 0, + "stored_compute_center_location": 1, + "stored_collector_location": 1, + "ror_organization_registry": 2, + "inherited": 3, + "nominatim_online_geocode": 4, + "local_registry": 8, + "local_registry_city": 9, + }.get(candidate.source, 9) + return (source_rank, precision_rank, -float(candidate.confidence or 0)) + + +class LocationPipeline: + """Orchestrate a sequence of resolvers. + + ``collect_candidates`` runs every resolver and returns *all* deduped + candidates plus the queries each resolver attempted (useful for + user-facing "why didn't this work?" diagnostics). + + ``resolve_best`` returns the top candidate per + :func:`default_candidate_sort_key` (or a custom sort). + """ + + def __init__( + self, + resolvers: Sequence[LocationResolver], + *, + sort_key=default_candidate_sort_key, + failure_reason: str = ( + "Could not resolve to renderable coordinates from any configured resolver." + ), + ) -> None: + self._resolvers = list(resolvers) + self._sort_key = sort_key + self._failure_reason = failure_reason + + @property + def resolvers(self) -> tuple[LocationResolver, ...]: + return tuple(self._resolvers) + + def collect_candidates( + self, query: LocationQuery + ) -> tuple[list[LocationCandidate], list[str]]: + candidates: list[LocationCandidate] = [] + attempted: list[str] = [] + seen_keys: set[tuple[str, str, str]] = set() + + for resolver in self._resolvers: + output = resolver.resolve(query) + for q in output.attempted_queries: + if q and q not in attempted: + attempted.append(q) + for candidate in output.candidates: + key = ( + candidate.source, + f"{candidate.latitude:.4f}", + f"{candidate.longitude:.4f}", + ) + if key in seen_keys: + continue + seen_keys.add(key) + candidates.append(candidate) + + candidates.sort(key=self._sort_key) + return candidates, attempted + + def resolve_best(self, query: LocationQuery) -> ResolutionResult: + candidates, attempted = self.collect_candidates(query) + if candidates: + return ResolutionResult( + location=candidates[0], + diagnostic=None, + attempted_queries=tuple(attempted), + ) + return ResolutionResult( + location=None, + diagnostic=ResolutionDiagnostic( + failure_reason=self._failure_reason, + attempted_queries=tuple(attempted), + name=query.name, + country=query.country, + city=query.city, + site=str(query.extra.get("site")) if query.extra.get("site") else None, + operator=str(query.extra.get("operator")) + if query.extra.get("operator") + else None, + ), + attempted_queries=tuple(attempted), + ) diff --git a/backend/app/services/location/resolvers/__init__.py b/backend/app/services/location/resolvers/__init__.py new file mode 100644 index 00000000..aaf5da41 --- /dev/null +++ b/backend/app/services/location/resolvers/__init__.py @@ -0,0 +1,20 @@ +"""Built-in resolver implementations.""" + +from .inherit import InheritFromAnotherEntityResolver +from .nominatim import ( + NominatimResolver, + build_default_nominatim_geocoder, + interpret_geocode_result, +) +from .registry import RegistryResolver, default_score_alias_match +from .source_coordinates import SourceCoordinatesResolver + +__all__ = [ + "InheritFromAnotherEntityResolver", + "NominatimResolver", + "RegistryResolver", + "SourceCoordinatesResolver", + "build_default_nominatim_geocoder", + "default_score_alias_match", + "interpret_geocode_result", +] diff --git a/backend/app/services/location/resolvers/inherit.py b/backend/app/services/location/resolvers/inherit.py new file mode 100644 index 00000000..46d39d90 --- /dev/null +++ b/backend/app/services/location/resolvers/inherit.py @@ -0,0 +1,31 @@ +"""Resolver that inherits a candidate from another entity's resolution. + +Used by BGP events to pick up the location of their owning collector. The +``source_lookup`` callable is the only domain coupling — it receives the +incoming :class:`LocationQuery` and returns either an already-resolved +:class:`LocationCandidate` (typically by querying another pipeline) or +``None`` to signal "no parent location available". +""" + +from __future__ import annotations + +from typing import Callable + +from ..models import LocationCandidate, LocationQuery, ResolverOutput + + +class InheritFromAnotherEntityResolver: + def __init__( + self, + *, + source_lookup: Callable[[LocationQuery], LocationCandidate | None], + name: str = "inherited", + ) -> None: + self.name = name + self._lookup = source_lookup + + def resolve(self, query: LocationQuery) -> ResolverOutput: + result = self._lookup(query) + if result is None: + return ResolverOutput() + return ResolverOutput(candidates=(result,)) diff --git a/backend/app/services/location/resolvers/nominatim.py b/backend/app/services/location/resolvers/nominatim.py new file mode 100644 index 00000000..59030961 --- /dev/null +++ b/backend/app/services/location/resolvers/nominatim.py @@ -0,0 +1,292 @@ +"""Nominatim-backed online geocoder. + +The actual HTTP call is encapsulated in :func:`build_default_nominatim_geocoder` +which returns an ``lru_cache``-wrapped function. Domain modules typically: + +1. Build a default geocoder via :func:`build_default_nominatim_geocoder`. +2. Re-export it under a stable module-level name (e.g. ``_geocode_online``). +3. Pass a *late-binding lambda* (``lambda q: _geocode_online(q)``) to + :class:`NominatimResolver`. + +This ensures tests that ``monkeypatch.setattr(module, "_geocode_online", ...)`` +can swap the geocoder behavior without touching pipeline construction. +""" + +from __future__ import annotations + +import time +from functools import lru_cache +from typing import Any, Callable + +import httpx + +from ..models import LocationCandidate, LocationQuery, ResolverOutput +from ..text import ( + coerce_str, + normalize_country_text, + normalize_text, + parse_float, +) + +NOMINATIM_SEARCH_URL = "https://nominatim.openstreetmap.org/search" +DEFAULT_USER_AGENT = "planet-earth-location-resolver/1.0" +DEFAULT_MIN_INTERVAL_SECONDS = 1.1 +DEFAULT_TIMEOUT_SECONDS = 8.0 + + +def build_default_nominatim_geocoder( + *, + user_agent: str = DEFAULT_USER_AGENT, + min_interval_seconds: float = DEFAULT_MIN_INTERVAL_SECONDS, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + cache_size: int = 512, +) -> Callable[[str], dict[str, Any] | None]: + """Return a cached, rate-limited Nominatim geocoder.""" + + last_request_at = [0.0] + + @lru_cache(maxsize=cache_size) + def geocode(query: str) -> dict[str, Any] | None: + if not query: + return None + elapsed = time.monotonic() - last_request_at[0] + if elapsed < min_interval_seconds: + time.sleep(min_interval_seconds - elapsed) + last_request_at[0] = time.monotonic() + response = httpx.get( + NOMINATIM_SEARCH_URL, + params={ + "q": query, + "format": "jsonv2", + "limit": 1, + "addressdetails": 1, + }, + headers={"User-Agent": user_agent}, + timeout=timeout_seconds, + ) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, list) or not payload: + return None + result = payload[0] + if not isinstance(result, dict): + return None + return result + + return geocode + + +_DEFAULT_SITE_CATEGORIES = frozenset( + { + "amenity", + "office", + "building", + "industrial", + "research", + "university", + "education", + "tourism", + "shop", + "man_made", + "campus", + "research_institute", + } +) + + +def interpret_geocode_result( + result: dict[str, Any], + *, + matched_fields: tuple[str, ...], + context_country: str | None, + site_categories: frozenset[str] = _DEFAULT_SITE_CATEGORIES, + site_promoting_match_fields: frozenset[str] = frozenset( + {"site", "operator", "name"} + ), +) -> tuple[float, float, dict[str, Any], str] | None: + """Validate a Nominatim raw result. Returns (lat, lon, address, classification).""" + latitude = parse_float(result.get("lat")) + longitude = parse_float(result.get("lon")) + if latitude in (None, 0.0) or longitude in (None, 0.0): + return None + + address = result.get("address") if isinstance(result.get("address"), dict) else {} + if not isinstance(address, dict): + address = {} + + has_city_level = bool( + address.get("city") + or address.get("town") + or address.get("village") + or address.get("municipality") + or address.get("hamlet") + or address.get("suburb") + ) + osm_class = str(result.get("class") or "").lower() + osm_type = str(result.get("type") or "").lower() + is_site_like = osm_class in site_categories or osm_type in site_categories + if not has_city_level and not is_site_like: + return None + + if context_country: + normalized_context = normalize_text(normalize_country_text(context_country)) + normalized_result = normalize_text( + normalize_country_text(address.get("country")) + ) + if ( + normalized_context + and normalized_result + and normalized_context != normalized_result + ): + return None + + classification = ( + "site" + if ( + is_site_like + and has_city_level + and any(field in site_promoting_match_fields for field in matched_fields) + ) + else "city" + ) + return float(latitude), float(longitude), address, classification + + +def _candidate_from_geocode( + *, + query: LocationQuery, + geocode_query: str, + matched_fields: tuple[str, ...], + raw_result: dict[str, Any], + interpret: Callable[..., tuple[float, float, dict[str, Any], str] | None], + source: str, + site_confidence: float, + city_confidence: float, +) -> LocationCandidate | None: + interpreted = interpret( + raw_result, + matched_fields=matched_fields, + context_country=query.country, + ) + if not interpreted: + return None + latitude, longitude, address, classification = interpreted + city = ( + address.get("city") + or address.get("town") + or address.get("village") + or address.get("municipality") + or query.city + or None + ) + region = address.get("state") or address.get("region") + country = address.get("country") or query.country or None + display_name = raw_result.get("display_name") or geocode_query + confidence = city_confidence if classification == "city" else site_confidence + + extra = query.extra or {} + suggested_registry_entry = { + "canonical_name": ( + (query.aliases[0] if query.aliases else None) + or query.name + or display_name + ), + "aliases": list( + { + value + for value in [ + query.name, + *query.aliases, + coerce_str(extra.get("operator")), + coerce_str(extra.get("site")), + ] + if value + } + ), + "operator": coerce_str(extra.get("operator")) or None, + "site": coerce_str(extra.get("site")) + or coerce_str(extra.get("organization")) + or None, + "country": country, + "city": city, + "region": region, + "latitude": latitude, + "longitude": longitude, + "precision": classification, + "confidence": confidence, + "source_note": ( + f"Resolved via Nominatim query '{geocode_query}' → {display_name}" + ), + } + return LocationCandidate( + latitude=latitude, + longitude=longitude, + display_name=display_name, + precision=classification, + confidence=confidence, + query=geocode_query, + source=source, + source_note=f"Nominatim search result: {display_name}", + matched_fields=matched_fields, + needs_confirmation=True, + city=city, + region=region, + country=country, + matched_location_name=display_name, + location_verified_at=None, + suggested_registry_entry=suggested_registry_entry, + ) + + +class NominatimResolver: + """Run a domain-specific query plan against Nominatim.""" + + def __init__( + self, + *, + query_plan_builder: Callable[ + [LocationQuery], list[tuple[str, tuple[str, ...]]] + ], + geocoder: Callable[[str], dict[str, Any] | None], + name: str = "nominatim_online_geocode", + site_confidence: float = 0.72, + city_confidence: float = 0.62, + interpret: Callable[..., tuple[float, float, dict[str, Any], str] | None] = ( + interpret_geocode_result + ), + ) -> None: + self.name = name + self._query_plan_builder = query_plan_builder + self._geocoder = geocoder + self._site_confidence = site_confidence + self._city_confidence = city_confidence + self._interpret = interpret + + def resolve(self, query: LocationQuery) -> ResolverOutput: + plan = self._query_plan_builder(query) + candidates: list[LocationCandidate] = [] + attempted: list[str] = [] + for geocode_query, matched_fields in plan: + attempted.append(geocode_query) + try: + raw_result = self._geocoder(geocode_query) + except Exception: + continue + if not raw_result: + continue + candidate = _candidate_from_geocode( + query=query, + geocode_query=geocode_query, + matched_fields=matched_fields, + raw_result=raw_result, + interpret=self._interpret, + source=self.name, + site_confidence=self._site_confidence, + city_confidence=self._city_confidence, + ) + if candidate is not None: + candidates.append(candidate) + return ResolverOutput( + candidates=tuple(candidates), + attempted_queries=tuple(attempted), + ) diff --git a/backend/app/services/location/resolvers/registry.py b/backend/app/services/location/resolvers/registry.py new file mode 100644 index 00000000..d844afe8 --- /dev/null +++ b/backend/app/services/location/resolvers/registry.py @@ -0,0 +1,323 @@ +"""Resolver that matches a query against a local JSON registry. + +Registry schema (a single JSON file): + + { + "locations": [ + { + "canonical_name": "...", + "aliases": ["...", "..."], + "operator": "...", + "site": "...", + "city": "...", + "country": "...", + "region": "...", + "latitude": 0.0, + "longitude": 0.0, + "precision": "precise" | "site" | "city", + "confidence": 0.0, + "verification_status": "verified", + "source_note": "...", + "verified_at": "YYYY-MM-DD" + } + ], + "city_fallbacks": [ {city, country, latitude, longitude, ...} ] + } +""" + +from __future__ import annotations + +import json +from functools import lru_cache +from pathlib import Path +from typing import Any, Callable, Iterable + +from ..models import ( + RENDERABLE_PRECISIONS, + LocationCandidate, + LocationQuery, + ResolverOutput, +) +from ..text import ( + city_key, + normalize_country_text, + normalize_text, + parse_float, +) + +# Field-priority weights when scoring "this query field text contains this +# alias text". Tuned to match the legacy compute-center ordering — name beats +# site beats operator beats city — which generalizes well to other domains. +_DEFAULT_FIELD_PRIORITY = { + "name": 8, + "site": 6, + "operator": 5, + "city": 3, +} + + +def default_score_alias_match( + alias_field: str, record_field: str, alias_text: str +) -> int: + score = max(0, len(alias_text)) + score += _DEFAULT_FIELD_PRIORITY.get(alias_field, 1) + if alias_field == record_field: + score += 4 + if alias_field == "name" and record_field in {"name", "name_short", "alias"}: + score += 6 + if alias_field == "site" and record_field in {"site", "organization"}: + score += 4 + if alias_field == "operator" and record_field in {"operator", "organization"}: + score += 4 + return score + + +@lru_cache(maxsize=32) +def _load_registry_file(path: str) -> dict[str, Any]: + with Path(path).open("r", encoding="utf-8") as handle: + return json.load(handle) + + +@lru_cache(maxsize=32) +def _build_alias_index( + path: str, +) -> tuple[tuple[dict[str, Any], tuple[tuple[str, str], ...]], ...]: + index: list[tuple[dict[str, Any], tuple[tuple[str, str], ...]]] = [] + for entry in _load_registry_file(path).get("locations", []): + aliases: list[tuple[str, str]] = [] + seen: set[str] = set() + for alias in [entry.get("canonical_name"), *(entry.get("aliases") or [])]: + normalized = normalize_text(alias) + if normalized and normalized not in seen: + aliases.append(("name", normalized)) + seen.add(normalized) + for field_name in ("operator", "site", "city"): + value = entry.get(field_name) + normalized = normalize_text(value) + if normalized and normalized not in seen: + aliases.append((field_name, normalized)) + seen.add(normalized) + index.append((entry, tuple(aliases))) + return tuple(index) + + +def _query_corpus(query: LocationQuery) -> dict[str, str]: + """Map a query into normalized strings keyed by source field.""" + fields: dict[str, str] = { + "name": query.name or "", + "city": query.city or "", + "country": query.country or "", + } + for alias in query.aliases: + if alias and alias != query.name: + fields["name_short"] = alias + break + extra = query.extra or {} + for key in ("site", "operator", "organization"): + value = extra.get(key) + if value: + fields[key] = str(value) + return {key: normalize_text(value) for key, value in fields.items() if value} + + +def _country_compatible(entry: dict[str, Any], query: LocationQuery) -> bool: + record_country = normalize_country_text(query.country) + entry_country = normalize_country_text(entry.get("country")) + if not record_country or not entry_country: + return True + return normalize_text(record_country) == normalize_text(entry_country) + + +def _normalized_alias_matches(alias_normalized: str, record_text: str) -> bool: + alias_tokens = alias_normalized.split() + record_tokens = record_text.split() + if not alias_tokens or not record_tokens: + return False + if len(alias_tokens) == 1: + return alias_tokens[0] in record_tokens + window_size = len(alias_tokens) + return any( + record_tokens[index : index + window_size] == alias_tokens + for index in range(0, len(record_tokens) - window_size + 1) + ) + + +def _entry_to_candidate( + entry: dict[str, Any], + *, + matched_alias: str, + matched_fields: Iterable[str], + source: str, + score_explainer: str, + confidence_floor: float, +) -> LocationCandidate: + canonical_name = entry.get("canonical_name") or matched_alias + # Registry entries are treated as candidates unless explicitly verified. + # This prevents migrated hard-coded hints from appearing as factual + # location evidence. + is_verified = entry.get("verification_status") == "verified" + precision = entry.get("precision") or "city" + if precision not in RENDERABLE_PRECISIONS: + precision = "city" + fields_summary = ", ".join(sorted(set(matched_fields))) or "name" + confidence_value = parse_float(entry.get("confidence")) + confidence = ( + float(confidence_value) + if confidence_value is not None + else confidence_floor + ) + return LocationCandidate( + latitude=float(parse_float(entry.get("latitude")) or 0.0), + longitude=float(parse_float(entry.get("longitude")) or 0.0), + display_name=canonical_name, + precision=precision, + confidence=confidence, + query=f"local_registry::{matched_alias or canonical_name}", + source=source, + source_note=entry.get("source_note") + or f"{score_explainer}: matched {fields_summary}", + matched_fields=tuple(sorted(set(matched_fields))) or ("name",), + needs_confirmation=bool(entry.get("needs_confirmation")) or not is_verified, + city=entry.get("city"), + region=entry.get("region"), + country=entry.get("country"), + matched_location_name=canonical_name, + location_verified_at=entry.get("verified_at") if is_verified else None, + suggested_registry_entry=None, + ) + + +class RegistryResolver: + """Match a query against a JSON registry (plus its city_fallbacks table).""" + + def __init__( + self, + *, + registry_path: Path | str, + name: str = "local_registry", + city_fallback_source: str = "local_registry_city", + city_fallback_confidence_default: float = 0.65, + confidence_default: float = 0.85, + score_alias_match: Callable[[str, str, str], int] = default_score_alias_match, + ) -> None: + self.name = name + self._registry_path = str(Path(registry_path)) + self._city_fallback_source = city_fallback_source + self._city_fallback_confidence_default = city_fallback_confidence_default + self._confidence_default = confidence_default + self._score = score_alias_match + + def reload(self) -> None: + """Drop the cached registry — useful when the JSON file is edited.""" + _load_registry_file.cache_clear() + _build_alias_index.cache_clear() + + def resolve(self, query: LocationQuery) -> ResolverOutput: + candidates: list[LocationCandidate] = [] + candidates.extend(self._registry_candidates(query)) + city_candidate = self._city_fallback_candidate(query) + if city_candidate is not None: + candidates.append(city_candidate) + return ResolverOutput(candidates=tuple(candidates)) + + # ── internals ────────────────────────────────────────────── + + def _registry_candidates( + self, query: LocationQuery + ) -> list[LocationCandidate]: + corpus = _query_corpus(query) + if not corpus: + return [] + + # When the query carries a name (a record-specific identifier), require + # at least one alias match against a name-class field — otherwise a + # generic shared field like operator="RIPE NCC" would promote every + # registry entry that lists that operator, regardless of whether the + # name matches. + query_has_name = bool(corpus.get("name") or corpus.get("name_short")) + + results: list[LocationCandidate] = [] + for entry, aliases in _build_alias_index(self._registry_path): + best_alias = "" + best_score = 0 + matched_fields: list[str] = [] + matched_via_name_alias = False + for alias_field, alias_normalized in aliases: + for record_field, record_text in corpus.items(): + if not _normalized_alias_matches(alias_normalized, record_text): + continue + score = self._score( + alias_field, record_field, alias_normalized + ) + if score > best_score or ( + score == best_score + and len(alias_normalized) > len(best_alias) + ): + best_score = score + best_alias = alias_normalized + if record_field not in matched_fields: + matched_fields.append(record_field) + if alias_field == "name" and record_field in {"name", "name_short"}: + matched_via_name_alias = True + if not matched_fields or best_score <= 0: + continue + if query_has_name and not matched_via_name_alias: + continue + if not _country_compatible(entry, query): + continue + results.append( + _entry_to_candidate( + entry, + matched_alias=best_alias, + matched_fields=matched_fields, + source=self.name, + score_explainer="Registry alias match", + confidence_floor=self._confidence_default, + ) + ) + return results + + def _city_fallback_candidate( + self, query: LocationQuery + ) -> LocationCandidate | None: + country = normalize_country_text(query.country) + city = city_key(query.city) + if not country or not city: + return None + + for fallback in _load_registry_file(self._registry_path).get( + "city_fallbacks", [] + ): + fallback_country = normalize_country_text(fallback.get("country")) + fallback_city = city_key(fallback.get("city")) + if fallback_country != country or fallback_city != city: + continue + confidence_value = parse_float(fallback.get("confidence")) + confidence = ( + float(confidence_value) + if confidence_value is not None + else self._city_fallback_confidence_default + ) + return LocationCandidate( + latitude=float(parse_float(fallback.get("latitude")) or 0.0), + longitude=float(parse_float(fallback.get("longitude")) or 0.0), + display_name=fallback.get("city") or "", + precision="city", + confidence=confidence, + query=( + f"city_fallback::{fallback.get('city')}, " + f"{fallback.get('country')}" + ), + source=self._city_fallback_source, + source_note=fallback.get("source_note") + or f"City fallback for {fallback.get('city')}, {fallback.get('country')}", + matched_fields=("city", "country"), + needs_confirmation=False, + city=fallback.get("city"), + region=fallback.get("region"), + country=fallback.get("country"), + matched_location_name=fallback.get("city"), + location_verified_at=fallback.get("verified_at"), + suggested_registry_entry=None, + ) + return None diff --git a/backend/app/services/location/resolvers/source_coordinates.py b/backend/app/services/location/resolvers/source_coordinates.py new file mode 100644 index 00000000..f9283fb5 --- /dev/null +++ b/backend/app/services/location/resolvers/source_coordinates.py @@ -0,0 +1,42 @@ +"""Resolver that consumes lat/lon already present on the source record.""" + +from __future__ import annotations + +from ..models import LocationCandidate, LocationQuery, ResolverOutput +from ..text import normalize_country_text + + +class SourceCoordinatesResolver: + """Pass-through for records that already carry valid coordinates.""" + + name = "source_coordinates" + + def __init__(self, *, source: str = "source_coordinates") -> None: + self._source = source + + def resolve(self, query: LocationQuery) -> ResolverOutput: + lat = query.source_latitude + lon = query.source_longitude + if lat in (None, 0.0) or lon in (None, 0.0): + return ResolverOutput() + + country = normalize_country_text(query.country) or query.country + candidate = LocationCandidate( + latitude=float(lat), + longitude=float(lon), + display_name=query.name or "", + precision="precise", + confidence=1.0, + query="source_coordinates", + source=self._source, + source_note="Source record provided valid coordinates.", + matched_fields=("source_coordinates",), + needs_confirmation=False, + city=query.city, + region=query.region, + country=country, + matched_location_name=query.name, + location_verified_at=None, + suggested_registry_entry=None, + ) + return ResolverOutput(candidates=(candidate,)) diff --git a/backend/app/services/location/text.py b/backend/app/services/location/text.py new file mode 100644 index 00000000..3db7c2ed --- /dev/null +++ b/backend/app/services/location/text.py @@ -0,0 +1,41 @@ +"""Text-normalization helpers shared by every resolver.""" + +from __future__ import annotations + +import re +from typing import Any + +from app.core.countries import normalize_country + + +def parse_float(value: Any) -> float | None: + try: + if value in (None, ""): + return None + return float(value) + except (TypeError, ValueError): + return None + + +def coerce_str(value: Any) -> str: + if value in (None, ""): + return "" + return str(value).strip() + + +def normalize_text(value: Any) -> str: + if value in (None, ""): + return "" + normalized = str(value).casefold() + normalized = re.sub(r"[^a-z0-9一-鿿]+", " ", normalized) + return re.sub(r"\s+", " ", normalized).strip() + + +def normalize_country_text(value: Any) -> str: + normalized = normalize_country(value) + return normalized or coerce_str(value) + + +def city_key(city: Any) -> str: + text = coerce_str(city).split(",", 1)[0] + return normalize_text(text) diff --git a/backend/app/services/persistent_logs.py b/backend/app/services/persistent_logs.py new file mode 100644 index 00000000..838c9f7c --- /dev/null +++ b/backend/app/services/persistent_logs.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from typing import Any + +from app.core.logging import get_logger, sanitize_log_value +from app.core.request_context import get_request_id +from app.db.session import async_session_factory +from app.models.system_log import AuditLog, SystemLog + +logger = get_logger(__name__) + + +async def record_system_log( + *, + source: str, + level: str, + message: str, + service: str | None = None, + module: str | None = None, + event: str | None = None, + request_id: str | None = None, + trace_id: str | None = None, + user_id: int | None = None, + category: str | None = None, + context: dict[str, Any] | None = None, +) -> None: + try: + async with async_session_factory() as session: + session.add( + SystemLog( + source=source, + service=service, + module=module, + event=event, + level=level.lower(), + message=str(sanitize_log_value(message)), + request_id=request_id or get_request_id(), + trace_id=trace_id, + user_id=user_id, + category=category, + context=sanitize_log_value(context or {}), + ) + ) + await session.commit() + except Exception: + logger.exception_event( + "Failed to persist system log", + event="system_log.persist.failed", + context={"event_name": event, "source": source}, + ) + + +async def record_audit_log( + *, + action: str, + actor_id: int | None = None, + actor_name: str | None = None, + target_type: str | None = None, + target_id: str | None = None, + result: str | None = None, + request_id: str | None = None, + ip: str | None = None, + details: dict[str, Any] | None = None, +) -> None: + try: + async with async_session_factory() as session: + session.add( + AuditLog( + actor_id=actor_id, + actor_name=actor_name, + action=action, + target_type=target_type, + target_id=target_id, + result=result, + request_id=request_id or get_request_id(), + ip=ip, + details=sanitize_log_value(details or {}), + ) + ) + await session.commit() + except Exception: + logger.exception_event( + "Failed to persist audit log", + event="audit_log.persist.failed", + context={"action": action}, + ) diff --git a/backend/app/services/scheduler.py b/backend/app/services/scheduler.py index a16d84ff..d96ddf98 100644 --- a/backend/app/services/scheduler.py +++ b/backend/app/services/scheduler.py @@ -1,7 +1,6 @@ """Task Scheduler for running collection jobs.""" import asyncio -import logging from datetime import UTC, datetime, timedelta from typing import Any, Dict, Optional @@ -9,13 +8,19 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.triggers.interval import IntervalTrigger from sqlalchemy import select +from app.core.logging import get_logger from app.db.session import async_session_factory from app.core.time import to_iso8601_utc from app.models.datasource import DataSource from app.models.task import CollectionTask from app.services.collectors.registry import collector_registry +from app.services.datasource_connectivity import ( + build_builtin_connectivity_checksum, + get_builtin_effective_candidate, + save_connectivity_success, +) -logger = logging.getLogger(__name__) +logger = get_logger(__name__) scheduler = AsyncIOScheduler() RUNNING_TASK_GUARD_TIMEOUT_MINUTES = 90 @@ -54,7 +59,11 @@ async def _update_next_run_at(datasource: DataSource, session) -> None: async def _apply_datasource_schedule(datasource: DataSource, session) -> None: collector = collector_registry.get(datasource.source) if not collector: - logger.warning("Collector not found for datasource %s", datasource.source) + logger.warning_event( + "Collector not found for datasource", + event="collector.schedule.collector_missing", + context={"collector_name": datasource.source}, + ) return collector_registry.set_active(datasource.source, datasource.is_active) @@ -72,13 +81,17 @@ async def _apply_datasource_schedule(datasource: DataSource, session) -> None: replace_existing=True, kwargs={"collector_name": datasource.source}, ) - logger.info( - "Scheduled collector: %s (every %sm)", - datasource.source, - datasource.frequency_minutes, + logger.info_event( + "Scheduled collector", + event="collector.schedule.updated", + context={"collector_name": datasource.source, "frequency_minutes": datasource.frequency_minutes}, ) else: - logger.info("Collector disabled: %s", datasource.source) + logger.info_event( + "Collector disabled", + event="collector.schedule.disabled", + context={"collector_name": datasource.source}, + ) await _update_next_run_at(datasource, session) @@ -87,18 +100,30 @@ async def run_collector_task(collector_name: str): """Run a single collector task.""" collector = collector_registry.get(collector_name) if not collector: - logger.error("Collector not found: %s", collector_name) + logger.error_event( + "Collector not found", + event="collector.run.collector_missing", + context={"collector_name": collector_name}, + ) return async with async_session_factory() as db: result = await db.execute(select(DataSource).where(DataSource.source == collector_name)) datasource = result.scalar_one_or_none() if not datasource: - logger.error("Datasource not found for collector: %s", collector_name) + logger.error_event( + "Datasource not found for collector", + event="collector.run.datasource_missing", + context={"collector_name": collector_name}, + ) return if not datasource.is_active: - logger.info("Skipping disabled collector: %s", collector_name) + logger.info_event( + "Skipping disabled collector", + event="collector.run.skipped_disabled", + context={"collector_name": collector_name}, + ) return running_result = await db.execute( @@ -122,10 +147,10 @@ async def run_collector_task(collector_name: str): and (now - started_at) > timedelta(minutes=RUNNING_TASK_GUARD_TIMEOUT_MINUTES) ) if not is_stale: - logger.warning( - "Skipping collector %s trigger because task %s is already running", - collector_name, - existing_running.id, + logger.warning_event( + "Skipping collector trigger because task is already running", + event="collector.run.skipped_already_running", + context={"collector_name": collector_name, "task_id": existing_running.id}, ) return @@ -143,31 +168,64 @@ async def run_collector_task(collector_name: str): else stale_reason ) await db.commit() - logger.warning( - "Marked stale running task %s as failed before rerun of %s", - existing_running.id, - collector_name, + logger.warning_event( + "Marked stale running task as failed before rerun", + event="collector.run.stale_task_failed", + context={"collector_name": collector_name, "task_id": existing_running.id}, ) try: collector._datasource_id = datasource.id - logger.info("Running collector: %s (datasource_id=%s)", collector_name, datasource.id) + logger.info_event( + "Running collector", + event="collector.run.started", + context={"collector_name": collector_name, "datasource_id": datasource.id}, + ) task_result = await collector.run(db) datasource.last_run_at = datetime.now(UTC) datasource.last_status = task_result.get("status") + if datasource.last_status == "success": + effective_candidate = await get_builtin_effective_candidate(db, datasource.source) + checksum, _credential_context = await build_builtin_connectivity_checksum( + datasource.source, + effective_candidate["endpoint"], + effective_candidate["auth_type"], + effective_candidate["headers"], + effective_candidate["config"], + db, + ) + await save_connectivity_success( + db, + datasource.source, + checksum, + {"status_code": None}, + connected_by="collection", + ) await _update_next_run_at(datasource, db) - logger.info("Collector %s completed: %s", collector_name, task_result) + logger.info_event( + "Collector completed", + event="collector.run.completed", + context={"collector_name": collector_name, "datasource_id": datasource.id, "result": task_result}, + ) except asyncio.CancelledError: datasource.last_run_at = datetime.now(UTC) datasource.last_status = "cancelled" await db.commit() - logger.warning("Collector %s cancelled by operator", collector_name) + logger.warning_event( + "Collector cancelled by operator", + event="collector.run.cancelled", + context={"collector_name": collector_name, "datasource_id": datasource.id}, + ) raise except Exception as exc: datasource.last_run_at = datetime.now(UTC) datasource.last_status = "failed" await db.commit() - logger.exception("Collector %s failed: %s", collector_name, exc) + logger.exception_event( + "Collector failed", + event="collector.run.failed", + context={"collector_name": collector_name, "datasource_id": datasource.id, "error": str(exc)}, + ) async def cleanup_stale_running_tasks(max_age_hours: int = 2) -> int: @@ -194,7 +252,11 @@ async def cleanup_stale_running_tasks(max_age_hours: int = 2) -> int: if stale_tasks: await db.commit() - logger.warning("Cleaned up %s stale running collection task(s)", len(stale_tasks)) + logger.warning_event( + "Cleaned up stale running collection tasks", + event="collector.cleanup.stale_tasks_cleaned", + context={"count": len(stale_tasks)}, + ) return len(stale_tasks) @@ -203,14 +265,14 @@ def start_scheduler() -> None: """Start the scheduler.""" if not scheduler.running: scheduler.start() - logger.info("Scheduler started") + logger.info_event("Scheduler started", event="scheduler.started") def stop_scheduler() -> None: """Stop the scheduler.""" if scheduler.running: scheduler.shutdown(wait=False) - logger.info("Scheduler stopped") + logger.info_event("Scheduler stopped", event="scheduler.stopped") async def sync_scheduler_with_datasources() -> None: @@ -271,12 +333,20 @@ def run_collector_now(collector_name: str) -> bool: """Run a collector immediately (not scheduled).""" collector = collector_registry.get(collector_name) if not collector: - logger.error("Collector not found: %s", collector_name) + logger.error_event( + "Collector not found", + event="collector.trigger.collector_missing", + context={"collector_name": collector_name}, + ) return False existing_task = get_running_collector_task(collector_name) if existing_task is not None and not existing_task.done(): - logger.warning("Collector %s is already running in-memory; skipping duplicate trigger", collector_name) + logger.warning_event( + "Collector is already running in-memory; skipping duplicate trigger", + event="collector.trigger.skipped_already_running", + context={"collector_name": collector_name}, + ) return False try: @@ -289,10 +359,18 @@ def run_collector_now(collector_name: str) -> bool: RUNNING_COLLECTOR_TASKS.pop(collector_name, None) task.add_done_callback(_cleanup_task) - logger.info("Triggered collector: %s", collector_name) + logger.info_event( + "Triggered collector", + event="collector.trigger.started", + context={"collector_name": collector_name}, + ) return True except Exception as exc: - logger.error("Failed to trigger collector %s: %s", collector_name, exc) + logger.error_event( + "Failed to trigger collector", + event="collector.trigger.failed", + context={"collector_name": collector_name, "error": str(exc)}, + ) return False diff --git a/backend/app/services/system_control.py b/backend/app/services/system_control.py index 7176177e..45de6a3e 100644 --- a/backend/app/services/system_control.py +++ b/backend/app/services/system_control.py @@ -19,6 +19,10 @@ ALLOWED_ACTIONS: dict[str, dict[str, Any]] = { "command": ["./planet.sh", "restart", "-b"], "recovery_mode": "backend", }, + "restart-frontend": { + "command": ["./planet.sh", "restart", "-f"], + "recovery_mode": "frontend", + }, "restart-ai-provider": { "command": ["./planet.sh", "restart", "-a"], "recovery_mode": "ai-provider", diff --git a/backend/app/services/system_logs.py b/backend/app/services/system_logs.py new file mode 100644 index 00000000..136cf373 --- /dev/null +++ b/backend/app/services/system_logs.py @@ -0,0 +1,532 @@ +from __future__ import annotations + +import json +import re +import shutil +import subprocess + +from collections import Counter, deque +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from app.core.security import redis_client + +DEFAULT_LOG_LINE_LIMIT = 200 +MAX_LOG_LINE_LIMIT = 1000 +BUFFER_LOG_LIMIT = 1000 +BUFFER_LOG_TTL_SECONDS = 7 * 24 * 60 * 60 +LOG_BUFFER_KEY_PREFIX = "planet:system_logs" + +LOG_LEVEL_ERROR = "error" +LOG_LEVEL_WARNING = "warning" +LOG_LEVEL_INFO = "info" +LOG_LEVEL_DEBUG = "debug" +LOG_LEVEL_ALL = "all" + +SUPPORTED_LOG_LEVELS = { + LOG_LEVEL_ALL, + LOG_LEVEL_ERROR, + LOG_LEVEL_WARNING, + LOG_LEVEL_INFO, + LOG_LEVEL_DEBUG, +} + +LOG_LEVEL_ALIASES = { + "warn": LOG_LEVEL_WARNING, + "warning": LOG_LEVEL_WARNING, + "err": LOG_LEVEL_ERROR, + "error": LOG_LEVEL_ERROR, + "info": LOG_LEVEL_INFO, + "information": LOG_LEVEL_INFO, + "debug": LOG_LEVEL_DEBUG, + "trace": LOG_LEVEL_DEBUG, + "critical": LOG_LEVEL_ERROR, + "fatal": LOG_LEVEL_ERROR, +} + +TIMESTAMP_FORMATS = ( + "%Y-%m-%d %H:%M:%S.%f", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", +) + +LEVEL_PATTERNS = ( + ("CRITICAL", LOG_LEVEL_ERROR), + ("FATAL", LOG_LEVEL_ERROR), + ("ERROR", LOG_LEVEL_ERROR), + ("WARNING", LOG_LEVEL_WARNING), + ("WARN", LOG_LEVEL_WARNING), + ("INFO", LOG_LEVEL_INFO), + ("DEBUG", LOG_LEVEL_DEBUG), + ("TRACE", LOG_LEVEL_DEBUG), +) + +LEADING_LEVEL_PATTERN = re.compile( + r"^\s*(?:\[[^\]]+\]\s*)?(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG|TRACE)\b[:\s-]*", + re.IGNORECASE, +) +EMBEDDED_LEVEL_PATTERN = re.compile( + r"\b(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG|TRACE)\b", + re.IGNORECASE, +) +CONTROL_CHAR_PATTERN = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") + + +@dataclass(frozen=True) +class LogSource: + source_id: str + name: str + kind: str + location: str + description: str + category: str + status: str = "ok" + buffer_key: str | None = None + container_name: str | None = None + + +@dataclass +class StructuredLogEntry: + timestamp: datetime | None + level: str | None + display_line: str + raw_line: str + search_text: str + + +@dataclass +class DailyLogMarker: + date_token: str + total: int + dominant_level: str + + +LOG_SOURCES: dict[str, LogSource] = { + "backend": LogSource( + source_id="backend", + name="后端服务", + kind="file", + location="/tmp/planet_backend.log", + description="FastAPI 后端、调度器和采集任务共享日志。", + category="service", + ), + "frontend": LogSource( + source_id="frontend", + name="前端开发服务", + kind="file", + location="/tmp/planet_frontend.log", + description="控制台与 Earth 前端开发服务输出。", + category="service", + ), + "ai-provider": LogSource( + source_id="ai-provider", + name="AI Provider", + kind="docker", + location="docker://planet_aiprovider", + description="AI Provider 容器实时输出日志。", + category="service", + container_name="planet_aiprovider", + ), + "earth-client": LogSource( + source_id="earth-client", + name="Earth 浏览器端", + kind="buffer", + location="redis://planet:system_logs:earth-client", + description="Earth 浏览器端上报的运行时错误与关键业务日志。", + category="client", + buffer_key=f"{LOG_BUFFER_KEY_PREFIX}:earth-client", + ), +} + + +def normalize_log_level(level: str | None) -> str: + if level is None: + return LOG_LEVEL_ALL + normalized = str(level).strip().lower() + if normalized in {"", LOG_LEVEL_ALL}: + return LOG_LEVEL_ALL + return LOG_LEVEL_ALIASES.get(normalized, LOG_LEVEL_ALL) + + +def normalize_log_levels(level: str | None = None, levels: str | None = None) -> tuple[str, ...]: + normalized_levels: list[str] = [] + if levels: + for item in str(levels).split(","): + normalized = normalize_log_level(item) + if normalized != LOG_LEVEL_ALL and normalized not in normalized_levels: + normalized_levels.append(normalized) + normalized_level = normalize_log_level(level) + if normalized_level != LOG_LEVEL_ALL and normalized_level not in normalized_levels: + normalized_levels.append(normalized_level) + return tuple(normalized_levels) + + +def get_source_status(source: LogSource) -> str: + if source.kind == "file": + path = Path(source.location) + if not path.exists(): + return "missing" + return "ok" if path.stat().st_size > 0 else "empty" + if source.kind == "docker": + return "ok" if shutil.which("docker") else "docker_unavailable" + if source.kind == "buffer": + if not source.buffer_key: + return "source_unavailable" + try: + return "ok" if redis_client.llen(source.buffer_key) > 0 else "empty" + except Exception: + return "source_unavailable" + return "source_unavailable" + + +def list_log_sources() -> list[dict[str, str]]: + items: list[dict[str, str]] = [] + for source in LOG_SOURCES.values(): + items.append( + { + "source_id": source.source_id, + "name": source.name, + "kind": source.kind, + "location": source.location, + "description": source.description, + "category": source.category, + "status": get_source_status(source), + } + ) + return items + + +def get_buffer_log_key(source_id: str) -> str: + return f"{LOG_BUFFER_KEY_PREFIX}:{source_id}" + + +def append_buffer_log( + source_id: str, + *, + level: str, + message: str, + context: dict[str, Any] | None = None, +) -> None: + payload = { + "timestamp": datetime.now(tz=UTC).isoformat(), + "level": normalize_log_level(level), + "message": message, + "context": context or {}, + } + buffer_key = get_buffer_log_key(source_id) + redis_client.rpush(buffer_key, json.dumps(payload, ensure_ascii=False)) + redis_client.ltrim(buffer_key, -BUFFER_LOG_LIMIT, -1) + redis_client.expire(buffer_key, BUFFER_LOG_TTL_SECONDS) + + +def parse_timestamp(raw_value: str | None) -> datetime | None: + if not raw_value: + return None + candidate = str(raw_value).strip() + if not candidate: + return None + candidate = candidate.replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(candidate) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + except ValueError: + pass + for fmt in TIMESTAMP_FORMATS: + try: + return datetime.strptime(candidate, fmt).replace(tzinfo=UTC) + except ValueError: + continue + return None + + +def parse_prefixed_timestamp(line: str) -> tuple[datetime | None, str]: + stripped = line.strip() + if not stripped: + return None, "" + for prefix_length in (35, 32, 29, 26, 23, 19): + if len(stripped) < prefix_length: + continue + prefix = stripped[:prefix_length] + timestamp = parse_timestamp(prefix) + if timestamp is not None: + return timestamp, stripped[prefix_length:].lstrip() + first_token = stripped.split(maxsplit=1)[0] + timestamp = parse_timestamp(first_token) + if timestamp is not None: + remainder = stripped[len(first_token):].lstrip() + return timestamp, remainder + return None, stripped + + +def infer_log_level_from_text(text: str, *, allow_embedded: bool = True) -> str | None: + leading_match = LEADING_LEVEL_PATTERN.match(text) + if leading_match: + return normalize_log_level(leading_match.group(1)) + + if allow_embedded: + embedded_match = EMBEDDED_LEVEL_PATTERN.search(text) + if embedded_match: + return normalize_log_level(embedded_match.group(1)) + upper_text = text.upper() + for pattern, normalized in LEVEL_PATTERNS: + if f"{pattern}:" in upper_text or f"{pattern} " in upper_text: + return normalized + return None + + +def build_display_line(timestamp: datetime | None, level: str | None, message: str) -> str: + message_part = message.strip() if message else "" + parts = [] + if timestamp is not None: + parts.append(timestamp.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S")) + if level: + parts.append(level.upper()) + if message_part: + parts.append(message_part) + return " ".join(parts).strip() + + +def sanitize_text_log_line(line: str) -> str: + return CONTROL_CHAR_PATTERN.sub("", line) + + +def parse_text_log_entry(line: str) -> StructuredLogEntry: + sanitized_line = sanitize_text_log_line(line).rstrip("\n") + timestamp, remainder = parse_prefixed_timestamp(sanitized_line) + level = infer_log_level_from_text(remainder or sanitized_line, allow_embedded=False) + display_line = sanitized_line + return StructuredLogEntry( + timestamp=timestamp, + level=level, + display_line=display_line, + raw_line=display_line, + search_text=display_line.lower(), + ) + + +def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry: + timestamp = parse_timestamp(str(payload.get("timestamp", "")).strip()) + level = normalize_log_level(payload.get("level")) + if level == LOG_LEVEL_ALL: + level = None + message = str(payload.get("message", "")).strip() + context = payload.get("context") + context_map = context if isinstance(context, dict) else {} + context_fragments = [] + for key in ("category", "module", "url", "detail"): + value = str(context_map.get(key, "")).strip() + if value: + context_fragments.append(f"{key}={value}") + message_with_context = " | ".join([message, *context_fragments]) if context_fragments else message + display_line = build_display_line(timestamp, level, message_with_context) + search_text = " ".join( + [ + message, + json.dumps(context_map, ensure_ascii=False, sort_keys=True), + display_line, + ] + ).lower() + return StructuredLogEntry( + timestamp=timestamp, + level=level, + display_line=display_line, + raw_line=json.dumps(payload, ensure_ascii=False, sort_keys=True), + search_text=search_text, + ) + + +def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: + path = Path(source.location) + if not path.exists(): + return [] + with path.open("r", encoding="utf-8", errors="replace") as handle: + recent_lines = deque(handle, maxlen=scan_limit) + return [ + parse_text_log_entry(line) + for line in recent_lines + if sanitize_text_log_line(line).strip() + ] + + +def read_docker_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: + if not shutil.which("docker") or not source.container_name: + return [] + try: + completed = subprocess.run( + [ + "docker", + "logs", + "--timestamps", + "--tail", + str(scan_limit), + source.container_name, + ], + capture_output=True, + text=True, + check=False, + ) + except OSError: + return [] + if completed.returncode != 0: + return [] + return [ + parse_text_log_entry(line) + for line in completed.stdout.splitlines() + if line.strip() + ] + + +def read_buffer_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: + if not source.buffer_key: + return [] + try: + raw_items = redis_client.lrange(source.buffer_key, -scan_limit, -1) + except Exception: + return [] + entries: list[StructuredLogEntry] = [] + for raw_item in raw_items: + try: + payload = json.loads(raw_item) + except json.JSONDecodeError: + entries.append(parse_text_log_entry(str(raw_item))) + continue + if isinstance(payload, dict): + entries.append(build_buffer_entry(payload)) + else: + entries.append(parse_text_log_entry(str(raw_item))) + return entries + + +def read_source_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: + if source.kind == "file": + return read_file_entries(source, scan_limit) + if source.kind == "docker": + return read_docker_entries(source, scan_limit) + if source.kind == "buffer": + return read_buffer_entries(source, scan_limit) + return [] + + +def matches_levels(entry: StructuredLogEntry, selected_levels: tuple[str, ...]) -> bool: + if not selected_levels: + return True + return entry.level in selected_levels + + +def matches_date_range( + entry: StructuredLogEntry, + start_date: str | None, + end_date: str | None, +) -> bool: + if not start_date and not end_date: + return True + if entry.timestamp is None: + return False + date_token = entry.timestamp.astimezone(UTC).date().isoformat() + if start_date and date_token < start_date: + return False + if end_date and date_token > end_date: + return False + return True + + +def matches_search(entry: StructuredLogEntry, search: str | None) -> bool: + if search is None: + return True + query = search.strip().lower() + if not query: + return True + return query in entry.search_text + + +def build_daily_log_markers(entries: list[StructuredLogEntry]) -> list[dict[str, Any]]: + grouped: dict[str, list[StructuredLogEntry]] = {} + for entry in entries: + if entry.timestamp is None: + continue + date_token = entry.timestamp.astimezone(UTC).date().isoformat() + grouped.setdefault(date_token, []).append(entry) + + markers: list[DailyLogMarker] = [] + for date_token, group in sorted(grouped.items()): + level_counts = Counter( + entry.level + for entry in group + if entry.level in SUPPORTED_LOG_LEVELS and entry.level != LOG_LEVEL_ALL + ) + dominant_level = LOG_LEVEL_INFO + if level_counts: + dominant_level = sorted( + level_counts.items(), + key=lambda item: ( + -item[1], + ("error", "warning", "info", "debug").index(item[0]), + ), + )[0][0] + markers.append( + DailyLogMarker( + date_token=date_token, + total=len(group), + dominant_level=dominant_level, + ) + ) + return [marker.__dict__ for marker in markers] + + +def read_log_snapshot( + source_id: str, + limit: int, + *, + level: str = LOG_LEVEL_ALL, + levels: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + search: str | None = None, +) -> dict[str, Any] | None: + source = LOG_SOURCES.get(source_id) + if source is None: + return None + + selected_levels = normalize_log_levels(level, levels) + search_query = (search or "").strip() + scan_limit = max(min(MAX_LOG_LINE_LIMIT * 5, 5000), limit * 5, BUFFER_LOG_LIMIT if source.kind == "buffer" else 1000) + all_entries = read_source_entries(source, scan_limit) + marker_entries = [ + entry + for entry in all_entries + if matches_levels(entry, selected_levels) and matches_search(entry, search_query) + ] + filtered_entries = [ + entry + for entry in marker_entries + if matches_date_range(entry, start_date, end_date) + ] + visible_entries = filtered_entries[-limit:] + + compatibility_level = selected_levels[0] if len(selected_levels) == 1 else LOG_LEVEL_ALL + return { + "source_id": source.source_id, + "name": source.name, + "kind": source.kind, + "location": source.location, + "description": source.description, + "category": source.category, + "status": get_source_status(source), + "level": compatibility_level, + "selected_levels": list(selected_levels), + "search_query": search_query, + "available_levels": [ + LOG_LEVEL_ALL, + LOG_LEVEL_ERROR, + LOG_LEVEL_WARNING, + LOG_LEVEL_INFO, + LOG_LEVEL_DEBUG, + ], + "daily_markers": build_daily_log_markers(marker_entries), + "line_limit": limit, + "line_count": len(visible_entries), + "lines": [entry.display_line for entry in visible_entries], + } diff --git a/backend/app/services/vessel_aggregation_strategy.py b/backend/app/services/vessel_aggregation_strategy.py new file mode 100644 index 00000000..33c516a2 --- /dev/null +++ b/backend/app/services/vessel_aggregation_strategy.py @@ -0,0 +1,198 @@ +"""Persistence + validation for the v4 vessel_ais aggregation strategy.""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.system_setting import SystemSetting + +VESSEL_AGGREGATION_STRATEGY_CATEGORY = "vessel_aggregation_strategy" + +DYNAMIC_FIELDS: tuple[str, ...] = ("lat", "lon", "sog", "cog", "heading", "nav_status") +STATIC_FIELDS: tuple[str, ...] = ( + "name", + "callsign", + "imo", + "flag", + "vessel_type", + "vessel_type_name", + "length", + "width", + "draught", +) +ALLOWED_FIELDS: frozenset[str] = frozenset(DYNAMIC_FIELDS + STATIC_FIELDS) +ALLOWED_DYNAMIC_MODES: frozenset[str] = frozenset({"newest"}) +ALLOWED_STATIC_MODES: frozenset[str] = frozenset({"source_priority", "non_empty", "newest", "locked"}) +ALLOWED_LOCKED_DYNAMIC_MODES: frozenset[str] = frozenset({"newest", "source_priority", "locked"}) + + +DEFAULT_STRATEGY: dict[str, Any] = { + "version": 1, + "vessel_ais": { + "source_priority": ["aisstream_vessels", "barentswatch_vessels"], + "field_rules": {}, + "freshness": { + "realtime_stream_seconds": 900, + "polling_seconds": 3600, + }, + "allow_dynamic_lock": False, + }, +} + + +class StrategyValidationError(ValueError): + """Raised when a saved strategy payload is malformed.""" + + +def _coerce_str_list(value: Any, *, label: str) -> list[str]: + if value is None: + return [] + if not isinstance(value, list): + raise StrategyValidationError(f"{label} must be a list of source names") + out: list[str] = [] + for item in value: + if not isinstance(item, str) or not item.strip(): + raise StrategyValidationError(f"{label} entries must be non-empty strings") + out.append(item.strip()) + return out + + +def validate_strategy(payload: dict[str, Any]) -> dict[str, Any]: + """Validate and normalize a strategy payload. Raise StrategyValidationError on issues.""" + + if not isinstance(payload, dict): + raise StrategyValidationError("strategy payload must be an object") + + vessel_ais = payload.get("vessel_ais") + if not isinstance(vessel_ais, dict): + raise StrategyValidationError("strategy.vessel_ais is required and must be an object") + + allow_dynamic_lock = bool(vessel_ais.get("allow_dynamic_lock", False)) + source_priority = _coerce_str_list( + vessel_ais.get("source_priority"), + label="vessel_ais.source_priority", + ) + + raw_rules = vessel_ais.get("field_rules") or {} + if not isinstance(raw_rules, dict): + raise StrategyValidationError("vessel_ais.field_rules must be an object") + field_rules: dict[str, dict[str, Any]] = {} + for field, rule in raw_rules.items(): + if field not in ALLOWED_FIELDS: + raise StrategyValidationError(f"unknown vessel_ais field: {field}") + if not isinstance(rule, dict): + raise StrategyValidationError(f"field_rules.{field} must be an object") + mode = str(rule.get("mode") or "").strip() + if not mode: + raise StrategyValidationError(f"field_rules.{field}.mode is required") + is_dynamic = field in DYNAMIC_FIELDS + if is_dynamic: + allowed_modes = ALLOWED_LOCKED_DYNAMIC_MODES if allow_dynamic_lock else ALLOWED_DYNAMIC_MODES + if mode not in allowed_modes: + if not allow_dynamic_lock: + raise StrategyValidationError( + f"field_rules.{field}.mode='{mode}' requires allow_dynamic_lock=true" + ) + raise StrategyValidationError( + f"field_rules.{field}.mode must be one of {sorted(allowed_modes)}" + ) + else: + if mode not in ALLOWED_STATIC_MODES: + raise StrategyValidationError( + f"field_rules.{field}.mode must be one of {sorted(ALLOWED_STATIC_MODES)}" + ) + normalized_rule: dict[str, Any] = {"mode": mode} + rule_priority = rule.get("source_priority") + if rule_priority is not None: + normalized_rule["source_priority"] = _coerce_str_list( + rule_priority, + label=f"field_rules.{field}.source_priority", + ) + if mode == "locked": + locked_source = rule.get("locked_source") + if not isinstance(locked_source, str) or not locked_source.strip(): + raise StrategyValidationError( + f"field_rules.{field}.locked_source must be a non-empty string when mode=locked" + ) + normalized_rule["locked_source"] = locked_source.strip() + field_rules[field] = normalized_rule + + raw_freshness = vessel_ais.get("freshness") or {} + if not isinstance(raw_freshness, dict): + raise StrategyValidationError("vessel_ais.freshness must be an object") + freshness: dict[str, int] = {} + for key in ("realtime_stream_seconds", "polling_seconds"): + value = raw_freshness.get(key, DEFAULT_STRATEGY["vessel_ais"]["freshness"][key]) + try: + seconds = int(value) + except (TypeError, ValueError) as exc: + raise StrategyValidationError(f"freshness.{key} must be an integer") from exc + if seconds < 0: + raise StrategyValidationError(f"freshness.{key} must be non-negative") + freshness[key] = seconds + + return { + "version": int(payload.get("version") or 0) + 1, + "vessel_ais": { + "source_priority": source_priority, + "field_rules": field_rules, + "freshness": freshness, + "allow_dynamic_lock": allow_dynamic_lock, + }, + } + + +async def _select_setting(db: AsyncSession) -> SystemSetting | None: + result = await db.execute( + select(SystemSetting).where(SystemSetting.category == VESSEL_AGGREGATION_STRATEGY_CATEGORY) + ) + return result.scalar_one_or_none() + + +def _current_version(setting: SystemSetting | None) -> int: + if setting is None: + return 0 + payload = setting.payload or {} + return int(payload.get("version") or 0) + + +async def load_strategy(db: AsyncSession) -> dict[str, Any]: + setting = await _select_setting(db) + if setting is None or not isinstance(setting.payload, dict): + return DEFAULT_STRATEGY + payload = setting.payload + if "vessel_ais" not in payload: + return DEFAULT_STRATEGY + return payload + + +async def save_strategy(db: AsyncSession, payload: dict[str, Any]) -> dict[str, Any]: + """Validate + persist; bumps version automatically.""" + + existing = await _select_setting(db) + incoming = dict(payload) + incoming.setdefault("version", _current_version(existing)) + validated = validate_strategy(incoming) + + if existing is None: + existing = SystemSetting(category=VESSEL_AGGREGATION_STRATEGY_CATEGORY, payload=validated) + db.add(existing) + else: + existing.payload = validated + await db.commit() + return validated + + +async def reset_strategy(db: AsyncSession) -> dict[str, Any]: + existing = await _select_setting(db) + payload = {**DEFAULT_STRATEGY, "version": _current_version(existing) + 1} + if existing is None: + existing = SystemSetting(category=VESSEL_AGGREGATION_STRATEGY_CATEGORY, payload=payload) + db.add(existing) + else: + existing.payload = payload + await db.commit() + return payload diff --git a/backend/app/services/vessel_ais_aggregation.py b/backend/app/services/vessel_ais_aggregation.py new file mode 100644 index 00000000..c8d45904 --- /dev/null +++ b/backend/app/services/vessel_ais_aggregation.py @@ -0,0 +1,698 @@ +"""AIS raw observation and aggregation support for vessel collectors.""" + +from datetime import UTC, datetime, timedelta +from hashlib import sha256 +import json +from typing import Any, Iterable + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth +from app.services.vessel_aggregation_strategy import ( + DEFAULT_STRATEGY, + load_strategy, +) +from app.services.vessel_types import normalize_vessel_type_name + +VESSEL_AIS_SCHEMA = "vessel_ais" +DEFAULT_AGGREGATION_WINDOW_HOURS = 24 +BARENTSWATCH_DELIVERY_MODE = "polling" +BARENTSWATCH_TRANSPORT = "http" +AISSTREAM_DELIVERY_MODE = "realtime_stream" +AISSTREAM_TRANSPORT = "websocket" +DELIVERY_MODE_PRIORITY = { + "realtime_stream": 40, + "batch_stream": 30, + "polling": 20, + "snapshot": 10, +} +DYNAMIC_FIELDS = ("lat", "lon", "sog", "cog", "heading", "nav_status") +CONFLICT_FIELDS = ( + "name", + "callsign", + "imo", + "flag", + "vessel_type", + "vessel_type_name", + "length", + "width", + "draught", +) + + +def _json_default(value: Any) -> Any: + if isinstance(value, datetime): + return value.astimezone(UTC).isoformat() + return str(value) + + +def _stable_payload(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=_json_default) + + +def _jsonable(value: Any) -> Any: + if isinstance(value, datetime): + return value.astimezone(UTC).isoformat() + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, list): + return [_jsonable(item) for item in value] + return value + + +def _coerce_datetime(value: Any) -> datetime | None: + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=UTC) + if isinstance(value, (int, float)): + timestamp = float(value) + if timestamp > 10_000_000_000: + timestamp /= 1000 + return datetime.fromtimestamp(timestamp, UTC) + if isinstance(value, str) and value: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + except ValueError: + return None + return None + + +def build_observation_hash( + *, + source: str, + entity_key: str, + message_type: str | None, + observed_at: datetime, + normalized_payload: dict[str, Any], + source_message_id: str | None = None, +) -> str: + """Build a deterministic idempotency key for one source-level AIS observation.""" + + if source_message_id: + basis = { + "source": source, + "entity_key": entity_key, + "source_message_id": source_message_id, + } + else: + basis = { + "source": source, + "entity_key": entity_key, + "message_type": message_type, + "observed_at": observed_at.astimezone(UTC).isoformat(), + "payload": normalized_payload, + } + return sha256(_stable_payload(basis).encode("utf-8")).hexdigest() + + +def build_field_conflict_candidates( + observations: Iterable[AISRawObservation], + fields: Iterable[str] = CONFLICT_FIELDS, +) -> list[dict[str, Any]]: + """Return current field disagreements from raw observations without mutating state.""" + + candidates_by_field: dict[str, dict[str, Any]] = {} + for observation in observations: + payload = observation.normalized_payload or {} + for field in fields: + value = payload.get(field) + if value in (None, ""): + continue + field_candidates = candidates_by_field.setdefault(field, {}) + field_candidates[observation.source] = value + + conflicts = [] + for field, candidates in sorted(candidates_by_field.items()): + unique_values = {_stable_payload(value) for value in candidates.values()} + if len(unique_values) <= 1: + continue + conflicts.append( + { + "field": field, + "candidates": candidates, + "status": "candidate", + } + ) + return conflicts + + +def _payload_value(payload: dict[str, Any], field: str) -> Any: + value = payload.get(field) + return None if value in (None, "") else value + + +def _clean_text(value: Any) -> str | None: + if value in (None, ""): + return None + text = str(value).strip() + return text or None + + +def _raw_metadata_value(observation: AISRawObservation, field: str) -> Any: + raw_payload = observation.raw_payload or {} + metadata = raw_payload.get("MetaData") if isinstance(raw_payload, dict) else None + if not isinstance(metadata, dict): + return None + if field == "name": + return _clean_text(metadata.get("ShipName") or metadata.get("ship_name") or metadata.get("name")) + return None + + +def _delivery_priority(observation: AISRawObservation) -> int: + return DELIVERY_MODE_PRIORITY.get(str(observation.delivery_mode or ""), 0) + + +def _has_valid_position(payload: dict[str, Any]) -> bool: + try: + lat = float(payload.get("lat")) + lon = float(payload.get("lon")) + except (TypeError, ValueError): + return False + return -90 <= lat <= 90 and -180 <= lon <= 180 + + +def _is_future_observation(observation: AISRawObservation, now: datetime) -> bool: + return observation.observed_at > now + + +def _strategy_source_rank( + source: str, + strategy: dict[str, Any], +) -> int: + priority = (strategy.get("vessel_ais") or {}).get("source_priority") or [] + if source in priority: + return len(priority) - priority.index(source) + return 0 + + +def _is_stream_stale( + observation: AISRawObservation, + *, + now: datetime, + strategy: dict[str, Any], +) -> bool: + delivery_mode = str(observation.delivery_mode or "") + freshness = (strategy.get("vessel_ais") or {}).get("freshness") or {} + if delivery_mode == "realtime_stream": + window = int(freshness.get("realtime_stream_seconds", 0) or 0) + else: + window = int(freshness.get("polling_seconds", 0) or 0) + if window <= 0: + return False + return (now - observation.observed_at).total_seconds() > window + + +def _select_position_observation( + observations: list[AISRawObservation], + *, + now: datetime, + strategy: dict[str, Any] | None = None, +) -> tuple[AISRawObservation | None, list[str]]: + strategy = strategy or DEFAULT_STRATEGY + rejected_flags: list[str] = [] + fresh_candidates: list[AISRawObservation] = [] + stale_candidates: list[AISRawObservation] = [] + for observation in observations: + payload = observation.normalized_payload or {} + if not _has_valid_position(payload): + rejected_flags.append("invalid_position") + continue + if _is_future_observation(observation, now): + rejected_flags.append("future_timestamp") + continue + if _is_stream_stale(observation, now=now, strategy=strategy): + stale_candidates.append(observation) + rejected_flags.append("freshness_fallback") + continue + fresh_candidates.append(observation) + + candidates = fresh_candidates or stale_candidates + if not candidates: + return None, sorted(set(rejected_flags)) + + candidates.sort( + key=lambda item: ( + item.observed_at, + _delivery_priority(item), + _strategy_source_rank(item.source, strategy), + item.collected_at, + item.id or 0, + ), + reverse=True, + ) + return candidates[0], sorted(set(rejected_flags)) + + +def _select_static_field( + observations: list[AISRawObservation], + field: str, + strategy: dict[str, Any] | None = None, +) -> tuple[Any, str | None, str | None]: + strategy = strategy or DEFAULT_STRATEGY + candidates = [] + for observation in observations: + value = _payload_value(observation.normalized_payload or {}, field) + if value is None: + value = _raw_metadata_value(observation, field) + if value is None: + continue + candidates.append((observation, value)) + + if not candidates: + return None, None, None + + field_rules = (strategy.get("vessel_ais") or {}).get("field_rules") or {} + rule = field_rules.get(field) or {"mode": "source_priority"} + mode = rule.get("mode") + + if mode == "locked": + locked_source = rule.get("locked_source") + for observation, value in candidates: + if observation.source == locked_source: + return value, observation.source, "locked" + + if mode in ("source_priority", "locked"): + priority = rule.get("source_priority") or (strategy.get("vessel_ais") or {}).get("source_priority") or [] + ranked = sorted( + candidates, + key=lambda item: ( + priority.index(item[0].source) if item[0].source in priority else len(priority) + 1, + -_delivery_priority(item[0]), + -(item[0].observed_at.timestamp() if item[0].observed_at else 0), + ), + ) + observation, value = ranked[0] + return value, observation.source, "source_priority" + + if mode == "newest": + ranked = sorted( + candidates, + key=lambda item: (item[0].observed_at, _delivery_priority(item[0]), item[0].id or 0), + reverse=True, + ) + observation, value = ranked[0] + return value, observation.source, "newest_observation" + + # default / non_empty: prefer delivery mode priority, then newest + candidates.sort( + key=lambda item: ( + _delivery_priority(item[0]), + item[0].observed_at, + item[0].collected_at, + item[0].id or 0, + ), + reverse=True, + ) + selected_observation, selected_value = candidates[0] + unique_values = {_stable_payload(value) for _, value in candidates} + reason = "delivery_mode_priority" if len(unique_values) > 1 else "non_empty_priority" + return selected_value, selected_observation.source, reason + + +def _build_source_summary(observations: list[AISRawObservation]) -> dict[str, dict[str, Any]]: + summary: dict[str, dict[str, Any]] = {} + for observation in observations: + source_summary = summary.setdefault( + observation.source, + { + "observation_count": 0, + "latest_observed_at": None, + "delivery_mode": observation.delivery_mode, + "transport": observation.transport, + "message_types": [], + }, + ) + source_summary["observation_count"] += 1 + latest_observed_at = source_summary["latest_observed_at"] + if latest_observed_at is None or observation.observed_at > latest_observed_at: + source_summary["latest_observed_at"] = observation.observed_at + if observation.message_type and observation.message_type not in source_summary["message_types"]: + source_summary["message_types"].append(observation.message_type) + return summary + + +def _build_aggregated_vessel( + entity_key: str, + observations: list[AISRawObservation], + *, + now: datetime, + strategy: dict[str, Any] | None = None, +) -> dict[str, Any] | None: + strategy = strategy or DEFAULT_STRATEGY + position_observation, rejected_flags = _select_position_observation( + observations, now=now, strategy=strategy + ) + if position_observation is None: + return None + + payload = position_observation.normalized_payload or {} + mmsi = int(entity_key) + result: dict[str, Any] = { + "mmsi": mmsi, + "lat": float(payload["lat"]), + "lon": float(payload["lon"]), + "received_at": position_observation.observed_at, + "field_sources": {}, + "selected_reasons": {}, + "source_summary": _build_source_summary(observations), + "quality_flags": sorted( + set((position_observation.quality_flags or []) + rejected_flags) + ), + "aggregation_strategy_version": int(strategy.get("version") or 0), + } + + for field in DYNAMIC_FIELDS: + value = _payload_value(payload, field) + if field in ("lat", "lon") or value is not None: + result[field] = value + result["field_sources"][field] = position_observation.source + result["selected_reasons"][field] = "newest_observation" + + for field in CONFLICT_FIELDS: + selected_value, selected_source, reason = _select_static_field( + observations, field, strategy=strategy + ) + if selected_value is None: + continue + result[field] = selected_value + result["field_sources"][field] = selected_source + result["selected_reasons"][field] = reason + + result["name"] = result.get("name") or f"MMSI {mmsi}" + result["vessel_type_name"] = result.get("vessel_type_name") or normalize_vessel_type_name( + result.get("vessel_type") + ) + return result + + +async def _upsert_conflict_records( + db: AsyncSession, + entity_key: str, + observations: list[AISRawObservation], + aggregated: dict[str, Any], +) -> int: + conflicts = build_field_conflict_candidates(observations) + now = datetime.now(UTC) + for conflict in conflicts: + field = conflict["field"] + result = await db.execute( + select(AISConflictRecord) + .where(AISConflictRecord.target_schema == VESSEL_AIS_SCHEMA) + .where(AISConflictRecord.entity_key == entity_key) + .where(AISConflictRecord.field == field) + .limit(1) + ) + record = result.scalar_one_or_none() + if record is None: + record = AISConflictRecord( + target_schema=VESSEL_AIS_SCHEMA, + entity_key=entity_key, + field=field, + ) + db.add(record) + record.candidates = conflict["candidates"] + record.selected_source = (aggregated.get("field_sources") or {}).get(field) + record.selected_value = aggregated.get(field) + record.selected_reason = (aggregated.get("selected_reasons") or {}).get(field) + record.resolved_by = "system" + record.status = "open" + record.updated_at = now + return len(conflicts) + + +def _group_observations(observations: Iterable[AISRawObservation]) -> dict[str, list[AISRawObservation]]: + grouped: dict[str, list[AISRawObservation]] = {} + for observation in observations: + grouped.setdefault(str(observation.entity_key), []).append(observation) + return grouped + + +async def record_vessel_ais_observation( + db: AsyncSession, + *, + source: str, + normalized_payload: dict[str, Any], + raw_payload: dict[str, Any] | None = None, + delivery_mode: str, + transport: str, + message_type: str | None = "PositionReport", + source_message_id: str | None = None, + observed_at: datetime | None = None, + collected_at: datetime | None = None, + quality_flags: list[str] | None = None, +) -> AISRawObservation | None: + """Insert one raw observation if the source-level fact has not already been stored.""" + + entity_key = str(normalized_payload["mmsi"]) + collected_at = collected_at or datetime.now(UTC) + observed_at = ( + _coerce_datetime(observed_at) + or _coerce_datetime(normalized_payload.get("received_at")) + or collected_at + ) + normalized_json = _jsonable(normalized_payload) + raw_json = _jsonable(raw_payload or {}) + + observation_hash = build_observation_hash( + source=source, + entity_key=entity_key, + message_type=message_type, + observed_at=observed_at, + normalized_payload=normalized_json, + source_message_id=source_message_id, + ) + existing_result = await db.execute( + select(AISRawObservation.id).where(AISRawObservation.observation_hash == observation_hash) + ) + if existing_result.scalar_one_or_none() is not None: + return None + + observation = AISRawObservation( + target_schema=VESSEL_AIS_SCHEMA, + source=source, + entity_key=entity_key, + delivery_mode=delivery_mode, + transport=transport, + message_type=message_type, + source_message_id=source_message_id, + observation_hash=observation_hash, + observed_at=observed_at, + collected_at=collected_at, + normalized_payload=normalized_json, + raw_payload=raw_json, + quality_flags=quality_flags or [], + ) + db.add(observation) + return observation + + +async def aggregate_vessel_observations( + db: AsyncSession, + observations: Iterable[AISRawObservation], + *, + write_conflicts: bool = False, + strategy: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + strategy = strategy if strategy is not None else await _safe_load_strategy(db) + now = datetime.now(UTC) + vessels = [] + for entity_key, entity_observations in _group_observations(observations).items(): + aggregated = _build_aggregated_vessel( + entity_key, entity_observations, now=now, strategy=strategy + ) + if aggregated is None: + continue + if write_conflicts: + aggregated["conflict_count"] = await _upsert_conflict_records( + db, + entity_key, + entity_observations, + aggregated, + ) + else: + aggregated["conflict_count"] = len(build_field_conflict_candidates(entity_observations)) + vessels.append(aggregated) + + vessels.sort(key=lambda item: item.get("received_at") or datetime.min.replace(tzinfo=UTC), reverse=True) + return vessels + + +async def _safe_load_strategy(db: AsyncSession) -> dict[str, Any]: + """Tolerate fake test sessions where load_strategy may misbehave.""" + try: + return await load_strategy(db) + except Exception: + return DEFAULT_STRATEGY + + +async def get_aggregated_vessels( + db: AsyncSession, + *, + bbox: tuple[float, float, float, float] | None = None, + limit: int | None = None, + observed_since: datetime | None = None, +) -> list[dict[str, Any]]: + observed_since = observed_since or ( + datetime.now(UTC) - timedelta(hours=DEFAULT_AGGREGATION_WINDOW_HOURS) + ) + stmt = ( + select(AISRawObservation) + .where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA) + .where(AISRawObservation.observed_at >= observed_since) + .order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc()) + ) + if limit and limit > 0: + stmt = stmt.limit(max(limit * 20, limit)) + + result = await db.execute(stmt) + if not hasattr(result, "scalars"): + return [] + vessels = await aggregate_vessel_observations(db, result.scalars().all()) + + if bbox is not None: + lon_min, lat_min, lon_max, lat_max = bbox + vessels = [ + vessel + for vessel in vessels + if lon_min <= float(vessel["lon"]) <= lon_max + and lat_min <= float(vessel["lat"]) <= lat_max + ] + + if limit and limit > 0: + return vessels[:limit] + return vessels + + +async def get_aggregated_vessel(db: AsyncSession, mmsi: int) -> dict[str, Any] | None: + observations = await get_vessel_raw_observations(db, mmsi, limit=1000) + vessels = await aggregate_vessel_observations(db, observations) + return vessels[0] if vessels else None + + +async def get_aggregated_vessel_track( + db: AsyncSession, + mmsi: int, + *, + cutoff: datetime, +) -> list[dict[str, Any]]: + result = await db.execute( + select(AISRawObservation) + .where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA) + .where(AISRawObservation.entity_key == str(mmsi)) + .where(AISRawObservation.observed_at >= cutoff) + .order_by(AISRawObservation.observed_at.asc(), AISRawObservation.id.asc()) + ) + if not hasattr(result, "scalars"): + return [] + + points: list[dict[str, Any]] = [] + seen: set[tuple[str, float, float, str]] = set() + for observation in result.scalars().all(): + payload = observation.normalized_payload or {} + if not _has_valid_position(payload): + continue + lat = float(payload["lat"]) + lon = float(payload["lon"]) + key = ( + observation.observed_at.isoformat(), + round(lat, 5), + round(lon, 5), + observation.source, + ) + if key in seen: + continue + seen.add(key) + points.append( + { + "lat": lat, + "lon": lon, + "observed_at": observation.observed_at, + "source": observation.source, + "selected_reason": "track_timeline", + "quality_flags": observation.quality_flags or [], + } + ) + return points + + +async def update_ais_source_health( + db: AsyncSession, + *, + source: str, + connection_state: str, + observed_count: int = 0, + last_seen_at: datetime | None = None, + last_success_at: datetime | None = None, + last_error: str | None = None, + lag_seconds: float | None = None, +) -> AISSourceHealth: + """Upsert the health row for an AIS source.""" + + now = datetime.now(UTC) + health = await db.get(AISSourceHealth, source) + if health is None: + health = AISSourceHealth(source=source) + db.add(health) + + health.connection_state = connection_state + health.last_seen_at = last_seen_at or health.last_seen_at + health.last_success_at = last_success_at or health.last_success_at + health.last_error = last_error + health.message_rate = float(observed_count) + health.lag_seconds = lag_seconds + health.updated_at = now + return health + + +async def count_unique_raw_vessel_mmsi( + db: AsyncSession, + *, + observed_since: datetime | None = None, +) -> int: + """Count unique raw vessel MMSI values for HUD counts; never aggregates.""" + from sqlalchemy import func as sa_func + + unique_mmsi_stmt = ( + select(AISRawObservation.entity_key) + .where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA) + .distinct() + ) + if observed_since is not None: + unique_mmsi_stmt = unique_mmsi_stmt.where( + AISRawObservation.observed_at >= observed_since, + ) + + result = await db.execute( + select(sa_func.count()).select_from(unique_mmsi_stmt.subquery()), + ) + return int(result.scalar() or 0) + + +async def get_vessel_raw_observations( + db: AsyncSession, + mmsi: int, + *, + limit: int = 100, +) -> list[AISRawObservation]: + result = await db.execute( + select(AISRawObservation) + .where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA) + .where(AISRawObservation.entity_key == str(mmsi)) + .order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc()) + .limit(limit) + ) + return list(result.scalars().all()) + + +async def get_vessel_conflict_records( + db: AsyncSession, + mmsi: int, +) -> list[AISConflictRecord]: + result = await db.execute( + select(AISConflictRecord) + .where(AISConflictRecord.target_schema == VESSEL_AIS_SCHEMA) + .where(AISConflictRecord.entity_key == str(mmsi)) + .order_by(AISConflictRecord.updated_at.desc(), AISConflictRecord.id.desc()) + ) + return list(result.scalars().all()) diff --git a/backend/app/services/vessel_enrichment.py b/backend/app/services/vessel_enrichment.py new file mode 100644 index 00000000..ef4d6512 --- /dev/null +++ b/backend/app/services/vessel_enrichment.py @@ -0,0 +1,109 @@ +"""v5 vessel enrichment service. + +Read-only side: `get_vessel_enrichment_bundle` is the only path the +aggregation/detail endpoints use. It never reaches out to third parties; it +just returns whatever the upsert side has already cached. Expired rows are +filtered out so old data never leaks back into the live UI. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.vessel_enrichment import VesselMediaEnrichment, VesselProfileEnrichment + + +def _coerce_datetime(value: Any) -> datetime | None: + if value in (None, ""): + return None + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=UTC) + if isinstance(value, (int, float)): + ts = float(value) + if ts > 10_000_000_000: + ts /= 1000 + return datetime.fromtimestamp(ts, UTC) + if isinstance(value, str): + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + except ValueError: + return None + return None + + +def _build_payload(record, *, now: datetime) -> dict[str, Any] | None: + if record is None: + return None + expires_at = record.expires_at + if isinstance(expires_at, datetime): + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=UTC) + if expires_at < now: + return None + return record.to_dict() + + +async def get_vessel_enrichment_bundle(db: AsyncSession, mmsi: int) -> dict[str, Any]: + now = datetime.now(UTC) + profile = await db.get(VesselProfileEnrichment, mmsi) + media = await db.get(VesselMediaEnrichment, mmsi) + return { + "mmsi": mmsi, + "profile": _build_payload(profile, now=now), + "media": _build_payload(media, now=now), + } + + +async def upsert_vessel_profile_enrichment( + db: AsyncSession, + *, + mmsi: int, + payload: dict[str, Any], +) -> dict[str, Any]: + record = await db.get(VesselProfileEnrichment, mmsi) + if record is None: + record = VesselProfileEnrichment(mmsi=mmsi) + db.add(record) + return _apply_upsert(record, payload) + + +async def upsert_vessel_media_enrichment( + db: AsyncSession, + *, + mmsi: int, + payload: dict[str, Any], +) -> dict[str, Any]: + record = await db.get(VesselMediaEnrichment, mmsi) + if record is None: + record = VesselMediaEnrichment(mmsi=mmsi) + db.add(record) + return _apply_upsert(record, payload) + + +def _apply_upsert(record, payload: dict[str, Any]) -> dict[str, Any]: + if not isinstance(payload, dict): + raise ValueError("enrichment payload must be an object") + body = payload.get("payload") + if body is not None and not isinstance(body, dict): + raise ValueError("payload.payload must be an object") + if body is not None: + record.payload = body + if "source" in payload and isinstance(payload["source"], str) and payload["source"].strip(): + record.source = payload["source"].strip() + fetched_at = _coerce_datetime(payload.get("fetched_at")) + record.fetched_at = fetched_at or datetime.now(UTC) + record.expires_at = _coerce_datetime(payload.get("expires_at")) + confidence = payload.get("confidence") + if confidence is not None: + try: + record.confidence = float(confidence) + except (TypeError, ValueError): + record.confidence = None + if "reference_url" in payload: + ref = payload.get("reference_url") + record.reference_url = str(ref) if ref else None + return record.to_dict() diff --git a/backend/app/services/vessel_types.py b/backend/app/services/vessel_types.py new file mode 100644 index 00000000..156cb183 --- /dev/null +++ b/backend/app/services/vessel_types.py @@ -0,0 +1,31 @@ +"""Shared AIS vessel type helpers.""" + +from typing import Any + +VESSEL_TYPE_NAMES = { + 30: "Fishing", + 35: "Military", + 60: "Passenger", + 70: "Cargo", + 80: "Tanker", +} + + +def normalize_vessel_type_name(vessel_type: Any) -> str: + """Map AIS numeric vessel type codes to display buckets.""" + + try: + type_code = int(float(vessel_type)) + except (TypeError, ValueError): + return "Other" + if 70 <= type_code <= 79: + return "Cargo" + if 80 <= type_code <= 89: + return "Tanker" + if 60 <= type_code <= 69: + return "Passenger" + if type_code == 30: + return "Fishing" + if type_code == 35: + return "Military" + return VESSEL_TYPE_NAMES.get(type_code, "Other") diff --git a/backend/scripts/system_restart_runner.py b/backend/scripts/system_restart_runner.py index a30ed2d6..c92bcea8 100644 --- a/backend/scripts/system_restart_runner.py +++ b/backend/scripts/system_restart_runner.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse import os +import shlex import subprocess import sys import time @@ -59,6 +60,8 @@ def wait_for_recovery(action: str) -> tuple[bool, str]: recovery_mode = get_action_recovery_mode(action) if recovery_mode == "backend": return wait_for_http("http://localhost:8000/health"), "backend health recovery" + if recovery_mode == "frontend": + return wait_for_http("http://localhost:3000"), "frontend entrypoint recovery" if recovery_mode == "ai-provider": return wait_for_http("http://localhost:8010/health"), "ai provider health recovery" if recovery_mode == "database": @@ -108,8 +111,9 @@ def main() -> int: ) append_task_log(args.task_id, "restart command started") + shell_command = shlex.join(command) completed = subprocess.run( - command, + ["zsh", "-ic", shell_command], cwd=str(ROOT_DIR), env=env, capture_output=True, diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index a34a9bb8..3f12b53a 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -2,10 +2,45 @@ import pytest import asyncio -from typing import AsyncGenerator -from unittest.mock import AsyncMock, MagicMock, patch +import json +from unittest.mock import AsyncMock, MagicMock -from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker +from sqlalchemy.ext.asyncio import AsyncSession + + +@pytest.fixture(autouse=True) +def bgp_collector_location_cache(): + """Mirror app startup seeding for tests that call sync BGP helpers.""" + from app.services.bgp_collector_locations import ( + SEED_PATH, + set_bgp_collector_location_cache, + ) + + payload = json.loads(SEED_PATH.read_text(encoding="utf-8")) + cache = {} + for entry in payload.get("locations", []): + collector_id = next( + alias for alias in entry.get("aliases", []) if str(alias).startswith("rrc") + ) + cache[collector_id] = { + "city": entry.get("city"), + "country": entry.get("country"), + "latitude": entry.get("latitude"), + "longitude": entry.get("longitude"), + "precision": entry.get("precision") or "city", + "source": "legacy_seed", + "needs_confirmation": True, + "matched_location_name": entry.get("site") or collector_id, + "verified_at": None, + "confidence": entry.get("confidence"), + "operator": entry.get("operator"), + "site": entry.get("site"), + "verification_status": "unverified", + "source_note": entry.get("source_note"), + } + set_bgp_collector_location_cache(cache) + yield + set_bgp_collector_location_cache({}) @pytest.fixture(scope="session") diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 2822bf82..d917ab6a 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -35,6 +35,7 @@ async def test_health_check(): data = response.json() assert data["status"] == "healthy" assert "version" in data + assert response.headers["x-request-id"] @pytest.mark.asyncio @@ -161,6 +162,345 @@ async def test_alerts_endpoint_with_auth(auth_headers): app.dependency_overrides.clear() +@pytest.mark.asyncio +async def test_system_log_sources_requires_super_admin(auth_headers): + def override_get_current_user(): + return User( + id=1, + username="testuser", + email="test@example.com", + password_hash="hashed", + role="admin", + is_active=True, + ) + + app.dependency_overrides = { + __import__("app.core.security", fromlist=["get_current_user"]).get_current_user: override_get_current_user, + } + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/system/logs/sources", headers=auth_headers) + assert response.status_code == 403 + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_system_log_sources_with_super_admin(auth_headers): + def override_get_current_user(): + return User( + id=1, + username="root", + email="root@example.com", + password_hash="hashed", + role="super_admin", + is_active=True, + ) + + app.dependency_overrides = { + __import__("app.core.security", fromlist=["get_current_user"]).get_current_user: override_get_current_user, + } + transport = ASGITransport(app=app) + try: + with patch( + "app.api.v1.system_control.list_log_sources", + return_value=[ + { + "source_id": "backend", + "name": "后端服务", + "kind": "file", + "location": "/tmp/planet_backend.log", + "description": "FastAPI 后端、调度器和采集任务共享日志。", + "category": "service", + "status": "ok", + } + ], + ): + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/system/logs/sources", headers=auth_headers) + assert response.status_code == 200 + data = response.json() + assert data["items"][0]["source_id"] == "backend" + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_system_log_snapshot_with_super_admin(auth_headers): + def override_get_current_user(): + return User( + id=1, + username="root", + email="root@example.com", + password_hash="hashed", + role="super_admin", + is_active=True, + ) + + app.dependency_overrides = { + __import__("app.core.security", fromlist=["get_current_user"]).get_current_user: override_get_current_user, + } + transport = ASGITransport(app=app) + try: + with patch( + "app.api.v1.system_control.read_log_snapshot", + return_value={ + "source_id": "backend", + "name": "后端服务", + "kind": "file", + "location": "/tmp/planet_backend.log", + "description": "FastAPI 后端、调度器和采集任务共享日志。", + "category": "service", + "status": "ok", + "level": "all", + "selected_levels": [], + "search_query": "", + "available_levels": ["all", "error", "warning", "info", "debug"], + "daily_markers": [], + "line_limit": 50, + "line_count": 2, + "lines": ["line 1", "line 2"], + }, + ): + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/system/logs/backend?limit=50", headers=auth_headers) + assert response.status_code == 200 + data = response.json() + assert data["source_id"] == "backend" + assert data["line_count"] == 2 + assert data["lines"] == ["line 1", "line 2"] + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_system_log_snapshot_supports_level_filter(auth_headers): + def override_get_current_user(): + return User( + id=1, + username="root", + email="root@example.com", + password_hash="hashed", + role="super_admin", + is_active=True, + ) + + app.dependency_overrides = { + __import__("app.core.security", fromlist=["get_current_user"]).get_current_user: override_get_current_user, + } + transport = ASGITransport(app=app) + try: + with patch( + "app.api.v1.system_control.read_log_snapshot", + return_value={ + "source_id": "backend", + "name": "后端服务", + "kind": "file", + "location": "/tmp/planet_backend.log", + "description": "FastAPI 后端、调度器和采集任务共享日志。", + "category": "service", + "status": "ok", + "level": "error", + "selected_levels": ["error"], + "search_query": "", + "available_levels": ["all", "error", "warning", "info", "debug"], + "daily_markers": [], + "line_limit": 50, + "line_count": 1, + "lines": ["ERROR: failed"], + }, + ): + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/system/logs/backend?limit=50&level=error", headers=auth_headers) + assert response.status_code == 200 + data = response.json() + assert data["level"] == "error" + assert data["lines"] == ["ERROR: failed"] + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_system_log_snapshot_supports_date_range_filter(auth_headers): + def override_get_current_user(): + return User( + id=1, + username="root", + email="root@example.com", + password_hash="hashed", + role="super_admin", + is_active=True, + ) + + app.dependency_overrides = { + __import__("app.core.security", fromlist=["get_current_user"]).get_current_user: override_get_current_user, + } + transport = ASGITransport(app=app) + try: + with patch( + "app.api.v1.system_control.read_log_snapshot", + return_value={ + "source_id": "backend", + "name": "后端服务", + "kind": "file", + "location": "/tmp/planet_backend.log", + "description": "FastAPI 后端、调度器和采集任务共享日志。", + "category": "service", + "status": "ok", + "level": "all", + "selected_levels": [], + "search_query": "", + "available_levels": ["all", "error", "warning", "info", "debug"], + "daily_markers": [], + "line_limit": 50, + "line_count": 1, + "lines": ["2026-04-23 INFO: service started"], + }, + ) as mock_read_log_snapshot: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get( + "/api/v1/system/logs/backend?limit=50&start_date=2026-04-20&end_date=2026-04-23", + headers=auth_headers, + ) + assert response.status_code == 200 + mock_read_log_snapshot.assert_called_once_with( + "backend", + 50, + level="all", + levels=None, + start_date="2026-04-20", + end_date="2026-04-23", + search=None, + ) + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_system_log_snapshot_supports_levels_and_search_filter(auth_headers): + def override_get_current_user(): + return User( + id=1, + username="root", + email="root@example.com", + password_hash="hashed", + role="super_admin", + is_active=True, + ) + + app.dependency_overrides = { + __import__("app.core.security", fromlist=["get_current_user"]).get_current_user: override_get_current_user, + } + transport = ASGITransport(app=app) + try: + with patch( + "app.api.v1.system_control.read_log_snapshot", + return_value={ + "source_id": "backend", + "name": "后端服务", + "kind": "file", + "location": "/tmp/planet_backend.log", + "description": "FastAPI 后端、调度器和采集任务共享日志。", + "category": "service", + "status": "ok", + "level": "all", + "selected_levels": ["error", "warning"], + "search_query": "timeout", + "available_levels": ["all", "error", "warning", "info", "debug"], + "daily_markers": [], + "line_limit": 50, + "line_count": 1, + "lines": ["2026-04-23 10:00:00 ERROR timeout"], + }, + ) as mock_read_log_snapshot: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get( + "/api/v1/system/logs/backend?limit=50&levels=error,warning&search=timeout", + headers=auth_headers, + ) + assert response.status_code == 200 + mock_read_log_snapshot.assert_called_once_with( + "backend", + 50, + level="all", + levels="error,warning", + start_date=None, + end_date=None, + search="timeout", + ) + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_system_log_snapshot_rejects_invalid_date_range(auth_headers): + def override_get_current_user(): + return User( + id=1, + username="root", + email="root@example.com", + password_hash="hashed", + role="super_admin", + is_active=True, + ) + + app.dependency_overrides = { + __import__("app.core.security", fromlist=["get_current_user"]).get_current_user: override_get_current_user, + } + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get( + "/api/v1/system/logs/backend?start_date=2026-04-31", + headers=auth_headers, + ) + assert response.status_code == 400 + assert "start_date must be in YYYY-MM-DD format" in response.json()["detail"] + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_ingest_earth_client_log_accepts_public_events(): + transport = ASGITransport(app=app) + try: + with patch("app.api.v1.system_control.append_buffer_log") as mock_append_buffer_log: + with patch("app.api.v1.system_control.record_system_log", new_callable=AsyncMock) as mock_record_system_log: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.post( + "/api/v1/system/logs/earth-client", + json={ + "level": "error", + "message": "登陆点加载失败: 登陆点接口返回 HTTP 500", + "category": "startup-load", + "module": "layer-startup", + }, + ) + assert response.status_code == 200 + data = response.json() + assert data["accepted"] is True + assert data["source_id"] == "earth-client" + mock_append_buffer_log.assert_called_once() + mock_record_system_log.assert_awaited_once() + persisted_kwargs = mock_record_system_log.await_args.kwargs + assert persisted_kwargs["source"] == "earth-client" + assert persisted_kwargs["event"] == "earth.client.runtime_log" + assert persisted_kwargs["category"] == "startup-load" + assert persisted_kwargs["level"] == "error" + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_request_id_header_is_echoed_when_provided(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/health", headers={"X-Request-ID": "planet-test-request"}) + assert response.status_code == 200 + assert response.headers["x-request-id"] == "planet-test-request" + + @pytest.mark.asyncio async def test_invalid_token(): """Test that invalid token is rejected""" @@ -263,6 +603,8 @@ async def test_ai_situational_analysis_returns_503_when_disabled(auth_headers): assert "content_blocks" in data assert "text_blocks" in data assert "thinking_blocks" in data + finally: + app.dependency_overrides.clear() @pytest.mark.asyncio @@ -382,8 +724,6 @@ async def test_save_playground_session_with_auth(auth_headers): assert data["state"]["objective"] == "测试目标" finally: app.dependency_overrides.clear() - finally: - app.dependency_overrides.clear() @pytest.mark.asyncio diff --git a/backend/tests/test_bgp.py b/backend/tests/test_bgp.py index 6482adf0..f76dc7f6 100644 --- a/backend/tests/test_bgp.py +++ b/backend/tests/test_bgp.py @@ -1,6 +1,7 @@ """Tests for BGP observability helpers.""" from datetime import UTC, datetime, timedelta +from types import SimpleNamespace import pytest from httpx import ASGITransport, AsyncClient @@ -54,11 +55,34 @@ class _FakeResult: def scalars(self): return _FakeScalarResult(self._rows) + def all(self): + if self._rows and all(isinstance(row, BGPObservation) for row in self._rows): + return [ + (row.prefix, row.origin_asn, row.collector, row.collector_geo) + for row in self._rows + ] + return self._rows + + def scalar(self): + if not self._rows: + return 0 + first = self._rows[0] + if isinstance(first, (int, float, str)): + return first + if isinstance(first, tuple) and len(first) == 1: + return first[0] + return len(self._rows) + def fetchall(self): return self._rows def fetchone(self): - return self._rows[0] if self._rows else None + if not self._rows: + return None + first = self._rows[0] + if isinstance(first, CollectedData): + return {"extra_data": first.extra_data} + return first class _FakeAsyncSession: @@ -988,7 +1012,7 @@ async def test_infer_related_infrastructure_links_nearby_cables(): data_type="cable", extra_data={"cable_id": 20}, ) - db = _FakeAsyncSession([[landing], [relation], [cable]]) + db = _FakeAsyncSession([[landing, relation, cable]]) result = await infer_related_infrastructure( db, @@ -1012,27 +1036,37 @@ async def test_infer_related_infrastructure_links_nearby_cables(): @pytest.mark.asyncio async def test_build_bgp_collector_coverage_summarizes_observations(): now = datetime.now(UTC) - obs_one = BGPObservation( - source="ris_live_bgp", + aggregate = SimpleNamespace( + collector="rrc00", + observation_count=2, + prefix_count=2, + origin_asn_count=2, + peer_asn_count=2, + recent_15m_observation_count=2, + recent_24h_observation_count=2, + recent_7d_observation_count=2, + recent_15m_prefix_count=2, + recent_24h_prefix_count=2, + recent_7d_prefix_count=2, + latest_observed_at=now + timedelta(minutes=5), + ) + latest = SimpleNamespace( + collector="rrc00", + latest_event_type="withdrawal", + country="Netherlands", + city="Amsterdam", + ) + top_event = SimpleNamespace( collector="rrc00", - prefix="203.0.113.0/24", - origin_asn=64496, - peer_asn=3333, event_type="announcement", - observed_at=now, - collector_geo={"city": "Amsterdam", "country": "Netherlands"}, + count=1, ) - obs_two = BGPObservation( - source="ris_live_bgp", + scope = SimpleNamespace( collector="rrc00", - prefix="198.51.100.0/24", - origin_asn=64497, - peer_asn=3334, - event_type="withdrawal", - observed_at=now + timedelta(minutes=5), - collector_geo={"city": "Amsterdam", "country": "Netherlands"}, + country="Netherlands", + city="Amsterdam", ) - db = _FakeAsyncSession([[obs_one, obs_two]]) + db = _FakeAsyncSession([[aggregate], [latest], [top_event], [scope]]) coverage = await build_bgp_collector_coverage(db, source_filter=BGP_SOURCES) @@ -1363,18 +1397,39 @@ async def test_bgp_event_summary_api_returns_aggregates(): @pytest.mark.asyncio async def test_bgp_collectors_api_returns_coverage(): now = datetime.now(UTC) - observation = BGPObservation( - id=1, - source="ris_live_bgp", + aggregate = SimpleNamespace( collector="rrc00", - peer_asn=3333, - prefix="203.0.113.0/24", - event_type="announcement", - origin_asn=64496, - observed_at=now, - collector_geo={"city": "Amsterdam", "country": "Netherlands"}, + observation_count=1, + prefix_count=1, + origin_asn_count=1, + peer_asn_count=1, + recent_15m_observation_count=1, + recent_24h_observation_count=1, + recent_7d_observation_count=1, + recent_15m_prefix_count=1, + recent_24h_prefix_count=1, + recent_7d_prefix_count=1, + latest_observed_at=now, + ) + latest = SimpleNamespace( + collector="rrc00", + latest_event_type="announcement", + country="Netherlands", + city="Amsterdam", + ) + top_event = SimpleNamespace( + collector="rrc00", + event_type="announcement", + count=1, + ) + scope = SimpleNamespace( + collector="rrc00", + country="Netherlands", + city="Amsterdam", + ) + db = _FakeAsyncSession( + [[aggregate], [latest], [top_event], [scope], [aggregate], [latest], [top_event], [scope]] ) - db = _FakeAsyncSession([[observation], [observation]]) client = await _bgp_test_client(db) try: diff --git a/backend/tests/test_bgp_collector_locations.py b/backend/tests/test_bgp_collector_locations.py new file mode 100644 index 00000000..62665ec9 --- /dev/null +++ b/backend/tests/test_bgp_collector_locations.py @@ -0,0 +1,149 @@ +"""Tests for the BGP collector + event location services.""" + +from __future__ import annotations + +import pytest + +from app.services import bgp_collector_locations +from app.services.bgp_collector_locations import ( + RIPE_RIS_COLLECTOR_COORDS, + collect_bgp_collector_location_candidates, + iter_known_collector_names, + resolve_bgp_collector_location, +) +from app.services.bgp_event_locations import ( + resolve_bgp_event_geo_dict, + resolve_bgp_event_location, +) + + +def test_legacy_dict_view_preserves_backward_compatible_keys(): + rrc00 = RIPE_RIS_COLLECTOR_COORDS["rrc00"] + assert rrc00["city"] == "Amsterdam" + assert rrc00["country"] == "Netherlands" + assert rrc00["latitude"] == pytest.approx(52.3676) + assert rrc00["longitude"] == pytest.approx(4.9041) + # New richer fields layered on top. + assert rrc00["precision"] == "city" + assert rrc00["source"] == "legacy_seed" + assert rrc00["needs_confirmation"] is True + + +def test_every_legacy_collector_present(): + expected = { + "rrc00", "rrc01", "rrc03", "rrc04", "rrc05", "rrc06", "rrc07", + "rrc10", "rrc11", "rrc12", "rrc13", "rrc14", "rrc15", "rrc16", + "rrc18", "rrc19", "rrc20", "rrc21", "rrc22", "rrc23", "rrc24", + "rrc25", "rrc26", + } + assert set(iter_known_collector_names()) == expected + + +def test_resolve_bgp_collector_returns_stored_location(): + result = resolve_bgp_collector_location("rrc12") + assert result.location is not None + assert result.location.city == "Frankfurt" + assert result.location.country == "Germany" + assert result.location.precision == "city" + assert result.location.source == "legacy_seed" + assert result.location.needs_confirmation is True + + +def test_resolve_unknown_bgp_collector_returns_diagnostic(monkeypatch): + monkeypatch.setattr(bgp_collector_locations, "_geocode_online", lambda q: None) + result = resolve_bgp_collector_location("rrc-doesnotexist") + assert result.location is None + assert result.diagnostic is not None + assert result.diagnostic.failure_reason + + +def test_collect_bgp_collector_candidates_uses_stored_context_without_registry(monkeypatch): + bgp_collector_locations._geocode_online.cache_clear() + + def _fake_geocode(query): + assert "CIXP" in query or "Geneva" in query + return { + "lat": "46.2044", + "lon": "6.1432", + "display_name": "Geneva, Switzerland", + "address": {"city": "Geneva", "country": "Switzerland"}, + } + + monkeypatch.setattr(bgp_collector_locations, "_geocode_online", _fake_geocode) + candidates, attempted = collect_bgp_collector_location_candidates( + collector="rrc04", + ) + assert attempted, "stored context should feed online query attempts" + assert candidates, "online geocoding should produce at least one candidate" + best = candidates[0] + assert best.source == "nominatim_online_geocode" + assert best.needs_confirmation is True + assert all(candidate.source != "local_registry" for candidate in candidates) + + +def test_collect_bgp_collector_candidates_uses_nominatim_when_registry_misses(monkeypatch): + bgp_collector_locations._geocode_online.cache_clear() + + def _fake_geocode(query): + if "Lyon" not in query and "France-IX" not in query and "FR-IX" not in query: + return None + return { + "lat": "45.764", + "lon": "4.8357", + "display_name": "Lyon, Auvergne-Rhône-Alpes, France", + "address": {"city": "Lyon", "country": "France"}, + } + + monkeypatch.setattr(bgp_collector_locations, "_geocode_online", _fake_geocode) + candidates, attempted = collect_bgp_collector_location_candidates( + collector="rrc-mystery", + city="Lyon", + country="France", + ) + assert attempted, "Nominatim plan should run" + online = [c for c in candidates if c.source == "nominatim_online_geocode"] + assert online, "online resolver must produce a candidate when registry misses" + assert online[0].needs_confirmation is True + + +# ── BGP event resolver ───────────────────────────────────────────── + + +def test_event_resolver_inherits_from_owning_collector(): + geo = resolve_bgp_event_geo_dict("rrc25") + assert geo["city"] == "Amsterdam" + assert geo["country"] == "Netherlands" + assert geo["source"] == "inherited_from_collector" + assert geo["precision"] == "city" + + +def test_event_resolver_does_not_match_unrelated_collectors(): + """Regression: passing operator=RIPE NCC must NOT make every collector match.""" + rrc12 = resolve_bgp_event_geo_dict("rrc12") + rrc25 = resolve_bgp_event_geo_dict("rrc25") + assert rrc12["city"] == "Frankfurt" + assert rrc25["city"] == "Amsterdam" + assert rrc12["latitude"] != rrc25["latitude"] + + +def test_event_resolver_uses_source_coordinates_when_present(): + geo = resolve_bgp_event_geo_dict( + "rrc12", + source_latitude=12.34, + source_longitude=56.78, + ) + assert geo["latitude"] == pytest.approx(12.34) + assert geo["longitude"] == pytest.approx(56.78) + assert geo["precision"] == "precise" + assert geo["source"] == "source_coordinates" + + +def test_event_resolver_returns_empty_for_unknown_collector_without_source_coords(): + geo = resolve_bgp_event_geo_dict("rrc-doesnotexist") + assert geo == {} + + +def test_event_resolver_full_result_carries_diagnostic_on_miss(): + result = resolve_bgp_event_location(collector="rrc-doesnotexist") + assert result.location is None + assert result.diagnostic is not None diff --git a/backend/tests/test_collectors.py b/backend/tests/test_collectors.py index 149f0b5c..d747f3a6 100644 --- a/backend/tests/test_collectors.py +++ b/backend/tests/test_collectors.py @@ -1,11 +1,14 @@ """Unit tests for data collectors""" import pytest -from datetime import datetime -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, patch +from app.core.datasource_defaults import DEFAULT_DATASOURCES +from app.services.credential_guides import DEFAULT_CREDENTIAL_GUIDES from app.services.collectors.top500 import TOP500Collector -from app.services.collectors.base import BaseCollector, HTTPCollector +from app.services.collectors.registry import collector_registry +from app.services.datasource_connectivity import SUPPORTED_CREDENTIAL_PROVIDERS +from app.models.task import CollectionTask class TestBaseCollector: @@ -19,6 +22,31 @@ class TestBaseCollector: assert collector.module == "L1" assert collector.frequency_hours == 4 + @pytest.mark.asyncio + async def test_update_phase_progress_tracks_phase_fields(self, mock_db_session): + """Test phase-level progress updates independently from record totals""" + collector = TOP500Collector() + task = CollectionTask(datasource_id=1, status="running", phase="fetching") + collector._current_task = task + collector._db_session = mock_db_session + + with patch.object(collector, "_publish_task_update", new=AsyncMock()) as publish: + await collector.update_phase_progress( + current=512, + total=1024, + unit="bytes", + message="Downloading dataset", + commit=True, + ) + + assert task.phase_progress == 50.0 + assert task.phase_current == 512 + assert task.phase_total == 1024 + assert task.phase_unit == "bytes" + assert task.phase_message == "Downloading dataset" + mock_db_session.commit.assert_awaited_once() + publish.assert_awaited_once() + class TestTOP500Collector: """Tests for TOP500Collector""" @@ -119,3 +147,30 @@ class TestHTTPCollector: assert hasattr(collector, "parse_response") assert callable(collector.fetch) assert callable(collector.parse_response) + + +def test_aisstream_collector_is_registered(): + collector = collector_registry.get("aisstream_vessels") + + assert collector is not None + assert collector.data_type == "vessel_ais" + + +def test_supported_credential_collectors_have_guides_and_connectivity_provider(): + missing: list[str] = [] + for source, info in DEFAULT_DATASOURCES.items(): + if not info.get("requires_credentials"): + continue + if info.get("credential_status") != "supported": + continue + + provider = info.get("credential_provider") + if not provider: + missing.append(f"{source}: missing credential_provider") + continue + if provider not in DEFAULT_CREDENTIAL_GUIDES: + missing.append(f"{source}: missing credential guide for {provider}") + if provider not in SUPPORTED_CREDENTIAL_PROVIDERS: + missing.append(f"{source}: missing connectivity provider for {provider}") + + assert missing == [] diff --git a/backend/tests/test_custom_datasource_runtime_live.py b/backend/tests/test_custom_datasource_runtime_live.py new file mode 100644 index 00000000..dd036e23 --- /dev/null +++ b/backend/tests/test_custom_datasource_runtime_live.py @@ -0,0 +1,149 @@ +"""End-to-end integration test for the custom WebSocket datasource runner. + +Boots an in-process WebSocket server that mimics the bun mock AIS server +(`scripts/mock-ais-ws-server.ts`) and runs the real +`run_mapped_websocket_config` against it. Catches regressions where the +runner stops connecting, fails to extract the configured message path, +or quietly drops mapped records before broadcasting. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +import websockets + +from app.models.datasource_config import DataSourceConfig +from app.services import custom_datasource_runtime +from app.services.custom_datasource_runtime import run_mapped_websocket_config + + +def _make_payload(seq: int) -> str: + return json.dumps( + { + "type": "vessel", + "sequence": seq, + "data": { + "mmsi": str(999_000_000 + seq), + "name": f"MOCK VESSEL {seq:03d}", + "lat": 36.20 + seq * 0.001, + "lon": 14.20 + seq * 0.001, + "sog": 12.0, + "cog": 90.0, + "heading": 90, + "vessel_type": 70, + "vessel_type_name": "Cargo", + "received_at": datetime.now(UTC).isoformat(), + }, + } + ) + + +@asynccontextmanager +async def _mock_ais_server(emit_count: int): + received_subscribe: list[str] = [] + + async def handler(ws): + try: + try: + msg = await asyncio.wait_for(ws.recv(), timeout=0.5) + received_subscribe.append(msg) + except (asyncio.TimeoutError, websockets.ConnectionClosed): + pass + for seq in range(1, emit_count + 1): + await ws.send(_make_payload(seq)) + await asyncio.sleep(0.01) + # keep the socket open briefly so the runner observes the messages + await asyncio.sleep(0.05) + except websockets.ConnectionClosed: + return + + async with websockets.serve(handler, "127.0.0.1", 0) as server: + port = next(iter(server.sockets)).getsockname()[1] + yield port, received_subscribe + + +@pytest.mark.asyncio +async def test_websocket_runner_streams_from_live_mock(monkeypatch): + mapping = SimpleNamespace( + id=11, + version=3, + target_schema="vessel_ais", + mapping_json={ + "source": {"items_path": "$"}, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "lat": {"path": "$.lat", "type": "float"}, + "lon": {"path": "$.lon", "type": "float"}, + "name": {"path": "$.name", "type": "string"}, + "vessel_type": {"path": "$.vessel_type", "type": "integer", "default": None}, + "vessel_type_name": {"path": "$.vessel_type_name", "type": "string", "default": None}, + "sog": {"path": "$.sog", "type": "float", "default": None}, + "cog": {"path": "$.cog", "type": "float", "default": None}, + "heading": {"path": "$.heading", "type": "integer", "default": None}, + "received_at": {"path": "$.received_at", "type": "datetime"}, + }, + }, + ) + + class FakeResult: + def scalar_one_or_none(self): + return mapping + + class FakeDB: + async def execute(self, _stmt): + return FakeResult() + + persist = AsyncMock(return_value=1) + monkeypatch.setattr(custom_datasource_runtime, "persist_mapped_records", persist) + + async with _mock_ais_server(emit_count=3) as (port, received_subscribe): + result = await run_mapped_websocket_config( + FakeDB(), + DataSourceConfig( + id=99, + name="mock_ais_ws", + source_type="websocket", + endpoint=f"ws://127.0.0.1:{port}", + auth_type="none", + headers={}, + config={ + "ws_message_path": "$.data", + "ws_subscribe_message": { + "type": "subscribe", + "anchor": {"lat": 36.2, "lon": 14.2}, + "spread_km": 50, + "rate_hz": 1, + }, + "debug_max_messages": 2, + "delivery_mode": "realtime_stream", + "ws_reconnect": False, + }, + ), + use_config_debug_max_messages=True, + ) + + assert result["status"] == "success" + assert result["messages_seen"] == 2 + assert result["written_count"] == 2 + assert result["mapped_count"] == 2 + assert result["target_schema"] == "vessel_ais" + # subscribe message must reach the server unchanged + assert received_subscribe, "runner did not forward ws_subscribe_message" + parsed = json.loads(received_subscribe[0]) + assert parsed["type"] == "subscribe" + assert parsed["anchor"] == {"lat": 36.2, "lon": 14.2} + assert parsed["rate_hz"] == 1 + # mapped records carry the real MMSIs from the mock stream + persisted_records = [] + for call in persist.await_args_list: + persisted_records.extend(call.kwargs["records"]) + assert {record["mmsi"] for record in persisted_records} == {999_000_001, 999_000_002} + assert all(record["vessel_type"] == 70 for record in persisted_records) + assert all(record["vessel_type_name"] == "Cargo" for record in persisted_records) diff --git a/backend/tests/test_datasource_mapping.py b/backend/tests/test_datasource_mapping.py new file mode 100644 index 00000000..aca3d040 --- /dev/null +++ b/backend/tests/test_datasource_mapping.py @@ -0,0 +1,328 @@ +from types import SimpleNamespace + +import pytest +from unittest.mock import AsyncMock +from httpx import ASGITransport, AsyncClient + +from app.api.v1.datasource_config import get_ai_provider_client +from app.core.websocket import broadcaster as broadcaster_module +from app.core.security import get_current_user +from app.core.target_schema_registry import get_target_schema, list_target_schemas +from app.main import app +from app.models.user import User +from app.models.datasource_config import DataSourceConfig +from app.services import custom_datasource_runtime +from app.services.custom_datasource_runtime import run_mapped_websocket_config +from app.services.datasource_mapping import execute_mapping, persist_mapped_records, redact_for_llm + + +SAMPLE_AIS = { + "data": [ + { + "mmsi": "257123000", + "latitude": "59.91", + "longitude": "10.75", + "speedOverGround": "12.4", + "timestamp": "2026-04-28T00:00:00Z", + "api_token": "secret-value", + } + ] +} + + +def test_registry_exposes_v1_target_schemas(): + keys = {schema["key"] for schema in list_target_schemas()} + + assert {"vessel_ais", "geo_points", "generic_records"}.issubset(keys) + assert get_target_schema("vessel_ais").destination == "vessel_position" + + +def test_mapping_engine_maps_and_validates_vessel_ais(): + mapping = { + "source": {"items_path": "$.data[*]"}, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "lat": {"path": "$.latitude", "type": "float"}, + "lon": {"path": "$.longitude", "type": "float"}, + "sog": {"path": "$.speedOverGround", "type": "float"}, + "received_at": {"path": "$.timestamp", "type": "datetime"}, + }, + } + + result = execute_mapping(SAMPLE_AIS, mapping, "vessel_ais") + + assert result["mapped_count"] == 1 + assert result["failed_count"] == 0 + assert result["records"][0]["mmsi"] == 257123000 + assert result["records"][0]["lat"] == 59.91 + + +def test_mapping_engine_reports_schema_errors(): + mapping = { + "source": {"items_path": "$.data[*]"}, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "lat": {"path": "$.missing_lat", "type": "float"}, + "lon": {"path": "$.longitude", "type": "float"}, + }, + } + + result = execute_mapping(SAMPLE_AIS, mapping, "vessel_ais") + + assert result["mapped_count"] == 0 + assert result["failed_count"] == 1 + assert any("lat" in error for error in result["errors"][0]["errors"]) + + +def test_redact_for_llm_masks_secret_like_fields(): + redacted = redact_for_llm(SAMPLE_AIS) + + assert redacted["data"][0]["api_token"] == "[REDACTED]" + + +@pytest.mark.asyncio +async def test_persist_mapped_records_writes_generic_records(): + class FakeDB: + def __init__(self): + self.added = [] + self.committed = False + + def add(self, value): + self.added.append(value) + + async def commit(self): + self.committed = True + + db = FakeDB() + + count = await persist_mapped_records( + db, + datasource_name="custom_weather", + datasource_config_id=42, + target_schema="generic_records", + records=[{"source_id": "row-1", "data": {"temp": 25}}], + mapping_version=3, + ) + + assert count == 1 + assert db.committed is True + assert db.added[0].source == "custom_weather" + assert db.added[0].data_type == "generic_records" + assert db.added[0].extra_data["mapping_version"] == 3 + + +@pytest.mark.asyncio +async def test_persist_mapped_vessel_records_writes_raw_and_broadcasts(monkeypatch): + record_observation = AsyncMock(return_value=object()) + update_health = AsyncMock() + broadcast_custom = AsyncMock() + monkeypatch.setattr( + "app.services.vessel_ais_aggregation.record_vessel_ais_observation", + record_observation, + ) + monkeypatch.setattr( + "app.services.vessel_ais_aggregation.update_ais_source_health", + update_health, + ) + monkeypatch.setattr(broadcaster_module, "broadcast_custom", broadcast_custom) + + class FakeDB: + def __init__(self): + self.committed = False + + async def commit(self): + self.committed = True + + db = FakeDB() + + count = await persist_mapped_records( + db, + datasource_name="mock_ais_ws", + datasource_config_id=42, + target_schema="vessel_ais", + records=[ + { + "mmsi": 999000001, + "lat": 31.2, + "lon": 121.4, + "name": "MOCK VESSEL 001", + "received_at": "2026-05-01T00:00:00Z", + } + ], + mapping_version=1, + delivery_mode="realtime_stream", + transport="websocket", + ) + + assert count == 1 + assert db.committed is True + record_observation.assert_awaited_once() + assert record_observation.await_args.kwargs["source"] == "mock_ais_ws" + assert record_observation.await_args.kwargs["delivery_mode"] == "realtime_stream" + assert record_observation.await_args.kwargs["transport"] == "websocket" + update_health.assert_awaited_once() + broadcast_custom.assert_awaited_once() + assert broadcast_custom.await_args.args[0] == "vessels" + assert broadcast_custom.await_args.args[1]["vessels"][0]["mmsi_display"] == "999000001" + + +@pytest.mark.asyncio +async def test_custom_websocket_runner_maps_and_persists_vessel_records(monkeypatch): + mapping = SimpleNamespace( + id=7, + version=2, + target_schema="vessel_ais", + mapping_json={ + "source": {"items_path": "$"}, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "lat": {"path": "$.lat", "type": "float"}, + "lon": {"path": "$.lon", "type": "float"}, + "name": {"path": "$.name", "type": "string"}, + "received_at": {"path": "$.received_at", "type": "datetime"}, + }, + }, + ) + + class FakeResult: + def scalar_one_or_none(self): + return mapping + + class FakeDB: + async def execute(self, _stmt): + return FakeResult() + + class FakeWebSocket: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def send(self, _message): + return None + + async def recv(self): + return ( + '{"type":"vessel","data":{"mmsi":"999000001","name":"MOCK VESSEL 001",' + '"lat":31.2,"lon":121.4,"received_at":"2026-05-01T00:00:00Z"}}' + ) + + persist = AsyncMock(return_value=1) + monkeypatch.setattr(custom_datasource_runtime, "_connect_websocket", AsyncMock(return_value=FakeWebSocket())) + monkeypatch.setattr(custom_datasource_runtime, "persist_mapped_records", persist) + + result = await run_mapped_websocket_config( + FakeDB(), + DataSourceConfig( + id=42, + name="mock_ais_ws", + source_type="websocket", + endpoint="ws://localhost:8787/ais", + auth_type="none", + headers={}, + config={"ws_message_path": "$.data", "debug_max_messages": 1}, + ), + ) + + assert result["status"] == "success" + assert result["messages_seen"] == 1 + assert result["written_count"] == 1 + persist.assert_awaited_once() + assert persist.await_args.kwargs["datasource_name"] == "mock_ais_ws" + assert persist.await_args.kwargs["records"][0]["mmsi"] == 999000001 + assert persist.await_args.kwargs["delivery_mode"] == "realtime_stream" + assert persist.await_args.kwargs["transport"] == "websocket" + + +@pytest.mark.asyncio +async def test_mapping_preview_api_uses_deterministic_engine(): + def override_get_current_user(): + return User( + id=1, + username="testuser", + email="test@example.com", + password_hash="hashed", + role="admin", + is_active=True, + ) + + app.dependency_overrides = {get_current_user: override_get_current_user} + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.post( + "/api/v1/datasources/mappings/preview", + json={ + "sample_payload": SAMPLE_AIS, + "target_schema": "vessel_ais", + "mapping_json": { + "source": {"items_path": "$.data[*]"}, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "lat": {"path": "$.latitude", "type": "float"}, + "lon": {"path": "$.longitude", "type": "float"}, + }, + }, + }, + ) + finally: + app.dependency_overrides.clear() + + assert response.status_code == 200 + payload = response.json() + assert payload["success"] is True + assert payload["preview"]["records"][0]["mmsi"] == 257123000 + + +@pytest.mark.asyncio +async def test_mapping_propose_api_redacts_sample_before_ai(): + seen_context = {} + + class FakeAIClient: + async def analyze(self, request, request_id=None): + seen_context.update(request.context) + return SimpleNamespace( + content=( + '{"source":{"items_path":"$.data[*]"},"fields":{' + '"mmsi":{"path":"$.mmsi","type":"integer"},' + '"lat":{"path":"$.latitude","type":"float"},' + '"lon":{"path":"$.longitude","type":"float"}}}' + ) + ) + + def override_get_current_user(): + return User( + id=1, + username="testuser", + email="test@example.com", + password_hash="hashed", + role="admin", + is_active=True, + ) + + def override_ai_client(): + return FakeAIClient() + + app.dependency_overrides = { + get_current_user: override_get_current_user, + get_ai_provider_client: override_ai_client, + } + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.post( + "/api/v1/datasources/mappings/propose", + json={ + "sample_payload": SAMPLE_AIS, + "target_schema": "vessel_ais", + "use_ai": True, + }, + ) + finally: + app.dependency_overrides.clear() + + assert response.status_code == 200 + payload = response.json() + assert payload["mapping_json"]["meta"]["generated_by"] == "ai_provider" + assert seen_context["sample_payload"]["data"][0]["api_token"] == "[REDACTED]" diff --git a/backend/tests/test_docs_gatekeeper.py b/backend/tests/test_docs_gatekeeper.py new file mode 100644 index 00000000..123a0a61 --- /dev/null +++ b/backend/tests/test_docs_gatekeeper.py @@ -0,0 +1,115 @@ +"""Docs Gatekeeper API tests.""" + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.api.v1 import docs as docs_api +from app.main import app +from app.models.user import User + + +def make_user(role: str = "viewer", groups: list[str] | None = None) -> User: + user = User( + id=1, + username="docs-user", + email="docs@example.com", + password_hash="x", + role=role, + is_active=True, + ) + user.gatekeeper_groups = groups or [] + return user + + +async def get_json(path: str, user: User | None = None): + if user is not None: + async def override_user(): + return user + + app.dependency_overrides[docs_api.get_optional_current_user] = override_user + + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + return await client.get(path) + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_public_catalog_only_for_anonymous_user(): + response = await get_json("/api/v1/docs/catalog") + + assert response.status_code == 200 + items = response.json()["items"] + assert {item["access"] for item in items} == {"public"} + assert {item["slug"] for item in items if item["lang"] == "zh"} == { + "overview", + "quickstart", + "manual", + "location-pipeline-user", + } + + +@pytest.mark.asyncio +async def test_anonymous_can_read_public_doc(): + response = await get_json("/api/v1/docs/zh/quickstart") + + assert response.status_code == 200 + assert response.json()["access"] == "public" + assert "快速开始" in response.json()["markdown"] + + +@pytest.mark.asyncio +async def test_anonymous_protected_doc_requires_authentication(): + response = await get_json("/api/v1/docs/zh/backend-collectors") + + assert response.status_code == 401 + + +@pytest.mark.asyncio +async def test_viewer_without_group_cannot_read_developer_doc(): + response = await get_json( + "/api/v1/docs/zh/backend-collectors", + make_user(role="viewer"), + ) + + assert response.status_code == 403 + + +@pytest.mark.asyncio +async def test_developer_group_can_read_developer_but_not_admin_doc(): + user = make_user(role="viewer", groups=["docs_developer"]) + + developer_response = await get_json("/api/v1/docs/zh/backend-collectors", user) + admin_response = await get_json("/api/v1/docs/zh/backend-system-service-control", user) + + assert developer_response.status_code == 200 + assert developer_response.json()["access"] == "docs_developer" + assert admin_response.status_code == 403 + + +@pytest.mark.asyncio +async def test_admin_and_super_admin_can_read_admin_docs(): + admin_response = await get_json( + "/api/v1/docs/zh/backend-system-service-control", + make_user(role="admin"), + ) + super_admin_response = await get_json( + "/api/v1/docs/zh/backend-system-service-control", + make_user(role="super_admin"), + ) + + assert admin_response.status_code == 200 + assert super_admin_response.status_code == 200 + + +@pytest.mark.asyncio +async def test_unknown_language_slug_and_path_traversal_do_not_read_files(): + bad_lang = await get_json("/api/v1/docs/fr/quickstart") + bad_slug = await get_json("/api/v1/docs/zh/not-a-doc") + traversal = await get_json("/api/v1/docs/zh/..%2Fmanual") + + assert bad_lang.status_code == 404 + assert bad_slug.status_code == 404 + assert traversal.status_code == 404 diff --git a/backend/tests/test_earth_news.py b/backend/tests/test_earth_news.py new file mode 100644 index 00000000..83661b0c --- /dev/null +++ b/backend/tests/test_earth_news.py @@ -0,0 +1,49 @@ +from datetime import UTC, datetime + +from app.services.earth_news import ParsedNewsItem, _serialize_item + + +def test_serialize_item_includes_region_anchor_for_cruise(): + item = ParsedNewsItem( + id="google-apac:test", + title="Example APAC story", + summary="Example summary", + url="https://example.com/story", + source="Example Source", + feed_name="Global Monitor / APAC", + feed_region="asia-pacific", + homepage_url="https://example.com", + published_at=datetime(2026, 4, 23, 2, 30, tzinfo=UTC), + ) + + payload = _serialize_item(item, active_region="asia-pacific") + + assert payload["latitude"] == 1.3521 + assert payload["longitude"] == 103.8198 + assert payload["location_label"] == "亚太" + assert payload["location_inferred"] is True + assert payload["is_focus_match"] is True + assert payload["published_at"] == "2026-04-23T02:30:00Z" + + +def test_serialize_item_falls_back_to_global_anchor(): + item = ParsedNewsItem( + id="custom:test", + title="Fallback story", + summary="Fallback summary", + url="https://example.com/fallback", + source="Fallback Source", + feed_name="Fallback Feed", + feed_region="unknown-region", + homepage_url="https://example.com", + published_at=None, + ) + + payload = _serialize_item(item, active_region="americas") + + assert payload["latitude"] == 20.0 + assert payload["longitude"] == 0.0 + assert payload["location_label"] == "全球" + assert payload["location_inferred"] is True + assert payload["is_focus_match"] is False + assert payload["published_at"] is None diff --git a/backend/tests/test_location_pipeline.py b/backend/tests/test_location_pipeline.py new file mode 100644 index 00000000..93c415a2 --- /dev/null +++ b/backend/tests/test_location_pipeline.py @@ -0,0 +1,429 @@ +"""Tests for the shared location resolution pipeline. + +Validates the abstraction itself: the protocol contract, the orchestrator, +each built-in resolver, and the pluggability promise (a custom resolver can +be slotted in without touching consumers). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from app.services.location import ( + InheritFromAnotherEntityResolver, + LocationCandidate, + LocationPipeline, + LocationQuery, + NominatimResolver, + RegistryResolver, + ResolverOutput, + SourceCoordinatesResolver, +) + + +# ── Test fixtures ──────────────────────────────────────────────────── + + +@pytest.fixture +def tmp_registry(tmp_path: Path) -> Path: + payload = { + "locations": [ + { + "canonical_name": "Test Site Alpha", + "aliases": ["alpha", "alpha-one", "Acme HQ"], + "operator": "Acme Networks", + "site": "Acme HQ", + "city": "Lyon", + "country": "France", + "latitude": 45.764, + "longitude": 4.8357, + "precision": "site", + "confidence": 0.92, + "source_note": "Test fixture", + "verified_at": "2026-05-08", + }, + { + "canonical_name": "Test Site Bravo", + "aliases": ["bravo"], + "operator": "Acme Networks", + "site": "Bravo POP", + "city": "Berlin", + "country": "Germany", + "latitude": 52.52, + "longitude": 13.405, + "precision": "city", + "confidence": 0.85, + }, + ], + "city_fallbacks": [ + { + "city": "Bhutan-Capital", + "country": "Bhutan", + "latitude": 27.4728, + "longitude": 89.639, + "precision": "city", + "confidence": 0.5, + } + ], + } + path = tmp_path / "registry.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +# ── SourceCoordinatesResolver ──────────────────────────────────────── + + +def test_source_coordinates_resolver_passes_through_valid_coordinates(): + resolver = SourceCoordinatesResolver() + query = LocationQuery( + name="Acme HQ", + source_latitude=45.0, + source_longitude=4.0, + country="France", + ) + output = resolver.resolve(query) + assert len(output.candidates) == 1 + candidate = output.candidates[0] + assert candidate.latitude == 45.0 + assert candidate.longitude == 4.0 + assert candidate.precision == "precise" + assert candidate.source == "source_coordinates" + assert candidate.needs_confirmation is False + + +def test_source_coordinates_resolver_skips_zero_coordinates(): + resolver = SourceCoordinatesResolver() + output = resolver.resolve( + LocationQuery(name="X", source_latitude=0.0, source_longitude=0.0) + ) + assert output.candidates == () + + +def test_source_coordinates_resolver_skips_when_missing(): + resolver = SourceCoordinatesResolver() + output = resolver.resolve(LocationQuery(name="X")) + assert output.candidates == () + + +# ── RegistryResolver ───────────────────────────────────────────────── + + +def test_registry_resolver_matches_alias(tmp_registry): + resolver = RegistryResolver(registry_path=tmp_registry) + resolver.reload() + output = resolver.resolve( + LocationQuery(name="alpha", country="France") + ) + candidates = list(output.candidates) + assert candidates, "should match registry entry" + assert any(c.matched_location_name == "Test Site Alpha" for c in candidates) + alpha = next(c for c in candidates if c.matched_location_name == "Test Site Alpha") + assert alpha.precision == "site" + assert alpha.confidence == pytest.approx(0.92) + assert alpha.needs_confirmation is True + assert alpha.location_verified_at is None + + +def test_registry_resolver_filters_country_mismatch(tmp_registry): + resolver = RegistryResolver(registry_path=tmp_registry) + resolver.reload() + # alpha is in France; query says Spain → should reject + output = resolver.resolve( + LocationQuery(name="alpha", country="Spain") + ) + assert all( + c.matched_location_name != "Test Site Alpha" for c in output.candidates + ) + + +def test_registry_resolver_emits_city_fallback_candidate(tmp_registry): + resolver = RegistryResolver(registry_path=tmp_registry) + resolver.reload() + output = resolver.resolve( + LocationQuery(city="Bhutan-Capital", country="Bhutan") + ) + candidates = list(output.candidates) + assert candidates, "city fallback should fire" + assert any(c.source == "local_registry_city" for c in candidates) + + +# ── NominatimResolver ─────────────────────────────────────────────── + + +def test_nominatim_resolver_calls_geocoder_with_plan_queries(): + calls = [] + + def fake_geocoder(query: str): + calls.append(query) + return { + "lat": "12.34", + "lon": "56.78", + "display_name": "Test City, Country", + "address": {"city": "Test City", "country": "Country"}, + } + + def plan(query: LocationQuery): + return [ + ("primary query", ("name",)), + ("secondary query", ("city",)), + ] + + resolver = NominatimResolver( + query_plan_builder=plan, + geocoder=fake_geocoder, + ) + output = resolver.resolve(LocationQuery(name="X", country="Country")) + assert calls == ["primary query", "secondary query"] + assert output.attempted_queries == ("primary query", "secondary query") + assert len(output.candidates) == 2 + assert all(c.precision == "city" for c in output.candidates) + assert all(c.needs_confirmation for c in output.candidates) + + +def test_nominatim_resolver_skips_when_geocoder_returns_none(): + resolver = NominatimResolver( + query_plan_builder=lambda q: [("only", ("name",))], + geocoder=lambda q: None, + ) + output = resolver.resolve(LocationQuery(name="X")) + assert output.candidates == () + assert output.attempted_queries == ("only",) + + +def test_nominatim_resolver_swallows_exceptions_per_query(): + def boom(query): + raise RuntimeError("network down") + + resolver = NominatimResolver( + query_plan_builder=lambda q: [("a", ()), ("b", ())], + geocoder=boom, + ) + output = resolver.resolve(LocationQuery(name="X")) + assert output.candidates == () + assert output.attempted_queries == ("a", "b") + + +# ── InheritFromAnotherEntityResolver ──────────────────────────────── + + +def test_inherit_resolver_returns_provided_candidate(): + sentinel = LocationCandidate( + latitude=10.0, + longitude=20.0, + display_name="Inherited", + precision="city", + confidence=0.7, + query="inherit::test", + source="inherited", + source_note=None, + matched_fields=("collector",), + needs_confirmation=False, + ) + resolver = InheritFromAnotherEntityResolver( + source_lookup=lambda q: sentinel + ) + output = resolver.resolve(LocationQuery(name="X")) + assert output.candidates == (sentinel,) + + +def test_inherit_resolver_skips_when_lookup_returns_none(): + resolver = InheritFromAnotherEntityResolver(source_lookup=lambda q: None) + assert resolver.resolve(LocationQuery(name="X")).candidates == () + + +# ── LocationPipeline orchestration ────────────────────────────────── + + +def test_pipeline_aggregates_candidates_across_resolvers(tmp_registry): + pipeline = LocationPipeline( + [ + SourceCoordinatesResolver(), + RegistryResolver(registry_path=tmp_registry), + NominatimResolver( + query_plan_builder=lambda q: [("nominatim attempt", ("name",))], + geocoder=lambda q: { + "lat": "1.0", + "lon": "2.0", + "display_name": "Online City", + "address": {"city": "Online City", "country": "France"}, + }, + ), + ] + ) + pipeline.resolvers[1].reload() + candidates, attempted = pipeline.collect_candidates( + LocationQuery( + name="alpha", + country="France", + source_latitude=44.0, + source_longitude=5.0, + ) + ) + sources = {c.source for c in candidates} + assert "source_coordinates" in sources + assert "local_registry" in sources + assert "nominatim_online_geocode" in sources + assert "nominatim attempt" in attempted + + +def test_pipeline_dedupes_by_source_and_coordinates(): + same = LocationCandidate( + latitude=1.0, + longitude=2.0, + display_name="dup", + precision="city", + confidence=0.5, + query="x", + source="dup_source", + source_note=None, + matched_fields=(), + needs_confirmation=False, + ) + + class _DupResolver: + name = "dup_source" + + def resolve(self, query): + return ResolverOutput(candidates=(same, same)) + + pipeline = LocationPipeline([_DupResolver()]) + candidates, _ = pipeline.collect_candidates(LocationQuery(name="X")) + assert len(candidates) == 1 + + +def test_registry_short_aliases_do_not_match_inside_larger_tokens(tmp_path: Path): + registry_path = tmp_path / "registry.json" + registry_path.write_text( + json.dumps( + { + "locations": [ + { + "canonical_name": "Aurora", + "aliases": ["Aurora", "ANL"], + "site": "DOE/SC/Argonne National Laboratory", + "country": "United States", + "city": "Lemont", + "latitude": 41.713, + "longitude": -87.982, + "precision": "site", + }, + { + "canonical_name": "Venado", + "aliases": ["Venado"], + "site": "DOE/NNSA/LANL", + "country": "United States", + "city": "Los Alamos", + "latitude": 35.8443, + "longitude": -106.2872, + "precision": "site", + }, + ], + "city_fallbacks": [], + } + ), + encoding="utf-8", + ) + resolver = RegistryResolver(registry_path=registry_path) + resolver.reload() + + output = resolver.resolve( + LocationQuery( + name="Venado", + country="United States", + extra={"site": "DOE/NNSA/LANL"}, + ) + ) + + assert len(output.candidates) == 1 + assert output.candidates[0].matched_location_name == "Venado" + + +def test_pipeline_resolve_best_returns_highest_priority(): + online = LocationCandidate( + latitude=10.0, + longitude=20.0, + display_name="online", + precision="city", + confidence=0.9, + query="x", + source="nominatim_online_geocode", + source_note=None, + matched_fields=(), + needs_confirmation=True, + ) + source = LocationCandidate( + latitude=11.0, + longitude=21.0, + display_name="src", + precision="precise", + confidence=1.0, + query="x", + source="source_coordinates", + source_note=None, + matched_fields=(), + needs_confirmation=False, + ) + + class _StubResolver: + def __init__(self, c, name): + self._c = c + self.name = name + + def resolve(self, query): + return ResolverOutput(candidates=(self._c,)) + + pipeline = LocationPipeline( + [ + _StubResolver(online, "online"), + _StubResolver(source, "src"), + ] + ) + result = pipeline.resolve_best(LocationQuery(name="X")) + assert result.location is source, "source_coordinates should beat nominatim" + + +def test_pipeline_returns_diagnostic_when_nothing_resolves(): + pipeline = LocationPipeline([SourceCoordinatesResolver()]) + result = pipeline.resolve_best(LocationQuery(name="X", country="Bhutan")) + assert result.location is None + assert result.diagnostic is not None + assert result.diagnostic.country == "Bhutan" + + +def test_pluggability_custom_resolver_works_without_changing_pipeline(): + """Validates the abstraction promise: a new algorithm = a new class.""" + + class _PeeringDBStubResolver: + name = "fake_peeringdb" + + def resolve(self, query): + asn = (query.extra or {}).get("asn") + if asn != 174: + return ResolverOutput() + return ResolverOutput( + candidates=( + LocationCandidate( + latitude=1.0, + longitude=2.0, + display_name="Cogent HQ", + precision="site", + confidence=0.8, + query=f"peeringdb::{asn}", + source="peeringdb_stub", + source_note="Stub for testing", + matched_fields=("asn",), + needs_confirmation=False, + ), + ) + ) + + pipeline = LocationPipeline([_PeeringDBStubResolver()]) + candidates, _ = pipeline.collect_candidates( + LocationQuery(name="X", extra={"asn": 174}) + ) + assert len(candidates) == 1 + assert candidates[0].source == "peeringdb_stub" diff --git a/backend/tests/test_logging.py b/backend/tests/test_logging.py new file mode 100644 index 00000000..53f790ff --- /dev/null +++ b/backend/tests/test_logging.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import logging + +from io import StringIO + +from app.core.logging import PlanetContextFilter, PlanetFormatter, get_logger +from app.core.request_context import set_request_id + + +def _capture_output(callback): + stream = StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(PlanetFormatter(datefmt="%Y-%m-%d %H:%M:%S")) + handler.addFilter(PlanetContextFilter()) + + adapter = get_logger("tests.logging") + target_logger = adapter.logger + original_handlers = list(target_logger.handlers) + original_level = target_logger.level + original_propagate = target_logger.propagate + + target_logger.handlers = [handler] + target_logger.setLevel(logging.INFO) + target_logger.propagate = False + + try: + callback(adapter) + finally: + handler.flush() + target_logger.handlers = original_handlers + target_logger.setLevel(original_level) + target_logger.propagate = original_propagate + + return stream.getvalue() + + +def test_structured_logger_injects_request_id_and_event(): + set_request_id("req-test-123") + try: + output = _capture_output( + lambda logger: logger.info_event( + "collector started", + event="collector.run.started", + context={"collector_name": "bgp_news"}, + ) + ) + finally: + set_request_id(None) + + assert "request_id=req-test-123" in output + assert "event=collector.run.started" in output + assert "service=backend" in output + assert '"collector_name": "bgp_news"' in output + + +def test_structured_logger_redacts_sensitive_text_and_context(): + set_request_id("req-test-redact") + try: + output = _capture_output( + lambda logger: logger.error_event( + "Authorization: Bearer super-secret-token", + event="auth.token.failed", + context={ + "token": "plain-secret", + "nested": {"password": "hunter2"}, + "safe": "visible", + }, + ) + ) + finally: + set_request_id(None) + + assert "super-secret-token" not in output + assert "plain-secret" not in output + assert "hunter2" not in output + assert "[REDACTED]" in output + assert '"safe": "visible"' in output diff --git a/backend/tests/test_models.py b/backend/tests/test_models.py index 33c4aae9..d63846ab 100644 --- a/backend/tests/test_models.py +++ b/backend/tests/test_models.py @@ -121,6 +121,25 @@ class TestCollectionTaskModel: ) assert task.records_processed == 100 + def test_task_with_phase_progress(self): + """Test collection task phase-level progress fields""" + task = CollectionTask( + datasource_id=1, + status="running", + phase="fetching", + phase_progress=42.5, + phase_message="Downloading dataset", + phase_current=1024, + phase_total=4096, + phase_unit="bytes", + ) + assert task.phase == "fetching" + assert task.phase_progress == 42.5 + assert task.phase_message == "Downloading dataset" + assert task.phase_current == 1024 + assert task.phase_total == 4096 + assert task.phase_unit == "bytes" + def test_task_error_message(self): """Test collection task with error message""" task = CollectionTask( diff --git a/backend/tests/test_system_logs.py b/backend/tests/test_system_logs.py new file mode 100644 index 00000000..f2b8d127 --- /dev/null +++ b/backend/tests/test_system_logs.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import json + +from pathlib import Path + +from app.services import system_logs + + +class FakeRedis: + def __init__(self) -> None: + self.store: dict[str, list[str]] = {} + + def rpush(self, key: str, value: str) -> None: + self.store.setdefault(key, []).append(value) + + def ltrim(self, key: str, start: int, end: int) -> None: + items = self.store.get(key, []) + normalized_end = None if end == -1 else end + 1 + self.store[key] = items[start:normalized_end] + + def expire(self, key: str, seconds: int) -> None: + return None + + def lrange(self, key: str, start: int, end: int) -> list[str]: + items = self.store.get(key, []) + normalized_end = None if end == -1 else end + 1 + return items[start:normalized_end] + + def llen(self, key: str) -> int: + return len(self.store.get(key, [])) + + +def test_read_log_snapshot_uses_structured_buffer_timestamp_level_and_search(monkeypatch): + fake_redis = FakeRedis() + monkeypatch.setattr(system_logs, "redis_client", fake_redis) + monkeypatch.setattr( + system_logs, + "LOG_SOURCES", + { + "earth-client": system_logs.LogSource( + source_id="earth-client", + name="Earth 浏览器端", + kind="buffer", + location="redis://planet:system_logs:earth-client", + description="Earth 浏览器端上报日志", + category="client", + buffer_key=system_logs.get_buffer_log_key("earth-client"), + ) + }, + ) + + fake_redis.rpush( + system_logs.get_buffer_log_key("earth-client"), + json.dumps( + { + "timestamp": "2026-04-22T10:15:30Z", + "level": "warning", + "message": "news feed degraded", + "context": {"module": "news", "detail": "timeout"}, + }, + ensure_ascii=False, + ), + ) + fake_redis.rpush( + system_logs.get_buffer_log_key("earth-client"), + json.dumps( + { + "timestamp": "2026-04-23T06:01:00Z", + "level": "error", + "message": "landing points failed", + "context": {"module": "layer-startup", "detail": "http 500"}, + }, + ensure_ascii=False, + ), + ) + + snapshot = system_logs.read_log_snapshot( + "earth-client", + 50, + levels="error,warning", + start_date="2026-04-23", + end_date="2026-04-23", + search="landing", + ) + + assert snapshot is not None + assert snapshot["selected_levels"] == ["error", "warning"] + assert snapshot["search_query"] == "landing" + assert snapshot["line_count"] == 1 + assert snapshot["lines"][0].startswith("2026-04-23 06:01:00 ERROR landing points failed") + assert snapshot["daily_markers"] == [ + {"date_token": "2026-04-23", "total": 1, "dominant_level": "error"} + ] + + +def test_read_log_snapshot_parses_file_timestamp_and_builds_markers(tmp_path: Path, monkeypatch): + log_path = tmp_path / "backend.log" + log_path.write_text( + "\n".join( + [ + "2026-04-22 08:00:00 INFO service booted", + "2026-04-23 09:15:00 WARNING disk pressure detected", + "2026-04-23 09:16:00 ERROR sync failed", + "2026-04-24 10:00:00 DEBUG collector trace", + ] + ), + encoding="utf-8", + ) + + monkeypatch.setattr( + system_logs, + "LOG_SOURCES", + { + "backend": system_logs.LogSource( + source_id="backend", + name="后端服务", + kind="file", + location=str(log_path), + description="测试文件日志", + category="service", + ) + }, + ) + + snapshot = system_logs.read_log_snapshot( + "backend", + 50, + levels="warning,error", + search="failed", + ) + + assert snapshot is not None + assert snapshot["line_count"] == 1 + assert snapshot["lines"] == ["2026-04-23 09:16:00 ERROR sync failed"] + assert snapshot["daily_markers"] == [ + {"date_token": "2026-04-23", "total": 1, "dominant_level": "error"} + ] + assert snapshot["status"] == "ok" + + +def test_append_buffer_log_persists_normalized_level(monkeypatch): + fake_redis = FakeRedis() + monkeypatch.setattr(system_logs, "redis_client", fake_redis) + + system_logs.append_buffer_log( + "earth-client", + level="warn", + message="feed delayed", + context={"module": "news"}, + ) + + stored_items = fake_redis.lrange(system_logs.get_buffer_log_key("earth-client"), 0, -1) + payload = json.loads(stored_items[0]) + assert payload["level"] == "warning" + assert payload["message"] == "feed delayed" + + +def test_infer_log_level_prefers_leading_prefix_over_query_string(): + line = 'INFO: 127.0.0.1 - "GET /api/v1/system/logs/backend?limit=200&level=error&levels=error HTTP/1.1" 200 OK' + + entry = system_logs.parse_text_log_entry(line) + + assert entry.level == "info" + + +def test_parse_text_log_entry_does_not_promote_exception_context_to_error(): + line = "websockets.exceptions.ConnectionClosedError: sent 1011 (internal error) keepalive ping timeout" + + entry = system_logs.parse_text_log_entry(line) + + assert entry.level is None + + +def test_parse_text_log_entry_still_detects_explicit_error_prefix(): + line = "ERROR: [Errno 98] Address already in use" + + entry = system_logs.parse_text_log_entry(line) + + assert entry.level == "error" + + +def test_read_log_snapshot_strips_nul_bytes_from_file_lines(tmp_path: Path, monkeypatch): + log_path = tmp_path / "backend.log" + log_path.write_bytes( + ( + b"INFO: service booted\n" + b"ERROR: bind failed\n" + + b"\x00" * 32 + + b"2026-04-23 23:41:32 INFO service=backend message=request served\n" + ) + ) + + monkeypatch.setattr( + system_logs, + "LOG_SOURCES", + { + "backend": system_logs.LogSource( + source_id="backend", + name="后端服务", + kind="file", + location=str(log_path), + description="测试文件日志", + category="service", + ) + }, + ) + + snapshot = system_logs.read_log_snapshot("backend", 50) + + assert snapshot is not None + assert snapshot["line_count"] == 3 + assert snapshot["lines"] == [ + "INFO: service booted", + "ERROR: bind failed", + "2026-04-23 23:41:32 INFO service=backend message=request served", + ] diff --git a/backend/tests/test_vessel_aggregation_strategy.py b/backend/tests/test_vessel_aggregation_strategy.py new file mode 100644 index 00000000..a849c348 --- /dev/null +++ b/backend/tests/test_vessel_aggregation_strategy.py @@ -0,0 +1,161 @@ +"""Tests for the v4 vessel_ais aggregation strategy.""" + +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest + +from app.models.vessel import AISRawObservation +from app.services.vessel_aggregation_strategy import ( + DEFAULT_STRATEGY, + StrategyValidationError, + validate_strategy, +) +from app.services.vessel_ais_aggregation import aggregate_vessel_observations + + +def _obs(*, source: str, mmsi: int, observed_at: datetime, **payload) -> AISRawObservation: + payload = {"mmsi": mmsi, "lat": 50.0, "lon": 10.0, **payload} + delivery_mode = "realtime_stream" if source == "aisstream_vessels" else "polling" + transport = "websocket" if source == "aisstream_vessels" else "http" + return AISRawObservation( + target_schema="vessel_ais", + source=source, + entity_key=str(mmsi), + delivery_mode=delivery_mode, + transport=transport, + message_type="PositionReport", + observation_hash=f"{source}:{mmsi}:{observed_at.isoformat()}", + observed_at=observed_at, + collected_at=observed_at, + normalized_payload=payload, + raw_payload=payload, + quality_flags=[], + ) + + +def test_validate_rejects_unknown_field(): + with pytest.raises(StrategyValidationError, match="unknown vessel_ais field"): + validate_strategy({"vessel_ais": {"field_rules": {"definitely_not_a_field": {"mode": "newest"}}}}) + + +def test_validate_rejects_dynamic_lock_without_flag(): + with pytest.raises(StrategyValidationError, match="allow_dynamic_lock"): + validate_strategy( + { + "vessel_ais": { + "field_rules": {"lat": {"mode": "source_priority"}}, + "allow_dynamic_lock": False, + } + } + ) + + +def test_validate_allows_dynamic_lock_with_flag(): + normalized = validate_strategy( + { + "version": 0, + "vessel_ais": { + "field_rules": {"lat": {"mode": "source_priority", "source_priority": ["barentswatch_vessels"]}}, + "allow_dynamic_lock": True, + }, + } + ) + assert normalized["vessel_ais"]["field_rules"]["lat"]["mode"] == "source_priority" + assert normalized["version"] == 1 + + +def test_validate_increments_version(): + first = validate_strategy({"version": 5, "vessel_ais": {}}) + assert first["version"] == 6 + + +@pytest.mark.asyncio +async def test_strategy_field_rule_promotes_specific_source(monkeypatch): + now = datetime(2026, 5, 4, 12, 0, tzinfo=timezone.utc) + + obs_a = _obs( + source="aisstream_vessels", + mmsi=257123000, + observed_at=now, + name="AISSTREAM ONE", + vessel_type_name="Cargo", + ) + obs_b = _obs( + source="barentswatch_vessels", + mmsi=257123000, + observed_at=now - timedelta(seconds=1), + name="BARENTSWATCH ONE", + vessel_type_name="Cargo", + ) + + strategy = { + "version": 7, + "vessel_ais": { + "source_priority": [], + "field_rules": { + "name": {"mode": "source_priority", "source_priority": ["barentswatch_vessels", "aisstream_vessels"]}, + }, + "freshness": {"realtime_stream_seconds": 0, "polling_seconds": 0}, + "allow_dynamic_lock": False, + }, + } + + db = AsyncMock() + vessels = await aggregate_vessel_observations( + db, + [obs_a, obs_b], + write_conflicts=False, + strategy=strategy, + ) + assert len(vessels) == 1 + vessel = vessels[0] + assert vessel["name"] == "BARENTSWATCH ONE" + assert vessel["field_sources"]["name"] == "barentswatch_vessels" + assert vessel["selected_reasons"]["name"] == "source_priority" + assert vessel["aggregation_strategy_version"] == 7 + + +@pytest.mark.asyncio +async def test_strategy_freshness_falls_back_to_polling_when_realtime_stale(): + now = datetime(2026, 5, 4, 12, 0, tzinfo=timezone.utc) + + stale_realtime = _obs( + source="aisstream_vessels", + mmsi=257123000, + observed_at=now - timedelta(hours=1), + lat=58.0, + lon=10.0, + ) + fresh_polling = _obs( + source="barentswatch_vessels", + mmsi=257123000, + observed_at=now - timedelta(seconds=30), + lat=60.0, + lon=11.0, + ) + + strategy = { + "version": 1, + "vessel_ais": { + "source_priority": ["aisstream_vessels", "barentswatch_vessels"], + "field_rules": {}, + "freshness": {"realtime_stream_seconds": 900, "polling_seconds": 7200}, + "allow_dynamic_lock": False, + }, + } + + db = AsyncMock() + vessels = await aggregate_vessel_observations( + db, + [stale_realtime, fresh_polling], + write_conflicts=False, + strategy=strategy, + ) + assert vessels[0]["field_sources"]["lat"] == "barentswatch_vessels" + assert vessels[0]["lat"] == 60.0 + + +def test_default_strategy_is_stable(): + assert DEFAULT_STRATEGY["vessel_ais"]["allow_dynamic_lock"] is False + assert "freshness" in DEFAULT_STRATEGY["vessel_ais"] diff --git a/backend/tests/test_vessel_enrichment.py b/backend/tests/test_vessel_enrichment.py new file mode 100644 index 00000000..74f9e1ce --- /dev/null +++ b/backend/tests/test_vessel_enrichment.py @@ -0,0 +1,155 @@ +"""Tests for v5 enrichment + conflict promote-to-rule.""" + +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest + +from app.models.vessel import AISConflictRecord, AISRawObservation +from app.models.vessel_enrichment import VesselMediaEnrichment, VesselProfileEnrichment +from app.services.vessel_ais_aggregation import aggregate_vessel_observations +from app.services.vessel_enrichment import ( + _apply_upsert, + get_vessel_enrichment_bundle, +) + + +class _StoreSession: + """Minimal AsyncSession stand-in that tracks mmsi-keyed enrichment + a strategy.""" + + def __init__(self, *, profile=None, media=None, conflicts=None): + self.profile = profile + self.media = media + self.conflicts = list(conflicts or []) + self.added: list = [] + self.committed = False + + async def get(self, model, key): + if model is VesselProfileEnrichment: + return self.profile if self.profile and self.profile.mmsi == key else None + if model is VesselMediaEnrichment: + return self.media if self.media and self.media.mmsi == key else None + return None + + +@pytest.mark.asyncio +async def test_enrichment_bundle_filters_expired_records(): + now = datetime.now(timezone.utc) + fresh = VesselProfileEnrichment( + mmsi=257123000, + source="local_cache", + payload={"vessel_subtype": "Container"}, + fetched_at=now - timedelta(hours=1), + expires_at=now + timedelta(days=7), + confidence=0.9, + ) + expired_media = VesselMediaEnrichment( + mmsi=257123000, + source="vesselfinder", + payload={"images": ["https://example.com/a.jpg"]}, + fetched_at=now - timedelta(days=30), + expires_at=now - timedelta(days=1), + ) + db = _StoreSession(profile=fresh, media=expired_media) + + bundle = await get_vessel_enrichment_bundle(db, 257123000) + + assert bundle["profile"]["payload"]["vessel_subtype"] == "Container" + assert bundle["media"] is None + + +def test_apply_upsert_preserves_payload_and_metadata(): + record = VesselProfileEnrichment(mmsi=257123000) + out = _apply_upsert( + record, + { + "source": "vesselfinder", + "payload": {"vessel_subtype": "Container", "operator": "Maersk"}, + "expires_at": "2026-12-31T00:00:00Z", + "confidence": 0.85, + "reference_url": "https://www.vesselfinder.com/vessels/257123000", + }, + ) + assert out["payload"]["operator"] == "Maersk" + assert out["confidence"] == 0.85 + assert record.reference_url == "https://www.vesselfinder.com/vessels/257123000" + assert record.expires_at is not None + assert record.expires_at.year == 2026 + + +def _obs(*, source: str, mmsi: int, observed_at, **payload) -> AISRawObservation: + payload = {"mmsi": mmsi, "lat": 60.0, "lon": 5.0, **payload} + delivery_mode = "realtime_stream" if source == "aisstream_vessels" else "polling" + transport = "websocket" if source == "aisstream_vessels" else "http" + return AISRawObservation( + target_schema="vessel_ais", + source=source, + entity_key=str(mmsi), + delivery_mode=delivery_mode, + transport=transport, + message_type="PositionReport", + observation_hash=f"{source}:{mmsi}:{observed_at.isoformat()}", + observed_at=observed_at, + collected_at=observed_at, + normalized_payload=payload, + raw_payload=payload, + quality_flags=[], + ) + + +@pytest.mark.asyncio +async def test_promoted_rule_wins_during_aggregation(): + """Simulate the strategy that conflict-promote-to-rule writes.""" + now = datetime.now(timezone.utc) + obs_a = _obs( + source="aisstream_vessels", + mmsi=257111000, + observed_at=now, + name="STREAM NAME", + vessel_type_name="Cargo", + ) + obs_b = _obs( + source="barentswatch_vessels", + mmsi=257111000, + observed_at=now - timedelta(seconds=1), + name="REST NAME", + vessel_type_name="Cargo", + ) + promoted_strategy = { + "version": 99, + "vessel_ais": { + "source_priority": [], + "field_rules": { + "name": {"mode": "source_priority", "source_priority": ["barentswatch_vessels"]} + }, + "freshness": {"realtime_stream_seconds": 0, "polling_seconds": 0}, + "allow_dynamic_lock": False, + }, + } + + db = AsyncMock() + vessels = await aggregate_vessel_observations( + db, + [obs_a, obs_b], + write_conflicts=False, + strategy=promoted_strategy, + ) + assert vessels[0]["name"] == "REST NAME" + assert vessels[0]["selected_reasons"]["name"] == "source_priority" + assert vessels[0]["aggregation_strategy_version"] == 99 + + +def test_conflict_record_holds_selected_source(): + """Sanity: the promote-to-rule API reads selected_source from this column.""" + record = AISConflictRecord( + target_schema="vessel_ais", + entity_key="257111000", + field="name", + candidates={"a": "X", "b": "Y"}, + selected_source="barentswatch_vessels", + selected_value="Y", + selected_reason="delivery_mode_priority", + ) + serialized = record.to_dict() + assert serialized["selected_source"] == "barentswatch_vessels" + assert serialized["field"] == "name" diff --git a/backend/tests/test_vessels.py b/backend/tests/test_vessels.py new file mode 100644 index 00000000..6f37efac --- /dev/null +++ b/backend/tests/test_vessels.py @@ -0,0 +1,675 @@ +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.api.v1 import visualization +from app.api.v1.visualization import convert_vessels_to_geojson +from app.db.session import get_db +from app.main import app +from app.models.vessel import AISRawObservation, VesselPosition, VesselStatic +from app.services import barentswatch +from app.services.collectors.aisstream import AISStreamCollector +from app.services.collectors.vessel_ais import VesselAISCollector +from app.services.vessel_ais_aggregation import ( + aggregate_vessel_observations, + build_field_conflict_candidates, + build_observation_hash, + record_vessel_ais_observation, +) + + +def test_vessel_collector_transforms_barentswatch_like_records(): + collector = VesselAISCollector() + records = collector.transform( + [ + { + "mmsi": "257123000", + "lat": "59.91", + "lon": "10.73", + "sog": 12.4, + "cog": 214, + "nav_status": 0, + "shipType": 70, + "name": "OSLO TRADER", + }, + {"mmsi": "bad", "lat": 120, "lon": 10}, + ] + ) + + assert len(records) == 1 + assert records[0]["mmsi"] == 257123000 + assert records[0]["vessel_type_name"] == "Cargo" + assert records[0]["lat"] == pytest.approx(59.91) + + +def test_vessel_observation_hash_is_stable_for_same_payload(): + observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc) + payload = { + "mmsi": 257123000, + "lat": 59.91, + "lon": 10.73, + "received_at": observed_at, + } + + first = build_observation_hash( + source="barentswatch_vessels", + entity_key="257123000", + message_type="PositionReport", + observed_at=observed_at, + normalized_payload=payload, + ) + second = build_observation_hash( + source="barentswatch_vessels", + entity_key="257123000", + message_type="PositionReport", + observed_at=observed_at, + normalized_payload=dict(reversed(payload.items())), + ) + + assert first == second + assert len(first) == 64 + + +@pytest.mark.asyncio +async def test_record_vessel_ais_observation_skips_existing_hash(): + observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc) + + class _Result: + def scalar_one_or_none(self): + return 123 + + class _Session: + def __init__(self): + self.added = [] + + async def execute(self, _stmt): + return _Result() + + def add(self, item): + self.added.append(item) + + db = _Session() + observation = await record_vessel_ais_observation( + db, + source="barentswatch_vessels", + normalized_payload={ + "mmsi": 257123000, + "lat": 59.91, + "lon": 10.73, + "received_at": observed_at, + }, + delivery_mode="polling", + transport="http", + observed_at=observed_at.isoformat(), + ) + + assert observation is None + assert db.added == [] + + +def test_build_field_conflict_candidates_from_raw_observations(): + observations = [ + AISRawObservation( + source="barentswatch_vessels", + normalized_payload={"name": "OSLO TRADER", "flag": "NO"}, + ), + AISRawObservation( + source="aisstream_vessels", + normalized_payload={"name": "OSLO TRADER II", "flag": "NO"}, + ), + ] + + conflicts = build_field_conflict_candidates(observations) + + assert conflicts == [ + { + "field": "name", + "candidates": { + "aisstream_vessels": "OSLO TRADER II", + "barentswatch_vessels": "OSLO TRADER", + }, + "status": "candidate", + } + ] + + +@pytest.mark.asyncio +async def test_aggregate_vessel_observations_prefers_realtime_and_records_conflict(): + observed_at = datetime.now(timezone.utc) - timedelta(minutes=5) + + class _Result: + def scalar_one_or_none(self): + return None + + class _Session: + def __init__(self): + self.added = [] + + async def execute(self, _stmt): + return _Result() + + def add(self, item): + self.added.append(item) + + db = _Session() + observations = [ + AISRawObservation( + id=1, + source="barentswatch_vessels", + entity_key="257123000", + delivery_mode="polling", + transport="http", + observed_at=observed_at, + collected_at=observed_at, + normalized_payload={ + "mmsi": 257123000, + "name": "OSLO TRADER", + "lat": 59.91, + "lon": 10.73, + }, + ), + AISRawObservation( + id=2, + source="aisstream_vessels", + entity_key="257123000", + delivery_mode="realtime_stream", + transport="websocket", + observed_at=observed_at + timedelta(seconds=10), + collected_at=observed_at + timedelta(seconds=10), + normalized_payload={ + "mmsi": 257123000, + "vessel_type": 79, + "lat": 59.92, + "lon": 10.74, + }, + raw_payload={"MetaData": {"ShipName": "OSLO TRADER II "}}, + ), + ] + + vessels = await aggregate_vessel_observations(db, observations) + + assert vessels[0]["lat"] == pytest.approx(59.92) + assert vessels[0]["field_sources"]["lat"] == "aisstream_vessels" + assert vessels[0]["name"] == "OSLO TRADER II" + assert vessels[0]["vessel_type_name"] == "Cargo" + assert vessels[0]["source_summary"]["aisstream_vessels"]["observation_count"] == 1 + assert vessels[0]["source_summary"]["barentswatch_vessels"]["delivery_mode"] == "polling" + assert vessels[0]["conflict_count"] == 0 + assert db.added == [] + + +@pytest.mark.asyncio +async def test_vessel_collector_writes_raw_observations_only(monkeypatch): + collector = VesselAISCollector() + collector.update_progress = AsyncMock() + record_observation = AsyncMock() + update_health = AsyncMock() + broadcast_custom = AsyncMock() + monkeypatch.setattr( + "app.services.collectors.vessel_ais.record_vessel_ais_observation", + record_observation, + ) + monkeypatch.setattr( + "app.services.collectors.vessel_ais.update_ais_source_health", + update_health, + ) + monkeypatch.setattr( + "app.services.collectors.vessel_ais.broadcaster.broadcast_custom", + broadcast_custom, + ) + + class _Session: + def __init__(self): + self.added = [] + self.committed = False + + async def get(self, *_args): + return None + + def add(self, item): + self.added.append(item) + + async def execute(self, _stmt): + return None + + async def commit(self): + self.committed = True + + db = _Session() + observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc) + + saved = await collector._save_data( + db, + [ + { + "mmsi": 257123000, + "name": "OSLO TRADER", + "lat": 59.91, + "lon": 10.73, + "received_at": observed_at, + } + ], + ) + + assert saved == 1 + assert db.committed is True + # BarentsWatch must funnel through the unified AIS pipeline only — no legacy writes. + assert not any(isinstance(item, VesselStatic) for item in db.added) + assert not any(isinstance(item, VesselPosition) for item in db.added) + record_observation.assert_awaited_once() + assert record_observation.await_args.kwargs["source"] == "barentswatch_vessels" + assert record_observation.await_args.kwargs["normalized_payload"]["mmsi"] == 257123000 + update_health.assert_awaited_once() + broadcast_custom.assert_awaited_once() + assert broadcast_custom.await_args.args[0] == "vessels" + assert broadcast_custom.await_args.args[1]["action"] == "upsert" + assert broadcast_custom.await_args.args[1]["vessels"][0]["mmsi_display"] == "257123000" + + +def test_aisstream_collector_normalizes_position_report(): + collector = AISStreamCollector() + + records = collector.transform( + [ + { + "MessageType": "PositionReport", + "MetaData": { + "MMSI": 257123000, + "ShipName": "OSLO TRADER ", + "time_utc": "2026-04-30T12:00:00Z", + }, + "Message": { + "PositionReport": { + "Latitude": 59.91, + "Longitude": 10.73, + "Sog": 12.4, + "Cog": 214, + "TrueHeading": 215, + "NavigationalStatus": 0, + } + }, + } + ] + ) + + assert len(records) == 1 + assert records[0]["mmsi"] == 257123000 + assert records[0]["lat"] == pytest.approx(59.91) + assert records[0]["name"] == "OSLO TRADER" + assert records[0]["_message_type"] == "PositionReport" + + +def test_aisstream_collector_maps_ship_static_type_name(): + collector = AISStreamCollector() + + records = collector.transform( + [ + { + "MessageType": "ShipStaticData", + "MetaData": { + "MMSI": 257123000, + "time_utc": "2026-04-30T12:00:00Z", + }, + "Message": { + "ShipStaticData": { + "Name": "OSLO TRADER", + "Type": 79, + "CallSign": "LAAB", + } + }, + } + ] + ) + + assert len(records) == 1 + assert records[0]["vessel_type"] == 79 + assert records[0]["vessel_type_name"] == "Cargo" + + +@pytest.mark.asyncio +async def test_aisstream_collector_writes_only_raw_observations(monkeypatch): + collector = AISStreamCollector() + collector.update_progress = AsyncMock() + record_observation = AsyncMock(return_value=object()) + update_health = AsyncMock() + monkeypatch.setattr( + "app.services.collectors.aisstream.record_vessel_ais_observation", + record_observation, + ) + monkeypatch.setattr( + "app.services.collectors.aisstream.update_ais_source_health", + update_health, + ) + + class _Session: + def __init__(self): + self.added = [] + self.committed = False + + def add(self, item): + self.added.append(item) + + async def commit(self): + self.committed = True + + db = _Session() + saved = await collector._save_data( + db, + [ + { + "mmsi": 257123000, + "lat": 59.91, + "lon": 10.73, + "received_at": datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc), + "_message_type": "PositionReport", + } + ], + ) + + assert saved == 1 + assert db.added == [] + assert db.committed is True + record_observation.assert_awaited_once() + assert record_observation.await_args.kwargs["source"] == "aisstream_vessels" + update_health.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_aisstream_stream_record_broadcasts_vessel_delta(monkeypatch): + collector = AISStreamCollector() + record_observation = AsyncMock(return_value=object()) + update_health = AsyncMock() + broadcast_custom = AsyncMock() + monkeypatch.setattr( + "app.services.collectors.aisstream.record_vessel_ais_observation", + record_observation, + ) + monkeypatch.setattr( + "app.services.collectors.aisstream.update_ais_source_health", + update_health, + ) + monkeypatch.setattr( + "app.services.collectors.aisstream.broadcaster.broadcast_custom", + broadcast_custom, + ) + + class _Session: + async def commit(self): + pass + + created = await collector._save_stream_record( + _Session(), + { + "mmsi": 257123000, + "lat": 59.91, + "lon": 10.73, + "cog": 214, + "received_at": datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc), + }, + ) + + assert created is True + record_observation.assert_awaited_once() + broadcast_custom.assert_awaited_once() + assert broadcast_custom.await_args.args[0] == "vessels" + assert broadcast_custom.await_args.args[1]["action"] == "upsert" + assert broadcast_custom.await_args.args[1]["vessels"][0]["mmsi_display"] == "257123000" + + +def test_barentswatch_reads_credentials_from_zshrc(tmp_path): + zshrc = tmp_path / ".zshrc" + zshrc.write_text( + "\n".join( + [ + "export BARENTSWATCH_CLIENT_ID='client-from-zshrc'", + 'export BARENTSWATCH_CLIENT_SECRET="secret-from-zshrc" # local dev credential', + ] + ), + encoding="utf-8", + ) + + values = barentswatch._read_zshrc_env(zshrc) + + assert values["BARENTSWATCH_CLIENT_ID"] == "client-from-zshrc" + assert values["BARENTSWATCH_CLIENT_SECRET"] == "secret-from-zshrc" + + +@pytest.mark.asyncio +async def test_barentswatch_resolves_config_from_zshrc(tmp_path, monkeypatch): + zshrc = tmp_path / ".zshrc" + zshrc.write_text( + "\n".join( + [ + "export BARENTSWATCH_CLIENT_ID=client-from-zshrc", + "export BARENTSWATCH_CLIENT_SECRET=secret-from-zshrc", + ] + ), + encoding="utf-8", + ) + monkeypatch.delenv("BARENTSWATCH_CLIENT_ID", raising=False) + monkeypatch.delenv("BARENTSWATCH_CLIENT_SECRET", raising=False) + monkeypatch.delenv("BARRENTSWATCH_CLIENT_ID", raising=False) + monkeypatch.delenv("BARRENTSWATCH_CLIENT_SECRET", raising=False) + monkeypatch.setattr(barentswatch.Path, "home", lambda: tmp_path) + + config = await barentswatch.resolve_barentswatch_config(None) + + assert config.client_id == "client-from-zshrc" + assert config.client_secret == "secret-from-zshrc" + assert config.credential_source == "~/.zshrc" + + +def test_convert_vessels_to_geojson(): + position = VesselPosition( + mmsi=257123000, + lat=59.91, + lon=10.73, + sog=12.4, + cog=214, + heading=215, + nav_status=0, + received_at=datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc), + ) + static = VesselStatic( + mmsi=257123000, + name="OSLO TRADER", + vessel_type=70, + vessel_type_name="Cargo", + flag="NO", + length=185, + ) + + payload = convert_vessels_to_geojson([(position, static)]) + + assert payload["type"] == "FeatureCollection" + assert payload["features"][0]["geometry"]["coordinates"] == [10.73, 59.91] + assert payload["features"][0]["properties"]["mmsi"] == 257123000 + assert payload["features"][0]["properties"]["vessel_type_name"] == "Cargo" + + +def test_convert_vessels_to_geojson_dedupes_mmsi_rows(): + first = VesselPosition( + mmsi=257123000, + lat=59.91, + lon=10.73, + received_at=datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc), + ) + duplicate = VesselPosition( + mmsi=257123000, + lat=60.01, + lon=10.83, + received_at=datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc), + ) + other = VesselPosition( + mmsi=257456000, + lat=60.3, + lon=5.3, + received_at=datetime(2026, 4, 28, 0, 59, tzinfo=timezone.utc), + ) + + payload = convert_vessels_to_geojson( + [ + (first, VesselStatic(mmsi=257123000, name="OSLO TRADER")), + (duplicate, VesselStatic(mmsi=257123000, name="OSLO TRADER DUP")), + (other, VesselStatic(mmsi=257456000, name="BERGEN FERRY")), + ] + ) + + mmsis = [feature["properties"]["mmsi"] for feature in payload["features"]] + assert mmsis == [257123000, 257456000] + assert payload["features"][0]["geometry"]["coordinates"] == [10.73, 59.91] + + +@pytest.mark.asyncio +async def test_vessels_geojson_endpoint_filters_type_and_bbox(): + now = datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc) + rows = [ + ( + VesselPosition(mmsi=1, lat=59.9, lon=10.7, received_at=now), + VesselStatic(mmsi=1, name="Cargo Ship", vessel_type=70, vessel_type_name="Cargo"), + ), + ( + VesselPosition(mmsi=2, lat=60.3, lon=5.3, received_at=now - timedelta(minutes=1)), + VesselStatic(mmsi=2, name="Passenger Ship", vessel_type=60, vessel_type_name="Passenger"), + ), + ] + + class _Result: + def all(self): + return rows + + class _FakeSession: + async def execute(self, _query): + return _Result() + + async def override_get_db(): + yield _FakeSession() + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get( + "/api/v1/visualization/geo/vessels", + params={"bbox": "0,50,20,70", "type": "cargo", "limit": 0}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["count"] == 1 + assert data["features"][0]["properties"]["name"] == "Cargo Ship" + assert data["stats"]["by_type"]["Cargo"] == 1 + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_vessels_geojson_merges_raw_and_legacy_sources(monkeypatch): + now = datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc) + monkeypatch.setattr( + visualization, + "get_aggregated_vessels", + AsyncMock( + return_value=[ + { + "mmsi": 1, + "lat": 59.9, + "lon": 10.7, + "received_at": now, + "name": "AISSTREAM SHIP", + "vessel_type_name": "Cargo", + "source_summary": {"aisstream_vessels": {"message_types": ["PositionReport"]}}, + } + ] + ), + ) + rows = [ + ( + VesselPosition(mmsi=1, lat=60.0, lon=10.8, received_at=now), + VesselStatic(mmsi=1, name="LEGACY DUP", vessel_type_name="Cargo"), + ), + ( + VesselPosition(mmsi=2, lat=60.3, lon=5.3, received_at=now), + VesselStatic(mmsi=2, name="BARENTSWATCH ONLY", vessel_type_name="Passenger"), + ), + ] + + class _Result: + def all(self): + return rows + + class _FakeSession: + async def execute(self, _query): + return _Result() + + async def override_get_db(): + yield _FakeSession() + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/visualization/geo/vessels") + + assert response.status_code == 200 + data = response.json() + names = {feature["properties"]["mmsi"]: feature["properties"]["name"] for feature in data["features"]} + assert data["count"] == 2 + assert names == {1: "AISSTREAM SHIP", 2: "BARENTSWATCH ONLY"} + assert data["diagnostics"]["legacy_backfilled_mmsi"] == 1 + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_vessel_name_fallbacks_reports_mmsi_display_names(monkeypatch): + now = datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc) + monkeypatch.setattr( + visualization, + "get_aggregated_vessels", + AsyncMock( + return_value=[ + { + "mmsi": 257123000, + "lat": 59.9, + "lon": 10.7, + "received_at": now, + "name": "MMSI 257123000", + "vessel_type_name": "Other", + "source_summary": { + "aisstream_vessels": { + "latest_observed_at": now, + "message_types": ["PositionReport"], + } + }, + } + ] + ), + ) + + class _Result: + def all(self): + return [] + + class _FakeSession: + async def execute(self, _query): + return _Result() + + async def override_get_db(): + yield _FakeSession() + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/visualization/vessels/name-fallbacks") + + assert response.status_code == 200 + data = response.json() + assert data["count"] == 1 + assert data["items"][0]["mmsi"] == "257123000" + assert data["items"][0]["message_types"] == ["PositionReport"] + finally: + app.dependency_overrides.clear() diff --git a/backend/tests/test_visualization_compute_centers.py b/backend/tests/test_visualization_compute_centers.py new file mode 100644 index 00000000..5e4d8d15 --- /dev/null +++ b/backend/tests/test_visualization_compute_centers.py @@ -0,0 +1,1001 @@ +from datetime import datetime, timezone + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.api.v1.visualization import convert_compute_centers_to_geojson +import app.services.compute_center_locations as compute_center_locations +from app.db.session import get_db +from app.main import app +from app.models.collected_data import CollectedData + + +def _build_record( + *, + record_id: int, + source: str, + data_type: str, + name: str, + country: str, + city: str, + latitude: float, + longitude: float, + metadata: dict, +): + return CollectedData( + id=record_id, + source=source, + data_type=data_type, + source_id=f"{source}-{record_id}", + name=name, + extra_data={ + "country": country, + "city": city, + "latitude": latitude, + "longitude": longitude, + **metadata, + }, + collected_at=datetime(2026, 4, 22, tzinfo=timezone.utc), + reference_date=datetime(2026, 4, 21, tzinfo=timezone.utc), + is_current=True, + ) + + +def test_convert_compute_centers_to_geojson_unifies_sources(): + top500_record = _build_record( + record_id=1, + source="top500", + data_type="supercomputer", + name="Frontier", + country="United States", + city="Oak Ridge", + latitude=35.93, + longitude=-84.31, + metadata={ + "rank": 1, + "manufacturer": "HPE", + "organization": "ORNL", + "rmax": 1102000.0, + "cores": 8730112, + "power": 21510.0, + }, + ) + gpu_record = _build_record( + record_id=2, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Colossus", + country="United States", + city="Memphis", + latitude=35.15, + longitude=-90.05, + metadata={ + "organization": "xAI", + "gpu_type": "H100", + "gpu_count": 100000, + "value": "20000", + "unit": "TFlop/s", + }, + ) + + payload = convert_compute_centers_to_geojson([top500_record, gpu_record]) + + assert payload["type"] == "FeatureCollection" + assert len(payload["features"]) == 2 + + supercomputer_feature = payload["features"][0] + assert supercomputer_feature["properties"]["site_type"] == "supercomputer" + assert supercomputer_feature["properties"]["capacity_unit"] == "GFlops" + assert supercomputer_feature["properties"]["capacity_band"] == "exascale" + assert supercomputer_feature["properties"]["operator"] == "ORNL" + assert supercomputer_feature["properties"]["location_precision"] == "precise" + assert supercomputer_feature["properties"]["is_estimated"] is False + assert supercomputer_feature["properties"]["location_source"] == "source_coordinates" + assert supercomputer_feature["properties"]["location_confidence"] == 1.0 + + gpu_feature = payload["features"][1] + assert gpu_feature["properties"]["site_type"] == "gpu_cluster" + assert gpu_feature["properties"]["vendor"] == "H100" + assert gpu_feature["properties"]["gpu_count"] == 100000 + assert gpu_feature["properties"]["capacity_band"] == "large" + assert gpu_feature["properties"]["location_precision"] == "precise" + + +def test_convert_compute_centers_to_geojson_accepts_source_coordinate_aliases(): + record = _build_record( + record_id=3, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Alias Coordinates", + country="United States", + city="New York", + latitude=0.0, + longitude=0.0, + metadata={ + "latitude": "", + "longitude": "", + "location": { + "lat": 40.7128, + "lng": -74.0060, + }, + "value": "1200", + "unit": "TFlop/s", + }, + ) + + payload = convert_compute_centers_to_geojson([record]) + + assert len(payload["features"]) == 1 + feature = payload["features"][0] + assert feature["geometry"]["coordinates"] == [-74.006, 40.7128] + assert feature["properties"]["location_source"] == "source_coordinates" + + +def test_compute_center_source_coordinates_win_over_stored_location(): + compute_center_locations.set_compute_center_location_cache({ + "top500:top500-31": { + "source": "top500", + "source_id": "top500-31", + "name": "Stored Wrong", + "latitude": 1.0, + "longitude": 2.0, + "precision": "city", + "confidence": 0.5, + "needs_confirmation": True, + } + }) + record = _build_record( + record_id=31, + source="top500", + data_type="supercomputer", + name="Source Wins", + country="United States", + city="Oak Ridge", + latitude=35.93, + longitude=-84.31, + metadata={"organization": "ORNL"}, + ) + + payload = convert_compute_centers_to_geojson([record]) + + assert payload["features"][0]["geometry"]["coordinates"] == [-84.31, 35.93] + assert payload["features"][0]["properties"]["location_source"] == "source_coordinates" + compute_center_locations.set_compute_center_location_cache({}) + + +def test_compute_center_geojson_uses_stored_location_when_source_coords_missing(): + compute_center_locations.set_compute_center_location_cache({ + "epoch_ai_gpu:epoch_ai_gpu-32": { + "source": "epoch_ai_gpu", + "source_id": "epoch_ai_gpu-32", + "name": "Stored Cluster", + "city": "Memphis", + "country": "United States", + "latitude": 35.1495, + "longitude": -90.049, + "precision": "city", + "confidence": 0.72, + "location_source": "manual_selection", + "source_note": "Saved by user", + "needs_confirmation": False, + "verified_at": "2026-05-08T00:00:00Z", + } + }) + record = _build_record( + record_id=32, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Stored Cluster", + country="United States", + city="", + latitude=0.0, + longitude=0.0, + metadata={"value": "1200", "unit": "TFlop/s"}, + ) + + payload = convert_compute_centers_to_geojson([record]) + + assert len(payload["features"]) == 1 + feature = payload["features"][0] + assert feature["geometry"]["coordinates"] == [-90.049, 35.1495] + assert feature["properties"]["location_source"] == "stored_compute_center_location" + assert feature["properties"]["needs_confirmation"] is False + compute_center_locations.set_compute_center_location_cache({}) + + +def test_convert_compute_centers_to_geojson_does_not_use_registry_aliases(): + registry_record = _build_record( + record_id=3, + source="top500", + data_type="supercomputer", + name="Frontier", + country="United States", + city="", + latitude=0.0, + longitude=0.0, + metadata={ + "organization": "Oak Ridge National Laboratory", + "rmax": 1102000.0, + }, + ) + + payload = convert_compute_centers_to_geojson([registry_record]) + + assert payload["features"] == [] + assert len(payload["unresolved"]) == 1 + assert payload["unresolved"][0]["name"] == "Frontier" + assert "source coords" in payload["unresolved"][0]["failure_reason"] + + +def test_convert_compute_centers_to_geojson_does_not_use_city_fallback(): + city_record = _build_record( + record_id=4, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Sample GPU Cluster", + country="United States", + city="San Francisco, CA", + latitude=0.0, + longitude=0.0, + metadata={ + "organization": "Sample Operator", + "value": "10000", + "unit": "TFlop/s", + }, + ) + + payload = convert_compute_centers_to_geojson([city_record]) + + assert payload["features"] == [] + assert len(payload["unresolved"]) == 1 + assert payload["unresolved"][0]["city"] == "San Francisco, CA" + + +def test_convert_compute_centers_to_geojson_does_not_online_geocode_on_startup(monkeypatch): + compute_center_locations._geocode_online.cache_clear() + + def _explode(_query): + raise AssertionError("startup GeoJSON must not call online geocoding") + + monkeypatch.setattr(compute_center_locations, "_geocode_online", _explode) + country_record = _build_record( + record_id=4, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Unknown Cluster", + country="France", + city="", + latitude=0.0, + longitude=0.0, + metadata={ + "organization": "Unknown Operator", + "value": "10000", + "unit": "TFlop/s", + }, + ) + + payload = convert_compute_centers_to_geojson([country_record]) + + assert payload["features"] == [] + assert len(payload["unresolved"]) == 1 + assert payload["unresolved"][0]["operator"] == "Unknown Operator" + + +def test_convert_compute_centers_to_geojson_records_diagnostics_when_online_geocode_fails(monkeypatch): + compute_center_locations._geocode_online.cache_clear() + monkeypatch.setattr(compute_center_locations, "_geocode_online", lambda _query: None) + country_record = _build_record( + record_id=5, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Unknown French Cluster", + country="France", + city="", + latitude=0.0, + longitude=0.0, + metadata={ + "organization": "Unknown Operator", + "value": "10000", + "unit": "TFlop/s", + }, + ) + + payload = convert_compute_centers_to_geojson([country_record]) + + assert payload["features"] == [] + assert len(payload["unresolved"]) == 1 + diagnostic = payload["unresolved"][0] + assert diagnostic["record_id"] == 5 + assert diagnostic["source_id"] == "epoch_ai_gpu-5" + assert diagnostic["country"] == "France" + assert diagnostic["operator"] == "Unknown Operator" + assert diagnostic["failure_reason"] + assert diagnostic["attempted_queries"] == [] + + +def test_convert_compute_centers_to_geojson_records_diagnostics_when_no_country(monkeypatch): + compute_center_locations._geocode_online.cache_clear() + monkeypatch.setattr(compute_center_locations, "_geocode_online", lambda _query: None) + unknown_record = _build_record( + record_id=6, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Unknown Offshore Cluster", + country="", + city="", + latitude=0.0, + longitude=0.0, + metadata={ + "organization": "Unknown Operator", + "value": "10000", + "unit": "TFlop/s", + }, + ) + + payload = convert_compute_centers_to_geojson([unknown_record]) + + assert payload["features"] == [] + assert len(payload["unresolved"]) == 1 + assert payload["unresolved"][0]["failure_reason"] + + +def test_convert_compute_centers_to_geojson_never_emits_zero_coordinates(monkeypatch): + compute_center_locations._geocode_online.cache_clear() + + def _zero_geocode(query): + return { + "lat": "0", + "lon": "0", + "display_name": "Null Island", + "address": {"city": "", "country": ""}, + } + + monkeypatch.setattr(compute_center_locations, "_geocode_online", _zero_geocode) + record = _build_record( + record_id=7, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Null Island Cluster", + country="", + city="", + latitude=0.0, + longitude=0.0, + metadata={"organization": "Null Inc"}, + ) + + payload = convert_compute_centers_to_geojson([record]) + + for feature in payload["features"]: + coords = feature["geometry"]["coordinates"] + assert coords[0] not in (0, 0.0) + assert coords[1] not in (0, 0.0) + + +def test_convert_compute_centers_to_geojson_rejects_country_or_unknown_precision(monkeypatch): + compute_center_locations._geocode_online.cache_clear() + monkeypatch.setattr(compute_center_locations, "_geocode_online", lambda _query: None) + record = _build_record( + record_id=8, + source="top500", + data_type="supercomputer", + name="Phantom System", + country="Liechtenstein", + city="", + latitude=0.0, + longitude=0.0, + metadata={"organization": "Phantom Operator", "rmax": 100.0}, + ) + + payload = convert_compute_centers_to_geojson([record]) + + for feature in payload["features"]: + assert feature["properties"]["location_precision"] in {"precise", "site", "city"} + assert payload["features"] == [] + assert payload["unresolved"], "phantom record must surface as diagnostic" + + +def test_resolve_full_returns_diagnostic_for_unresolved(monkeypatch): + compute_center_locations._geocode_online.cache_clear() + monkeypatch.setattr(compute_center_locations, "_geocode_online", lambda _query: None) + record = _build_record( + record_id=11, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Phantom Cluster", + country="Bhutan", + city="", + latitude=0.0, + longitude=0.0, + metadata={"organization": "Mystery Operator"}, + ) + result = compute_center_locations.resolve_compute_center_location_full(record, record.extra_data) + assert result.location is None + assert result.diagnostic is not None + assert result.diagnostic.failure_reason + assert result.diagnostic.country == "Bhutan" + + +def test_collect_location_candidates_ignores_registry_and_uses_online(monkeypatch): + compute_center_locations._geocode_online.cache_clear() + + def _fake_ror(query): + assert query == "Oak Ridge National Laboratory" + return { + "id": "https://ror.org/01qz5mb56", + "names": [ + {"types": ["ror_display"], "value": "Oak Ridge National Laboratory"} + ], + "locations": [ + { + "geonames_id": 4646571, + "geonames_details": { + "name": "Oak Ridge", + "country_subdivision_name": "Tennessee", + "country_name": "United States", + "lat": 36.01036, + "lng": -84.26964, + }, + } + ], + } + + monkeypatch.setattr(compute_center_locations, "_lookup_ror_organization", _fake_ror) + monkeypatch.setattr(compute_center_locations, "_geocode_online", lambda _query: None) + candidates, attempted = compute_center_locations.collect_location_candidates( + name="Frontier", + operator="Oak Ridge National Laboratory", + country="United States", + ) + assert candidates, "online source-traced query must produce a candidate" + best = candidates[0] + assert best.source == "ror_organization_registry" + assert best.precision == "city" + assert best.needs_confirmation is True + assert attempted[0] == "ror:Oak Ridge National Laboratory" + + +def test_collect_location_candidates_returns_online_when_registry_misses(monkeypatch): + compute_center_locations._geocode_online.cache_clear() + + def _fake_geocode(query): + if "Lyon" not in query and "Mystery Operator" not in query and "Lyon, France" not in query: + return None + return { + "lat": "45.7640", + "lon": "4.8357", + "display_name": "Lyon, Auvergne-Rhône-Alpes, France", + "address": {"city": "Lyon", "state": "Auvergne-Rhône-Alpes", "country": "France"}, + } + + monkeypatch.setattr(compute_center_locations, "_geocode_online", _fake_geocode) + monkeypatch.setattr(compute_center_locations, "_lookup_ror_organization", lambda _query: None) + candidates, attempted = compute_center_locations.collect_location_candidates( + name="Mystery System", + operator="Mystery Operator", + city="Lyon", + country="France", + ) + assert candidates, "online geocoding must produce a candidate" + online_candidates = [c for c in candidates if c.source == "nominatim_online_geocode"] + assert online_candidates, "must include at least one online candidate" + online = online_candidates[0] + assert online.precision == "city" + assert online.needs_confirmation is True + assert online.suggested_registry_entry is not None + assert attempted, "must record attempted query strings" + + +def test_collect_location_candidates_failure_returns_attempted_queries(monkeypatch): + compute_center_locations._geocode_online.cache_clear() + monkeypatch.setattr(compute_center_locations, "_lookup_ror_organization", lambda _query: None) + monkeypatch.setattr(compute_center_locations, "_geocode_online", lambda _query: None) + candidates, attempted = compute_center_locations.collect_location_candidates( + name="Mystery Offshore Cluster", + operator="Mystery Operator", + country="Bhutan", + ) + assert candidates == [] + assert attempted, "even on failure we record attempted queries for diagnostics" + + +@pytest.mark.asyncio +async def test_compute_centers_geojson_endpoint_returns_stats(): + records = [ + _build_record( + record_id=1, + source="top500", + data_type="supercomputer", + name="Frontier", + country="United States", + city="Oak Ridge", + latitude=35.93, + longitude=-84.31, + metadata={"rank": 1, "rmax": 1102000.0}, + ), + _build_record( + record_id=2, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Colossus", + country="United States", + city="Memphis", + latitude=35.15, + longitude=-90.05, + metadata={"value": "20000", "unit": "TFlop/s"}, + ), + ] + + class _ScalarResult: + def __init__(self, rows): + self._rows = rows + + def scalars(self): + class _Scalars: + def __init__(self, rows): + self._rows = rows + + def all(self): + return self._rows + + return _Scalars(self._rows) + + class _FakeSession: + async def execute(self, _query): + return _ScalarResult(records) + + async def override_get_db(): + yield _FakeSession() + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/visualization/geo/compute-centers") + + assert response.status_code == 200 + data = response.json() + assert data["count"] == 2 + assert data["stats"]["supercomputers"] == 1 + assert data["stats"]["gpu_clusters"] == 1 + assert data["features"][0]["properties"]["data_type"] == "compute_center" + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_visualization_geo_summary_returns_counts(monkeypatch): + records = [ + _build_record( + record_id=1, + source="arcgis_cables", + data_type="submarine_cable", + name="Test Cable", + country="", + city="", + latitude=0, + longitude=0, + metadata={ + "route_coordinates": [[[0, 0], [1, 1]]], + "status": "active", + }, + ), + _build_record( + record_id=2, + source="arcgis_landing_points", + data_type="landing_point", + name="Test Landing", + country="United States", + city="New York", + latitude=40.7, + longitude=-74.0, + metadata={"city_id": 10}, + ), + _build_record( + record_id=3, + source="celestrak_tle", + data_type="satellite_tle", + name="TESTSAT", + country="", + city="", + latitude=0, + longitude=0, + metadata={ + "norad_cat_id": 12345, + "tle_line1": "1 12345U 98067A 24001.00000000 .00000000 00000-0 00000-0 0 9991", + "tle_line2": "2 12345 51.6000 100.0000 0001000 10.0000 20.0000 15.50000000 01", + }, + ), + _build_record( + record_id=4, + source="top500", + data_type="supercomputer", + name="Frontier", + country="United States", + city="Oak Ridge", + latitude=35.93, + longitude=-84.31, + metadata={"rank": 1, "rmax": 1102000.0}, + ), + _build_record( + record_id=5, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Colossus", + country="United States", + city="Memphis", + latitude=35.15, + longitude=-90.05, + metadata={"value": "20000", "unit": "TFlop/s"}, + ), + ] + + class _ScalarResult: + def __init__(self, rows=None, scalar_value=None): + self._rows = rows or [] + self._scalar_value = scalar_value + + def scalar(self): + return self._scalar_value + + def all(self): + return list(self._rows) + + def scalars(self): + class _Scalars: + def __init__(self, rows): + self._rows = rows + + def all(self): + return self._rows + + return _Scalars(self._rows) + + class _FakeSession: + async def execute(self, query): + query_text = str(query).lower() + if "bgp_incidents" in query_text: + return _ScalarResult(scalar_value=2) + if "bgp_anomalies" in query_text: + return _ScalarResult(scalar_value=3) + if "ais_raw_observations" in query_text or "vessel_position" in query_text: + return _ScalarResult(rows=[]) + return _ScalarResult(rows=records) + + async def get(self, *_args, **_kwargs): + return None + + async def override_get_db(): + yield _FakeSession() + + async def _fake_build_bgp_collector_coverage(*_args, **_kwargs): + return [ + {"collector": "rrc00"}, + {"collector": "rrc01"}, + ] + + monkeypatch.setattr( + "app.api.v1.visualization.build_bgp_collector_coverage", + _fake_build_bgp_collector_coverage, + ) + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/visualization/geo/summary") + + assert response.status_code == 200 + stats = response.json()["stats"] + assert stats["cable_count"] == 1 + assert stats["landing_point_count"] == 1 + assert stats["satellite_count"] == 1 + assert stats["compute_center_count"] == 2 + assert stats["supercomputer_count"] == 1 + assert stats["gpu_cluster_count"] == 1 + assert stats["bgp_event_count"] == 2 + assert stats["bgp_incident_count"] == 2 + assert stats["bgp_anomaly_count"] == 3 + assert stats["bgp_collector_count"] == 2 + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_collect_location_endpoint_returns_candidates_for_known_record(monkeypatch): + def _fake_ror(query): + assert query == "Oak Ridge National Laboratory" + return { + "id": "https://ror.org/01qz5mb56", + "names": [ + {"types": ["ror_display"], "value": "Oak Ridge National Laboratory"} + ], + "locations": [ + { + "geonames_id": 4646571, + "geonames_details": { + "name": "Oak Ridge", + "country_subdivision_name": "Tennessee", + "country_name": "United States", + "lat": 36.01036, + "lng": -84.26964, + }, + } + ], + } + + monkeypatch.setattr(compute_center_locations, "_lookup_ror_organization", _fake_ror) + monkeypatch.setattr(compute_center_locations, "_geocode_online", lambda _query: None) + + target_record = _build_record( + record_id=42, + source="top500", + data_type="supercomputer", + name="Frontier", + country="United States", + city="", + latitude=0.0, + longitude=0.0, + metadata={"organization": "Oak Ridge National Laboratory", "rmax": 1102000.0}, + ) + + class _ScalarResult: + def __init__(self, rows): + self._rows = rows + + def scalars(self): + class _Scalars: + def __init__(self, rows): + self._rows = rows + + def first(self): + return self._rows[0] if self._rows else None + + def all(self): + return self._rows + + return _Scalars(self._rows) + + class _FakeSession: + async def execute(self, _query): + return _ScalarResult([target_record]) + + async def override_get_db(): + yield _FakeSession() + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.post( + "/api/v1/visualization/compute-centers/top500-42/collect-location", + json={ + "name": "Frontier", + "operator": "Oak Ridge National Laboratory", + "country": "United States", + }, + ) + assert response.status_code == 200 + body = response.json() + assert body["success"] is True + assert body["candidates"], "must include candidates" + best = body["best_candidate"] + assert best["precision"] in {"precise", "site", "city"} + assert best["source"] == "ror_organization_registry" + assert best["needs_confirmation"] is True + assert best["matched_fields"], "matched_fields must be populated" + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_collect_location_endpoint_returns_failure_reason(monkeypatch): + monkeypatch.setattr(compute_center_locations, "_lookup_ror_organization", lambda _query: None) + monkeypatch.setattr(compute_center_locations, "_geocode_online", lambda _query: None) + + class _ScalarResult: + def __init__(self, rows): + self._rows = rows + + def scalars(self): + class _Scalars: + def __init__(self, rows): + self._rows = rows + + def first(self): + return self._rows[0] if self._rows else None + + def all(self): + return self._rows + + return _Scalars(self._rows) + + class _FakeSession: + async def execute(self, _query): + return _ScalarResult([]) + + async def override_get_db(): + yield _FakeSession() + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.post( + "/api/v1/visualization/compute-centers/epoch-mystery-99/collect-location", + json={ + "name": "Mystery Cluster", + "operator": "Mystery Operator", + "country": "Bhutan", + }, + ) + assert response.status_code == 200 + body = response.json() + assert body["success"] is False + assert body["failure_reason"] + assert body["candidates"] == [] + assert body["attempted_queries"], "must include attempted queries" + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_save_location_endpoint_upserts_and_geojson_can_render(): + target_record = _build_record( + record_id=52, + source="epoch_ai_gpu", + data_type="gpu_cluster", + name="Saved Cluster", + country="United States", + city="", + latitude=0.0, + longitude=0.0, + metadata={"value": "1200", "unit": "TFlop/s"}, + ) + + class _ScalarResult: + def __init__(self, rows): + self._rows = rows + + def scalars(self): + class _Scalars: + def __init__(self, rows): + self._rows = rows + + def first(self): + return self._rows[0] if self._rows else None + + def all(self): + return self._rows + + return _Scalars(self._rows) + + class _FakeSession: + def __init__(self): + self.saved = [] + + async def execute(self, _query): + if self.saved: + return _ScalarResult(self.saved) + return _ScalarResult([target_record]) + + async def scalar(self, _query): + return None + + def add(self, record): + self.saved.append(record) + + async def commit(self): + return None + + async def refresh(self, _record): + return None + + fake_session = _FakeSession() + + async def override_get_db(): + yield fake_session + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.post( + "/api/v1/visualization/compute-centers/epoch_ai_gpu-52/location", + json={ + "source": "epoch_ai_gpu", + "name": "Saved Cluster", + "latitude": 35.1495, + "longitude": -90.049, + "precision": "city", + "confidence": 0.72, + "location_source": "ror_organization_registry", + "source_note": "Selected by user", + "raw_payload": {"source": "ror_organization_registry"}, + }, + ) + assert response.status_code == 200 + body = response.json() + assert body["success"] is True + assert fake_session.saved + + payload = convert_compute_centers_to_geojson([target_record]) + assert len(payload["features"]) == 1 + feature = payload["features"][0] + assert feature["geometry"]["coordinates"] == [-90.049, 35.1495] + assert feature["properties"]["location_source"] == "stored_compute_center_location" + finally: + app.dependency_overrides.clear() + compute_center_locations.set_compute_center_location_cache({}) + + +def test_resolution_chain_orders_source_coords_first(monkeypatch): + def _explode(_query): + raise AssertionError("source coords must short-circuit before online geocoding") + + monkeypatch.setattr(compute_center_locations, "_geocode_online", _explode) + record = _build_record( + record_id=20, + source="top500", + data_type="supercomputer", + name="Frontier", + country="United States", + city="Oak Ridge", + latitude=35.93, + longitude=-84.31, + metadata={"organization": "ORNL"}, + ) + result = compute_center_locations.resolve_compute_center_location_full(record, record.extra_data) + assert result.is_resolved + assert result.location.location_precision == "precise" + assert result.location.location_source == "source_coordinates" + + +def test_no_country_centroid_or_major_compute_city_fallback(monkeypatch): + monkeypatch.setattr(compute_center_locations, "_geocode_online", lambda _query: None) + record = _build_record( + record_id=21, + source="top500", + data_type="supercomputer", + name="Phantom System", + country="France", + city="", + latitude=0.0, + longitude=0.0, + metadata={"organization": "Phantom Operator"}, + ) + result = compute_center_locations.resolve_compute_center_location_full(record, record.extra_data) + assert result.location is None, "must NOT fall back to country centroid or hashed major city" + assert result.diagnostic is not None + assert result.diagnostic.failure_reason + + +def test_repository_has_no_forbidden_precision_tokens(): + """Static guard: forbidden fallback strategies must not regress into the codebase. + + Each forbidden token may appear at most once per target file, and only inside + the FORBIDDEN_PRECISIONS guard list (so we still reject them at runtime). + """ + from pathlib import Path + + backend_root = Path(__file__).resolve().parents[1] + forbidden_tokens = ( + "country_centroid", + "country_major_compute_city", + "estimated_country", + ) + targets = [ + backend_root / "app" / "services" / "compute_center_locations.py", + backend_root / "app" / "api" / "v1" / "visualization.py", + ] + for target in targets: + text = target.read_text(encoding="utf-8") + for token in forbidden_tokens: + occurrences = text.count(token) + assert occurrences <= 1, ( + f"{token} appears {occurrences} times in {target}; " + "should only appear in FORBIDDEN_PRECISIONS guard list." + ) + if occurrences == 1: + assert "FORBIDDEN_PRECISIONS" in text, ( + f"{token} appears in {target} outside the FORBIDDEN_PRECISIONS guard" + ) diff --git a/backend/tests/test_websocket_manager.py b/backend/tests/test_websocket_manager.py new file mode 100644 index 00000000..498447a9 --- /dev/null +++ b/backend/tests/test_websocket_manager.py @@ -0,0 +1,46 @@ +import pytest + +from app.core.websocket.manager import ConnectionManager + + +class FakeWebSocket: + def __init__(self): + self.accepted = False + self.sent = [] + self.closed = False + + async def accept(self): + self.accepted = True + + async def send_json(self, message): + self.sent.append(message) + + async def close(self): + self.closed = True + + +@pytest.mark.asyncio +async def test_channel_subscribers_receive_channel_broadcasts(): + manager = ConnectionManager() + socket = FakeWebSocket() + + await manager.connect(socket, "user-1") + manager.subscribe(socket, ["dashboard"]) + await manager.broadcast({"type": "data_frame", "channel": "dashboard"}, channel="dashboard") + + assert socket.accepted is True + assert socket.sent == [{"type": "data_frame", "channel": "dashboard"}] + + +@pytest.mark.asyncio +async def test_disconnect_removes_channel_subscriptions(): + manager = ConnectionManager() + socket = FakeWebSocket() + + await manager.connect(socket, "user-1") + manager.subscribe(socket, ["dashboard"]) + manager.disconnect(socket, "user-1") + await manager.broadcast({"type": "data_frame", "channel": "dashboard"}, channel="dashboard") + + assert socket.sent == [] + assert "dashboard" not in manager.channel_subscriptions diff --git a/database_schema.sql b/database_schema.sql index eccef58d..17b58bd4 100644 --- a/database_schema.sql +++ b/database_schema.sql @@ -86,6 +86,12 @@ CREATE TABLE collection_tasks ( id BIGSERIAL PRIMARY KEY, datasource_id INTEGER NOT NULL REFERENCES data_sources(id) ON DELETE CASCADE, status task_status NOT NULL DEFAULT 'pending', + phase VARCHAR(30) DEFAULT 'queued', + phase_progress FLOAT, + phase_message VARCHAR(255), + phase_current BIGINT, + phase_total BIGINT, + phase_unit VARCHAR(30), started_at TIMESTAMP WITH TIME ZONE, completed_at TIMESTAMP WITH TIME ZONE, records_processed INTEGER DEFAULT 0, diff --git a/docker-compose.local-model.yml b/docker-compose.local-model.yml index f7720f0b..0927245a 100644 --- a/docker-compose.local-model.yml +++ b/docker-compose.local-model.yml @@ -18,6 +18,9 @@ services: build: context: . dockerfile: aiprovider/Dockerfile + args: + PYTHON_IMAGE: ${PYTHON_IMAGE:-python:3.14-slim} + UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:latest} container_name: planet_aiprovider ports: - "8010:8010" diff --git a/docker-compose.simple.yml b/docker-compose.simple.yml index 04d9272a..c8c2e3fd 100644 --- a/docker-compose.simple.yml +++ b/docker-compose.simple.yml @@ -5,6 +5,12 @@ services: build: context: . dockerfile: aiprovider/Dockerfile + args: + PYTHON_IMAGE: ${PYTHON_IMAGE:-python:3.14-slim} + UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:latest} + env_file: + - ./aiprovider/.env + - ${PLANET_AI_PROVIDER_RUNTIME_ENV_FILE:-./aiprovider/.env} container_name: planet_aiprovider ports: - "8010:8010" diff --git a/docker-compose.yml b/docker-compose.yml index 329ecff7..3ee9f9eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,12 @@ services: build: context: . dockerfile: aiprovider/Dockerfile + args: + PYTHON_IMAGE: ${PYTHON_IMAGE:-python:3.14-slim} + UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:latest} env_file: - ./aiprovider/.env + - ${PLANET_AI_PROVIDER_RUNTIME_ENV_FILE:-./aiprovider/.env} container_name: planet_aiprovider ports: - "8010:8010" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cdea90d8..a20fd6f1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,465 @@ This project follows the repository versioning rule: - `improvement` -> `+0.0.1`（bugfix + 小功能混合） - `bugfix` -> `+0.0.1` +## [0.49.0] — 2026-05-08 + +Released: 2026-05-08 + +### ✨ Features +- 新增统一地理位置解析 Pipeline，支持 SourceCoordinates / Nominatim / Registry / Inherit 多策略链式 resolver。 +- 新增 BGP 采集站与算力中心地理定位服务（`bgp_collector_locations`、`compute_center_locations`、`bgp_event_locations`）。 +- 新增 Docs Gatekeeper 带鉴权文档 API（`/api/v1/docs`），按用户权限动态返回文档目录与内容。 +- 新增 Earth 全球新闻栏（`/api/v1/news/earth-feed`），根据地球视角坐标推断地区并聚合多源 RSS 信息流。 +- Earth 新增 Mobile 算力中心国家高亮（`mobile-center-country-highlight.js`）。 + +--- + +## [0.48.0] — 2026-05-07 + +Released: 2026-05-07 + +### ✨ Highlights +- 自定义数据源新增 REST / WebSocket 映射运行时，并提供本地 AIS mock WebSocket，用于实时船只 upsert 链路验证。 +- AIS 原始观测、聚合策略、字段来源、冲突记录与船舶 enrichment 继续完善，Earth 船只实时展示链路更接近生产数据形态。 +- Earth 全球态势 summary 改为轻量 SQL 聚合，并在卫星 current 异常时回退到最近有效 TLE 批次，避免统计接口被大规模明细读取拖慢。 + +### 🔧 Improvements +- 修复 `/geo/summary` 与 `/geo/satellites` 在大表下加载慢或超时的问题，并补充 `collected_data` 与 AIS raw 相关索引。 +- WebSocket 管理器支持匿名连接、频道订阅清理和更稳的连接生命周期测试，前端 WebSocket candidates / fallback 更可靠。 +- `planet.sh` 强化端口释放、端口诊断和前端启动流程，mock AIS server 提供 Bun 脚本入口。 + +--- + +## [0.47.0] — 2026-04-30 + +Released: 2026-04-30 + +### ✨ Highlights +- 新增 AISStream WebSocket 船只采集器，并将 AIS 多源数据写入原始观测层，由聚合接口统一去重、合并和解释字段来源。 +- 设置页新增 AISStream API Key、采集范围 preset、运行状态、连接验证和凭证教程入口，让全球 AIS 采集链路可配置、可观察。 +- Earth 船只图层默认不再限制 5000 艘，并统一 marker 颜色、详情卡、hover 和搜索结果的船型归一化显示。 + +### 🔧 Improvements +- 聚合接口新增 `field_sources`、`selected_reasons`、`source_summary`、`quality_flags` 和冲突记录调试接口，动态字段默认优先采用更新的实时流观测。 +- AISStream 标准化支持 `MetaData.ShipName` 船名兜底，并将 AIS 数字船型映射为 Cargo / Tanker / Passenger / Fishing / Military。 +- 将仓库 docs 技能改为通用文档工作流，Planet 专属白名单、双语、裸文件标题和凭证教程规则迁移到 `docs/documentation-coverage-rules.md`。 +- 更新 AIS v4/v5 TODO 与计划文档，明确后续聚合策略配置、船舶资料 enrichment 和媒体缓存边界。 + +--- + +## [0.46.3] — 2026-04-30 + +Released: 2026-04-30 + +### 🐛 Fixes +- 优化 Starlink footprint 显示后的地球拖拽性能，避免旋转地球时每帧重建 footprint 大网格，同时保持现有视觉效果不变。 +- 恢复点击线缆后的呼吸透明度动画，让 locked / hover 线缆重新使用既有 pulse 配置。 + +--- + +## [0.46.2] — 2026-04-30 + +Released: 2026-04-30 + +### 🐛 Fixes +- 修复 Earth 启动时高清材质、云图和图层可见性绕过 `startupPriority` 的问题，统一由启动队列按文档顺序加载。 +- 修复保存为关闭的高清材质/图层仍会先加载再关闭的问题，并保持海陆基座作为国界线图层的常驻底图。 +- 修复搜索跳转会误关媒体面板、船只轨迹末端不贴合当前船只、Iridium footprint 被地表层遮挡等 Earth 交互问题。 + +### 📝 Documentation +- 更新 Earth 图层顺序、样式参考、使用手册和 AIS 聚合计划，补齐中英文说明与后续接入策略。 + +--- + +## [0.46.1] — 2026-04-30 + +Released: 2026-04-30 + +### 🐛 Fixes +- 修复新增 technical docs 文件存在但未进入 Docs 前端白名单时，侧栏不显示且 Markdown 链接无法解析到 `/docs/` 的问题。 +- 补齐数据源/采集器连接验证与 Earth Interactable 使用说明的英文文档，保证公开 Docs 切换 EN 时同名页面可访问。 +- 清理中英文 technical docs 中裸 `.md` 文件名链接标题，改为面向读者的语义标题。 + +### 📝 Documentation +- 将 Docs 前端白名单、公开文档双语配对、裸文件名链接标题三项检查写入 Claude 与 Codex 的 docs 技能流程。 + +--- + +## [0.46.0] — 2026-04-30 + +Released: 2026-04-30 + +### ✨ Highlights +- Earth 新增通用 Interactable 图标层，船只、算力中心、BGP 事件与观测站统一使用批量 Points、屏幕拾取、状态 glow 和状态缩放。 +- BGP 事件保留向外扩散圈，观测站保留雷达扫描层，并与 Interactable 主图标解耦到稳定的地表渲染层级。 +- 登陆点回归黄色球形 Sprite，贴近海缆层级并保持更稳定的地表显示和遮挡表现。 + +### 🔧 Improvements +- 新增 SVG asset 到 canvas texture 的 Interactable 资产加载路径，支持统一图标资源、缓存和可选染色。 +- 同坐标 Interactable 自动做地表切向避让，降低重叠物件无法选择的问题。 +- 优化 Earth toolbar 初始尺寸注入，避免首次显示原始尺寸后再跳到缩放尺寸。 +- 补充 Interactable 计划、使用说明、图层顺序和 Earth 前端上下文文档。 +- 修复船只 hover/locked 状态仅发光但放大反馈不明显的问题，将已有状态缩放接入通用图标层。 + +--- + +## [0.45.0] — 2026-04-29 + +### ✨ Highlights +- 采集任务新增阶段级量化进度，`fetching` 可展示百分比、阶段说明和字节下载量。 +- AI Provider 启动链路支持从 `aiprovider/.env` 与 `~/.zshrc` 注入运行期配置，并避免密钥/模型变化触发镜像重建。 +- AI Provider Docker build context 收敛到服务必需文件，`uv sync` 接入 BuildKit 缓存以减少重复下载。 + +### 🔧 Improvements +- IPtoASN、OpenGeoFeed、NRO delegated 下载型采集器接入真实字节进度上报。 +- 数据源页、采集中任务弹窗和任务历史页展示阶段摘要，并在 tooltip 中保留完整进度细节。 +- 调整 Earth 船只默认高度偏移，进一步贴近地表展示。 + +--- + +## [0.44.2] — 2026-04-29 + +### 📝 Documentation +- 补充 Earth 船只图层技术文档，记录分桶 `THREE.Points` 批量渲染、同尺寸交互 overlay 和屏幕空间 picking 的设计约束。 +- 同步 Earth 渲染图层顺序和样式参考，明确 AIS 船只 renderOrder、depthTest、图标尺寸、航向分桶与 hover 命中半径。 +- 更新船只渲染性能计划状态，标注 `0.44.1` 已落地的实现与后续全球 AIS / LOD 演进方向。 + +--- + +## [0.44.1] — 2026-04-29 + +### 🐛 Fixes +- 修复 Earth 船只图层拖动不跟手的问题，将普通船只从独立 Sprite 切换为按航向分桶的批量 Points 渲染。 +- 修正船只 hover/click 拾取错位，改为屏幕空间命中检测并在拖拽/惯性期间跳过 hover 拾取。 +- 统一 AIS 船只普通态与交互态方向，并让 hover/locked glow 与普通图标保持同尺寸覆盖。 +- 恢复船只深度测试并收敛默认图标尺寸，避免北部岛屿/冰面附近出现明显压盖陆地的视觉问题。 + +--- + +## [0.44.0] — 2026-04-29 + +### ✨ Highlights +- 重构数据源与采集器设置边界：数据源页回归目录和采集触发，采集器 endpoint、请求头、凭证与连接验证统一进入设置页 +- BarentsWatch AIS 完整接入凭证解析、连接检查、默认教程、AI 生成教程和船只采集/可视化链路 +- Earth 新增船只图例、缩放反馈胶囊、缩放感知拖拽灵敏度，并将船只渲染性能优化方案沉淀到 plans +- 仪表盘重启服务新增前端重启 action，并让 runner 通过 `~/.zshrc` 继承本地环境变量 + +### 🔧 Improvements +- 采集器连接状态改为基于成功采集或手动连接校验 checksum 判断，避免只依赖前端样式状态 +- 数据源页新增采集中任务标签和任务进度弹窗，内置与自定义数据源统一展示 +- Earth 国界线进一步贴近地表，并补充船只图层渲染顺序、样式和用户手册说明 +- docs skill 与 Claude/Codex 文档流程补齐技术文档和 plans 的职责边界 + +--- + +## [0.43.1] — 2026-04-28 + +### 🐛 Fixes +- 修正 `planet.sh` 在全量 restart 后启动 AI Provider 时的提示语义，避免把预期内未就绪描述成异常不健康 + +--- + +## [0.43.0] — 2026-04-28 + +### ✨ Highlights +- 新增 Earth 船舶追踪链路，接入 BarentsWatch AIS 凭证配置、采集器、后端 vessel 模型/API 与前端 Earth 船舶图层 +- 新增自定义数据源映射流程，支持样本抓取、目标 schema、AI 辅助生成映射、预览校验和映射执行 +- Settings 拆分 AI Provider 与采集器凭证配置，DataSources 只保留采集状态、运行参数和必要引导 + +### 🔧 Improvements +- AI Provider 支持运行时 LLM 配置、provider preset 下拉与刷新，并在 Playground 中引导到 AI 配置页 +- Markdown 渲染器补齐代码块复制按钮、语言标签、任务列表、图片、自动链接、删除线和文档主题样式 +- Docs 公开导航改为显式元数据白名单，避免开发任务文档自动出现在“其他”分组 +- 将 Codex/Claude cleanup、docs、goal-driven、release 流程补充 CLI-first 约束，并把 `rules.md` 整理成可按模块加载的工程规则 + +--- + +## [0.42.2] — 2026-04-28 + +### 🐛 Fixes +- Docs 中文模式下补齐左侧分组、文档标题、页头分类与搜索结果分类翻译，并更新文档站品牌标题/副标题文案 + +--- + +## [0.42.1] — 2026-04-28 + +### 🐛 Fixes +- 修正 release skill 的 feature 版本计算规则：minor 进位时 patch 必须重置为 `0`，例如 `0.41.2` 应发布为 `0.42.0` + +--- + +## [0.42.0] — 2026-04-28 + +### ✨ Highlights +- 新增公开 `/docs` 文档站，支持中英文技术文档、使用手册、Quickstart、搜索、目录锚点与浅色/深色/跟随系统主题 +- Earth 在无高清材质时新增轻量 Fresnel 边缘提示，并调整卫星覆盖默认显示与地表材质可读性 + +### 🔧 Improvements +- 将技术文档整理为 `docs/technical/zh` 与 `docs/technical/en`，并补充控制台、`planet.sh`、Earth 与公共组件使用说明 +- 新增 `SegmentedControl` 公共滑块组件，支持缩放参数，复用到 docs 语言与主题切换 +- Markdown 渲染器接入自定义滚动条，表格与代码块在深色模式和 overflow 场景下保持可读 +- Docs 搜索结果支持内部滚动、点击外部关闭、重新聚焦恢复上次搜索结果 +- Earth 工具栏展开状态与设置持久化版本迁移继续收口，改善默认面板和快捷关闭行为 + +--- + +## [0.41.2] — 2026-04-27 + +### 🔧 Improvements +- `planet.sh` 启动链路新增 verbose 滚动输出窗口，并在后端端口占用时打印目标地址和监听进程诊断 +- Docker 构建支持通过 build args 覆盖 Python 与 uv 镜像，方便 Docker Hub 不稳定时切换镜像源 + +### 🐛 Fixes +- Earth 海缆登陆点改为基于相机射线与地球遮挡判断可见性，修复旋转后 pin 可见性滞后一帧的问题 + +### 🔧 Improvements +- `docker-compose*.yml` 为 AI Provider 构建传入 `PYTHON_IMAGE` / `UV_IMAGE` 参数，默认仍使用官方镜像 +- 后端启动失败遇到 `Address already in use` 时输出 `lsof`、`ss` 与 PID 命令行信息 +- verbose 模式下 AI Provider build、后端与前端启动日志会在 spinner 下方保留最新 5 行滚动展示 + +--- + +## [0.41.1] — 2026-04-27 + +### 🐛 Fixes +- 修复新闻直播面板设置项持久化失效：`closeTransientMobileOverlays` 通过旁路路径隐藏面板导致下次 persist 快照到错误状态，改为不重新从 DOM 读取面板可见性 +- 修复登陆点 pin 在地球侧面被半截遮挡：改为在接近地平线前（dot < 0.05）主动隐藏，避免深度测试切片 + +### 🔧 Improvements +- 将所有画布绘制的图标抽取为 SVG，存入 `frontend/public/earth/assets/icons/`，新增图标规范到 `rules.md` + +--- + +## [0.41.0] — 2026-04-27 + +### ✨ Highlights +- Earth 图层系统完成地表到天空的注册顺序与关注优先的面板顺序拆分，支持基座海陆色块、国界、高清材质、云图、地形、算力、BGP、卫星、轨迹与海缆的稳定层级 +- 国界层新增真实行政区轮廓交互与中国/台湾联动高亮，修复高清材质、地形、footprint、卫星与经纬线之间的遮挡和 hover 竞争 + +### 🔧 Improvements +- 新增无轮廓基座地图，所有图层关闭时仍保留 `#010609` 海洋与 `#080f1b` 陆地色块 +- 将大气云图抽象为独立图层并接入桌面/移动端图层开关、持久化状态与启动同步 +- 高清材质改为独立纹理覆盖层，地形显示在高清材质上方，并在高清材质关闭/恢复时保持原地形开关意图 +- 补充 Earth 渲染层级与图层样式文档，记录正式图层名、变量名、材质颜色、线宽与 renderOrder + +--- + +## [0.40.5] — 2026-04-26 + +### 🔧 Improvements +- 卫星拖尾改用 Instanced screen-space ribbon，单 draw call 渲染所有轨迹段，支持像素级宽度控制 +- Iridium 地面覆盖重写为球面投影径向网格，修复填充光晕不可见问题；新增外圈 LineLoop +- 搜索面板打开时改用双 rAF 延迟聚焦输入框，确保 CSS 过渡完成后焦点可靠触发 +- 代码清理：提取 `IRIDIUM_OVERLAY_COLOR`、`IRIDIUM_REFERENCE_ALTITUDE_KM` 常量，消除重复三角函数调用 + +--- + +## [0.39.0] — 2026-04-24 + +## [0.40.4] — 2026-04-26 + +### 🔧 Improvements +- 新增页面可见性恢复处理，页面从后台切回前台时主动刷新卫星位置，避免累积后台时间在下一帧一次性回放 +- 抽出卫星轨迹状态与轨迹几何清理 helper，统一后台恢复与清空数据时的轨迹重置路径 + +### 🐛 Fixes +- 修复页面在后台停留较久后恢复前台时，卫星轨迹因超大 `deltaTime` 突然跳变、拖尾异常拉长的问题 +- 修复后台恢复后首帧仍沿用旧轨迹缓存，导致轨迹与当前卫星位置短时错位的问题 + +--- + +## [0.40.3] — 2026-04-25 + +### 🔧 Improvements +- 卫星点云升级为自定义 ShaderMaterial，支持 per-point alpha 控制，锁定/悬停卫星从点云中精确隐藏 +- 修复锁定环与自发光选中标记的 depthTest 错误（false → true），消除远端渲染穿透 artifact +- 新增锁定环悬停态缩放与线宽（LOCKED_RING_HOVER_SCALE / LOCKED_RING_HOVER_LINE_WIDTH） +- 修复 updateLockedDotWorldTransform / updateLockedHaloWorldTransform 未强制刷新 matrixWorld 导致的位置漂移 + +--- + +## [0.40.2] — 2026-04-24 + +### 🔧 Improvements +- 卫星点大小随镜头缩放动态调整，拉近变大、拉远变小，响应与相机距离线性对应 +- 调小卫星点默认基础尺寸（dotSize 2.8），缩放范围更合理 + +--- + +## [0.40.1] — 2026-04-24 + +### 🔧 Improvements +- 卫星选中标记（lockedring / lockeddot / 光晕）颜色统一跟随图例轨道倾角分类配色 +- 修复 Starlink footprint 在特定视角下遮蔽卫星点的渲染顺序问题（Group renderOrder 影响子 Mesh 排序） +- footprint 材质改为 `depthTest: false` + 相机朝向 limbFade，替代 polygonOffset 深度竞争方案 +- 修复选中海缆时误触发附近卫星高亮（该行为属于 BGP 事件点逻辑，不应用于海缆） + +--- + +## [0.40.0] — 2026-04-24 + +### ✨ Highlights +- Earth 卫星 footprint 正式按星座能力分层：Starlink 保留专用地表覆盖，Iridium 改为独立外圈覆盖表达，其它非 Starlink 星座不再误用同一套 footprint +- Earth 卫星详情卡补齐覆盖能力与当前显示说明，用户现在可以直接看见每颗卫星为什么显示 footprint、为何回退为自身发光 + +### 🔧 Improvements +- 后端可视化接口新增并透传 `constellation_group` 与 `footprint_policy`，前端据此执行 capability-gated footprint renderer +- 新增 Iridium 独立 coverage ring adapter，并继续保留 Starlink 专用 footprint 调校与昼夜可读性增强 +- 新增 Earth 卫星 footprint 策略技术文档，明确 GNSS、generic LEO、GEO 与 Iridium 的显示边界 + +### 🐛 Fixes +- 修复前后端对 Iridium footprint policy 命名不一致，导致策略分发语义含混的问题 +- 清理 Starlink footprint 渲染中的未使用常量与过时命名，减少后续继续调校时的歧义 + +--- + +## [0.39.0] — 2026-04-24 + +### ✨ Highlights +- 后端正式落下统一结构化日志地基：请求上下文、事件名、脱敏与持久化链路开始收口为可扩展的企业级日志体系 +- 系统日志页重构为真正的日志工作台：顶部筛选更紧凑，终端日志区成为主视觉，移动端 Earth 新闻/态势细节交互继续补稳 + +### 🔧 Improvements +- 新增 `backend/app/core/logging.py`，统一 `request_id`、`service`、`event` 注入与敏感字段脱敏，并接入后端主入口、调度器、缓存、数据库和可视化链路 +- 系统日志页筛选区重排为更紧凑的两层结构，信息摘要并入终端工具栏 tooltip，日志终端区留出更稳定的按钮避让空间 +- Earth 移动端态势抽屉补齐宽度约束与图例换行规则，新闻详情抽屉在巡航切换时可同步更新标题和摘要 + +### 🐛 Fixes +- 修复 `/tmp/planet_backend.log` 中混入空字节时，日志摘要条行数与实际可见日志不一致的问题 +- 修复移动端“态势”tab 在内容渲染后被图例文本撑宽、超出一屏的问题 +- 修复移动端新闻详情抽屉在巡航切换下一条新闻时标题更新但 summary 不同步的问题 + +--- + +## [0.38.0] — 2026-04-23 + +### ✨ Highlights +- Earth 新闻正式接入通用巡航层：新闻和 BGP 统一进入可配置巡航模块，桌面端与移动端都能在巡航聚焦时展示对应新闻卡片 +- 系统日志页升级为结构化过滤链路：按真实时间戳、结构化级别和字符串检索统一筛选，不再依赖前端或后端从日志文本里猜结果 + +### 🔧 Improvements +- 新闻巡航补齐业务适配层：按发生地与时间生成巡航目标，桌面端与移动端统一标题 + summary 卡片风格，并增加连线与打字机摘要展示 +- 日志页筛选体验重排，统一服务源、级别、行数、时间和检索布局，日历标记改为由后端返回的结构化每日聚合结果驱动 +- 后端补充 `system_logs` 结构化解析与多级别精确过滤能力，Earth 浏览器端日志缓冲与系统日志 API 现在走同一套筛选语义 + +### 🐛 Fixes +- 修复新闻巡航模块开启后难以关闭、桌面/移动端设置状态互相污染的问题 +- 修复新闻巡航卡片缺少摘要、移动端详情样式不统一、新闻巡航缺少连线的问题 +- 修复日志级别筛选会被访问日志 query string 中的 `level=error` 等参数污染，从而把 `INFO` 行误判为 `ERROR` 的问题 + +--- + +## [0.37.2] — 2026-04-23 + +### ✨ Highlights +- Earth 图层系统新增经纬线开关，桌面图层面板与移动端抽屉都可直接控制 + +### 🔧 Improvements +- 经纬线正式接入 Earth layer registry，复用现有图层切换、移动端图层卡片与设置持久化流 + +### 🐛 Fixes +- 修复经纬线只能默认常驻、无法作为独立图层开关控制的问题 + +--- + +## [0.37.1] — 2026-04-23 + +### ✨ Highlights +- `planet.sh` 后端重启链路修复 `uvicorn --reload` 残留 worker 场景，`restart` 现在能真正替换旧实例 + +### 🔧 Improvements +- 收口后端清理逻辑，统一按 `uvicorn` 进程、端口占用进程和进程组执行清理，减少 reload 场景漏杀分支 + +### 🐛 Fixes +- 修复部分机器执行 `./planet.sh restart --allow-lan` 后后端仍停留旧实例，导致 `/api/v1/visualization/geo/compute-centers` 返回 `404` 的问题 + +--- + +## [0.35.1] — 2026-04-22 +## [0.37.0] — 2026-04-23 + +### ✨ Highlights +- Earth 连线系统正式从巡航里解耦成通用 callout connector：桌面端和移动端统一支持对象级锚点、四边切换与临界区边缘滑动 +- BGP 巡航展示继续收口为稳定的“先定位卡片、再连真实锚点、再展示卡片”链路，移动端 popup 与桌面 info panel 的路线规则统一 + +### 🔧 Improvements +- connector 配置从 `CRUISE_CONFIG` 拆到独立 `CONNECTOR_CONFIG`，默认类名、动画名和实例命名也全部去 cruise 语义 +- 移动端 popup 增加更稳定的 dock/obstacle 处理，拖动卡片时连线起终点会持续按几何关系自适应刷新 +- Earth 多个图层与控制逻辑继续收口，补充算力中心/BGP 风格对齐、layer panel 与相关交互细节调整 + +### 🐛 Fixes +- 修复巡航模式下终点只像“视觉锚点”而不是真实绑定对象的问题，卡片拖动后终点现在会跟随 +- 修复移动端与桌面端多类连线路线异常：压线、反向、临界区折返、起点遮挡事件点等问题 +- 修复对象矩形临界区内连线仍强制中点到中点导致路线像“先钻进 source 内部”再出去的问题 + +--- + +## [0.35.1] — 2026-04-22 +## [0.36.0] — 2026-04-22 + +### ✨ Highlights +- Earth 新增统一“算力中心”图层：接入超算与 GPU 集群，支持搜索、统计、图例、详情卡与独立图层开关 +- 算力中心支持精确位置与估算位置两种状态，估算点会以问号角标区分，避免数据不全时整批节点在地图上消失 + +### 🔧 Improvements +- Earth 详情卡拖拽与地球拖拽交互继续收口，减少拖动卡片和旋转地球时的选中文本与 pointer 竞争 +- `planet.sh` 改为通过独立脚本计算 AI Provider 依赖指纹，降低与根仓库依赖版本文件的无关耦合 +- README 补充 WSL / Windows 局域网访问排查与转发配置说明，便于开发环境联调 + +### 🐛 Fixes +- 修复 Earth 算力中心图层在无原始坐标时无法显示的问题，支持站点提示和国家级估算回退 +- 修复信息卡拖拽事件可能被卡片级 stopPropagation 吞掉，导致拖拽流中断的问题 + +--- + +## [0.35.1] — 2026-04-22 + +### ✨ Highlights +- Earth 统计展示改为统一 `data-earth-stat` 绑定机制，桌面 HUD 和移动端抽屉复用同一套状态更新入口 + +### 🔧 Improvements +- 收口海缆、登陆点、卫星、BGP 事件与 BGP 状态的统计写入逻辑，减少后续继续补桌面/移动双写分支的成本 + +### 🐛 Fixes +- 修复移动端态势抽屉中的海缆、登陆点与 BGP 统计在图层切换后可能停留旧值的问题 + +--- + +## [0.35.0] — 2026-04-22 + +### ✨ Highlights +- Earth 移动端底部抽屉系统全面上线：响应式布局自动切换、Tab 导航、手势上拉/下滑开合、惯性速度判定 +- 移动端点击可交互物件（海缆、登陆点、卫星、BGP）后弹出智能定位悬浮卡片，可拖动，点击跳转详情 + +### 🔧 Improvements +- 抽屉把手区域缩小至 36px（collapsed 时仅露出把手，不遮挡地球操作区） +- 抽屉定期弹跳动画提示用户可上拉，5 秒间隔，打开后自动停止 +- 通知胶囊位置调整，不再覆盖品牌 logo +- 移动端单指旋转、双指捏合缩放地球，触控事件冲突修复（pointer-events 级联） + +### 🐛 Fixes +- 修复移动端抽屉 shell 因 layout 高度（240px+）遮挡地球触控区域，pointer-events 改为按层级精确控制 +- 修复悬浮卡片因 setPointerCapture 在 iOS Safari 抑制合成 click 事件导致无法点击的问题 + +--- + +## [0.34.0] — 2026-04-22 + +### ✨ Highlights +- Earth 搜索面板正式接入，支持搜索海缆、登陆点、卫星、BGP 事件与观测站，并可直接聚焦到对应对象 +- `planet.sh --allow-lan` 打通 Bun + Vite 的局域网开放链路，启动成功后自动打印推荐访问地址与后端健康检查地址 + +### 🔧 Improvements +- 前端开发启动链统一改成 Bun 直接执行 Vite 入口，不再依赖 shell 中额外暴露的 Node 路径 +- Earth 搜索结果接入登陆点详情卡片与对象聚焦，搜索后可直接进入对应详情流 +- `planet.sh` 补充局域网 IPv4 自动识别与推荐地址输出，减少 WSL 局域网调试成本 + +### 🐛 Fixes +- 修复 `./planet.sh restart --allow-lan` 全量重启时未把 `--allow-lan` 继续传给 `start()`，导致前端退回本机监听的问题 +- 修复 WSL + Bun 环境下前端偶发因 Vite 启动链不稳定而无法正确监听 `0.0.0.0:3000` 的问题 + +--- + ## [0.33.0] — 2026-04-22 ### ✨ Highlights @@ -65,8 +524,6 @@ This project follows the repository versioning rule: --- -## [0.31.0] — 2026-04-21 - ## [0.31.2] — 2026-04-21 ### ✨ Highlights @@ -103,6 +560,8 @@ This project follows the repository versioning rule: --- +## [0.31.0] — 2026-04-21 + ### ✨ Features - Earth 新增"巡航展示"模式：自动轮播 BGP 异常事件，逐帧追踪连接线位置，支持外部交互立即中断序列（cancel notifier 模式） - 巡航目标事件点高亮显示：hover 外观 + 锁定脉冲动画，并与点击行为统一展示周边受影响卫星与海缆 @@ -116,8 +575,6 @@ This project follows the repository versioning rule: --- -## [0.29.1] — 2026-04-20 - ## [0.30.0] — 2026-04-21 ### ✨ Features diff --git a/docs/documentation-coverage-rules.md b/docs/documentation-coverage-rules.md new file mode 100644 index 00000000..a3b517bf --- /dev/null +++ b/docs/documentation-coverage-rules.md @@ -0,0 +1,117 @@ +# Documentation Coverage Rules + +This file contains Planet-specific documentation coverage rules. Documentation skills and agents should read this file before deciding which docs to update. Keep tool-specific workflow in skills; keep product and repository rules here. + +## Scope Rules + +- User-visible workflow changes must update `docs/technical/zh/manual.md` and usually `docs/technical/zh/quickstart.md`. +- If an English counterpart exists for user-facing docs such as `manual.md` or `quickstart.md`, update `docs/technical/en/...` enough that it does not contradict the Chinese source. +- Control console page responsibility changes must update `docs/technical/zh/frontend-admin-frontend-context.md`. +- Earth frontend behavior changes must update `docs/technical/zh/earth-frontend-context.md`. +- Earth layer additions, `renderOrder`, altitude/radius offsets, depth strategy, pointer picking, legend modes, or layer panel/startup ordering must update `docs/technical/zh/earth-render-layer-order.md`. +- Earth layer visual style or legend symbol/color semantics should also update `docs/technical/zh/earth-layer-style-reference.md` when that reference is affected. +- Collector, datasource, credential, settings, connectivity, scheduler, or API changes must update the relevant backend docs, especially `docs/technical/zh/backend-collectors.md` and any datasource/settings-specific doc. +- When a change turns an old plan assumption into current behavior, update the relevant `docs/plans/*.md` with a status note instead of leaving contradictory instructions. +- Search docs for stale terms introduced by the change, for example old tab names, old route responsibilities, obsolete auth assumptions, or renamed UI labels. + +## Public Docs Rules + +- If adding a new technical document, add it to `docs/technical/zh/README.md` when it should be discoverable from the technical docs index. +- If a technical document should be visible in the public Docs page or linked from a technical README, register it in `frontend/src/pages/Docs/docs-content.ts` under `DOCS_METADATA`. Files under `docs/technical/{zh,en}/` are not automatically routable. +- For every public technical doc, keep the bilingual file pair in sync by filename: `docs/technical/zh/.md` and `docs/technical/en/.md`. If content is intentionally Chinese-only or English-only, state that intentionally in the final note. +- Public docs should use readable link text, not raw filenames such as `manual.md`. + +## Credential Collector Rules + +- Any built-in collector marked `requires_credentials: true` and `credential_status: supported` must have: + - a `credential_provider` in `backend/app/core/datasource_defaults.py`; + - a default credential guide in `backend/app/services/credential_guides.py`; + - a supported connectivity provider in `backend/app/services/datasource_connectivity.py`; + - settings UI guidance or a credential form in `frontend/src/pages/Settings/Settings.tsx`; + - a regression test that fails if the guide/provider is missing. + +## Recommended Checks + +Run the checks that match the affected docs. + +### Duplicate Bilingual Docs + +```bash +python - <<'PY' +from pathlib import Path +same = [] +for en in sorted(Path("docs/technical/en").glob("*.md")): + zh = Path("docs/technical/zh") / en.name + if zh.exists() and en.read_text() == zh.read_text(): + same.append(en.name) +if same: + raise SystemExit("identical en/zh docs: " + ", ".join(same)) +print("no identical en/zh docs") +PY +``` + +### Language-Less Technical Links + +```bash +rg -n "/home/ray/dev/linkong/planet/docs/technical/(?!zh|en)" docs/technical/zh --pcre2 +``` + +This should return no matches. + +### Public Docs Registry + +```bash +python - <<'PY' +import re +from pathlib import Path + +metadata = Path("frontend/src/pages/Docs/docs-content.ts").read_text() +known = set(re.findall(r"'([^']+\.md)':\s*\{", metadata)) +known.add("README.md") + +missing = [] +for readme in [Path("docs/technical/zh/README.md"), Path("docs/technical/en/README.md")]: + if not readme.exists(): + continue + for href in re.findall(r"\]\(([^)]+\.md)\)", readme.read_text()): + path = Path(href) + if "docs/technical/" not in href: + continue + filename = path.name + if filename not in known: + missing.append(f"{readme}: {filename}") + +if missing: + raise SystemExit("docs README links missing DOCS_METADATA: " + ", ".join(missing)) +print("docs README links are whitelisted") +PY +``` + +### Public Bilingual Pairs + +```bash +python - <<'PY' +import re +from pathlib import Path + +metadata = Path("frontend/src/pages/Docs/docs-content.ts").read_text() +filenames = sorted(set(re.findall(r"'([^']+\.md)':\s*\{", metadata)) - {"README.md"}) +missing = [] +for filename in filenames: + for lang in ("zh", "en"): + path = Path("docs/technical") / lang / filename + if not path.exists(): + missing.append(str(path)) +if missing: + raise SystemExit("missing bilingual docs: " + ", ".join(missing)) +print("public docs have zh/en file pairs") +PY +``` + +### Raw Filename Link Titles + +```bash +rg -n "\[[^]]+\.md\]\(" docs/technical/zh docs/technical/en +``` + +This should return no matches for polished public docs. diff --git a/docs/plans/README.md b/docs/plans/README.md index 2062a1c2..a31060b9 100644 --- a/docs/plans/README.md +++ b/docs/plans/README.md @@ -16,11 +16,21 @@ 当前重点入口： +- [earth-mobile-drawer-ui-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-mobile-drawer-ui-plan.md) +- [earth-compute-center-bgp-style-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-compute-center-bgp-style-plan.md) - [earth-renderer-architecture-separation-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-renderer-architecture-separation-plan.md) +- [earth-country-boundary-overlay-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-country-boundary-overlay-plan.md) - [earth-predicted-orbit-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-predicted-orbit-plan.md) - [earth-webgl-instancing-satellites-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-webgl-instancing-satellites-plan.md) - [earth-real-terrain-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-real-terrain-plan.md) - [earth-news-source-configuration-and-collector-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-news-source-configuration-and-collector-plan.md) +- [earth-news-cruise-summary-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-news-cruise-summary-plan.md) +- [earth-vessel-rendering-performance-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-vessel-rendering-performance-plan.md) +- [AIS 多源采集、冲突记录与聚合接口计划](/home/ray/dev/linkong/planet/docs/plans/earth-vessel-ais-aggregation-plan.md) +- [earth-interactable-layer-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-interactable-layer-plan.md) +- [frontend-public-docs-site-plan.md](/home/ray/dev/linkong/planet/docs/plans/frontend-public-docs-site-plan.md) +- [Docs Gatekeeper 鉴权系统计划](/home/ray/dev/linkong/planet/docs/plans/docs-gatekeeper-auth-plan.md) +- [Location Resolver 共享管线计划](/home/ray/dev/linkong/planet/docs/plans/location-resolver-shared-pipeline-plan.md) - [frontend-ai-playground-development-plan.md](/home/ray/dev/linkong/planet/docs/plans/frontend-ai-playground-development-plan.md) - [ue5-mvp-fused-plan.md](/home/ray/dev/linkong/planet/docs/plans/ue5-mvp-fused-plan.md) diff --git a/docs/plans/custom-source-live-mock-plan.md b/docs/plans/custom-source-live-mock-plan.md new file mode 100644 index 00000000..1e6fb231 --- /dev/null +++ b/docs/plans/custom-source-live-mock-plan.md @@ -0,0 +1,384 @@ +# Custom Source Live Mock 计划 + +**状态**：实施中 +**创建日期**：2026-05-01 +**任务名**：`Custom Source Live Mock` +**核心目标**：把自定义源升级为同时支持 REST 与 WebSocket 的可映射采集入口，并提供本地 AIS mock WebSocket 服务，用于验证 Earth 船只实时新增与 upsert 链路。 + +## 背景 + +真实 AIS 接口变化频率不可控，无法稳定验证 Earth 页面“不刷新也能看到新船只”的实时链路。当前系统已经有自定义源基础设施： + +- `datasource_configs` 保存 endpoint、auth、headers、config。 +- `datasource_mapping_templates` 保存目标 schema 的确定性映射模板。 +- `run-mapped` 支持保存后的自定义 REST 源通过 active mapping 写入目标数据。 + +但现有能力主要面向 REST sample 和批量 mapping，缺少以下能力： + +- 自定义源不能明确选择 `REST` 或 `WebSocket` 采集模式。 +- WebSocket 长连接、订阅消息、重连、消息路径提取还没有通用 runtime。 +- `vessel_ais` 自定义数据写入后需要进入 AIS raw observation 和 `vessels` WS channel，才能真实验证 Earth 实时 upsert。 +- 删除自定义源时没有清晰的数据清理选项。 +- 设置中心里“采集调度 / 凭证 / 自定义源”入口混杂，用户很难判断该在哪里配置。 + +## 已确认决策 + +| 项目 | 决策 | +|-----|------| +| 计划名称 | `Custom Source Live Mock` | +| 自定义源传输类型 | 支持 `REST` 与 `WebSocket` | +| 采集写入方式 | 先映射到目标 schema，再由 destination handler 写入 | +| AIS mock 目标 | 优先打通 `vessel_ais`，验证 Earth 船只实时新增和同 MMSI upsert | +| mock 服务 runtime | 使用 `bun` 启动本地 mock WS 服务 | +| 凭证配置 | 支持 headers、bearer、api key、basic，并保留 query/header API key 位置配置 | +| 删除策略 | 删除自定义源时允许选择是否删除该源写入的数据 | +| 合并语义 | 自定义源必须选择“合并到哪个内置数据”，作为内置源的补充数据进入同一聚合链路 | +| UI 方向 | 自定义源创建和维护放在“配置中心 > 采集器设置”的采集器下拉框内联入口；数据源页保留总览与运行控制 | + +## 范围 + +### 本阶段要做 + +- 自定义源可选择 `REST` 或 `WebSocket`。 +- 自定义源支持请求头、凭证、query params、body、WS subscribe message。 +- WebSocket 自定义源支持长连接、重连、消息解析、mapping、写入。 +- `vessel_ais` 自定义源写入 AIS raw observations，并广播 `vessels` channel。 +- 提供 mock AIS WS 服务，持续发送新增 MMSI 和位置变更。 +- 删除自定义源时提供“是否删除该源数据”的选项。 +- 梳理设置中心信息架构，明确后续 UI 重构方向。 + +### 暂不做 + +- 不新增任意动态数据库表。 +- 不允许用户提交可执行脚本作为 mapping。 +- 不让 LLM 进入正式采集链路。 +- 不把 mock 数据直接写 legacy `vessel_position`，优先写 AIS raw observations，保持可追踪和可删除。 +- 不在本阶段完成完整 `Earth Live Sync`，但要为后续 summary invalidation 留出 hook。 + +## 现状入口 + +| 能力 | 当前位置 | +|-----|----------| +| 自定义源配置模型 | `backend/app/models/datasource_config.py` | +| 自定义源 mapping 模型 | `backend/app/models/datasource_mapping.py` | +| 自定义源 API | `backend/app/api/v1/datasource_config.py` | +| 目标 schema registry | `backend/app/core/target_schema_registry.py` | +| mapping engine | `backend/app/services/datasource_mapping.py` | +| 数据源总览 UI | `frontend/src/pages/DataSources/DataSources.tsx` | +| 采集器设置 UI | `frontend/src/pages/Settings/Settings.tsx` | + +## 目标架构 + +```mermaid +flowchart LR + A[Custom Source Config] --> B{source_type} + B -->|rest| C[Mapped REST Runner] + B -->|websocket| D[Mapped WS Runner] + C --> E[Mapping Engine] + D --> E + E --> F[Target Schema Validator] + F --> G{Destination Handler} + G -->|vessel_ais| H[AIS Raw Observations] + H --> I[AIS Aggregation] + H --> J[vessels WS Channel] + J --> K[Earth Vessel Upsert] +``` + +## 数据配置设计 + +短期可以继续复用 `DataSourceConfig`，避免大迁移。语义约定如下： + +| 字段 | 用途 | +|-----|------| +| `name` | 自定义源唯一名称，例如 `mock_ais_ws` | +| `source_type` | `rest` 或 `websocket` | +| `endpoint` | `http(s)://...` 或 `ws(s)://...` | +| `auth_type` | `none`、`bearer`、`api_key`、`basic` | +| `auth_config` | token、api_key、key name、basic username/password 等 | +| `headers` | 静态请求头 | +| `config` | method、params、body、timeout、retry、WS 订阅消息、重连策略、消息路径等 | + +建议 `config` 结构： + +```json +{ + "transport": "websocket", + "delivery_mode": "realtime_stream", + "merge_target_source": "barentswatch_vessels", + "target_schema": "vessel_ais", + "method": "GET", + "params": {}, + "body": null, + "timeout": 30, + "retry": 3, + "ws_subscribe_message": {"type": "subscribe", "channel": "vessels"}, + "ws_message_path": "$.data", + "ws_items_path": "$.vessels[*]", + "ws_reconnect": true, + "reconnect_delay_seconds": 3, + "debug_max_messages": null, + "delete_policy": "config_only" +} +``` + +## 后端实施计划 + +### Phase 1 — 自定义源类型与连接测试 + +- 允许 `source_type` 为 `rest` 或 `websocket`。 +- REST 连接测试保留现有 HTTP 请求逻辑。 +- WebSocket 连接测试新增： + - 校验 endpoint 必须是 `ws://` 或 `wss://`。 + - 注入 headers 和 auth。 + - 连接后可选发送 `ws_subscribe_message`。 + - 读取一条消息或超时返回诊断。 + +### Phase 2 — Mapped REST Runner 补齐 + +现有 `run-mapped` 继续作为 REST 一次性采集入口，补齐： + +- `GET/POST` method。 +- query params。 +- JSON body。 +- headers 和 auth 注入。 +- sample limit 与响应大小限制。 +- `vessel_ais` destination handler。 + +### Phase 3 — Mapped WebSocket Runner + +新增通用 WebSocket runner，读取 `DataSourceConfig + active mapping`： + +- 建立长连接。 +- 发送可选订阅消息。 +- 循环接收消息。 +- JSON parse。 +- 按 `ws_message_path/ws_items_path` 提取 item 或 list。 +- 使用 mapping engine 转换。 +- 使用 target schema validator 校验。 +- 调用 destination handler 写入。 +- 更新采集任务状态： + - `connecting` + - `streaming` + - `reconnecting` + - `stopped` +- 维护运行指标： + - `messages_seen` + - `records_written` + - `unique_entities` + - `last_message_at` + - `last_error` +- 后台长连接不读取 `config.debug_max_messages`；该字段只用于显式的一次性调试运行，避免正式 WS 流被测试上限截断。 + +### Phase 4 — Destination Handler + +为 target schema 建立明确写入处理器。 + +`vessel_ais` handler： + +- 写入 `AISRawObservation`。 +- `source = datasource.name`。 +- `delivery_mode` 来自 config，默认 WS 为 `realtime_stream`、REST 为 `polling`。 +- `transport` 来自 `source_type`。 +- 生成幂等 observation hash。 +- 更新 AIS source health。 +- 广播 `vessels` channel，payload 使用当前 Earth 已支持的 upsert 格式。 + +`generic_records` handler： + +- 写入通用 collected data 或后续 generic store。 +- 不直接进入 Earth。 + +### Phase 5 — 删除与数据清理 + +删除自定义源时新增清理策略： + +| 选项 | 行为 | +|-----|------| +| 只删除配置 | 删除 `datasource_configs`，保留 mapping 和历史数据需要另行处理 | +| 删除配置和 mapping | 删除配置及对应 `datasource_mapping_templates` | +| 删除配置、mapping 和该源数据 | 同时删除该源写入的数据 | + +数据删除范围： + +- `collected_data.source == datasource.name` +- `ais_raw_observations.source == datasource.name` +- `ais_source_health.source == datasource.name` + +不建议直接删除 legacy `vessel_position`，因为当前 legacy 表不带 source，无法安全归因。自定义 AIS 源应优先只写 raw observations。 + +删除数据后应触发： + +- `vessels` channel 的 reload/invalidation 事件，提示 Earth 重新拉船只聚合。 +- 后续接入 `Earth Live Sync` 后，触发 `earth_summary` invalidation。 + +### Phase 6 — Mock AIS WebSocket 服务 + +新增脚本： + +`scripts/mock-ais-ws-server.ts` + +运行方式建议： + +```bash +bun run mock:ais-ws +``` + +服务行为： + +- 监听 `ws://localhost:8787/ais`。 +- 接受任意客户端连接。 +- 可记录收到的 subscribe message。 +- 每 1-2 秒发送一条 AIS-like JSON。 +- 每隔 N 条生成新 MMSI，验证船只数量增长。 +- 已存在 MMSI 随时间改变 `lat/lon/cog/heading`，验证同 MMSI upsert。 +- 支持固定 seed，保证测试可复现。 + +示例 payload： + +```json +{ + "type": "vessel", + "data": { + "mmsi": "999000001", + "name": "MOCK VESSEL 001", + "lat": 31.23, + "lon": 121.47, + "sog": 12.4, + "cog": 86, + "heading": 90, + "received_at": "2026-05-01T00:00:00Z" + } +} +``` + +## 前端实施计划 + +### 信息架构调整 + +自定义源不作为割裂的新入口，而是作为内置采集器的补充源，直接纳入“配置中心 > 采集器设置”的采集器选择器： + +- 采集器下拉框同时展示内置采集器和自定义补充源。 +- 下拉框右侧提供加号按钮，用于添加自定义源。 +- 新建自定义源时必须选择“合并到内置数据”，例如合并到 `barentswatch_vessels`。 +- 选择自定义源后，右侧基础配置区域沿用正常采集器配置形态，支持连接测试、保存、endpoint、headers、auth、高级 JSON。 +- 自定义源比内置源多一个“删除自定义源”按钮。 +- 删除时弹出确认框，可勾选“同时删除该自定义源生成的所有数据”。 + +数据源页保留： + +- 内置源总览。 +- 内置源最近状态。 +- 内置源手动触发。 +- 不展示自定义源管理入口；自定义源创建、维护、删除统一在采集器设置中完成。 + +### 自定义源表单 + +新增或重构自定义源表单： + +- 源名称。 +- 类型：`REST` / `WebSocket`。 +- 合并到内置数据：必选，用于声明该源补充哪个内置数据域。 +- endpoint。 +- method/body/params，仅 REST 显示。 +- subscribe message/message path/items path，仅 WS 显示。 +- auth type。 +- headers。 +- target schema。 +- sample/test 按钮。 +- mapping assistant/preview。 +- 保存并运行。 + +### 删除确认 + +删除自定义源时弹出确认： + +- 默认只删除配置。 +- 可勾选删除 mapping。 +- 可勾选删除该源写入的数据。 +- 显示将删除的数据范围和不可恢复提示。 + +## 验证方案 + +### Mock WS 验证路径 + +1. 启动 mock 服务： + +```bash +bun run mock:ais-ws +``` + +2. 新建自定义源： + +| 字段 | 值 | +|-----|----| +| name | `mock_ais_ws` | +| source_type | `websocket` | +| endpoint | `ws://localhost:8787/ais` | +| merge_target_source | `barentswatch_vessels` | +| target_schema | `vessel_ais` | +| ws_message_path | `$.data` | + +3. 保存 active mapping： + +```json +{ + "source": { + "items_path": "$" + }, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "name": {"path": "$.name", "type": "string"}, + "lat": {"path": "$.lat", "type": "float"}, + "lon": {"path": "$.lon", "type": "float"}, + "sog": {"path": "$.sog", "type": "float", "default": null}, + "cog": {"path": "$.cog", "type": "float", "default": null}, + "heading": {"path": "$.heading", "type": "integer", "default": null}, + "received_at": {"path": "$.received_at", "type": "datetime", "default": null} + } +} +``` + +4. 启动自定义源。 + +5. 打开 Earth 船只图层，不刷新页面观察： + +- `vessels` WS channel 收到 `source = mock_ais_ws`。 +- HUD 船只数在新 MMSI 到达时增加。 +- 地球出现 `MOCK VESSEL`。 +- 同 MMSI 后续消息更新位置和航向，不重复叠加。 + +### 自动化测试 + +后端测试： + +- WebSocket 自定义源连接测试。 +- WS message path 和 items path 提取。 +- mapping 到 `vessel_ais`。 +- 写入 AIS raw observation。 +- 广播 `vessels` channel。 +- 删除自定义源时按策略删除 mapping 和源数据。 + +前端测试： + +- REST/WS 表单条件显示。 +- 删除确认选项。 +- mock 源配置保存 payload。 +- mapping preview 展示错误和成功记录。 + +## 风险与约束 + +- WebSocket 自定义源是长连接，不能沿用一次性 REST 进度条。 +- 如果 mock 源写 legacy vessel 表，删除会变得不安全，因此先只写 raw observations。 +- 自定义 WS 可能消息量很大，必须有 backpressure、日志限流和任务取消能力。 +- 任意外部 WS 不能信任 payload，必须经过 mapping 和 schema validation。 +- headers/auth 不能进入 LLM mapping prompt。 + +## 交付顺序 + +1. Mock AIS WS 服务。 +2. 后端自定义 WS runner。 +3. `vessel_ais` destination handler 和 `vessels` broadcast。 +4. 删除自定义源及数据清理。 +5. 设置中心采集器下拉框内联自定义源 UI。 +6. 配置中心信息架构重整。 +7. 与 `Earth Live Sync` 对接 summary invalidation。 diff --git a/docs/plans/datasource-custom-api-mapping-plan.md b/docs/plans/datasource-custom-api-mapping-plan.md new file mode 100644 index 00000000..93ebbc13 --- /dev/null +++ b/docs/plans/datasource-custom-api-mapping-plan.md @@ -0,0 +1,426 @@ +# 自定义 API 数据源与 LLM 映射系统 — 实施计划 + +**状态**：规划中 +**创建日期**：2026-04-28 +**核心原则**：LLM 辅助生成映射配置；生产采集使用确定性转换引擎 + +## 已确认决策 + +| 项目 | 决策 | +|-----|------| +| 自定义 API 的定位 | 作为内置数据源的补充入口，不直接等同于 Earth 新功能 | +| LLM 的职责 | 探索未知 API、分析样本 JSON、生成 mapping 草案 | +| 采集时是否调用 LLM | 不调用；采集链路必须确定性、可审计、可复现 | +| 自定义数据如何进入 Earth | 必须映射到已支持的目标 schema，或先进入通用数据沉淀 | +| 外部凭证放置位置 | Settings / 外部集成统一管理 provider token；DataSources 引用 provider profile | +| TimescaleDB | 放入 TODO；高频时序数据稳定后再评估迁移 | + +--- + +## 一、背景与问题 + +当前系统已经有 `datasource_configs`，可以配置自定义数据源的 endpoint、auth、headers、config，也已经有部分 collector 会读取这些配置。但这只能解决“怎么请求数据”，还没有解决以下问题： + +- API 返回 JSON 后，如何转换成系统已有领域模型。 +- 自定义数据源是补充已有能力，还是全新数据沉淀。 +- 转换规则由谁生成、谁校验、谁执行。 +- 未知数据是否能自动在 Earth 上展示。 +- 外部 token 是放在全局配置中心，还是放在每个 datasource 下。 + +专业做法是把“请求配置”“外部凭证”“目标 schema”“字段映射”“采集执行”拆开： + +- Settings 管外部集成凭证，例如 AI Provider、BarentsWatch、未来付费 AIS API。 +- DataSources 管具体数据源实例，例如 endpoint、调度频率、目标 schema、mapping 版本。 +- LLM 只在配置阶段辅助生成 mapping，不进入生产采集链路。 +- Earth 只消费明确 schema 的数据，不消费任意未知 JSON。 + +--- + +## 二、目标架构 + +### 2.1 自定义 API 数据源生命周期 + +```mermaid +flowchart LR + A[配置 endpoint/auth/request] --> B[抓取 sample JSON] + B --> C[选择目标 schema] + C --> D[LLM 生成 mapping 草案] + D --> E[确定性 mapping engine 预览] + E --> F[schema validation] + F --> G[保存 mapping version] + G --> H[scheduler 执行 mapped collector] + H --> I[写入目标表或 generic_records] +``` + +### 2.2 目标 schema 分层 + +| schema | 用途 | Earth 可视化 | +|-------|------|-------------| +| `vessel_ais` | 船只 AIS 位置、航速、航向、MMSI 等 | 进入船舶图层 | +| `geo_points` | 通用点位数据，包含经纬度、名称、类型、时间 | 进入通用 geo layer（TODO） | +| `news_events` | 新闻/事件类数据，带时间、地点、摘要、来源 | 复用新闻/事件链路 | +| `compute_centers` | 算力中心、机房、数据中心数据 | 复用算力中心图层 | +| `generic_records` | 未知结构化数据沉淀 | 不直接展示 | + +v1 建议优先实现： + +- `vessel_ais` +- `geo_points` +- `generic_records` + +其他 schema 可先在 registry 中预留名称，但不承诺完整落库与可视化。 + +### 2.3 LLM 的边界 + +LLM 可以做： + +- 根据 API 文档或 sample JSON 解释字段含义。 +- 推荐目标 schema。 +- 生成 mapping JSON 草案。 +- 给出字段置信度和需要人工确认的字段。 +- 帮用户发现分页、数组路径、时间字段、坐标字段。 + +LLM 不应该做： + +- 在正式采集时参与每批数据转换。 +- 生成并执行 Python/JavaScript 代码。 +- 接触 API key、bearer token、basic auth password。 +- 自动创建新的 Earth 图层或数据库表。 + +--- + +## 三、后端实施计划 + +### Phase 1 — Target Schema Registry + +新增代码级 registry，统一描述系统支持的目标数据类型。 + +每个 target schema 至少包含： + +- `key`：例如 `vessel_ais`。 +- `label`：前端展示名称。 +- `description`：适用场景。 +- `fields`：字段名、类型、是否必填、说明、示例。 +- `validator`：Pydantic 或等价校验器。 +- `destination`：写入目标，例如 vessel 表、generic_records、future geo layer。 + +示例概念： + +```json +{ + "key": "vessel_ais", + "fields": [ + {"name": "mmsi", "type": "integer", "required": true}, + {"name": "lat", "type": "float", "required": true}, + {"name": "lon", "type": "float", "required": true}, + {"name": "sog", "type": "float", "required": false}, + {"name": "cog", "type": "float", "required": false}, + {"name": "received_at", "type": "datetime", "required": false} + ] +} +``` + +### Phase 2 — Mapping Template Model + +新增 mapping 配置持久化表，建议命名为 `datasource_mapping_templates`。 + +关键字段： + +- `id` +- `datasource_config_id` +- `target_schema` +- `mapping_json` +- `sample_payload_hash` +- `validation_status` +- `version` +- `is_active` +- `created_at` +- `updated_at` + +`mapping_json` 是声明式 DSL，不允许任意代码执行。 + +示例： + +```json +{ + "source": { + "items_path": "$.data.vessels[*]" + }, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "lat": {"path": "$.latitude", "type": "float"}, + "lon": {"path": "$.longitude", "type": "float"}, + "sog": {"path": "$.speedOverGround", "type": "float", "default": null}, + "received_at": {"path": "$.timestamp", "type": "datetime"} + } +} +``` + +### Phase 3 — Deterministic Mapping Engine + +实现独立 mapping engine，输入 sample/raw payload 和 mapping JSON，输出目标 schema 记录。 + +v1 支持能力： + +- JSONPath/JMESPath 风格路径提取。 +- 数组展开。 +- 默认值。 +- 基础类型转换：string、integer、float、boolean、datetime。 +- 坐标范围校验。 +- 简单枚举映射。 +- 错误收集：缺字段、类型转换失败、路径不存在。 + +明确不支持： + +- 任意表达式执行。 +- 用户提交脚本。 +- LLM runtime 修复。 + +### Phase 4 — LLM Mapping Assistant API + +新增配置阶段 API： + +- `POST /api/v1/datasources/custom/sample` + - 按 datasource 请求配置抓取 sample JSON。 +- `GET /api/v1/datasources/target-schemas` + - 返回可选目标 schema 和字段说明。 +- `POST /api/v1/datasources/mappings/propose` + - 输入 sample JSON + target schema，调用 AI provider 生成 mapping 草案。 +- `POST /api/v1/datasources/mappings/preview` + - 使用确定性 mapping engine 预览转换结果。 +- `POST /api/v1/datasources/mappings` + - 保存 mapping 版本。 +- `PUT /api/v1/datasources/mappings/{id}` + - 更新 mapping，生成新版本或覆盖草稿。 +- `POST /api/v1/datasources/{id}/run-mapped` + - 手动触发一次 mapped collector。 + +安全要求： + +- `propose` 请求发送给 LLM 前必须脱敏 sample。 +- auth headers、token、password 不进入 prompt。 +- LLM 返回结果必须再经过 mapping schema 校验。 + +### Phase 5 — Generic Mapped HTTP Collector + +新增通用 collector： + +- 读取 `DataSourceConfig` 请求配置。 +- 读取 active mapping template。 +- 拉取 API 数据。 +- 使用 mapping engine 转换。 +- 使用 target schema validator 校验。 +- 调用 destination handler 写入目标表或 generic storage。 +- 将失败记录写入错误日志或 dead-letter 结构。 + +对于 `generic_records`： + +- 保存 datasource id。 +- 保存 target schema。 +- 保存 normalized JSON。 +- 保存 raw payload 摘要或 raw reference。 +- 保存采集时间、source timestamp、mapping version。 + +--- + +## 四、前端实施计划 + +### Phase 1 — Settings 外部集成 + +Settings 中保留统一外部集成配置： + +- AI Provider：base URL、model、API key。 +- BarentsWatch：client id/client secret 或 bearer token。 +- 未来付费接口：AISHub、MarineTraffic、VesselFinder 等 provider profile。 + +DataSources 不直接管理全局 secret，只引用 provider profile。 + +### Phase 2 — Settings 自定义源向导 + +自定义数据源配置入口应放在 `/settings` 的“采集器设置”或后续专门的自定义采集器设置区。`/datasources` 保持数据源目录和采集触发职责，不再承载编辑入口。 + +自定义数据源配置改成向导或右侧 drawer： + +1. Request + - endpoint + - method + - auth profile + - headers + - query/body config + - schedule +2. Sample + - 点击抓取 sample + - 展示 JSON tree + - 支持选择数组根路径 +3. Target Schema + - 选择 `vessel_ais`、`geo_points`、`generic_records` + - 展示该 schema 必填字段 +4. Mapping Proposal + - 调用 LLM 生成 mapping 草案 + - 显示字段匹配置信度 + - 标出需要人工确认的字段 +5. Preview + - 用确定性 engine 预览前 N 条转换结果 + - 展示校验错误 +6. Save & Enable + - 保存 mapping version + - 启用调度或仅保存草稿 + +### Phase 3 — 运维视图 + +为 mapped datasource 展示： + +- 上次运行时间。 +- 成功记录数。 +- 失败记录数。 +- 当前 mapping version。 +- 目标 schema。 +- 最近错误。 +- 手动运行按钮。 + +--- + +## 五、数据库与存储策略 + +### v1：继续使用 PostgreSQL + +PostgreSQL 可以承载当前规模的采集、关系查询、JSONB 沉淀和基础时序查询。v1 不必因为“时序数据”立刻引入 TimescaleDB。 + +适合继续用 PostgreSQL 的场景： + +- 数据量可控。 +- 最近状态查询为主。 +- 历史保留窗口较短。 +- 查询模式还没稳定。 +- 需要快速迭代 schema 与 mapping。 + +### TODO：TimescaleDB + +以下条件满足后，再评估 TimescaleDB： + +- AIS、遥测、轨迹类数据达到高频持续写入。 +- 需要按时间窗口做聚合、降采样、retention policy。 +- 单表时间序列查询明显成为瓶颈。 +- 历史轨迹保留从 24h 扩展到数周或数月。 + +候选迁移对象： + +- `vessel_position` +- future telemetry tables +- future generic time-series records + +备选方案： + +- PostgreSQL 原生按天/月分区。 +- TimescaleDB hypertable。 +- 热数据 PostgreSQL，冷数据对象存储。 + +--- + +## 六、安全与治理 + +### Secret 管理 + +- Settings 中保存 provider credentials。 +- API 返回配置时必须 mask secret。 +- LLM prompt 只能包含脱敏 sample 和 schema 说明。 +- 后续 TODO：引入字段级加密或 KMS。 + +### Mapping 治理 + +- 每次 mapping 变更保留版本。 +- active mapping 只能有一个。 +- 允许保存 draft mapping。 +- 运行记录关联 mapping version。 +- 校验失败不能自动启用。 + +### 错误处理 + +常见错误类型： + +- API 401/403：凭证错误或过期。 +- API 429：限流，需要调整 schedule。 +- JSON path 不存在：上游结构变化。 +- 类型转换失败：mapping 规则错误。 +- schema validation failed：转换结果不满足目标模型。 + +每次运行需要记录： + +- datasource id。 +- mapping version。 +- started_at / finished_at。 +- fetched count。 +- mapped count。 +- written count。 +- failed count。 +- error summary。 + +--- + +## 七、测试计划 + +### Backend Unit Tests + +- mapping engine： + - path 提取。 + - 数组展开。 + - 默认值。 + - 类型转换。 + - datetime parse。 + - 枚举映射。 + - 缺字段错误。 +- target schema registry： + - `vessel_ais` 必填字段校验。 + - `geo_points` 经纬度范围校验。 + - `generic_records` 接受未知结构。 +- LLM assistant： + - mock provider 返回 mapping。 + - 验证 secret 不进入 prompt。 + - 验证非法 mapping 被拒绝。 + +### Backend Integration Tests + +- sample JSON -> propose mapping -> preview -> save mapping。 +- mapped collector 使用保存的 mapping 写入 `generic_records`。 +- `vessel_ais` sample 写入船舶相关目标结构。 +- 上游 JSON 结构变化时，运行失败并记录错误。 + +### Frontend Tests + +- 自定义数据源向导完整流程。 +- 未配置 AI Provider 时，提示去 Settings 配置，但允许手写 mapping。 +- LLM 返回不完整 mapping 时，Preview 阶段显示校验错误。 +- 保存 mapping 后展示 active version 和运行状态。 + +--- + +## 八、分期工作量 + +| 阶段 | 内容 | 估算 | +|-----|------|------| +| Phase 0 | 完成本规划、确认 schema registry 设计 | 0.5 天 | +| Phase 1 | target schema registry + mapping template model | 1–2 天 | +| Phase 2 | deterministic mapping engine | 2–3 天 | +| Phase 3 | sample/propose/preview/save API | 2–3 天 | +| Phase 4 | DataSources 自定义源向导 | 3–5 天 | +| Phase 5 | generic mapped collector + run history | 2–4 天 | +| Phase 6 | vessel_ais / geo_points destination handler | 2–4 天 | + +--- + +## 九、当前差距与下一步 + +当前差距： + +- `datasource_configs` 只描述请求配置，不描述目标 schema 和 mapping。 +- 自定义源没有 sample -> schema -> mapping -> preview -> save 的闭环。 +- 生产采集还没有通用 mapped collector。 +- Settings 与 DataSources 的职责边界需要在 UI 上进一步明确。 +- Earth 还没有通用 `geo_points` 图层。 + +下一步建议： + +1. 先实现 target schema registry 和 mapping engine，不急着接 LLM。 +2. 用固定 sample JSON 做 `vessel_ais` 和 `generic_records` 的单元测试。 +3. 再接 LLM propose API，让 LLM 产出的只是 mapping 草案。 +4. 最后做前端向导，把人工确认和 preview 放到启用之前。 diff --git a/docs/plans/docs-gatekeeper-auth-plan.md b/docs/plans/docs-gatekeeper-auth-plan.md new file mode 100644 index 00000000..2898cf52 --- /dev/null +++ b/docs/plans/docs-gatekeeper-auth-plan.md @@ -0,0 +1,92 @@ +# Docs Gatekeeper 鉴权系统计划 + +**状态**：已实现，当前行为见 [Docs Gatekeeper 开发说明](/home/ray/dev/linkong/planet/docs/technical/zh/docs-gatekeeper-development.md) +**创建日期**：2026-05-08 +**核心目标**：把 `/docs` 从前端公开打包 Markdown 改成后端受控读取，并通过用户 Gatekeeper 权限组划分公开文档、用户文档、开发文档和管理/运维文档。 + +## 背景 + +当前 Docs 页面通过前端 `import.meta.glob(...?raw)` 把 `docs/technical/{zh,en}` 中注册过的 Markdown 直接打进前端 bundle。即使在前端隐藏目录或增加路由守卫，受保护 Markdown 仍可能出现在构建产物中，无法形成真正鉴权。 + +本阶段需要把文档正文读取迁到后端，并让后端根据当前用户身份返回可见目录和正文。Earth 仍保持公开访问，其它控制台模块暂不改变既有鉴权。 + +## 鉴权模型 + +保留现有 `users.role`，新增 `gatekeeper_groups` 作为可叠加的权限组。`role` 继续用于控制台和系统操作；Gatekeeper 只负责 Docs 等内容权限。 + +默认权限： + +| 身份 | 默认 Docs 能力 | +| --- | --- | +| 未登录访客 | `public` | +| 普通登录用户 | `public`，以及用户被分配的 Gatekeeper 组 | +| `admin` | `docs_admin`，并隐含 `docs_developer` / `docs_user` | +| `super_admin` | 全部 Docs 权限 | + +Gatekeeper 组： + +- `docs_user`：登录用户操作类文档。 +- `docs_developer`：开发、前端、后端、Earth 实现文档。 +- `docs_admin`：运维、服务控制、凭证、环境变量和敏感操作文档。 + +## 初步文档划分 + +`public`： + +- `README.md` +- `quickstart.md` +- `manual.md` + +`docs_developer`： + +- `earth-frontend-context.md` +- `earth-interactable-usage.md` +- `earth-layer-style-reference.md` +- `earth-render-layer-order.md` +- `earth-satellite-footprint-policy.md` +- `earth-bgp-context.md` +- `earth-news-live-streams-collector-format.md` +- `earth-toolbar-overlay-coordination.md` +- `frontend-admin-frontend-context.md` +- `frontend-layout-guidelines.md` +- `backend-collectors.md` +- `datasource-collector-settings-connectivity.md` +- `backend-datasources-api-performance.md` +- `agents-aiprovider.md` + +`docs_admin`： + +- `backend-system-service-control.md` +- `ops-docker-compose-buildx-upgrade.md` +- `ops-planet-sh-startup.md` + +## 实施要点 + +后端新增： + +- `GET /api/v1/docs/catalog`：返回当前用户可见文档目录；未登录只返回 `public`。 +- `GET /api/v1/docs/{lang}/{slug}`：返回单篇 Markdown；未登录访问受保护文档返回 `401`，已登录无权限返回 `403`。 +- 服务端维护文档 metadata 白名单，禁止任意路径读取。 + +用户管理新增： + +- `users.gatekeeper_groups` JSON 字段。 +- 用户列表、创建和编辑支持展示/配置 Gatekeeper 权限组。 +- 只有 `super_admin` 能编辑 Gatekeeper 权限组。 + +前端 Docs 改造： + +- 移除 Markdown raw import 作为正文来源。 +- 从后端 catalog 构建目录和搜索记录。 +- 从后端 content API 加载正文。 +- 对 `401` 显示登录入口，对 `403` 显示无权限提示。 + +## 验证 + +- 未登录用户只能看到和读取 `public` 文档。 +- 未登录直接访问受保护文档返回 `401` 并显示登录提示。 +- 无 Gatekeeper 组的普通用户访问开发文档返回 `403`。 +- `docs_developer` 用户能读开发文档，不能读管理/运维文档。 +- `admin` 和 `super_admin` 能读管理/运维文档。 +- 未知 slug、未知语言和路径穿越字符串不能读取文件。 +- 前端构建产物不再包含受保护 Markdown raw import 生成的文档模块。 diff --git a/docs/plans/earth-compute-center-bgp-style-plan.md b/docs/plans/earth-compute-center-bgp-style-plan.md new file mode 100644 index 00000000..da414c17 --- /dev/null +++ b/docs/plans/earth-compute-center-bgp-style-plan.md @@ -0,0 +1,372 @@ +# Earth Compute Center BGP-Style Plan + +## Goal + +这份文档定义如何按照 BGP 模块的产品方式，把“算力中心”提升为 Earth 上的一级能力。 + +这里的“按 BGP 方式”指的是： + +- 有独立的数据语义和接口入口 +- 有独立的 Earth 图层与图例 +- 有独立的 hover / click / 选中态 / 详情卡 +- 有独立的统计口径与后续专题页扩展空间 + +这里的“按 BGP 方式”不指： + +- 机械复制 BGP 的 anomaly / incident / collector 三层事件模型 +- 为静态算力设施强行引入不必要的复杂告警语义 + +算力中心本质上更接近“长期基础设施分布层”，不是“高频动态异常层”。 +因此应该复用 BGP 的模块化方法，而不是照搬 BGP 的事件结构。 + +## Why + +当前仓库里已经有算力相关基础： + +- 后端已有 `top500` 和 `epoch_ai_gpu` 数据采集 +- 可视化接口已有 `/api/v1/visualization/geo/supercomputers` 和 `/api/v1/visualization/geo/gpu-clusters` +- Earth 信息卡已对 `supercomputer` 和 `gpu_cluster` 做了基础类型兼容 + +但当前能力还停留在“数据可取到”的阶段，没有形成像 BGP 那样完整的可视化模块： + +- Earth 缺少独立的算力图层加载模块 +- 缺少算力 marker 体系和视觉层级 +- 缺少算力图例、统计、开关和搜索接入 +- 缺少与海缆、BGP、卫星的关系表达 +- 缺少算力专题页和后续告警/研判扩展入口 + +所以当前真正的缺口不是“有没有数据”，而是“有没有产品级模块”。 + +## Core Principle + +算力中心应当采用和 BGP 一致的模块化分层： + +1. 数据层：稳定的数据契约和 GeoJSON 输出 +2. 渲染层：独立的 Earth 图层、marker 和视觉状态管理 +3. 交互层：hover、click、锁定态、详情卡、图例和统计 +4. 扩展层：后续专题页、关系分析、告警和 AI 研判 + +但语义上必须保持算力中心自身的特点： + +- `site / center` 是主对象，不是事件 +- `capacity / rank / vendor / operator / status` 是主信息，不是异常严重度 +- `distribution / concentration / dependency` 是后续分析方向，不是第一阶段必须项 + +## Recommended Scope + +第一版“算力中心”建议统一承载两类对象： + +- `supercomputer` +- `gpu_cluster` + +并在 Earth 上收口为一个主题层：`compute_centers` + +这样做有几个好处： + +- 用户看到的是统一的“算力基础设施”语义，而不是零散数据源 +- 后端仍可保留 `top500` 和 `epoch_ai_gpu` 的来源差异 +- 前端可以在一个图层里再细分两种 marker 语言 + +## Current Gap + +和 BGP 对比，当前差距主要在下面几层。 + +### 1. Data Contract Gap + +现在的算力 GeoJSON 还是通用 `collected_data` 输出思路，字段较轻： + +- `gpu_cluster` 只有基础名称和地点 +- `supercomputer` 只暴露一部分性能字段 +- 缺少统一的 `site_type / operator / capacity_band / source / updated_at / confidence` +- 缺少统一的算力层聚合出口 + +### 2. Earth Rendering Gap + +当前 Earth 里没有类似 `bgp.js` 的算力模块： + +- `constants.js` 没有算力 API 路径和视觉配置 +- `main.js` 没有算力加载、拾取、状态同步和 HUD 更新 +- `controls.js` 没有算力图层开关和启动加载优先级 +- `layer-startup-tasks.js` 没有算力启动任务 +- `legend.js` / `ui.js` 没有算力统计与图例模式 + +### 3. Interaction Gap + +虽然 `info-card.js` 支持基础字段，但还没有形成 BGP 那种完整交互链路： + +- 没有 hover / selected / dimmed 的视觉状态 +- 没有算力对象专属 tooltip 与摘要文案 +- 没有锁定后与其他基础设施的联动高亮 +- 没有搜索、统计卡和详情组织方式 + +### 4. Product Expansion Gap + +当前还没有“算力中心”专题页与分析语义： + +- 没有全球分布/国家聚合/厂商聚合视图 +- 没有算力与海缆/BGP/区域的关系表达 +- 没有 AI brief / assessment 的后续落点 + +## Architecture Direction + +推荐把算力中心做成“BGP 同级能力”，但采用更适合静态基础设施的结构。 + +### Backend + +建议新增统一聚合接口，例如： + +- `/api/v1/visualization/geo/compute-centers` + +它的职责是把： + +- `top500` +- `epoch_ai_gpu` + +统一转换成一个主题层输出，同时保留对象细分类型： + +- `site_type: supercomputer | gpu_cluster` + +建议统一字段至少包括： + +- `id` +- `name` +- `site_type` +- `country` +- `city` +- `latitude` +- `longitude` +- `operator` +- `vendor` +- `capacity_value` +- `capacity_unit` +- `capacity_band` +- `rank` +- `source` +- `updated_at` +- `location_precision` +- `geography_mode` +- `is_estimated` +- `estimated_reason` +- `metadata` + +这里建议优先做“统一聚合出口”，而不是一开始就新增独立数据库表。 + +原因： + +- 当前源数据更新频率低，先复用 `collected_data` 成本更低 +- 可以先把 Earth 产品体验做完整 +- 如果后续要做历史趋势、关系推断、告警，再评估是否拆成独立模型 + +### Frontend Earth + +建议新增独立模块，例如： + +- `frontend/public/earth/js/compute-centers.js` + +职责参照 `bgp.js`： + +- 拉取算力中心 GeoJSON +- 创建 marker +- 管理 hover / selected / dimmed 状态 +- 输出图例项 +- 输出统计摘要 +- 提供 overlay 和详情格式化辅助函数 + +推荐视觉分层： + +1. `supercomputer` 用更稳定、更规整的设施型符号 +2. `gpu_cluster` 用更活跃、更现代的密度型符号 +3. 选中态通过 halo / ring / related infrastructure highlight 表达 + +视觉上应避免把算力中心做成“BGP 事件点”那种高频脉冲风格。 +它应该更像长期存在的高价值设施。 + +## Phases + +## Phase 1: Unified Earth Layer + +目标： + +- 先把算力中心做成 Earth 上可用、可点、可解释的一级图层 + +工作项： + +- 新增统一算力 GeoJSON 接口 +- 新增 `compute-centers.js` +- 在 `constants.js` 增加 API 路径和视觉配置 +- 在 `controls.js` 增加算力图层开关与启动元数据 +- 在 `layer-startup-tasks.js` 增加算力启动加载任务 +- 在 `main.js` 接入算力拾取、hover、click、锁定态和 HUD 统计 +- 在 `ui.js` / `legend.js` / `index.html` 增加算力统计与图例入口 +- 在 `info-card.js` 提升算力详情字段组织 +- 对无法精确定位、但可按国家或弱线索推测的大概位置，仍然生成地图点位 +- 这类对象必须带显式“估算位置”状态，例如图标问号角标与详情说明 + +完成标准： + +- Earth 上能独立显示/隐藏算力中心 +- 两类对象有可区分的视觉表达 +- hover / click / 详情卡 / 图例 / 统计全部打通 +- 精确位置与估算位置在图标或文案上可区分，不会误导为同一精度 +- 不干扰现有海缆、卫星、BGP 的交互链路 + +## Phase 2: Relationship Layer + +目标： + +- 让算力中心不只是“点”，而是和其他基础设施产生上下文关系 + +工作项： + +- 建立算力中心与国家/区域聚合摘要 +- 增加与附近海缆登陆点的关系提示 +- 增加与 BGP 事件/观测范围的空间邻近提示 +- 增加与卫星覆盖或区域连通性的实验性提示 + +完成标准： + +- 点击算力中心时，用户能看到“它和哪些基础设施相关” +- 信息表达以辅助判断为主，不做夸张推断 + +## Phase 3: Compute Center Observatory + +目标： + +- 把算力中心从 Earth 图层扩展成独立专题观测能力 + +工作项： + +- 新增算力中心专题页 +- 提供国家/厂商/类型/容量分布统计 +- 支持列表、筛选、详情和历史快照 +- 预留 AI brief / assessment 入口 + +完成标准： + +- 算力中心不再只是 Earth 上的视觉点位 +- 能作为独立业务上下文进入日常观察与研判 + +## Phase 4: Alerts And Assessment + +目标： + +- 在不滥造“假动态告警”的前提下，引入真正有价值的变化感知 + +候选方向： + +- 新增大规模算力中心 +- 既有中心容量显著变化 +- 国家/区域集中度显著变化 +- 高价值中心与关键网络基础设施关系变化 + +完成标准： + +- 告警来自可解释的结构变化 +- 不把静态数据硬做成噪声式实时事件流 + +## Implementation Notes + +建议按下面顺序推进： + +1. 先统一 GeoJSON 契约 +2. 再做 Earth 独立模块和图层开关 +3. 再补详情卡、图例和统计 +4. 最后才做关系层和专题页 + +这样可以避免一开始把范围摊得过大。 + +## Unknown Location Strategy + +由于部分算力数据源不会直接提供经纬度，未知位置补全不能只依赖“继续找 API 字段”。 +更稳妥的方式是做成一条分层富化链路，而不是单一猜测规则。 + +推荐按下面优先级推进： + +1. 直接源信息 + +- 源记录显式给出 `latitude / longitude` +- 源记录给出 `city / region / facility / campus / operator` +- 源页面详情、内嵌 JSON、结构化元数据、新闻稿链接里能抽出地点线索 + +2. 名称与机构归一化 + +- 建立 `canonical_name / aliases / operator / facility` 归一化表 +- 把 `cluster name`、`operator`、`campus name` 归一到同一个实体 +- 优先解决同一对象多写法导致的命中失败，而不是先扩大猜测范围 + +3. 本地位置注册表 + +- 用仓库内可维护的 registry 保存高价值对象的位置知识 +- 每条记录至少包含：`canonical_name`、`aliases`、`operator`、`country`、`region`、`city`、`lat`、`lon`、`confidence`、`source_note` +- 转换层优先读取 registry，避免地点知识长期散落在转换代码里 + +4. 分层回退定位 + +- `precise` +- `estimated_site` +- `estimated_city` +- `estimated_region` +- `estimated_national_hub` +- `estimated_country` + +这里建议把“国家内主要算力城市”作为国家质心之前的一层。 +例如没有美国精确位置时，优先考虑已知的主要算力/数据中心城市候选，而不是直接落在几何质心。 + +5. 候选证据富化 + +- 如果源 API 无地点信息，可以允许采集链路读取公开辅助证据 +- 例如机构官网、数据中心介绍页、新闻稿、百科型页面、公开 PDF +- 但只提取“地点线索”，不把外部页面上的经纬度当真值直接写回 + +6. 人工校验闭环 + +- 对高价值且仍然未知的对象输出待核验清单 +- 把人工确认结果回写到位置注册表 +- 后续采集继续优先复用这层人工确认结果 + +### Additional Solution Paths + +除了静态映射表，还可以考虑下面这些办法： + +- 基于国家和运营方建立“主要园区候选集”，用稳定散列把同国未知节点分散到若干可信城市，而不是全部压到一个点 +- 基于数据中心/云厂商公开 region 列表建立 `operator -> city set` 候选映射，用于云 GPU 集群类对象 +- 把“估算依据”结构化，例如 `matched_alias`、`matched_operator`、`matched_city_text`、`fallback_country_hub` +- 给位置补全增加 `last_verified_at`，便于后续按时间重新校验老旧映射 +- 单独维护“不可可靠定位”状态；这类对象仍可在国家级聚合统计中出现，但可以允许用户在地图上过滤掉 +- 后续如果你们愿意投入更多，可把这条链路做成小型 enrichment pipeline，而不是仅在 API 转换时临时判断 + +## Non-Goals + +第一阶段不建议做这些内容： + +- 不复制 BGP 巡航模式到算力中心 +- 不先做复杂实时 websocket 推送 +- 不先引入独立 `compute_center_incident` 一类模型 +- 不先做全量 AI 分析面板 + +原因是算力中心的第一需求是“被看清楚”，不是“被实时播报”。 +但“被看清楚”不等于“只显示精确坐标对象”。 +对于没有精确经纬度、但能推测到国家或区域级位置的算力中心，应优先以上图并标注估算状态的方式处理，而不是直接在地图上消失。 + +## Acceptance Checklist + +- 后端存在统一的算力中心 GeoJSON 出口 +- Earth 有独立算力图层模块，而不是散落在 `main.js` +- 页面上有清晰的算力开关、图例和统计 +- `supercomputer` 和 `gpu_cluster` 在视觉和详情上都可区分 +- 估算位置对象在地图和详情中都有明确状态提示 +- 现有 BGP / 海缆 / 卫星功能无回归 +- 代码结构上为后续专题页和关系分析留出了明确扩展点 + +## Summary + +这项工作的本质不是“再多画几个点”。 + +它应该把算力中心从已有数据源，升级成与 BGP 同级的 Earth 观测主题： + +- 有独立语义 +- 有独立图层 +- 有独立交互 +- 有后续分析扩展能力 + +推荐先完成 Phase 1，把算力中心做成真正可用的 Earth 一级模块，再继续推进关系层和专题页。 diff --git a/docs/plans/earth-interactable-layer-plan.md b/docs/plans/earth-interactable-layer-plan.md new file mode 100644 index 00000000..29d67010 --- /dev/null +++ b/docs/plans/earth-interactable-layer-plan.md @@ -0,0 +1,313 @@ +# Earth Interactable Layer Plan + +## 背景 + +状态：Phase 1 已经开始落地，Phase 2 的 BGP 事件 / 观测站迁移和 Phase 3 的算力中心迁移也已完成。`frontend/public/earth/js/interactable.js` 已新增，AIS 船只、BGP 事件、BGP 观测站和算力中心图层已经改为通过 `createInteractableLayer()` 使用通用批量 `Points`、hover / locked overlay、默认 glow、状态更新、asset icon 预加载、屏幕空间 picking、固定 / 距离缩放和跨 Interactable 同坐标避让。登陆点因 `THREE.Points` 边缘深度裁切和贴地层级要求，已退回专用 `THREE.Sprite` 黄色球路径，并与海缆同高度同 renderOrder。后续阶段聚焦把可复用的扩圈 / 雷达扇形动画正式沉淀成 `animations` 扩展。 + +当前实现说明和接入示例见： + +- [earth-interactable-usage.md](/home/ray/dev/linkong/planet/docs/technical/zh/earth-interactable-usage.md) + +当前 AIS 船只图层已经形成了一个适合作为基准的交互图标模式： + +- 普通态使用批量 `THREE.Points` 渲染，避免每个对象一个 `Sprite` 带来的 draw call 和透明排序压力。 +- hover / locked 态使用单点 overlay 叠加 glow，不改变普通批次，交互反馈清晰且成本低。 +- moving / anchored 船只通过 canvas 点纹理表达不同形状，moving 船只还按航向分桶。 +- 拾取走屏幕空间命中，拖动和惯性期间跳过高频 hover picking。 +- 图层高度贴近地表，仅保留很小的深度余量，避免“浮在表面层”的观感。 + +这个模式不应该只服务船只。后续 BGP 事件、BGP 观测站、算力中心、新闻事件、告警、地面传感器等都可能需要“图标类可交互元素”。如果每个图层继续各写一套 icon、glow、hover、locked、动画、picking 和图例逻辑，视觉会漂移，性能策略也会重复分叉。登陆点已经验证为例外：需要完整贴地且不被球面边缘裁切时，专用 Sprite 路径比通用 `Points` 更合适。 + +目标是把船只图层的成功做法抽象成一个通用接口：业务图层只描述“要画什么、在哪里、怎么交互”，底层统一负责批量渲染、默认 glow、状态 overlay、动画槽位、拾取和生命周期。 + +## 目标 + +1. 建立统一的 Earth 交互图标接口，作为未来地表图标类元素的默认入口。 +2. 以 AIS 船只 glow 为默认 glow 视觉，其它图标默认沿用同一套 glow 质感。 +3. 保留图标颜色、状态颜色、hover 放大、locked 强调、dimmed 聚焦、动画扩展等能力。 +4. 支持 canvas / SVG / image icon，不强行要求所有图标都可重着色。 +5. 保持船只当前性能路线：批量绘制普通态，少量 overlay 处理交互态。 +6. 给 BGP 事件扩圈、BGP 观测站雷达扇形等补充动画留出正式扩展点。 + +## 非目标 + +- 不在第一阶段重写所有 Earth 图层。 +- 不把卫星、海缆、国家边界、真实地形这类非图标图层纳入同一个接口。 +- 不为了抽象牺牲业务图标的差异表达，例如船只航向、BGP 事件严重级别、观测站雷达扫掠。 +- 不要求图片图标支持运行时重着色；图片图标只能通过预制多状态图片或 overlay tint 做有限表达。 + +## 核心设计 + +建议新增一个通用模块，例如： + +```text +frontend/public/earth/js/interactable.js +``` + +它导出一个工厂或注册函数： + +```js +createInteractableLayer({ + id, + earth, + renderOrder, + altitudeOffset, + icon, + scale, + glow, + colors, + states, + animations, + picking, + data, + getPosition, + getKind, + getRotation, + getPayload, +}); +``` + +业务模块仍保留自己的数据加载、图例、详情卡字段和业务语义。例如 `vessels.js` 负责 AIS 数据和船型映射，但 icon 渲染、hover overlay、locked overlay、默认 glow 和屏幕空间 picking 可以逐步迁入 `interactable.js`。 + +## 参数草案 + +| 参数 | 类型 / 示例 | 默认值 | 说明 | +| --- | --- | --- | --- | +| `id` | `"vessels"` | 必填 | 图层唯一标识，用于 debug、picking、legend 和状态缓存。 | +| `earth` | `THREE.Object3D` | 必填 | 图层挂载目标，通常是 Earth root。 | +| `renderOrder` | `4.4` | `4` | 普通 icon 批次和 overlay 的基础渲染顺序。 | +| `altitudeOffset` | `0.2` | `0.2` | 图层高度，语义为 `CONFIG.earthRadius + altitudeOffset`。地表图标默认贴近真实地形基础层。 | +| `icon` | `{ type, source, draw, size, bins }` | 必填 | 图标来源。支持 canvas draw、SVG URL、image URL、内置 shape。 | +| `icon.fitSize` | `60` 或 `{ width: 60, height: 60 }` | `atlasCellSize` | asset 图标在 atlas canvas 内的最大绘制尺寸，默认居中等比 contain。SVG / 图片文件只负责原始形状，不需要为了显示大小手写 transform。 | +| `scale` | `{ base, min, max }` | `{ base: 1 }` | 基础缩放和距离稳定范围。当前船只可映射到 `VESSEL_POINT_SIZE` / `baseScale`。 | +| `sizeMode` | `"fixed" / "distance"` | `"fixed"` | 是否固定屏幕像素尺寸；非 fixed 时按相机到地表距离做比例缩放。 | +| `sizeScale` | `{ min, max, referenceFov }` | `{ min: 0.12, max: 3, referenceFov: 75 }` | `sizeMode !== "fixed"` 时的缩放限制和参考视角。 | +| `glow.enabled` | `true / false` | `true` | 是否启用默认 glow。默认 glow 以船只 hover / locked overlay 为基准。 | +| `glow.intensity` | `0.0 - 2.0` | `1` | glow 强度，内部映射到 canvas `shadowBlur`、opacity 或 shader uniform。 | +| `glow.colorMode` | `"state" / "icon" / "fixed"` | `"state"` | glow 颜色来源，默认跟随状态颜色。 | +| `hover.scale` | `1.0 - 2.0` | `1.18` | hover 放大倍率。当前船只保持同尺寸 glow overlay，接口仍保留放大能力供其它图层使用。 | +| `hover.mode` | `"scale" / "glow-only" / "custom"` | `"scale"` | hover 反馈方式。船只可用 `"glow-only"`，其它图标默认放大。 | +| `colors.normal` | `"#4A90D9"` | icon 原色 | 普通态颜色。只有可上色 icon 生效。 | +| `colors.hover` | `"#7dd3fc"` | normal | hover 态颜色。 | +| `colors.locked` | `"#ffffff"` | hover | locked 态颜色。 | +| `colors.dimmed` | `"#9B9B9B"` | normal | 聚焦其它对象时的弱化颜色。 | +| `colors.byKind` | `{ cargo: "#4A90D9" }` | `{}` | 按业务类型着色，如船型、BGP 严重级别。 | +| `colorable` | `true / false` | 由 icon 类型推断 | canvas shape 和 SVG mask 通常可上色；图片默认不可上色。 | +| `opacity` | `{ normal, hover, locked, dimmed }` | 船只当前值 | 各状态透明度。 | +| `rotation` | `{ enabled, bins, getAngle }` | disabled | 是否按角度分桶，例如船只按 COG 分 32 桶。 | +| `animations` | `IconAnimationSpec[]` | `[]` | 补充动画列表，例如扩圈、雷达扇形、脉冲、轨迹尾迹。 | +| `picking.radiusPx` | `22` | `20` | 屏幕空间命中半径。 | +| `picking.throttleMs` | `100` | `80` | hover picking 节流。 | +| `picking.skipWhileDragging` | `true` | `true` | 拖动和惯性期间跳过 hover picking。 | +| `zIndexPolicy` | `"surface-icon"` | `"surface-icon"` | 预设层级策略，避免每个业务图层手写高度和 renderOrder。 | +| `avoidance.enabled` | `true / false` | `true` | 是否参与跨 Interactable 的同坐标避让。默认开启，同一经纬度下的图标会沿地表切平面小幅排开，方便辨认和选择。 | +| `avoidance.radius` | `number` | `1.1` | 同坐标避让的第一圈半径，单位为地球本地坐标单位。 | +| `avoidance.precision` | `number` | `4` | 经纬度归并精度，默认约等于只处理几乎完全重叠的图标。 | +| `legend` | `{ label, color, shape }[]` | `[]` | 可选图例声明，业务层也可以继续自己导出。 | +| `metadata` | object | `{}` | 业务扩展数据，不参与渲染但参与 tooltip / info-card / search。 | + +## Icon 规格 + +图标输入建议分三类： + +```js +{ + type: "canvas-shape", + size: 128, + draw(ctx, state) { + // draw triangle / dot / custom shape + }, +} +``` + +```js +{ + type: "svg-mask", + source: "/earth/assets/icons/bgp-event-dot.svg", + colorable: true, +} +``` + +```js +{ + type: "image", + source: "/earth/assets/icons/vendor-logo.png", + colorable: false, + stateSources: { + hover: "/earth/assets/icons/vendor-logo-hover.png", + }, +} +``` + +颜色策略： + +- `canvas-shape` 默认可上色，适合船只、事件点、雷达站这类符号。 +- `svg-mask` 如果能作为 mask 使用，则可上色；如果是完整多色 SVG，则按图片处理。 +- `image` 默认不可上色；需要状态变化时使用 `stateSources` 或额外 glow / ring。 + +## 默认 Glow 规范 + +默认 glow 以当前船只 overlay 为视觉基准： + +- 普通态尽量不启用 glow，保持地图干净。 +- hover / locked 态叠加同位置 overlay。 +- glow 颜色默认跟随状态颜色或业务类型颜色。 +- glow blur 应该稳定，不随 camera zoom 夸张膨胀。 +- 允许通过 `glow.intensity` 控制强度，但不要让业务图层各自发明完全不同的光晕语言。 + +建议内部把 glow 拆成两个层次： + +1. `textureGlow`：canvas texture 里的 `shadowBlur`，适合小图标 hover / locked。 +2. `effectGlow`：额外 ring / halo / pulse，适合告警、BGP 事件和锁定强调。 + +## 状态模型 + +通用状态至少包含： + +| 状态 | 触发 | 默认表现 | +| --- | --- | --- | +| `normal` | 普通显示 | 批量 Points，使用 normal 颜色和 opacity。 | +| `hover` | 指针悬停 | 默认放大并显示 glow；船只可配置为同尺寸 glow-only。 | +| `locked` | 点击锁定 / 详情打开 | 强 glow、更高 opacity，可选 ring 或 pulse。 | +| `dimmed` | 聚焦其它对象 | 降低 opacity，保留上下文。 | +| `hidden` | 图层关闭或过滤 | 不参与绘制和 picking。 | +| `alert` | 业务告警 | 可叠加动画，不替代 locked 状态。 | + +状态更新需要增量化：只在 hover 目标、locked 目标、过滤条件、数据版本或相机距离阈值变化时更新，不在每帧遍历全部 icon 写材质属性。 + +## 动画扩展 + +动画不直接塞进 icon 基础参数，而是作为 `animations` 列表注册。每个动画声明自己的 geometry / material / update 策略： + +```js +{ + type: "expanding-ring", + when: ["alert", "locked"], + color: "state", + radiusPx: [10, 42], + durationMs: 1400, + opacity: [0.8, 0], +} +``` + +```js +{ + type: "radar-sweep", + when: ["normal", "hover", "locked"], + angleDeg: 72, + rotationMs: 2600, + opacity: 0.36, +} +``` + +首批建议内置动画： + +| 动画 | 用例 | 说明 | +| --- | --- | --- | +| `pulse-ring` | locked、告警点 | 原地呼吸环，强调选中对象。 | +| `expanding-ring` | BGP 事件 | 向外扩散的事件波纹。 | +| `radar-sweep` | BGP 观测站 | 扇形扫描，可持续旋转。 | +| `orbiting-dot` | 数据流 / collector 活跃态 | 小点绕 icon 环绕，表达活动状态。 | +| `trail` | 移动目标 | 可选短尾迹，船只或飞机类目标使用。 | + +动画必须支持批量或分组绘制，避免为每个对象创建独立的高频更新对象。只有 locked / hover / 少量 alert 对象可以使用单对象 overlay。 + +## 渲染策略 + +### 普通态 + +普通态优先使用分桶 `THREE.Points`： + +- 按 icon 类型、可上色策略、旋转分桶、纹理 key 分组。 +- 每组一个 `BufferGeometry`，存 `position`、`color`、必要的 `payloadIndex`。 +- `PointsMaterial.sizeAttenuation = false`，保持屏幕尺寸稳定。 +- `depthTest = true`，`depthWrite = false`，避免遮挡关系破坏地表。 + +### 交互态 + +hover / locked 使用少量 overlay： + +- overlay 复用 `THREE.Points` 单点对象或小型 ring mesh。 +- overlay texture 从统一 cache 获取。 +- overlay 更新只写当前 hover / locked 的 position、texture、opacity、size。 + +### 高密度升级 + +当某类图标超过分桶 Points 的舒适区，才考虑升级： + +- `InstancedBufferGeometry` billboard。 +- 自定义 shader 支持 per-instance rotation / scale / opacity。 +- 视口 bbox / LOD / cluster。 + +这个升级不应该改变业务接口，只替换底层 renderer。 + +## Picking 策略 + +沿用船只当前方向： + +- 默认屏幕空间 picking，而不是 Three.js 对每个 Sprite / Points 做 raycast。 +- 每个 icon 保留世界坐标和业务 payload。 +- 每次 pointer move 将候选点投影到屏幕，按半径和深度判断命中。 +- 拖动、惯性旋转、相机剧烈变化期间跳过 hover picking。 +- click 时允许做一次更精确的 picking。 + +后续可以按图层或经纬度网格增加空间索引，减少候选点数量。 + +## 与现有图层的迁移路径 + +### Phase 1：抽出船只基准能力 + +- 从 `vessels.js` 提取 texture cache、canvas icon draw、overlay glow、分桶 Points 创建、状态增量更新。 +- 保持 `vessels.js` 的公开 API 不变：`loadVessels()`、`toggleVessels()`、`getVesselMarkers()` 等继续可用。 +- 新模块先只服务船只，确保视觉没有回退。 + +### Phase 2：迁移 BGP 事件和观测站 + +- BGP 事件使用 `canvas-shape`，已接入 `Interactable`。 +- 严重级别映射到 `colors.byKind`，并通过通用 `getPointSizeMultiplier` 保留严重级别尺寸倍率。 +- 当前扩圈效果保留在 BGP 业务动画中，并跟随 `Interactable` marker 位置更新。 +- BGP 观测站主图标已接入 `Interactable`，活跃度映射到颜色和 `getPointSizeMultiplier`。 +- BGP 观测站 halo / 覆盖扇形继续由 BGP 业务动画表达扫描，并跟随 `Interactable` marker 位置更新。 + +### Phase 3：迁移算力中心并评估登陆点 + +- 算力中心保留现有业务 icon，但接入统一 hover / locked / glow。（已完成） +- 登陆点曾接入同一套 `Points` 渲染，但 pin 类 SVG 在地球边缘会被深度测试裁切；当前保留专用 `THREE.Sprite`，并使用 canvas 生成黄色扁平球，贴到海缆层级。 +- TODO：登陆点暂不迁移到完整 Interactable。后续若要统一交互接口，优先考虑 Sprite-backed adapter，只对齐 `getMarkers()`、`getPointerIntersections()`、`setMarkerState()`、`updateVisualState()` 等外观协议，不强行复用 `THREE.Points`、atlas 和跨图层避让。 +- 检查图例、搜索和 info-card 是否只依赖业务 payload，而不是依赖渲染对象类型。 + +### Phase 4：形成 Earth 图标层规范 + +- 在 `docs/technical/zh/earth-frontend-context.md` 记录当前实现入口。 +- 在 `docs/technical/zh/earth-layer-style-reference.md` 记录默认 glow、状态颜色、默认高度和动画参数。 +- 在 `docs/technical/zh/earth-render-layer-order.md` 记录 surface icon renderOrder 范围。 + +## 风险与约束 + +- 过早抽象可能让船只这种高质量基准被平均化，因此第一阶段必须以船只视觉不回退为验收标准。 +- 图片 icon 不可上色，接口需要明确 `colorable = false` 的行为，避免业务层误以为颜色一定生效。 +- 动画如果默认开启过多，会重新引入 overdraw 和每帧更新压力；默认只给 hover / locked 或少量 alert 使用。 +- 地形开启时，贴地 icon 需要在高度、`depthTest`、`polygonOffset` 和 renderOrder 之间保持平衡。 +- 统一 glow 不等于所有图标一模一样；业务可以调强度和颜色，但不应破坏整体视觉语言。 + +## 验收标准 + +1. 船只迁入通用接口后，普通态、hover、locked、航向、颜色、轨迹和 picking 行为保持一致。 +2. 新增一个 BGP 事件示例图层配置，不需要复制船只渲染代码即可得到 icon、glow、hover 和扩圈动画。 +3. 新增一个 BGP 观测站示例图层配置，不需要自写独立动画循环即可得到雷达扇形。 +4. 关闭图层后对应 icon、overlay、动画和 picking 全部停止。 +5. 高密度数据下普通态仍走批量绘制，hover / locked 只更新少量 overlay。 +6. 文档同步说明默认高度、默认 glow、状态模型和动画扩展点。 + +## 相关文件 + +| 文件 | 当前角色 | 未来关系 | +| --- | --- | --- | +| `frontend/public/earth/js/vessels.js` | 船只基准实现，包含分桶 Points、hover / locked overlay、默认 glow 形态 | Phase 1 的抽象来源 | +| `frontend/public/earth/js/constants.js` | 保存船只高度、颜色、透明度、轨迹参数 | 后续可加入通用 surface icon 默认配置 | +| `frontend/public/earth/js/bgp.js` | BGP 事件和观测站视觉逻辑 | BGP 事件和观测站主图标已接入 Interactable；扩圈、halo 和覆盖扇形仍保留业务动画 | +| `frontend/public/earth/js/compute-centers.js` | 算力中心 icon 和交互 | 已通过 Interactable 接入统一 Points、overlay、glow 和 picking | +| `frontend/public/earth/js/cables.js` | 登陆点 icon 和海缆线 | 登陆点当前使用专用 `THREE.Sprite` 黄色球，不再走 Interactable；海缆线仍独立渲染 | +| `frontend/public/earth/js/main.js` | 当前集中处理 hover、click、locked 和 info-card 入口 | 后续需要接入通用 icon picking 结果 | +| `docs/technical/zh/earth-layer-style-reference.md` | 当前视觉参数参考 | 实现后同步默认 glow 和通用参数 | +| `docs/technical/zh/earth-render-layer-order.md` | 当前层级参考 | 实现后同步 surface icon 层级范围 | diff --git a/docs/plans/earth-mobile-center-country-highlight-plan.md b/docs/plans/earth-mobile-center-country-highlight-plan.md new file mode 100644 index 00000000..efd25978 --- /dev/null +++ b/docs/plans/earth-mobile-center-country-highlight-plan.md @@ -0,0 +1,252 @@ +# Earth Mobile Center Country Highlight Plan + +## Goal + +移动端打开 Earth 国界图层后，用屏幕中心，也就是当前镜头正对的地球表面位置，自动识别所在国家，并高亮该国家国界。 + +桌面端仍保持现有 hover 行为。移动端不引入新的国界渲染体系，而是复用已有 `country-boundaries.js` 的 GeoJSON 命中和 hover 高亮能力。 + +## Criteria for success + +1. 移动端 `layout-mode-mobile` 下，国界图层开启后，屏幕中心所在国家会自动高亮。 +2. 移动端旋转、缩放、巡航或自动旋转地球时，高亮会跟随镜头中心更新。 +3. 屏幕中心落在海洋或没有命中地球时，国家高亮会清除。 +4. 国界图层关闭时，不执行中心国家识别，也不显示残留高亮。 +5. 桌面端 pointer hover 行为保持不变。 +6. 移动端抽屉、搜索、设置、媒体、详情等前景 UI 打开时，不因为用户操作 UI 产生明显误高亮或抖动。 +7. 中心识别有节流或状态缓存，不把 GeoJSON point-in-polygon 检测放到无条件每帧高频执行。 +8. 实现后能通过本地静态检查或前端构建，并用移动端 viewport 手动或 Playwright 验证核心场景。 + +## Existing pieces + +当前项目已经具备大部分基础能力： + +- [frontend/public/earth/js/country-boundaries.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/country-boundaries.js) + - `updateCountryBoundaryHover(coords)`：根据 `{ lat, lon }` 命中国家并更新高亮线。 + - `clearCountryBoundaryHover()`：清除当前 hover 高亮。 + - `getShowCountryBoundaries()`：判断国界线图层是否可见。 +- [frontend/public/earth/js/utils.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/utils.js) + - `screenToEarthCoords(clientX, clientY, camera, earth, domElement)`：屏幕坐标 raycast 到地球表面。 + - `vector3ToLatLon(vector)`：地球本地坐标转经纬度。 +- [frontend/public/earth/js/constants.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/constants.js) + - `COUNTRY_BOUNDARY_CONFIG` 已定义普通国界线和 hover 国界线样式。 +- 移动端布局状态已经通过 `layout-mode-mobile` body class 区分。 + +因此本需求的核心不是新增图层，而是补一个移动端中心取点控制器。 + +## Non-goals + +- 不改变桌面端 hover 交互。 +- 不替换 `countries-admin0.min.geojson` 数据源。 +- 不新增后端 API。 +- 不把国家面填充做成新的 selected country 面状 shader。 +- 不为移动端增加永久准星 UI，除非后续产品明确需要视觉准星。 + +## Implementation plan + +### 1. Add a small mobile center hover controller + +新增一个轻量函数，建议放在现有主循环附近或单独模块，例如： + +```text +frontend/public/earth/js/mobile-center-country-highlight.js +``` + +建议导出： + +```js +updateMobileCenterCountryHighlight({ + camera, + earth, + renderer, + now, + isBlocked, +}); + +clearMobileCenterCountryHighlight(); +``` + +职责： + +1. 判断是否处于移动端。 +2. 判断国界图层是否开启。 +3. 判断当前是否被移动端前景 UI 阻塞。 +4. 对 renderer canvas 中心点做 raycast。 +5. 命中地球后转经纬度。 +6. 调用 `updateCountryBoundaryHover({ lat, lon })`。 +7. 无命中或禁用时调用 `clearCountryBoundaryHover()`。 + +### 2. Use canvas center, not window center + +中心点应基于 renderer canvas rect 计算： + +```js +const rect = renderer.domElement.getBoundingClientRect(); +const clientX = rect.left + rect.width / 2; +const clientY = rect.top + rect.height / 2; +``` + +这样在移动端安全区、地址栏变化、viewport resize 或 canvas 非全屏时仍然准确。 + +### 3. Convert center point into country hover coords + +复用已有工具： + +```js +const point = screenToEarthCoords(clientX, clientY, camera, earth, renderer.domElement); +if (!point) { + clearCountryBoundaryHover(); + return; +} + +const coords = vector3ToLatLon(point); +updateCountryBoundaryHover(coords); +``` + +注意：`screenToEarthCoords` 返回的是 earth local point，符合 `vector3ToLatLon` 的输入语义。 + +### 4. Gate updates by mobile and foreground UI state + +建议新增一个本地判断函数： + +```js +function isMobileCenterCountryHighlightBlocked() { + return ( + !document.body.classList.contains("layout-mode-mobile") || + document.body.classList.contains("earth-search-open") || + document.body.classList.contains("earth-settings-open") || + document.body.classList.contains("earth-media-open") || + document.body.classList.contains("earth-info-open") + ); +} +``` + +如果移动端抽屉只是半收起、且没有覆盖中心视野，可以继续允许中心高亮。若实际体验里抽屉展开会遮挡中心点，再把 drawer open 状态纳入阻塞条件。 + +### 5. Throttle and cache center updates + +GeoJSON polygon 命中不应该无条件每帧执行。 + +第一版建议： + +- `throttleMs = 120` +- 缓存上次经纬度，中心点变化小于 `0.05` 度时跳过。 +- 禁用、切回桌面、图层关闭、UI 阻塞时立即清除一次高亮。 + +伪代码： + +```js +if (now - lastUpdateAt < 120) return; +if (Math.abs(coords.lat - lastLat) < 0.05 && Math.abs(coords.lon - lastLon) < 0.05) return; +``` + +### 6. Wire into the Earth animation loop + +在 [frontend/public/earth/js/main.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/main.js) 的动画循环中调用： + +```js +updateMobileCenterCountryHighlight({ + camera, + earth, + renderer, + now: performance.now(), + isBlocked: isMobileCenterCountryHighlightBlocked(), +}); +``` + +这样自动旋转、手势旋转、缩放和巡航都会自然更新。 + +### 7. Keep desktop hover unchanged + +桌面 pointer hover 仍然走当前逻辑。 + +移动端中心高亮只在 `layout-mode-mobile` 下生效，不应该监听 pointer move，也不应该抢占 desktop hover 状态。 + +### 8. Optional visual tuning + +第一版复用： + +- `COUNTRY_BOUNDARY_CONFIG.hoverLineColor` +- `COUNTRY_BOUNDARY_CONFIG.hoverLineOpacity` +- `COUNTRY_BOUNDARY_CONFIG.hoverGlowOpacity` + +如果移动端体验太强，可以后续加独立配置： + +```js +mobileCenterHoverLineOpacity +mobileCenterHoverGlowOpacity +``` + +但第一版不建议过早分叉样式。 + +## Verification + +### Static checks + +1. `npm` 前端构建或现有 lint/typecheck 命令通过。 +2. `rg` 确认新增函数只在移动端路径调用，不影响桌面 pointer hover。 +3. `git diff --stat` 和目标文件 diff 确认改动范围集中。 + +### Manual mobile checks + +使用移动端 viewport，例如 390x844： + +1. 打开 Earth。 +2. 开启国界图层。 +3. 转动地球到中国、美国、澳大利亚等大块陆地区域，确认中心国家国界高亮。 +4. 转动到太平洋或印度洋，确认高亮消失。 +5. 缩放地球，确认高亮仍跟随中心点。 +6. 打开移动端搜索、设置、媒体或详情面板，确认没有明显误高亮或抖动。 +7. 切回桌面 viewport，确认 hover 仍由鼠标位置控制。 + +### Playwright smoke check + +如果已有 Playwright 流程，建议补一个移动端 smoke： + +1. 设置 viewport 为手机尺寸。 +2. 打开 Earth 页面。 +3. 开启国界图层。 +4. 等待国界数据加载。 +5. 截图确认中心附近国家边界有 hover 高亮线。 + +这个 smoke 不必断言具体国家名称，因为当前功能核心是视觉高亮；更稳定的自动化可以后续通过暴露 debug state 实现。 + +## Risks and mitigations + +### Polygon hit cost too高 + +风险：移动端设备上频繁 `featureContains` 可能带来卡顿。 + +缓解： + +- 使用 `120ms` 节流。 +- 经纬度变化小于阈值时跳过。 +- 后续如仍慢，再为 GeoJSON features 预计算 bbox，先 bbox 粗筛再 point-in-polygon。 + +### UI blocking state 不完整 + +风险：某些移动端前景 UI 没有对应 body class，中心点被遮挡但高亮仍更新。 + +缓解： + +- 第一版覆盖现有主要 class。 +- 验证时记录遗漏项，补充到 `isMobileCenterCountryHighlightBlocked()`。 + +### Desktop hover 被移动端状态污染 + +风险：移动端中心高亮和桌面 hover 共用 `_hoveredFeature` 状态。 + +缓解： + +- 只在 `layout-mode-mobile` 下运行中心高亮。 +- 切出 mobile 或图层关闭时调用一次 `clearCountryBoundaryHover()`。 +- 不改 `updateCountryBoundaryHover()` 的语义。 + +## Milestones + +1. 设计落地：完成本 plan，明确目标和验收标准。 +2. 最小实现：新增移动端中心取点 controller，并接入 animation loop。 +3. 性能保护：加入节流、经纬度阈值和禁用态清理。 +4. 验证：本地构建通过，移动端 viewport 手动检查通过。 +5. 调优：根据截图或真机体验微调阻塞条件和节流阈值。 + diff --git a/docs/plans/earth-mobile-drawer-ui-plan.md b/docs/plans/earth-mobile-drawer-ui-plan.md new file mode 100644 index 00000000..8bfee7c0 --- /dev/null +++ b/docs/plans/earth-mobile-drawer-ui-plan.md @@ -0,0 +1,400 @@ +# Earth Mobile Drawer UI Plan + +## 背景 + +当前 Earth 移动端已经补上了基础触控能力，例如： + +- 单指拖拽旋转地球 +- 双指缩放 +- 点击阈值和基础事件隔离 + +但移动端 UI 仍然存在一个根本问题： + +它还在沿用桌面 HUD 的内容切分方式，只是把原来的 panel、modal、toolbar 改位置、改层级、改容器。这样虽然能快速复用旧代码，但手机端体验仍然是生硬的，因为： + +- 信息密度和结构是按桌面设计的 +- 面板标题、关闭、折叠、开关项是桌面心智，不是手机心智 +- 很多内容只是“被塞进抽屉”，而不是为抽屉重新设计 +- 设置里仍然带有“显示/隐藏某些 panel”的思路，但移动端本来就不应该存在那些独立 panel + +因此本计划进一步收紧： + +移动端不只是“底部抽屉化”，而是**重新设计一套 fit 抽屉体系的 mobile-first UI**。 + +## 新目标 + +1. 手机端不再使用现有 `toolbar` 作为主入口。 +2. 手机端不再使用现有独立 `panel / modal / sheet` 作为直接 UI 单元。 +3. 手机端统一采用“底部抽屉 + 顶部标题 + tab 切换 + 卡片内容”的单前景模式。 +4. 抽屉内部每个 tab 页面都按移动端重新设计内容结构，而不是直接复用旧 panel 结构。 +5. 设置页移除“显示/隐藏 panel”的桌面遗留配置。 +6. 媒体页拆成两个移动端页面：`新闻` 与 `TV`，都归入抽屉体系。 +7. 桌面端保持现有 HUD 体系，不回退。 + +## 核心原则 + +### 1. 只复用数据和状态，不复用桌面 UI 结构 + +可复用： + +- 图层注册表 +- 搜索结果数据 +- BGP / 海缆 / 卫星详情数据 +- 媒体数据 +- 旋转、缩放、选择、高亮等运行时状态 + +不直接复用： + +- 桌面 panel DOM 结构 +- 桌面 panel header / close / collapse 交互 +- 桌面 settings 项里的“显示某 panel”逻辑 +- 桌面媒体面板布局 + +### 2. 抽屉是唯一主前景层 + +移动端同一时刻只有一个主前景层：底部抽屉。 + +抽屉内部切换内容页，而不是多个悬浮层互相覆盖。 + +### 3. 每个 tab 都是移动端页面，而不是 panel 容器 + +抽屉中的每一项都应视为一个移动端子页面： + +- 有自己的标题 +- 有自己的内容层次 +- 有自己的滚动区域 +- 有自己的主操作 + +而不是简单挂一个旧面板进去。 + +### 4. 移动端状态提示不占据屏幕正中 + +桌面端当前很多通知、状态提示、胶囊消息更适合在屏幕上方居中出现，但移动端不应继续沿用这套布局。 + +移动端统一改为： + +- 通知栏放在右上角安全区 +- 胶囊提示放在右上角堆叠 +- 不遮挡地球中心视野 +- 不与底部抽屉主交互区冲突 + +## 交互模型 + +### 默认态 + +移动端默认只显示： + +- 地球主画布 +- 底部半露出的抽屉头部 + +不再单独显示上箭头按钮。 + +### 展开态 + +用户从底边直接上拉抽屉，或点击抽屉头部展开。 + +展开后显示： + +- 当前页面标题 +- tab 导航 +- 当前页面内容 + +### 收起态 + +用户下拉抽屉头部收起，或点击背景收起。 + +## 信息架构 + +移动端抽屉内的一级页面重定为： + +1. 图层 +2. 搜索 +3. 态势 +4. 新闻 +5. TV +6. 设置 +7. 详情（按需出现，不固定常驻 tab） + +其中 `新闻` 和 `TV` 不再共享同一个移动端媒体面板。 + +## 页面重设计要求 + +### 图层页 + +目标： + +- 成为移动端最核心的控制页 +- 强调快速开关，不强调桌面 panel 感 + +内容建议： + +- 顶部摘要：当前已启用图层数量 +- 图层列表卡片 +- 每个图层项只保留： + - 图标 + - 中文名 + - 英文副标题 + - 开关 +- 去掉桌面式 header / collapse / close 结构 + +### 搜索页 + +目标： + +- 成为抽屉中的完整搜索页 +- 避免看起来像桌面 modal 被塞进抽屉 + +内容建议： + +- 顶部搜索输入框 +- 搜索提示文案 +- 结果列表 +- 结果项更适合手指点击 +- 结果点击后： + - 聚焦地球对象 + - 自动切换到详情页 + +### 态势页 + +目标： + +- 合并原来的 `stats + legend` 思路 +- 成为移动端全局态势页 + +内容建议： + +- 顶部核心统计卡 + - 海缆数量 + - 登陆点数量 + - 卫星数量 + - BGP 事件数量 +- 当前关注层图例 +- BGP 状态摘要 +- 不再出现独立 legend 面板和独立 stats 面板 + +### 新闻页 + +目标： + +- 从原媒体面板中拆出单独的移动端新闻页 + +内容建议： + +- 当前区域焦点 +- 新闻源数量 +- 新闻卡片列表 +- 卡片内显示标题、来源、时间、区域 +- 外链操作更清晰 + +### TV 页 + +目标： + +- 从原媒体面板中拆出单独的移动端 TV 页 + +内容建议： + +- 顶部频道选择 +- 直播状态 +- 当前频道说明 +- 视频播放器区域 +- 刷新和外链按钮 + +不再保留桌面式“新闻/TV tab 共处一个 panel”的结构。 + +### 设置页 + +目标： + +- 只保留对移动端仍有意义的系统配置 + +必须移除： + +- 图层控制 panel 显示/隐藏 +- 图例 panel 显示/隐藏 +- 全球态势 panel 显示/隐藏 +- 媒体 panel 显示/隐藏 + +保留项建议： + +- 旋转模式 +- 日夜模式 +- 地球默认大小 +- 地形透明度 +- 系统入口 + +原因： + +移动端已经没有这些独立 panel 了，所以继续保留这些开关会制造错误心智。 + +### 详情页 + +目标： + +- 成为海缆 / BGP / 卫星对象的统一移动端详情页 + +内容建议： + +- 标题区 +- 类型标签 +- 关键属性列表 +- 相关对象摘要 +- 相关图层或态势提示 + +行为建议： + +- 点击对象后自动切入详情页 +- 搜索结果点击后也切入详情页 + +## 阶段重定义 + +### 阶段 2：抽屉壳层 + +目标： + +1. 实现底部抽屉基本壳层。 +2. 支持上拉展开、下拉收起、背景点击关闭。 +3. `mobile` 模式下隐藏旧 toolbar。 +4. `mobile` 模式下不再直接显示旧 panel。 + +完成标准： + +1. 手机端只有地球主视图和抽屉。 +2. 抽屉开合稳定。 + +### 阶段 3：基础页面重做 + +目标： + +1. 重新设计并实现图层页。 +2. 重新设计并实现搜索页。 +3. 重新设计并实现设置页。 + +完成标准： + +1. 这三个页面不再是旧 panel 原样移植。 +2. 设置页已移除 panel 可见性开关。 + +### 阶段 4：态势与详情重做 + +目标： + +1. 将 stats 和 legend 合并为新的态势页。 +2. 实现统一详情页。 +3. 对象点击与搜索结果点击都可切入详情页。 + +完成标准： + +1. 不再存在移动端独立 legend / stats 面板。 +2. 详情页成为统一对象信息入口。 + +### 阶段 5：媒体拆分重做 + +目标： + +1. 将原媒体面板拆成两个移动端页面：新闻页、TV 页。 +2. 分别重做这两个页面的布局。 +3. 保留各自必要操作，但不继续共享桌面 panel 结构。 + +完成标准： + +1. 新闻与 TV 各自成为独立移动端页面。 +2. 不再使用桌面媒体 panel 的 tab 结构作为移动端主体。 + +### 阶段 6：手感与真机修正 + +目标： + +1. 调整抽屉高度、节奏、手势阈值。 +2. 调整 tab 密度与文字层级。 +3. 优化 iPhone / Android 安全区。 +4. 优化抽屉滚动与地球拖拽边界。 + +完成标准： + +1. 抽屉和地球不会抢手势。 +2. 手机端各页面信息层次清晰。 +3. 真机下无遮挡、无死层、无错误交互心智。 + +## 技术落点调整 + +### [frontend/public/earth/index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) + +职责： + +- 只保留移动端抽屉壳层 +- 为各页面提供新的页面容器 + +不再把旧 panel 作为最终结构直接塞进抽屉。 + +### [frontend/public/earth/js/controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) + +职责： + +- 管理抽屉开合 +- 管理 tab 切换 +- 管理详情页切入 +- 管理 mobile / desktop 分流 + +### [frontend/public/earth/js/search.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/search.js) + +职责： + +- 保留搜索能力和结果逻辑 +- 输出给新的移动端搜索页 + +### [frontend/public/earth/js/info-card.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/info-card.js) + +职责： + +- 从桌面 info-card 逻辑中提取可复用的数据层 +- 服务新的移动端详情页 + +### [frontend/public/earth/js/tv.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/tv.js) + +职责： + +- 为新的 TV 页面提供数据和状态 +- 不再直接主导移动端媒体 panel 壳层 + +### [frontend/public/earth/js/news.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/news.js) + +职责： + +- 为新的新闻页面提供列表和区域焦点数据 + +### CSS + +需要新增真正的移动端页面样式，而不是继续在旧 panel class 上堆条件分支： + +- 图层页样式 +- 搜索页样式 +- 态势页样式 +- 新闻页样式 +- TV 页样式 +- 设置页样式 +- 详情页样式 +- 移动端右上角通知 / 胶囊提示样式 + +## 验收标准 + +1. `mobile` 模式下不再显示旧 toolbar。 +2. `mobile` 模式下不再把旧 panel 直接作为最终 UI。 +3. 图层、搜索、态势、新闻、TV、设置都是重新设计的移动端页面。 +4. 设置页不再包含移动端无意义的 panel 显示/隐藏项。 +5. 新闻与 TV 已拆分为两个移动端页面。 +6. legend / stats 已整合为态势页。 +7. 详情页成为统一对象详情入口。 +8. 移动端通知栏和胶囊提示已统一放到右上角安全区，而不是屏幕正中。 + +## 结论 + +本计划进一步明确： + +移动端目标不是“把桌面 HUD 放进抽屉”，而是“以抽屉为载体，重做一套适合手机端的信息页面”。 + +后续开发必须以此为准： + +- 复用数据 +- 重做界面 +- 清除桌面遗留心智 diff --git a/docs/plans/earth-news-cruise-summary-plan.md b/docs/plans/earth-news-cruise-summary-plan.md new file mode 100644 index 00000000..8b9869ca --- /dev/null +++ b/docs/plans/earth-news-cruise-summary-plan.md @@ -0,0 +1,366 @@ +# Earth 新闻巡航摘要增强计划 + +## 背景 + +`Earth` 的新闻巡航模式目前直接消费 `/api/v1/news/earth-feed` 返回的 `items[].summary`。这个字段主要来自 RSS/Atom 的 `description`、`summary` 或 `content`，再经过 HTML 清理与长度截断。 + +这个实现足够轻量，但在巡航展示里有三个问题： + +- 不是所有新闻源都会提供摘要，部分源只返回标题和链接。 +- 聚合源的摘要质量不稳定，可能只是重复标题、来源署名或片段文本。 +- 巡航模式需要更稳定的“态势说明”，否则新闻卡片的 `SUMMARY` 区域会显得空或信息密度不足。 + +目标不是把所有新闻都交给大模型，而是建立一个分层摘要管线：能用新闻源自带内容时零成本处理，需要增强时优先用本地模型，云端 LLM 只作为可控兜底。 + +## 当前相关实现 + +| 文件 | 作用 | +| --- | --- | +| `backend/app/services/earth_news.py` | 拉取 RSS/Atom 新闻源、解析标题/摘要、按区域聚合并返回 Earth 新闻 payload | +| `backend/app/api/v1/news.py` | 暴露 `/api/v1/news/earth-feed` | +| `frontend/public/earth/js/news.js` | 拉取新闻 payload 并渲染媒体面板新闻列表 | +| `frontend/public/earth/js/news-cruise-adapter.js` | 将新闻条目映射为巡航事件，并把 `summary` 传给信息卡 | +| `frontend/public/earth/js/info-card.js` | 展示新闻巡航卡片中的 `SUMMARY` | + +当前摘要生成逻辑集中在 `earth_news.py`： + +```python +summary = _extract_item_text(node, "description", "content") +clean_summary = _truncate(_strip_html(summary), 180) +``` + +这意味着后端还没有区分“摘要来自哪里”“质量是否足够”“是否需要异步增强”。 + +## 总体方案 + +采用四级摘要来源： + +| 优先级 | 来源 | 成本 | 适用情况 | 风险 | +| --- | --- | --- | --- | --- | +| 1 | 新闻源自带 summary / description | 低 | RSS/Atom 已提供可读摘要 | 字段可能为空或重复标题 | +| 2 | 本地规则提取 | 低 | 有正文片段但没有可靠摘要 | 只能抽取，不能真正概括 | +| 3 | 本地 Gemma/Ollama 摘要 | 中 | 巡航会展示且前两级质量不足 | 本地模型质量与机器性能相关 | +| 4 | 云端 LLM 兜底 | 高 | 用户手动增强、重点新闻、失败补偿 | 成本与网络依赖 | + +推荐默认策略： + +```text +provider summary -> extractive summary -> cached local model summary -> async local model summary -> optional cloud LLM +``` + +巡航 UI 永远先展示已有摘要，不等待模型调用。模型摘要在后台补齐，写入缓存后下一轮巡航或刷新时使用。 + +## 数据结构 + +后端应把原来的 `summary: str` 升级为可追踪的摘要元信息，同时为了兼容前端保留顶层 `summary` 字段。 + +建议新增结构： + +```json +{ + "summary": "短摘要文本", + "summary_meta": { + "source": "provider", + "quality": "good", + "generated_at": "2026-04-29T00:00:00Z", + "content_hash": "sha256:...", + "model": null, + "language": "zh-CN" + } +} +``` + +字段说明： + +| 字段 | 可选值 | 说明 | +| --- | --- | --- | +| `source` | `provider` / `extractive` / `local_llm` / `cloud_llm` / `fallback` | 摘要来源 | +| `quality` | `good` / `partial` / `poor` | 后端对摘要可用性的判断 | +| `generated_at` | ISO 时间 | 模型或规则生成时间 | +| `content_hash` | SHA-256 | 用于缓存命中和判断内容变化 | +| `model` | 字符串或 `null` | 例如 `gemma3:4b` | +| `language` | 语言代码 | 默认 `zh-CN`，也可跟随新闻语言 | + +前端第一阶段不需要显示 `summary_meta`，但可以用于后续调试面板或质量标记。 + +## 后端设计 + +### NewsSummaryService + +新增 `backend/app/services/news_summary.py`，提供统一入口： + +```python +async def resolve_news_summary(item: ParsedNewsItem, *, mode: str) -> NewsSummaryResult: + ... +``` + +核心职责： + +- 标准化新闻输入：标题、URL、来源、发布时间、摘要片段、正文片段。 +- 判断 provider summary 是否可用。 +- 生成本地规则摘要。 +- 查询模型摘要缓存。 +- 在允许时调用本地 Ollama/Gemma。 +- 在增强模式或手动触发时调用云端 LLM。 +- 返回摘要文本与 `summary_meta`。 + +### 摘要质量判断 + +第一版可以用轻量规则： + +- 少于 30 个字符：`poor` +- 与标题高度重复：`partial` +- 包含明显来源署名或聚合噪声：`partial` +- 60-180 个字符且不重复标题：`good` + +伪代码： + +```python +def score_summary(title: str, summary: str) -> SummaryQuality: + if len(summary.strip()) < 30: + return "poor" + if normalized_overlap(title, summary) > 0.75: + return "partial" + if looks_like_source_attribution(summary): + return "partial" + return "good" +``` + +### 本地规则摘要 + +如果新闻源没有摘要，但有 `content`、`description`、`snippet` 或正文片段： + +- 清理 HTML。 +- 去掉标题重复内容。 +- 去掉来源署名、发布时间、图片说明。 +- 优先取前 1-2 个完整句子。 +- 控制在 80-140 个中文字符或 40-80 个英文词。 + +### 本地 Gemma/Ollama Provider + +不要在业务里写死 Gemma，抽象为 `LocalLLMSummaryProvider`，默认可以指向 Ollama： + +```env +NEWS_SUMMARY_PROVIDER=ollama +OLLAMA_BASE_URL=http://localhost:11434 +NEWS_SUMMARY_MODEL=gemma3:4b +NEWS_SUMMARY_TIMEOUT_SECONDS=20 +NEWS_SUMMARY_MAX_INPUT_CHARS=5000 +NEWS_SUMMARY_MAX_PER_HOUR=60 +``` + +Ollama 请求示例： + +```http +POST /api/generate +Content-Type: application/json + +{ + "model": "gemma3:4b", + "prompt": "...", + "stream": false, + "options": { + "temperature": 0.2, + "num_predict": 180 + } +} +``` + +摘要 prompt 要强调“只基于原文”，避免模型补事实： + +```text +你是新闻摘要器。只根据输入新闻内容生成摘要，不要添加原文没有的信息。 +输出中文，1-2 句话，80-140 字。 +如果原文信息不足，只概括已知事实，不要推测。 + +标题：{title} +来源：{source} +发布时间：{published_at} +正文或片段： +{content} +``` + +### 缓存 + +需要缓存模型摘要，避免重复花时间和费用。 + +缓存 key： + +```text +sha256(url + title + published_at + normalized_content) +``` + +建议新增表或复用系统设置缓存。若要可查询与清理，推荐独立表： + +```text +news_summary_cache +- id +- cache_key +- url +- title +- content_hash +- summary +- source +- quality +- provider +- model +- generated_at +- expires_at +- failure_count +- last_error +``` + +缓存策略： + +- 同一 `cache_key` 命中后直接返回。 +- `provider` / `extractive` 可以短期缓存。 +- `local_llm` / `cloud_llm` 可以长缓存，内容 hash 变化才重算。 +- LLM 失败后记录 `failure_count`，短时间内不重复调用。 + +## 前端与巡航行为 + +前端第一阶段只需要继续使用 `item.summary`，不阻塞现有逻辑。 + +后续可选增强： + +- `news.js` 在渲染新闻列表时，如果 `summary_meta.quality === "poor"`，可以用更紧凑的标题卡样式。 +- `news-cruise-adapter.js` 选择巡航项时，可以优先选择 `summary_meta.quality !== "poor"` 的新闻。 +- `info-card.js` 不显示“AI 生成中”这类文案，避免把系统内部状态暴露给用户。 + +如果后端异步生成完成，可以通过下一次 `/api/v1/news/earth-feed` 刷新自然更新。第一版不需要 WebSocket。 + +## 调用策略 + +默认使用“省钱模式”： + +- 只处理本次 payload 中即将进入巡航队列的前 N 条。 +- `provider` 或 `extractive` 达到 `good` 时不调用模型。 +- 本地模型失败时不影响新闻 payload。 +- 云端 LLM 默认关闭，只允许手动增强或后台配置开启。 + +推荐限制： + +| 配置 | 默认值 | 说明 | +| --- | --- | --- | +| `NEWS_SUMMARY_MODE` | `economy` | `off` / `economy` / `enhanced` / `manual` | +| `NEWS_SUMMARY_CRUISE_PREFETCH_LIMIT` | `10` | 每次新闻 payload 预热多少条巡航摘要 | +| `NEWS_SUMMARY_MAX_PER_HOUR` | `60` | 本地模型每小时最多处理数量 | +| `NEWS_SUMMARY_CLOUD_MAX_PER_DAY` | `20` | 云端 LLM 每天最多处理数量 | +| `NEWS_SUMMARY_TIMEOUT_SECONDS` | `20` | 单条模型摘要超时 | +| `NEWS_SUMMARY_MAX_INPUT_CHARS` | `5000` | 输入截断上限 | + +## Gemma 本地部署建议 + +Gemma 适合作为“本地省钱层”，但不应成为强绑定依赖。建议通过 Ollama 接入，未来可切换 Qwen、Llama 或其他本地模型。 + +开发环境： + +```bash +ollama pull gemma3:4b +ollama serve +``` + +集成原则： + +- 后端只依赖 Ollama HTTP API，不直接依赖 Gemma SDK。 +- 模型名称来自配置，不写死在代码里。 +- 健康检查访问 `/api/tags` 或执行一条极短测试 prompt。 +- 如果 Ollama 不可用，摘要管线自动退回 `provider` / `extractive`。 + +中文新闻较多时，需要单独评估 Gemma 与 Qwen 系本地模型的中文摘要质量。不要只看单条效果，至少抽样 50 条新闻比较： + +- 事实准确性 +- 中文自然度 +- 长度稳定性 +- 延迟 +- 是否会补充原文没有的信息 + +## 云端 LLM 兜底 + +云端 LLM 不作为默认路径，只用于： + +- 用户点击“增强摘要”。 +- 管理员开启增强模式。 +- 本地模型连续失败且新闻进入重点巡航队列。 + +云端结果同样写入 `news_summary_cache`，并受每日限额控制。 + +## 分阶段实施 + +### 第一阶段：零成本摘要质量增强 + +- 在 `earth_news.py` 中引入 `summary_meta`。 +- 增加 provider summary 质量判断。 +- 增加本地规则摘要兜底。 +- `/api/v1/news/earth-feed` 保持兼容，继续返回顶层 `summary`。 +- 前端无需大改。 + +验收标准： + +- 没有摘要的新闻也能尽量得到短摘要。 +- `summary_meta.source` 和 `summary_meta.quality` 可用于调试。 +- 现有新闻面板和巡航模式不破坏。 + +### 第二阶段：本地 Gemma/Ollama 摘要 + +- 新增 `LocalLLMSummaryProvider`。 +- 接入 Ollama `/api/generate`。 +- 添加超时、输入截断、错误退避。 +- 增加模型摘要缓存。 +- 巡航 payload 后台预热前 N 条摘要。 + +验收标准： + +- Ollama 可用时，低质量摘要能被本地模型增强。 +- Ollama 不可用时，新闻接口仍然正常返回。 +- 同一新闻不会重复调用模型。 + +### 第三阶段：设置与可观测性 + +- 在设置中增加新闻摘要模式： + - `关闭` + - `省钱模式` + - `增强模式` + - `仅手动` +- 增加本地模型连通性检查。 +- 暴露缓存命中率、模型调用次数、失败次数。 +- 日志记录摘要来源和失败原因。 + +验收标准： + +- 用户可以不改环境变量就知道本地摘要服务是否可用。 +- 管理员能看出成本和失败情况。 + +### 第四阶段：云端 LLM 兜底 + +- 接入现有 AI Provider 或新增 cloud summary provider。 +- 增加每日限额与手动增强入口。 +- 对云端生成结果落缓存。 + +验收标准： + +- 云端调用可控、可关闭、可限流。 +- 云端失败不影响巡航。 + +## 风险与防护 + +| 风险 | 防护 | +| --- | --- | +| 本地模型生成不存在的事实 | prompt 明确禁止扩写；摘要只作为原文概括；保留来源链接 | +| 本地模型慢导致新闻接口卡住 | 模型摘要异步化；接口先返回已有摘要 | +| 成本失控 | 默认不启用云端；按小时/天限流；缓存命中优先 | +| 摘要语言不一致 | 配置目标语言，默认 `zh-CN` | +| 新闻源正文不足 | 只概括标题和片段，不强行扩写 | +| 模型服务不可用 | 自动回退，不影响巡航主流程 | + +## 推荐优先级 + +先做第一阶段和第二阶段的最小闭环： + +1. `summary_meta` + 质量判断。 +2. 本地规则摘要。 +3. Ollama provider。 +4. 缓存。 +5. 巡航前 N 条异步预热。 + +云端 LLM 和设置页可以后置。这样能先验证“摘要缺失比例、本地模型质量、实际延迟”三个关键问题，再决定是否投入更重的 UI 与云端增强。 diff --git a/docs/plans/earth-predicted-orbit-plan.md b/docs/plans/earth-predicted-orbit-plan.md index df963f24..6414b6c9 100644 --- a/docs/plans/earth-predicted-orbit-plan.md +++ b/docs/plans/earth-predicted-orbit-plan.md @@ -96,3 +96,11 @@ GEO 轨道点数高，采样率需要按轨道类型分层。 2. 解锁后轨道立即清除 3. 不同轨道类型下点数可控 4. 页面切换回来不会闪出旧轨道残留 + +## Satellite Footprint Follow-Up Items + +从技术文档迁出的 footprint 后续项，作为卫星覆盖能力的计划 backlog： + +1. 为 `iridium-next` 新建独立 footprint adapter。 +2. 在 UI 上补一个只读提示，让用户知道当前卫星是否支持 footprint。 +3. 如果未来拿到 GEO beam contour / operator metadata，再为 GEO 开 operator-specific footprint。 diff --git a/docs/plans/earth-renderer-architecture-separation-plan.md b/docs/plans/earth-renderer-architecture-separation-plan.md index 649c20ac..ec416478 100644 --- a/docs/plans/earth-renderer-architecture-separation-plan.md +++ b/docs/plans/earth-renderer-architecture-separation-plan.md @@ -25,6 +25,49 @@ - 状态和渲染更新散落在多个模块 - 后续再加新图层时容易复制旧逻辑 +## Current High-Frequency Risks + +### 1. Visual state and business state drift apart + +Earth 里最常见的 bug 不是“没渲染”，而是状态没有一起收口： + +- 图层关了，tooltip 还在 +- 锁定对象隐藏了，info card 还在 +- legend 没跟图层切换 +- loading 已结束，但按钮还像没开 + +后续架构治理需要把这类同步责任从临时 UI patch 转为统一状态流。 + +### 2. HUD layout fixes skip structure analysis + +Earth HUD 历史上反复出现： + +- 面板只剩一条缝 +- markdown 被裁掉 +- tabs / iframe 被 `overflow: hidden` 吃掉 + +这类问题应纳入布局治理计划，而不是散落在单个功能改动里临时修。 + +### 3. Transitional paths keep accumulating + +Earth 已经经历过多轮 HUD、toolbar、media panel 重构，容易留下： + +- 旧 helper +- 旧 class +- 旧 fallback 逻辑 +- 已废弃变体 + +架构分离阶段需要把 cleanup pass 作为计划项，而不是让技术上下文承担提醒职责。 + +### 4. Cruise logic and business events couple too deeply + +巡航相关风险是通用巡航层继续混入业务事件细节，导致 BGP、新闻、卫星、海缆各自复制一套状态机。 + +架构目标应保持： + +- 通用巡航层管理目标、队列、focus、停留、隐藏和切换 +- 业务模块只提供队列、坐标、卡片内容和高亮副作用 + ## Target Architecture Earth 对每类对象都尽量拆成三层： diff --git a/docs/plans/earth-vessel-ais-aggregation-plan.md b/docs/plans/earth-vessel-ais-aggregation-plan.md new file mode 100644 index 00000000..c46f2ac5 --- /dev/null +++ b/docs/plans/earth-vessel-ais-aggregation-plan.md @@ -0,0 +1,478 @@ +# AIS 多源采集、冲突记录与聚合接口计划 + +**状态**：v0-v3 已实现，v3.1-v3.4 为 v4/v5 前置稳定化任务，v4 / v5 已落最小可用子集 +**创建日期**：2026-04-30 +**核心原则**：采集器只写原始观测；去重、合并、冲突解释放在聚合接口中完成 + +## 已确认决策 + +| 项目 | 决策 | +|-----|------| +| AISStream 接入方式 | 单独实现 WebSocket 采集器，不塞进现有 BarentsWatch HTTP collector | +| 采集器职责 | 连接上游、标准化字段、写入原始观测，不直接决定最终展示值 | +| 去重合并位置 | 放在聚合服务和聚合 API 中，而不是散落在每个 collector 的保存逻辑里 | +| 冲突处理 | 先记录冲突事实和当前选择原因，后续再开放用户规则配置 | +| 默认可信度 | 同类 AIS 数据源优先按 `delivery_mode` 评估：`realtime_stream` 优于 `batch_stream`，再优于 `polling` 和 `snapshot` | +| 过期保护 | 实时流源断流超过 freshness 窗口后，不能仅凭“实时源”身份压过更新的轮询数据 | +| 源健康状态 | 聚合时必须参考采集器健康状态，不能只看配置中的理论优先级 | +| 媒体富化 | 船只图片等媒体信息不进入 AIS 实时聚合主链路，后续单独做 enrichment | +| v4/v5 顺序 | 在聚合完整性、AISStream 实时链路、采集状态语义和基础身份信息显示修好之前，不进入策略配置和 enrichment UI | + +## 背景 + +当前 AIS 链路以 BarentsWatch 为主。它是 HTTP polling 模式，覆盖挪威附近海域，适合作为稳定的免费起点，但不适合承担全球实时船只数据的全部职责。后续接入 AISStream 后，会出现同一个 MMSI 被多个来源同时上报的情况： + +- 位置、航速、航向可能在多个来源之间存在秒级差异。 +- 船名、IMO、呼号、船型、尺寸等静态字段可能不完整，甚至互相冲突。 +- WebSocket 或其他实时流通常更接近实时，但也可能断流或批量延迟。 +- 如果每个 collector 自己做去重合并，规则会分散、不可审计，也很难让用户后续配置“某个字段信任哪个来源”。 + +因此第一阶段不应让采集器直接覆盖最终船只表。更稳的方式是先保留观测事实，再由聚合接口统一给出当前展示视图。 + +## 目标架构 + +```mermaid +flowchart LR + A[BarentsWatch HTTP collector] --> D[AIS raw observations] + B[AISStream WebSocket collector] --> D + C[Custom mapped vessel_ais sources] --> D + D --> E[AIS aggregation service] + E --> F[Conflict records] + E --> G[GeoJSON vessels API] + E --> H[Vessel detail API] + I[Aggregation strategy config] --> E +``` + +### 原始观测层 + +原始观测层保存每个来源看到的事实。建议模型包含： + +| 字段 | 用途 | +|-----|------| +| `target_schema` | 例如 `vessel_ais` | +| `source` | 例如 `barentswatch_vessels`、`aisstream_vessels` | +| `entity_key` | AIS 使用 MMSI | +| `delivery_mode` | `realtime_stream`、`batch_stream`、`polling`、`snapshot` | +| `transport` | `websocket`、`sse`、`http`、`file` 等 | +| `observed_at` | 上游数据时间，优先使用 AIS 消息时间 | +| `collected_at` | 本系统接收或采集时间 | +| `source_message_id` | 上游消息 ID 或可推导 ID，没有则为空 | +| `observation_hash` | 幂等去重指纹，用于防止同一来源重复写入同一条观测 | +| `normalized_payload` | 标准化后的 AIS JSON | +| `raw_payload` | 可选，保存原始或裁剪后的上游记录 | +| `quality_flags` | 观测级质量标记，例如 `stale`、`position_jump`、`future_timestamp` | + +`delivery_mode` 和 `transport` 不应混为一谈。WebSocket 是传输方式；streaming 是交付模式。聚合可信度主要看 `delivery_mode`，`transport` 只作为辅助信息。 + +原始观测层需要做存储级幂等去重，但这里的去重不是业务合并。推荐使用 `source + entity_key + message_type + observed_at + payload_hash` 或上游稳定消息 ID 作为唯一约束，避免 WebSocket 重连、HTTP 重试或批量回放导致同一事实重复入库。 + +### 源健康状态 + +每个采集器应维护独立的健康状态，供聚合服务读取： + +| 字段 | 用途 | +|-----|------| +| `source` | 采集器标识 | +| `connection_state` | `connected`、`reconnecting`、`disconnected`、`disabled` 等 | +| `last_seen_at` | 最近收到上游消息或响应的时间 | +| `last_success_at` | 最近成功写入观测的时间 | +| `last_error` | 最近错误摘要 | +| `message_rate` | 最近窗口内的消息速率 | +| `lag_seconds` | 上游观测时间与本系统接收时间的延迟 | + +聚合优先级不能只看 `source_priority`。例如 `aisstream_vessels` 默认优先于 `barentswatch_vessels`，但如果它处于 `disconnected` 或 `lag_seconds` 超过 freshness 窗口，则动态字段应回退到更新的可用来源。 + +### 身份键边界 + +v1 可以继续用 MMSI 作为 `entity_key`，因为它是 AIS 动态消息里最稳定、最容易获得的主键。但文档和模型都要为后续扩展留出口：MMSI 可能复用、填错或缺少静态信息，后续身份解析应结合 `mmsi + imo + callsign + name + dimensions` 判断是否需要拆分或合并实体。 + +### 冲突记录层 + +聚合服务发现同一个实体、同一个字段存在多个非空不同值时，写入冲突记录。冲突记录不代表错误，只代表“有多个可用候选值”。 + +```json +{ + "target_schema": "vessel_ais", + "entity_key": "257123000", + "field": "name", + "candidates": { + "barentswatch_vessels": "OSLO TRADER", + "aisstream_vessels": "OSLO TRADER II" + }, + "selected_source": "aisstream_vessels", + "selected_value": "OSLO TRADER II", + "selected_reason": "delivery_mode_priority", + "resolved_by": "system", + "status": "open" +} +``` + +第一阶段只需要记录冲突和当前选择原因，不需要做人工逐条确认。后续 UI 的目标也不是让用户处理每条冲突，而是把冲突沉淀成字段级规则。 + +## 聚合规则 + +### 字段分类 + +| 类型 | 字段 | 默认策略 | +|-----|------|----------| +| 动态位置 | `lat`、`lon`、`sog`、`cog`、`heading`、`nav_status` | 优先最新 `observed_at`，同时间再按来源优先级 | +| 静态身份 | `name`、`callsign`、`imo`、`flag` | 非空优先，再按字段策略或来源优先级 | +| 静态规格 | `vessel_type`、`vessel_type_name`、`length`、`width`、`draught` | 非空优先；冲突时记录候选值 | +| 轨迹点 | `track_points` | 按时间线合并；同一时间窗口内相近点去重；保留点级 `source` | +| 元信息 | `field_sources`、`conflict_count`、`selected_reasons`、`quality_flags` | 聚合接口生成，便于调试和后续 UI 展示 | + +### 默认优先级 + +默认优先级应使用两个维度： + +```yaml +delivery_mode_priority: + - realtime_stream + - batch_stream + - polling + - snapshot + +transport_priority: + - websocket + - sse + - http + - file +``` + +`delivery_mode_priority` 是主判断。比如 AISStream 如果提供实时推送，应标记为 `realtime_stream + websocket`；BarentsWatch 当前是 `polling + http`。 + +### 断流保护 + +实时流不能永久凭身份占优。聚合时需要 freshness 窗口： + +```yaml +freshness: + realtime_stream_seconds: 900 + polling_seconds: 3600 +``` + +如果 `aisstream_vessels` 最近 15 分钟没有该 MMSI 的新观测，而 BarentsWatch 轮询源有更新位置，则位置类字段应采用 BarentsWatch 的更新观测，并记录选择原因 `newest_observation` 或 `freshness_fallback`。 + +### 异常位置保护 + +多源 AIS 接入后，聚合服务必须过滤或降权明显异常的位置观测： + +- 经纬度必须在合法范围内。 +- `observed_at` 不能明显来自未来。 +- 同一 MMSI 短时间内跨越不合理距离时，标记 `position_jump`，默认不直接采用该点。 +- 当异常点来自当前优先源时，应记录 `selected_reason = anomaly_rejected`，再回退到其他可用来源。 + +异常保护不应静默丢弃事实。原始观测仍应保留，聚合结果通过 `quality_flags` 和冲突记录解释为什么没有采用它。 + +### 轨迹聚合 + +轨迹接口不能简单拼接所有来源，否则前端会出现折返、抖动和重复点。默认规则： + +- 以 `observed_at` 排序，生成统一时间线。 +- 同一来源的完全重复点通过 `observation_hash` 去重。 +- 多来源在短时间窗口内上报的相近位置视为同一轨迹点，优先选择 freshness 和 source priority 更高的一条。 +- 每个轨迹点保留 `source`、`selected_reason` 和必要的 `quality_flags`。 +- 对被判定为 `position_jump` 的点，默认不进入展示轨迹，但可通过调试参数查看。 + +## 聚合接口 + +现有展示接口应逐步改为消费聚合服务，而不是自己直接拼 `VesselPosition + VesselStatic`。 + +```text +GET /api/v1/visualization/geo/vessels +GET /api/v1/visualization/vessels/{mmsi} +GET /api/v1/visualization/vessels/{mmsi}/track +GET /api/v1/visualization/vessels/{mmsi}/conflicts +``` + +GeoJSON properties 建议增加： + +```json +{ + "mmsi": 257123000, + "name": "OSLO TRADER", + "lat": 59.91, + "lon": 10.73, + "received_at": "2026-04-30T10:00:00Z", + "field_sources": { + "name": "aisstream_vessels", + "lat": "aisstream_vessels", + "lon": "aisstream_vessels", + "vessel_type": "barentswatch_vessels" + }, + "selected_reasons": { + "name": "delivery_mode_priority", + "lat": "newest_observation", + "vessel_type": "non_empty_priority" + }, + "quality_flags": [], + "conflict_count": 2 +} +``` + +## 开放配置计划 + +### Phase 1 — 内置默认策略和只读解释 + +- 实现后端默认策略。 +- 聚合接口返回 `field_sources`、`selected_reasons`、`conflict_count`。 +- 冲突记录可查询，但不允许用户修改。 +- 保持现有前端船只图层接口形状基本兼容，新增字段只作为调试和后续 UI 输入。 + +### Phase 2 — 系统设置中的 JSON/YAML 策略配置 + +新增系统设置项，例如： + +```yaml +collector_aggregation: + vessel_ais: + source_priority: + - aisstream_vessels + - barentswatch_vessels + field_rules: + name: + mode: source_priority + vessel_type: + mode: source_priority + source_priority: + - barentswatch_vessels + - aisstream_vessels + lat: + mode: newest + lon: + mode: newest +``` + +配置校验要求： + +- 未知 source 只警告，不阻断保存，便于先配置后启用。 +- 未知 field 必须拒绝，避免拼写错误悄悄失效。 +- 动态位置字段默认不允许被固定来源永久锁死，除非显式开启高级选项。 +- 空值不覆盖非空值是全局保护，不建议开放关闭。 + +### Phase 3 — 冲突治理 UI + +基于冲突记录提供页面或 drawer： + +- 查看某个 MMSI 的冲突字段。 +- 查看每个字段的候选来源和值。 +- 查看当前选择原因。 +- 将一次人工选择保存成字段规则，而不是只处理单条冲突。 +- 支持恢复默认策略。 + +## AISStream 采集器计划 + +AISStream 采集器单独实现，建议命名为 `aisstream_vessels`。它的职责是： + +- 维护 WebSocket 连接、订阅范围和重连。 +- 将上游 AIS 消息标准化为 `vessel_ais` payload。 +- 标记 `delivery_mode = realtime_stream`，`transport = websocket`。 +- 写入原始观测层。 +- 不直接 upsert 最终展示数据。 + +配置应放入采集器设置，而不是硬编码： + +```yaml +aisstream_vessels: + api_key: "${AISSTREAM_API_KEY}" + bounding_boxes: + - [[-180, -90], [180, 90]] + message_types: + - PositionReport + - ShipStaticData +``` + +默认不建议直接订阅全球范围。AISStream 采集器应支持以下订阅策略： + +- 使用配置的固定 `bounding_boxes`。 +- 后续支持按 Earth 当前视口或关注区域动态调整订阅范围。 +- 支持限制 `message_types`，避免静态信息、位置报告和扩展消息全量涌入。 +- 断线后使用指数退避重连，并把连接状态写入源健康状态。 +- 重连后可能收到重复或回放消息，因此必须依赖原始观测层的幂等去重。 + +### 媒体富化边界 + +VesselFinder 等服务里的船只图片不属于 AIS 实时数据本身。图片、船籍详情、公司信息等后续应作为独立 enrichment 链路： + +- 通过 MMSI、IMO、船名等字段异步查询。 +- 使用独立缓存和授权配置。 +- 不阻塞 `vessel_ais` 实时观测入库。 +- 聚合接口只暴露已经缓存好的媒体引用，不在请求链路中现场抓取。 + +## 版本拆分 + +计划先按 v0-v3 建立基础能力，再用 v3.1-v3.4 修复当前稳定性缺口，最后进入 v4/v5： + +### v0 — 聚合基础设施（已实现） + +目标是不改变前端展示行为，先把数据底座铺好。 + +1. 新增原始观测模型、冲突记录模型和源健康状态模型。 +2. 为现有 BarentsWatch collector 写入原始观测，同时保留现有 `vessel_position` / `vessel_static` 兼容写入。 +3. 实现存储级 `observation_hash` 幂等去重。 +4. 补基础管理命令或调试接口，用于查看某个 MMSI 的原始观测和冲突候选。 + +### v1 — 聚合读接口（已实现） + +目标是让展示接口开始消费聚合结果，但前端形状保持兼容。 + +1. 实现 AIS 聚合服务，先兼容读取现有表，再逐步切换到原始观测层。 +2. 将 `/geo/vessels` 和 `/vessels/{mmsi}` 改为走聚合服务。 +3. 将 `/vessels/{mmsi}/track` 改为走轨迹聚合逻辑。 +4. 返回 `field_sources`、`selected_reasons`、`quality_flags`、`conflict_count`。 +5. 加入 freshness fallback 和异常位置保护。 + +### v2 — AISStream WebSocket collector（已实现） + +目标是接入第二个真实 AIS 来源，并验证多源冲突和回退逻辑。 + +1. 实现 `aisstream_vessels` collector。 +2. 支持 API key、订阅范围、消息类型、重连和限流配置。 +3. 将 AISStream 写入原始观测层，不直接 upsert 最终展示表。 +4. 接入源健康状态和 message rate 统计。 +5. 提供 AISStream API Key 获取教程、设置页入口和连接验证支持。 +6. 为重复消息、断流回退、WS 优先级写集成测试。 + +### v3 — AISStream 可用性与配置体验（已实现） + +目标是让 AISStream 从“能采集”变成日常可观察、可调试、可配置的数据源。 + +1. 设置页展示 AISStream 运行状态：连接状态、最近收到、最近成功、本轮消息数、延迟和最近错误。 +2. AISStream 设置页提供常用采集范围 preset，并保留自定义 Bounding Boxes JSON。 +3. 聚合结果返回 `source_summary`，展示每艘船的来源、观测数量、最新观测时间、传输模式和消息类型。 +4. 保留 `field_sources` 和 `selected_reasons`，用于解释动态字段来自实时流、静态字段来自可用非空来源。 +5. 船名标准化会读取 AISStream `MetaData.ShipName`；船型展示会从 `vessel_type_name` 和 AIS 数字 `vessel_type` 共同归一化，保证 marker 颜色、详情卡、hover 和搜索结果一致。 +6. `/geo/vessels` 不再默认限制 5000 艘；不传 `limit` 或传 `limit=0` 表示全量返回，前端默认也不再二次裁剪到 5000。 + +### v3.1 — 聚合完整性修复（v4 前置） + +目标是先保证“所有已采集到的船都能显示”，BarentsWatch 不因为接入 AISStream 而被 raw observation 聚合结果遮蔽。 + +当前风险是 `/geo/vessels` 只要 raw observation 聚合返回非空，就直接使用 raw 聚合结果，不再补读兼容层 `vessel_position + vessel_static`。如果 raw observation 中只存在 AISStream 的几百艘船，或 BarentsWatch 历史数据没有完整回填到 raw 层，最终 Earth 就会只显示 AISStream 子集。 + +1. `/geo/vessels` 必须合并 raw observation 聚合结果和 legacy latest position 结果。 +2. raw 与 legacy 同一 MMSI 同时存在时只显示一艘，优先使用 raw 聚合结果及其 `field_sources` / `selected_reasons`。 +3. raw 中不存在的 BarentsWatch-only MMSI 必须从 `vessel_position + vessel_static` 补齐。 +4. `bbox`、`type`、`limit` 过滤必须作用在合并后的最终集合上；不传 `limit` 或 `limit=0` 仍表示全量返回。 +5. 增加诊断统计，至少能看到 raw AISStream unique MMSI、raw BarentsWatch unique MMSI、legacy unique MMSI、final merged unique MMSI 和被 legacy 补齐的数量。 +6. 为 raw 只有 AISStream 子集、legacy 有更多 BarentsWatch 船只的场景补回归测试。 + +### v3.2 — AISStream 真实时链路（v4 前置） + +目标是把 AISStream 从“一次 collector 收一批消息后结束”改成真正的 WebSocket 长连接实时数据源，并把实时变化推送到 Earth。 + +当前 `aisstream_vessels` 只在 collector `fetch()` 中连接 `wss://stream.aisstream.io/v0/stream`，默认收 `max_messages = 500` 条后结束。这不符合 WebSocket 流式数据源的运行语义，也不能保证新船、位置变化和航向变化实时出现在前端。 + +1. 为 AISStream 增加 streaming service / long-running runner，不再依赖单次 `fetch -> transform -> save -> completed` 表达实时采集。 +2. 外部 AISStream WebSocket 保持长连接，断线后指数退避重连，并持续更新 `AISSourceHealth`。 +3. 每条或小批量 AIS 消息标准化后写入 `ais_raw_observations`，按时间或数量短周期 commit，避免长事务堆积。 +4. 将新增船只、位置变化、航向变化和静态字段补充转换成 vessel delta。 +5. 通过应用内部 `/ws` 的 `vessels` channel 广播 delta，复用 `DataBroadcaster.broadcast_custom("vessels", payload)`。 +6. Earth 前端订阅 `vessels` channel，`vessels.js` 支持按 MMSI upsert marker，而不是每次全量 reload。 +7. 船只改变航向时，前端必须更新 course bin / marker bucket，避免 marker 方向滞后。 +8. freshness 超时或 AISStream 健康异常时，动态字段可回退到 BarentsWatch 最新可用观测。 + +### v3.3 — Streaming 采集状态语义（v4 前置） + +目标是让采集页面正确表达 AISStream 这类长连接数据源，不再使用一次性 REST collector 的完成型进度条。 + +REST collector 的自然状态是 `fetch -> transform -> save -> progress 0..100 -> completed`。AISStream 的自然状态应是 `connecting -> streaming -> reconnecting -> stopped/failed`，没有固定总量，也不应在收到一批消息后显示“采集完成”。 + +1. AISStream 采集状态使用 indeterminate / streaming 状态，而不是百分比完成进度条。 +2. 设置页运行状态卡展示连接状态、已运行时长、本轮消息数、新增观测数、unique MMSI、message rate、最近消息时间、延迟和最近错误。 +3. `phase_message` 使用“正在接收 AISStream 实时消息”“重连中”“已停止”等长连接语义。 +4. 停止、重连和配置变更要有明确操作入口；配置变化后必须安全重订阅。 +5. 后端任务状态不能因为没有 `total_records` 就长期显示 `0%` 或误判失败。 +6. WebSocket 健康状态和 collector task 状态要分离：上游短暂断线是 `reconnecting`，不是普通采集任务完成或失败。 + +### v3.4 — 船只身份字段和名称聚合修复（v4 前置） + +目标是把 MMSI、IMO、callsign 这类身份编号按字符串显示，并把仍然使用 MMSI 作为船名的记录视为信息聚合未完成，而不是正常船名。 + +1. 前端详情卡、hover、搜索结果和日志中的 `mmsi`、`imo`、`callsign` 必须作为 identifier 字段展示，禁止走 `toLocaleString()` 或数字千分位格式。 +2. GeoJSON 可增加 `mmsi_display` / `imo_display` 等字符串字段，但前端仍必须对 identifier key 做兜底格式保护。 +3. 聚合服务生成船名时，不能把 `MMSI 257123000` 当成真实 `name` 的成功结果；它只能作为 display fallback。 +4. 增加诊断查询，列出所有当前仍以 MMSI 号码或 `MMSI ` 作为船只名称的记录，包括： + - `vessel_static.name` 为空或等于 MMSI fallback 的 MMSI； + - raw observation 中没有任何非空 `name` / `MetaData.ShipName` / `ShipStaticData.Name` 的 MMSI； + - 聚合结果最终 `name` 仍为 fallback 的 MMSI； + - 每个 MMSI 的可用来源、最近观测时间、message types 和缺失原因。 +5. 对这些 fallback-name 船只建立待修复集合，优先通过 AISStream `ShipStaticData`、BarentsWatch 静态字段和后续 enrichment 缓存补齐。 +6. 船只详情面板需要区分“真实船名”和“显示兜底”：真实船名缺失时展示 `MMSI ` 可以继续作为标题，但字段来源应标注为 `fallback`，避免误以为聚合成功。 +7. 为 MMSI 千分位格式、fallback-name 诊断和名称来源解释补回归测试。 + +### v4 — 策略配置（v0 可用） + +目标是开放系统级配置，但仍以安全默认值兜底。 + +已落地的最小子集： + +1. 策略持久化在 `system_settings.category = 'vessel_aggregation_strategy'`，保存时自动版本递增。 +2. `app/services/vessel_aggregation_strategy.py` 暴露 `load_strategy / save_strategy / reset_strategy / validate_strategy`，并维护 `DEFAULT_STRATEGY` 兜底。 +3. 校验规则： + - 未知 `field_rules.` → `400 unknown vessel_ais field`； + - 未知 mode → `400 mode must be one of ...`； + - 动态字段（`lat/lon/sog/cog/heading/nav_status`）使用非 `newest` mode 时必须显式 `allow_dynamic_lock=true`，否则拒绝； + - `freshness.realtime_stream_seconds` / `polling_seconds` 必须为非负整数； + - `mode=locked` 必须带非空 `locked_source`。 +4. 聚合服务 `vessel_ais_aggregation.py` 在 `_select_position_observation` 中按 `freshness` 把过期实时流降级到 stale 候选；在 `_select_static_field` 中按 `field_rules.mode = source_priority / locked / newest / non_empty` 选源。 +5. 聚合输出每条 vessel 携带 `aggregation_strategy_version`，并在 `/geo/vessels` GeoJSON properties + `/vessels/{mmsi}` 详情中暴露。 +6. API： + - `GET /api/v1/vessel-aggregation/strategy` + - `PUT /api/v1/vessel-aggregation/strategy`（校验失败 400） + - `DELETE /api/v1/vessel-aggregation/strategy`（恢复默认并 bump version） + +未做项（留给 v4 后续）： + +- 系统设置 UI 中的策略编辑器尚未做，目前直接调 API； +- `transport_priority`、`quality_flags` 级别的策略尚未引入； +- `source_priority` 中的未知 source 不强校验，留给后续 warn-only 提示。 + +### v5 — 船舶资料 enrichment 与冲突治理（v0 可用） + +目标是把 AIS 实时流里不稳定或低频出现的静态信息，补成可缓存、可审计的船舶资料层，同时把冲突解释变成可操作能力。 + +已落地的最小子集： + +1. 新增模型 `app/models/vessel_enrichment.py::VesselProfileEnrichment` + `VesselMediaEnrichment`：以 `mmsi` 为主键，记录 `source / payload / fetched_at / expires_at / confidence / reference_url`；通过 `Base.metadata.create_all` 在 `init_db` 中建表。 +2. 服务 `app/services/vessel_enrichment.py` 提供 `upsert_vessel_profile_enrichment` / `upsert_vessel_media_enrichment` / `get_vessel_enrichment_bundle`；读路径只读缓存，过期记录（`expires_at < now`）直接过滤为 `None`，永不联网。 +3. 聚合接口在 `/api/v1/visualization/vessels/{mmsi}` 响应中追加 `enrichment.profile` 与 `enrichment.media` 字段（含 `source / fetched_at / expires_at / confidence / reference_url`）；命中失败时返回 `null`，不阻塞 AIS 实时链路。 +4. 冲突治理 API： + - `POST /api/v1/vessel-aggregation/conflicts/{mmsi}/{field}/promote-to-rule` 读取最近 `AISConflictRecord.selected_source`，写入 `field_rules[field] = {mode: source_priority, source_priority: []}` 并 bump version； + - `DELETE` 对应路径移除该 field 的覆盖，恢复默认。 +5. 前端 Earth `info-card.js` 渲染 `船舶资料` 区块：profile.payload 标量字段平铺、媒体 `images` 数组缩略图、来源 / 更新时间 / 置信度元数据；缓存命中失败回退到 `资料缓存中`；常规字段在 `field_sources` 命中时附带来源 tag。 + +未做项（留给 v5 后续）： + +- 没有真正的异步 enrichment 抓取作业；当前依赖外部脚本/管理 API 写入缓存； +- 冲突治理 UI 还没接入设置中心，目前只暴露 API； +- enrichment 命中状态尚未广播到 `vessels` channel，详情面板首次打开时按需请求即可。 + +## 测试计划 + +- 同一来源同一 `mmsi + observed_at + lat + lon` 重复记录只聚合一次。 +- 多来源同一 MMSI 的位置字段优先选择最新观测。 +- 实时流和轮询源同时间冲突时，实时流优先。 +- 实时流过期后，更新的轮询源可以接管动态字段。 +- 实时流源健康状态异常时，动态字段可以回退到更新的可用来源。 +- 静态字段不会被空值覆盖。 +- 静态字段冲突会写入冲突记录。 +- 明显异常位置不会进入默认展示轨迹，并会留下 `quality_flags`。 +- 同一时间窗口内多来源相近轨迹点只展示一个点。 +- AISStream 重连或回放导致的重复消息不会重复进入聚合结果。 +- raw observation 聚合结果和 legacy latest position 结果会按 MMSI 合并，BarentsWatch-only 船只不会因为 AISStream 子集存在而消失。 +- 不传 `limit` 或传 `limit=0` 时，`/geo/vessels` 全量返回合并后的船只集合。 +- AISStream 长连接收到新船、位置变化和航向变化后，会通过内部 `/ws` 的 `vessels` channel 推送增量。 +- AISStream streaming 状态不会显示成固定百分比完成进度条，也不会在收到一批消息后误报采集完成。 +- `mmsi`、`imo`、`callsign` 等身份编号在前端不显示千分位符。 +- 聚合结果中仍以 MMSI fallback 作为船名的记录可以被诊断查询完整列出，并带来源和缺失原因。 +- 字段级配置可以覆盖默认来源优先级。 +- 聚合接口在没有冲突表时仍可返回兼容 GeoJSON。 + +## 相关文件 + +- [实时船只监控系统计划](/home/ray/dev/linkong/planet/docs/plans/earth-vessel-tracking-plan.md) +- [自定义 API 数据源与 LLM 映射系统计划](/home/ray/dev/linkong/planet/docs/plans/datasource-custom-api-mapping-plan.md) +- [BarentsWatch AIS collector](/home/ray/dev/linkong/planet/backend/app/services/collectors/vessel_ais.py) +- [船只模型](/home/ray/dev/linkong/planet/backend/app/models/vessel.py) +- [可视化 API](/home/ray/dev/linkong/planet/backend/app/api/v1/visualization.py) diff --git a/docs/plans/earth-vessel-rendering-performance-plan.md b/docs/plans/earth-vessel-rendering-performance-plan.md new file mode 100644 index 00000000..7db14adc --- /dev/null +++ b/docs/plans/earth-vessel-rendering-performance-plan.md @@ -0,0 +1,249 @@ +# Earth Vessel Rendering Performance Plan + +## 当前状态 + +该计划的前端核心部分已经在 `0.44.1` 落地，但最终实现不是原文设想的 `InstancedBufferGeometry` quad，而是更稳的分桶 `THREE.Points` 方案： + +- 普通船只按 moving / anchored 和 `VESSEL_COURSE_BINS` 航向分桶，使用 `PointsMaterial` 批量绘制。 +- 航行船只仍是带方向的三角形，停泊或低速船只仍是圆点。 +- hover / locked 不再放大成世界尺寸 Sprite，而是在原点位叠加同尺寸单点 glow overlay。 +- picking 改为屏幕空间命中，拖动和惯性期间跳过 hover picking。 +- 普通态关闭 glow，交互态才显示 glow，降低 overdraw 并让默认地图更干净。 + +后续如果需要全球 AIS 或更高船只密度，再评估是否从分桶 `Points` 升级到真正 instanced quad 或视口 bbox / LOD。 + +## 背景 + +Earth 船只图层已经形成了一套较好的视觉语言： + +- 航行船只使用三角形标记 +- 标记按航向旋转 +- 停泊或低速船只使用圆点 +- 不同船型使用不同颜色 +- hover / locked 状态有 glow、透明度和聚焦反馈 +- 标记带有轻微 glow / soft edge，和 Earth HUD 的观感一致 + +当前性能问题不应通过降级成无方向、无船型语义的普通小点来解决。目标是在保留现有观赏性的前提下，把底层从“每艘船一个 Sprite 对象”优化为批量绘制和轻量交互。 + +## 当前问题判断 + +卫星图层能承载几万个对象，是因为它主要走 `THREE.Points` / `BufferGeometry` / instanced trail 路径。船只图层目前每艘船创建一个 `THREE.Sprite` 和独立 `SpriteMaterial`，这会带来： + +- draw call 随船只数量增长 +- 透明 sprite 排序和 overdraw 成本上升 +- 每帧遍历所有船只更新 opacity / scale / visible +- pointer move 时对船只 sprite 做对象级 raycast +- hover reset 时全量遍历 marker + +因此，即使免费 BarentsWatch AIS 只开放挪威周边数据，前端仍可能因为对象级 sprite、raycast 和每帧全量更新出现地球拖动卡顿。 + +## 目标 + +1. 保留当前船只标记的视觉质量。 +2. 保留 hover tooltip、点击详情、lock、轨迹等交互。 +3. 显著降低 draw call、每帧 JS 遍历和 pointer picking 成本。 +4. 为后续全球 AIS 或更高船只数量预留扩展空间。 + +## 非目标 + +- 不把船只降级为普通无方向 `Points`。 +- 不取消船型颜色、航向三角和停泊圆点。 +- 不为了短期性能直接删除 hover / click 交互。 + +## Phase 1：交互路径止血 + +这一阶段不改视觉，只减少 pointer move 和 hover 状态开销。 + +### 1. 拖动和惯性期间跳过船只 picking + +地球拖动时用户主要关注视角变化，不需要每个 pointer move 都命中船只。 + +处理方式： + +- `isDragging === true` 时跳过船只 hover picking。 +- 惯性旋转期间也跳过船只 hover picking。 +- 拖动结束后再恢复 hover 检测。 + +### 2. vessel hover picking 节流 + +对船只 hover 命中增加节流，例如 `80ms ~ 120ms` 一次。鼠标高速移动时复用上一次 hover 状态，不在每个 pointer event 上都做 raycast。 + +### 3. hover reset 从全量遍历改为增量更新 + +当前 `resetTransientVesselStates()` 会遍历所有船只。改为记录： + +- `hoveredVessel` +- `lockedObject` + +当 hover 目标变化时，只更新旧 hover 和新 hover。 + +### 4. 点击路径只在 click 时做一次精确 picking + +点击仍保留精确命中，但只在 click 事件里执行，不参与拖动和高频 pointer move。 + +## Phase 2：每帧更新减负 + +这一阶段仍保留 `Sprite` 外观，但减少每帧对全部 marker 的写操作。 + +### 1. `updateVesselVisualState()` 增量化 + +当前每帧都会遍历船只并写： + +- `marker.material.opacity` +- `marker.scale` +- `marker.visible` + +优化方向： + +- 图层关闭时直接 return。 +- 没有船只时直接 return。 +- 只有以下状态变化时才更新 marker： + - show/hide 变化 + - hover 变化 + - locked 变化 + - camera zoom / distance scale 变化超过阈值 + - focus dim 状态变化 + +### 2. 缓存 distance scale + +`getDistanceScale(camera)` 可以按 camera distance 或 zoom 阈值缓存。缩放没有明显变化时，不必每帧重设所有船只 scale。 + +### 3. 降低透明 overdraw + +在不破坏视觉的前提下微调： + +- marker 基础尺寸 +- glow blur 半径 +- 最大 size stabilization + +目标是减少屏幕空间重叠面积，而不是改变符号设计。 + +## Phase 3：保留视觉的批量渲染（已落地为分桶 Points） + +原设想是把每艘船的视觉从 `THREE.Sprite` 迁移为 instanced sprite batch。实际落地时选择了更稳的分桶 `THREE.Points`： + +- 不依赖自定义 shader。 +- 不依赖 `Points` 自带 raycaster。 +- 用 canvas 纹理保留三角、圆点、船型颜色和航向。 +- 用 hover / locked 单点 overlay 保留交互 glow。 + +如果未来全球 AIS 导致分桶 `Points` 仍不够，再升级到 instanced quad。 + +### 1. 原候选方案：instanced quad + +每艘船仍然显示为带贴图/软边的 billboard，但底层使用： + +- `THREE.InstancedBufferGeometry` +- 每类船只一个或少量 material +- per-instance attributes + +可按形状和船型拆 batch： + +- moving cargo +- moving tanker +- moving passenger +- moving fishing +- moving military +- moving other +- anchored / slow dot + +这样 draw call 从“每艘船一个”变为“每类船只一个”。 + +### 2. 原候选方案：per-instance attributes + +每个 instance 存： + +- position +- color +- rotation +- scale +- opacity +- state +- mmsi / data index + +hover、locked、dimmed 可通过更新少量 instance attribute 实现，不再逐个修改 material。 + +### 3. 当前落地方案：分桶 `THREE.Points` + +当前实现按以下方式复刻视觉： + +- moving 船只按 `VESSEL_COURSE_BINS` 做航向分桶。 +- anchored / slow 船只使用圆点分桶。 +- 每个分桶生成一组 `THREE.Points`，共享 `PointsMaterial` 和 canvas 点纹理。 +- `VESSEL_CONFIG.colors` 仍通过 vertex colors 表示船型颜色。 +- hover / locked 在原位置叠加同尺寸单点 overlay，普通态不带 glow，交互态才带 glow。 + +这样 draw call 从“每艘船一个”变为“每个形状 / 航向分桶一组”，同时避免自定义 shader 的兼容风险。 + +### 4. 复刻当前视觉 + +视觉上继续使用当前 canvas texture 或等效 shader： + +- moving 使用三角形纹理 +- anchored 使用圆点纹理 +- 保留 soft glow +- 保留航向 rotation +- 保留 hover / locked 放大 + +因此用户看到的效果应与当前船只图层基本一致。 + +## Phase 4：picking 改造 + +批量渲染后不再适合对所有 sprite object 做 `raycaster.intersectObjects()`。 + +### 1. 屏幕空间 picking + +参考卫星 picking： + +1. 过滤背面船只。 +2. 将候选船只世界坐标投影到屏幕。 +3. 用鼠标位置计算距离。 +4. 取距离最近且小于半径阈值的船只。 + +### 2. 可选空间索引 + +如果后续船只数量明显上升，可增加轻量空间索引： + +- 经纬度网格 bucket +- 屏幕空间 bucket +- viewport bbox 过滤 + +第一阶段不必引入复杂索引。 + +## Phase 5：数据层和 LOD + +当接入全球 AIS 或船只数量显著增加时，再做数据层优化。 + +### 1. 请求视口范围 + +前端请求 `/api/v1/visualization/geo/vessels` 时带上当前视口 `bbox`，减少无关船只。 + +### 2. 后端排序策略 + +从单纯 `received_at desc` 改为综合排序： + +- 数据新鲜度 +- 船型优先级 +- 当前视口相关性 +- 是否正在航行 + +### 3. 远景聚合 + +远景可显示聚合或 top N，近景展开单船。 + +## 验收指标 + +1. 船只视觉效果保持当前质量：三角、圆点、颜色、航向、hover、lock 都保留。 +2. 开启船只图层后拖动地球不应明显掉帧。 +3. pointer move 不应因为船只 hover 导致卡顿。 +4. 船只数量达到 `1000` 级别时仍可顺畅旋转地球。 +5. `renderer.info.render.calls` 相比 Sprite 版本显著下降。 +6. hover / click 命中体验不低于当前版本。 + +## 建议落地顺序 + +1. 先做 Phase 1，快速恢复地球拖动手感。 +2. 再做 Phase 2，减少每帧 JS 写操作。 +3. Phase 3 和 Phase 4 已按分桶 `THREE.Points` + 屏幕空间 picking 落地。 +4. Phase 5 等全球船只数据或数量压力出现后再推进。 +5. 如果分桶 `THREE.Points` 达到瓶颈，再评估 instanced quad。 diff --git a/docs/plans/earth-vessel-tracking-plan.md b/docs/plans/earth-vessel-tracking-plan.md new file mode 100644 index 00000000..e1ccfc05 --- /dev/null +++ b/docs/plans/earth-vessel-tracking-plan.md @@ -0,0 +1,281 @@ +# 实时船只监控系统 — 实施计划 + +**状态**：规划中 +**创建日期**：2026-04-27 +**优先数据源**：BarentsWatch AIS（免费但需要 OAuth client credentials）→ AISHub / MarineTraffic（TODO，付费） + +## 已确认决策 + +| 项目 | 决策 | +|-----|------| +| 数据源 | BarentsWatch 先行；AISHub / MarineTraffic TODO | +| 船只规模 | BarentsWatch 阶段全部显示；全球数据接入后按需加船型过滤（默认 Cargo + Tanker + Passenger） | +| 更新频率 | 准实时：前端 5 分钟轮询，后端 Collector 每分钟拉取写库 | +| 历史轨迹 | 保留（`vessel_position` 表保留 24h，后期按需扩展） | +| 推送方式 | 前端展示仍可先用 HTTP 拉取聚合结果；AISStream 等实时源应单独实现 WebSocket 采集器 | + +--- + +## 一、技术背景 + +船只通过 AIS（自动识别系统）每 2–10 秒广播位置、航速、航向、目的地等信息。全球约 50 万艘持证船只在线，实时数据通过以下方式获取： + +| 来源类型 | 典型服务 | 覆盖范围 | 成本 | 状态 | +|---------|---------|---------|------|------| +| **BarentsWatch AIS API** | live.ais.barentswatch.no | 挪威海域实时 | 免费，需要 AIS API client credentials | **当前使用** | +| **AISHub** | aishub.net | 全球实时 | 免费/小额 | TODO：付费接入 | +| **MarineTraffic API** | marinetraffic.com | 全球实时 | $50–$500/月 | TODO：评估 tier | +| **VesselFinder API** | vesselfinder.com | 全球实时 | $50–$300/月 | TODO：备选 | +| **自建 SDR 接收** | RTL-SDR + AIS-catcher | 仅本地 30–50km | 硬件 $30 | 不考虑 | +| **NOAA 历史数据** | Marine Cadastre | 美国近海历史 | 免费 | 可用于冷启动 | + +### BarentsWatch AIS API + +- 端点：`https://live.ais.barentswatch.no/v1/latest/combined` +- 需要在 BarentsWatch developer portal 创建 `AIS - API` client，通过 client credentials 获取 `scope=ais` 的 access token 后请求 AIS endpoint +- 字段：mmsi, lat, lon, sog, cog, heading, nav_status, name, vessel_type, flag +- 刷新频率：数据约 30–60s 更新一次，可随意轮询 + +### TODO：多源 AIS 与实时流接入 + +- [ ] 接入 AISStream WebSocket 采集器，作为 BarentsWatch 覆盖不足的实时补充 +- [ ] 将 BarentsWatch、AISStream、自定义 `vessel_ais` 映射源统一写入原始观测层 +- [ ] 通过聚合接口做去重、字段合并、冲突记录和默认来源选择 +- [ ] 开放字段级聚合策略配置，让用户决定不同字段优先信任哪个来源 +- [ ] 评估 AISHub 订阅（全球覆盖，约 $30/月），接入全球实时流 +- [ ] 评估 MarineTraffic API tier，对比 AISHub 数据质量与成本 +- [ ] 实现多数据源适配器，通过 `datasource_config` 切换 +- [ ] 真实高频 AIS 稳定接入后，评估将 `vessel_position` 迁移为 TimescaleDB hypertable（保留 Postgres 原生分区作为备选） + +多源 AIS 的详细设计见 [AIS 多源采集、冲突记录与聚合接口计划](/home/ray/dev/linkong/planet/docs/plans/earth-vessel-ais-aggregation-plan.md)。 + +--- + +## 二、实施计划 + +### Phase 0 — 数据源验证与链路打通（1–2 天） + +- 接入 BarentsWatch AIS API，验证 OAuth token、数据格式与字段 +- 构建全球 mock 数据生成器（用于前端渲染压测，补充 BarentsWatch 的地域限制） +- 确认前端可渲染船只点，整条链路走通 + +### Phase 1 — 后端基础设施（3–4 天） + +#### 1.1 数据库 Schema + +```sql +-- 船只静态信息（每 6h 刷新一次） +CREATE TABLE vessel_static ( + mmsi BIGINT PRIMARY KEY, + name VARCHAR(128), + callsign VARCHAR(16), + vessel_type SMALLINT, + vessel_type_name VARCHAR(64), + flag VARCHAR(4), -- ISO 国家码 + length FLOAT, + width FLOAT, + draught FLOAT, + imo BIGINT, + updated_at TIMESTAMPTZ +); + +-- 船只实时位置（高频写入，保留 24h 轨迹） +CREATE TABLE vessel_position ( + id BIGSERIAL PRIMARY KEY, + mmsi BIGINT NOT NULL, + lat FLOAT NOT NULL, + lon FLOAT NOT NULL, + sog FLOAT, -- Speed over ground（节） + cog FLOAT, -- Course over ground（度） + heading SMALLINT, -- 真北航向 + nav_status SMALLINT, -- 0=航行 1=锚泊 5=停靠 ... + received_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_vessel_pos_mmsi_time ON vessel_position(mmsi, received_at DESC); +CREATE INDEX idx_vessel_pos_time ON vessel_position(received_at DESC); + +-- 最新位置物化视图（地图渲染主数据源，避免全表扫描） +CREATE MATERIALIZED VIEW vessel_latest AS +SELECT DISTINCT ON (mmsi) + vp.*, vs.name, vs.vessel_type_name, vs.flag, vs.length +FROM vessel_position vp +LEFT JOIN vessel_static vs USING (mmsi) +ORDER BY mmsi, received_at DESC; + +CREATE UNIQUE INDEX ON vessel_latest(mmsi); +``` + +> 后期如需完整历史轨迹查询，迁移 `vessel_position` 到 TimescaleDB 或按天分区。 + +#### 1.2 Collector：VesselAISCollector + +文件：`backend/app/services/collectors/vessel_ais.py` + +- 继承 `BaseCollector`，注册到 `collector_registry` +- 轮询间隔：30–60s（由数据源限速决定） +- 支持多数据源切换，通过 `datasource_config` 配置 URL + API Key +- 写入逻辑：upsert `vessel_latest`，append `vessel_position` +- 接入现有调度系统（`scheduler.py`） + +#### 1.3 API 端点 + +``` +GET /api/v1/visualization/geo/vessels + ?bbox=lon_min,lat_min,lon_max,lat_max # 视口裁剪 + ?type=cargo,tanker,passenger # 船型过滤 + ?limit=0 # 可选；不传或 0 表示不裁剪数量 +→ GeoJSON FeatureCollection（Point） + +GET /api/v1/visualization/vessels/{mmsi} # 单船详情 +GET /api/v1/visualization/vessels/{mmsi}/track # 历史轨迹（默认 6h） + ?hours=6 +→ GeoJSON LineString +``` + +GeoJSON Feature 格式： + +```json +{ + "type": "Feature", + "geometry": { "type": "Point", "coordinates": [lon, lat] }, + "properties": { + "mmsi": 123456789, + "name": "EVER GIVEN", + "vessel_type": 70, + "vessel_type_name": "Cargo", + "flag": "PA", + "sog": 12.4, + "cog": 247.0, + "heading": 245, + "nav_status": 0, + "length": 400, + "received_at": "2026-04-27T10:00:00Z" + } +} +``` + +#### 1.4 更新机制 + +**前端聚合结果拉取 + 后端实时采集**： + +- 前端 `setInterval(fetchVessels, 5 * 60 * 1000)` 定期拉取最新快照 +- 后端 BarentsWatch collector 继续以 HTTP polling 方式采集 +- AISStream 等实时源以独立 WebSocket collector 写入原始观测层 +- 展示接口从聚合服务读取当前船只视图，而不是由单个 collector 决定最终展示值 +- 前端默认不再给 `/geo/vessels` 传 `limit=5000`，`VESSEL_CONFIG.maxRenderedMarkers = 0` 表示不做前端数量裁剪；后续如性能不足再引入显式 LOD 上限 +- marker 颜色、详情卡、hover 和搜索结果必须共享 `vessel_type_display` 船型归一化结果，避免 AIS 数字类型码已驱动颜色但卡片仍显示 `Other` +- 前端是否升级为 WebSocket delta push 是独立优化，不影响后端采集器可以使用 WebSocket 接上游实时源 + +--- + +### Phase 2 — 前端渲染（3–4 天） + +文件：`frontend/public/earth/js/vessels.js` + +#### 2.1 渲染方案 + +参考现有卫星系统（`satellites.js`）的 InstancedMesh 模式： + +- `THREE.InstancedMesh`：每个实例 = 一艘船，矩阵包含位置 + 旋转（朝向 COG） +- 行进船：三角箭头图标，朝向 COG 方向 +- 静止/锚泊船：圆点图标 +- SVG 图标输出到 `frontend/public/earth/assets/icons/vessel-arrow.svg` 和 `vessel-dot.svg` + +#### 2.2 船型颜色规范 + +| 船型 | 颜色 | +|-----|------| +| 货轮 Cargo | `#4A90D9` 蓝 | +| 油轮 Tanker | `#E85D04` 橙红 | +| 客船 Passenger | `#06D6A0` 绿 | +| 渔船 Fishing | `#FFD166` 黄 | +| 军舰 Military | `#73797E` 灰 | +| 其他 | `#9B9B9B` 浅灰 | +| 锚泊/停靠 | 降低饱和度 0.4x | + +#### 2.3 LOD（相机距离细节层次） + +| 相机距离 | 渲染策略 | +|---------|---------| +| > 400 | 默认渲染当前接口返回的全部船只；如性能不足，再引入可配置 LOD 上限 | +| 200–400 | 默认渲染当前接口返回的全部船只；如性能不足，再引入可配置 LOD 上限 | +| < 200 | 渲染当前视口 bbox 内全部船只 | + +前端根据相机位置动态计算 bbox，附加到 API 请求中。 + +#### 2.4 图层集成 + +接入现有图层系统，新增"船只"图层项，支持： +- 图层开/关，状态持久化 +- 子过滤（按船型选择显示哪类，可在图例或设置面板中配置） +- 与海缆、BGP、卫星层级共存（renderOrder 待定，参考现有层级文档） + +#### 2.5 Info Card + +复用 `showInfoCard` 机制，点击船只弹出： + +``` +EVER GIVEN 🚢 +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +MMSI 123456789 +IMO 9811000 +旗帜 巴拿马 🇵🇦 +船型 散货轮 +当前航速 12.4 kn +航向 247° +状态 航行中 +目的地 ROTTERDAM +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +[ 查看轨迹 ] [ MarineTraffic ↗ ] +``` + +#### 2.6 轨迹可视化 + +点击"查看轨迹" → 请求 `/vessels/{mmsi}/track` → 用 `THREE.CatmullRomCurve3` 渲染插值轨迹线，风格与海缆一致。 + +--- + +### Phase 3 — 功能完善（2–3 天） + +| 功能 | 说明 | +|-----|------| +| **船只搜索** | 接入现有搜索面板，按名称 / MMSI 搜索 | +| **统计 HUD** | 显示当前在线船只数、各类型分布 | +| **密度热图** | 超低 zoom 时切换为 hex-bin 热力图（避免点云爆炸） | +| **港口标注** | 加载 WorldPorts 数据集，显示主要港口标记 | +| **关键水道监控** | 马六甲、霍尔木兹、苏伊士等高亮 + 流量统计 | + +--- + +### Phase 4 — 性能与生产化（2–3 天） + +- `vessel_position` 按天分区，7 天自动清理 +- TODO：真实数据量达到百万级/日后，将 `vessel_position` 升级为 TimescaleDB hypertable，配置 retention policy 与压缩策略 +- GeoJSON endpoint 用 Redis 缓存 15s +- 若需 bbox 精确查询，引入 PostGIS `geography` + `ST_DWithin` +- InstancedMesh + frustum culling，目标 5 万船只 60fps + +--- + +## 三、工作量估算 + +| Phase | 内容 | 估计时间 | +|-------|-----|---------| +| Phase 0 | 数据源验证、mock | 1–2 天 | +| Phase 1 | 后端 Schema + Collector + API | 3–4 天 | +| Phase 2 | 前端渲染（InstancedMesh + 图层 + Info Card） | 3–4 天 | +| Phase 3 | 搜索 + 统计 + 轨迹 | 2–3 天 | +| Phase 4 | 性能优化 + 生产数据源接入 | 2–3 天 | +| **合计** | | **约 2–3 周** | + +--- + +## 四、参考资料 + +- BarentsWatch AIS API 文档：https://www.barentswatch.no/en/developer/ais-api/ +- MarineTraffic API：https://www.marinetraffic.com/en/ais-api-services +- AISHub：https://www.aishub.net/api +- AIS 导航状态码：ITU-R M.1371-5 +- 船型编码（vessel_type）：ITU/IMO AIS Message 5 Type and Cargo +- WorldPorts 数据集：https://msi.nga.mil/Publications/WPI diff --git a/docs/plans/enterprise-logging-system-plan.md b/docs/plans/enterprise-logging-system-plan.md new file mode 100644 index 00000000..4b7e667a --- /dev/null +++ b/docs/plans/enterprise-logging-system-plan.md @@ -0,0 +1,793 @@ +# Planet 企业级日志系统实施计划 + +## Goal + +把 Planet 当前“能看一点运行输出”的日志能力，升级为一套真正可用、可定位、可纠错、可追责、可演进的企业级日志系统。 + +这里的“企业级”不是指一上来就接入很重的外部平台，而是指这套系统需要同时满足下面五件事： + +1. 排障可用 +2. 历史可查 +3. 业务可解释 +4. 权限操作可追责 +5. 出错后能够反向定位到请求、任务、模块和操作者 + +最终目标不是“把更多 stdout 放到日志页里”，而是建立一套统一的日志契约与落地链路： + +- 统一日志字段 +- 统一事件命名 +- 统一采集入口 +- 统一查询视图 +- 清晰的实时日志、持久化事件、审计日志分层 + +## Why + +当前仓库已经有一些日志基础，但离真正可用的日志系统还有明显距离。 + +已有基础： + +- 后端运行日志可通过 `/tmp/planet_backend.log` 查看 +- 前端开发服务日志可通过 `/tmp/planet_frontend.log` 查看 +- AI Provider 可从 Docker 容器读取日志 +- Earth 浏览器端关键日志可上报到后端并进入 Redis 缓冲 +- 已有 `system_logs` / `audit_logs` 持久化能力 +- 管理台已有“系统日志”页面，支持来源、级别、日期、搜索 + +当前缺口： + +- 后端日志仍以 `uvicorn` / 文本输出为主，不是统一结构化事件流 +- 不同模块的日志格式不一致，很多地方只有 message，没有 event 语义 +- 还没有统一的后端 logger 封装与字段注入机制 +- 前端虽然能上报错误，但还没有统一 logger API 和统一事件词汇 +- Earth 与管理台之间的错误事件还没有形成可串联的事件链路 +- 历史持久化还偏点状，很多高价值失败并没有系统性落库 +- 系统日志页当前更像“运行输出查看器”，不是“多层日志查询台” +- 审计日志与运行日志尚未形成明确的产品级联动 + +所以当前真正的问题不是“有没有日志页”，而是： + +**当前系统能看见输出，但还不能稳定回答“发生了什么、影响了谁、在哪条链路上坏了、是否已修复、是谁触发的”。** + +## Current State + +截至 2026-04-23，当前代码中的日志相关能力大致如下。 + +### 1. 日志来源 + +当前系统日志页主要读取以下来源： + +- `backend` + 读取 `/tmp/planet_backend.log` +- `frontend` + 读取 `/tmp/planet_frontend.log` +- `ai-provider` + 读取 Docker 容器日志 +- `earth-client` + 读取 Redis 缓冲的浏览器端日志 + +这些来源定义在： + +- [backend/app/services/system_logs.py](/home/ray/dev/linkong/planet/backend/app/services/system_logs.py) + +### 2. 当前日志读取模型 + +当前 `read_log_snapshot()` 的职责是： + +- 读取某个来源的最近若干行 +- 解析基础级别与时间 +- 按级别、日期、搜索进行过滤 +- 返回用于日志页展示的快照 + +这个模型适合“运维查看器”，但不适合企业级日志系统，原因是： + +- 读取基于文本尾部扫描，不是基于事件模型 +- 不同来源的结构粒度完全不同 +- 过滤依赖文本解析，准确率有限 +- 没有请求、任务、用户、资源、动作等核心关联字段 + +### 3. 已有持久化能力 + +当前已经存在两个持久化入口： + +- `record_system_log(...)` +- `record_audit_log(...)` + +位置： + +- [backend/app/services/persistent_logs.py](/home/ray/dev/linkong/planet/backend/app/services/persistent_logs.py) + +这说明系统并不是从 0 开始，但也说明当前最大的问题是： + +**持久化能力存在，但没有成为统一默认路径。** + +### 4. 已有 request_id 基础 + +当前系统已具备 `request_id` 相关基础，部分持久化能力也会尝试写入 `request_id`。 + +这为后续做： + +- 请求链路排障 +- 前后端关联查询 +- 任务执行追踪 + +提供了很好的基础。 + +### 5. 当前日志页定位 + +当前日志页已经具备： + +- 来源切换 +- 级别筛选 +- 日期筛选 +- 搜索 +- 文本控制台视图 + +但它仍然是“单层视图”： + +- 上面是筛选器 +- 下面是一块文本控制台 + +它还不是： + +- 运行日志 + 事件日志 + 审计日志 的统一入口 +- 也没有事件详情、关联跳转、纠错建议、链路追踪能力 + +## Core Principles + +这套日志系统后续必须遵循下面几个原则。 + +### 1. 分层，而不是混存 + +日志必须拆成三层： + +1. 运行日志 +2. 持久化事件日志 +3. 审计日志 + +它们的用途不同，绝不能继续混成一个概念。 + +#### 运行日志 + +用于： + +- 实时排障 +- 观察服务运行状态 +- 看 stdout / stderr / exception / collector 输出 + +特点： + +- 数据量大 +- 时效性强 +- 保留周期短 +- 不要求每条都落库 + +#### 持久化事件日志 + +用于： + +- 记录高价值错误 +- 记录关键业务失败 +- 支撑历史追溯 +- 支撑趋势分析 + +特点： + +- 只持久化有价值事件 +- 必须结构化 +- 必须有统一 event 命名 + +#### 审计日志 + +用于： + +- 留痕 +- 追责 +- 还原高权限操作 + +特点： + +- 必须单独建模 +- 不与普通运行日志混用 + +### 2. 结构化优先 + +正式日志必须可拆字段，不能长期依赖自由文本。 + +最低要求至少能拿到： + +- `timestamp` +- `level` +- `service` +- `module` +- `event` +- `message` +- `request_id` +- `trace_id` +- `user_id` / `actor` +- `context` + +### 3. 事件命名优先于 message 命名 + +人看的 message 可以变化，但机器查询和跨模块关联必须依赖稳定事件名。 + +例如： + +- `collector.run.started` +- `collector.run.completed` +- `collector.run.failed` +- `earth.layer.load_failed` +- `earth.cruise.route_build_failed` +- `system.restart_task.failed` +- `auth.websocket.invalid_token` + +### 4. 查询链路必须可串联 + +企业级日志系统的核心不是“有很多日志”，而是“能串起来”。 + +最终一条高价值事件，至少要能回链到下面任意几类对象： + +- 某个请求 +- 某个任务 +- 某个用户 +- 某个数据源 +- 某个 Earth 模块 +- 某个管理动作 + +### 5. 默认脱敏 + +日志体系必须明确禁止记录： + +- token +- password +- Authorization header +- cookie +- session +- 明文敏感个人信息 + +并且需要有统一脱敏器，而不是靠调用者自觉。 + +### 6. “可纠错”不是一句口号 + +这里的“可纠错”至少包含三层： + +1. 日志字段足够解释错误，方便人排查 +2. 系统能识别常见错误模式并给出纠偏建议 +3. 关键错误支持闭环动作，例如重试、重建索引、重新触发采集、跳转到对应对象 + +也就是说，这套日志系统最终不只是“告诉你出错了”，而要尽量接近“告诉你为什么出错、怎么修、去哪修”。 + +## Non-Goals + +第一阶段不追求： + +- 全量接入 ELK / Loki / Datadog / OpenTelemetry 全家桶 +- 做分布式 trace 全链路可视化大屏 +- 把所有历史日志都迁进数据库 +- 先做特别复杂的规则引擎 + +第一阶段追求的是： + +- 在当前仓库和当前部署方式下，先把基础日志体系做正确 +- 再为后续平台化接入预留好接口 + +## Target Architecture + +推荐目标架构如下。 + +### Layer 1: Runtime Logs + +职责： + +- 承载后端、前端开发服务、容器输出、浏览器端缓冲事件 +- 提供最近窗口内的实时查看能力 + +来源： + +- 文件 +- Docker +- Redis 缓冲 +- 后续可扩展到 stdout collector + +接口： + +- `GET /api/v1/system/logs/sources` +- `GET /api/v1/system/logs/{source_id}` + +这层继续保留，但需要做结构化增强和来源补强。 + +### Layer 2: Persistent System Events + +职责： + +- 只存高价值事件 +- 供历史追溯、事件列表、趋势和纠错使用 + +数据来源： + +- 后端关键异常 +- 浏览器端关键失败 +- 采集器/调度器关键失败 +- 业务关键告警与降级事件 + +接口建议： + +- `GET /api/v1/system/events` +- `GET /api/v1/system/events/{id}` +- `POST /api/v1/system/events/{id}/actions/...`（后续） + +### Layer 3: Audit Logs + +职责： + +- 留痕高权限操作 +- 记录操作者、对象、结果、请求号 + +接口建议： + +- `GET /api/v1/system/audit-logs` + +### Layer 4: Error Intelligence / Triage + +职责： + +- 对高频错误做归类 +- 对已知错误给出解释与建议动作 +- 对相同错误进行 fingerprint 聚合 + +这是“可纠错”能力的关键层。 + +建议字段： + +- `fingerprint` +- `root_cause_type` +- `known_fix_hint` +- `runbook_url` +- `related_resource_type` +- `related_resource_id` + +## Canonical Event Model + +推荐统一事件字段模型如下。 + +### Runtime Log Record + +```json +{ + "timestamp": "2026-04-23T10:15:30Z", + "level": "error", + "service": "backend", + "module": "app.services.scheduler", + "event": "collector.run.failed", + "message": "Collector bgp_news failed", + "request_id": "req_xxx", + "trace_id": "trace_xxx", + "user_id": null, + "actor": null, + "resource_type": "collector", + "resource_id": "bgp_news", + "context": { + "datasource_id": 12, + "exception_type": "TimeoutError" + } +} +``` + +### Persistent System Event + +```json +{ + "id": 1024, + "event": "earth.layer.load_failed", + "level": "error", + "source": "earth-client", + "service": "earth", + "module": "cables", + "message": "Failed to load cable layer", + "fingerprint": "earth.layer.load_failed:cables:network_timeout", + "request_id": "req_xxx", + "trace_id": null, + "user_id": 1, + "resource_type": "earth_layer", + "resource_id": "cables", + "category": "visualization", + "status": "open", + "context": { + "url": "/api/v1/visualization/geo/cables" + }, + "created_at": "2026-04-23T10:15:30Z" +} +``` + +### Audit Log + +```json +{ + "id": 88, + "action": "system.restart_task.requested", + "actor_id": 1, + "actor_name": "root", + "target_type": "restart_task", + "target_id": "restart_20260423_xxx", + "result": "success", + "request_id": "req_xxx", + "ip": "127.0.0.1", + "details": { + "action": "restart_backend" + }, + "created_at": "2026-04-23T10:15:30Z" +} +``` + +## Implementation Plan + +## Phase 0: Logging Inventory And Naming Freeze + +目标： + +- 先统一“记录什么”和“怎么命名”，避免后面越做越乱 + +工作项： + +- 盘点当前所有 `logging.getLogger` 使用点 +- 盘点裸 `print` +- 盘点 `record_system_log` / `record_audit_log` 已落点位 +- 建立统一事件命名表 +- 定义 service / module / category / resource 字段枚举 +- 输出日志字段白名单和脱敏规范 + +完成标准： + +- 有一份稳定的事件命名清单 +- 有一份字段规范清单 +- 后续新增日志不再“临时起名” + +## Phase 1: Backend Structured Logging Foundation + +目标： + +- 把后端从“散落 logging + 文本输出”升级成“统一结构化 logger” + +工作项： + +- 新增统一后端 logger helper，例如 `app/core/logging.py` +- 自动注入： + - `service` + - `module` + - `request_id` + - `trace_id` +- 增加统一脱敏 filter +- 把关键模块先切到统一 logger： + - API 层 + - scheduler + - collectors + - websocket + - visualization + - system control +- 约束： + - 正式路径禁止裸 `print` + - 正式异常优先 `logger.exception(..., extra={...})` + +完成标准： + +- 后端关键模块都有稳定 `event` +- request 日志和异常日志能挂上 `request_id` +- 不再依赖只看 `uvicorn` 原生文本输出来定位问题 + +## Phase 2: Persistent Event Layer + +目标： + +- 把“值得长期保留的错误和关键事件”系统性落库 + +工作项： + +- 重新定义 `record_system_log()` 的使用边界 +- 明确哪些事件必须持久化： + - API 关键失败 + - 调度器失败 + - 采集器失败 + - Earth 客户端关键错误 + - 数据源不可用 + - 业务降级与恢复 +- 补齐字段： + - `event` + - `resource_type` + - `resource_id` + - `category` + - `fingerprint` + - `status` +- 增加高频错误去重/聚合策略 + +完成标准： + +- 高价值错误不再只存在于运行日志里 +- 能查询最近一周/一月的关键失败事件 +- 相同错误具备聚合基础 + +## Phase 3: Frontend And Earth Unified Logger + +目标： + +- 把前端从“点状 error 上报”升级成统一前端事件流 + +工作项： + +- 在前端新增统一 logger API +- 统一方法： + - `debug` + - `info` + - `warn` + - `error` +- 统一字段： + - `page` + - `module` + - `event` + - `message` + - `url` + - `user_agent` + - `context` +- Earth 模块优先接入： + - layer load failed + - cruise build failed + - popup render failed + - connector render failed + - websocket dropped +- 管理台优先接入： + - settings save failed + - datasource toggle failed + - restart task submit failed + +完成标准： + +- 前端日志事件名与后端可对齐 +- Earth 和管理台关键失败不再只停留在 console +- 浏览器端关键问题能进入统一系统日志/事件层 + +## Phase 4: Audit Logging Completion + +目标： + +- 把管理员与高权限操作真正做成企业级审计 + +工作项： + +- 扩大审计覆盖面： + - 系统重启 + - 数据源启停 + - 调度规则变更 + - 配置变更 + - 人工触发采集 + - 删除/修改关键配置 +- 增加字段： + - actor + - target + - before / after + - request_id + - IP +- 审计页支持： + - 动作筛选 + - 操作者筛选 + - 时间筛选 + - 目标对象筛选 + +完成标准： + +- 所有高权限操作都能追到人、时间、对象、结果 + +## Phase 5: Log Console To Enterprise Observability UI + +目标： + +- 把当前“系统日志”页升级为真正的多层日志工作台 + +工作项： + +- 将页面拆为三个主视图： + 1. 运行日志 + 2. 关键事件 + 3. 审计日志 +- 运行日志视图： + - 保留大控制台 + - 支持来源、级别、日期、搜索 +- 关键事件视图： + - 列表化展示高价值事件 + - 支持聚合、状态、指纹、对象筛选 +- 审计视图： + - 列表化展示管理员动作 +- 增加详情抽屉： + - 原始 message + - context + - request_id + - related resource + - recommended action + +完成标准： + +- 日志页不再只是“终端文本窗口” +- 运维排障、历史追溯、审计留痕三者分层清晰 + +## Phase 6: Corrective Intelligence + +目标： + +- 让系统从“能看日志”进化到“能辅助修错” + +工作项： + +- 引入错误 fingerprint +- 对已知错误配置： + - 根因类型 + - 修复建议 + - runbook 链接 + - 推荐动作 +- 支持常见纠错动作： + - 重试采集任务 + - 重载配置 + - 跳转到对应模块/资源 + - 打开相关日志过滤视图 +- 高频错误支持聚合与静默窗口 + +完成标准： + +- 已知错误能给出明确建议 +- 运维不需要每次都从零猜 + +## Recommended Module Changes + +### Backend + +建议新增/增强的模块： + +- `backend/app/core/logging.py` + - 统一 logger 封装 + - formatter + - filter + - request/trace 注入 +- `backend/app/services/persistent_logs.py` + - 扩展字段 + - 统一持久化策略 +- `backend/app/services/system_logs.py` + - 逐步从“文本尾部查看器”升级为“运行日志聚合器” +- `backend/app/services/log_classification.py` + - 指纹 + - 根因分类 + - 纠错建议 +- `backend/app/api/v1/system_control.py` + - 补充事件 / 审计 / 日志多视图接口 + +### Frontend + +建议新增/增强： + +- `frontend/src/lib/logger.ts` + - 统一前端 logger API +- `frontend/src/pages/Logs/Logs.tsx` + - 升级为多层工作台 +- `frontend/public/earth/js/...` + - 各 Earth 模块接入统一事件 logger + +## Event Naming Convention + +建议采用： + +`...` + +示例： + +- `collector.datasource.run.started` +- `collector.datasource.run.failed` +- `earth.layer.cables.load.failed` +- `earth.cruise.route.build.failed` +- `system.restart_task.requested` +- `system.restart_task.completed` +- `auth.websocket.connect.failed` +- `settings.datasource.priority.updated` + +规则： + +- 不用自然语言句子 +- 不把 ID 塞进 event 名里 +- 资源对象通过字段承载，不通过 event 名承载 + +## Query Model + +最终推荐支持的查询维度： + +- 时间范围 +- level +- source +- service +- module +- event +- request_id +- trace_id +- user_id / actor +- resource_type / resource_id +- category +- fingerprint +- status +- full-text search + +## Retention Strategy + +推荐保留策略： + +- 运行日志： + - 文件 / 容器 / Redis 缓冲保留短周期 +- 持久化事件： + - 保留中长期 +- 审计日志： + - 长期保留 + +初版可以先这样： + +- 运行日志：7 到 14 天 +- 关键事件：90 到 180 天 +- 审计日志：180 天以上 + +后续再根据存储与合规要求调整。 + +## Security And Compliance + +必须落实： + +- 敏感字段脱敏 +- 前端上报白名单 +- 防止日志注入 +- 审计日志不可被普通管理员随意篡改 +- 高敏感纠错动作必须再次鉴权 + +## Success Criteria + +当下面这些条件成立时，才算这套日志系统真的“成了”： + +1. 一个后端请求失败时，能通过 `request_id` 在运行日志、持久化事件、审计日志之间串联查询 +2. 一个 Earth 前端错误能定位到页面、模块、事件名和上下文 +3. 一个采集器失败能同时看到运行日志、持久化事件和可执行纠错动作 +4. 一个管理员操作能查到操作者、目标对象、结果和 request_id +5. 日志页不再只是文本控制台，而是完整的“运行日志 / 关键事件 / 审计日志”工作台 +6. 高频已知错误能聚合并给出修复建议 + +## Delivery Order + +推荐严格按下面顺序做，不要乱跳： + +1. Phase 0 命名与字段规范冻结 +2. Phase 1 后端结构化 logging 基础 +3. Phase 2 高价值事件持久化 +4. Phase 3 前端 / Earth 统一 logger +5. Phase 4 审计覆盖补齐 +6. Phase 5 日志工作台 UI 重构 +7. Phase 6 指纹 / 纠错 / runbook + +原因： + +- 如果不先统一字段和命名，后面 UI 和持久化会越来越乱 +- 如果不先做后端结构化基础，前端上报再多也串不起来 +- 如果不先补持久化层，就只有“实时可看”，没有“历史可查” + +## First Actionable Milestone + +如果要从明天就开始做，最合理的第一个里程碑是： + +### M1: 让后端关键路径全部拥有统一结构化事件 + +范围： + +- API 请求入口/出口 +- scheduler +- collectors +- websocket +- visualization +- system control + +交付物： + +- 统一 logger helper +- 统一 event naming 表 +- 统一 request_id 注入 +- 统一脱敏策略 +- 关键模块替换完成 + +完成这个里程碑后，Planet 才算真正拥有了“企业级日志系统的地基”。 + diff --git a/docs/plans/frontend-markdown-renderer-plan.md b/docs/plans/frontend-markdown-renderer-plan.md new file mode 100644 index 00000000..2cea1a70 --- /dev/null +++ b/docs/plans/frontend-markdown-renderer-plan.md @@ -0,0 +1,97 @@ +# Markdown 渲染器完善计划 + +## 背景 + +Planet 控制台当前有三类主要 Markdown 使用场景： + +- 文档中心：技术文档、计划文档、运行手册。 +- AI Playground：模型回复、分析结果、代码片段。 +- BGP 简报：由系统生成并保存的态势报告。 + +这些场景都复用 `frontend/src/components/MarkdownRenderer/MarkdownRenderer.tsx`。因此 Markdown 能力应该集中在共享渲染器内完成，页面只负责传入内容、链接转换和布局约束，不能让每篇文档或每个页面手写复制按钮、表格样式、列表样式等交互细节。 + +## 目标 + +建设一个稳定、可复用、适合技术文档和 AI 输出的 Markdown 渲染器，优先覆盖常用语法、代码块操作和清晰的阅读样式，并为后续语法高亮、锚点导航、内容安全策略留出接口。 + +## 成功标准 + +- 代码块支持 fenced language、语言标签、复制按钮、复制成功状态和横向滚动。 +- 常用块语法稳定渲染：标题 1-6、段落、引用、分割线、表格、无序列表、有序列表、任务列表。 +- 常用行内语法稳定渲染：链接、自动链接、图片、行内代码、粗体、斜体、删除线。 +- 文档中心、AI Playground、BGP 简报继续复用同一个组件，不出现页面级重复实现。 +- 样式在普通业务面板和文档中心都有合理表现，文档中心可以通过 `.docs-markdown` 覆盖主题变量。 +- 前端 TypeScript build 通过，`git diff --check` 无空白错误。 + +## 当前实施范围 + +### 第一阶段：共享渲染器补齐 + +- 在 `MarkdownRenderer` 内解析 fenced code block 的语言信息。 +- 引入 `MarkdownCodeBlock` 子组件，负责语言标签、复制按钮和复制状态。 +- 保留现有 `Scrollbar` 横向滚动能力，避免长代码撑破页面。 +- 扩展标题渲染到 h1-h6，并保留 `getHeadingId` 对文档目录的支持。 +- 扩展列表解析，支持 `-`、`*`、`+`、`1.`、`1)` 和 GitHub 风格任务列表。 +- 扩展行内解析，支持图片、自动链接、删除线。 + +### 第二阶段：样式统一 + +- 全局 Markdown 样式覆盖业务场景，保持紧凑、清晰、可扫描。 +- 文档中心用 `.docs-markdown` 适配主题变量，避免硬编码颜色破坏明暗主题。 +- 代码块 toolbar 和 copy button 不依赖具体页面。 +- 图片默认响应式展示，避免超出内容区域。 + +### 第三阶段：验证 + +- 使用前端 build 验证 TypeScript 和 Vite 构建。 +- 使用 `git diff --check` 验证补丁格式。 +- 手动检查至少一个文档页中代码块复制按钮、语言标签和表格滚动是否出现。 + +## 后续增强项 + +### 语法高亮 + +当前不新增高亮依赖，避免一次性引入过重运行时代码。后续可以在以下方案中二选一： + +- `shiki`：适合文档中心，视觉质量高，但包体和初始化成本更高。 +- `highlight.js`：接入简单，覆盖语言广，但样式控制需要额外约束。 + +建议当文档代码块数量稳定增加后再引入，并做按需加载或懒加载。 + +### 更完整 CommonMark 支持 + +当前渲染器覆盖 Planet 常见内容，不追求完整 CommonMark 兼容。后续如果需要完整规范，建议切换到成熟生态： + +- `react-markdown` +- `remark-gfm` +- `rehype-sanitize` +- `rehype-slug` + +切换前需要评估：链接转换、目录 ID、现有样式、AI 输出安全策略和包体影响。 + +### 安全策略 + +目前渲染器不解析原始 HTML，这是正确默认值。后续如需支持 HTML，必须先明确： + +- 是否允许用户输入 Markdown。 +- 是否需要 HTML 白名单。 +- 是否需要 `rehype-sanitize`。 +- 图片和链接是否需要域名策略。 + +### 文档页能力 + +可继续补齐： + +- 标题锚点悬浮复制。 +- Mermaid 图表。 +- 代码块折叠。 +- 文档内搜索结果定位到代码块。 +- 复制按钮埋点，用于判断文档片段是否真正被使用。 + +## 维护约束 + +- Markdown 语法能力优先放在共享渲染器，不在具体文档页面散落实现。 +- 文档内容只表达内容，不承载 UI 行为。 +- 新增 Markdown 能力必须同时考虑文档中心、AI Playground、BGP 简报三个调用方。 +- 不解析原始 HTML，除非同步引入明确的 sanitize 策略。 +- 与主题相关的样式优先走页面容器变量覆盖，不在组件内写死文档中心颜色。 diff --git a/docs/plans/frontend-public-docs-site-plan.md b/docs/plans/frontend-public-docs-site-plan.md new file mode 100644 index 00000000..f26c6e73 --- /dev/null +++ b/docs/plans/frontend-public-docs-site-plan.md @@ -0,0 +1,486 @@ +# Frontend Public Docs Site Plan + +## 目标 + +新增一个公开访问的 `/docs` 页面，作为 Planet 的开发设计文档与使用手册入口。 + +这个页面应类似常见开源软件文档站： + +- 不需要登录即可访问 +- 与 `/earth` 和 admin 后台平级，但视觉和信息架构独立 +- 直接整理并展示仓库内 `docs/technical` 的 Markdown 文档 +- 支持搜索、分类导航、文档目录和内部跳转 +- 让 `docs/technical` 继续作为文档真源，避免页面内容和仓库文档漂移 + +## 非目标 + +本阶段不做： + +- 后端全文搜索服务 +- 数据库驱动的 CMS +- 独立文档构建系统，例如 Docusaurus / VitePress +- 每篇文档单独手写 React 页面 +- 用户权限、编辑器、在线保存或评论功能 +- 把 `docs/plans`、`docs/deprecated` 全量公开为正式手册 + +后续可以再决定是否把 plans / deprecated 做成独立的“路线图 / 历史归档”分区。 + +## 技术路线 + +### 推荐方案：Markdown 直接渲染 + +使用 Vite 在前端构建阶段直接加载 `docs/technical/**/*.md`： + +```ts +const modules = import.meta.glob('../../../docs/technical/**/*.md', { + query: '?raw', + import: 'default', +}) +``` + +这样每篇 Markdown 文件仍然留在仓库文档目录中，`/docs` 页面只是读取、索引和渲染这些文档。 + +当前项目已经满足主要前提： + +- 前端使用 Vite + React +- `frontend/vite.config.ts` 已配置 `server.fs.allow: ['..']` +- 已有 `MarkdownRenderer` 可作为基础 +- `docs/technical` 文档数量较少，前端本地搜索足够 + +### 不推荐方案：每篇文档单独写 React + +不建议把每篇文档重写成 `.tsx` 页面，因为： + +- 文档会出现两份真源 +- 修改技术文档时还要同步 UI 页面 +- 计划文档、技术上下文、变量表这类内容天然适合 Markdown +- 后续新增文档的成本会变高 + +只有当某篇文档需要强交互演示、实时图表或复杂 UI 时，才考虑给该文档补充一个 React 组件扩展。 + +## 信息架构 + +### 公开路由 + +新增： + +- `/docs` +- `/docs/:slug` + +路由行为： + +- `/docs` 默认打开 `docs/technical/README.md`，或打开人工指定的首页文档 +- `/docs/:slug` 打开对应技术文档 +- 未找到文档时显示 docs 专属 404，而不是跳回 admin +- `/docs` 加入 `App.tsx` 的公开路由白名单 + +### 文档分类 + +将 `docs/technical` 中的现有文档整理进以下分组： + +#### Overview + +- `README.md` + +#### Earth + +- `earth-frontend-context.md` +- `earth-layer-style-reference.md` +- `earth-render-layer-order.md` +- `earth-satellite-footprint-policy.md` +- `earth-bgp-context.md` +- `earth-news-live-streams-collector-format.md` + +#### Frontend + +- `frontend-admin-frontend-context.md` +- `frontend-layout-guidelines.md` + +#### Backend + +- `backend-collectors.md` +- `backend-system-service-control.md` + +#### Agents + +- `agents-aiprovider.md` + +#### Ops + +- `ops-docker-compose-buildx-upgrade.md` + +### 页面布局 + +桌面端： + +- 顶部：产品名、搜索框、当前文档标题 +- 左侧：文档分组导航 +- 中间：Markdown 正文 +- 右侧：当前文档目录，也就是 h2 / h3 anchors + +移动端： + +- 顶部固定搜索入口 +- 导航折叠为抽屉或下拉 +- 正文单列显示 +- 当前文档目录折叠为“本文目录” + +视觉风格： + +- 像开源软件 docs 页面，清晰、安静、可长时间阅读 +- 不复用 admin 后台的重操作感布局 +- 不做 Earth 的沉浸式深色 HUD 风格 +- 优先阅读性、扫描效率和代码/表格可读性 + +## 前端实现设计 + +### 文件结构 + +建议新增： + +```text +frontend/src/pages/Docs/ + Docs.tsx + docs-content.ts + docs-search.ts + docs-slugs.ts + Docs.css +``` + +可选拆分： + +```text +frontend/src/pages/Docs/components/ + DocsSidebar.tsx + DocsSearch.tsx + DocsToc.tsx + DocsMarkdown.tsx +``` + +如果初版代码量不大，可以先保持在 `Docs.tsx` + 少量 helper 文件中，避免过度拆分。 + +### 文档注册表 + +创建一个 registry，负责将 Markdown 文件路径映射为文档元信息： + +```ts +interface DocsEntry { + slug: string + path: string + title: string + group: string + order: number + loader: () => Promise +} +``` + +slug 规则： + +- `docs/technical/README.md` -> `overview` +- `docs/technical/earth-layer-style-reference.md` -> `earth-layer-style-reference` +- 只暴露稳定 slug，不暴露本机绝对路径 + +标题规则： + +- 优先读取 Markdown 第一个 `# heading` +- 没有 h1 时用人工 registry title +- 再 fallback 到文件名转换标题 + +### Markdown 渲染 + +初版可以复用现有： + +- [frontend/src/components/MarkdownRenderer/MarkdownRenderer.tsx](/home/ray/dev/linkong/planet/frontend/src/components/MarkdownRenderer/MarkdownRenderer.tsx) + +但建议增强或包装为 docs 专用渲染： + +- heading 生成稳定 `id` +- 右侧 TOC 使用同一套 heading 解析结果 +- 内部 Markdown 链接转换为 `/docs/:slug` +- 外部链接保留 `target="_blank" rel="noreferrer"` +- 表格横向滚动 +- 代码块保留等宽字体和语言标记 +- 支持 GitHub 风格的相对文档链接 + +内部链接转换示例： + +- `earth-render-layer-order.md` -> `/docs/earth-render-layer-order` +- `./earth-layer-style-reference.md` -> `/docs/earth-layer-style-reference` +- `/home/ray/dev/linkong/planet/docs/technical/foo.md` -> `/docs/foo` + +对非 `docs/technical` 的链接： + +- 初版可保留原始链接文本 +- 或显示为不可跳转的 repo path +- 后续再扩展为跨文档区导航 + +### 搜索 + +初版使用纯前端本地搜索。 + +索引字段： + +- title +- slug +- group +- headings +- markdown 正文纯文本 + +搜索策略： + +- 页面首次加载后异步加载所有 `docs/technical` Markdown +- 生成内存索引 +- 用户输入时本地过滤 +- 简单打分即可： + - 标题命中权重最高 + - heading 命中其次 + - 文件名 / slug 命中其次 + - 正文命中最低 + +搜索结果展示： + +- 文档标题 +- 分组 +- 命中的 heading 或正文摘要 +- 点击跳转到文档 + +当前只有 13 篇文档，不需要 Lunr、Fuse 或后端搜索。后续文档数量显著增长时，再考虑引入轻量搜索库。 + +### 路由接入 + +修改： + +- [frontend/src/App.tsx](/home/ray/dev/linkong/planet/frontend/src/App.tsx) + +新增 lazy import： + +```ts +const Docs = lazy(() => import('./pages/Docs/Docs')) +``` + +公开路由： + +```ts +const publicPaths = new Set(['/', '/earth', '/docs']) +``` + +注意：`/docs/:slug` 不能只用精确匹配 `Set`。 + +建议改为： + +```ts +const isPublicRoute = + window.location.pathname === '/' || + window.location.pathname === '/earth' || + window.location.pathname === '/docs' || + window.location.pathname.startsWith('/docs/') +``` + +新增 routes： + +```tsx +} /> +} /> +``` + +### 样式 + +建议独立 `Docs.css`，不依赖 admin 页面布局。 + +核心样式要求： + +- 文档正文最大宽度控制在适合阅读的范围 +- 表格横向滚动，不撑破布局 +- 代码块横向滚动 +- 左侧导航固定或 sticky +- 右侧 TOC sticky +- 移动端隐藏右侧 TOC，导航折叠 +- 搜索结果浮层或独立面板不遮挡正文阅读 + +注意： + +- 不做营销 hero +- 不做卡片堆叠式首页 +- 首页第一屏应直接是文档入口和内容，而不是宣传页 + +## 实施阶段 + +### Phase 1：基础文档站 + +目标： + +- `/docs` 可公开访问 +- 能看到 `docs/technical` 文档列表 +- 能打开每篇 Markdown +- 能基本渲染标题、段落、列表、代码块、表格 + +任务： + +- 新增 `Docs` 页面 +- 新增 docs registry +- 接入 Vite raw Markdown loading +- 接入 `/docs` 和 `/docs/:slug` +- 加入公开路由白名单 +- 初版 CSS 布局 + +验收： + +- 未登录访问 `/docs` 不跳转登录 +- `/docs/earth-layer-style-reference` 可打开样式参考文档 +- `/docs/backend-collectors` 可打开后端采集器文档 +- 构建通过：`source ~/.zshrc && bun run build` + +### Phase 2：搜索与 TOC + +目标： + +- 支持本地搜索所有 technical 文档 +- 当前文档右侧显示目录 +- 搜索结果可跳转 + +任务： + +- 实现 heading parser +- 实现 TOC 组件 +- 实现 search index +- 搜索结果显示文档标题、分组和摘要 +- 当前文档标题与 active nav 高亮 + +验收： + +- 搜索 `Fresnel` 能找到 Earth 图层样式文档 +- 搜索 `collector` 能找到 backend collectors +- 点击搜索结果进入对应文档 +- 右侧 TOC 点击后滚动到对应 heading + +### Phase 3：链接清理与文档体验 + +目标： + +- Markdown 内部链接在 docs 站内自然跳转 +- 长表格、代码块、绝对路径链接的显示更友好 + +任务： + +- 转换 `docs/technical/*.md` 相对链接 +- 转换 repo 内 technical 文档绝对路径 +- 外链新窗口打开 +- 文件路径链接以代码样式显示 +- 增强空状态和 404 + +验收： + +- 从 `docs/technical/README.md` 点击 technical 文档链接进入 `/docs/:slug` +- 不支持的 repo 内路径不会导致前端崩溃 +- 外部链接行为正常 + +### Phase 4：文档内容整理 + +目标： + +- `docs/technical` 的首页适合作为公开手册入口 +- 每篇文档标题、摘要和分类清晰 + +任务： + +- 检查每篇文档是否有唯一 h1 +- 给 README 补公开手册导览 +- 必要时补文档摘要 +- 保持文档内容仍然服务开发维护，不改成营销语气 + +验收： + +- `/docs` 首页能说明各技术文档用途 +- 左侧分类和 README 内容一致 +- 没有明显重复、过期或找不到的主入口 + +## 需要改动的文件 + +预计新增： + +- `frontend/src/pages/Docs/Docs.tsx` +- `frontend/src/pages/Docs/Docs.css` +- `frontend/src/pages/Docs/docs-content.ts` +- `frontend/src/pages/Docs/docs-search.ts` + +预计修改： + +- `frontend/src/App.tsx` +- `frontend/src/components/MarkdownRenderer/MarkdownRenderer.tsx` 或新增 docs 专用 wrapper +- `docs/technical/README.md` + +可选修改： + +- `frontend/src/index.css`，只放全局极少量 docs shell reset 时才需要 +- `docs/CHANGELOG.md`，实施完成后记录 +- `docs/version-history.md`，若进入版本发布流程再更新 + +## 风险与注意事项 + +### 构建路径风险 + +Vite 从 `frontend/src` 读取 `../../../docs/technical/**/*.md` 时，需要确认开发和生产构建都可解析。 + +缓解： + +- 使用相对路径 glob +- 构建验证必须跑 `source ~/.zshrc && bun run build` +- 不使用运行时 `fetch('/docs/...')` 读取仓库文件，避免生产环境缺文件 + +### Markdown 能力不足 + +现有 `MarkdownRenderer` 是轻量实现，可能不完整支持所有 GitHub Markdown。 + +缓解： + +- 初版优先覆盖当前 `docs/technical` 实际用到的语法 +- 若后续需要脚注、嵌套列表、复杂代码高亮，再考虑引入 `react-markdown` 等依赖 + +### Bundle 体积 + +把所有 Markdown 打进前端 bundle 会增加体积。 + +当前文档数量少，风险可接受。 + +缓解： + +- 使用 lazy page chunk +- Markdown loader 保持异步 +- 搜索索引在 `/docs` 页面内初始化，不影响 `/earth` 和 admin 首屏 + +### 公开内容边界 + +`docs/technical` 会被公开展示，需要避免包含密钥、内部机器地址、临时方案或不应公开的操作细节。 + +缓解： + +- 实施前快速审阅 `docs/technical` +- 暂不公开 `docs/plans` 和 `docs/deprecated` +- 以后如需公开更多文档，先建立 allowlist + +## 验收清单 + +- `/docs` 未登录可访问 +- `/docs/:slug` 未登录可访问 +- `/docs` 不影响 `/earth` +- 未登录访问 admin 仍然跳登录 +- 左侧导航包含所有 `docs/technical` 文档 +- 文档按 Overview / Earth / Frontend / Backend / Agents / Ops 分类 +- Markdown 表格正常显示并可横向滚动 +- 代码块正常显示并可横向滚动 +- 搜索可搜索标题、heading 和正文 +- 搜索结果点击可跳转 +- 当前文档 TOC 可跳转 +- 不存在的 slug 显示 docs 404 +- `source ~/.zshrc && bun run build` 通过 + +## 后续增强 + +- 给文档页面增加复制 heading 链接按钮 +- 给代码块增加复制按钮 +- 增加“上一页 / 下一页”导航 +- 增加最近更新信息 +- 从 git metadata 读取文档更新时间 +- 引入轻量全文搜索库 +- 支持 plans / deprecated 独立分区 +- 增加页面内反馈入口 diff --git a/docs/plans/location-resolver-shared-pipeline-plan.md b/docs/plans/location-resolver-shared-pipeline-plan.md new file mode 100644 index 00000000..d54caa4e --- /dev/null +++ b/docs/plans/location-resolver-shared-pipeline-plan.md @@ -0,0 +1,127 @@ +# Location Resolver Shared Pipeline Plan + +**状态**：已实现，当前用户流程见 [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md)，开发接口见 [通用位置估算管线开发说明](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-development.md)。 + +## Goal + +把"给定一条记录，决定它的 lat/lon"这件事抽象成一条统一的可插拔管线，让算力中心、BGP 观测站、BGP 事件——以及未来任何需要位置估算的实体——共用同一套接口。新算法（peeringdb 设施查询、IXP 表、用户认领的精确点位等）通过实现一个 Resolver 类即可挂入，不需要改任何上层调用方。 + +## Background + +### 实施前现状 + +- **算力中心** (`backend/app/services/compute_center_locations.py`) 早期曾使用源坐标 → 本地 JSON 注册表 → 城市兜底 → Nominatim 在线地理编码。后续为避免硬编码位置污染事实链路，算力中心本地注册表已移除；主地图只使用源坐标，手动候选采集使用 ROR 和 Nominatim。 +- **BGP 观测站** (`collectors/bgp_common.py:RIPE_RIS_COLLECTOR_COORDS`) 是一张写死的字典，26 个 RIPE RIS collector 的城市级坐标。新增 collector / 升级到设施级精度都得改 Python。 +- **BGP 事件**继承所属 collector 的城市级坐标（`BGPObservation.collector_geo`）。 +- 用户原本以为 BGP 观测站位置是通过 iptoasn 推断的——其实 iptoasn 只用于前缀级国家归属（`bgp_enrichment.py`），不影响 marker 坐标。 + +### 痛点 + +1. 算力中心那条 4 层链路写死在算力中心模块里，BGP 想用得复制一遍。 +2. 三类实体各走各的坐标策略，缺统一抽象。 +3. 未来要插更精的算法（peeringdb / IXP / 用户认领），现在没有挂入点。 + +## Design + +### 接口契约 + +`backend/app/services/location/`： + +- `models.py` —— `LocationQuery`（输入）、`LocationCandidate`（候选）、`ResolverOutput`（单 resolver 输出）、`ResolutionResult`/`ResolutionDiagnostic`（管线最终结果） +- `pipeline.py` —— `LocationResolver` Protocol、`LocationPipeline` 编排器 +- `resolvers/source_coordinates.py` —— 记录自带 lat/lon 时直通 +- `resolvers/registry.py` —— 本地 JSON 注册表（locations + city_fallbacks），按别名得分 +- `resolvers/nominatim.py` —— 通用 Nominatim 客户端（rate-limited + LRU 缓存）+ 可注入 query plan +- `resolvers/inherit.py` —— 从外部回调取候选（事件继承 collector 用） +- `text.py` —— 文本规范化共享工具 + +核心 Protocol： + +```python +class LocationResolver(Protocol): + name: str + def resolve(self, query: LocationQuery) -> ResolverOutput: ... +``` + +`LocationPipeline.collect_candidates()` 跑全部 resolver，聚合所有候选，按 `(source_rank, precision_rank, -confidence)` 排序去重；`resolve_best()` 选 top 候选。 + +### 各领域管线 + +```python +# compute_center_locations.py（重构后，公共 API 不变） +COMPUTE_CENTER_PIPELINE = LocationPipeline([ + SourceCoordinatesResolver(), +]) + +COMPUTE_CENTER_COLLECTION_PIPELINE = LocationPipeline([ + SourceCoordinatesResolver(), + ROROrganizationResolver(), + NominatimResolver(query_plan_builder=_compute_center_query_plan, + geocoder=lambda q: _geocode_online(q)), +]) + +# bgp_collector_locations.py（新） +BGP_COLLECTOR_PIPELINE = LocationPipeline([ + SourceCoordinatesResolver(), + StoredCollectorLocationResolver(), +]) + +BGP_COLLECTOR_COLLECTION_PIPELINE = LocationPipeline([ + SourceCoordinatesResolver(), + NominatimResolver(query_plan_builder=_bgp_collector_query_plan, + geocoder=lambda q: _geocode_online(q)), +]) + +# bgp_event_locations.py（新） +BGP_EVENT_PIPELINE = LocationPipeline([ + SourceCoordinatesResolver(), + InheritFromAnotherEntityResolver(source_lookup=_inherit_from_owning_collector), + # 占位：将来插 ASNFacilityResolver / PrefixGeoResolver +]) +``` + +### 关键设计决策 + +1. **算力中心公共 API 完全不变**：`resolve_compute_center_location()`、`collect_location_candidates()`、`ComputeCenterLocation` dataclass、`_geocode_online` 模块级符号都保留，前端 / 上层调用方零改动；现有 19 个回归测试全绿。 +2. **`_geocode_online` 用 lambda 晚绑定**：`NominatimResolver(geocoder=lambda q: _geocode_online(q))` 能让测试 `monkeypatch.setattr(module, "_geocode_online", fake)` 继续生效。 +3. **`RIPE_RIS_COLLECTOR_COORDS` 自动从 DB-backed cache 重建**：启动时 seed/refresh `bgp_collector_locations` 维表，再原地刷新旧 `{rrcXX → {city, country, lat, lon}}` 字典。下游消费者（`bgp_collectors.py`、序列化、detector）不动即可获得新元数据。 +4. **修复隐藏 bug**：BGP collector 不再通过 registry/operator 模糊匹配晋升候选，避免 `operator="RIPE NCC"` 让每个事件都落到 `rrc00`。 +5. **事件继承走严格名字查询**：事件继承不跑 collector 的完整 pipeline，改成直接查 DB-backed cache。"改进位置"用户触发流程只跑源坐标和在线地理编码候选。 + +## Files + +### 新增 +- `backend/app/services/location/__init__.py` +- `backend/app/services/location/models.py` +- `backend/app/services/location/pipeline.py` +- `backend/app/services/location/text.py` +- `backend/app/services/location/resolvers/__init__.py` +- `backend/app/services/location/resolvers/source_coordinates.py` +- `backend/app/services/location/resolvers/registry.py` +- `backend/app/services/location/resolvers/nominatim.py` +- `backend/app/services/location/resolvers/inherit.py` +- `backend/app/services/bgp_collector_locations.py` +- `backend/app/services/bgp_event_locations.py` +- `backend/app/models/bgp_collector_location.py` +- `backend/tests/test_location_pipeline.py`（16 用例） +- `backend/tests/test_bgp_collector_locations.py`（11 用例） + +### 修改 +- `backend/app/services/compute_center_locations.py` —— 改为薄包装 +- `backend/app/services/collectors/bgp_common.py` —— 删除写死字典，改调 `resolve_bgp_event_geo_dict()` +- `backend/app/api/v1/bgp.py` —— 新增 `POST /api/v1/bgp/collectors/{collector_id}/collect-location` +- `frontend/public/earth/js/info-card.js` —— `renderComputeCenterCollectSection` → `renderLocationCollectSection`，BGP collector 走通用化路径 +- `frontend/public/earth/js/compute-centers.js` —— 新增通用 `collectLocationCandidates(endpoint, payload)` +- `frontend/public/earth/js/main.js` —— `previewComputeCenterCandidate` → `previewLocationCandidate`，事件名改为 `earth:preview-location-candidate` + +## Verification + +- `uv run pytest backend/tests/test_visualization_compute_centers.py` —— 19 个用例全绿（公共 API 未改） +- `uv run pytest backend/tests/test_location_pipeline.py backend/tests/test_bgp_collector_locations.py` —— 16 + 11 用例全绿 +- 抽象可插拔性测试：`test_pluggability_custom_resolver_works_without_changing_pipeline` —— 临时实现 `_PeeringDBStubResolver` 直接接入 `LocationPipeline`，验证管线不需要改一行就能识别新 source + +## Out of scope + +- 持久化用户认领的精确坐标（写回 JSON 注册表）—— `suggested_registry_entry` 字段已就绪，工作流单独立项 +- 真正实现 `ASNFacilityResolver` / `PrefixGeoResolver` —— 接口已留好，具体算法（peeringdb / IXP 表 / iptoasn 升级）单独立项 +- 算力中心 / 观测站 marker 合并避让 —— 上一轮已用 `SURFACE_AVOIDANCE_PROFILES.city` + halo 收敛解决 diff --git a/docs/technical/README.md b/docs/technical/README.md deleted file mode 100644 index a8a585de..00000000 --- a/docs/technical/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Technical Docs - -这里放“当前实现和当前结构”的文档，重点回答： - -- 现在代码是怎么组织的 -- 当前入口在哪 -- 状态和组件如何工作 -- 后续改动应该沿着哪条实现边界继续走 - -适合放入这里的内容： - -- 前端上下文 -- Earth 前端结构 -- 后端运行控制 -- collector 现状 -- 采集格式约定 - -不适合放入这里的内容： - -- 尚未完成的 roadmap -- 未来迭代方案 -- 大范围重构计划 - -这些应放入： - -- [docs/plans/README.md](/home/ray/dev/linkong/planet/docs/plans/README.md) diff --git a/docs/technical/earth-frontend-context.md b/docs/technical/earth-frontend-context.md deleted file mode 100644 index b2d6c5e9..00000000 --- a/docs/technical/earth-frontend-context.md +++ /dev/null @@ -1,381 +0,0 @@ -# Earth Frontend Context - -本文件描述当前 Earth 大屏前端的真实结构，重点是帮助后续继续改 HUD、图层、媒体面板、真实地形、BGP 可视化时，不再重复踩结构和状态同步上的坑。 - -相关规则建议一起参考： - -- [rules.md](/home/ray/dev/linkong/planet/rules.md) -- [frontend-layout-guidelines.md](/home/ray/dev/linkong/planet/docs/technical/frontend-layout-guidelines.md) - -## 当前目标 - -Earth 前端不是普通管理页，它是独立的大屏展示前端。当前产品目标是： - -- 维持地球视图的空间感和可读性 -- 让 HUD、图层、媒体面板、BGP、卫星、海缆等保持统一交互 -- 把加载中、已启用、已隐藏、锁定中这类状态做清楚 - -## 当前入口 - -React 路由入口： - -- [Earth.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Earth/Earth.tsx) - -当前做法很简单： - -- React 页面只负责提供一个全屏 `iframe` -- 真正的 Earth 应用运行在： - - [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) - -所以 Earth 前端本质上是 `public/earth` 下的一套独立静态应用。 - -## 当前文件分层 - -### 1. 页面入口与结构 - -- [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) - -职责： - -- HUD 基础 DOM -- 图层面板 -- 媒体面板 -- 工具栏 -- 设置弹窗 -- 兼容旧元素 id - -### 2. 主运行时 - -- [main.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/main.js) - -职责： - -- 地球初始化 -- Three.js 场景组装 -- 数据加载与刷新 -- 各图层集成 -- Earth 级别状态同步 - -### 3. 地球控制层 - -- [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) - -职责： - -- 工具栏交互 -- 图层面板交互 -- 旋转/缩放/布局 -- HUD 面板拖拽 -- 图层开关状态机 -- Earth 设置读取、持久化与重置 - -这份文件是 Earth 前端当前最核心的 UI 控制入口。 - -### 4. UI 与状态消息 - -- [ui.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/ui.js) - -职责： - -- loading 面板 -- status message -- tooltip / error / 清理逻辑 - -### 5. 地球与地形 - -- [earth.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/earth.js) -- [terrain.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/terrain.js) - -职责： - -- 地球球体、云层、大气 -- 真实地形 mesh -- terrain tile 拉取、解码、位移、着色 - -### 6. 图层模块 - -- [satellites.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/satellites.js) -- [cables.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/cables.js) -- [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) -- [bgp-cruise-adapter.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp-cruise-adapter.js) -- [news.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/news.js) -- [tv.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/tv.js) -- [layer-startup-tasks.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/layer-startup-tasks.js) -- [cruise-sequencer.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/cruise-sequencer.js) -- [callout-connector.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/callout-connector.js) - -职责： - -- 各自的数据层 -- 开关行为 -- 面板内容 -- hover/lock/selection 语义 - -其中 Earth 启动加载链现在也拆成了两层： - -- `controls.js` - - 提供图层注册表与启动元信息 -- `layer-startup-tasks.js` - - 提供图层启动任务注册表 - - 通过 `registerLayerStartupTask(id, taskFactory)` 扩展启动任务 -- `main.js` - - 只负责读取排序后的启动图层，再按映射执行队列 - -其中巡航模式现在已经拆成两层： - -- `cruise-sequencer.js` - - 负责目标队列顺序、停留时长、切换节奏、打断与恢复 -- `callout-connector.js` - - 负责卡片连线 SVG、路径计算与绘制动画 -- `bgp-cruise-adapter.js` - - 负责 BGP 巡航展示适配：目标排序、卡片落点、连线路径、focus/overlay/info-card 时序 - -当前 BGP 巡航只是这套能力的一个调用方，不应再把“按队列巡航”和“BGP 事件展示”混写在同一个状态机里。 - -## 当前样式分层 - -Earth 的 CSS 不是一份大样式表，而是分层管理： - -- [base.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/base.css) -- [hud.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/hud.css) -- [toolbar.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/toolbar.css) -- [layer-panel.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/layer-panel.css) -- [info-panel.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/info-panel.css) -- [legend.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/legend.css) -- [earth-stats.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/earth-stats.css) -- [coordinates-display.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/coordinates-display.css) -- [tv-panel.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/tv-panel.css) - -当前建议： - -- 通用 HUD 壳层写进 `hud.css` -- 单一面板特性写进各自子文件 -- 不要把业务状态样式再散回 `index.html` - -## 当前图层开关状态语义 - -Earth 图层按钮现在不应再只有“开/关”两态，而应支持： - -- `inactive` -- `active` -- `loading` - -当前入口在： - -- [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) -- [layer-button-state.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/layer-button-state.js) - -关键函数： - -- `updateLayerButtonState(button, isActive)` -- `setLayerButtonState(button, options)` - -`setLayerButtonState` 负责： - -- `loading` 样式 -- `aria-busy` -- 按钮禁用 -- tooltip 更新 -- 绑定状态文本更新 -- 可选同步 `active` - -因此后续如果别的图层也需要异步启用，应该直接走这套状态机，而不是再手写一套临时 loading class。 - -另外，Earth 图层控制现在已经收成“注册表驱动”： - -- 图层元数据 - - `id` - - `icon` - - `label` - - `meta` - - `buttonId` - - `persist` - - `startupPriority` - - `startupMode` - - `startupLabel` - - `startupMessage` -- 图层行为 - - `getVisible()` - - `setVisible(next, options)` - -当前入口仍在 [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js)。 - -这意味着后续新增图层时，优先应补一条图层注册定义，而不是同时去改： - -- 图层面板 HTML -- 持久化快照 -- 初始化恢复 -- click 绑定 - -这四处现在都应该由注册表派生。 - -其中： - -- `startupPriority` - - 描述图层参与启动加载时的顺序 -- `startupMode` - - `visible` - - 仅当前图层处于启用/可见状态时，才加入启动加载队列 - - `preload` - - 即使当前图层未显示，也会参与启动预加载 - -当前 `main.js` 会通过注册表读取排序后的启动图层列表，再动态拼装启动加载队列，而不是手写一串固定步骤。像 BGP 这类需要尽早准备数据、但不一定默认显示的图层，应该优先走 `startupMode: "preload"`，而不是在启动流程里写隐式特判。 - -此外，启动阶段给用户看的提示文案也应尽量从注册表派生： - -- `startupLabel` - - 用于描述当前启动任务的业务名称 -- `startupMessage` - - 用于描述启动中的提示文案 - - 可以是字符串 - - 也可以是对象，用于像海缆这种“准备阶段 / 主加载阶段”两段式文案 - -这样后续新增会参与启动加载的图层时，顺序、模式和提示文案都在同一处定义，不需要再去 `main.js` 里补第二套常量。 - -### `data-status-target` - -图层按钮可以通过： - -- `data-status-target` - -指向一个状态文本节点。当前 terrain 已接入： - -- 按钮：`#toggle-terrain` -- 状态节点：`#terrain-status` - -以后别的异步图层也可以沿用这套约定。 - -## 当前设置持久化 - -Earth 设置面板当前由 [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) 统一负责： - -- 捕获默认值 -- 从 `localStorage` 读取上次设置 -- 初始化应用当前设置 -- 用户变更后即时持久化 -- 一键重置回默认值 - -当前持久化的范围是： - -- 旋转模式 -- 地球默认大小（作为重置视角、缩放重置和巡航视图的默认 zoom 真源） -- HUD 面板显示/隐藏 -- 图层控制开关：`地形 / 卫星 / 轨迹 / 海缆 / BGP` -- 地形透明度 - -也就是说，Earth 设置不是一次性 UI 状态了，而是本地设备级偏好。后续如果再加入新的设置项，应优先接入同一条持久化链，而不是各自散着写 `localStorage`。 - -## 当前地形链路 - -真实地形首次启用会慢，原因不只是一个： - -1. 需要拉取 Terrarium 瓦片 -2. 需要解码图片 -3. 需要按顶点采样高程 -4. 需要重新写入 geometry 和 color -5. 需要重新计算法线与包围体 - -当前入口在： - -- [terrain.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/terrain.js) - -当前已经做了两层体验优化： - -1. 图层开关 loading 状态持续可见 -2. 页面空闲时会预热 `ensureTerrainReady()` - -也就是说，后续再继续优化 terrain 时，优先顺序应该是： - -1. 先保证用户感知正确 -2. 再压缩首次等待 -3. 最后才做更激进的几何/瓦片优化 - -## 当前高频风险点 - -### 1. 视觉状态和业务状态不同步 - -Earth 里最常见的 bug 不是“没渲染”，而是： - -- 图层关了，tooltip 还在 -- 锁定对象隐藏了，info card 还在 -- legend 没跟图层切换 -- loading 已结束，但按钮还像没开 - -后续改动必须优先检查状态同步。 - -### 2. HUD 布局问题先查结构，不要先打 CSS 补丁 - -Earth HUD 历史上反复出现： - -- 面板只剩一条缝 -- markdown 被裁掉 -- tabs/iframe 被 `overflow: hidden` 吃掉 - -优先检查： - -1. 谁负责高度 -2. 谁负责滚动 -3. 哪一层在裁剪 - -不要上来先加 `overflow: hidden` 或额外包装层。 - -### 3. Transitional path 必须收口 - -Earth 已经经历过多轮 HUD、toolbar、media panel 重构，所以最容易积累： - -- 旧 helper -- 旧 class -- 旧 fallback 逻辑 -- 已废弃变体 - -每次大功能完成后，都要做一次 cleanup pass。 - -### 4. 巡航与业务事件不要再深度耦合 - -当前正确边界应该是： - -- 通用巡航层只知道： - - 当前目标 - - 队列顺序 - - 相机 focus - - 停留 / 隐藏 / 切换 -- 业务模块只负责： - - 提供目标队列 - - 提供 focus 坐标 - - 提供卡片内容 - - 提供高亮/图层副作用 - -如果以后再给海缆、卫星或新闻做巡航，不应复制一套新的 `main.js` 状态变量，而应复用： - -- [cruise-sequencer.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/cruise-sequencer.js) -- [callout-connector.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/callout-connector.js) -- [bgp-cruise-adapter.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp-cruise-adapter.js) 这种业务适配层模式 - -## 当前推荐改动方式 - -如果后续继续改 Earth，建议按这个顺序： - -1. 先确认改的是： - - Three.js 渲染层 - - HUD 结构层 - - 图层状态层 - - 面板内容层 -2. 如果涉及图层按钮，优先接入统一状态机 -3. 如果涉及可见性切换，检查 tooltip / legend / info-card / lock 是否一起收口 -4. 如果涉及面板布局，先查结构再动 CSS - -## 当前与控制台前端的边界 - -Earth 前端和控制台前端不是同一套 UI 系统： - -- 控制台前端：React + Ant Design 工作台 -- Earth 前端：`public/earth` 原生 HUD + Three.js 展示面 - -因此： - -- Earth 不应该直接复用 Ant Table / AppLayout 语义 -- 控制台也不应该照搬 Earth HUD 动画和玻璃层语言 - -控制台相关结构见： - -- [admin-frontend-context.md](/home/ray/dev/linkong/planet/docs/technical/frontend-admin-frontend-context.md) diff --git a/docs/technical/en/README.md b/docs/technical/en/README.md new file mode 100644 index 00000000..06b9065b --- /dev/null +++ b/docs/technical/en/README.md @@ -0,0 +1,43 @@ +# Technical Docs + +This directory holds "current implementation and current structure" documentation, focusing on: + +- How the code is organized right now +- Where the current entry points are +- How state and components work +- Which implementation boundaries future changes should follow + +What belongs here: + +- Quickstart and user manual +- Frontend context +- Earth frontend structure +- Earth satellite footprint policy +- Earth render layer order +- Earth layer style property index +- Backend runtime control +- Collector status +- Collector settings and connectivity validation +- Earth Interactable integration +- Collection format conventions + +## Entry Points + +- [Quickstart](/home/ray/dev/linkong/planet/docs/technical/en/quickstart.md): The shortest path to getting Planet running from scratch +- [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md): Complete usage guide for the console, `planet.sh`, Earth, and Docs +- [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md): Collect and preview coordinate candidates for compute centers and BGP collectors on Earth +- [Collector Settings and Connectivity Validation](/home/ray/dev/linkong/planet/docs/technical/en/datasource-collector-settings-connectivity.md): Data source catalog, collector settings, connectivity validation, and BarentsWatch credentials +- [Shared Location Resolution Pipeline Development Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-development.md): Backend location resolver / pipeline interfaces, registries, and extension points +- [Docs Gatekeeper Development Guide](/home/ray/dev/linkong/planet/docs/technical/en/docs-gatekeeper-development.md): Backend Docs catalog, Markdown content loading, and Gatekeeper permission groups +- [Earth Interactable Usage](/home/ray/dev/linkong/planet/docs/technical/en/earth-interactable-usage.md): API, lifecycle, and integration examples for Earth surface icon Interactable +- [Earth Toolbar and Overlay Coordination](/home/ray/dev/linkong/planet/docs/technical/en/earth-toolbar-overlay-coordination.md): Closing matrix and integration rules for toolbar buttons, search, settings, news, and layer overlays + +What does not belong here: + +- Incomplete roadmaps +- Future iteration plans +- Large-scale refactor proposals + +Those belong in: + +- [Plans Index](/home/ray/dev/linkong/planet/docs/plans/README.md) diff --git a/docs/technical/agents-aiprovider.md b/docs/technical/en/agents-aiprovider.md similarity index 100% rename from docs/technical/agents-aiprovider.md rename to docs/technical/en/agents-aiprovider.md diff --git a/docs/technical/en/backend-collectors.md b/docs/technical/en/backend-collectors.md new file mode 100644 index 00000000..edbea3b2 --- /dev/null +++ b/docs/technical/en/backend-collectors.md @@ -0,0 +1,393 @@ +# Data Collectors + +## I. System Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Data Collection Architecture │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ TOP500 │ │ Epoch AI │ │ HuggingFace │ │ +│ │ Collector │ │ Collector │ │ Collector │ │ +│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ +│ │ │ │ │ +│ └───────────────────┼───────────────────┘ │ +│ ▼ │ +│ ┌─────────────────────┐ │ +│ │ BaseCollector │◄── Base class (unified) │ +│ │ run() method │ │ +│ └─────────┬───────────┘ │ +│ │ │ +│ ┌─────────────────┼─────────────────┐ │ +│ ▼ ▼ ▼ │ +│ ┌───────────┐ ┌───────────┐ ┌───────────┐ │ +│ │ fetch() │ │transform()│ │ _save_data│ │ +│ │ raw data │ │ transform │ │ save to DB│ │ +│ └───────────┘ └───────────┘ └───────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────┐ │ +│ │ CollectedData table│◄── Unified storage │ +│ └─────────────────────┘ │ +│ │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ Scheduler (APScheduler) │ │ +│ │ Scheduled tasks: every 4h/6h/12h/1d auto-execute │ │ +│ └─────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## II. Pipeline + +```python +# 1. Scheduler triggers (scheduled or manual) +# ↓ + +# 2. run() executes the full pipeline +async def run(self, db): + # 2.1 Check if collector is enabled + if not collector_registry.is_active(self.name): + return {"status": "skipped"} + + # 2.2 Record task start + task = CollectionTask(status="running") + db.add(task) + await db.commit() + + # 2.3 FETCH — get raw data (implemented by subclass) + raw_data = await self.fetch() + + # 2.4 TRANSFORM — convert to unified format + data = self.transform(raw_data) + + # 2.5 SAVE — persist to database + records_count = await self._save_data(db, data) + + # 2.6 Record task completion + task.status = "success" + task.records_processed = records_count + await db.commit() +``` + +**Core file**: `backend/app/services/collectors/base.py` + +## III. Collector List + +| Collector | Data type | Content | Frequency | +|-----------|-----------|---------|-----------| +| TOP500 | supercomputer | Global supercomputer rankings (compute, performance) | 4 hours | +| Epoch AI | gpu_cluster | GPU compute cluster info | 6 hours | +| HuggingFace Models | model | AI model information | 12 hours | +| HuggingFace Datasets | dataset | Dataset information | 12 hours | +| HuggingFace Spaces | space | Demo applications | 1 day | +| PeeringDB | ixp/network/facility | Internet exchange points / networks / facilities | 1-2 days | +| TeleGeography | submarine_cable | Submarine cable information | 7 days | +| BarentsWatch AIS | vessel | AIS vessel positions, speed, heading, MMSI, and related fields | Collector settings | +| AISStream Vessels | vessel_ais | AIS WebSocket realtime stream, written to the raw observation layer and displayed through aggregation | Collector settings | + +AIS vessel collectors use a different persistence path from regular `CollectedData` collectors. BarentsWatch, AISStream, and custom `vessel_ais` sources write into the AIS raw observation layer first, then the aggregation service merges those observations into the GeoJSON and detail payloads used by the Earth vessel layer. This preserves source, transport, field conflicts, and observation time instead of letting one realtime source overwrite the final display table. + +TOP500 and Epoch AI compute sources do not always provide usable coordinates. The unified Earth compute-center endpoint uses only valid source-provided coordinates or `compute_center_locations` dimension-table coordinates during the main map startup path; records without coordinates are returned as `unresolved` instead of being rendered from a local registry, country centroid, or guessed city. When users manually collect candidates, the backend queries ROR and Nominatim/OpenStreetMap from source fields; accepted candidates are saved into `compute_center_locations` and rendered from that table on the next layer refresh. + +## IV. Data Format (stored in CollectedData table) + +```python +# Each collector's parse_response() return format +{ + "source_id": "top500_1", # Original system ID (required) + "name": "El Capitan", # Name (required) + "description": "System desc...", # Description + "country": "United States", # Country + "city": "Livermore, CA", # City + "latitude": "37.6819", # Latitude (string) + "longitude": "-121.7681", # Longitude (string) + "value": "1742.00", # Performance value (e.g. compute) + "unit": "PFlop/s", # Unit + "metadata": { # Extra data (JSON) + "rank": 1, + "r_peak": 2746.38, + "cores": 11039616 + }, + "reference_date": "2025-11-01" # Data reference date +} +``` + +## V. Database Schema + +**CollectedData table** (`collected_data`) + +| Field | Type | Description | +|-------|------|-------------| +| id | SERIAL | Primary key | +| source | VARCHAR(100) | Data source name (top500, huggingface, etc.) | +| source_id | VARCHAR(100) | Original data ID | +| data_type | VARCHAR(50) | Data type (supercomputer, model, etc.) | +| name | VARCHAR(500) | Name | +| title | VARCHAR(500) | Title | +| description | TEXT | Description | +| country | VARCHAR(100) | Country | +| city | VARCHAR(100) | City | +| latitude | VARCHAR(50) | Latitude | +| longitude | VARCHAR(50) | Longitude | +| value | VARCHAR(100) | Performance value | +| unit | VARCHAR(20) | Unit | +| metadata | JSONB | Extra metadata | +| collected_at | TIMESTAMP | Collection time | +| reference_date | TIMESTAMP | Data reference date | +| is_valid | INTEGER | Whether valid | + +**Core file**: `backend/app/models/collected_data.py` + +## VI. TOP500 Collector Example (full pipeline) + +```python +# 1. fetch() — get HTML from the web +async def fetch(self): + url = "https://top500.org/lists/top500/list/2025/11/" + response = await client.get(url) + return response.text # returns HTML + +# 2. parse_response() — parse HTML into unified format +def parse_response(self, html): + soup = BeautifulSoup(html, "html.parser") + table = soup.find("table") + + for row in table.find_all("tr")[1:]: # skip header + cells = row.find_all("td") + + entry = { + "source_id": f"top500_{cells[0].text}", + "name": cells[1].text.strip(), + "country": cells[2].text.strip(), + "city": "", + "latitude": "", + "longitude": "", + "value": "1742.00", + "unit": "PFlop/s", + "metadata": { + "rank": 1, + "cores": "11340000" + }, + "reference_date": "2025-11-01" + } + data.append(entry) + + return data + +# 3. run() automatically calls _save_data() to save to database +``` + +**Core file**: `backend/app/services/collectors/top500.py` + +## VII. Scheduler + +```python +# Register all collectors into scheduled tasks at startup +def start_scheduler(): + for name, collector in collectors.items(): + if collector_registry.is_active(name): + scheduler.add_job( + run_collector_task, + trigger=IntervalTrigger(hours=collector.frequency_hours), + id=name, + name=name + ) +``` + +| Collector | Frequency | +|-----------|-----------| +| TOP500 | Every 4 hours | +| Epoch AI | Every 6 hours | +| HuggingFace | Every 12 hours | +| PeeringDB | Every 1-2 days | +| TeleGeography | Every 7 days | + +**Core file**: `backend/app/services/scheduler.py` + +## VIII. Code Files + +``` +backend/app/services/collectors/ +├── base.py # Base class: run() pipeline, _save_data() persistence +├── registry.py # Collector registry +├── scheduler.py # Scheduled task dispatch (APScheduler) +├── top500.py # TOP500 collector +├── epoch_ai.py # Epoch AI collector +├── huggingface.py # HuggingFace collector +├── peeringdb.py # PeeringDB collector +├── telegeraphy.py # TeleGeography submarine cable collector +├── vessel_ais.py # BarentsWatch AIS vessel collector +└── aisstream.py # AISStream WebSocket vessel collector + +backend/app/services/ +├── custom_datasource_runtime.py # Custom REST / WebSocket mapping runtime +├── datasource_mapping.py # Deterministic field mapping and target writes +├── vessel_ais_aggregation.py # AIS raw observation writes and aggregate reads +├── vessel_aggregation_strategy.py # Multi-source field selection, freshness fallback, and conflict records +└── vessel_enrichment.py # Vessel profile enrichment cache + +backend/app/models/ +├── collected_data.py # Unified data model +└── vessel_enrichment.py # Vessel enrichment cache +``` + +## IX. Credentialed Collectors + +Some collectors require external service credentials: + +| Collector | Credential provider | Credential sources | +| --- | --- | --- | +| `barentswatch_vessels` | `barentswatch` | Console collector settings, environment variables, `~/.zshrc` | +| `aisstream_vessels` | `aisstream` | Console collector settings, environment variables, `~/.zshrc` for connectivity checks; save it in collector settings or inject it into the backend environment for collection | +| `spacetrack_tle` | `spacetrack` | Environment variables, `~/.zshrc` | + +### BarentsWatch AIS + +BarentsWatch AIS credential resolution is centralized in: + +- [barentswatch.py](/home/ray/dev/linkong/planet/backend/app/services/barentswatch.py) + +`VesselAISCollector` only collects and transforms AIS data. It no longer reads environment variables or builds token requests directly. It uses: + +- `resolve_barentswatch_config()` +- `fetch_barentswatch_access_token()` + +Resolution priority: + +1. `DataSourceConfig.auth_config` +2. `DataSourceConfig.config` +3. Environment variables +4. `~/.zshrc` + +Supported variables: + +```bash +export BARENTSWATCH_CLIENT_ID="..." +export BARENTSWATCH_CLIENT_SECRET="..." +``` + +Historical misspellings are also supported: + +```bash +export BARRENTSWATCH_CLIENT_ID="..." +export BARRENTSWATCH_CLIENT_SECRET="..." +``` + +Connectivity validation requests `https://id.barentswatch.no/connect/token` for an access token with `scope=ais`, then requests the AIS endpoint with `Authorization: Bearer `. + +### AISStream Realtime Vessels + +AISStream uses the `wss://stream.aisstream.io/v0/stream` WebSocket endpoint. Its default runtime is a long-lived realtime collector rather than the traditional REST pattern of one request, progress to 100%, then completion. + +Runtime configuration: + +- `api_key`: read first from `DataSourceConfig.auth_config.api_key` or `config.api_key`; it can also come from the backend process environment variable `AISSTREAM_API_KEY`. +- `bounding_boxes`: AISStream subscription bounds. The default example is global `[[[-90, -180], [90, 180]]]`; demos and production runs should usually start with a smaller area. +- `message_types`: defaults to `PositionReport` and `ShipStaticData`. +- `streaming_enabled`: enables long-lived streaming by default; disabling it falls back to batch-style `fetch -> transform -> save`. +- `streaming_max_messages`: test-only stop limit. Non-zero values stop the stream after the requested number of messages. +- `reconnect_delay_seconds` and `receive_timeout_seconds`: control reconnect delay and idle receive waits. + +State semantics: + +- `connecting`: connecting to AISStream. +- `streaming`: receiving realtime messages; `records_processed` means messages seen, usually without a fixed total or percentage. +- `reconnecting`: upstream or network interruption; the collector records `AISSourceHealth` and waits before reconnecting. +- `stopped` / `cancelled`: stopped by a test limit or user action. + +AISStream connectivity validation reads the saved collector configuration, environment variables, and `AISSTREAM_API_KEY` in `~/.zshrc` through `datasource_connectivity.py`. For actual collection, the most reliable path is saving the API key in `Settings -> Collector Settings -> AISStream Vessels`; if the key only lives in `~/.zshrc`, confirm that the backend process inherited it. + +### AIS Raw Observations And Aggregation + +AIS observations do not directly replace final vessel records. They are first saved as raw observations: + +- `source` records the origin, such as `barentswatch_vessels`, `aisstream_vessels`, or a custom source name. +- `delivery_mode` captures realtime quality; `realtime_stream` outranks `polling`. +- `transport` records `websocket` or `http`. +- Dynamic fields such as position, speed, and course are selected by freshness and source priority. +- Static fields prefer non-empty values; conflicting candidates are recorded for detail and diagnostics views. + +Earth still reads vessel data from: + +```http +GET /api/v1/visualization/geo/vessels +GET /api/v1/visualization/vessels/{mmsi} +GET /api/v1/visualization/vessels/{mmsi}/track +GET /api/v1/visualization/vessels/{mmsi}/conflicts +GET /api/v1/visualization/vessels/aggregation/diagnostics +``` + +`/geo/vessels` merges raw observation aggregation with the legacy BarentsWatch latest-position tables so adding AISStream does not hide historical BarentsWatch-only vessels. + +## X. Collector Settings And Connectivity Validation + +The console "Collector Settings" page owns endpoint, headers, timeouts, retries, and credentials for all built-in collectors. Connectivity is derived by the backend checksum rather than by frontend button styling: + +- endpoint +- auth type +- headers +- config +- credential provider +- credential fingerprint + +Related APIs: + +```http +GET /api/v1/datasources/configs/all +POST /api/v1/datasources/configs/builtin/connection-status +POST /api/v1/datasources/configs/builtin/connect +POST /api/v1/settings/integrations/barentswatch/connect +GET /api/v1/settings/credential-guides/{provider} +POST /api/v1/settings/credential-guides/{provider}/generate +POST /api/v1/settings/credential-guides/{provider}/reset +``` + +See [Collector Settings and Connectivity Validation](/home/ray/dev/linkong/planet/docs/technical/en/datasource-collector-settings-connectivity.md) for the full flow. + +## XI. Data Usage + +Collected data ultimately: + +1. **Visualization** — displays supercomputers, GPU clusters, and submarine cables' geographic positions +2. **Situational analysis** — global compute distribution statistics and growth trends +3. **Alert system** — detects changes to important nodes + +## XII. Collector Registration + +Collectors are automatically registered at application startup: + +```python +# backend/app/services/collectors/__init__.py + +collector_registry.register(TOP500Collector()) +collector_registry.register(EpochAIGPUCollector()) +collector_registry.register(HuggingFaceModelCollector()) +collector_registry.register(HuggingFaceDatasetCollector()) +collector_registry.register(HuggingFaceSpacesCollector()) +collector_registry.register(PeeringDBIXPCollector()) +collector_registry.register(PeeringDBNetworkCollector()) +collector_registry.register(PeeringDBFacilityCollector()) +collector_registry.register(TeleGeographyCableCollector()) +collector_registry.register(TeleGeographyLandingPointCollector()) +collector_registry.register(TeleGeographyCableSystemCollector()) +``` + +**Core file**: `backend/app/services/collectors/registry.py` + +## XIII. Triggering Collection + +### Method 1: Scheduled + +At startup, APScheduler automatically creates scheduled tasks based on each collector's `frequency_hours` setting. + +### Method 2: Manual API trigger + +```bash +# Trigger TOP500 collection +curl -X POST http://localhost:8000/api/v1/datasources/1/trigger \ + -H "Authorization: Bearer " +``` + +**Core file**: `backend/app/api/v1/datasources.py` diff --git a/docs/technical/en/backend-datasources-api-performance.md b/docs/technical/en/backend-datasources-api-performance.md new file mode 100644 index 00000000..06b192db --- /dev/null +++ b/docs/technical/en/backend-datasources-api-performance.md @@ -0,0 +1,99 @@ +# DataSources List API Performance Optimization + +## Background + +`GET /api/v1/datasources` is the core API for the Data Sources page. Slow responses directly block page rendering. + +## Query Path Before Optimization + +`_load_datasource_list_context` used to run these queries sequentially: + +| Order | Function | Query | Bottleneck | +| --- | --- | --- | --- | +| 1 | `_load_latest_running_tasks` | `collection_tasks` window query; stale check depends on this result | Must be serial | +| 2 | `_load_latest_completed_tasks` | `collection_tasks` window query for latest completed tasks | Serial wait | +| 3 | `_load_datasource_data_counts` | `COUNT(*) GROUP BY source` on `collected_data` | Slow full-table scan | +| 4 | `_load_datasource_endpoint_overrides` | Simple `datasource_configs` SELECT | Serial wait | + +## Phase 1: Parallelization + +The independent queries 2, 3, and 4 were moved to `asyncio.gather` with separate sessions: + +```python +async def _fetch_completed(): + async with async_session_factory() as s: + return await _load_latest_completed_tasks(s, datasource_ids) + +async def _fetch_counts(): + async with async_session_factory() as s: + return await _load_datasource_data_counts(s, sources) + +async def _fetch_overrides(): + async with async_session_factory() as s: + return await _load_datasource_endpoint_overrides(s, sources) + +completed_tasks, data_counts, endpoint_overrides = await asyncio.gather( + _fetch_completed(), _fetch_counts(), _fetch_overrides(), +) +``` + +SQLAlchemy `AsyncSession` does not support concurrent use from multiple coroutines, so every parallel branch needs its own session. + +## Phase 2: Remove Heavy Queries + +### Remove `_load_datasource_data_counts` + +`data_count` was only used by the frontend to show an edge-case `(0 records)` hint in the latest collection column. It was not worth keeping a `COUNT(*) GROUP BY` full-table scan. + +- Frontend `(0 records)` display logic was removed. +- `data_count` was removed from the `BuiltInDataSource` interface. + +### Remove `_load_latest_completed_tasks` + +`last_status` and `last_run_at` are already written to the `DataSource` model when collectors finish, so the list endpoint no longer needs to join `collection_tasks`: + +```python +# Before: completed_tasks query required +last_run_at = datasource.last_run_at or (last_task.completed_at if last_task else None) +last_status = datasource.last_status or (last_task.status if last_task else None) + +# After: read model fields directly +last_run_at = datasource.last_run_at +last_status = datasource.last_status +``` + +`last_records_processed` was removed as well because it came from completed task rows and is not displayed in the list. + +## Query Path After Optimization + +```text +datasources SELECT -> required primary data +_load_latest_running_tasks -> required for running state and stale check +_load_datasource_endpoint_overrides -> required for endpoint overrides and collector settings display +``` + +The endpoint now runs three queries instead of five. The last two run sequentially because running tasks are needed for stale checks and endpoint overrides are lightweight. + +## Frontend `triggerDatasource` Double Refresh Fix + +`triggerDatasource` previously called `fetchData()` twice: + +```typescript +// Before +} else { + window.setTimeout(() => { fetchData() }, 800) +} +fetchData() + +// After: mutually exclusive +if (res.data.task_id) { + fetchData() +} else { + window.setTimeout(fetchData, 800) +} +``` + +## Related Files + +- [datasources.py](/home/ray/dev/linkong/planet/backend/app/api/v1/datasources.py): `_load_datasource_list_context`, `list_datasources` +- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx): `BuiltInDataSource`, `triggerDatasource` diff --git a/docs/technical/backend-system-service-control.md b/docs/technical/en/backend-system-service-control.md similarity index 100% rename from docs/technical/backend-system-service-control.md rename to docs/technical/en/backend-system-service-control.md diff --git a/docs/technical/en/datasource-collector-settings-connectivity.md b/docs/technical/en/datasource-collector-settings-connectivity.md new file mode 100644 index 00000000..40193768 --- /dev/null +++ b/docs/technical/en/datasource-collector-settings-connectivity.md @@ -0,0 +1,447 @@ +# Collector Settings and Connectivity Validation + +## Background + +The console now separates the "data source catalog" from "collector configuration": + +- `/datasources` + - Lists all data sources, including built-in and custom sources. + - Clicking a name only opens an information drawer. + - Focuses on status, manual collection, and running collection tasks. +- `/settings?tab=collector_credentials` + - Displays as "Collector Settings". + - Owns endpoint, headers, base parameters, and credentials. + - Every collector exposes a connection button for health checks. + +This reduces first-use confusion: API endpoints, headers, credentials, and custom source configuration all belong to collector settings instead of being scattered across the data source list and system settings. + +## User-Facing Rules + +Connection state is not a frontend styling state. The backend derives it from the current configuration checksum and previously validated records. + +A built-in collector is considered "connected" when either condition is true: + +- The current configuration has successfully collected data. +- The user clicked the connection button for the current configuration and backend validation succeeded. + +If endpoint, headers, base configuration, or credential fingerprint changes after the last successful validation, the state returns to "needs reconnection". + +## Frontend Entry Points + +### Data Source Catalog + +Files: + +- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx) +- [index.css](/home/ray/dev/linkong/planet/frontend/src/index.css) + +Current behavior: + +- Built-in and custom data sources are merged into a `UnifiedDataSource` list. +- The table only keeps view, collect, and status actions. +- Clicking the name opens a read-only drawer. +- The drawer shows: + - Whether the source is built in + - Whether it is enabled + - Module, priority, and frequency + - Endpoint + - Headers + - Base configuration + - Whether credentials are required +- When tasks are running, the top progress area shows a clickable `Collecting N` pill. +- Clicking `Collecting N` opens a task list modal with per-task progress. + +`data-source-bulk-toolbar__running-pill` is the styling entry point for the "Collecting" pill. It is aligned with other status tags, while hover treatment, arrow affordance, and blue outline indicate interactivity. + +### Collector Settings + +File: + +- [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx) + +Current behavior: + +- The `collector_credentials` tab is displayed as "Collector Settings". +- A select lists built-in collectors and supports maintaining custom supplemental sources that merge into built-in data. +- The only button beside the select is a plug icon for health checks. +- Status tags below the select show: + - `Credentials required` / `No credentials required` + - Module + - `Enabled` / `Disabled` + - `Unchecked` / `Available` / `Unavailable` + - Whether the endpoint is overridden +- Collectors that require credentials place the credential card above base configuration. +- Collectors without credentials only show base configuration. +- The AISStream collector uses WebSocket semantics: connecting, streaming, reconnecting, or stopped. It does not use a fixed completion percentage. +- Custom source editing lives in collector settings. The data source catalog keeps overview, run controls, and read-only drawers. + +The connection button uses an inline Tabler-style plug icon with `plug-connected` semantics, avoiding the older refresh icon for a connection action. + +## Backend APIs + +### Data Source Configuration List + +```http +GET /api/v1/datasources/configs/all +``` + +Returns a merged view of YAML default data sources and database overrides. This route must be declared before `/configs/{config_id}`; otherwise FastAPI treats `all` as a path parameter and returns 422. + +Returned fields include: + +- `name` +- `default_url` +- `endpoint` +- `is_overridden` +- `is_active` +- `source_type` +- `auth_type` +- `headers` +- `config` +- `config_id` +- `description` + +Before returning `config`, internal connectivity validation fields are removed so the frontend does not display validation metadata as user configuration. + +### Built-In Collector Connection Status + +```http +POST /api/v1/datasources/configs/builtin/connection-status +``` + +Purpose: + +- Accept a candidate configuration. +- Compute its checksum. +- Determine whether the current configuration is already connected. + +The current frontend mostly performs an immediate check through the connection button and does not strongly depend on this endpoint. It remains the backend basis for future save-button disabling and restoring initial page state. + +### Built-In Collector Connectivity Validation + +```http +POST /api/v1/datasources/configs/builtin/connect +``` + +Purpose: + +- Free collectors request the endpoint directly. +- Credentialed collectors go through their credential provider. +- Successful validation writes a system-level connection record. + +Successful responses include: + +- `success` +- `connected` +- `checksum` +- `stage` +- `message` +- `response_time_ms` +- `credential_provider` +- `credential_source` + +### BarentsWatch AIS Connectivity Validation + +```http +POST /api/v1/settings/integrations/barentswatch/connect +GET /api/v1/settings/integrations/barentswatch/connectivity +``` + +BarentsWatch uses separate endpoints because draft credentials must be validated before saving: + +- Use draft `client_id` / `client_secret` to fetch a token. +- Use that token to request the AIS endpoint. +- After success, write a built-in collector connection record using the draft credential fingerprint. + +## Connectivity Validation Service + +File: + +- [datasource_connectivity.py](/home/ray/dev/linkong/planet/backend/app/services/datasource_connectivity.py) + +Core responsibilities: + +- Compute built-in collector configuration checksums. +- Read credentials from environment variables and `~/.zshrc`. +- Determine whether the current configuration is already connected. +- Run endpoint health checks. +- Save successful connection records. + +### Checksum Inputs + +The checksum includes: + +- Collector name +- Endpoint +- Auth type +- Headers +- Config after removing internal validation fields +- Credential provider +- Credential fingerprint + +The credential fingerprint is a hash of credential content. Plaintext credentials are not written into connection records. + +### Connection Records + +Successful connection records are written to `SystemSetting`: + +```text +category = datasource_connectivity_validations +``` + +The payload uses collector source as the key: + +```json +{ + "barentswatch_vessels": { + "checksum": "...", + "status": "success", + "validated_at": "2026-04-29T00:00:00+00:00", + "status_code": 200, + "credential_source": "datasource_config", + "connected_by": "connection_button" + } +} +``` + +`connected_by` currently has two sources: + +- `connection_button`: the user manually clicked the connection button. +- `collection`: a collection task completed successfully, so the system recorded the current effective configuration as connected. + +### Successful Collection Means Connected + +After a successful collection, the scheduler writes a connection record: + +- [scheduler.py](/home/ray/dev/linkong/planet/backend/app/services/scheduler.py) + +This prevents collectors that already have data from asking the user to validate again. Reconnection is only required when the configuration checksum changes. + +## BarentsWatch AIS Credential Chain + +Files: + +- [barentswatch.py](/home/ray/dev/linkong/planet/backend/app/services/barentswatch.py) +- [vessel_ais.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/vessel_ais.py) + +Resolution priority: + +1. `DataSourceConfig.auth_config` +2. `DataSourceConfig.config` +3. Environment variables +4. `~/.zshrc` + +Supported environment variables: + +```bash +export BARENTSWATCH_CLIENT_ID="..." +export BARENTSWATCH_CLIENT_SECRET="..." +``` + +Historical misspellings are also supported: + +```bash +export BARRENTSWATCH_CLIENT_ID="..." +export BARRENTSWATCH_CLIENT_SECRET="..." +``` + +Token request rules: + +- Token URL: `https://id.barentswatch.no/connect/token` +- `Content-Type`: `application/x-www-form-urlencoded` +- Body: + - `grant_type=client_credentials` + - `client_id` + - `client_secret` + - `scope=ais` + +AIS request rules: + +- Default endpoint: `https://live.ais.barentswatch.no/v1/latest/combined` +- Header: `Authorization: Bearer ` + +`VesselAISCollector` no longer reads environment variables directly. It goes through `resolve_barentswatch_config()` and `fetch_barentswatch_access_token()` so settings, connectivity validation, and collection do not fork into three credential flows. + +## AISStream Collector Chain + +Files: + +- [aisstream.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/aisstream.py) +- [vessel_ais_aggregation.py](/home/ray/dev/linkong/planet/backend/app/services/vessel_ais_aggregation.py) + +AISStream uses a WebSocket realtime stream. The collector writes only to the `ais_raw_observations` raw observation layer; it does not directly overwrite the final vessel display table. The aggregation API handles multi-source deduplication, field selection, and conflict records. + +Configuration: + +- `api_key`: stored in `DataSourceConfig.auth_config`, or provided through `AISSTREAM_API_KEY`. +- `endpoint`: defaults to `wss://stream.aisstream.io/v0/stream`. +- `message_types`: defaults to `PositionReport` and `ShipStaticData`. +- `bounding_boxes`: AISStream format is `[[[lat_min, lon_min], [lat_max, lon_max]]]`; the settings page provides global, Norway / North Sea, Europe coast, East Asia, and North America coast presets. +- `max_messages` and `receive_timeout_seconds`: control the batch-style WebSocket collection window. + +Normalization: + +- `PositionReport` mainly provides position, speed, course, heading, and navigation status. +- Vessel names can be filled from `MetaData.ShipName` even when the message body has no `name`. +- Vessel type usually comes from lower-frequency `ShipStaticData.Type`; the backend maps AIS numeric type codes to Cargo / Tanker / Passenger / Fishing / Military. +- If a vessel has not yet produced a static message, its aggregated type can still be `Other`; v5 vessel profile enrichment is planned to fill that gap. + +Connectivity validation reads saved configuration, environment variables, and `AISSTREAM_API_KEY` from `~/.zshrc`. For actual collection, prefer saving the API key in collector settings. If the key only lives in `~/.zshrc`, confirm that the backend process inherited it; otherwise validation may pass while the collector runtime cannot read the key. + +## Custom REST / WebSocket Mapping Runtime + +Files: + +- [custom_datasource_runtime.py](/home/ray/dev/linkong/planet/backend/app/services/custom_datasource_runtime.py) +- [datasource_mapping.py](/home/ray/dev/linkong/planet/backend/app/services/datasource_mapping.py) + +Custom sources are supplemental inputs for existing target schemas, not isolated data islands. The most complete target today is `vessel_ais`: a custom REST or WebSocket source is mapped deterministically, written into AIS raw observations, and then pushed to Earth through the `vessels` WebSocket channel. + +### Configuration Semantics + +Important fields: + +- `source_type`: `rest` / `http` / `websocket` / `ws`. +- `endpoint`: REST uses `http(s)://`; WebSocket uses `ws(s)://`. +- `auth_type`: `none`, `bearer`, `api_key`, or `basic`. +- `headers`: static request headers. +- `auth_config`: token, API key, or basic username/password; API keys can be sent by header or query. +- `config.target_schema`: for example `vessel_ais`. +- `config.delivery_mode`: REST defaults to `polling`; WebSocket defaults to `realtime_stream`. +- `config.merge_target_source`: records which built-in source this custom source supplements, such as `barentswatch_vessels`. + +The REST runner supports: + +- `GET` / `POST` +- query params +- JSON body +- headers and auth injection +- active mapping writes into the target schema + +The WebSocket runner supports: + +- endpoint format validation +- headers and auth injection +- optional `ws_subscribe_message` +- `ws_message_path` / `ws_items_path` extraction +- reconnects +- `debug_max_messages` debug limits +- background stream start / stop / status + +Related APIs: + +```http +POST /api/v1/datasources/custom/sample +GET /api/v1/datasources/target-schemas +POST /api/v1/datasources/{config_id}/run-mapped +POST /api/v1/datasources/{config_id}/stop-mapped +GET /api/v1/datasources/{config_id}/mapped-status +DELETE /api/v1/datasources/configs/{config_id}?delete_mappings=true&delete_source_data=true +``` + +`run-mapped?background=true` only matters for WebSocket sources and starts a background stream. REST sources remain one-shot collection runs. + +### Delete And Data Cleanup + +Deleting a custom source has three levels: + +- Delete configuration only: preserve mapping and historical data. +- Delete configuration and mapping: also delete mapping templates for that config. +- Delete configuration, mapping, and source data: delete that source's `collected_data`, `ais_raw_observations`, and `ais_source_health`. + +When deleted `vessel_ais` source data affects Earth, the backend broadcasts `reload_required` on the `vessels` channel so Earth reloads aggregated vessels. Legacy `vessel_position` rows are not deleted by custom source because that table cannot safely attribute rows back to a custom source. + +### Local AIS Mock WebSocket + +File: + +- [mock-ais-ws-server.ts](/home/ray/dev/linkong/planet/scripts/mock-ais-ws-server.ts) + +Run: + +```bash +bun run mock:ais-ws +``` + +The mock service continuously sends AIS-like JSON to validate the chain: WebSocket custom source -> mapping -> AIS raw observation -> `vessels` channel -> Earth vessel upsert. Typical config: + +```json +{ + "source_type": "websocket", + "endpoint": "ws://localhost:8787", + "config": { + "target_schema": "vessel_ais", + "delivery_mode": "realtime_stream", + "merge_target_source": "barentswatch_vessels", + "ws_message_path": "$.data", + "ws_items_path": "$.vessels[*]", + "ws_reconnect": true + } +} +``` + +## Credential Guide + +File: + +- [credential_guides.py](/home/ray/dev/linkong/planet/backend/app/services/credential_guides.py) + +APIs: + +```http +GET /api/v1/settings/credential-guides/{provider} +POST /api/v1/settings/credential-guides/{provider}/generate +POST /api/v1/settings/credential-guides/{provider}/reset +``` + +Currently supported: + +- `barentswatch` +- `aisstream` + +The default guide includes the official BarentsWatch tutorial: + +```text +https://developer.barentswatch.no/docs/tutorial +``` + +If the user clicks that the tutorial is not useful, the backend sends the default prompt to AI Provider, generates a new Chinese tutorial, and saves it to `SystemSetting`: + +```text +category = collector_credential_guides +``` + +Reset deletes the custom tutorial and restores the default guide. + +## Save Rules + +When built-in collector configuration is saved, the internal `connectivity_validation` field is removed so validation state does not mix with user configuration. + +BarentsWatch `client_secret` has special handling: + +- The input shows a masked preview. +- If the submitted value still matches the masked preview, the backend keeps the old secret. +- If a new value is submitted, the secret is replaced. +- The previous separate "clear current secret" checkbox is no longer provided. + +## Test Coverage + +Related tests: + +- [test_vessels.py](/home/ray/dev/linkong/planet/backend/tests/test_vessels.py) + +Added coverage: + +- BarentsWatch credentials can be parsed from `~/.zshrc`. +- When environment variables are empty, `resolve_barentswatch_config()` can fall back to `~/.zshrc`. +- Vessel data conversion and GeoJSON output remain compatible. + +## Current Provider Coverage + +Credential providers currently supported: + +- `barentswatch` +- `aisstream` +- `spacetrack` + +Other collectors with `requires_credentials=true` return that their credential chain has not been wired yet, and the frontend shows `Unavailable`. diff --git a/docs/technical/en/docs-gatekeeper-development.md b/docs/technical/en/docs-gatekeeper-development.md new file mode 100644 index 00000000..fb1c3f01 --- /dev/null +++ b/docs/technical/en/docs-gatekeeper-development.md @@ -0,0 +1,116 @@ +# Docs Gatekeeper Development Guide + +Docs Gatekeeper moves `/docs` from "bundle all Markdown into the frontend" to "return catalog and content from the backend according to permissions." Its goal is to keep public manuals, user docs, developer docs, and admin/ops docs in one searchable Docs page while making every protected Markdown body pass through a server-side whitelist and authorization check. + +For the user workflow, see the Docs section in [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md). + +## Authorization Model + +Docs uses two permission layers: + +- `users.role`: preserved for console/system permissions. +- `users.gatekeeper_groups`: Docs content permission groups. + +Groups: + +| Group | Purpose | +| --- | --- | +| `docs_user` | User-operation docs | +| `docs_developer` | Earth, frontend, backend, collector, and AI Provider development docs | +| `docs_admin` | Service control, operations, environment, and sensitive-operation docs | + +Inheritance: + +- Anonymous users can only read `public`. +- `docs_developer` includes `docs_user`. +- `docs_admin` includes `docs_developer` and `docs_user`. +- `admin` and `super_admin` receive all Docs permissions by default. + +## Backend Entry Points + +Files: + +- [docs.py](/home/ray/dev/linkong/planet/backend/app/api/v1/docs.py) +- [docs_gatekeeper.py](/home/ray/dev/linkong/planet/backend/app/services/docs_gatekeeper.py) +- [user.py](/home/ray/dev/linkong/planet/backend/app/models/user.py) +- [users.py](/home/ray/dev/linkong/planet/backend/app/api/v1/users.py) + +APIs: + +```http +GET /api/v1/docs/catalog +GET /api/v1/docs/{lang}/{slug} +``` + +`catalog` returns only documents visible to the current user. The content endpoint validates language, slug, and file existence through the metadata whitelist before checking access: + +- Anonymous protected-doc request: `401`. +- Authenticated but insufficient permissions: `403`. +- Unknown language, unknown slug, or missing file: `404`. + +Markdown bodies can only come from whitelisted files under `docs/technical/{zh,en}/`; arbitrary path reads are not allowed. + +## Metadata Source + +Server-side metadata lives in [docs_gatekeeper.py](/home/ray/dev/linkong/planet/backend/app/services/docs_gatekeeper.py): + +```python +DocsMetadata( + "manual.md", + "manual", + "public", + "Manual", + 2, + "Planet 使用手册", + "Planet Manual", +) +``` + +When adding a public technical doc: + +- Add both Chinese and English Markdown files. +- Add filename, slug, access, group, order, and titles to server `DOCS_METADATA`. +- Add matching metadata to frontend [docs-content.ts](/home/ray/dev/linkong/planet/frontend/src/pages/Docs/docs-content.ts) so navigation titles and sorting stay aligned. +- Update `docs/technical/zh/README.md` and `docs/technical/en/README.md` when the document should be discoverable from the README. + +## User Management + +The `users` table has `gatekeeper_groups JSONB DEFAULT '[]'`. Startup [session.py](/home/ray/dev/linkong/planet/backend/app/db/session.py) applies `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` for existing local databases. + +The user API: + +- Writes `gatekeeper_groups` during user creation. +- Validates group names on update: only `docs_user`, `docs_developer`, and `docs_admin` are accepted. +- Allows only `super_admin` to modify Gatekeeper groups. + +Frontend [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Users/Users.tsx) displays group tags and provides a multi-select in the edit form. Non-`super_admin` users see the field disabled, and submission removes `gatekeeper_groups` before sending. + +## Frontend Docs Loading + +Files: + +- [Docs.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Docs/Docs.tsx) +- [docs-content.ts](/home/ray/dev/linkong/planet/frontend/src/pages/Docs/docs-content.ts) +- [docs-search.ts](/home/ray/dev/linkong/planet/frontend/src/pages/Docs/docs-search.ts) + +Key changes: + +- Remove `import.meta.glob(...?raw)` as the Markdown content source. +- Load `/api/v1/docs/catalog` to build the visible navigation. +- Load `/api/v1/docs/{lang}/{slug}` for document bodies. +- Index search only across currently visible docs, loading Markdown from the backend as needed. +- Show login state for `401`, permission state for `403`, and unavailable-doc state for `404`. + +## Test Coverage + +Relevant tests: + +- [test_docs_gatekeeper.py](/home/ray/dev/linkong/planet/backend/tests/test_docs_gatekeeper.py) + +Tests should cover: + +- Anonymous users only see public docs. +- Protected content returns `401` or `403` appropriately. +- `docs_developer` can read developer docs but not admin docs. +- `admin` and `super_admin` can read admin docs. +- Unknown slugs, unknown languages, and path traversal strings cannot read files. diff --git a/docs/technical/earth-bgp-context.md b/docs/technical/en/earth-bgp-context.md similarity index 98% rename from docs/technical/earth-bgp-context.md rename to docs/technical/en/earth-bgp-context.md index bd0eb8ef..931e286e 100644 --- a/docs/technical/earth-bgp-context.md +++ b/docs/technical/en/earth-bgp-context.md @@ -187,7 +187,7 @@ Current reality: - that is expected, because incidents are aggregated and de-noised - but incident-first rendering makes the Earth view look too quiet unless there is another always-available activity layer -Implementation detail for the recommended `activity layer` is expanded in [bgp-region-aggregation-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-bgp-region-aggregation-plan.md). +Implementation detail for the recommended `activity layer` is expanded in the [BGP Region Aggregation Plan](/home/ray/dev/linkong/planet/docs/plans/earth-bgp-region-aggregation-plan.md). So the immediate next milestone is: diff --git a/docs/technical/en/earth-frontend-context.md b/docs/technical/en/earth-frontend-context.md new file mode 100644 index 00000000..7747cffa --- /dev/null +++ b/docs/technical/en/earth-frontend-context.md @@ -0,0 +1,267 @@ +# Earth Frontend Context + +This document describes the current real structure of the Earth display frontend. The focus is on helping future changes to the HUD, layers, media panel, real terrain, and BGP visualization avoid repeating past structural and state-sync pitfalls. + +Related references: + +- [Project Rules](/home/ray/dev/linkong/planet/rules.md) +- [Frontend Layout Guidelines](/home/ray/dev/linkong/planet/docs/technical/en/frontend-layout-guidelines.md) + +## Current Goal + +The Earth frontend is not an ordinary admin page — it is an independent large-screen display frontend. Current product goals: + +- Maintain the spatial depth and readability of the globe view +- Keep HUD, layers, media panel, BGP, satellites, cables, and similar elements in a unified interaction model +- Clearly represent states like loading, enabled, hidden, and locked + +## Current Entry Point + +React route entry: + +- [Earth.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Earth/Earth.tsx) + +The current approach is simple: + +- The React page only provides a full-screen `iframe` +- The actual Earth application runs at: + - [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) + +Earth frontend is essentially a standalone static application under `public/earth`. + +## Current File Layers + +### 1. Page Entry and Structure + +- [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) + +Responsibilities: + +- Base HUD DOM +- Layer panel +- Media panel +- Toolbar +- Settings dialog +- Legacy element ID compatibility + +### 2. Main Runtime + +- [main.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/main.js) + +Responsibilities: + +- Globe initialization +- Three.js scene assembly +- Data loading and refresh +- Layer module integration +- Earth-level state synchronization + +### 3. Earth Control Layer + +- [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) + +Responsibilities: + +- Toolbar interaction +- Layer panel interaction +- Rotation / zoom / layout +- HUD panel drag +- Layer toggle state machine +- Earth settings read, persist, and reset + +This is currently the most critical UI control entry point for the Earth frontend. + +### 4. UI and Status Messages + +- [ui.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/ui.js) + +Responsibilities: + +- Loading panel +- Status message +- Tooltip / error / cleanup logic + +### 5. Globe and Terrain + +- [earth.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/earth.js) +- [terrain.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/terrain.js) + +Responsibilities: + +- Globe sphere, cloud layer, atmosphere +- Real terrain mesh +- Terrain tile fetch, decode, displacement, and shading + +### 6. Layer Modules + +- [satellites.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/satellites.js) +- [cables.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/cables.js) +- [vessels.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/vessels.js) +- [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) +- [bgp-cruise-adapter.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp-cruise-adapter.js) +- [compute-centers.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/compute-centers.js) renders supercomputer and GPU-cluster markers. The backend renders compute centers only from source-provided coordinates or `compute_center_locations` dimension-table coordinates during startup; manual candidate collection can query ROR and Nominatim/OpenStreetMap, and the layer keeps the `?` badge for unconfirmed positions while the details card shows precision, confidence, source notes, and verification date. +- [country-boundaries.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/country-boundaries.js) + +Each module is responsible for its own: + +- Data fetching +- Three.js mesh creation and update +- State tracking (loaded, visible, hover, locked) +- Self-cleanup (dispose on scene destroy) + +The compute-center layer row has a notification badge for GeoJSON `unresolved` records. The badge means "no trustworthy coordinates, cannot render on the globe"; it is different from the `?` marker drawn on already positioned but unconfirmed compute centers. Clicking the badge opens a fixed info card beside the layer panel. Row-level `采集` fetches candidates only. Header-level `一键采用` processes the queue top-to-bottom, saves the highest-confidence valid candidate, removes successful rows, renumbers the list, and dispatches `earth:compute-center-unresolved-count-change` so the badge updates immediately. When the batch ends, `earth:compute-center-location-saved` refreshes the real layer. + +### AIS Vessel Layer + +The vessel layer fetches `/api/v1/visualization/geo/vessels` and renders the aggregated AIS GeoJSON through `createInteractableLayer()`. By default it does not send a `limit` parameter, and `VESSEL_CONFIG.maxRenderedMarkers = 0` means the frontend does not clip the result to 5000 vessels. A positive `options.limit` or positive `maxRenderedMarkers` can still be used as an explicit temporary cap. + +Vessel color and vessel type text must use the same normalized classification. `vessels.js` derives `type` from both `vessel_type_name` and the AIS numeric `vessel_type` code; that `type` drives marker color. It also derives `vessel_type_display`, which `main.js` uses for the info card, hover summary, and search result subtitle. Do not make the info card read only the raw `vessel_type_name`, because AISStream can provide a numeric type while the raw name is still `Other`. + +AISStream `PositionReport` messages commonly carry live position and `MetaData.ShipName`, while vessel type usually comes from lower-frequency `ShipStaticData.Type`. The backend normalizes `MetaData.ShipName` into the vessel name and maps numeric type codes into Cargo / Tanker / Passenger / Fishing / Military where available. Missing type detail should wait for a static AIS message or the planned vessel profile enrichment; the frontend should not invent a more specific type. + +### 7. HUD Panels and Search + +- [hud-panels.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/hud-panels.js) +- [info-card.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/info-card.js) +- [search.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/search.js) +- [legend.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/legend.js) + +### 8. Cruise Mode + +- [cruise-sequencer.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/cruise-sequencer.js) +- [callout-connector.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/callout-connector.js) + +The cruise sequencer handles generic logic: current target, queue order, camera focus, and dwell / hide / switch. Business modules supply target queues and content — they should not contain camera control logic. + +### 9. Constants + +- [constants.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/constants.js) + +All material, layer, satellite, BGP, cable, terrain, celestial, and other style parameters are maintained here. Do not scatter magic numbers in module files. + +## Current Style Layers + +CSS files in `frontend/public/earth/css/` each correspond to a specific component scope. Do not write global Earth styles into `base.css` unless they genuinely apply to everything. + +## Current Layer Toggle State Semantics + +### `data-status-target` + +Layer toggle buttons use `data-status-target` attributes to link button state to layer state. The state machine in `controls.js` handles: + +- `loading`: showing the loading indicator +- `enabled`: layer is active +- `hidden`: layer is hidden +- `error`: layer failed to load + +This is the canonical way to synchronize button visual state with actual layer state. Do not maintain separate boolean flags for button display. + +Terrain should not block startup when it is not the restored visible layer. After deferred layer visibility settings are applied, `controls.js` schedules `scheduleTerrainPrefetch()` only when HD texture is enabled, terrain is not ready, and no prefetch is already running. The prefetch uses `setTimeout` plus `requestIdleCallback` so cloud, HD texture, and startup layer work keep first-screen priority. + +## Current Settings Persistence + +Earth settings are stored in `localStorage`. The key is typically a namespaced string defined in `constants.js`. `controls.js` handles read, write, and reset. + +Settings that affect visual layers (terrain opacity, day/night mode, satellite display style, etc.) are read during initialization and applied immediately. + +## Current Terrain Pipeline + +1. `terrain.js` creates a sphere geometry with enough segments +2. On load, fetches Terrarium-format elevation tiles from the backend +3. Decodes R/G/B into elevation values +4. Displaces vertex positions radially based on elevation +5. Applies a vertex alpha that fades terrain edges at coastlines +6. Terrain writes to the scene as a mesh above the HD texture layer + +When HD texture is off, terrain is temporarily hidden and its state is remembered. When HD texture comes back on, terrain restores its prior visibility. + +Terrain tile fetching is batched. `terrain.js` deduplicates required Terrarium tile keys and sends chunks sized by `TERRAIN_CONFIG.batchRequestSize` to `/api/v1/visualization/terrain/terrarium/batch`. The backend proxies S3 Terrarium tiles with an in-memory LRU cache, per-batch deduplication, and bounded concurrency. The single tile endpoint remains for fallback paths and browser cache semantics. + +## Current High-Frequency Risk Points + +### 1. Visual State and Business State Out of Sync + +The most common class of Earth bugs: + +- Button shows "loaded," but layer has no objects rendered +- Button shows "hidden," but objects are still visible +- Loading ended, but button still looks like it hasn't + +All future changes must prioritize checking state sync. + +### 2. HUD Layout: Check Structure First, Not CSS Patches + +Earth HUD has repeatedly experienced: + +- Panel compressed to a sliver +- Markdown content clipped +- Tabs/iframe content consumed by `overflow: hidden` + +Inspection order: + +1. Who is responsible for height +2. Who is responsible for scrolling +3. Which layer is doing the clipping + +Do not immediately add `overflow: hidden` or extra wrapper layers. + +### 3. Transitional Paths Must Be Closed Off + +Earth has gone through multiple rounds of HUD, toolbar, and media panel refactoring, making it easy to accumulate: + +- Old helpers +- Old classes +- Old fallback logic +- Deprecated variants + +After each major feature is complete, do a cleanup pass. + +### 4. Cruise Mode and Business Events Must Not Be Deeply Coupled + +The correct boundary: + +- The generic cruise layer only knows: + - Current target + - Queue order + - Camera focus + - Dwell / hide / switch +- Business modules only supply: + - Target queues + - Focus coordinates + - Card content + - Highlight / layer side effects + +If future cable, satellite, or news cruise is added, do not copy a new set of `main.js` state variables. Instead reuse: + +- [cruise-sequencer.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/cruise-sequencer.js) +- [callout-connector.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/callout-connector.js) +- The business adapter pattern from [bgp-cruise-adapter.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp-cruise-adapter.js) + +## Recommended Change Approach + +For future Earth changes: + +1. First identify what you're changing: + - Three.js rendering layer + - HUD structure layer + - Layer state layer + - Panel content layer +2. If involving layer buttons, connect to the unified state machine +3. If involving visibility toggle, check whether tooltip / legend / info-card / lock all close together +4. If involving panel layout, check structure before touching CSS + +## Current Boundary with the Console Frontend + +The Earth frontend and the console frontend are not the same UI system: + +- Console frontend: React + Ant Design workbench +- Earth frontend: native HUD + Three.js display under `public/earth` + +Therefore: + +- Earth should not directly reuse Ant Table / AppLayout semantics +- The console should not copy Earth HUD animations and glass-layer design language + +For console structure, see: + +- [Admin Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/frontend-admin-frontend-context.md) diff --git a/docs/technical/en/earth-interactable-usage.md b/docs/technical/en/earth-interactable-usage.md new file mode 100644 index 00000000..252bcfa8 --- /dev/null +++ b/docs/technical/en/earth-interactable-usage.md @@ -0,0 +1,270 @@ +# Earth Interactable Usage + +`Interactable` is the shared rendering entry point for icon-like interactive elements on the Earth surface. It extracts the pattern proven by the vessel layer into reusable behavior: normal state uses batched `THREE.Points`, hover and locked states use small overlays, picking uses screen-space hit testing, icon assets are normalized into canvas textures, and the shared layer handles glow, state, size, ground rendering, and same-coordinate avoidance. + +Currently integrated layers: + +| Layer | Business File | Icon Source | Extra Animation | +| --- | --- | --- | --- | +| AIS vessels | `frontend/public/earth/js/vessels.js` | canvas draw, moving triangle / anchored dot | Vessel tracks are still maintained by the business layer | +| Compute centers | `frontend/public/earth/js/compute-centers.js` | `assets/icons/compute-*.svg` | Estimated-location `?` badge is added through `icon.afterDraw()` | +| BGP events | `frontend/public/earth/js/bgp.js` | canvas draw, symbol by event type | Expanding rings are still maintained by the BGP business layer | +| BGP observers | `frontend/public/earth/js/bgp.js` | `assets/icons/bgp-broadcast-pin.svg` | Halo, activity core, coverage wedge, and radar sweep remain in the BGP business layer | + +Landing sites were previously attempted on Interactable, but pin-style SVGs were fragmented by `THREE.Points` depth testing near the Earth edge. They now use a dedicated `THREE.Sprite` path with a yellow flat-sphere texture generated by canvas. The old SVG assets remain in `assets/icons/`, but landing sites no longer depend on SVG at runtime. + +## Why Interactable Exists + +Before this layer, each surface icon layer could easily reimplement its own version of: + +- icon texture generation +- hover / locked state +- glow styling +- picking radius +- zoom-dependent size strategy +- overlap avoidance for identical coordinates + +When this logic is scattered across business files, visual behavior drifts and later tuning becomes layer-by-layer repair. The boundary of `Interactable` is: the shared layer owns how icons remain stable on Earth and how they are selected; the business layer owns where data comes from, what the icon means, what detail cards show, and whether extra animation exists. + +## Entry Point + +```javascript +import { createInteractableLayer } from "./interactable.js"; +``` + +Core call shape: + +```javascript +const layer = createInteractableLayer({ + id: "example", + objectType: "example_object", + renderOrder: 4.4, + altitudeOffset: 0.2, + pointSize: 34, + icon: { + draw(context, options) { + // draw canvas icon + }, + }, + getPosition: (item) => ({ + latitude: item.latitude, + longitude: item.longitude, + }), + getKind: (item) => item.kind || "default", +}); +``` + +Business modules usually expose only a thin wrapper: + +```javascript +export function getExampleMarkers() { + return layer.getMarkers(); +} + +export function getExamplePointerIntersections(options) { + return layer.getPointerIntersections(options); +} + +export function setExampleMarkerState(marker, state = "normal") { + layer.setMarkerState(marker, state); +} + +export function updateExampleVisualState(lockedObjectType, lockedObject, camera) { + layer.updateVisualState(lockedObjectType, lockedObject, camera); +} +``` + +## Configuration + +| Option | Default | Description | +| --- | --- | --- | +| `id` | required | Unique layer id used for group name, avoidance registration, and debug. | +| `objectType` | `id` | Business type written to `marker.userData.type`; the main interaction layer uses it to identify locked objects. | +| `renderOrder` | `4` | Base render order for normal points and hover / locked overlays. | +| `altitudeOffset` | `0.2` | Business altitude, used as `CONFIG.earthRadius + altitudeOffset` for the original surface position. | +| `pointSize` | `32` | Base screen pixel size used by both normal points and overlays. | +| `sizeMode` | `"fixed"` | Fixed screen size by default; non-`"fixed"` modes scale by camera distance. | +| `sizeScale` | `{ referenceFov: 75, min: 0.12, max: 3 }` | Scaling bounds when `sizeMode !== "fixed"`. | +| `atlasCellSize` | `128` | Canvas texture cell size for icons. | +| `colors` | `{}` | Supports `normal`, flattened kind keys, and `byKind`. | +| `opacity` | `{ normal: 0.88, dimmed: 0.26, hover: 0.98, locked: 1 }` | Opacity per state. | +| `stateScale` | `{ hover: 1, locked: 1, dimmed: 1 }` | Size multiplier per state. | +| `pulse` | `{}` | Optional locked-state breathing scale, with `enabled`, `speed`, and `amplitude`. | +| `avoidance` | `{ enabled: true, precision: 4, radius: 1.1, step: 0.35 }` | Same-coordinate avoidance across Interactable layers. | +| `icon` | required | Icon source, supporting canvas draw, SVG / image asset, state asset, anchor, and post-processing. | +| `getPosition(item)` | required | Returns `{ latitude, longitude }` or `THREE.Vector3`. | +| `getKind(item)` | `item.type || "default"` | Returns a business kind for color and texture buckets. | +| `getRotationBin(marker)` | `0` | Returns a rotation bucket, such as 32 heading buckets for vessels. | +| `getBucketKey(marker)` | `String(getRotationBin(marker))` | Returns a texture / geometry bucket key. | +| `getPointSizeMultiplier(marker)` | `1` | Per-marker size multiplier. BGP events use severity; observers use activity. | +| `getUserData(item)` | `item` | Business fields written onto the marker. | + +## Icon Configuration + +`icon.anchor` is optional and defaults to `{ x: 0.5, y: 0.5 }`, meaning the texture center aligns with the marker coordinate. It is only suitable for small visual anchor offsets. If the icon body is large and must remain fully visible at the Earth edge, such as the old landing-site pin, it should not be forced through `THREE.Points + depthTest`; the body will be clipped by Earth depth. + +### Canvas Icons + +Canvas icons fit vessels and BGP events where symbols need to be drawn dynamically by state or rotation: + +```javascript +const vesselIconLayer = createInteractableLayer({ + id: "vessels", + objectType: "vessel", + pointSize: 34, + icon: { + draw(context, { marker, rotationBin = 0, glow = false, color = "#ffffff" }) { + if (!marker.userData.anchored) { + context.rotate((rotationBin / 32) * Math.PI * 2); + } + context.fillStyle = color; + context.shadowColor = color; + context.shadowBlur = glow ? 14 : 0; + context.beginPath(); + context.moveTo(0, -37); + context.lineTo(28, 32); + context.lineTo(0, 17); + context.lineTo(-28, 32); + context.closePath(); + context.fill(); + }, + }, + getRotationBin: getCourseBin, + getBucketKey: (marker) => `${marker.userData.anchored ? "anchored" : "moving"}:${getCourseBin(marker)}`, +}); +``` + +When `icon.coordinates !== "canvas"`, `Interactable` translates the context to the atlas center first. Vessel-style icons that already draw around center coordinates do not need to declare `coordinates`. + +### SVG / Image Asset Icons + +Asset icons fit facilities such as compute centers and BGP observers: + +```javascript +const computeCenterIconLayer = createInteractableLayer({ + id: "computeCenters", + objectType: "compute_center", + pointSize: 36, + atlasCellSize: 128, + icon: { + coordinates: "canvas", + colorable: false, + fitSize: 60, + glowBlur: 16, + getSource({ marker, item }) { + const siteType = marker?.userData?.site_type || item?.site_type || "gpu_cluster"; + return COMPUTE_CENTER_ICON_SOURCES[siteType]; + }, + afterDraw(context, { marker, item }) { + if (marker?.userData?.is_estimated ?? item?.is_estimated) { + drawComputeCenterEstimatedBadge(context, true); + } + }, + }, +}); +``` + +Asset conventions: + +- SVG / image files live in `frontend/public/earth/assets/icons/` and are referenced as `/earth/assets/icons/name.svg`. +- Original SVGs should keep a standard `viewBox` and paths; avoid hard-coding transform only for display size. +- Display size is controlled by `icon.fitSize`; it can be a number, `{ width, height }`, or a function. +- If `icon.colorable !== false` and state colors are provided, the shared layer first draws the asset to a temporary canvas and then tints it with `source-in`. +- Multicolor images or SVGs that should not be tinted must set `colorable: false`. + +## Lifecycle + +Typical load flow: + +```javascript +export async function loadExampleLayer(_scene, earth) { + clearExampleData(earth); + + const markerData = await fetchExampleData(); + await layer.preloadAssets(markerData); + layer.setData(markerData); + layer.attach(earth); + layer.setVisible(showExampleLayer); + + return { totalCount: layer.getCount() }; +} +``` + +Method responsibilities: + +| Method | Description | +| --- | --- | +| `preloadAssets(items)` | Collects asset sources that may be used by normal / hover / locked states and preloads them with browser `Image`. Canvas-drawn icons can skip this. | +| `setData(items)` | Clears old points, creates markers, registers avoidance, and rebuilds `THREE.Points` by bucket. | +| `attach(parent)` | Mounts the layer group onto the Earth root. | +| `setVisible(next)` | Controls visibility for the group, points, and overlays. | +| `setMarkerState(marker, state)` | Sets `normal` / `hover` and other states, then invalidates visual state. | +| `updateVisualState(focusType, focusObject, camera)` | Updates normal opacity / size and refreshes hover / locked overlays. | +| `getPointerIntersections(options)` | Runs screen-space picking and returns hits sorted by pixel distance. | +| `clearData(parent)` | Unregisters avoidance, disposes geometry / material, clears markers, and removes the group from the parent. | + +## Picking Integration + +`Interactable` does not depend on the default Three.js raycast for `Points`. The main interaction layer passes Earth, camera, pointer, and hit radius: + +```javascript +const intersects = getVesselPointerIntersections({ + earth, + camera, + pointer, + radiusPx: 22, + width: window.innerWidth, + height: window.innerHeight, +}); +``` + +The shared layer: + +1. Converts the camera position into Earth-local coordinates. +2. Skips markers on the back side. +3. Projects marker world position into screen coordinates. +4. Uses `radiusPx` for pixel-distance hits. +5. Returns the nearest candidate objects. + +Earth dragging, inertia, and hover throttling still belong to `main.js` because they depend on global input state. + +## Same-Coordinate Avoidance + +Avoidance is enabled by default and applies to all layers created through `createInteractableLayer()`. The shared layer builds an `icon_avoidance_key` from latitude / longitude or `THREE.Vector3`, then arranges markers with the same key into a small circle along the surface tangent plane. + +Key points: + +- `icon_base_position` keeps the original business position. +- Avoidance only changes rendering and picking position. It does not change business latitude / longitude. +- When a single marker returns to its original position, it uses the business surface position computed from `altitudeOffset`. +- When multiple markers share coordinates, the first ring uses `avoidance.radius`; later rings add `avoidance.step`. + +If a business layer must stay exactly on the original point, disable avoidance explicitly: + +```javascript +createInteractableLayer({ + id: "strict-layer", + avoidance: { enabled: false }, +}); +``` + +## Business Animation Boundary + +`Interactable` currently owns only the icon body and common hover / locked overlays. Complex animations remain in business modules, but should follow the Interactable marker position: + +- BGP event expanding rings are independent ring sprites created by `bgp.js`, updated every frame with `position.copy(marker.position)`. +- BGP observer halo, status core, coverage halo, and coverage wedge are managed by `bgp.js`; the icon body is managed by Interactable. +- Vessel tracks remain in `vessels.js` because they depend on track data loaded after a click. + +This boundary avoids pushing every animation type into the shared interface too early. If multiple layers reuse the same animation type later, it can move into an Interactable `animations` extension. + +## New Layer Checklist + +1. Prepare marker data in the business file and keep required business fields. +2. Choose an icon type: canvas draw, SVG / image asset, or dynamic `getSource()`. +3. Configure `pointSize`, `icon.fitSize`, `colors`, `opacity`, and `stateScale`. +4. Provide `getPointSizeMultiplier()` if business-specific size variation is needed. +5. Provide `getRotationBin()` and a stable `getBucketKey()` if rotation exists. +6. During load, call `preloadAssets()` before `setData()`, `attach()`, and `setVisible()`. +7. Wire `getPointerIntersections()` in `main.js` and reuse the existing hover / locked state update flow. +8. Record altitude, `renderOrder`, `pointSize`, and animation ordering in the layer style index and render order documents. diff --git a/docs/technical/en/earth-layer-style-reference.md b/docs/technical/en/earth-layer-style-reference.md new file mode 100644 index 00000000..d7b984ee --- /dev/null +++ b/docs/technical/en/earth-layer-style-reference.md @@ -0,0 +1,250 @@ +# Earth Layer Style Property Index + +This document records the material, color, opacity, line width, radius offset, and `renderOrder` style properties of all Earth frontend layers. For layer ordering relationships, see [Earth Render Layer Order](/home/ray/dev/linkong/planet/docs/technical/en/earth-render-layer-order.md). + +## Naming Conventions + +| Category | Convention | Example | +| --- | --- | --- | +| Global config objects | `*_CONFIG` | `COUNTRY_BOUNDARY_CONFIG` | +| Layer radius offsets | `*AltitudeOffset` / `radiusOffset` | `lineAltitudeOffset`, `GRID_CONFIG.radiusOffset` | +| Opacity | `*Opacity` | `hoverLineOpacity` | +| Render order | `*RenderOrder` | `textureOverlayRenderOrder` | +| Color | `*Color`, hex number or CSS color value | `lineColor`, `colors.supercomputer` | +| Line width | `lineWidth` / `*LineWidth` | `GRID_CONFIG.lineWidth` | + +## Earth Base and HD Texture + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Earth base radius | `CONFIG.earthRadius` | `100` | `earth.js:createEarth()` | +| Earth base color | `EARTH_MATERIAL_CONFIG.color` | `0x010609` | `MeshPhongMaterial.color` | +| Earth base emissive | `EARTH_MATERIAL_CONFIG.emissive` | `0x010609` | `MeshPhongMaterial.emissive` | +| Earth base specular | `EARTH_MATERIAL_CONFIG.specular` | `0x1a2d45` | `MeshPhongMaterial.specular` | +| Earth base shininess | `EARTH_MATERIAL_CONFIG.shininess` | `12` | `MeshPhongMaterial.shininess` | +| Earth base opacity | `EARTH_MATERIAL_CONFIG.opacity` | `1` | `MeshPhongMaterial.opacity` | +| HD texture radius offset | `EARTH_MATERIAL_CONFIG.textureOverlayAltitudeOffset` | `0.1` | Standalone HD texture sphere radius | +| HD texture opacity | `EARTH_MATERIAL_CONFIG.textureOverlayOpacity` | `0.88` | HD texture `MeshPhongMaterial.opacity` | +| HD texture renderOrder | `EARTH_MATERIAL_CONFIG.textureOverlayRenderOrder` | `0.96` | `_earthTextureOverlay.renderOrder` | +| HD texture specular | `EARTH_MATERIAL_CONFIG.textureOverlaySpecular` | `0x05080d` | Reduces specular highlight in direct-light areas to avoid blown-out texture | +| HD texture shininess | `EARTH_MATERIAL_CONFIG.textureOverlayShininess` | `4` | Reduces specular concentration | +| HD texture color multiplier | inline | `0xffffff` | `_earthTextureOverlayMaterial.color` | + +## Earth Occluder and Day/Night + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Occluder radius factor | `EARTH_MATERIAL_CONFIG.occluderRadiusFactor` | `0.999` | Depth occluder sphere radius | +| Occluder segments | `EARTH_MATERIAL_CONFIG.occluderSegments` | `48` | Occluder geometry segments | +| Occluder renderOrder | inline | `-1` | `occluder.renderOrder` | +| Day/night sun direction | `EARTH_MATERIAL_CONFIG.dayNight.sunDirection` | `{ x: 1, y: 0.2, z: 0.4 }` | Custom day/night shader | +| Night-side minimum brightness | `EARTH_MATERIAL_CONFIG.dayNight.nightFloor` | `0.24` | Shader uniform | +| Day-side boost | `EARTH_MATERIAL_CONFIG.dayNight.dayBoost` | `1.12` | Shader uniform | +| Twilight width | `EARTH_MATERIAL_CONFIG.dayNight.twilightWidth` | `0.2` | Shader uniform | +| Twilight intensity | `EARTH_MATERIAL_CONFIG.dayNight.twilightIntensity` | `0.14` | Shader uniform | +| Twilight color | `EARTH_MATERIAL_CONFIG.dayNight.twilightColor` | `0x4ea0ff` | Shader uniform | +| Night tint color | `EARTH_MATERIAL_CONFIG.dayNight.nightTintColor` | `0x0b1830` | Shader uniform | +| Night tint intensity | `EARTH_MATERIAL_CONFIG.dayNight.nightTintIntensity` | `0.08` | Shader uniform | + +## Atmospheric Glow and Clouds + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Inner atmosphere radius factor | `EARTH_MATERIAL_CONFIG.atmosInnerRadiusFactor` | `1.01` | `atmosInnerGeo` | +| Inner atmosphere segments | `EARTH_MATERIAL_CONFIG.atmosInnerSegments` | `64` | `atmosInnerGeo` | +| Inner atmosphere color | `EARTH_MATERIAL_CONFIG.atmosInnerColor` | `[0.25, 0.62, 1.0]` | Shader RGB | +| Inner atmosphere rim power | `EARTH_MATERIAL_CONFIG.atmosInnerRimPower` | `3.2` | Shader rim falloff | +| Inner atmosphere intensity | `EARTH_MATERIAL_CONFIG.atmosInnerIntensity` | `0.18` | Shader alpha multiplier | +| Outer atmosphere radius factor | `EARTH_MATERIAL_CONFIG.atmosOuterRadiusFactor` | `1.016` | `atmosOuterGeo` | +| Outer atmosphere segments | `EARTH_MATERIAL_CONFIG.atmosOuterSegments` | `48` | `atmosOuterGeo` | +| Outer atmosphere color | `EARTH_MATERIAL_CONFIG.atmosOuterColor` | `[0.18, 0.45, 0.9]` | Shader RGB | +| Outer atmosphere rim power | `EARTH_MATERIAL_CONFIG.atmosOuterRimPower` | `5.0` | Shader rim falloff | +| Outer atmosphere intensity | `EARTH_MATERIAL_CONFIG.atmosOuterIntensity` | `0.02` | Shader alpha multiplier | +| Atmosphere blending | inline | `THREE.AdditiveBlending` | `ShaderMaterial.blending` | +| Atmosphere renderOrder | inline | `1` | `atmosInner/Outer.renderOrder` | +| No-HD-texture rim glow color | `EARTH_MATERIAL_CONFIG.rimGlowColor` | `[0.35, 0.65, 1.0]` | Fresnel shell RGB when HD texture is hidden or unavailable | +| No-HD-texture rim glow power | `EARTH_MATERIAL_CONFIG.rimGlowPower` | `3.8` | Shader rim falloff; higher = narrower edge | +| No-HD-texture rim glow intensity | `EARTH_MATERIAL_CONFIG.rimGlowIntensity` | `0.28` | Shader alpha multiplier | +| No-HD-texture rim glow segments | `EARTH_MATERIAL_CONFIG.rimGlowSegments` | `64` | `earth-rim-glow` geometry segments | +| Cloud layer radius offset | `CLOUD_LAYER_CONFIG.radiusOffset` | `3` | Cloud sphere radius | +| Cloud layer segments | `CLOUD_LAYER_CONFIG.widthSegments / heightSegments` | `64 / 64` | Cloud sphere geometry | +| Cloud layer opacity | `CLOUD_LAYER_CONFIG.opacity` | `0.15` | `MeshPhongMaterial.opacity` | +| Cloud texture | `CLOUD_LAYER_CONFIG.textureUrl` | `"./assets/earth_clouds_1024.png"` | Cloud texture map | +| Cloud blending | inline | `THREE.AdditiveBlending` | `MeshPhongMaterial.blending` | + +## Land/Ocean Base and Country Borders + +The land/ocean base is an Earth base-map asset and preloads at startup; the "Border Lines" layer toggle only controls normal border lines, hover lines, and interactive hover. + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Country border data path | `COUNTRY_BOUNDARY_CONFIG.dataPath` | `"/earth/data/countries-admin0.min.geojson"` | GeoJSON input | +| Ocean fill color | local `OCEAN_HEX` | `0x010609` | Land/ocean base canvas background | +| Land fill color | `COUNTRY_BOUNDARY_CONFIG.landColor` | `0x080f1b` | Land/ocean base canvas land | +| Land/ocean base opacity | `COUNTRY_BOUNDARY_CONFIG.landOpacity` | `1.0` | `MeshBasicMaterial.opacity` | +| Land/ocean base radius offset | `COUNTRY_BOUNDARY_CONFIG.landAltitudeOffset` | `0.08` | `country-land-ocean` radius | +| Land/ocean base renderOrder | `COUNTRY_BOUNDARY_CONFIG.landRenderOrder` | `0.86` | `country-land-ocean.renderOrder` | +| Land/ocean mask size | `landMaskWidth / landMaskHeight` | `2048 / 1024` | Canvas / DataTexture size | +| Country tint color | `COUNTRY_BOUNDARY_CONFIG.tintColor` | `0x0b1830` | Tint when HD texture is off | +| Country tint radius offset | `COUNTRY_BOUNDARY_CONFIG.tintAltitudeOffset` | `0.04` | `country-tint` radius | +| Country tint renderOrder | `COUNTRY_BOUNDARY_CONFIG.tintRenderOrder` | `0.2` | `country-tint.renderOrder` | +| Border line color | `COUNTRY_BOUNDARY_CONFIG.lineColor` | `0x7fc7ff` | Normal border line | +| Border line opacity | `COUNTRY_BOUNDARY_CONFIG.lineOpacity` | `0.58` | Normal border line opacity | +| Border dimmed opacity on hover | `COUNTRY_BOUNDARY_CONFIG.dimmedLineOpacity` | `0.18` | Normal border opacity during hover | +| Border line radius offset | `COUNTRY_BOUNDARY_CONFIG.lineAltitudeOffset` | `0.115` | Normal border line radius; slightly above HD texture `0.10` and below terrain base `0.16` to reduce floating | +| Border line renderOrder | `COUNTRY_BOUNDARY_CONFIG.lineRenderOrder` | `2.2` | Normal border line level | +| Border hover color | `COUNTRY_BOUNDARY_CONFIG.hoverLineColor` | `0xff3b1f` | Neon red-orange | +| Border hover opacity | `COUNTRY_BOUNDARY_CONFIG.hoverLineOpacity` | `1.0` | Hover line opacity | +| Border hover radius offset | `COUNTRY_BOUNDARY_CONFIG.hoverAltitudeOffset` | `0.14` | Hover line radius; close to the surface but above normal border lines | +| Border hover renderOrder | `COUNTRY_BOUNDARY_CONFIG.hoverLineRenderOrder` | `2.3` | Hover line level | +| Border hover glow opacity | `COUNTRY_BOUNDARY_CONFIG.hoverGlowOpacity` | `0.38` | Glow line opacity | +| Border hover glow line width | `COUNTRY_BOUNDARY_CONFIG.hoverGlowLineWidth` | `3` | Glow `LineBasicMaterial.linewidth` | +| Border hover glow level offset | `COUNTRY_BOUNDARY_CONFIG.hoverGlowRenderOrderOffset` | `0.01` | Glow renderOrder = `2.29` | +| Border hover glow radius offset | `COUNTRY_BOUNDARY_CONFIG.hoverGlowRadiusOffset` | `0.04` | Glow radius = hover radius + 0.04 | + +## Real Terrain + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Terrain tile size | `TERRAIN_CONFIG.tileSize` | `256` | Terrarium tile read size | +| Terrain base zoom | `TERRAIN_CONFIG.baseZoom` | `4` | Terrain sampling zoom | +| Terrain geometry segments | `geometryWidthSegments / geometryHeightSegments` | `320 / 320` | Terrain sphere geometry | +| Terrain base radius offset | `TERRAIN_CONFIG.baseRadiusOffset` | `0.16` | Terrain overlays HD texture | +| Terrain exaggeration | `TERRAIN_CONFIG.exaggeration` | `34` | Elevation to world units | +| Terrain land fade height | `TERRAIN_CONFIG.landRevealFadeMeters` | `220` | Vertex alpha for coastline fade | +| Terrain opacity | `TERRAIN_CONFIG.opacity` | `0.68` | `MeshPhongMaterial.opacity` | +| Terrain color | `TERRAIN_CONFIG.color` | `0x8aa884` | `MeshPhongMaterial.color` | +| Terrain emissive | `TERRAIN_CONFIG.emissive` | `0x030704` | Reduces self-emission to preserve terrain shading | +| Terrain specular | `TERRAIN_CONFIG.specular` | `0x344438` | Gives terrain local sheen without boosting HD texture brightness | +| Terrain shininess | `TERRAIN_CONFIG.shininess` | `16` | Tightens terrain highlight | +| Terrain renderOrder | inline | `1.2` | `terrain.renderOrder` | +| Terrain polygonOffset | inline | `factor -1`, `units -1` | Reduces z-fighting near sphere surface | + +## Grid Lines + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Grid radius offset | `GRID_CONFIG.radiusOffset` | `0.14` | Grid sphere radius | +| Grid color | `GRID_CONFIG.color` | `0xc0e0ff` | `LineBasicMaterial.color` | +| Grid opacity | `GRID_CONFIG.opacity` | `0.08` | `LineBasicMaterial.opacity` | +| Grid line width | `GRID_CONFIG.lineWidth` | `1` | `LineBasicMaterial.linewidth` | +| Grid renderOrder | `GRID_CONFIG.renderOrder` | `2.05` | Grid level | +| Latitude step | `GRID_CONFIG.latitudeStep` | `15` | Latitude line generation step | +| Longitude step | `GRID_CONFIG.longitudeStep` | `30` | Longitude line generation step | +| Segment sample step | `GRID_CONFIG.segmentStep` | `5` | Grid line sample step | + +## Submarine Cables and Landing Points + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Default cable color | `CABLE_COLORS.default` | `0xffff44` | Used when no data color available | +| Cable radius offset | `CABLE_CONFIG.line.altitudeOffset` | `0.2` | Cable line radius | +| Cable line width | `CABLE_CONFIG.line.lineWidth` | `1` | `LineBasicMaterial.linewidth` | +| Cable opacity | `CABLE_CONFIG.line.opacity` | `1.0` | Cable line opacity | +| Cable renderOrder | `CABLE_CONFIG.line.renderOrder` | `1` | Cable line level | +| Landing point radius offset | `CABLE_CONFIG.landingPoint.altitudeOffset` | `0.2` | Same surface height as cable lines, avoiding a floating marker | +| Landing point sprite height | local `LANDING_POINT_SPRITE_HEIGHT` | `3` | `THREE.Sprite` base height | +| Landing point reference FOV | local `LANDING_POINT_SIZE_REFERENCE_FOV` | `75` | Matches the current Earth camera FOV | +| Landing point scale minimum | local `LANDING_POINT_SIZE_SCALE_MIN` | `0.16` | Minimum multiplier after roughly 200% zoom, limiting high-zoom screen footprint; `3 * 0.16 = 0.48` | +| Landing point scale maximum | local `LANDING_POINT_SIZE_SCALE_MAX` | `3` | Maximum multiplier at far distance; current minimum zoom reaches roughly `2.50` | +| Landing point atlas size | local `LANDING_POINT_ATLAS_CELL_SIZE` | `128` | Canvas flat shaded sphere texture size | +| Landing point color | `CABLE_CONFIG.landingPoint.color` | `0xffaa00` | `SpriteMaterial.color` | +| Landing point opacity | `CABLE_CONFIG.landingPoint.opacity` | `1.0` | `SpriteMaterial.opacity` | +| Landing point renderOrder | `CABLE_CONFIG.landingPoint.renderOrder` | `1` | Same level as cable lines; `depthTest: false` keeps the ball whole, while camera-to-center globe occlusion hides back-side points | +| Landing point dim brightness | `landingPointVisual.dimBrightness` | `0.62` | Dim state color multiplier | +| Related landing point opacity | `landingPointVisual.related.opacityBase / opacityPulse` | `0.8 / 0.2` | Highlight pulse | +| Dimmed landing point color | `landingPointVisual.dimmed.colorRGB` | `{ r: 180, g: 116, b: 28 }` | Dim state color; avoids dark base showing through as a dark hole | +| Dimmed landing point opacity | `landingPointVisual.dimmed.opacity` | `0.78` | Dim state opacity; no longer uses low alpha blending with dark base | + +## Satellites, Trails, and Footprints + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Satellite display radius offset | `SATELLITE_CONFIG.displayAltitudeOffset` | `8` | Satellite point position | +| Satellite dot base pixel size | `SATELLITE_CONFIG.dotBaseSize` | `2.8` | Point shader size | +| Satellite backdrop dot scale | `SATELLITE_CONFIG.dotBackdropScale` | `1.28` | Backdrop dot size | +| Satellite dot opacity range | `dotOpacityMin / dotOpacityMax` | `0.7 / 1.0` | Breathing animation | +| Satellite dot breathing speed | `SATELLITE_CONFIG.dotBreathingSpeed` | `0.12` | Dot opacity animation | +| Satellite backdrop renderOrder | inline | `5` | `satelliteBackdropPoints.renderOrder` | +| Satellite dot renderOrder | inline | `6` | `satellitePoints.renderOrder` | +| Satellite trail length | `SATELLITE_CONFIG.trailLength` | `10` | Trail buffer | +| Satellite trail line width | `SATELLITE_CONFIG.trailLineWidth` | `3` | Ribbon shader uniform | +| Selected ring size | `SATELLITE_CONFIG.ringSize` | `0.07` | Hover / locked ring sprite | +| Satellite overlay renderOrder | `SATELLITE_CONFIG.overlayRenderOrder` | `12` | Locked ring / halo / orbit | +| Footprint renderOrder | local `GROUND_FOOTPRINT_RENDER_ORDER` | `3` | Starlink footprint fill and Iridium coverage ring; must stay above land / texture / terrain surface layers | + +## AIS Vessels + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Vessel radius offset | `VESSEL_CONFIG.altitudeOffset` | `0.2` | Normal marker position, close to the real terrain base layer | +| Vessel track radius offset | `VESSEL_CONFIG.track.altitudeOffset` | `0.2` | Selected vessel track line, aligned to the vessel marker radius; the frontend anchors the track endpoint to the current marker position | +| Vessel renderOrder | local `VESSEL_RENDER_ORDER` | `4.4` | Normal marker and interactive overlay | +| Vessel track renderOrder | `VESSEL_RENDER_ORDER - 0.1` | `4.3` | Below vessel markers | +| Vessel point pixel size | local `VESSEL_POINT_SIZE` | `34` | Shared size for normal markers and hover / locked overlays | +| Default vessel render cap | `VESSEL_CONFIG.maxRenderedMarkers` | `0` | `0` means the frontend does not clip by default; positive values send `limit` and clip markers | +| Vessel texture canvas size | local `VESSEL_ATLAS_CELL_SIZE` | `128` | Canvas point texture | +| Course bucket count | local `VESSEL_COURSE_BINS` | `32` | Moving vessels are bucketed by COG to reduce draw calls while preserving direction | +| Vessel hover picking throttle | local `VESSEL_HOVER_PICK_INTERVAL_MS` | `100` | `main.js` hover picking | +| Vessel screen hit radius | local `VESSEL_POINTER_RADIUS_PX` | `22` | `main.js` screen-space picking | + +AIS vessel markers use batched `THREE.Points`, not one `THREE.Sprite` per vessel. Moving vessels stay triangular, anchored or slow vessels stay circular, and hover / locked states add a same-size glow overlay. Vessel type color and info-card type text must come from the same normalized result: `vessels.js` reads both backend `vessel_type_name` and AIS numeric `vessel_type`, derives the color-driving `type`, then exposes `vessel_type_display` for the info card, hover summary, and search results. + +## Compute Centers + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Compute center radius offset | `COMPUTE_CENTER_CONFIG.altitudeOffset` | `0.48` | Marker position | +| Compute center point size | local `COMPUTE_CENTER_POINT_SIZE` | `36` | Shared Interactable base size for normal markers and hover / locked overlays | +| Compute center asset fit size | local `COMPUTE_CENTER_ICON_FIT_SIZE` | `60` | Maximum SVG asset draw size inside the `128x128` atlas canvas, controlled by `icon.fitSize` | +| Compute center base opacity | `COMPUTE_CENTER_CONFIG.marker.baseOpacity` | `0.88` | Normal `PointsMaterial.opacity` | +| Supercomputer marker scale | `COMPUTE_CENTER_CONFIG.marker.supercomputerScale` | `12` | Legacy Sprite scale; not directly used by the current Interactable path | +| GPU cluster marker scale | `COMPUTE_CENTER_CONFIG.marker.gpuClusterScale` | `12` | Legacy Sprite scale; not directly used by the current Interactable path | +| Hover scale | `COMPUTE_CENTER_CONFIG.marker.hoverScale` | `1.16` | Hover overlay size multiplier | +| Locked scale | `COMPUTE_CENTER_CONFIG.marker.lockedScale` | `1.22` | Locked overlay size multiplier, with pulse | +| Dimmed scale / opacity | `dimmedScale / dimmedOpacity` | `0.82 / 0.34` | Dim state | +| Supercomputer color | `COMPUTE_CENTER_CONFIG.colors.supercomputer` | `"#38bdf8"` | Marker texture | +| GPU cluster color | `COMPUTE_CENTER_CONFIG.colors.gpu_cluster` | `"#2dd4bf"` | Marker texture | +| Linked color | `COMPUTE_CENTER_CONFIG.colors.linked` | `"#f8fafc"` | Linked state | +| Compute center renderOrder | local `COMPUTE_CENTER_RENDER_ORDER` | `4.5` | Surface facility below satellites | + +## BGP Observation + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| BGP event radius offset | `BGP_CONFIG.altitudeOffset` | `0.48` | BGP event Interactable marker | +| BGP collector radius offset | `BGP_CONFIG.collectorAltitudeOffset` | `0.2` | BGP collector Interactable marker, aligned with the vessel layer | +| BGP event point size | local `BGP_EVENT_POINT_SIZE` | `34` | Event Interactable base size, adjusted by severity through `getPointSizeMultiplier()` | +| BGP event symbol draw size | local `BGP_EVENT_SYMBOL_SIZE` | `60` | Event canvas symbol draw size inside the `128x128` atlas | +| BGP collector point size | local `BGP_COLLECTOR_POINT_SIZE` | `36` | Collector Interactable base size, adjusted by activity through `getPointSizeMultiplier()` | +| BGP collector asset fit size | local `BGP_COLLECTOR_ICON_FIT_SIZE` | `60` | Maximum `bgp-broadcast-pin.svg` draw size inside the atlas canvas | +| Event base scale | `BGP_CONFIG.marker.eventBaseScale` | `6.2` | Event ring anchor | +| Collector base scale | `BGP_CONFIG.marker.collectorBaseScale` | `7.4` | Collector halo / coverage animation anchor | +| Hover / dim scale | `hoverScale / dimmedScale` | `1.16 / 0.92` | Interaction states | +| Normal event opacity | `BGP_CONFIG.opacity.normal` | `0.78` | BGP event Interactable normal state | +| Hover opacity | `BGP_CONFIG.opacity.hover` | `1.0` | Hover state | +| Dimmed opacity | `BGP_CONFIG.opacity.dimmed` | `0.24` | Dim state | +| Collector opacity | `BGP_CONFIG.opacity.collector` | `0.62` | Collector state | +| Critical color | `BGP_CONFIG.severityColors.critical` | `0xff4d4f` | Critical event | +| High color | `BGP_CONFIG.severityColors.high` | `0xff9f43` | High-severity event | +| Medium color | `BGP_CONFIG.severityColors.medium` | `0xffd166` | Medium-severity event | +| Low color | `BGP_CONFIG.severityColors.low` | `0x4dabf7` | Low-severity event | +| Collector base color | `BGP_CONFIG.collectorColor` | `0x6db7ff` | Default collector color | +| Region color | `BGP_CONFIG.regionColor` | `0x2dd4bf` | Region overlay | + +## Celestial and Starfield + +| Name | Variable | Current Value | Location / Notes | +| --- | --- | --- | --- | +| Sky sphere radius | `CELESTIAL_CONFIG.skyRadius` | `2600` | Celestial background | +| Sky opacity | `CELESTIAL_CONFIG.skyOpacity` | `1` | Background material | +| Sun distance / scale | `sunDistance / sunScale` | `2150 / 78` | Sun sprite | +| Moon distance / scale | `moonDistance / moonScale` | `2050 / 38` | Moon sprite | +| Sun halo scale | `CELESTIAL_CONFIG.sunHaloScale` | `136` | Sun halo | +| Moon halo scale | `CELESTIAL_CONFIG.moonHaloScale` | `62` | Moon halo | +| Sun light color / intensity | `sunLightColor / sunLightIntensity` | `0xfff4df / 1.02` | Scene light | +| Back light color / intensity | `backLightColor / backLightIntensity` | `0x2b4c78 / 0.3` | Scene light | +| Star count | `STARFIELD_CONFIG.count` | `8000` | `createStars()` | +| Star radius range | `minRadius + radiusJitter` | `800 + 200` | Random distribution | +| Star color | `STARFIELD_CONFIG.color` | `0xffffff` | `PointsMaterial.color` | +| Star size | `STARFIELD_CONFIG.size` | `0.5` | `PointsMaterial.size` | diff --git a/docs/technical/en/earth-news-live-streams-collector-format.md b/docs/technical/en/earth-news-live-streams-collector-format.md new file mode 100644 index 00000000..1d64b635 --- /dev/null +++ b/docs/technical/en/earth-news-live-streams-collector-format.md @@ -0,0 +1,175 @@ +# News Live Streams Collector Format + +The `news_live_streams` collector accepts a "channel directory JSON" as input rather than scraping web pages directly. + +Goals: + +- Allow the backend to stably ingest live news streams from around the world +- Ensure the Earth page TV module always consumes a consistent structure +- Make it easy to integrate channel directories like `worldmonitor` that mix YouTube / HLS / iframe sources + +## Recommended JSON Structure + +```json +{ + "sources": [ + { + "id": "bbc-world-news", + "name": "BBC World News", + "provider": "BBC", + "region": "UK", + "language": "en", + "source_type": "youtube", + "youtube_video_id": "dQw4w9WgXcQ", + "youtube_channel": "https://www.youtube.com/@BBCNews", + "embed_url": "", + "stream_url": "", + "homepage_url": "https://www.youtube.com/@BBCNews/live", + "poster_url": "", + "sort_order": 220, + "is_enabled": true, + "notes": "Primary English global news channel" + }, + { + "id": "france24-en", + "name": "France 24 English", + "provider": "France 24", + "region": "France", + "language": "en", + "source_type": "hls", + "stream_url": "https://example.com/live.m3u8", + "homepage_url": "https://www.france24.com/en/live", + "sort_order": 230, + "is_enabled": true + }, + { + "id": "cctv4-page", + "name": "CCTV-4 Chinese International", + "provider": "CCTV", + "region": "China", + "language": "zh-CN", + "source_type": "iframe", + "embed_url": "https://tv.cctv.com/live/cctv4/", + "homepage_url": "https://tv.cctv.com/live/cctv4/", + "sort_order": 10, + "is_enabled": true + } + ] +} +``` + +## Field Conventions + +- `id`: unique identifier, should be stable +- `name`: channel display name +- `provider`: provider name +- `region`: country or region +- `language`: language code +- `source_type`: `iframe` / `hls` / `video` / `external` / `youtube` +- `embed_url`: page suitable for iframe embedding +- `stream_url`: direct video stream URL +- `homepage_url`: official website or channel page +- `youtube_video_id`: YouTube live video ID +- `youtube_channel`: YouTube channel handle or channel URL +- `poster_url`: cover image, optional +- `sort_order`: sort value, smaller = higher in the list +- `is_enabled`: whether enabled +- `notes`: brief notes + +## Panel Behavior Conventions + +- `youtube` + - Prefers `youtube_video_id` + - When embedding is not possible, at least keep `youtube_channel` or `homepage_url` for external opening +- `hls` / `video` + - Prefers `stream_url` +- `iframe` + - Prefers `embed_url` +- `external` + - No embedding attempt; only keeps external open link + +## Current Implementation Status + +- The backend settings page supports manually maintaining channel directories +- The Earth TV module merges: + - Manually configured sources + - Sources collected by the `news_live_streams` collector +- The current default fallback source is CCTV-4 Chinese International +- When no override is configured, `news_live_streams` defaults to `iptv-org`: + - `channels.json` + - `streams.json` + - `logos.json` + and automatically filters for news-category channel directories + +## Collector Configuration + +`news_live_streams` does not need a separate new page; it reuses Collector Settings under `/settings`: + +- `endpoint` + - Channel directory JSON API URL +- `auth_type` + - `none` / `bearer` / `api_key` / `basic` +- `headers` + - Additional request headers +- `config` + - Collector request and parsing behavior + +### Supported `config` Fields + +```json +{ + "timeout": 30, + "method": "GET", + "params": { + "region": "global" + }, + "body_type": "json", + "body": { + "include_disabled": false + }, + "response_path": "payload.channels" +} +``` + +- `timeout`: request timeout in seconds +- `method`: `GET` or `POST` +- `params`: query parameter object +- `body_type`: `json` or `form` +- `body`: request body for `POST` +- `json_body`: explicit JSON request body, takes priority over `body` +- `form_body`: explicit form request body, takes priority over `body` +- `response_path`: path to the channel array in the response JSON, supports dot notation, e.g.: + - `payload.channels` + - `data.items` + - `result.streams` + +### Authentication Details + +- `bearer`: uses `Authorization: Bearer ` +- `api_key`: sent as request header by default; if `auth_config.in = "query"`, sent as query param +- `basic`: uses HTTP Basic Authorization + +## Compatible Response Structures + +The collector first tries to read: + +- Top-level array +- Or an array under these common fields: + - `sources` + - `streams` + - `channels` + - `items` + - `results` + - `data` + +It also accepts these field aliases: + +- `id` / `source_id` / `slug` / `channel_id` / `code` +- `name` / `title` / `channel` / `display_name` +- `provider` / `publisher` / `network` +- `stream_url` / `stream` / `playback_url` / `hls_url` / `m3u8_url` +- `embed_url` / `embed` / `page_url` +- `homepage_url` / `source_url` / `website` +- `language` / `lang` / `locale` +- `youtube_video_id` / `video_id` +- `youtube_channel` / `channel_handle` diff --git a/docs/technical/en/earth-render-layer-order.md b/docs/technical/en/earth-render-layer-order.md new file mode 100644 index 00000000..3f15329c --- /dev/null +++ b/docs/technical/en/earth-render-layer-order.md @@ -0,0 +1,56 @@ +# Earth Render Layer Order + +This document records the current Earth renderer's layer order and the intent of each layer. When adjusting `renderOrder`, radius offsets, depth strategy, or pointer interaction, update this document accordingly. + +Note: the layer control panel order and the registration / startup load order are two separate semantics. + +| Order type | Current sequence | Notes | +| --- | --- | --- | +| Control panel order | Cables → Trails → Satellites → Compute Centers → BGP → Terrain → HD Texture → Cloud Layer → Border Lines → Grid | Controlled by `displayOrder`, sorted by operational relevance. | +| Registration / startup load order | Grid → Border Lines / Land-Ocean Base → HD Texture → Cloud Layer → Cables → Compute Centers → BGP → Satellites | Controlled by registration order and `startupPriority`, sorted surface-to-sky; the startup queue reads persisted layer visibility first, skips normal layers explicitly saved as hidden, and HD Texture does not download the texture when disabled; Border Lines are the exception: the land-ocean base always preloads, while the persisted state only controls interactive border lines and hover; Trails and Terrain are dependency/optional display layers and do not participate in normal startup data loading. | + +## Surface Layer Stack + +| Order | Layer | Source | Render / Radius Strategy | Depth / Interaction Strategy | Notes | +| --- | --- | --- | --- | --- | --- | +| -1000 | Celestial background mesh | `celestial.js` | Background sphere | Not part of surface picking | Behind all Earth content. | +| -1 | Earth occluder sphere | `earth.js` | Invisible inner sphere | Writes depth buffer | Occludes objects behind the Earth. | +| 0 | Earth base sphere | `earth.js` | `CONFIG.earthRadius` | Surface picking fallback target | Dark base; still visible when all optional map layers are off. | +| 0.2 | Country dark tint | `country-boundaries.js` | `tintAltitudeOffset` | Raycast disabled | Used when HD texture is off. | +| 0.86 | Land/ocean base fill | `country-boundaries.js` | `landAltitudeOffset`; ocean `#010609`, land `#080f1b` | Raycast disabled | Base map remains usable even when country borders are off. | +| 0.96 | HD Earth texture | `earth.js` | `textureOverlayAltitudeOffset` | Surface picking target when visible | HD texture always overlays the land/ocean base fill. | +| 1 | Atmospheric glow and clouds | `earth.js` | Atmosphere / cloud spheres | Not in normal object selection path | Cloud layer controlled by the "Cloud Layer" toggle. | +| 1 | Submarine cables | `cables.js` | `CABLE_CONFIG.line.renderOrder` | Cable picking path | Preserves existing cable layer level. | +| 1.2 | Real terrain | `earth.js`, `terrain.js` | `TERRAIN_CONFIG.baseRadiusOffset` plus terrain displacement | Raycast disabled | Terrain overlays HD texture; temporarily hidden when HD texture is off, restores to prior state when re-enabled. | +| 2.05 | Grid lines | `earth.js` | `CONFIG.earthRadius + 0.14` | Raycast disabled | Low-opacity lines over HD texture. | +| 2.2 | Country borders | `country-boundaries.js` | `lineAltitudeOffset` | Raycast disabled | Only needs to stay above HD texture. | +| 2.29 | Country border hover glow | `country-boundaries.js` | Hover radius + glow offset | `depthTest: false`, raycast disabled | Additive glow to reinforce border edge and terrain hover visibility. | +| 2.3 | Country border hover line | `country-boundaries.js` | `hoverAltitudeOffset` | `depthTest: false`, raycast disabled | Neon red-orange hover line; China and Taiwan share the same highlight group. | +| 3 | Satellite footprint fill / Iridium coverage ring | `satellites.js`, `iridium-footprint-adapter.js` | `GROUND_FOOTPRINT_RENDER_ORDER` | depth-tested; Iridium adapter fill / ring use the same renderOrder | Footprint above land / texture / terrain and country borders, below compute centers and satellites. | +| 3-5 | BGP markers and overlays | `bgp.js` | Each marker's own renderOrder | BGP picking path | Preserves existing BGP visual level. | +| 4.5 | Compute centers | `compute-centers.js` | `COMPUTE_CENTER_RENDER_ORDER` | Compute center picking path | Surface facilities, below satellites. | +| 5 | Satellite background dot | `satellites.js` | Fixed renderOrder | Screen-space satellite picking | Below satellite dots. | +| 6 | Satellite dots | `satellites.js` | Fixed renderOrder | Screen-space satellite picking | Satellite dots above footprints and compute centers. | +| 12+ | Satellite locked ring, halo, predicted orbit | `satellites.js` | `SATELLITE_CONFIG.overlayRenderOrder` and offsets | Satellite overlay path | Used for selected/locked satellite emphasis. | +| 98-100 | Sun / moon halo and sprite | `celestial.js` | Fixed renderOrder | Celestial picking disabled | Foreground celestial sprites. | + +## Toggle Behavior + +| Toggle | Behavior | +| --- | --- | +| HD texture off | Hides HD texture, enables country tint / base surface, disables terrain and day/night toggle interaction, and remembers terrain and day/night previous states. | +| HD texture on | Restores HD texture and the remembered terrain / day/night states. | +| Terrain on | Displayed above HD texture, but below country border hover, footprints, satellites, and other emphasis layers. | +| Cloud layer | Only controls cloud mesh visibility. | +| Border Lines off | Hides only interactive border lines and hover, clearing hover state; the land/ocean base fill remains as the Earth base map. | + +## Interaction Rules + +| Interaction | Current Rule | +| --- | --- | +| Earth coordinate hover | When HD texture is visible, uses the HD texture overlay as the surface picking target; otherwise uses the Earth base sphere. | +| Country border hover | Converts surface pick coordinates to lat/lon, then uses GeoJSON point-in-polygon; the border hover line itself does not receive raycasts. | +| Country border hover visual | On hover, dims normal border lines and draws no-depth-test glow and solid lines. | +| China / Taiwan hover | `CHN` and `TWN` are grouped in the same hover highlight group; the tooltip still shows the actually-hit feature. | +| Terrain | Acts as a visual layer only; `terrain.raycast` is disabled. | +| Satellites | Uses screen-space satellite picking to prevent footprints or surface layers from blocking satellite clicks. | diff --git a/docs/technical/en/earth-satellite-footprint-policy.md b/docs/technical/en/earth-satellite-footprint-policy.md new file mode 100644 index 00000000..57e25ceb --- /dev/null +++ b/docs/technical/en/earth-satellite-footprint-policy.md @@ -0,0 +1,198 @@ +# Earth Satellite Footprint Policy + +This document records the current product boundary, data rationale, and implemented behavior for `footprint` in the Earth satellite layer. The goal is to prevent the Starlink-specific ground coverage model from being misapplied to other constellations. + +Related context: + +- [Earth Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/earth-frontend-context.md) +- [Backend Collectors](/home/ray/dev/linkong/planet/docs/technical/en/backend-collectors.md) +- [backend/app/services/collectors/celestrak.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/celestrak.py) +- [frontend/public/earth/js/satellites.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/satellites.js) + +## Current Goal + +- Define which non-Starlink satellites should not show a ground footprint +- Define which constellations may have their own footprint in the future but cannot reuse the Starlink bowtie / GSO-gap model +- Solidify this policy as an executable implementation boundary, not leave it scattered across visual parameters + +## Current Local Categories + +Current CelesTrak satellite groups in [backend/app/services/collectors/celestrak.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/celestrak.py) include: + +- `starlink` +- `gps-ops` +- `galileo` +- `glonass` +- `beidou` +- `leo` +- `geo` +- `iridium-next` + +Non-Starlink categories: + +- `gps-ops` +- `galileo` +- `glonass` +- `beidou` +- `leo` +- `geo` +- `iridium-next` + +## Research Conclusions + +### 1. GNSS / RNSS: `gps-ops`, `galileo`, `glonass`, `beidou` + +Do not draw a localized ground footprint by default. + +Reason: + +- Public sources emphasize `Earth-pointing`, `Earth coverage`, `continuous global coverage` +- The public semantic of these systems is global navigation / timing coverage, not the localized spot footprint associated with Starlink's end-user service + +More appropriate representation: + +- Default: show only the satellite body and orbit +- If future needs require showing "service reachability," only a weak global coverage semantic is appropriate — do not draw a localized ground spot + +References: + +- [GPS III EC Antenna Patterns](https://www.navcen.uscg.gov/sites/default/files/pdf/gps/GPS_ZIP/GPS_III_EC_Antenna_Patterns_SVN_74_75_76_77_78.pdf) +- [ESA Galileo satellites](https://www.esa.int/Applications/Satellite_navigation/Galileo/Galileo_satellites) +- [Navipedia Galileo General Introduction](https://gssc.esa.int/navipedia/index.php/Galileo_General_Introduction) +- [BeiDou official overview](https://www.beidou.gov.cn/xt/gfxz/201812/P020190117356387956569.pdf) +- [GPS.gov GNSS overview](https://www.gps.gov/systems/gnss/) + +### 2. `iridium-next` + +Can have a footprint, but cannot reuse Starlink's single bowtie footprint. + +Reason: + +- Iridium NEXT public documentation emphasizes a fixed multi-spot beam system +- Public examples commonly show `48 fixed spot beams in 4 tiers` +- This is not the same problem as Starlink's "single satellite, single primary footprint, with GSO gap" business visualization + +More appropriate representation: + +- Default: still do not draw a Starlink-style ground footprint +- Future implementation: connect an independent Iridium multi-beam adapter layer +- Visually closer to multi-beam clusters / honeycomb / layered beams, not a single bowtie spot + +Reference: + +- [Iridium Satellite Spot Beam Coverage on the US](https://www.mathworks.com/help/phased/ug/iridium-satellite-spot-beam-coverage-on-the-us-1.html) + +### 3. `geo` + +Do not draw a unified footprint by default. + +Reason: + +- GEO communication satellites may use global beam, zone beam, spot beam, or steerable spot beam +- Without operator / payload / beam contour metadata, drawing a unified footprint is very likely incorrect + +More appropriate representation: + +- Default: show only the GEO belt and satellite parking position semantics +- Only allow footprint drawing when beam contour / operator metadata is available + +Reference: + +- [ITU Handbook on Satellite](https://www.itu.int/dms_pub/itu-r/opb/hdb/R-HDB-42-2002-PDF-E.pdf) + +### 4. `leo` (generic) + +Do not draw a footprint by default. + +Reason: + +- The `leo` group is too mixed — it may include communication, remote sensing, experimental, and observation satellites +- Without mission / payload / antenna pattern metadata, there is no basis for a service-coverage visualization + +More appropriate representation: + +- Default: show only the satellite and orbit +- Future: if subdivided by operator / mission subtype, decide then whether to introduce an independent coverage mode + +## Product Policy + +Current unified policy: + +- `Starlink` + - Keep the current dedicated `ground_footprint` logic +- `Iridium NEXT` + - Reserve an independent adapter layer + - Do not reuse Starlink footprint currently +- `GPS / Galileo / GLONASS / BeiDou` + - No ground footprint +- `GEO` + - No footprint without beam metadata +- `Generic LEO` + - No footprint without mission metadata + +## Implemented Behavior + +This implementation only does the minimum executable version and does not change existing Starlink visual parameters: + +1. Backend passes constellation group and footprint policy hint to the frontend + +- CelesTrak collector stores `GROUP` in `metadata.constellation_group` +- Visualization API outputs: + - `properties.constellation_group` + - `properties.footprint_policy` + +Current policy values: + +- `starlink_ground_footprint` +- `iridium_coverage_ring` +- `none` + +Relevant code: + +- [backend/app/services/collectors/celestrak.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/celestrak.py) +- [backend/app/api/v1/visualization.py](/home/ray/dev/linkong/planet/backend/app/api/v1/visualization.py) + +2. Frontend makes footprint a capability-gated renderer + +- `ground_footprint` is only actually enabled when `footprint_policy === starlink_ground_footprint` +- `iridium-next` no longer falls back to a placeholder branch; it goes through an independent Iridium coverage ring adapter +- Other non-Starlink satellites automatically fall back to `self_glow` even if the user globally selects `ground_footprint` + +Relevant code: + +- [frontend/public/earth/js/satellites.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/satellites.js) +- [frontend/public/earth/js/iridium-footprint-adapter.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/iridium-footprint-adapter.js) + +3. Satellite info card shows capability, not just orbital parameters + +- Satellite details now clearly display: + - `Constellation / Group` + - `Coverage Capability` + - `Current Display` + - `Coverage Model` +- Users can directly see: + - Whether the current satellite supports footprint + - Whether the current display has been fallen back due to capability gating + - That Iridium and Starlink use different models + +Relevant code: + +- [frontend/public/earth/js/main.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/main.js) +- [frontend/public/earth/js/info-card.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/info-card.js) + +## Current Implementation Boundary + +This boundary must be maintained: + +- Starlink's footprint parameters and shader logic serve Starlink only +- Non-Starlink capability decisions belong to the "policy layer / adapter layer" +- Do not re-mix different constellations' coverage models into the same parameter set +- `iridium-next` has been separated into an independent adapter and should continue along this boundary rather than adding more if/else to the existing Starlink bowtie + +## Recommended Next Steps + +If continuing forward, the recommended order is: + +1. Create a dedicated footprint adapter for `iridium-next` +2. Add a read-only indicator in the UI to tell users whether the current satellite supports footprint +3. If GEO beam contour / operator metadata becomes available, enable operator-specific footprint for GEO diff --git a/docs/technical/en/earth-toolbar-overlay-coordination.md b/docs/technical/en/earth-toolbar-overlay-coordination.md new file mode 100644 index 00000000..7e92050c --- /dev/null +++ b/docs/technical/en/earth-toolbar-overlay-coordination.md @@ -0,0 +1,86 @@ +# Earth Toolbar And Overlay Coordination + +This document describes the current coordination rules between the Earth toolbar buttons and the search panel, settings modal, news/live panel, and layer panel. Use this matrix when changing interactions, adding buttons, or adjusting panels so one action does not close an unrelated overlay. + +Related entries: + +- [Earth Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/earth-frontend-context.md) +- [Frontend Layout Guidelines](/home/ray/dev/linkong/planet/docs/technical/en/frontend-layout-guidelines.md) + +## Toolbar Button Directory + +The toolbar is marked by `.earth-toolbar-btn` in [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html): + +| ID | Title | Type | Overlay / action | +|----|-------|------|------------------| +| `layer-action` | Layers | Overlay toggle | HUD panel `layer-toggles` on desktop / mobile drawer `layers` card | +| `search-action` | Search | Overlay toggle | Search panel on desktop / mobile drawer `search` card | +| `rotate-toggle` | Auto rotate | Standalone toggle | No overlay | +| `toggle-tv` | News live | Overlay toggle | Media panel `media-panel` with TV and News tabs | +| `reload-data` | Reload data | Standalone action | No overlay | +| `zoom-trigger` | Zoom control | Floating menu | Zoom floating menu | +| `settings-trigger` | Settings | Overlay toggle | Settings modal on desktop / mobile drawer `settings` card | +| `reset-view` | Reset view | Standalone action | No overlay | +| `layout-toggle` | Maximize layout | Standalone toggle | No overlay | + +## Shared Coordination Entry Point + +[controls.js::closeTransientMobileOverlays](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) is the shared coordinator for deciding what should close when an overlay opens. + +Every path that opens a fullscreen-style overlay calls `closeTransientMobileOverlays({ except })`, where `except` names the overlay that should stay open: + +```js +closeTransientMobileOverlays({ except: "search" }); +closeTransientMobileOverlays({ except: "settings" }); +closeTransientMobileOverlays({ except: "media" }); +closeTransientMobileOverlays({ except: "layer-toggles" }); +``` + +Current `except` values are `"search"`, `"settings"`, `"media"`, `"layer-toggles"`, or omitted to close all transient overlays. + +## Close Matrix + +`close` means the overlay closes; `keep` means it remains open. + +| Action | Search | Settings | Mobile layers drawer | News/live | +|--------|:------:|:--------:|:--------------------:|:---------:| +| Open search (`except: "search"`) | self | close | close | keep | +| Open settings (`except: "settings"`) | close | self | close | keep | +| Open news/live (`except: "media"`) | close | close | close | self | +| Open mobile layers (`except: "layer-toggles"`) | close | close | self | close | +| Close all (`except: null`) | close | close | close | close | + +Examples: + +- Clicking toolbar Settings closes search and the mobile layer drawer, but keeps news/live open. +- Clicking toolbar Layers on mobile opens the `layers` drawer and closes search, settings, and news. +- Clicking News Live closes search, settings, and the layer drawer, then toggles the media panel. + +## Design Rules + +1. **Floating menus such as `zoom-trigger` are not overlays.** They use `bindFloatingMenu` and are managed separately by `closeFloatingMenus()`. Opening any overlay first closes floating menus. +2. **Desktop `layer-toggles` is a persistent HUD panel.** `closeTransientMobileOverlays` only closes it when `activeMobileDrawerId === "layer-toggles"`, so desktop search, settings, and news do not disturb the layer panel. +3. **News/live is independent from settings.** Users often adjust collector settings while watching news, so opening settings does not close the media panel. This became an invariant after the May 2026 coordination patch. +4. **Search and news are both primary information overlays.** Search opens without closing news, and news opens without closing search. If product direction changes, update both sides in `closeTransientMobileOverlays` so the matrix stays symmetric. +5. **Mobile drawers are fullscreen-focus states.** Any mobile drawer, whether layers, search, or settings, uses `setMobileDrawerState` and closes other overlays. +6. **Escape has a fixed close order.** See [controls.js::setupKeyboardControls](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js): search, settings, mobile drawer, floating menu, toolbar hub, locked object. + +## Adding A Button Or Overlay + +1. Add the button in the `.earth-toolbar` container in [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html), using the existing `floating-btn liquid-glass-surface earth-toolbar-btn` class pattern. +2. Decide whether it is a standalone action, a floating menu, or a mutually coordinated overlay. +3. For a coordinated overlay, call `closeTransientMobileOverlays({ except: "" })` when opening it. +4. Add the reciprocal close branch inside `closeTransientMobileOverlays`, so other overlays can close yours. +5. If the new overlay should coexist with an existing overlay, exclude that peer on both sides of the matrix. +6. Add an Escape close path in `setupKeyboardControls`. +7. On mobile, use `setMobileDrawerState({ open: true, card: "" })` for drawer-style panels. + +## Current Implementation Locations + +- Coordinator: [controls.js::closeTransientMobileOverlays](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- Settings overlay: [controls.js::openSettingsModal / closeSettingsModal](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- Search overlay: [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js), imported from the search module +- News/live overlay: [tv.js::setTVPanelVisible](/home/ray/dev/linkong/planet/frontend/public/earth/js/tv.js), with the News tab in [news.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/news.js) +- Mobile layer drawer: [controls.js::setMobileDrawerState](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- Floating menu: [controls.js::bindFloatingMenu](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- Toolbar DOM: [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) diff --git a/docs/technical/en/frontend-admin-frontend-context.md b/docs/technical/en/frontend-admin-frontend-context.md new file mode 100644 index 00000000..ed5130ec --- /dev/null +++ b/docs/technical/en/frontend-admin-frontend-context.md @@ -0,0 +1,295 @@ +# Admin Frontend Context + +This document describes the current real structure of the console frontend. The goal is to help future page development, table refactoring, layout governance, and state consolidation quickly find the right entry points. + +Related references: + +- [Project Rules](/home/ray/dev/linkong/planet/rules.md) +- [Frontend Layout Guidelines](/home/ray/dev/linkong/planet/docs/technical/en/frontend-layout-guidelines.md) + +## Current Goal + +The console frontend is a backend workbench, not a display-style dashboard. Current constraints: + +- Pages default to a single-screen work area +- Primary interaction happens through in-module scrolling, not relying on the whole page growing infinitely +- Lists, tables, and analysis pages prioritize keeping the main work area visible +- Common layout, scrollbar, and table scroll behavior should be reused across pages + +## Current Route Entry Points + +Main entry point: + +- [App.tsx](/home/ray/dev/linkong/planet/frontend/src/App.tsx) + +Current admin-related routes: + +- `/admin` +- `/users` +- `/datasources` +- `/data` +- `/alerts/system` +- `/alerts/bgp` +- `/alerts/situational` +- `/bgp` +- `/playground` +- `/settings` + +`/earth` is a standalone display page and is not part of the console shell. + +## Current Page Shell + +The console shared shell is at: + +- [AppLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/components/AppLayout/AppLayout.tsx) + +Responsibilities: + +- Left-side navigation +- Collapse and expand +- Current account / version information +- Content area height closure +- Site-wide unified sidebar scrollbar + +Current structure: + +```tsx + + ... + + +
{children}
+ + + +``` + +Future console pages should adapt to this shell rather than redefining full-page height semantics. + +## Current Shared Components + +### 1. `Scrollbar` + +File: + +- [Scrollbar.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/Scrollbar.tsx) + +Purpose: + +- Ordinary content containers like the console sidebar +- Internally manages visibility, thumb size, drag, and dual-axis overflow detection + +Current constraint: + +- The scrollbar must be a floating overlay that does not participate in layout +- Should leave no visible trace when there is no overflow +- Real scrolling is still handled by the native container; only the visible layer and interaction layer are replaced + +### 2. `ScrollbarOverlay` + +File: + +- [ScrollbarOverlay.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/ScrollbarOverlay.tsx) + +Purpose: + +- Areas like Ant Table that already have an internal scroll container +- Does not take over scroll semantics; only adds a new scrollbar visible layer + +Current usage: + +- Data sources +- Collected data +- User management +- Settings page +- Alerts page +- BGP page + +### 3. `TableScrollRegion` + +File: + +- [TableScrollRegion.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/TableScrollRegion.tsx) + +Purpose: + +- Provides a unified wrapper for table scroll areas +- New table pages should reuse this rather than repeating the "table area + overlay scrollbar" boilerplate + +### 4. `SegmentedControl` + +Files: + +- [SegmentedControl.tsx](/home/ray/dev/linkong/planet/frontend/src/components/SegmentedControl/SegmentedControl.tsx) +- [SegmentedControl.css](/home/ray/dev/linkong/planet/frontend/src/components/SegmentedControl/SegmentedControl.css) + +Purpose: + +- Segmented controls for language, theme, mode, or other 2 to 3 option settings +- Settings that need the shared animated slider, active state, and compact button layout +- The `/docs` footer language switcher and theme switcher already reuse it + +Interface semantics: + +- `options`: each option contains `value` and `label`, with optional `icon` and `title` +- `value`: current active value +- `onChange`: called when the selected option changes +- `ariaLabel`: accessible name for the control +- `className`: page-level hook for size or local style overrides + +Current constraints: + +- The component owns slider count, position, and spring-like transition +- Feature pages should only pass options and state, not recreate private slider DOM +- Prefer CSS variable overrides for colors instead of hard-coding theme colors in feature components +- Best for a small set of mutually exclusive choices; do not use it as a long list, navigation menu, or select replacement + +### 5. `MarkdownRenderer` + +File: + +- [MarkdownRenderer.tsx](/home/ray/dev/linkong/planet/frontend/src/components/MarkdownRenderer/MarkdownRenderer.tsx) + +Purpose: + +- Renders Markdown content for `/docs` +- Supports headings, lists, blockquotes, code blocks, tables, and basic inline formatting +- Code blocks and tables reuse `Scrollbar` so horizontal content does not blow out the docs page +- Docs content is returned by backend `/api/v1/docs/...` endpoints according to Gatekeeper permissions; the frontend only renders content visible to the current user + +Current constraints: + +- It is not a full GitHub Markdown engine; it only covers the syntax currently needed by project docs +- Internal document links should be converted to `/docs/:slug` through `transformLink` +- Heading anchors are injected through `getHeadingId`, keeping route state outside the renderer + +### 6. `TableActions` + +File: + +- [TableActions.tsx](/home/ray/dev/linkong/planet/frontend/src/components/TableActions/TableActions.tsx) + +Purpose: + +- Shared action entry for table operation columns +- Shows inline actions when expanded +- Uses a more-actions dropdown when collapsed + +Companion export: + +- `actionCellProps`: for action-column `onCell`, preventing action buttons from being ellipsized or wrapped + +## Current State Sources + +### 1. Auth State + +File: + +- [auth.ts](/home/ray/dev/linkong/planet/frontend/src/stores/auth.ts) + +Responsibilities: + +- Token +- Current user +- Gatekeeper groups +- Login / logout + +`App.tsx` uses it to decide whether to redirect to the login page. `/docs` remains a public route, but the backend decides the visible catalog and content from the token; anonymous visitors only receive public docs. + +### 2. Business Data Gateway + +AI / situational awareness related services are currently in: + +- [http-gateway.ts](/home/ray/dev/linkong/planet/frontend/src/services/situational-awareness/http-gateway.ts) +- [port.ts](/home/ray/dev/linkong/planet/frontend/src/services/situational-awareness/port.ts) +- [types.ts](/home/ray/dev/linkong/planet/frontend/src/services/situational-awareness/types.ts) + +Constraints: + +- Pages must not scatter URL construction directly +- Define boundaries through port/types first +- Then implement via http/mock gateway + +## Current Page Layer Recommendations + +### 1. Dashboard and Summary Pages + +Example: + +- [Dashboard.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Dashboard/Dashboard.tsx) + +Priority goals: + +- Stable header +- Summary cards compact first +- Main work area occupies primary height + +### 2. Table Pages + +Examples: + +- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx) +- [DataList.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataList/DataList.tsx) +- [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Users/Users.tsx) +- [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx) + +Constraints: + +- Prefer internal scrolling +- Do not let tables blow out the full page +- New table areas should reuse `TableScrollRegion` / `ScrollbarOverlay` + +### 3. Complex Workspace Pages + +Examples: + +- [BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx) +- [Playground.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Playground/Playground.tsx) + +Constraints: + +- Tab content must not share the same height logic +- Table tabs, Markdown tabs, and config tabs each need their own scroll responsibility +- AI result areas and long text areas should maintain a minimum readable height + +## Current Layout Constraints + +These principles have been repeatedly validated in the project: + +1. Parent container height chain must close +2. `min-height: 0` must not be omitted +3. Overflow responsibility must be explicit +4. Do not use `overflow: hidden` to mask structural issues +5. Do not compress the main work area to make summary cards show completely +6. Custom scrollbars must be floating overlays; they must not squeeze content width + +For detailed experience, see: + +- [Frontend Layout Guidelines](/home/ray/dev/linkong/planet/docs/technical/en/frontend-layout-guidelines.md) + +## Recommended Change Approach + +For future console page changes: + +1. Confirm whether the page is a summary page, table page, or complex workspace +2. Integrate into the existing shell and scroll semantics first +3. Reuse shared scroll components +4. Handle visual and detail interactions last + +Do not write local CSS patches first, then retrofit the structure. + +## Current Clear Boundary + +The console frontend and the Earth frontend are not the same system: + +- Console frontend: React + Ant Design workbench +- Earth frontend: independent native HUD system under `public/earth` + +Therefore: + +- Do not move Earth's HUD / animations / state machine directly into the console +- Do not force the console's table / scroll strategy onto the Earth HUD + +For Earth-related structure, see: + +- [Earth Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/earth-frontend-context.md) diff --git a/docs/technical/en/frontend-layout-guidelines.md b/docs/technical/en/frontend-layout-guidelines.md new file mode 100644 index 00000000..5b54ff3f --- /dev/null +++ b/docs/technical/en/frontend-layout-guidelines.md @@ -0,0 +1,309 @@ +# Frontend Layout Guidelines + +Admin pages in this project default to a "single-screen workspace" layout standard. The goal is not to prevent all overflow, but to ensure that under common desktop viewports: + +- The main page structure is visible within one screen +- The user can simultaneously see the page header, summary area, and main workspace +- Overflow content scrolls within its module, rather than stretching the entire page vertically + +Current recommended reference implementations: + +- [frontend/src/pages/BGP/BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx) +- [frontend/src/index.css](/home/ray/dev/linkong/planet/frontend/src/index.css) + +## Core Principles + +### 1. Pages Should Prioritize a Single-Screen Workspace + +Admin pages default to: + +- Header: title, description, main actions +- Main workspace: stats cards, tables, charts, lists, tabs + +Recommended structure: + +```tsx + +
+
...
+
...
+
+ +``` + +Total page height should be bounded within the `AppLayout` content area, not allowed to grow naturally downward without limit. + +### 2. Scrolling Should Happen Inside Modules + +If tables, logs, long lists, or chart details overflow their space: + +- Let the card scroll internally +- Let the table scroll internally +- Let the tab content area scroll internally + +Do not rely on full-page scrolling to "solve" the space problem. + +### 3. The Main Workspace Must Get the Most Space + +The most important module on a page must be the visual and spatial lead. Typically ensure: + +- Header always visible +- Summary area height controlled +- Main table / chart / analysis area occupies more than 50% of visible height + +If a page has multiple large modules, priority order is: + +1. First compress the description and summary areas +2. Then move secondary modules into tabs or switch views +3. Only then consider adding more full-page scrolling + +### 4. Small Screens and High Zoom Must Enter Compact Mode + +When window height is low, width is narrow, or system zoom is high, actively switch to a compact layout: + +- Reduce card padding +- Reduce header and cell spacing +- Convert summary area to a more compact single-row or horizontal-scroll layout +- Move secondary modules into tabs, drawers, or collapsed areas + +Compact mode goal: maintain usability, not just shrink all text and controls. + +### 5. Overflow Responsibility Must Be Explicit + +Large content blocks on the page must explicitly define: + +- Who is responsible for filling remaining height +- Who is responsible for clipping +- Who is responsible for scrolling + +Common requirements: + +- Parent container chain needs `min-height: 0` +- Workspace containers typically need `display: flex` +- The real scroll node must explicitly use `overflow: auto` + +### 6. Cards Must Not Be Compressed to Unreadable + +Historical problems have not been "missing scrollbars," but: + +- Cards compressed by `flex` to only a tiny visible area +- Text can render but cannot be read completely +- Content exists but is cut off by `overflow: hidden` + +Future constraints: + +- First ensure cards have a readable minimum height +- If further compression affects readability, switch to internal scrolling +- Do not compress body text, tables, or description areas into unreadable strips just to "maintain one screen" + +### 7. Tabs Are Not Inherently Safe Layout Containers + +Historical regressions with Tabs include: + +- Hidden tab panes reappearing due to custom `display: flex` +- All tabs having the same height/overflow rules forced on them +- Table tabs work, but markdown / help / diagnostics tabs get crushed + +Constraints: + +- Each type of content inside `Tabs` must define its own layout strategy +- Table tab: "fixed height + internal scrolling" +- Docs/Markdown tab: better as "tab pane self-scrolls + content normal document flow" +- If overriding component library styles, verify the hidden state still holds + +### 8. Summary Areas Should Enter Compact Mode First, Not Compress Body + +Historical experience shows the top summary cards are most often mishandled: + +- They frequently get forcibly narrowed to "fit everything" +- Then the body, tables, and AI result areas all lose their main space + +Unified constraint: + +- On small screens or high zoom, summary cards should first: + - Reduce padding + - Switch to horizontal scrolling + - Switch to a more compact grid +- Do not sacrifice the main workspace's visible area first + +### 9. Long-Document Content Should Prioritize Reading Experience + +Content like the following cannot directly apply "table workspace" logic: + +- AI briefs +- Runtime logs +- Raw JSON +- Help text +- Multi-paragraph descriptive text + +These areas should prioritize: + +- Stable title and meta information visibility +- Body has a clear minimum readable height +- Body scroll strategy defined separately +- Support for Markdown tables, dividers, quotes, code blocks + +### 10. Height Critical Paths Should Use Fewer Wrapper Layers + +Many scroll problems historically were not in the component itself, but came from an extra wrapper layer: + +- Height chain broken +- `min-height: 0` not passed down +- `overflow` responsibility absorbed + +Therefore: + +- For height-critical areas, prefer the most direct DOM structure +- When using `Space`, extra wrapper `div`, or third-party layout containers, verify they don't change scroll and height semantics +- If an area shows "content is there but only a sliver is visible," first suspect an intermediate wrapper layer + +## Historical Pitfalls + +From Earth, Playground, BGP, DataSources page bugfixes, several high-frequency pitfall types: + +### 1. Using `overflow: hidden` to Mask Layout Problems + +Superficially the page looks "clean," but actually causes: + +- Content getting clipped +- Tab content reduced to a sliver +- Panel renders successfully but users can't see it + +Correct approach: + +- Let the real content node scroll +- Don't let upper containers unconditionally clip all child content + +### 2. Treating All Tabs as the Same Content Type + +Tables, Markdown, help cards, and log streams have completely different space requirements. + +Correct approach: + +- Table: fixed workspace + internal scrolling +- Document: normal flow content + pane-level scrolling +- Side description: content-driven height, not forced to fill + +### 3. Only Doing Visual Shrinking, Not Space Reallocation + +This causes: + +- Card text truncated +- Table shows only 1-2 rows +- Buttons and filters crammed together + +Correct approach: + +- Compact mode prioritizes re-layout +- Summary area horizontal scrolling +- Collapse / hide secondary modules + +### 4. Incomplete Parent Container Height Chain + +This is the most common cause of internal scrolling failing. + +Inspection order: + +1. Does the outer layer actually have a determined height? +2. Does the flex parent have `min-height: 0`? +3. Does the real scroll node explicitly use `overflow: auto`? +4. Have intermediate wrapper layers silently changed layout semantics? + +### 5. UI State and Display State Out of Sync + +Repeated in Earth-related changes: + +- Layer hidden, but hover/lock still active +- Tooltip still showing stale object +- Legend not switching with the state + +These constraints also apply to admin pages: + +- Hidden, unmounted, or switched-out content should not retain active interaction state + +## Recommended Implementation Patterns + +### Page Shell + +Reuse existing common structures in the project: + +- `.dashboard-content-inner` +- `.page-shell` +- `.page-shell__header` +- `.page-shell__body` +- `.table-scroll-region` + +Do not invent a completely different height and scroll semantics for each page. + +### Table Workspace + +Recommended pattern: + +```tsx + +
+ + + +``` + +Requirements: + +- Tables should scroll inside their card +- `scroll.y` should come from actual available height calculation, not a completely static magic number +- Parent container chain must ensure header, body, content overflow all close inside the table + +### Multi-Module Pages + +If a page has: + +- Summary cards +- Table +- Anomaly details +- Recent events + +Do not simply stack all modules vertically. Prefer: + +- Top summary + single main workspace at bottom +- Tab-switch multiple secondary data views +- Left-right split with each column scrolling independently + +## Discouraged Patterns + +The following patterns are considered non-compliant with this project's page standard: + +- Relying on full-page vertical scrolling to display the main workspace +- Stacking 3-4 large cards vertically on one page, each wanting to display fully +- Table without internal scrolling, causing only 1-2 rows visible after zoom +- Parent container missing `min-height: 0`, causing internal scrolling to fail +- Only doing visual shrinking without addressing real space allocation + +## Page Acceptance Checklist + +Before submitting, check at minimum: + +- Can page header, summary area, and main workspace appear simultaneously? +- Does the main workspace get the most height on the page? +- When table or detail overflows, does the scrollbar appear inside the module? +- Is the card compressed to the point where text doesn't display completely? If so, has it switched to internal scrolling? +- Is it still usable at browser zoom `125%` / `150%`? +- In a low-height window, is there still a reasonable number of visible content rows? +- Are Tabs, Card, Table still operable when overflowing? +- Do non-table tabs (Markdown, help text, logs) have their own independent and reasonable scroll strategy? + +## Implementation Order + +When adding or refactoring admin pages, design in this order: + +1. Define the main workspace first +2. Determine which modules must always be visible +3. Then handle styling and visual hierarchy + +Simply put: + +- First ensure correct space allocation +- Then handle scroll boundaries +- Finally handle aesthetics diff --git a/docs/technical/en/location-pipeline-development.md b/docs/technical/en/location-pipeline-development.md new file mode 100644 index 00000000..6147156a --- /dev/null +++ b/docs/technical/en/location-pipeline-development.md @@ -0,0 +1,200 @@ +# Shared Location Resolution Pipeline Development Guide + +`backend/app/services/location/` is the shared abstraction for any "given a record, decide its lat/lon" workflow. Compute centers, BGP collectors, and BGP events now run on this pipeline. Future entities such as satellite ground stations, user-claimed points, and IXP facilities should plug in here instead of creating another geocoding path. + +For the user workflow, see [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md). + +## Design Goals + +Historically compute centers had their own four-tier chain, BGP collectors used a hard-coded dictionary, and BGP events inherited collector coordinates. These implementations did not share code, and new algorithms had no stable insertion point. + +The refactored rules: + +- Share the `LocationResolver` protocol and `LocationPipeline` orchestrator. +- Domain modules only build `LocationQuery` and choose resolver order. +- New algorithms join by adding resolver classes, without changing ingestion, API, or frontend envelopes. +- Earth renders only city-level or better locations. +- Local JSON registries are not runtime candidate sources for compute centers or BGP collectors; persisted location facts live in database dimension tables. + +## Core Interfaces + +```python +@dataclass(frozen=True) +class LocationQuery: + name: str | None + aliases: tuple[str, ...] + city: str | None + country: str | None + region: str | None + source_latitude: float | None + source_longitude: float | None + extra: Mapping[str, Any] +``` + +```python +@dataclass(frozen=True) +class LocationCandidate: + latitude: float + longitude: float + display_name: str + precision: str + confidence: float + source: str + needs_confirmation: bool + matched_fields: tuple[str, ...] + suggested_registry_entry: dict | None +``` + +```python +class LocationResolver(Protocol): + name: str + def resolve(self, query: LocationQuery) -> ResolverOutput: ... +``` + +`LocationPipeline.collect_candidates()` returns sorted candidates plus `attempted_queries`; `resolve_best()` returns the best candidate with diagnostics. The default sort key ranks source, precision, and confidence, then deduplicates candidates with the same source and rounded coordinates. + +## Built-In Resolvers + +| Resolver | File | Responsibility | +| --- | --- | --- | +| `SourceCoordinatesResolver` | `resolvers/source_coordinates.py` | Emits `precision="precise"` when the record already has lat/lon | +| `RegistryResolver` | `resolvers/registry.py` | Legacy generic resolver; current compute-center and BGP runtime paths do not use it to generate candidates | +| `NominatimResolver` | `resolvers/nominatim.py` | Runs a domain query plan against Nominatim with LRU cache and rate limiting | +| `InheritFromAnotherEntityResolver` | `resolvers/inherit.py` | Wraps an externally resolved entity location as a candidate | + +`RegistryResolver` remains available for future controlled import scenarios, but it should not be reconnected as a hard-coded hint source for compute centers or BGP. Matching common fields such as `operator` or `city` was the main reason multiple entities could collapse onto the same point. + +## Current Domain Pipelines + +### Compute Centers + +Entry points: + +- [compute_center_locations.py](/home/ray/dev/linkong/planet/backend/app/services/compute_center_locations.py) + +Resolver order: + +```python +SourceCoordinatesResolver() +StoredComputeCenterLocationResolver() +``` + +The main map startup path is source coordinates first, then the database-backed current-location table. The table is `compute_center_locations`, keyed by `(source, source_id)`, and stores manually accepted locations or true coordinates migrated from source records. `init_db()` only migrates source records that already contain real coordinates; it does not import old hard-coded hints and does not run ROR, Nominatim, or LLM geocoding during startup. + +Candidate collection is intentionally separate from rendering. `collect_location_candidates()` builds ROR and Nominatim/OpenStreetMap queries from source fields, but it does not emit the current `compute_center_locations` row as a candidate. After a user accepts a candidate, the save endpoint upserts it into the dimension table; the next map refresh renders it through `StoredComputeCenterLocationResolver`. + +`resolve_compute_center_location()`, `resolve_compute_center_location_full()`, and `collect_location_candidates()` remain the domain API. `visualization.py` consumes that API and no longer owns coordinate hints, country-centroid fallbacks, or Nominatim details. + +GeoJSON output includes only `RENDERABLE_PRECISIONS`. Unresolved records are returned in `unresolved` with `failure_reason`, `attempted_queries`, `source_id`, `record_id`, and related diagnostics. + +### BGP Collectors + +Entry points: + +- [bgp_collector_locations.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_collector_locations.py) +- [bgp_collector_location.py](/home/ray/dev/linkong/planet/backend/app/models/bgp_collector_location.py) + +Resolver order: + +```python +SourceCoordinatesResolver() +StoredCollectorLocationResolver() +NominatimResolver(_bgp_collector_query_plan) +``` + +The 23 RIPE RIS collector coordinates moved from the old table into the `bgp_collector_locations` dimension table with `source=legacy_seed` and `needs_confirmation=true`. The legacy dictionary is still maintained from the DB-backed cache for compatibility; manual candidate collection uses stored site/city/country as context but does not emit stored rows as candidates. + +### BGP Events + +Entry point: + +- [bgp_event_locations.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_event_locations.py) + +Resolver order: + +```python +SourceCoordinatesResolver() +InheritFromAnotherEntityResolver(_inherit_from_owning_collector) +``` + +Event inheritance uses a strict owning-collector lookup and does not run the full fuzzy collector registry. Future ASN facility, PrefixGeo, or PeeringDB resolvers can be inserted after inheritance. + +## API Envelope + +```http +POST /api/v1/visualization/compute-centers/{source_id}/collect-location +POST /api/v1/visualization/compute-centers/{source_id}/location +POST /api/v1/bgp/collectors/{collector_id}/collect-location +``` + +Both `collect-location` endpoints return the same envelope: + +```json +{ + "success": true, + "candidates": [], + "best_candidate": {}, + "attempted_queries": [], + "context": {} +} +``` + +`POST /api/v1/visualization/compute-centers/{source_id}/location` upserts the candidate selected by the frontend into `compute_center_locations`. Manual saves default to `needs_confirmation=false`, `verification_status="verified"`, and a `verified_at` timestamp. Future automated staging can pass `needs_confirmation=true` explicitly. + +The frontend [info-card.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/info-card.js) renders the shared candidate list and preview events. The compute-center layer button shows an `unresolved` badge; clicking it opens the unresolved queue. Row-level `采集` only fetches candidates. Header-level `一键采用` walks the queue top-to-bottom, picks the highest-confidence candidate with valid coordinates, saves it, removes the row, renumbers the list, and dispatches `earth:compute-center-unresolved-count-change` so the badge updates immediately. When the batch finishes, `earth:compute-center-location-saved` refreshes the real layer. + +If the remaining records have no city-level candidates, the batch must not invent coordinates. The UI keeps those rows and shows the backend `failure_reason` plus attempted queries. + +## Adding A Resolver + +A resolver only needs `name` and `resolve()`, returning `ResolverOutput`. + +```python +class PeeringDBFacilityResolver: + name = "peeringdb_facility" + + def __init__(self, client): + self._client = client + + def resolve(self, query): + asn = query.extra.get("origin_asn") + if not asn: + return ResolverOutput() + return ResolverOutput(candidates=tuple( + LocationCandidate( + latitude=f.latitude, + longitude=f.longitude, + display_name=f.name, + precision="site", + confidence=0.78, + query=f"peeringdb::{asn}", + source=self.name, + source_note=f"PeeringDB facility for AS{asn}", + matched_fields=("origin_asn",), + needs_confirmation=False, + city=f.city, + country=f.country, + ) + for f in self._client.facilities_for_asn(asn) + )) +``` + +Wire it in: + +```python +BGP_EVENT_PIPELINE = LocationPipeline([ + SourceCoordinatesResolver(), + InheritFromAnotherEntityResolver(source_lookup=...), + PeeringDBFacilityResolver(client=peeringdb_client), +]) +``` + +## Test Coverage + +Relevant tests: + +- [test_location_pipeline.py](/home/ray/dev/linkong/planet/backend/tests/test_location_pipeline.py) +- [test_bgp_collector_locations.py](/home/ray/dev/linkong/planet/backend/tests/test_bgp_collector_locations.py) +- [test_visualization_compute_centers.py](/home/ray/dev/linkong/planet/backend/tests/test_visualization_compute_centers.py) + +Coverage focuses on resolver pluggability, registry alias guards, BGP collector legacy dictionary compatibility, compute-center public API compatibility, and non-renderable locations being returned as `unresolved`. diff --git a/docs/technical/en/location-pipeline-user.md b/docs/technical/en/location-pipeline-user.md new file mode 100644 index 00000000..04977335 --- /dev/null +++ b/docs/technical/en/location-pipeline-user.md @@ -0,0 +1,127 @@ +# Earth Location Candidate Collection User Guide + +Location candidate collection helps fill or verify coordinates for compute centers and BGP collectors on Earth. Users do not type coordinates by hand; the backend ranks source coordinates, open organization-registry results, and online geocoding results into a previewable candidate list. + +## Supported Entities + +Currently supported: + +- Compute centers: TOP500 supercomputers and Epoch AI GPU clusters. +- BGP collectors: RIPE RIS `rrcXX` collectors. + +BGP events inherit the location of their owning collector. Events do not have a separate collection button yet; future ASN facility, prefix geography, or PeeringDB resolvers should use the same pipeline. + +## What Users See + +Clicking a compute center or BGP collector on Earth opens a detail card with location fields: + +| Field | Meaning | +| --- | --- | +| Location precision | Precise coordinates, site-level, city-level, or unconfirmed | +| Location source | Source coordinates, ROR organization registry, Nominatim online search, or stored BGP collector locations | +| Location confidence | Relative confidence reported by the backend resolver | +| Verification status | Confirmed, estimated, or online result pending confirmation | +| Resolution reason | Why the location was selected | +| Matched location name | Canonical name from an open source, online result, or stored collector location | +| Verified at | Verification date for confirmed locations; online candidates are usually empty | + +Compute-center GeoJSON no longer renders country centroids, unknown locations, or `[0, 0]` placeholders. Records that cannot reach city-level precision are returned in the endpoint's `unresolved` list and can be improved through candidate collection. + +A compute center with a `?` marker on Earth is not unresolved. It already has coordinates, but the coordinates still need confirmation, either because `needs_confirmation=true` or because the source is online geocoding. Truly unresolved records have no trustworthy coordinates and are therefore absent from the globe. + +## Collect Candidates + +1. Open `http://localhost:3000/earth`. +2. Enable the `Compute centers` or `BGP observation` layer. +3. Click an object to open its detail card. +4. Click `自动采集坐标候选` or `重新自动采集坐标`. +5. Wait for up to five candidates to appear. +6. Click `预览` on a candidate row; Earth flies to that latitude and longitude. + +Candidate rows show: + +- Candidate name. +- Precision: precise, site, or city. +- Resolver source. +- Confidence. +- Coordinates. + +Clicking `保存` on a candidate row writes the selected compute-center candidate into the location dimension table. After the save succeeds, the compute-center layer refreshes; if the record was previously in the unresolved queue, the unresolved count decreases. + +## Unresolved Queue And Adopt All + +The notification badge on the compute-center layer row shows the current unresolved count. Clicking it opens a fixed queue beside the layer panel: + +1. The queue contains only compute centers without trustworthy coordinates. +2. Row-level `采集` calls the candidate endpoint and shows up to five previewable candidates. +3. Header-level `一键采用` walks the list from top to bottom, chooses the highest-confidence candidate with valid coordinates, and saves it. +4. Each successful save immediately removes that row, renumbers the remaining rows, and updates the badge count. +5. When the batch completes, the frontend refreshes the compute-center layer so UI state and backend state converge. + +If a record has no saveable candidate, the system does not invent a country centroid, vendor headquarters, or hard-coded hint. The row stays in the queue with the backend failure reason and attempted queries so an operator can supply better evidence later. + +## Backend APIs + +The frontend buttons call: + +```http +POST /api/v1/visualization/compute-centers/{source_id}/collect-location +POST /api/v1/visualization/compute-centers/{source_id}/location +POST /api/v1/bgp/collectors/{collector_id}/collect-location +``` + +Both `collect-location` endpoints use the same response shape: + +```json +{ + "success": true, + "candidates": [], + "best_candidate": {}, + "attempted_queries": [], + "context": {} +} +``` + +When no candidate reaches city-level precision, `success` is `false` and the response includes `failure_reason` plus the attempted queries. This helps distinguish missing source fields, open-source gaps, and online geocoding misses. + +## Registry Maintenance + +Compute centers and BGP collectors no longer maintain local candidate registries. Compute-center accepted locations are stored in the `compute_center_locations` database dimension table keyed by `(source, source_id)`. BGP collector current locations are stored in the `bgp_collector_locations` database dimension table; the old RIPE RIS city-level coordinates are used only as initialization seed data and still require confirmation. + +For compute centers, prefer maintaining: + +- `source` / `source_id`: for example `top500` + `top500_50`. +- `name` / `operator` / `site`. +- `city` / `country`. +- `latitude` / `longitude`. +- `precision`: `precise`, `site`, or `city`. +- `confidence`: confidence from 0 to 1. +- `location_source` / `source_url` / `source_note` / `raw_payload`: evidence source. +- `needs_confirmation` / `verification_status` / `verified_at`: manual verification status and date. + +For BGP collectors, prefer maintaining: + +- `collector_id`: for example `rrc12`. +- `site` / `operator`: site and operator. +- `city` / `country` / `region`. +- `latitude` / `longitude`. +- `precision`: `precise`, `site`, or `city`. +- `confidence`: confidence from 0 to 1. +- `source` / `source_url` / `raw_payload`: evidence source. +- `verification_status` / `verified_at`: manual verification status and date. + +If only the city is known, use city-level precision. Do not enter a precise-looking coordinate that has not been verified. + +## Common Questions + +### Why are some compute centers missing on Earth? + +Earth only renders coordinates that reach city-level precision or better. If source data, verified storage, and online geocoding all fail, the record is returned as `unresolved` instead of being rendered at a misleading country center or `[0, 0]`. + +### Why do online results need confirmation? + +Nominatim/OpenStreetMap results may match same-name cities, organizations, or campuses. They are useful for previewing candidates, but should be manually confirmed before being persisted as verified locations. + +### Why do BGP events no longer all land in Amsterdam? + +The old behavior could match common fields like `operator="RIPE NCC"` and incorrectly promote `rrc00`. BGP event inheritance now uses a strict owning-collector lookup in the DB-backed cache instead of registry fuzzy matching. diff --git a/docs/technical/en/manual.md b/docs/technical/en/manual.md new file mode 100644 index 00000000..3b2ae7ef --- /dev/null +++ b/docs/technical/en/manual.md @@ -0,0 +1,627 @@ +# Planet Manual + +This manual is for daily use, demos, development integration, and local operations. It covers four core entry points: + +- `planet.sh`: local start, stop, restart, health check, and log access +- Earth: public 3D situational awareness page +- Console: admin backend (login required) +- Docs: backend Gatekeeper-controlled documentation; basic usage docs are public, while developer and operations docs require permission groups + +For the shortest path to getting started, see [Quickstart](/home/ray/dev/linkong/planet/docs/technical/en/quickstart.md). + +## Entry Overview + +After a default startup, the common URLs are: + +| Name | URL | Login Required | Description | +| --- | --- | --- | --- | +| Earth | `http://localhost:3000/earth` | No | 3D globe, layers, BGP, satellites, cables, news situational awareness | +| Docs | `http://localhost:3000/docs` | Partly | Usage docs are public; developer, backend, and operations docs require Gatekeeper groups | +| Console | `http://localhost:3000/admin` | Yes | Data, config, alerts, logs, and situational observation | +| AI Playground | `http://localhost:3000/playground` | Yes | AI Provider status and debugging | +| Backend API Docs | `http://localhost:8000/docs` | Depends on endpoint | FastAPI / OpenAPI documentation | + +## planet.sh + +`planet.sh` is the main control script for local development and demos. Use it to manage services rather than manually starting frontend, backend, database, and AI Provider separately. + +### Start + +```bash +./planet.sh start +``` + +Default behavior: + +- Starts PostgreSQL and Redis +- Starts AI Provider +- Starts the backend API +- Starts the frontend Vite dev server +- Outputs Earth, console, Playground, and backend API doc URLs + +Specify custom ports: + +```bash +./planet.sh start -b 8001 -f 3001 -a 8101 +``` + +Parameters: + +| Flag | Meaning | +| --- | --- | +| `-b ` | Backend port | +| `-f ` | Frontend port | +| `-a ` | AI Provider port | +| `--allow-lan` | Enable LAN access | +| `--verbose` | Show more command output during execution | + +### AI Provider Environment and Builds + +AI Provider runtime configuration can live in `aiprovider/.env` or in matching variables in `~/.zshrc`. `planet.sh` reads simple `export AI_...=...` / `AI_...=...` lines and passes them to the container at startup. + +Changing model, API key, or base URL does not rebuild the image. Restart only AI Provider to pick up runtime configuration changes: + +```bash +./planet.sh restart -a +``` + +For complex shell expansion in `~/.zshrc`, opt in explicitly: + +```bash +PLANET_LOAD_ZSHRC_ENV=source ./planet.sh start -a +``` + +To ignore `~/.zshrc` during troubleshooting: + +```bash +PLANET_LOAD_ZSHRC_ENV=0 ./planet.sh start -a +``` + +The AI Provider Docker build context is intentionally limited to the files required by the service, and `uv sync` uses a BuildKit cache mount so dependency downloads are reused after the first build. + +### Stop + +```bash +./planet.sh stop +``` + +Stops: + +- Backend +- AI Provider +- Frontend +- PostgreSQL +- Redis + +### Restart + +Full restart: + +```bash +./planet.sh restart +``` + +Per-module restart: + +```bash +./planet.sh restart -b +./planet.sh restart -f +./planet.sh restart -a +./planet.sh restart -d +``` + +| Flag | Effect | +| --- | --- | +| `-b` | Backend only | +| `-f` | Frontend only | +| `-a` | AI Provider only | +| `-d` | Database only | + +Per-module restarts are preferred during development — they avoid interrupting unrelated services. + +### Create User + +```bash +./planet.sh createuser +``` + +Used to create a console login account before first use. The script interactively prompts for username, password, and role. + +### Health Check + +```bash +./planet.sh health +``` + +Checks: + +- `planet_*` container status +- Backend `/health` +- AI Provider `/health` +- Frontend reachability + +If something shows offline, check the corresponding logs first. + +### Logs + +Recent logs: + +```bash +./planet.sh log +``` + +Follow logs: + +```bash +./planet.sh log -f +./planet.sh log -b +./planet.sh log -a +``` + +| Flag | Log source | +| --- | --- | +| `-f` / `--frontend` | `/tmp/planet_frontend.log` | +| `-b` / `--backend` | `/tmp/planet_backend.log` | +| `-a` / `--ai-provider` | `planet_aiprovider` container logs | + +### LAN Access + +```bash +./planet.sh start --allow-lan +``` + +Useful for: + +- Starting in WSL, accessing from Windows browser +- Demos on phone or tablet +- Another machine on the same LAN accessing the same dev instance + +`--allow-lan` only makes the frontend and backend listen on `0.0.0.0`. When Planet runs in WSL, Windows can usually reach it through `localhost`, but access from a phone or another computer through `http://:3000` still depends on Windows port forwarding and firewall rules. + +Use this order to diagnose: + +```bash +# From WSL or the shell running Planet +curl http://localhost:3000 +curl http://localhost:8000/health +ss -ltnp | grep -E ':3000|:8000' +``` + +If this shows `0.0.0.0:3000` and `0.0.0.0:8000`, but the LAN IP still fails, configure Windows from an elevated PowerShell: + +```powershell +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 + +New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 +New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 +``` + +## Earth + +Earth is the public 3D situational awareness page, accessed at: + +```text +http://localhost:3000/earth +``` + +It is a standalone frontend. The actual page lives at: + +- `frontend/public/earth/index.html` +- `frontend/public/earth/js/` +- `frontend/public/earth/css/` + +The React route `/earth` simply hosts it in an iframe. + +### Main Uses + +Earth is used to observe in a single globe view: + +- BGP events, anomalies, and situational posture +- Satellites and orbital trails +- Submarine cables and landing points +- Compute centers +- AIS vessels +- Border lines, grid lines, HD texture, cloud layer, terrain +- Live news streams and situational news +- Search and focused object details + +### Layer Control + +The right-side layer panel toggles visualization layers on or off. + +Common layers include: + +- Grid lines +- Border lines +- HD texture +- Atmospheric cloud layer +- Submarine cables +- Compute centers +- BGP observation +- AIS vessels +- Satellites +- Orbital trails +- Terrain + +Some layers have dependencies: + +- Terrain requires HD texture +- Trails require Satellites +- When HD texture is off, the globe shows the base map and edge glow effect + +### Legend + +The lower-left legend follows the currently focused or enabled layer. + +Current legend modes include: + +- Cables +- Satellites +- Border lines +- Compute centers +- BGP +- AIS vessels + +AIS vessel legend entries are grouped by vessel type: cargo, tanker, passenger, fishing, military, anchored/slow, and other. Triangle markers represent moving vessels; dots represent anchored or slow vessels. + +### Search + +Earth search finds current globe objects, such as: + +- Submarine cables +- Landing points +- Satellites +- Compute centers +- BGP events +- BGP collectors + +Search results can be used to quickly locate objects and open their details. + +### Location Candidate Collection + +Compute-center and BGP collector detail cards can collect candidate coordinates automatically. After clicking an object, use `自动采集坐标候选` or `重新自动采集坐标`; the backend ranks source coordinates, open organization lookups, and Nominatim online search results. Stored BGP collector locations are used as query context only and are not emitted as candidates. + +Candidates can be previewed directly on Earth. Compute-center candidates can be saved into the `compute_center_locations` dimension table from the detail card, then the layer refreshes immediately. The notification badge on the compute-center layer row shows unresolved records that cannot be rendered; clicking it opens the queue, where users can collect individual candidates or use `一键采用` to save the highest-confidence candidate top-to-bottom. Records without candidates stay in the queue and are not replaced by country centroids or hard-coded hints. See [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md) for the full workflow. + +### Settings + +The settings panel contains: + +- Rotation mode / cruise mode +- Cruise modules: BGP, News +- Satellite display style: self-glow, real ground footprint +- Day/night mode +- Panel visibility toggles +- Globe default size +- Terrain opacity +- Reset settings + +These settings are stored in browser local storage. They revert to defaults if you switch browsers or clear site data. + +### View Controls + +Earth supports mouse, touchpad, and touchscreen interaction. + +Common controls: + +| Action | Result | +| --- | --- | +| Left-button drag | Rotates the globe | +| One-finger drag | Rotates the globe on touch devices | +| Mouse wheel | Zooms the view in or out | +| Two-finger pinch | Zooms the view on touch devices | +| Zoom buttons | Adjust zoom in fixed steps | +| Click the zoom percent | Resets to the default zoom | + +When zooming, the top capsule briefly shows the current zoom level, for example `Zoom 180%`. This indicates view zoom only, not data loading progress. Loading status takes priority and will not be interrupted by zoom feedback. + +Drag sensitivity adjusts automatically based on the current zoom. Around the default view it keeps the normal rotation feel; when zoomed in, dragging becomes progressively finer for inspecting a region, vessel, satellite, or BGP event; when zoomed out, dragging is slightly faster for global browsing. + +### Cruise Mode + +Cruise mode makes Earth automatically cycle through focus targets. + +Current cruise modules: + +- BGP +- News + +Suitable for demos, monitoring displays, or unattended presentations. + +### Mobile + +Earth has a mobile drawer layout. On small screens: + +- Layer controls open in a mobile drawer +- Search, settings, and details use mobile panels +- Main interactions remain centered on globe object clicks, search, and layer toggles + +### Common Issues + +#### Earth Won't Open + +Check whether the frontend is online: + +```bash +./planet.sh health +./planet.sh log -f +``` + +If the frontend port is not `3000`, use the actual port shown at startup. + +#### Layer Has No Data + +Check the backend and data sources: + +```bash +./planet.sh health +./planet.sh log -b +``` + +Then open the console and check: + +- `/datasources` +- `/data` +- `/bgp` + +#### Satellites, BGP, or Cables Load Slowly + +These layers may depend on backend APIs, external data sources, or first-run collection tasks. Wait for startup tasks to finish before checking logs and console data source status. + +## Console + +Console entry point: + +```text +http://localhost:3000/admin +``` + +The console requires login. Create a user first if this is your first time: + +```bash +./planet.sh createuser +``` + +### Page Structure + +The console uses React + Ant Design, with a left-side menu organized by work domain. + +Common pages: + +| Page | Route | Purpose | +| --- | --- | --- | +| Dashboard | `/admin` | System overview | +| Earth | `/earth` | Opens the public Earth page | +| Data Sources | `/datasources` | View data sources and trigger collection | +| Collected Data | `/data` | View collected data | +| BGP Observation | `/bgp` | BGP situational data | +| System Alerts | `/alerts/system` | System-level alerts | +| BGP Alerts | `/alerts/bgp` | BGP-related alerts | +| Situational Alerts | `/alerts/situational` | Situational assessment alerts | +| AI Playground | `/playground` | AI Provider debugging | +| System Logs | `/logs` | View system logs (typically super admin only) | +| Users | `/users` | User management | +| Settings | `/settings` | System config and TV live stream sources | + +### Data Sources + +`/datasources` shows collection sources and triggers collection. It is now a data source directory that lists built-in and custom sources in one table. + +Common operations: + +- View data source status +- Trigger collection +- View recent collection tasks +- Open the read-only detail drawer for endpoint, headers, runtime config, and built-in/custom source type + +If a category of objects is missing on Earth, start here to confirm the data source is available. + +The data source name opens an information drawer only. Endpoint, credentials, headers, and custom source configuration are maintained under `/settings` collector settings. + +When collection tasks are running, the progress area shows a clickable `Collecting N` pill. Clicking it opens a modal with each running task's phase, progress, and processed count. + +### Collected Data + +`/data` shows the collected data table. + +Useful for diagnosing: + +- Whether data has entered the system +- Whether data update times match expectations +- Whether a data source produced valid records + +### BGP Observation + +`/bgp` is the BGP-focused page. + +It complements the BGP layer on Earth: + +- Earth emphasizes spatial posture and visual focus +- The console BGP page emphasizes lists, status, details, and assessment + +### Alerts + +Alert entry points: + +- `/alerts/system` +- `/alerts/bgp` +- `/alerts/situational` + +Used to view system, network, and situational alerts. + +### System Settings + +`/settings` manages system-level configuration. + +Current common uses: + +- System settings +- TV live stream source configuration +- Collector settings +- External integrations and AI Provider configuration + +Available configuration depends on the current user's role. + +#### Collector Settings + +`/settings?tab=collector_credentials` is currently displayed as Collector Settings. It manages connection settings for all collectors, not only credentials. + +Use it to: + +1. Select a collector from the dropdown. +2. Review tags such as `Requires credentials`, module, enabled state, and `Unchecked` / `Available` / `Unavailable`. +3. Click the plug icon next to the selector to run a health check. +4. Edit endpoint, request headers, timeout, and retry settings. +5. Save the collector settings. + +Free collectors are checked by requesting their endpoint directly. Credentialed collectors use their credential provider. If endpoint or credential fingerprint changes after the last successful validation, the collector must be checked again. + +The system treats a collector as connected when the current configuration has either collected data successfully or passed the manual connection check. + +#### BarentsWatch AIS Credentials + +`BarentsWatch AIS` is a credentialed built-in collector. Its credential card appears above the basic configuration card. + +Configured fields: + +- `Client ID` +- `Client Secret` +- `Endpoint` + +If a secret is already configured, the input shows a masked preview. Keeping that preview unchanged preserves the stored secret; entering a new value replaces it. + +BarentsWatch AIS credentials can be read from: + +1. Collector settings saved in the console. +2. Backend environment variables: + - `BARENTSWATCH_CLIENT_ID` + - `BARENTSWATCH_CLIENT_SECRET` + - historical spellings: `BARRENTSWATCH_CLIENT_ID`, `BARRENTSWATCH_CLIENT_SECRET` +3. matching `export` lines in `~/.zshrc`. + +If connection fails, the page opens the credential guide. The guide can be regenerated through AI Provider or reset to the default guide. The default guide points users to the official BarentsWatch tutorial and emphasizes selecting `AIS - API`, not the regular `BarentsWatch - API`. + +### System Logs + +`/logs` views system logs. If the menu item is not visible, the current user likely lacks the required role. + +Common troubleshooting sequence: + +```bash +./planet.sh health +./planet.sh log +``` + +Then open `/logs` for more structured runtime information. + +## Docs + +Documentation site: + +```text +http://localhost:3000/docs +``` + +Docs content is read through backend APIs by permission. The frontend no longer bundles all Markdown files directly. Source files still live in: + +```text +docs/technical/zh/ (Chinese) +docs/technical/en/ (English) +``` + +Anonymous visitors only see `public` docs such as the overview, quickstart, and manual. Logged-in users can see more technical docs when assigned Gatekeeper groups: + +- `docs_user`: user-operation docs. +- `docs_developer`: Earth, frontend, backend, collector, and AI Provider development docs. +- `docs_admin`: service control, operations, environment variable, and sensitive-operation docs. + +`admin` receives admin-doc access by default, and `super_admin` can read all Docs content. Gatekeeper groups are configured in the console Users page. + +Docs supports: + +- Category navigation +- Markdown rendering +- Tables and code blocks +- In-document table of contents +- Search across currently visible docs +- Internal links between technical documents + +When adding a new technical document, check: + +- Does it have a clear top-level heading +- Does it need to be added to backend Docs metadata for category and ordering +- Should it be classified as `public`, `docs_user`, `docs_developer`, or `docs_admin` + +## Development Command Conventions + +Frontend commands must use Bun: + +```bash +cd frontend +bun install +bun run dev +bun run build +``` + +Do not use `npm run ...`. The project uses Bun in WSL / Windows mixed environments to avoid Node/npm path compatibility issues. + +Verify the frontend build: + +```bash +source ~/.zshrc && bun run build +``` + +## Troubleshooting Order + +When something goes wrong, follow this sequence: + +1. Check service status: + +```bash +./planet.sh health +``` + +2. Check recent logs: + +```bash +./planet.sh log +``` + +3. Check per-module logs: + +```bash +./planet.sh log -f +./planet.sh log -b +./planet.sh log -a +``` + +4. Restart only the affected module: + +```bash +./planet.sh restart -f +./planet.sh restart -b +./planet.sh restart -a +``` + +5. If database or cache is abnormal, restart the database: + +```bash +./planet.sh restart -d +``` + +6. If still unrecovered, do a full restart: + +```bash +./planet.sh restart +``` + +## Related Docs + +- [Quickstart](/home/ray/dev/linkong/planet/docs/technical/en/quickstart.md) +- [Admin Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/frontend-admin-frontend-context.md) +- [Earth Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/earth-frontend-context.md) +- [Earth Layer Style Reference](/home/ray/dev/linkong/planet/docs/technical/en/earth-layer-style-reference.md) +- [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md) +- [System Service Control](/home/ray/dev/linkong/planet/docs/technical/en/backend-system-service-control.md) +- [Backend Collectors](/home/ray/dev/linkong/planet/docs/technical/en/backend-collectors.md) diff --git a/docs/technical/en/ops-docker-compose-buildx-upgrade.md b/docs/technical/en/ops-docker-compose-buildx-upgrade.md new file mode 100644 index 00000000..c25327e9 --- /dev/null +++ b/docs/technical/en/ops-docker-compose-buildx-upgrade.md @@ -0,0 +1,105 @@ +# Docker + Compose + Buildx Upgrade Guide + +Process: remove old version → install new version → verify + +--- + +# 1. Remove Old Version + +## Remove apt-installed packages + +```bash +sudo apt remove -y docker.io docker-compose docker-compose-v2 docker-doc podman-docker containerd runc +``` + +--- + +## Remove system `docker-compose` (V1) + +```bash +sudo rm -f "$(which docker-compose 2>/dev/null)" +``` + +--- + +## Find and remove manually installed Buildx plugin + +```bash +docker info | sed -n '/Plugins:/,/^ Server:/p' | grep -A2 buildx +``` + +Get the `Path` from the output, then run: + +```bash +rm -f +``` + +--- + +## Clean up unused dependencies + +```bash +sudo apt autoremove -y +``` + +--- + +# 2. Install Official Docker + +Includes Docker Engine, Docker Compose plugin, and Docker Buildx plugin. + +## Install dependencies + +```bash +sudo apt update +sudo apt install -y ca-certificates curl gnupg +``` + +--- + +## Add Docker GPG key + +```bash +sudo install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \ + sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg +``` + +--- + +## Add official repository + +```bash +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ +sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +``` + +--- + +## Install Docker + Compose + Buildx + +```bash +sudo apt update +sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +``` + +--- + +# 3. Verify Installation + +```bash +docker --version +docker compose version +docker buildx version +``` + +--- + +# 4. Common Commands + +```bash +docker compose up -d +docker compose down +docker buildx build . +``` diff --git a/docs/technical/en/ops-planet-sh-startup.md b/docs/technical/en/ops-planet-sh-startup.md new file mode 100644 index 00000000..61941fba --- /dev/null +++ b/docs/technical/en/ops-planet-sh-startup.md @@ -0,0 +1,205 @@ +# `planet.sh` Startup Performance Optimization + +## Background + +`planet.sh` manages start, stop, restart, health checks, and logs for all local services. The previous implementation had several startup issues: + +1. AI Provider rebuilt every time, even when code had not changed. +2. Port cleanup could wait up to 45 seconds. +3. Port bind detection used a Python subprocess, adding about 300 ms per call. +4. Plain `restart` and `restart -b` behaved differently. + +## Issue 1: AI Provider Rebuilt Every Time + +### Root Cause + +The build stamp file lived under `/tmp/`. After WSL or Linux restart, `/tmp` is cleared, so the `stamp_non_empty` condition failed and the script decided to rebuild: + +```bash +# All three conditions had to be true to skip rebuild +image_exists AND stamp_non_empty AND fingerprint_match +``` + +### Fix + +The stamp file moved to a persistent cache path: + +```bash +AI_PROVIDER_BUILD_STAMP_FILE="$HOME/.cache/planet/aiprovider_build.sha256" +``` + +Writing the stamp creates the directory first: + +```bash +write_ai_provider_build_stamp() { + mkdir -p "$(dirname "$AI_PROVIDER_BUILD_STAMP_FILE")" + compute_ai_provider_build_fingerprint > "$AI_PROVIDER_BUILD_STAMP_FILE" +} +``` + +### Faster Fingerprint + +The previous implementation tarred the whole `aiprovider/` directory before hashing, which could take seconds in large trees. The new version uses `find + stat` and reads only file metadata: + +```bash +compute_ai_provider_build_fingerprint() { + find aiprovider \ + -type f \ + ! -path '*/__pycache__/*' \ + ! -name '.env' \ + ! -name '.env.*' \ + ! -name '*.pyc' \ + ! -name '*.pyo' \ + | LC_ALL=C sort \ + | xargs -r stat --format="%Y %s %n" 2>/dev/null + sha256sum docker-compose.yml docker-compose.simple.yml 2>/dev/null + python3 "$SCRIPT_DIR/scripts/compute_aiprovider_dependency_fingerprint.py" 2>/dev/null +} +``` + +This is roughly 10 times faster for many-small-file workloads while preserving the same practical rebuild signal. `.env` and `.env.*` are excluded because runtime model, key, and Base URL changes should not force an image rebuild. + +### Docker Build Context + +AI Provider only needs root `pyproject.toml`, `uv.lock`, and `aiprovider/` source code. Sending the entire repository as Docker build context wastes time on frontend assets, PDFs, historical data, and Unreal files. + +The root `.dockerignore` now narrows the context: + +```dockerignore +** + +!pyproject.toml +!uv.lock +!aiprovider/ +!aiprovider/** + +aiprovider/.env +aiprovider/.env.* +!aiprovider/.env.example +``` + +The Dockerfile copies only AI Provider inputs: + +```dockerfile +COPY pyproject.toml uv.lock /app/ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + +COPY aiprovider /app/aiprovider +``` + +`uv sync` uses a BuildKit cache mount. The first build may still depend on network speed, but later builds reuse `/root/.cache/uv`. + +### Runtime Configuration + +Before starting AI Provider, `planet.sh` generates a temporary env-file and passes it to Compose or the manual `docker run` fallback. Configuration priority: + +1. `aiprovider/.env` +2. simple `export AI_...=...` or `AI_...=...` lines from `~/.zshrc` + +The default parser is static and only covers AI Provider, image, and proxy variables. It avoids executing interactive shell initialization. Complex shell expansion can be enabled explicitly: + +```bash +PLANET_LOAD_ZSHRC_ENV=source ./planet.sh start -a +``` + +To ignore personal shell config during debugging: + +```bash +PLANET_LOAD_ZSHRC_ENV=0 ./planet.sh start -a +``` + +### Skip-Rebuild Behavior + +When the fingerprint matches, the script skips `docker compose build` and starts the existing container: + +```bash +docker start planet_aiprovider +``` + +`docker stop` stops the container without deleting the image. `cleanup_exit_containers` removes exited containers but not images, so the next `docker start` can reuse the existing image. + +## Issue 2: Slow Port Cleanup + +### Cause + +`wait_for_port_release` could wait up to 45 seconds by default: 15 attempts times 3 seconds. + +### Fix + +Background process cleanup now uses a 3-second timeout: TERM, 1.5 seconds, KILL, 1.5 seconds. + +```bash +PORT_RELEASE_ATTEMPTS=15 +PORT_RELEASE_INTERVAL=0.2 + +wait_for_port_release "$port" 15 0.2 +``` + +`wait_for_port_release` accepts optional parameters so different situations can choose different timeouts. + +## Issue 3: Port Detection Used Python + +### Cause + +`can_bind_port` used `python3 -c "import socket..."`; each call cost about 300 ms. + +### Fix + +Prefer system tools and keep Python as a fallback: + +```bash +can_bind_port() { + local port="$1" + if command -v ss >/dev/null 2>&1; then + ! ss -tlnH 2>/dev/null | awk '{print $4}' | grep -qE ":${port}$" + return + fi + if command -v lsof >/dev/null 2>&1; then + [ -z "$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null)" ] + return + fi + python3 - "$port" <<'PY' +import sys, socket +p = int(sys.argv[1]) +s = socket.socket() +s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +try: + s.bind(("", p)); s.close(); sys.exit(0) +except OSError: + sys.exit(1) +PY +} +``` + +Frontend startup now has an additional pre-start cleanup retry layer: + +- `PORT_PRESTART_RETRIES`: defaults to 3 attempts. +- `PORT_PRESTART_RETRY_INTERVAL`: defaults to 2 seconds. + +`kill_port_if_requested()` only kills processes when the current environment can identify listening PIDs. If no PID is visible but the port still cannot bind, it logs diagnostics and lets the service startup flow make the final decision. `start_frontend_with_retry()` only enters the pre-cleanup retry path when a listener PID is visible, so the script no longer spends its retry budget repeatedly killing nothing while a host-side or external network namespace is still releasing the port. Seeing "no listener found but port still unavailable" on the first restart usually means the external environment is still releasing the port, not that a local process cleanup loop is useful. + +## Issue 4: `restart` Behavior + +Before the stamp path fix: + +- `restart -b`: stop all services, check fingerprint, rebuild only when needed, then start. +- plain `restart`: stop all services, then often rebuild AI Provider because `/tmp` lost the stamp. + +After moving the stamp file, plain `restart` uses the same `stop + start` behavior and the same fingerprint check as `restart -b`. + +## Other Cleanup + +Two redundant `sleep 3` waits were removed because health checks already cover the same readiness: + +- `start_backend_service`: post-database-health-check sleep. +- `restart_database_service`: post-restart sleep. + +## Related Files + +- [planet.sh](/home/ray/dev/linkong/planet/planet.sh) +- [.dockerignore](/home/ray/dev/linkong/planet/.dockerignore) +- [aiprovider/Dockerfile](/home/ray/dev/linkong/planet/aiprovider/Dockerfile) +- [docker-compose.yml](/home/ray/dev/linkong/planet/docker-compose.yml) +- [docker-compose.simple.yml](/home/ray/dev/linkong/planet/docker-compose.simple.yml) +- [compute_aiprovider_dependency_fingerprint.py](/home/ray/dev/linkong/planet/scripts/compute_aiprovider_dependency_fingerprint.py) diff --git a/docs/technical/en/quickstart.md b/docs/technical/en/quickstart.md new file mode 100644 index 00000000..7cb880e7 --- /dev/null +++ b/docs/technical/en/quickstart.md @@ -0,0 +1,231 @@ +# Quickstart + +This guide is for developers or demo operators starting Planet for the first time. The goal is to get services running via the shortest path and know which URLs to open. + +## Prerequisites + +Recommended: run in a WSL / Linux shell. + +You need: + +- Docker / Docker Compose available +- `uv` and `bun` accessible in the current shell +- Repository cloned locally + +On a new machine, run the bootstrap script first: + +```bash +./scripts/bootstrap-dev.sh +``` + +This script checks and syncs common dependencies, and generates if missing: + +- `backend/.env` +- `aiprovider/.env` +- `frontend/.env.local` + +Personal AI Provider configuration can also live in `~/.zshrc`. `planet.sh` reads simple `export AI_...=...` / `AI_...=...` lines and passes them to the AI Provider container. After changing model, key, or base URL, restart only AI Provider: + +```bash +./planet.sh restart -a +``` + +Collector credentials such as AISStream and BarentsWatch can also start in `~/.zshrc` for connectivity validation: + +```bash +export AISSTREAM_API_KEY="..." +export BARENTSWATCH_CLIENT_ID="..." +export BARENTSWATCH_CLIENT_SECRET="..." +``` + +For actual collection, prefer saving credentials in `Settings -> Collector Settings`, especially for AISStream's long-lived WebSocket collector. That keeps connectivity validation, backend collection tasks, and Earth realtime vessel aggregation on the same configuration source. + +## 1. Start Services + +From the repository root: + +```bash +./planet.sh start +``` + +After startup, the key URLs are: + +| Entry | Default URL | Purpose | +| --- | --- | --- | +| Earth | `http://localhost:3000/earth` | Public 3D Earth visualization | +| Console | `http://localhost:3000/admin` | Admin console (login required) | +| Docs | `http://localhost:3000/docs` | Usage docs are public; developer and operations docs require Gatekeeper groups | +| AI Playground | `http://localhost:3000/playground` | AI debugging (login required) | +| Backend API Docs | `http://localhost:8000/docs` | FastAPI / OpenAPI interface docs | + +If the default ports are taken, specify custom ports: + +```bash +./planet.sh start -f 3001 -b 8001 -a 8101 +``` + +## 2. Create a Login User + +The console requires login. For first-time use: + +```bash +./planet.sh createuser +``` + +Follow the prompts to enter username, password, and role. + +To read developer or operations docs, log in as `super_admin` and assign Gatekeeper groups from the Users page. Use `docs_developer` for development docs and `docs_admin` for service-control and operations docs. + +## 3. Open Earth + +Visit: + +```text +http://localhost:3000/earth +``` + +Earth is a public page — no login required. + +Once in, verify: + +- The globe renders correctly +- The right-side layer panel can toggle layers on/off +- Search can find cables, satellites, compute centers, BGP events +- Compute-center and BGP collector detail cards can collect and preview coordinate candidates; the compute-center unresolved badge can open the queue and save candidates +- Mouse drag, wheel zoom, and zoom percent feedback work correctly +- Settings panel can switch cruise mode, day/night mode, satellite display style + +## 4. Open the Console + +Visit: + +```text +http://localhost:3000/admin +``` + +The console manages data sources, collected data, situational observation, alerts, system logs, and configuration. + +First-time inspection checklist: + +- `/datasources`: data source directory and collection triggers; endpoint, headers, and credentials are configured under `/settings` collector settings +- `/data`: collected data +- `/bgp`: BGP situational view +- `/alerts/system`: system alerts +- `/settings`: system configuration + +## 5. Check Service Health + +```bash +./planet.sh health +``` + +This shows container status and checks: + +- Backend +- AI Provider +- Frontend + +## 6. View Logs + +Recent logs: + +```bash +./planet.sh log +``` + +Follow a specific service: + +```bash +./planet.sh log -f +./planet.sh log -b +./planet.sh log -a +``` + +Flags: + +- `-f`: frontend logs +- `-b`: backend logs +- `-a`: AI Provider logs + +## 7. Common Restarts + +Frontend only: + +```bash +./planet.sh restart -f +``` + +Backend only: + +```bash +./planet.sh restart -b +``` + +AI Provider only: + +```bash +./planet.sh restart -a +``` + +Database only: + +```bash +./planet.sh restart -d +``` + +Full restart: + +```bash +./planet.sh restart +``` + +## 8. LAN Access + +To allow a Windows browser, phone, or another device on the same network: + +```bash +./planet.sh start --allow-lan +``` + +This makes the frontend and backend listen on a LAN-accessible address. + +Note: `--allow-lan` only makes Planet listen on `0.0.0.0`; it does not automatically expose WSL services through the Windows LAN IP. A common pattern is: + +- `localhost:3000` / `localhost:8000` works inside WSL +- `localhost:3000` / `localhost:8000` works on Windows +- `http://:3000` fails from a phone or another computer + +That usually means Windows still needs port forwarding or firewall rules. + +If access fails, check from the shell running Planet: + +```bash +curl http://localhost:3000 +curl http://localhost:8000/health +ss -ltnp | grep -E ':3000|:8000' +``` + +If WSL is listening on `0.0.0.0:3000` and `0.0.0.0:8000` but the LAN IP still fails, configure Windows forwarding and firewall rules from an elevated PowerShell: + +```powershell +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 + +New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 +New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 +``` + +## 9. Stop Services + +```bash +./planet.sh stop +``` + +This shuts down the frontend, backend, AI Provider, PostgreSQL, and Redis. + +## Next Steps + +- Full usage guide: [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md) +- Console structure: [Admin Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/frontend-admin-frontend-context.md) +- Earth structure: [Earth Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/earth-frontend-context.md) +- Backend collectors: [Backend Collectors](/home/ray/dev/linkong/planet/docs/technical/en/backend-collectors.md) diff --git a/docs/technical/frontend-admin-frontend-context.md b/docs/technical/frontend-admin-frontend-context.md deleted file mode 100644 index bbcab832..00000000 --- a/docs/technical/frontend-admin-frontend-context.md +++ /dev/null @@ -1,236 +0,0 @@ -# Admin Frontend Context - -本文件描述当前控制台前端的真实结构，目标是帮助后续页面开发、表格改造、布局治理和状态收口时快速找到正确入口。 - -相关规则建议一起参考： - -- [rules.md](/home/ray/dev/linkong/planet/rules.md) -- [frontend-layout-guidelines.md](/home/ray/dev/linkong/planet/docs/technical/frontend-layout-guidelines.md) - -## 当前目标 - -控制台前端承担的是后台工作台，而不是展示型大屏。当前约束是： - -- 页面默认遵循单屏工作区 -- 主交互在内部模块滚动，而不是依赖整页无限变长 -- 列表、表格、分析页优先保证主工作区可见 -- 通用布局、滚动条、表格滚动行为尽量复用，不要每页各写一套 - -## 当前路由入口 - -主入口在： - -- [App.tsx](/home/ray/dev/linkong/planet/frontend/src/App.tsx) - -当前后台相关路由包括： - -- `/admin` -- `/users` -- `/datasources` -- `/data` -- `/alerts/system` -- `/alerts/bgp` -- `/alerts/situational` -- `/bgp` -- `/playground` -- `/settings` - -`/earth` 是独立展示页，不属于控制台骨架。 - -## 当前页面骨架 - -控制台公共壳层在： - -- [AppLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/components/AppLayout/AppLayout.tsx) - -职责： - -- 左侧导航 -- 折叠与展开 -- 当前账号/版本信息 -- 内容区高度闭合 -- 全站统一侧边栏滚动条 - -当前结构是： - -```tsx - - ... - - -
{children}
- - - -``` - -后续控制台页面应优先适配这套壳层，而不是重新定义全页高度语义。 - -## 当前共享组件 - -### 1. `Scrollbar` - -文件： - -- [Scrollbar.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/Scrollbar.tsx) - -用途： - -- 控制台侧边栏这类普通内容容器 -- 组件内部管理可见性、thumb 尺寸、拖拽和双轴 overflow 判定 - -当前约束： - -- 滚动条必须是浮层，不参与布局 -- 无 overflow 时不应留下可见痕迹 -- 真实滚动仍交给原生容器，只替换可见层和交互层 - -### 2. `ScrollbarOverlay` - -文件： - -- [ScrollbarOverlay.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/ScrollbarOverlay.tsx) - -用途： - -- Ant Table 这类内部已有滚动容器的区域 -- 不接管滚动语义，只叠加新的滚动条可见层 - -当前使用场景： - -- 数据源 -- 采集数据 -- 用户管理 -- 设置页 -- 告警页 -- BGP 页面 - -### 3. `TableScrollRegion` - -文件： - -- [TableScrollRegion.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/TableScrollRegion.tsx) - -用途： - -- 为表格滚动区提供统一包裹层 -- 后续新表格页优先复用，不要重复写“表格区域 + overlay scrollbar”样板 - -### 4. 其他共享组件 - -- [MarkdownRenderer.tsx](/home/ray/dev/linkong/planet/frontend/src/components/MarkdownRenderer/MarkdownRenderer.tsx) -- [TableActions.tsx](/home/ray/dev/linkong/planet/frontend/src/components/TableActions/TableActions.tsx) - -## 当前状态来源 - -### 1. 认证状态 - -文件： - -- [auth.ts](/home/ray/dev/linkong/planet/frontend/src/stores/auth.ts) - -职责： - -- token -- 当前用户 -- 登录/退出 - -`App.tsx` 用它判断是否进入登录页。 - -### 2. 业务数据网关 - -目前 AI / 态势感知相关服务集中在： - -- [http-gateway.ts](/home/ray/dev/linkong/planet/frontend/src/services/situational-awareness/http-gateway.ts) -- [port.ts](/home/ray/dev/linkong/planet/frontend/src/services/situational-awareness/port.ts) -- [types.ts](/home/ray/dev/linkong/planet/frontend/src/services/situational-awareness/types.ts) - -约束： - -- 页面不要直接散落拼 URL -- 先通过 port/types 定义边界 -- 再由 http/mock gateway 实现 - -## 当前页面分层建议 - -### 1. 仪表盘和摘要型页面 - -例如： - -- [Dashboard.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Dashboard/Dashboard.tsx) - -优先目标： - -- 页头稳定 -- 摘要卡片先紧凑化 -- 主工作区占据主要高度 - -### 2. 表格型页面 - -例如： - -- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx) -- [DataList.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataList/DataList.tsx) -- [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Users/Users.tsx) -- [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx) - -约束： - -- 优先内部滚动 -- 不要让表格撑爆整页 -- 新表格区域优先复用 `TableScrollRegion` / `ScrollbarOverlay` - -### 3. 复杂工作区页面 - -例如： - -- [BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx) -- [Playground.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Playground/Playground.tsx) - -约束： - -- Tabs 里的内容不能套同一套高度逻辑 -- 表格 tab、Markdown tab、配置 tab 要各自定义滚动责任 -- AI 结果区、长文本区优先保证最小可读高度 - -## 当前布局约束 - -这些原则已经在项目里反复验证过： - -1. 父容器高度链要闭合 -2. `min-height: 0` 不能漏 -3. overflow 责任必须明确 -4. 不要用 `overflow: hidden` 掩盖结构问题 -5. 不要为了摘要卡完整显示去压缩主工作区 -6. 自定义滚动条必须是浮层，不得挤压内容宽度 - -详细经验见： - -- [frontend-layout-guidelines.md](/home/ray/dev/linkong/planet/docs/technical/frontend-layout-guidelines.md) - -## 当前推荐改动方式 - -如果后续继续改后台页面，建议按这个顺序： - -1. 先确认页面属于摘要页、表格页还是复杂工作区 -2. 先接入现有壳层和滚动语义 -3. 优先复用共享滚动组件 -4. 最后再改视觉和细节交互 - -不要先写局部 CSS 补丁，再回头补结构。 - -## 当前明显边界 - -控制台前端和 Earth 前端不是一套系统： - -- 控制台前端是 React + Ant Design 工作台 -- Earth 前端是 `public/earth` 下的独立原生 HUD 系统 - -因此： - -- 不要把 Earth 的 HUD/动画/状态机直接挪进控制台 -- 不要把控制台表格/滚动策略硬套到 Earth HUD - -Earth 相关结构见： - -- [earth-frontend-context.md](/home/ray/dev/linkong/planet/docs/technical/earth-frontend-context.md) diff --git a/docs/technical/zh/README.md b/docs/technical/zh/README.md new file mode 100644 index 00000000..392e2734 --- /dev/null +++ b/docs/technical/zh/README.md @@ -0,0 +1,41 @@ +# 技术文档 + +这里放“当前实现和当前结构”的文档，重点回答： + +- 现在代码是怎么组织的 +- 当前入口在哪 +- 状态和组件如何工作 + +适合放入这里的内容： + +- 快速开始和使用手册 +- 前端上下文 +- Earth 前端结构 +- Earth 卫星覆盖策略 +- Earth 渲染图层顺序 +- Earth 图层样式属性索引 +- 后端运行控制 +- 采集器现状 +- 采集器设置与连接验证 +- 采集格式约定 + +## 使用入口 + +- [快速开始](/home/ray/dev/linkong/planet/docs/technical/zh/quickstart.md)：从零启动 Planet 的最短路径 +- [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md)：控制台、`planet.sh`、Earth 和 Docs 的完整使用手册 +- [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md)：在 Earth 上为算力中心和 BGP 观测站采集、预览坐标候选 +- [数据源、采集器设置与连接验证](/home/ray/dev/linkong/planet/docs/technical/zh/datasource-collector-settings-connectivity.md)：数据源目录、采集器设置、连接验证、BarentsWatch 凭证链路 +- [通用位置估算管线开发说明](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-development.md)：后端 location resolver / pipeline 的接口、注册表和扩展方式 +- [Docs Gatekeeper 开发说明](/home/ray/dev/linkong/planet/docs/technical/zh/docs-gatekeeper-development.md)：后端 Docs 目录、正文读取和 Gatekeeper 权限组实现 +- [Earth 可交互图标接入](/home/ray/dev/linkong/planet/docs/technical/zh/earth-interactable-usage.md)：Earth 地表可交互图标 `Interactable` 的接口、生命周期和接入示例 +- [Earth 工具栏与浮层协同](/home/ray/dev/linkong/planet/docs/technical/zh/earth-toolbar-overlay-coordination.md)：工具栏按钮与搜索 / 设置 / 新闻 / 图层浮层之间的关闭矩阵和接入规则 + +不适合放入这里的内容： + +- 尚未完成的路线图 +- 未来迭代方案 +- 大范围重构计划 + +这些应放入： + +- [计划文档索引](/home/ray/dev/linkong/planet/docs/plans/README.md) diff --git a/docs/technical/zh/agents-aiprovider.md b/docs/technical/zh/agents-aiprovider.md new file mode 100644 index 00000000..a85b8aea --- /dev/null +++ b/docs/technical/zh/agents-aiprovider.md @@ -0,0 +1,333 @@ +# AI Provider 指南 + +## 概览 + +`aiprovider` 是 Planet 的模型适配服务。 + +它把模型厂商差异隔离在主后端之外，让系统其它部分可以调用稳定的业务 API： + +- 调用方服务 -> `planet backend` +- `planet backend` -> `aiprovider` +- `aiprovider` -> 具体模型提供方 + +推荐默认方式： + +- 外部调用方和跨服务调用方统一调用 `planet backend` +- 只有基础设施级内部任务才直接调用 `aiprovider` + +## 职责边界 + +`backend` 负责： + +- 身份认证和权限控制 +- 业务层请求整理 +- 稳定的 `/api/v1/ai/...` 接口 +- 面向 `aiprovider` 的内部服务认证 + +`aiprovider` 负责： + +- 模型协议适配 +- 基于 `.env` 选择 provider +- 超时和轻量重试 +- 通过 `X-Request-ID` 串联请求追踪 + +当前配置采用类似 OpenClaw 的拆分方式： + +- `AI_PROVIDER` 标识厂商或逻辑 provider +- `AI_PROVIDER_API` 标识实际请求协议适配器 + +这个拆分能更清楚地表达 MiniMax、Claude 兼容网关、自托管 OpenAI 兼容服务等情况，避免把所有含义塞进一个配置项。 + +## 支持的 Provider + +`aiprovider` 当前支持以下 provider 标识： + +- `openai` +- `anthropic` +- `minimax` +- `ollama` + +支持的请求适配器： + +- `openai-completions` +- `anthropic-messages` +- `ollama-generate` + +仍然兼容的历史别名： + +- `openai_compatible` +- `anthropic_compatible` +- `claude_compatible` + +推荐映射关系： + +- `vLLM`、`LM Studio`、`One API`：`AI_PROVIDER=openai`，`AI_PROVIDER_API=openai-completions` +- `MiniMax`：`AI_PROVIDER=minimax`，`AI_PROVIDER_API=anthropic-messages` +- Claude 兼容网关：`AI_PROVIDER=anthropic`，`AI_PROVIDER_API=anthropic-messages` +- `Ollama`：`AI_PROVIDER=ollama`，`AI_PROVIDER_API=ollama-generate` + +## API 面 + +### 主后端 API + +推荐使用的稳定入口： + +- `GET /api/v1/ai/provider/status` +- `POST /api/v1/ai/situational-awareness/analyze` + +认证方式： + +- `Authorization: Bearer ` + +可选追踪头： + +- `X-Request-ID: ` + +后端会把 `X-Request-ID` 透传给 `aiprovider`，并在响应中返回同一个 header。 + +### AI Provider 内部 API + +仅供内部调用的接口： + +- `GET /v1/provider/status` +- `POST /v1/analyze` + +认证方式： + +- `X-Provider-Token: ` + +可选追踪头： + +- `X-Request-ID: ` + +## 请求示例 + +### 通过后端调用 + +```bash +curl -X POST http://localhost:8000/api/v1/ai/situational-awareness/analyze \ + -H "Authorization: Bearer " \ + -H "X-Request-ID: bgp-incident-20260407-001" \ + -H "Content-Type: application/json" \ + -d '{ + "title": "BGP异常研判", + "objective": "总结当前风险并给出处置建议", + "observations": [ + "collector A 在 5 分钟内出现多次 origin 变更", + "异常集中在同一地区前缀" + ], + "constraints": [ + "不要编造不存在的数据", + "区分事实和推断" + ], + "context": { + "source": "bgp-monitor", + "severity": "high" + } + }' +``` + +### 直接调用 `aiprovider` + +```bash +curl -X POST http://localhost:8010/v1/analyze \ + -H "X-Provider-Token: change_me" \ + -H "X-Request-ID: ai-batch-job-001" \ + -H "Content-Type: application/json" \ + -d '{ + "title": "链路波动分析", + "objective": "给出简要态势摘要和下一步建议", + "observations": [ + "多个节点出现延迟上升" + ], + "constraints": [ + "不要假设根因已经确认" + ], + "context": { + "region": "APAC" + } + }' +``` + +## 响应结构 + +后端和 `aiprovider` 返回相同的 payload 结构： + +```json +{ + "provider": "minimax", + "api": "anthropic-messages", + "model": "MiniMax-M2.7", + "content": "1) 态势摘要 ...", + "content_blocks": [], + "text_blocks": [], + "thinking_blocks": [], + "raw_response": {} +} +``` + +两个服务都会返回： + +- `X-Request-ID: ` + +## 配置 + +### 后端 + +推荐的后端 `.env`： + +```env +AI_PROVIDER_SERVICE_URL=http://localhost:8010 +AI_PROVIDER_SERVICE_TOKEN=change_me +AI_PROVIDER_TIMEOUT_SECONDS=60 +AI_PROVIDER_RETRY_ATTEMPTS=2 +``` + +参考文件： + +- [backend/.env.example](/home/ray/dev/linkong/planet/backend/.env.example) + +### AI Provider + +参考文件： + +- [aiprovider/.env.example](/home/ray/dev/linkong/planet/aiprovider/.env.example) + +前端本地参考： + +- [frontend/.env.example](/home/ray/dev/linkong/planet/frontend/.env.example) + +通用配置： + +```env +SERVICE_NAME=planet-ai-provider +SERVICE_VERSION=0.1.0 +AI_PROVIDER_SERVICE_TOKEN=change_me +AI_TIMEOUT_SECONDS=60 +AI_HTTP_RETRY_ATTEMPTS=2 +AI_ANALYSIS_SYSTEM_PROMPT=你是态势感知分析助手。请基于输入的上下文、观测与约束，输出结构化、克制、可执行的分析。 +``` + +### OpenAI 兼容示例 + +```env +AI_PROVIDER=openai +AI_PROVIDER_API=openai-completions +AI_BASE_URL=http://127.0.0.1:8001/v1 +AI_API_KEY=local-key +AI_MODEL=your-local-model +``` + +### MiniMax 中国区示例 + +```env +AI_PROVIDER=minimax +AI_PROVIDER_API=anthropic-messages +AI_BASE_URL=https://api.minimaxi.com/anthropic +AI_API_KEY=sk-cp-xxxxx +AI_MODEL=MiniMax-M2.7 +AI_MAX_TOKENS=1200 +AI_ANTHROPIC_VERSION=2023-06-01 +``` + +MiniMax 说明： + +- 这里使用官方 MiniMax 示例中的 Anthropic Messages 请求结构。 +- 对 MiniMax，`aiprovider` 默认不会开启 `thinking`，除非调用方显式传入 `thinking` 对象。 +- 这个行为和 OpenClaw 对 MiniMax Anthropic 兼容接口的谨慎处理保持一致。 + +### Anthropic 兼容示例 + +```env +AI_PROVIDER=anthropic +AI_PROVIDER_API=anthropic-messages +AI_BASE_URL=https://your-claude-compatible-endpoint.example.com/anthropic +AI_API_KEY=your_api_key +AI_MODEL=your-model +AI_MAX_TOKENS=1200 +AI_ANTHROPIC_VERSION=2023-06-01 +``` + +### Ollama 示例 + +```env +AI_PROVIDER=ollama +AI_PROVIDER_API=ollama-generate +AI_BASE_URL=http://127.0.0.1:11434 +AI_API_KEY= +AI_MODEL=qwen2.5:7b +``` + +## 部署模式 + +### 单机部署 + +推荐的本地流程： + +- `backend` 运行在 `localhost:8000` +- `aiprovider` 运行在 `localhost:8010` +- 本地模型网关运行在 `localhost:11434` 或其它本地端口 + +仓库内已包含辅助入口： + +- [planet.sh](/home/ray/dev/linkong/planet/planet.sh) +- [docker-compose.local-model.yml](/home/ray/dev/linkong/planet/docker-compose.local-model.yml) + +### 多机部署 + +示例拓扑： + +- 应用机器：`backend` +- AI 网关机器：`aiprovider` +- 模型机器：本地模型服务或云代理 + +此时链路变成服务间 HTTP RPC： + +- caller -> backend +- backend -> `http://10.0.0.12:8010` +- `aiprovider` -> 模型端点 + +推荐的跨机器后端配置： + +```env +AI_PROVIDER_SERVICE_URL=http://10.0.0.12:8010 +AI_PROVIDER_SERVICE_TOKEN=change_me +AI_PROVIDER_TIMEOUT_SECONDS=60 +AI_PROVIDER_RETRY_ATTEMPTS=2 +``` + +推荐运行规则： + +- 将 `aiprovider` 放在私有网络内 +- 至少用 `X-Provider-Token` 保护它 +- 始终发送 `X-Request-ID` +- 除基础设施任务外，调用方优先走后端 API + +## 重试和失败行为 + +`backend -> aiprovider`： + +- 对轻量网络错误和 5xx 失败进行重试 +- provider 服务不可用时返回 `502` + +`aiprovider -> model provider`： + +- 对轻量网络错误和 5xx 失败进行重试 +- 模型提供方不可用时返回 `502` + +这个策略故意保持保守：它能吸收短暂抖动，但不会掩盖持续性错误。 + +## 运维说明 + +- `./planet.sh start` 会自动启动 `aiprovider` +- `./planet.sh restart -a` 只重启 `aiprovider` +- `./planet.sh log -a` 跟随查看 `aiprovider` 日志 +- `./planet.sh health` 会报告 `aiprovider` 健康状态 + +## 推荐调用策略 + +- 前端和应用服务：调用 `backend` +- 定时基础设施任务和诊断任务：可选直接调用 `aiprovider` +- 不要让多个业务服务分别接入模型厂商 + +这样可以集中管理 provider 切换，避免模型相关差异在系统里四处扩散。 diff --git a/docs/technical/backend-collectors.md b/docs/technical/zh/backend-collectors.md similarity index 57% rename from docs/technical/backend-collectors.md rename to docs/technical/zh/backend-collectors.md index 0a735e70..eb60fd95 100644 --- a/docs/technical/backend-collectors.md +++ b/docs/technical/zh/backend-collectors.md @@ -84,6 +84,13 @@ async def run(self, db): | HuggingFace Spaces | space | Demo应用 | 1天 | | PeeringDB | ixp/network/facility | 互联网交换点/网络/机房 | 1-2天 | | TeleGeography | submarine_cable | 海底光缆信息 | 7天 | +| Space-Track TLE | satellite_tle | 卫星轨道 TLE 数据 | 依采集器配置 | +| BarentsWatch AIS | vessel | 船只位置、航速、航向、MMSI 等 AIS 数据 | 依采集器配置 | +| AISStream Vessels | vessel_ais | AIS WebSocket 实时流，写入原始观测层并由聚合接口展示 | 依采集器配置 | + +AIS 船只类采集器和其它 `CollectedData` 采集器的落库路径不同。BarentsWatch、AISStream 和自定义 `vessel_ais` 源都会进入 AIS 原始观测层，随后由聚合服务合并成 Earth 船只图层使用的 GeoJSON 和详情数据。这样做可以保留来源、传输方式、字段冲突和观测时间，避免某个实时源直接覆盖最终展示表。 + +TOP500 和 Epoch AI 算力数据的公开源不总是提供可用经纬度。Earth 统一算力中心接口在主地图启动链路中只使用源数据自带坐标或 `compute_center_locations` 维表坐标；缺少坐标的记录会进入 `unresolved`，不会通过本地注册表、国家质心或猜测城市自动渲染。用户手动采集候选时，后端会用源字段调用 ROR 组织注册 API 和 Nominatim/OpenStreetMap 在线搜索；候选经前端保存后写入 `compute_center_locations`，后续地图刷新再从维表渲染。 ## 四、数据格式 (统一存储到 CollectedData 表) @@ -200,6 +207,30 @@ def start_scheduler(): **核心文件**: `backend/app/services/scheduler.py` +### 成功采集与连接状态 + +内置采集器成功采集后，调度器会记录当前有效配置已通过连接验证： + +```python +if datasource.last_status == "success": + effective_candidate = await get_builtin_effective_candidate(db, datasource.source) + checksum, _ = await build_builtin_connectivity_checksum(...) + await save_connectivity_success( + db, + datasource.source, + checksum, + {"status_code": None}, + connected_by="collection", + ) +``` + +这个记录用于控制台“采集器设置”中的连接状态判断：如果当前配置和成功采集时的 checksum 一致，就视为已连接，不要求用户再手动点击连接按钮。只有 endpoint、请求头、基础配置或凭证指纹变化时，才需要重新验证。 + +相关实现见： + +- [datasource_connectivity.py](/home/ray/dev/linkong/planet/backend/app/services/datasource_connectivity.py) +- [数据源、采集器设置与连接验证](/home/ray/dev/linkong/planet/docs/technical/zh/datasource-collector-settings-connectivity.md) + ## 八、相关代码文件 ``` @@ -211,13 +242,140 @@ backend/app/services/collectors/ ├── epoch_ai.py # Epoch AI采集器 ├── huggingface.py # HuggingFace采集器 ├── peeringdb.py # PeeringDB采集器 -└── telegeraphy.py # TeleGeography海底光缆采集器 +├── telegeraphy.py # TeleGeography海底光缆采集器 +├── vessel_ais.py # BarentsWatch AIS 船只采集器 +└── aisstream.py # AISStream WebSocket 船只采集器 + +backend/app/services/ +├── custom_datasource_runtime.py # 自定义 REST / WebSocket 映射运行时 +├── datasource_mapping.py # 确定性字段映射与目标写入 +├── vessel_ais_aggregation.py # AIS 原始观测写入与聚合读取 +├── vessel_aggregation_strategy.py # 多源字段选择、freshness fallback 和冲突记录 +└── vessel_enrichment.py # 船舶资料富化缓存 backend/app/models/ -└── collected_data.py # 统一数据模型 +├── collected_data.py # 统一数据模型 +└── vessel_enrichment.py # 船舶富化结果缓存 ``` -## 九、数据使用场景 +## 九、凭证型采集器 + +部分采集器需要外部服务凭证，例如： + +| 采集器 | credential provider | 凭证来源 | +| --- | --- | --- | +| `barentswatch_vessels` | `barentswatch` | 控制台采集器设置、环境变量、`~/.zshrc` | +| `aisstream_vessels` | `aisstream` | 控制台采集器设置、环境变量、`~/.zshrc`（连接验证可读；正式采集建议保存到采集器设置或注入后端环境） | +| `spacetrack_tle` | `spacetrack` | 环境变量、`~/.zshrc` | + +### BarentsWatch AIS + +BarentsWatch AIS 的凭证解析统一在： + +- [barentswatch.py](/home/ray/dev/linkong/planet/backend/app/services/barentswatch.py) + +`VesselAISCollector` 只负责采集和转换 AIS 数据，不再自己读取环境变量或拼 token 请求。它通过： + +- `resolve_barentswatch_config()` +- `fetch_barentswatch_access_token()` + +获取运行时配置。 + +解析优先级： + +1. `DataSourceConfig.auth_config` +2. `DataSourceConfig.config` +3. 环境变量 +4. `~/.zshrc` + +支持变量： + +```bash +export BARENTSWATCH_CLIENT_ID="..." +export BARENTSWATCH_CLIENT_SECRET="..." +``` + +并兼容历史拼写： + +```bash +export BARRENTSWATCH_CLIENT_ID="..." +export BARRENTSWATCH_CLIENT_SECRET="..." +``` + +连接验证会先请求 `https://id.barentswatch.no/connect/token` 获取 `scope=ais` 的 access token，再用 `Authorization: Bearer ` 请求 AIS endpoint。 + +### AISStream 实时船舶 + +AISStream 使用 `wss://stream.aisstream.io/v0/stream` WebSocket endpoint。默认运行方式是长连接实时采集，而不是传统 REST collector 的“请求一次、进度到 100%、完成”模型。 + +运行时配置： + +- `api_key`：优先从 `DataSourceConfig.auth_config.api_key` 或 `config.api_key` 读取；也可由后端进程环境变量 `AISSTREAM_API_KEY` 提供。 +- `bounding_boxes`：AISStream 订阅范围，默认示例为全球 `[[[-90, -180], [90, 180]]]`，生产或演示建议先缩小区域。 +- `message_types`：默认 `PositionReport` 和 `ShipStaticData`。 +- `streaming_enabled`：默认启用长连接；关闭后回退到批次式 `fetch -> transform -> save`。 +- `streaming_max_messages`：测试用上限，非 0 时收到指定消息数后停止。 +- `reconnect_delay_seconds`、`receive_timeout_seconds`：控制断线重连和空闲等待。 + +状态语义： + +- `connecting`：正在连接 AISStream。 +- `streaming`：持续接收实时消息，`records_processed` 表示已见消息数，通常没有固定总量和百分比。 +- `reconnecting`：上游断开或网络异常，采集器记录 `AISSourceHealth` 后等待重连。 +- `stopped` / `cancelled`：任务被测试上限或用户停止。 + +AISStream 连接验证会通过 `datasource_connectivity.py` 读取保存的采集器配置、环境变量和 `~/.zshrc` 中的 `AISSTREAM_API_KEY`。正式采集时，最稳妥的方式是把 API Key 保存到“设置 -> 采集器设置 -> AISStream 实时船舶”；如果只放在 `~/.zshrc`，需要确认后端进程实际继承到了该环境变量。 + +### AIS 原始观测与聚合 + +AIS 观测写入后不会直接替换最终船只记录，而是先保存为 raw observation： + +- `source` 记录来源，例如 `barentswatch_vessels`、`aisstream_vessels` 或自定义源名称。 +- `delivery_mode` 表达实时性，`realtime_stream` 优先于 `polling`。 +- `transport` 记录 `websocket` 或 `http`。 +- 位置、速度、航向等动态字段会按 freshness 和来源优先级选择。 +- 静态字段优先保留非空值；冲突候选会记录到详情接口，便于排查多源差异。 + +Earth 使用的接口仍是： + +```http +GET /api/v1/visualization/geo/vessels +GET /api/v1/visualization/vessels/{mmsi} +GET /api/v1/visualization/vessels/{mmsi}/track +GET /api/v1/visualization/vessels/{mmsi}/conflicts +GET /api/v1/visualization/vessels/aggregation/diagnostics +``` + +`/geo/vessels` 会合并 raw observation 聚合结果和 legacy BarentsWatch latest position 结果，避免只接入 AISStream 后把历史 BarentsWatch 船只遮蔽掉。 + +## 十、采集器设置与连接验证 + +控制台的“采集器设置”页提供所有内置采集器的 endpoint、请求头、超时、重试和凭证配置。连接验证不是只看前端按钮状态，而是由后端计算 checksum： + +- endpoint +- auth type +- headers +- config +- credential provider +- 凭证指纹 + +相关 API： + +```http +GET /api/v1/datasources/configs/all +POST /api/v1/datasources/configs/builtin/connection-status +POST /api/v1/datasources/configs/builtin/connect +POST /api/v1/settings/integrations/barentswatch/connect +GET /api/v1/settings/credential-guides/{provider} +POST /api/v1/settings/credential-guides/{provider}/generate +POST /api/v1/settings/credential-guides/{provider}/reset +``` + +更多细节见： + +- [数据源、采集器设置与连接验证](/home/ray/dev/linkong/planet/docs/technical/zh/datasource-collector-settings-connectivity.md) + +## 十一、数据使用场景 采集的数据最终会： @@ -225,7 +383,7 @@ backend/app/models/ 2. **态势分析** - 统计全球算力分布、增长趋势 3. **告警系统** - 检测重要节点变化 -## 十、采集器注册机制 +## 十二、采集器注册机制 采集器在应用启动时自动注册： @@ -247,7 +405,7 @@ collector_registry.register(TeleGeographyCableSystemCollector()) **核心文件**: `backend/app/services/collectors/registry.py` -## 十一、触发采集 +## 十三、触发采集 ### 方式一：定时触发 系统启动时，APScheduler会自动根据各采集器的`frequency_hours`设置定时任务。 diff --git a/docs/technical/zh/backend-datasources-api-performance.md b/docs/technical/zh/backend-datasources-api-performance.md new file mode 100644 index 00000000..aa60956e --- /dev/null +++ b/docs/technical/zh/backend-datasources-api-performance.md @@ -0,0 +1,99 @@ +# DataSources 列表接口性能优化 + +## 背景 + +`GET /api/v1/datasources` 是数据源管理页面的核心接口，响应慢会直接阻塞页面渲染。 + +## 优化前的查询链路 + +`_load_datasource_list_context` 按顺序执行以下查询： + +| 序号 | 函数 | 查询内容 | 瓶颈 | +|------|------|---------|------| +| 1 | `_load_latest_running_tasks` | collection_tasks 窗口函数，stale check 依赖此结果 | 必须串行 | +| 2 | `_load_latest_completed_tasks` | collection_tasks 窗口函数（最近完成任务） | 串行等待 | +| 3 | `_load_datasource_data_counts` | `COUNT(*) GROUP BY source` on collected_data | **最慢，全表扫描** | +| 4 | `_load_datasource_endpoint_overrides` | datasource_configs 简单 SELECT | 串行等待 | + +## 第一阶段：并行化 + +将 2/3/4 三个互不依赖的查询改为 `asyncio.gather` + 独立 session 并行执行： + +```python +async def _fetch_completed(): + async with async_session_factory() as s: + return await _load_latest_completed_tasks(s, datasource_ids) + +async def _fetch_counts(): + async with async_session_factory() as s: + return await _load_datasource_data_counts(s, sources) + +async def _fetch_overrides(): + async with async_session_factory() as s: + return await _load_datasource_endpoint_overrides(s, sources) + +completed_tasks, data_counts, endpoint_overrides = await asyncio.gather( + _fetch_completed(), _fetch_counts(), _fetch_overrides(), +) +``` + +> **注意**：SQLAlchemy `AsyncSession` 不支持在同一 session 上并发，每个协程必须独立开 session。 + +## 第二阶段：删除重量级查询 + +### 删除 `_load_datasource_data_counts` + +`data_count` 字段仅用于前端在"最近采集"列显示 `(0条)` 的边缘提示，不值得为此维持一次 `COUNT(*) GROUP BY` 全表扫描。 + +- 前端同步移除 `(0条)` 显示逻辑 +- 移除 `BuiltInDataSource` 接口中的 `data_count` 字段 + +### 删除 `_load_latest_completed_tasks` + +`last_status` 和 `last_run_at` 已由 collector 在任务完成时直接更新到 `DataSource` 模型字段，不需要再 JOIN collection_tasks 获取： + +```python +# 优化前：需要查 completed_tasks +last_run_at = datasource.last_run_at or (last_task.completed_at if last_task else None) +last_status = datasource.last_status or (last_task.status if last_task else None) + +# 优化后：直接读模型字段 +last_run_at = datasource.last_run_at +last_status = datasource.last_status +``` + +同步移除 `last_records_processed` 字段（来源是 completed_tasks，列表不显示此字段）。 + +## 优化后的查询链路 + +``` +datasources SELECT → 主数据，必须 +_load_latest_running_tasks → 必须（进行中状态 + stale check） +_load_datasource_endpoint_overrides → 必须（endpoint 覆盖，列表详情和采集器设置需要显示当前有效地址） +``` + +3 个查询（原来 5 个），后两个顺序执行（running tasks 先完成用于 stale check，endpoint overrides 轻量）。 + +## 前端 triggerDatasource 双调修复 + +`triggerDatasource` 中存在双重 `fetchData()` 调用： + +```typescript +// 修复前 +} else { + window.setTimeout(() => { fetchData() }, 800) // 无 task_id 时延迟刷 +} +fetchData() // 总是立即刷 → 与上面的延迟刷重叠 + +// 修复后（二者互斥） +if (res.data.task_id) { + fetchData() // 有 task_id：立即刷一次 +} else { + window.setTimeout(fetchData, 800) // 无 task_id：等 800ms 再刷一次 +} +``` + +## 相关文件 + +- `backend/app/api/v1/datasources.py` — `_load_datasource_list_context`、`list_datasources` +- `frontend/src/pages/DataSources/DataSources.tsx` — `BuiltInDataSource` interface、`triggerDatasource` diff --git a/docs/technical/zh/backend-system-service-control.md b/docs/technical/zh/backend-system-service-control.md new file mode 100644 index 00000000..2a8fb0ff --- /dev/null +++ b/docs/technical/zh/backend-system-service-control.md @@ -0,0 +1,333 @@ +# 系统服务控制 + +本文定义后台控制面动作与现有 `planet.sh` 服务管理命令之间的固定映射。 + +目标是在复用当前运维脚本语义的同时，不向前端或 API 调用方暴露任意 shell 执行能力。 + +## 范围 + +- 这套映射只用于管理端运维控制。 +- 控制面必须提交固定 action 名称，而不是原始 shell 命令。 +- 后端负责把允许的 action 翻译成固定的 `planet.sh` 调用。 + +## 设计规则 + +- 只允许执行白名单 action。 +- 前端绝不能发送任意 shell 字符串。 +- 后端必须从固定映射表构造命令参数。 +- 高风险 action 应限制为 `super_admin`。 +- 在 UI 连续性重要时，优先局部重启，而不是全栈重启。 + +## Action 映射 + +| Action 名称 | 用途 | `planet.sh` 命令 | 备注 | +| --- | --- | --- | --- | +| `restart-backend` | 只重启后端 API | `./planet.sh restart -b` | 页面通常短暂失联后由 `/health` 轮询恢复。 | +| `restart-frontend` | 只重启前端开发服务器 | `./planet.sh restart -f` | 页面入口会短暂不可用；UI 通过前端入口探测恢复后刷新。 | +| `restart-database` | 重启 PostgreSQL 和 Redis 容器 | `./planet.sh restart -d` | 适合数据库/缓存需要受控重启但不希望重启 UI 的场景。 | +| `restart-system` | 重启整个应用栈 | `./planet.sh restart` | 前端会短暂中断；UI 应进入引导恢复模式。 | +| `restart-backend-port` | 在指定端口重启后端 | `./planet.sh restart -b ` | 执行前必须由后端校验端口。 | +| `restart-frontend-port` | 在指定端口重启前端 | `./planet.sh restart -f ` | 执行前必须由后端校验端口。 | +| `health-check` | 读取当前服务健康状态 | `./planet.sh health` | 安全的只读运维动作。 | +| `show-logs-backend` | 查看后端日志 | `./planet.sh log -b` | 更适合 CLI/运维工具，不建议作为普通 Web UI 日志流。 | +| `show-logs-frontend` | 查看前端日志 | `./planet.sh log -f` | 更适合 CLI/运维工具，不建议作为普通 Web UI 日志流。 | + +## 默认不暴露到 UI 的能力 + +除非有明确产品需求并经过额外安全评审，否则以下脚本能力不应直接暴露到 Web UI： + +- `./planet.sh restart` +- `./planet.sh start` +- `./planet.sh stop` +- `./planet.sh createuser` +- 任何未来的原始 shell 透传能力 + +原因： + +- 全量重启可能打断当前控制会话； +- stop/start 影响面更大； +- 用户创建不是服务控制操作； +- 原始 shell 透传会引入不必要的权限风险。 + +## 第一阶段推荐 UI 契约 + +### 前端 action payload + +```json +{ + "action": "restart-backend" +} +``` + +### 后端命令解析 + +```text +restart-backend -> ["./planet.sh", "restart", "-b"] +restart-database -> ["./planet.sh", "restart", "-d"] +restart-system -> ["./planet.sh", "restart"] +restart-frontend -> ["./planet.sh", "restart", "-f"] +health-check -> ["./planet.sh", "health"] +``` + +## API 草案 + +### 主接口 + +- `POST /api/v1/system/restart-tasks` + +用途： + +- 创建受控重启任务； +- 将白名单 action 解析成固定 `planet.sh` 命令； +- 把执行交给外部 runner 或 detached subprocess。 + +### 请求体 + +```json +{ + "action": "restart-backend" +} +``` + +未来可选形态： + +```json +{ + "action": "restart-backend-port", + "port": 8000 +} +``` + +### 响应 + +```json +{ + "task_id": "restart_20260331_153000_ab12cd", + "action": "restart-backend", + "status": "queued", + "stage": "accepted", + "message": "Restart task accepted" +} +``` + +### 任务查询接口 + +- `GET /api/v1/system/restart-tasks/{task_id}` + +响应结构： + +```json +{ + "task_id": "restart_20260331_153000_ab12cd", + "action": "restart-backend", + "status": "queued", + "stage": "accepted", + "message": "Waiting for execution", + "requested_by": { + "id": 1, + "username": "admin" + }, + "created_at": "2026-03-31T15:30:00+08:00", + "updated_at": "2026-03-31T15:30:02+08:00" +} +``` + +### 可选日志接口 + +- `GET /api/v1/system/restart-tasks/{task_id}/logs` + +建议响应： + +```json +{ + "task_id": "restart_20260331_153000_ab12cd", + "lines": [ + "accepted restart-backend request", + "spawning restart command", + "waiting for backend shutdown", + "waiting for backend health recovery" + ] +} +``` + +日志接口在第一阶段不是必需项。首版可以只依赖任务状态加 `/health` 轮询。 + +## 任务状态模型 + +### Status + +- `queued` +- `running` +- `succeeded` +- `failed` +- `timeout` + +### Stage + +- `accepted` +- `spawning` +- `stopping` +- `starting` +- `waiting_for_health` +- `healthy` +- `failed` + +### 含义 + +- `status` 是高层终态/非终态状态。 +- `stage` 是面向运维人员和 UI 的执行阶段。 +- `message` 是 modal 或全屏遮罩中展示的短文本。 + +## 权限模型 + +- `restart-backend` 应要求 `super_admin`。 +- 权限检查应沿用 [users.py](/home/ray/dev/linkong/planet/backend/app/api/v1/users.py) 中已有的角色模式。 +- 前端可以对非 `super_admin` 隐藏控件，但后端必须继续强制鉴权。 + +## 存储模型 + +推荐第一阶段实现： + +- 将重启任务状态存入 Redis； +- 任务生命周期保持较短； +- 最近日志用有界列表保存。 + +建议 key： + +- `system:restart_task:{task_id}` +- `system:restart_task:{task_id}:logs` + +建议字段： + +- `task_id` +- `action` +- `status` +- `stage` +- `message` +- `requested_by_id` +- `requested_by_username` +- `created_at` +- `updated_at` + +## 执行模型 + +处理请求的 API 进程不应依赖自身持续存活来流式输出完整重启日志。 + +推荐执行流程： + +1. 校验调用方和 action +2. 在 Redis 中创建任务状态 +3. 将 action 解析为固定 `planet.sh` argv +4. 启动 detached executor +5. 返回 `task_id` +6. executor 在重启过程中更新任务状态 +7. 前端轮询健康状态和/或任务状态，直到服务恢复 + +推荐命令解析示例： + +```text +restart-backend -> ["./planet.sh", "restart", "-b"] +restart-frontend -> ["./planet.sh", "restart", "-f"] +restart-backend-port -> ["./planet.sh", "restart", "-b", ""] +health-check -> ["./planet.sh", "health"] +``` + +## 前端轮询流程 + +推荐第一阶段 UX： + +1. 用户点击 `重启后端` +2. 确认 modal 说明服务会短暂不可用 +3. 前端调用 `POST /api/v1/system/restart-tasks` +4. UI 进入阻塞式重启状态 +5. 前端每 `1-2s` 轮询 `/health` +6. 临时请求失败视为预期现象 +7. 连续 `2-3` 次健康检查成功后，前端刷新页面 + +可选增强轮询： + +1. 后端仍可达时轮询任务状态接口 +2. 断连开始后切换为 `/health` 恢复轮询 +3. 健康恢复后刷新页面 + +## 前端状态机 + +- `idle` +- `confirming` +- `submitting` +- `waiting_for_shutdown` +- `waiting_for_recovery` +- `recovered` +- `failed` +- `timeout` + +建议 UI 文案： + +- `已发送重启指令` +- `正在停止后端服务` +- `正在等待服务恢复` +- `服务已恢复，正在刷新页面` +- `恢复超时，请手动检查服务状态` + +## 当前 Dashboard 实现 + +Dashboard 当前已实现： + +- `restart-backend` +- `restart-frontend` +- `restart-ai-provider` +- `restart-database` +- `restart-system` +- `super_admin` 权限门禁 +- 任务创建接口 +- Redis 任务状态 +- 前端确认 modal +- 后端 `/health` 轮询 +- 前端入口轮询 +- 恢复后自动刷新页面 + +暂不实现： + +- 原始 shell 命令透传 +- 任意服务控制 +- 完整终端 stdout 流式输出 +- 多 action 并发重启队列 + +## 实现清单 + +### 后端 + +1. 在 `backend/app/api/v1/` 下新增专用系统控制 API 模块 +2. 增加基于白名单的 `planet.sh` action 解析器 +3. 将重启任务状态存入 Redis +4. 增加 detached restart-runner 脚本执行 +5. 暴露： + - `POST /api/v1/system/restart-tasks` + - `GET /api/v1/system/restart-tasks/{task_id}` + - 可选任务日志接口 +6. 对所有 restart-task 接口强制 `super_admin` 权限 + +### 前端 + +1. 在 dashboard 为 `super_admin` 增加 `重启服务` 控件 +2. 发送前展示确认 modal +3. 提交后将 modal 切换为阻塞式重启状态 +4. 后端重启使用 `/health` 轮询确认恢复 +5. 前端重启和完全重启使用前端入口探测确认恢复 +6. 连续健康检查成功后自动刷新页面 +7. 展示简短阶段日志，而不是原始终端流 + +### 运维说明 + +1. 优先使用局部重启，只有确实需要时才执行完全重启 +2. 前端重启会打断当前页面入口，必须进入恢复等待状态 +3. 命令执行必须始终从仓库根目录发起 +4. API 边界只能传递固定 action 名称 + +## 校验要求 + +- 拒绝任何不在白名单中的 action。 +- 如果增加带端口 action，端口必须校验为 `1..65535` 的整数。 +- 从仓库根目录解析命令，确保 `planet.sh` 的工作目录稳定。 +- detached runner 使用 `zsh -ic` 执行白名单命令，确保 `~/.zshrc` 中的本地环境变量进入重启流程。 +- 记录请求 action、操作者身份、执行开始时间和结果。 diff --git a/docs/technical/zh/datasource-collector-settings-connectivity.md b/docs/technical/zh/datasource-collector-settings-connectivity.md new file mode 100644 index 00000000..09af46e9 --- /dev/null +++ b/docs/technical/zh/datasource-collector-settings-connectivity.md @@ -0,0 +1,449 @@ +# 采集器设置与连接验证 + +## 背景 + +控制台现在把“数据源目录”和“采集器配置”拆开： + +- `/datasources` + - 展示所有数据源，包括内置和自定义。 + - 点击名称只打开信息抽屉。 + - 负责查看状态、触发采集和查看采集中任务。 +- `/settings?tab=collector_credentials` + - 显示为“采集器设置”。 + - 负责 endpoint、请求头、基础参数和凭证配置。 + - 所有采集器都提供连接按钮，用于健康检查。 + +这样做是为了减少首次使用时的认知分裂：接口地址、请求头、凭证和自定义源配置都属于“采集器设置”，而不是散落在数据源列表和系统设置多个入口里。 + +## 用户侧规则 + +连接状态不是前端样式状态，而是由后端根据配置 checksum 和已验证记录判断。 + +一个内置采集器被视为“已连接”需要满足任一条件： + +- 当前配置已经成功采集过数据。 +- 当前配置点击过连接按钮，并且后端验证成功。 + +如果 endpoint、请求头、基础配置或凭证指纹相对上次验证成功时发生变化，状态会回到“需要重新连接”。 + +## 前端入口 + +### 数据源目录 + +文件： + +- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx) +- [index.css](/home/ray/dev/linkong/planet/frontend/src/index.css) + +当前行为： + +- 内置数据源和自定义数据源合并为 `UnifiedDataSource` 列表。 +- 表格只保留查看、采集和状态类操作。 +- 名称点击打开只读抽屉。 +- 抽屉中展示： + - 是否内置 + - 是否启用 + - 模块、优先级、频率 + - endpoint + - 请求头 + - 基础配置 + - 是否需要凭证 +- 有任务运行时，顶部进度区显示 `采集中 N` 可点击标签。 +- 点击 `采集中 N` 打开任务列表弹窗，显示各任务进度。 + +`data-source-bulk-toolbar__running-pill` 是“采集中”标签的样式入口。它和其他状态标签同排，但通过 hover、箭头和蓝色描边表达可交互性。 + +### 采集器设置 + +文件： + +- [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx) + +当前行为： + +- `collector_credentials` tab 展示为“采集器设置”。 +- 下拉框列出内置采集器，并支持维护合并到内置数据的自定义补充源。 +- 下拉框右侧只有一个插头图标按钮，用于健康检查。 +- 下拉框下方用状态标签展示： + - `需要凭证` / `无需凭证` + - 模块 + - `启用` / `禁用` + - `未检查` / `可用` / `不可用` + - 是否覆盖 endpoint +- 需要凭证的采集器把凭证卡片放在基础配置上方。 +- 不需要凭证的采集器只显示基础配置。 +- AISStream 采集器使用 WebSocket 语义，状态会显示为连接中、实时接收、重连或停止，不使用固定百分比表达完成度。 +- 自定义源入口放在采集器设置内，不在数据源目录里重复提供编辑入口；数据源目录只保留总览、运行和只读抽屉。 + +连接按钮使用内联 Tabler 风格插头图标，来源语义对应 `plug-connected`，避免继续使用刷新图标表达连接动作。 + +## 后端接口 + +### 数据源配置列表 + +```http +GET /api/v1/datasources/configs/all +``` + +返回 YAML 默认数据源和数据库覆盖配置的合并结果。该路由必须定义在 `/configs/{config_id}` 之前，否则 `all` 会被 FastAPI 当成路径参数并触发 422。 + +返回字段包括： + +- `name` +- `default_url` +- `endpoint` +- `is_overridden` +- `is_active` +- `source_type` +- `auth_type` +- `headers` +- `config` +- `config_id` +- `description` + +`config` 返回前会移除内部连接验证字段，避免前端把校验元数据当成用户配置展示。 + +### 内置采集器连接状态 + +```http +POST /api/v1/datasources/configs/builtin/connection-status +``` + +用途： + +- 给定一份候选配置。 +- 计算 checksum。 +- 判断当前配置是否已经连接。 + +当前前端主要通过连接按钮即时检查，不强依赖这个接口，但它是后续保存按钮置灰、页面初始化状态恢复的后端依据。 + +### 内置采集器连接验证 + +```http +POST /api/v1/datasources/configs/builtin/connect +``` + +用途： + +- 免费采集器直接请求 endpoint。 +- 需要凭证的采集器走对应 credential provider。 +- 验证成功后写入系统级连接记录。 + +成功返回中会带： + +- `success` +- `connected` +- `checksum` +- `stage` +- `message` +- `response_time_ms` +- `credential_provider` +- `credential_source` + +### BarentsWatch AIS 连接验证 + +```http +POST /api/v1/settings/integrations/barentswatch/connect +GET /api/v1/settings/integrations/barentswatch/connectivity +``` + +BarentsWatch 使用独立接口，是因为它需要在保存前验证草稿凭证： + +- 使用草稿 `client_id` / `client_secret` 获取 token。 +- 使用 token 请求 AIS endpoint。 +- 连接成功后用草稿凭证指纹写入内置采集器连接记录。 + +## 连接校验服务 + +文件： + +- [datasource_connectivity.py](/home/ray/dev/linkong/planet/backend/app/services/datasource_connectivity.py) + +核心职责： + +- 计算内置采集器配置 checksum。 +- 读取环境变量和 `~/.zshrc` 中的凭证。 +- 判断当前配置是否已连接。 +- 执行 endpoint 健康检查。 +- 保存连接成功记录。 + +### checksum 组成 + +checksum 包含： + +- 采集器名称 +- endpoint +- auth type +- headers +- 去掉内部校验字段后的 config +- credential provider +- 凭证指纹 + +凭证指纹使用凭证内容 hash，不把明文凭证写入连接记录。 + +### 连接记录 + +连接成功记录写入 `SystemSetting`： + +```text +category = datasource_connectivity_validations +``` + +payload 以采集器 source 为 key： + +```json +{ + "barentswatch_vessels": { + "checksum": "...", + "status": "success", + "validated_at": "2026-04-29T00:00:00+00:00", + "status_code": 200, + "credential_source": "datasource_config", + "connected_by": "connection_button" + } +} +``` + +`connected_by` 当前有两个来源： + +- `connection_button` + - 用户手动点击连接按钮。 +- `collection` + - 采集任务成功完成，系统自动记录当前有效配置已连通。 + +### 成功采集即连接 + +调度器在采集成功后会调用连接记录写入逻辑： + +- [scheduler.py](/home/ray/dev/linkong/planet/backend/app/services/scheduler.py) + +这样已有数据的采集器不会要求用户重复验证。只有当配置 checksum 变化时，才需要重新点击连接。 + +## BarentsWatch AIS 凭证链路 + +文件： + +- [barentswatch.py](/home/ray/dev/linkong/planet/backend/app/services/barentswatch.py) +- [vessel_ais.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/vessel_ais.py) + +解析优先级： + +1. `DataSourceConfig.auth_config` +2. `DataSourceConfig.config` +3. 环境变量 +4. `~/.zshrc` + +支持的环境变量： + +```bash +export BARENTSWATCH_CLIENT_ID="..." +export BARENTSWATCH_CLIENT_SECRET="..." +``` + +也兼容历史拼写： + +```bash +export BARRENTSWATCH_CLIENT_ID="..." +export BARRENTSWATCH_CLIENT_SECRET="..." +``` + +Token 请求规则： + +- Token URL：`https://id.barentswatch.no/connect/token` +- `Content-Type`: `application/x-www-form-urlencoded` +- Body: + - `grant_type=client_credentials` + - `client_id` + - `client_secret` + - `scope=ais` + +AIS 请求规则： + +- Endpoint 默认：`https://live.ais.barentswatch.no/v1/latest/combined` +- Header：`Authorization: Bearer ` + +`VesselAISCollector` 不再自己读取环境变量，而是统一走 `resolve_barentswatch_config()` 和 `fetch_barentswatch_access_token()`，避免设置页、连接验证和采集器三套凭证逻辑分叉。 + +## AISStream 采集器链路 + +文件： + +- [aisstream.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/aisstream.py) +- [vessel_ais_aggregation.py](/home/ray/dev/linkong/planet/backend/app/services/vessel_ais_aggregation.py) + +AISStream 使用 WebSocket 实时流，采集器只写入 `ais_raw_observations` 原始观测层，不直接覆盖最终船只展示表。聚合接口负责多源去重、字段选择和冲突记录。 + +配置项： + +- `api_key`：保存在 `DataSourceConfig.auth_config`，也可用环境变量 `AISSTREAM_API_KEY`。 +- `endpoint`：默认 `wss://stream.aisstream.io/v0/stream`。 +- `message_types`：默认 `PositionReport` 和 `ShipStaticData`。 +- `bounding_boxes`：AISStream 格式为 `[[[lat_min, lon_min], [lat_max, lon_max]]]`，设置页提供全球、挪威 / 北海、欧洲近海、东亚、北美东西海岸 preset。 +- `max_messages` 和 `receive_timeout_seconds`：控制单次批次式 WebSocket 采集窗口。 + +标准化规则： + +- `PositionReport` 主要提供位置、速度、航向和状态。 +- 船名可以从 `MetaData.ShipName` 补入，即使消息体本身没有 `name`。 +- 船型通常来自低频 `ShipStaticData.Type`；后端会把 AIS 数字类型码映射为 Cargo / Tanker / Passenger / Fishing / Military。 +- 如果某艘船尚未收到静态消息，聚合结果的船型仍可能是 `Other`，后续由 v5 船舶资料 enrichment 补齐。 + +连接验证会读取保存配置、环境变量和 `~/.zshrc` 中的 `AISSTREAM_API_KEY`。正式采集时，推荐把 API Key 保存到采集器设置；如果只写在 `~/.zshrc`，需要确认后端进程实际继承了该变量，否则连接验证可能可用但 collector 运行时拿不到 key。 + +## 自定义 REST / WebSocket 映射运行时 + +文件： + +- [custom_datasource_runtime.py](/home/ray/dev/linkong/planet/backend/app/services/custom_datasource_runtime.py) +- [datasource_mapping.py](/home/ray/dev/linkong/planet/backend/app/services/datasource_mapping.py) + +自定义源现在不是独立的新数据孤岛，而是作为内置数据源的补充源写入目标 schema。当前最完整的目标是 `vessel_ais`：自定义 REST 或 WebSocket 源经过确定性 mapping 后写入 AIS raw observations，再通过 `vessels` WebSocket channel 推送给 Earth。 + +### 配置语义 + +关键字段： + +- `source_type`：`rest` / `http` / `websocket` / `ws`。 +- `endpoint`：REST 使用 `http(s)://`，WebSocket 使用 `ws(s)://`。 +- `auth_type`：`none`、`bearer`、`api_key`、`basic`。 +- `headers`：静态请求头。 +- `auth_config`：token、API key、basic 用户名密码，API key 支持 header 或 query。 +- `config.target_schema`：例如 `vessel_ais`。 +- `config.delivery_mode`：REST 默认 `polling`，WebSocket 默认 `realtime_stream`。 +- `config.merge_target_source`：记录该自定义源补充哪个内置数据，例如 `barentswatch_vessels`。 + +REST runner 支持： + +- `GET` / `POST` +- query params +- JSON body +- headers 和 auth 注入 +- active mapping 写入目标 schema + +WebSocket runner 支持： + +- endpoint 格式校验 +- headers 和 auth 注入 +- 可选 `ws_subscribe_message` +- `ws_message_path` / `ws_items_path` 提取消息主体或数组 +- 断线重连 +- `debug_max_messages` 调试上限 +- 后台 stream start / stop / status + +相关 API： + +```http +POST /api/v1/datasources/custom/sample +GET /api/v1/datasources/target-schemas +POST /api/v1/datasources/{config_id}/run-mapped +POST /api/v1/datasources/{config_id}/stop-mapped +GET /api/v1/datasources/{config_id}/mapped-status +DELETE /api/v1/datasources/configs/{config_id}?delete_mappings=true&delete_source_data=true +``` + +`run-mapped?background=true` 只对 WebSocket 源有意义，会启动后台 stream。REST 源仍是一次性采集。 + +### 删除与数据清理 + +删除自定义源时有三种层级： + +- 只删除配置：保留 mapping 和历史数据。 +- 删除配置和 mapping：同时删除该配置的 mapping 模板。 +- 删除配置、mapping 和该源数据：删除该源写入的 `collected_data`、`ais_raw_observations` 和 `ais_source_health`。 + +如果删除的是 `vessel_ais` 自定义源数据，后端会向 `vessels` channel 广播 `reload_required`，提示 Earth 重新拉取船只聚合结果。legacy `vessel_position` 不按自定义源直接删除，因为它没有可靠的 source 归因。 + +### 本地 AIS mock WebSocket + +文件： + +- [mock-ais-ws-server.ts](/home/ray/dev/linkong/planet/scripts/mock-ais-ws-server.ts) + +运行方式： + +```bash +bun run mock:ais-ws +``` + +mock 服务持续发送 AIS-like JSON，用于验证“WebSocket 自定义源 -> mapping -> AIS raw observation -> `vessels` channel -> Earth 船只 upsert”链路。典型配置： + +```json +{ + "source_type": "websocket", + "endpoint": "ws://localhost:8787", + "config": { + "target_schema": "vessel_ais", + "delivery_mode": "realtime_stream", + "merge_target_source": "barentswatch_vessels", + "ws_message_path": "$.data", + "ws_items_path": "$.vessels[*]", + "ws_reconnect": true + } +} +``` + +## 凭证教程 + +文件： + +- [credential_guides.py](/home/ray/dev/linkong/planet/backend/app/services/credential_guides.py) + +接口： + +```http +GET /api/v1/settings/credential-guides/{provider} +POST /api/v1/settings/credential-guides/{provider}/generate +POST /api/v1/settings/credential-guides/{provider}/reset +``` + +当前支持： + +- `barentswatch` +- `aisstream` + +默认教程包含 BarentsWatch 官方 tutorial 地址： + +```text +https://developer.barentswatch.no/docs/tutorial +``` + +如果用户点击“教程不好用”，后端会把默认 prompt 发给 AI Provider 生成新的中文教程，并保存到 `SystemSetting`： + +```text +category = collector_credential_guides +``` + +“重置”会删除自定义教程，恢复默认教程。 + +## 保存规则 + +内置采集器配置保存时会移除内部 `connectivity_validation` 字段，避免校验状态跟用户配置混在一起。 + +BarentsWatch `client_secret` 保存时有特殊处理： + +- 输入框显示脱敏预览。 +- 如果提交值仍等于脱敏预览，后端保留原 secret。 +- 如果输入新值，才替换 secret。 +- 不再提供单独“清除当前 secret”复选框。 + +## 测试覆盖 + +相关测试： + +- [test_vessels.py](/home/ray/dev/linkong/planet/backend/tests/test_vessels.py) + +新增覆盖： + +- 能从 `~/.zshrc` 解析 BarentsWatch 凭证。 +- 环境变量为空时，`resolve_barentswatch_config()` 能回退到 `~/.zshrc`。 +- 船只数据转换和 GeoJSON 输出保持兼容。 + +## 当前 Provider 覆盖 + +当前已经支持的凭证 provider： + +- `barentswatch` +- `aisstream` +- `spacetrack` + +其他 `requires_credentials=true` 的采集器如果还没有 provider，会返回“凭证链路尚未接入”，前端显示 `不可用`。 diff --git a/docs/technical/zh/docs-gatekeeper-development.md b/docs/technical/zh/docs-gatekeeper-development.md new file mode 100644 index 00000000..dd6a3188 --- /dev/null +++ b/docs/technical/zh/docs-gatekeeper-development.md @@ -0,0 +1,116 @@ +# Docs Gatekeeper 开发说明 + +Docs Gatekeeper 把 `/docs` 从“前端构建时打包所有 Markdown”改成“后端按权限返回目录和正文”。它的目标是让公开使用手册、用户文档、开发文档和管理/运维文档在同一个 Docs 页面内可检索，但正文读取必须经过服务端白名单和用户权限检查。 + +用户侧说明见 [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md) 的 Docs 章节。 + +## 鉴权模型 + +Docs 使用两层权限： + +- `users.role`：保留给控制台系统权限。 +- `users.gatekeeper_groups`：Docs 内容权限组。 + +权限组： + +| 组 | 用途 | +| --- | --- | +| `docs_user` | 用户操作类文档 | +| `docs_developer` | Earth、前端、后端、采集器和 AI Provider 开发文档 | +| `docs_admin` | 服务控制、运维、环境变量和敏感操作文档 | + +继承规则： + +- 未登录用户只能读 `public`。 +- `docs_developer` 隐含 `docs_user`。 +- `docs_admin` 隐含 `docs_developer` 和 `docs_user`。 +- `admin` 和 `super_admin` 默认拥有全部 Docs 权限。 + +## 后端入口 + +文件： + +- [docs.py](/home/ray/dev/linkong/planet/backend/app/api/v1/docs.py) +- [docs_gatekeeper.py](/home/ray/dev/linkong/planet/backend/app/services/docs_gatekeeper.py) +- [user.py](/home/ray/dev/linkong/planet/backend/app/models/user.py) +- [users.py](/home/ray/dev/linkong/planet/backend/app/api/v1/users.py) + +API： + +```http +GET /api/v1/docs/catalog +GET /api/v1/docs/{lang}/{slug} +``` + +`catalog` 只返回当前用户可见文档。正文接口会先校验语言、slug 和文件是否在 metadata 白名单里，再判断权限： + +- 未登录访问受保护文档：`401`。 +- 已登录但权限不足：`403`。 +- 未知语言、未知 slug 或文件不存在：`404`。 + +正文文件只能来自 `docs/technical/{zh,en}/` 下的白名单文件，不能通过路径拼接读取任意文件。 + +## Metadata 来源 + +当前服务端 metadata 维护在 [docs_gatekeeper.py](/home/ray/dev/linkong/planet/backend/app/services/docs_gatekeeper.py)： + +```python +DocsMetadata( + "manual.md", + "manual", + "public", + "Manual", + 2, + "Planet 使用手册", + "Planet Manual", +) +``` + +新增公开文档时，需要同步： + +- 新增中英文 Markdown 文件。 +- 在服务端 `DOCS_METADATA` 添加 filename、slug、access、group、order、标题。 +- 在前端 [docs-content.ts](/home/ray/dev/linkong/planet/frontend/src/pages/Docs/docs-content.ts) 添加同名 metadata，保持导航标题和排序一致。 +- 如果需要从 README 发现，更新 `docs/technical/zh/README.md` 和 `docs/technical/en/README.md`。 + +## 用户管理 + +`users` 表新增 `gatekeeper_groups JSONB DEFAULT '[]'`。启动时 [session.py](/home/ray/dev/linkong/planet/backend/app/db/session.py) 会用 `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` 补列，适配已有本地数据库。 + +用户 API 负责： + +- 创建用户时写入 `gatekeeper_groups`。 +- 更新用户时校验组名只能是 `docs_user`、`docs_developer`、`docs_admin`。 +- 只有 `super_admin` 能修改 Gatekeeper 权限组。 + +前端 [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Users/Users.tsx) 展示权限组标签，并在编辑表单中提供多选框。非 `super_admin` 打开的表单会禁用该字段，并在提交前移除 `gatekeeper_groups`。 + +## 前端 Docs 加载 + +文件： + +- [Docs.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Docs/Docs.tsx) +- [docs-content.ts](/home/ray/dev/linkong/planet/frontend/src/pages/Docs/docs-content.ts) +- [docs-search.ts](/home/ray/dev/linkong/planet/frontend/src/pages/Docs/docs-search.ts) + +关键变化： + +- 移除 `import.meta.glob(...?raw)` 作为正文来源。 +- 页面加载时请求 `/api/v1/docs/catalog` 构建当前可见目录。 +- 打开正文时请求 `/api/v1/docs/{lang}/{slug}`。 +- 搜索只索引当前用户可见文档，并按需从后端读取 Markdown。 +- `401` 显示登录提示，`403` 显示权限提示，`404` 显示文档不可用。 + +## 测试覆盖 + +相关测试： + +- [test_docs_gatekeeper.py](/home/ray/dev/linkong/planet/backend/tests/test_docs_gatekeeper.py) + +测试应覆盖： + +- 匿名用户只能看到 public 文档。 +- 受保护正文的 `401` / `403`。 +- `docs_developer` 可读开发文档但不能读管理文档。 +- `admin` 和 `super_admin` 可读管理文档。 +- 未知 slug、未知语言和路径穿越字符串不能读取文件。 diff --git a/docs/technical/zh/earth-bgp-context.md b/docs/technical/zh/earth-bgp-context.md new file mode 100644 index 00000000..fe578a39 --- /dev/null +++ b/docs/technical/zh/earth-bgp-context.md @@ -0,0 +1,355 @@ +# BGP 态势上下文 + +## 当前目标 + +BGP 模块正在从一个只展示异常的演示功能，演进为分层观测管线： + +`raw observations -> enrichment -> detectors -> incidents -> console/Earth visualization` + +实际产品目标已经不只是“在地球上显示事件”。当前目标是： + +1. 即使 incident 密度很低，也让 BGP 在 Earth 上保持可见存在感 +2. 让 incident 明显比 anomaly 更像高置信度事件层 +3. 即使没有活跃 incident，也能表达观测网络仍在运行 + +换句话说，Earth 应该表现为观测面，而不只是事件地图： + +- `collectors` 表达观测正在发生 +- `activity` 表达哪里的路由状态近期活跃或噪声较高 +- `incidents` 成为最高置信度的聚焦层 + +## 当前后端架构 + +### 数据层 + +1. `BGPObservation` + - 文件：`backend/app/models/bgp_observation.py` + - 用途：存储从实时/历史来源归一化后的原始路由观测。 + - 典型字段： + - `source` + - `collector` + - `peer_asn` + - `peer_ip` + - `prefix` + - `event_type` + - `as_path` + - `origin_asn` + - `next_hop` + - `communities` + - `observed_at` + - `raw_payload` + - `collector_geo` + - `ingest_batch_id` + +2. `BGPAnomaly` + - 文件：`backend/app/models/bgp_anomaly.py` + - 用途：保存原子级 detector 输出。 + - 当前 detector 输出类型包括： + - `origin_change` + - `more_specific_burst` + - `mass_withdrawal` + +3. `BGPIncident` + - 文件：`backend/app/models/bgp_incident.py` + - 用途：把原子 anomaly 聚合成人类和 UI 可消费的 incident 对象。 + +### 管线 + +主流程目前集中在： + +- `backend/app/services/collectors/bgp_common.py` +- `backend/app/services/bgp_enrichment.py` +- `backend/app/services/bgp_detectors.py` +- `backend/app/services/bgp_incidents.py` + +运行流程： + +1. 采集器抓取原始 BGP 数据 +2. `normalize_bgp_event()` 规范化 payload +3. observation 写入 `bgp_observations` +4. enrichment 为事件补充分析上下文 +5. detector 创建 `bgp_anomalies` +6. incident 聚合把 anomaly 汇总为 `bgp_incidents` + +### 当前接入来源 + +1. `RIPE RIS Live` + - 采集器文件：`backend/app/services/collectors/ris_live.py` + - 用于实时观测流。 + +2. `CAIDA BGPStream Backfill` + - 采集器文件：`backend/app/services/collectors/bgpstream.py` + - 用作历史/回填入口。 + +## 当前 enrichment 状态 + +已在以下文件实现 enrichment 骨架： + +- `backend/app/services/bgp_enrichment.py` + +当前 enrichment 内容： + +- prefix family / prefix length +- supernet / more-specific 推导 +- 去重 AS path +- path prepending 提示 +- collector 区域信息 +- prefix baseline 提示 +- new-origin 检测 +- 可用时从 PeeringDB 获取 ASN 组织画像 +- prefix scope / 受影响区域提示 +- prefix 地理来源优先级： + - `OpenGeoFeed`（override，高置信） + - `IPtoASN`（国家范围 baseline） + - `NRO delegated stats`（registry allocation fallback） + +当前限制： + +- `RPKI` 仍只是占位，返回 `unknown` +- 尚未集成真实 ROA 校验来源 +- `inetnum` / `inet6num` whois fallback 仍待实现 + +## 当前 API 面 + +主 API 文件： + +- `backend/app/api/v1/bgp.py` + +可用接口： + +- `/api/v1/bgp/events` +- `/api/v1/bgp/events/summary` +- `/api/v1/bgp/events/{id}` +- `/api/v1/bgp/anomalies` +- `/api/v1/bgp/anomalies/summary` +- `/api/v1/bgp/anomalies/{id}` +- `/api/v1/bgp/incidents` +- `/api/v1/bgp/incidents/summary` +- `/api/v1/bgp/incidents/{id}` + +可视化 GeoJSON 接口： + +- `backend/app/api/v1/visualization.py` +- `/api/v1/visualization/geo/bgp-collectors` +- `/api/v1/visualization/geo/bgp-anomalies` +- `/api/v1/visualization/geo/bgp-incidents` + +## 当前 Earth 行为 + +相关文件： + +- `frontend/public/earth/js/bgp.js` +- `frontend/public/earth/js/main.js` +- `frontend/public/earth/js/info-card.js` +- `frontend/public/earth/js/constants.js` +- `frontend/public/earth/index.html` + +当前设计： + +1. BGP 启用时始终显示 collectors。 +2. Incident marker 现在是 Earth BGP 的主 marker。 +3. 如果没有 incident，Earth 回退显示 anomaly marker。 +4. 如果也没有 anomaly，collector 仍然提供存在感。 +5. 专用 `activity layer` 现在增加： + - 每个 collector 最近 15 分钟活动 halo + - 基于活跃 collector 推导的区域聚合活动提示 +6. Incident marker 现在使用： + - 由符号驱动的事件核心 + - 向外扩散的环形脉冲 + - 相比旧版 Earth 更少的弥散 glow +7. 右侧统计现在显示： + - BGP events + - collector count + - BGP status summary + +这个方向是对的，但在低事件密度时期仍不完整。当前 Earth 在 incident 稀疏时仍可能显得过于安静，因为系统还缺少位于原始观测和 incident 聚焦之间的专用 `activity layer`。 + +当前 BGP 状态策略： + +- 有 incident：显示活跃 incident 数量 +- 无 incident 但有 anomaly：显示活跃 anomaly 数量，并在可用时显示活跃观测区域 +- 无 incident/anomaly 但有 activity：显示 `观测网络运行中` +- 无 incident/anomaly 但有 collectors：显示 `观测网络运行中 · 当前未发现聚合级事件` +- 完全无 BGP 数据：显示 `暂无观测数据` + +Earth info-card 策略： + +- `bgp` 卡片文案以 incident 为中心 +- `bgp_collector` 卡片显示 collector 位置和当前事件数 + +## 当前产品缺口 + +主要缺口不是架构正确性，而是低密度可视化策略。 + +当前事实： + +- incident 数量天然远低于 anomaly 数量 +- 这是预期行为，因为 incident 是聚合和去噪后的结果 +- 但 incident-first 渲染会让 Earth 显得过于安静，除非有另一层始终可用的 activity layer + +推荐 `activity layer` 的实现细节在 [BGP 区域聚合计划](/home/ray/dev/linkong/planet/docs/plans/earth-bgp-region-aggregation-plan.md) 中展开。 + +因此最近的里程碑是： + +`event map -> observability map` + +这意味着 Earth 需要三层同时可读： + +1. `observation layer` + - collectors + - recent collector activity + - baseline coverage +2. `activity layer` + - recent event density + - anomaly/noise hotspots + - regional activity scoring + - incident presence bonus +3. `incident layer` + - 稀疏但高度清晰的高置信事件对象 + - 符号化 marker + - 向外环形脉冲，而不是大面积弥散 glow + +## Incident 视觉方向 + +Earth 的 `incident` 层不应该像一大片发光区域，而应该像紧凑、高置信度的事件焦点。 + +设计原则： + +1. `incident` marker 应使用强主符号 + - 符号形状尽量承载类型含义 + - 示例： + - `origin_change`：类似三角警告 marker + - `mass_withdrawal`：告警/感叹号风格 marker + - `more_specific_burst`：分裂/放射 marker + +2. 强调应来自向外扩散的环形脉冲，而不是区域泛光 + - 使用紧凑高亮核心 + - 使用一个或多个扩张环形脉冲 + - 避免让事件中心变得模糊的大面积亮斑 + +3. `collector` 和 `incident` 必须保持视觉区别 + - collector 是观测基础设施 + - incident 是抽取后的事件焦点 + - collector activity 应比 incident pulse 更安静 + +4. 平静期仍需要观测存在感 + - collectors 和 activity layer 应让地图保持活跃 + - 一旦出现 incident，它们应明确压过附近 BGP 视觉元素 + +5. incident 地理位置应转向 `prefix-centric` + - collector 应保持证据来源身份，而不是主要事件位置 + - 推荐地理优先级： + - `prefix_geography` + - `prefix_scope` + - `ASN organization region` + - `collector centroid` 作为最终 fallback + - `prefix_scope` 应保持为由观测推导出的范围提示 + - 应新增真正面向 prefix 位置的 `prefix_geography` 层 + +参考灵感： + +- `World Monitor` + - 稀疏事件符号 + - 紧凑中心 + - 类似环形的向外脉冲 + - 比弥散 glow 更强的 incident 可读性 + +## 当前控制台行为 + +相关页面： + +- `frontend/src/pages/BGP/BGP.tsx` + +当前 BGP 控制台页面有三层： + +1. 观测摘要 + - 总事件数 + - collector 数量 + - prefix 数量 + +2. incident 摘要和 incident 表格 + +3. anomaly 详情表和最近 observation events + +这意味着即使 anomaly 为零，BGP 页面仍有可用信号。 + +## 已知产品/工程边界 + +1. 当前系统仍更接近事件看板，而不是完整 BGP sensing platform。 +2. RIS 覆盖范围仍需从较窄订阅范围继续扩展。 +3. BGPStream 历史数据仍不是完整 MRT-to-prefix 解码分析。 +4. Collector 地理位置仍高度依赖静态 RIPE RIS 映射。 +5. Incident 与海缆、IXP、区域之间的关联仍较弱，且处于早期阶段。 +6. Earth 当前可视化的是逻辑观测/影响结构，而不是真实物理流量路径。 + +## 测试状态 + +BGP 专项测试位于： + +- `backend/tests/test_bgp.py` + +当前已验证状态： + +- `backend/tests/test_bgp.py` 为 `25 passed` +- `backend/tests` 为 `62 passed` + +覆盖范围包括： + +- normalization +- observation serialization +- enrichment +- detectors，包括 route leak candidate 和 path flap +- incident aggregation +- batch anomaly creation +- BGP events/incidents API +- summary endpoints + +## 最相关文件 + +后端： + +- `backend/app/models/bgp_observation.py` +- `backend/app/models/bgp_anomaly.py` +- `backend/app/models/bgp_incident.py` +- `backend/app/services/collectors/bgp_common.py` +- `backend/app/services/bgp_enrichment.py` +- `backend/app/services/bgp_detectors.py` +- `backend/app/services/bgp_incidents.py` +- `backend/app/api/v1/bgp.py` +- `backend/app/api/v1/visualization.py` + +前端： + +- `frontend/src/pages/BGP/BGP.tsx` +- `frontend/public/earth/js/bgp.js` +- `frontend/public/earth/js/main.js` +- `frontend/public/earth/js/info-card.js` +- `frontend/public/earth/js/constants.js` +- `frontend/public/earth/index.html` + +## 推荐下一步 + +### 后端 / 检测优先级 + +1. 集成真实 RPKI 校验数据。 +2. 扩展实时 collector 覆盖范围，并更广泛纳入 withdrawals。 +3. 用更强启发式继续完善 route leak 和 path instability detector。 + +### 关联 / 叙事优先级 + +4. 强化 incident 聚合语义和标题。 +5. 增加 incident 与以下对象的弱关联： + - 海缆走廊 + - 登陆点 + - IXPs + - 其它流量异常来源 +6. 优化 Earth 中 collector 和 incident 之间的 hover/click 交接。 + +### 可视化优先级 + +7. 调整区域 activity scoring，让 activity layer 有信息量但不嘈杂。 +8. 随着新 detector 落地，增加更多 incident 符号类型。 +9. 增加真实 prefix geography 来源： + - `IPtoASN / IPtoCountry` 作为第一阶段可用数据集 + - `OpenGeoFeed` 作为更高质量 override 层 + - registry/whois 只作为 fallback diff --git a/docs/technical/zh/earth-frontend-context.md b/docs/technical/zh/earth-frontend-context.md new file mode 100644 index 00000000..e90cadea --- /dev/null +++ b/docs/technical/zh/earth-frontend-context.md @@ -0,0 +1,421 @@ +# Earth 前端结构 + +本文件描述当前 Earth 大屏前端的真实结构，重点是帮助后续继续改 HUD、图层、媒体面板、真实地形、BGP 可视化时，不再重复踩结构和状态同步上的坑。 + +相关规则建议一起参考： + +- [项目规则](/home/ray/dev/linkong/planet/rules.md) +- [前端布局指南](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-layout-guidelines.md) + +## 当前目标 + +Earth 前端不是普通管理页，它是独立的大屏展示前端。当前产品目标是： + +- 维持地球视图的空间感和可读性 +- 让 HUD、图层、媒体面板、BGP、卫星、海缆等保持统一交互 +- 把加载中、已启用、已隐藏、锁定中这类状态做清楚 + +## 当前入口 + +React 路由入口： + +- [Earth.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Earth/Earth.tsx) + +当前做法很简单： + +- React 页面只负责提供一个全屏 `iframe` +- 真正的 Earth 应用运行在： + - [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) + +所以 Earth 前端本质上是 `public/earth` 下的一套独立静态应用。 + +## 当前文件分层 + +### 1. 页面入口与结构 + +- [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) + +职责： + +- HUD 基础 DOM +- 图层面板 +- 媒体面板 +- 工具栏 +- 设置弹窗 +- 兼容旧元素 id + +### 2. 主运行时 + +- [main.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/main.js) + +职责： + +- 地球初始化 +- Three.js 场景组装 +- 数据加载与刷新 +- 各图层集成 +- Earth 级别状态同步 + +### 3. 地球控制层 + +- [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) + +职责： + +- 工具栏交互 +- 图层面板交互 +- 旋转/缩放/布局 +- HUD 面板拖拽 +- 图层开关状态机 +- Earth 设置读取、持久化与重置 + +这份文件是 Earth 前端当前最核心的 UI 控制入口。 + +### 4. UI 与状态消息 + +- [ui.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/ui.js) + +职责： + +- loading 面板 +- status message +- tooltip / error / 清理逻辑 + +当前 status message 有两类短提示： + +- 普通业务提示：通过 `showStatusMessage()` 入队显示。 +- 手势提示：通过 `showGestureStatusMessage()` 直接短暂显示，用于缩放视角时的 `缩放 N%`。 + +手势提示不会抢占 loading 状态。对应样式是 [hud.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/hud.css) 中的 `.earth-status-message.gesture`。 + +### 5. 地球与地形 + +- [earth.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/earth.js) +- [terrain.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/terrain.js) + +职责： + +- 地球球体、云层、大气 +- 真实地形 mesh +- terrain tile 拉取、解码、位移、着色 + +### 6. 图层模块 + +- [satellites.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/satellites.js) +- [cables.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/cables.js) +- [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) +- [bgp-cruise-adapter.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp-cruise-adapter.js) +- [vessels.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/vessels.js) +- [news.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/news.js) +- [tv.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/tv.js) +- [layer-startup-tasks.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/layer-startup-tasks.js) +- [cruise-sequencer.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/cruise-sequencer.js) +- [callout-connector.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/callout-connector.js) + +职责： + +- 各自的数据层 +- 开关行为 +- 面板内容 +- hover/lock/selection 语义 + +其中 Earth 启动加载链现在也拆成了两层： + +- `controls.js` + - 提供图层注册表与启动元信息 +- `layer-startup-tasks.js` + - 提供图层启动任务注册表 + - 通过 `registerLayerStartupTask(id, taskFactory)` 扩展启动任务 +- `main.js` + - 只负责读取排序后的启动图层，再按映射执行队列 + +其中巡航模式现在已经拆成两层： + +- `cruise-sequencer.js` + - 负责目标队列顺序、停留时长、切换节奏、打断与恢复 +- `callout-connector.js` + - 负责卡片连线 SVG、路径计算与绘制动画 +- `bgp-cruise-adapter.js` + - 负责 BGP 巡航展示适配：目标排序、卡片落点、连线路径、focus/overlay/info-card 时序 + +当前 BGP 巡航只是这套能力的一个调用方，不应再把“按队列巡航”和“BGP 事件展示”混写在同一个状态机里。 + +新闻巡航摘要计划见： + +- [Earth 新闻巡航摘要计划](/home/ray/dev/linkong/planet/docs/plans/earth-news-cruise-summary-plan.md) + +## 当前样式分层 + +Earth 的 CSS 不是一份大样式表，而是分层管理： + +- [base.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/base.css) +- [hud.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/hud.css) +- [toolbar.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/toolbar.css) +- [layer-panel.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/layer-panel.css) +- [info-panel.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/info-panel.css) +- [legend.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/legend.css) +- [earth-stats.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/earth-stats.css) +- [coordinates-display.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/coordinates-display.css) +- [tv-panel.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/tv-panel.css) + +当前建议： + +- 通用 HUD 壳层写进 `hud.css` +- 单一面板特性写进各自子文件 +- 不要把业务状态样式再散回 `index.html` + +## 当前图层开关状态语义 + +Earth 图层按钮现在不应再只有“开/关”两态，而应支持： + +- `inactive` +- `active` +- `loading` + +当前入口在： + +- [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- [layer-button-state.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/layer-button-state.js) + +关键函数： + +- `updateLayerButtonState(button, isActive)` +- `setLayerButtonState(button, options)` + +`setLayerButtonState` 负责： + +- `loading` 样式 +- `aria-busy` +- 按钮禁用 +- tooltip 更新 +- 绑定状态文本更新 +- 可选同步 `active` + +因此后续如果别的图层也需要异步启用，应该直接走这套状态机，而不是再手写一套临时 loading class。 + +另外，Earth 图层控制现在已经收成“注册表驱动”： + +- 图层元数据 + - `id` + - `icon` + - `label` + - `meta` + - `buttonId` + - `persist` + - `startupPriority` + - `startupMode` + - `startupLabel` + - `startupMessage` +- 图层行为 + - `getVisible()` + - `setVisible(next, options)` + +当前入口仍在 [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js)。 + +这意味着后续新增图层时，优先应补一条图层注册定义，而不是同时去改： + +- 图层面板 HTML +- 持久化快照 +- 初始化恢复 +- click 绑定 + +这四处现在都应该由注册表派生。 + +其中： + +- `startupPriority` + - 描述图层参与启动加载时的顺序 +- `startupMode` + - `visible` + - 仅当前图层处于启用/可见状态时，才加入启动加载队列 + - `preload` + - 即使当前图层未显示，也会参与启动预加载 + +当前 `main.js` 会通过注册表读取排序后的启动图层列表，再动态拼装启动加载队列，而不是手写一串固定步骤。像 BGP 这类需要尽早准备数据、但不一定默认显示的图层，应该优先走 `startupMode: "preload"`，而不是在启动流程里写隐式特判。 + +此外，启动阶段给用户看的提示文案也应尽量从注册表派生： + +- `startupLabel` + - 用于描述当前启动任务的业务名称 +- `startupMessage` + - 用于描述启动中的提示文案 + - 可以是字符串 + - 也可以是对象，用于像海缆这种“准备阶段 / 主加载阶段”两段式文案 + +这样后续新增会参与启动加载的图层时，顺序、模式和提示文案都在同一处定义，不需要再去 `main.js` 里补第二套常量。 + +### 船只图层与图例 + +AIS 船只图层入口： + +- [vessels.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/vessels.js) +- [interactable.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/interactable.js) + +船只图层当前负责： + +- 请求 `/api/v1/visualization/geo/vessels` +- 将聚合后的 AIS GeoJSON 转为地球局部坐标 marker 数据；请求默认不传 `limit`，后端和前端都不再默认裁剪到 5000 艘 +- 通过 `createInteractableLayer()` 注册 Interactable 图标层 +- 用按航向分桶的 `THREE.Points` 批量渲染普通船只 marker +- 按船型映射颜色；`vessels.js` 会用 `vessel_type_name` 和 AIS `vessel_type` 数字共同归一化船型 +- 根据航行/停泊状态绘制三角形或圆点纹理 +- 用单点 `THREE.Points` overlay 承载 hover / locked glow +- 支持 hover、lock、轨迹加载和视觉聚焦 + +船只图层不再是“每艘船一个 `THREE.Sprite`”。原始 Sprite 方案在拖动地球时会把透明对象排序、draw call 和对象级 raycast 成本全部放到主交互路径上；即使 BarentsWatch 免费 AIS 当前只覆盖挪威周边，也会让地球拖动明显不跟手。 + +当前设计把普通船只拆成少量批次： + +- moving / anchored 分开。 +- moving 船只按 `VESSEL_COURSE_BINS` 做航向分桶。 +- 每个批次是一组 `THREE.PointsMaterial`，位置和颜色写入 `BufferGeometry` attribute。 +- 普通态不带 glow；hover / locked 时才在相同点位叠加带 glow 的单点 overlay。 + +方向标准以 AIS `course / cog` 为准：从正北开始顺时针。普通态和交互态都通过同一套 canvas 旋转规则生成纹理，避免 hover 后箭头方向和原 marker 不一致。 + +船型展示也必须复用同一套归一化结果。`buildVesselMarkerData()` 会把后端的 `vessel_type_name` 和 AIS 数字类型码归一化为 `type`，用于 marker 颜色；同时生成 `vessel_type_display`，供详情卡、hover 简述和搜索结果显示。不要让详情卡直接只读原始 `vessel_type_name`，否则会出现 marker 已按 Cargo/Tanker 等颜色显示、卡片仍写 `Other` 的不一致。 + +AISStream 的 `PositionReport` 常带实时位置和 `MetaData.ShipName`，但船型通常来自低频 `ShipStaticData.Type`。后端会把 `MetaData.ShipName` 补进船名，并将类型码映射为 Cargo / Tanker / Passenger / Fishing / Military；仍缺失的船型需要等待静态 AIS 消息或后续船舶资料 enrichment，不能在前端凭颜色之外的信息臆造细分类。 + +船只 hover / click 也不再对渲染对象做 `raycaster.intersectObjects()`。`main.js` 只负责传入当前 Earth、camera、pointer 和命中半径，实际命中计算由 `interactable.js` 的图标层接口完成： + +1. 拖动地球或惯性旋转时跳过 hover picking。 +2. 对 hover picking 做轻量节流。 +3. 只保留正面船只作为候选。 +4. 将候选船只投影到屏幕坐标。 +5. 用 `VESSEL_POINTER_RADIUS_PX` 做像素距离命中，并取最近船只。 + +这样 picking 位置和用户看到的屏幕 marker 对齐，也避免 `Points` 自带 raycaster 在固定屏幕尺寸图标上的命中半径错位。 + +图例系统已经注册 `vessels` 模式： + +- [legend.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/legend.js) +- [legend.css](/home/ray/dev/linkong/planet/frontend/public/earth/css/legend.css) + +`getVesselLegendItems()` 返回带 `shape` 的图例项： + +- `shape: "vessel"`：三角形，表示航行船只。 +- `shape: "dot"`：圆点，表示停泊或低速状态。 + +图例项颜色来自 `VESSEL_CONFIG.colors`，不要在 `legend.css` 里重新定义业务颜色。新增船型时，应优先改 `vessels.js` 和 `constants.js` 的船型映射，再同步图例项。 + +`interactable.js` 是后续地表图标类图层的共用入口。它当前已经承载船只、BGP 事件、BGP 观测站和算力中心图层的批量 `Points`、texture cache、hover / locked overlay、默认 glow、状态增量更新和屏幕空间 picking；BGP 事件的向外扩散圈、BGP 观测站的 halo / 覆盖扇形仍由 `bgp.js` 保留业务动画，但图标本体和 pointer 命中已经接入通用层。新增小型、中心对齐、可以参与深度测试的图标类元素时，应优先复用这个接口，而不是再次复制船只渲染逻辑。 + +登陆点是当前明确保留的例外：它曾接入 `Interactable`，但 pin 类 SVG 在地球边缘会被 `THREE.Points` 的深度测试裁切成碎片；关闭 depthTest 又会破坏背面遮挡语义。因此登陆点退回 `cables.js` 内的专用 `THREE.Sprite` 路径，并改为 canvas 生成的黄色扁平球纹理。它的 `altitudeOffset` 和 `renderOrder` 与海缆线一致，避免漂在海缆之上；Sprite 本体关闭 `depthTest` 保持球完整，背面可见性由 `isFacingCamera()` 的球体遮挡判断控制。 + +图标资源可以继续用 canvas draw，也可以放到 `frontend/public/earth/assets/icons/` 后由 `Interactable` 预加载。asset 路径不会在每帧读取；图层加载阶段通过 `preloadAssets()` 只加载一次 SVG / 图片，之后按 `icon source + state + bucket + color` 生成 `CanvasTexture` 并复用。当前算力中心已经从 `assets/icons/compute-supercomputer.svg`、`assets/icons/compute-gpu-cluster.svg` 和备用 `assets/icons/compute-hdd-network.svg` 读取图标，再在 canvas 上叠加未确认位置的 `?` badge。算力中心后端在启动链路只渲染源数据自带坐标或 `compute_center_locations` 维表坐标；手动候选采集会调用 ROR 和 Nominatim/OpenStreetMap，并在 GeoJSON 或候选响应中返回位置精度、置信度、来源说明和核验时间；前端详情卡展示这些字段。 + +算力中心图层行左上角的通知气泡显示 GeoJSON `unresolved` 数量。这个数字表示“完全没有可信坐标、不能渲染到地球上”的记录，不等同于地图上带 `?` 的已定位待确认点。点击气泡会在图层面板右侧打开固定信息卡，信息卡内容区内部滚动，不随鼠标 hover 消失。列表中的单条 `采集` 只展示候选；顶部 `一键采用` 会按当前列表顺序逐条采集、保存最高置信候选，成功一条就移除一条、重新编号，并通过 `earth:compute-center-unresolved-count-change` 同步气泡数量。批量结束后再触发 `earth:compute-center-location-saved` 刷新真实图层。 + +asset 图标大小由 `Interactable` 的 `icon.fitSize` 控制。SVG / 图片文件应尽量保持原始 viewBox 和路径，不要为了在地球上显示成 60x60 而手写 `transform`；`drawAssetIcon()` 会把资源等比 contain 到指定尺寸并居中绘制到 atlas canvas。 + +`Interactable` 默认使用固定屏幕像素尺寸，适合船只、BGP 事件、BGP 观测站、算力中心这类需要稳定识别的图标。如果某类图标需要跟随相机距离缩放，可以把 `sizeMode` 设为非 `"fixed"`，并用 `sizeScale.min / max / referenceFov` 控制缩放范围；单个 marker 的业务尺寸差异可以通过 `getPointSizeMultiplier()` 表达，例如 BGP 事件按严重级别调整点大小，BGP 观测站按活跃度调整点大小。 + +`Interactable` 不再把图标本体额外抬离业务高度。`altitudeOffset` 就是 marker、hover glow、locked glow 和 picking 共同使用的地表高度；这样船只图标会继续贴着船只轨迹线，不会因为单独抬高显示位置而显得漂浮。后续如果要解决边缘 glow 裁切，应优先考虑 glow 纹理、overlay 尺寸或图层专属特效，而不是把通用图标层整体抬高。 + +跨 Interactable 的同坐标避让也在公共层处理。每个 marker 会保留 `icon_base_position` 作为业务原始位置；当多个 Interactable marker 归入同一个经纬度 key 时，公共层会把它们沿地表切平面排成小圈，并刷新已创建的 `THREE.Points` geometry。这样视觉位置和屏幕空间 picking 位置一致，不需要业务层再单独判断“算力中心和 BGP 事件重叠”这类场景。 + +接口细节、生命周期和接入示例见： + +- [Earth 可交互图标接入](/home/ray/dev/linkong/planet/docs/technical/zh/earth-interactable-usage.md) + +### 视角控制反馈 + +[controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) 统一维护 Earth 缩放状态。滚轮缩放、缩放按钮和触屏双指捏合最终都会更新 `zoomLevel`，并通过 `showZoomStatusCapsule()` 显示当前缩放比例： + +```javascript +showGestureStatusMessage(`缩放 ${Math.round(zoomLevel * 100)}%`, "info"); +``` + +该提示每 90ms 最多更新一次，显示 760ms 后淡出。它是视角反馈，不是数据加载进度，也不应该写进图层 loading 状态。 + +[main.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/main.js) 只负责在双指捏合缩放时调用 `setZoomLevel()` 和 `showZoomStatusCapsule()`。鼠标滚轮与缩放按钮的胶囊提示应继续放在 `controls.js`，避免同一种缩放反馈散落在多个模块。 + +拖拽地球的旋转灵敏度会根据当前缩放连续衰减，而不是按某个缩放阈值分段： + +```javascript +const scale = THREE.MathUtils.clamp( + Math.pow(zoom, -CONFIG.dragRotationZoomExponent), + CONFIG.dragRotationScaleMin, + CONFIG.dragRotationScaleMax, +); +``` + +调参入口在 [constants.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/constants.js)：`dragRotationFactorBase` 控制基础速度，`dragRotationZoomExponent` 控制放大后的衰减曲线，`dragRotationScaleMin` / `dragRotationScaleMax` 控制上下限。 + +### `data-status-target` + +图层按钮可以通过： + +- `data-status-target` + +指向一个状态文本节点。当前 terrain 已接入： + +- 按钮：`#toggle-terrain` +- 状态节点：`#terrain-status` + +地形不是默认可见图层时，启动期不会立即阻塞加载地形瓦片。`controls.js` 会在图层可见性恢复完成后才调度 `scheduleTerrainPrefetch()`，并且只在高清材质可用、地形尚未 ready、预取未开始时执行。预取使用 `setTimeout` + `requestIdleCallback`，避免和首屏云图、高清材质、图层启动队列抢主线程。 + +地形瓦片请求也不再逐个散发大量单 tile 请求。`terrain.js` 会把需要的 Terrarium tile 去重后按 `TERRAIN_CONFIG.batchRequestSize` 分批请求 `/api/v1/visualization/terrain/terrarium/batch`；后端用 LRU 内存缓存、批次去重和并发限制代理 S3 Terrarium tile。单 tile endpoint 仍保留给回退路径和浏览器缓存语义。 + +以后别的异步图层也可以沿用这套约定。 + +## 当前设置持久化 + +Earth 设置面板当前由 [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) 统一负责： + +- 捕获默认值 +- 从 `localStorage` 读取上次设置 +- 初始化应用当前设置 +- 用户变更后即时持久化 +- 一键重置回默认值 + +当前持久化的范围是： + +- 旋转模式 +- 地球默认大小（作为重置视角、缩放重置和巡航视图的默认 zoom 真源） +- HUD 面板显示/隐藏 +- 图层控制开关：`地形 / 卫星 / 轨迹 / 海缆 / BGP` +- 地形透明度 + +也就是说，Earth 设置不是一次性 UI 状态了，而是本地设备级偏好。后续如果再加入新的设置项，应优先接入同一条持久化链，而不是各自散着写 `localStorage`。 + +## 当前地形链路 + +真实地形首次启用会慢，原因不只是一个： + +1. 需要拉取 Terrarium 瓦片 +2. 需要解码图片 +3. 需要按顶点采样高程 +4. 需要重新写入 geometry 和 color +5. 需要重新计算法线与包围体 + +当前入口在： + +- [terrain.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/terrain.js) + +当前已经做了两层体验优化： + +1. 图层开关 loading 状态持续可见 +2. 页面空闲时会预热 `ensureTerrainReady()` + +## 当前巡航链路 + +当前巡航边界： + +- 通用巡航层包含： + - 当前目标 + - 队列顺序 + - 相机 focus + - 停留 / 隐藏 / 切换 +- 业务模块提供： + - 提供目标队列 + - 提供 focus 坐标 + - 提供卡片内容 + - 提供高亮/图层副作用 + +巡航模块的结构文件： + +- [cruise-sequencer.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/cruise-sequencer.js) +- [callout-connector.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/callout-connector.js) +- [bgp-cruise-adapter.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp-cruise-adapter.js) diff --git a/docs/technical/zh/earth-interactable-usage.md b/docs/technical/zh/earth-interactable-usage.md new file mode 100644 index 00000000..9605b156 --- /dev/null +++ b/docs/technical/zh/earth-interactable-usage.md @@ -0,0 +1,270 @@ +# Earth Interactable 使用说明 + +`Interactable` 是 Earth 地表“图标类可交互元素”的通用渲染入口。它把船只图层验证过的模式抽成公共能力：普通态用批量 `THREE.Points`，hover / locked 用少量 overlay，拾取走屏幕空间命中，图标资源统一转进 canvas texture，并在公共层处理 glow、状态、尺寸、贴地渲染和同坐标避让。 + +当前已接入： + +| 图层 | 业务文件 | 图标来源 | 补充动画 | +| --- | --- | --- | --- | +| AIS 船只 | `frontend/public/earth/js/vessels.js` | canvas draw，航行三角形 / 停泊圆点 | 船只轨迹仍由业务层维护 | +| 算力中心 | `frontend/public/earth/js/compute-centers.js` | `assets/icons/compute-*.svg` | 估算位置 `?` badge 通过 `icon.afterDraw()` 叠加 | +| BGP 事件 | `frontend/public/earth/js/bgp.js` | canvas draw，按事件类型绘制符号 | 向外扩散圈仍由 BGP 业务层维护 | +| BGP 观测站 | `frontend/public/earth/js/bgp.js` | `assets/icons/bgp-broadcast-pin.svg` | halo、活跃度 core、覆盖扇形和雷达扫掠仍由 BGP 业务层维护 | + +登陆点曾尝试接入 Interactable，但 pin 类 SVG 在地球边缘会被 `THREE.Points` 深度测试裁切成碎片；当前退回 `THREE.Sprite` 专用路径，并改为由 canvas 生成黄色扁平球纹理。旧 SVG 资产保留在 `assets/icons/` 目录中，但登陆点运行时不再依赖 SVG。 + +## 为什么需要 Interactable + +之前每个地表图标图层都容易各写一套： + +- icon texture 生成 +- hover / locked 状态 +- glow 样式 +- picking 命中半径 +- zoom 下的尺寸策略 +- 同经纬度对象重叠避让 + +这些逻辑如果分散在业务文件里，视觉会漂移，后续调参也会变成逐图层修补。`Interactable` 的边界是：公共层负责“图标怎么在地球上稳定显示和被选中”，业务层负责“数据从哪里来、图标表达什么语义、详情卡展示什么、是否有额外动画”。 + +## 入口 + +```javascript +import { createInteractableLayer } from "./interactable.js"; +``` + +核心调用形态： + +```javascript +const layer = createInteractableLayer({ + id: "example", + objectType: "example_object", + renderOrder: 4.4, + altitudeOffset: 0.2, + pointSize: 34, + icon: { + draw(context, options) { + // draw canvas icon + }, + }, + getPosition: (item) => ({ + latitude: item.latitude, + longitude: item.longitude, + }), + getKind: (item) => item.kind || "default", +}); +``` + +业务模块通常只暴露一层薄封装： + +```javascript +export function getExampleMarkers() { + return layer.getMarkers(); +} + +export function getExamplePointerIntersections(options) { + return layer.getPointerIntersections(options); +} + +export function setExampleMarkerState(marker, state = "normal") { + layer.setMarkerState(marker, state); +} + +export function updateExampleVisualState(lockedObjectType, lockedObject, camera) { + layer.updateVisualState(lockedObjectType, lockedObject, camera); +} +``` + +## 配置参数 + +| 参数 | 默认值 | 说明 | +| --- | --- | --- | +| `id` | 必填 | 图层唯一标识，用于 group name、避让注册和 debug。 | +| `objectType` | `id` | marker 写入 `userData.type` 的业务类型，主交互层用它判断 locked 对象。 | +| `renderOrder` | `4` | 普通 points 和 hover / locked overlay 的基础渲染顺序。 | +| `altitudeOffset` | `0.2` | 业务高度，按 `CONFIG.earthRadius + altitudeOffset` 计算原始地表位置。 | +| `pointSize` | `32` | 基准屏幕像素尺寸。普通 points 和 overlay 都以它为基础。 | +| `sizeMode` | `"fixed"` | 默认固定屏幕尺寸；非 `"fixed"` 时会按相机距离做比例缩放。 | +| `sizeScale` | `{ referenceFov: 75, min: 0.12, max: 3 }` | `sizeMode !== "fixed"` 时的缩放范围。 | +| `atlasCellSize` | `128` | icon canvas texture 尺寸。 | +| `colors` | `{}` | 支持 `normal`、按 kind 的平铺 key，以及 `byKind`。 | +| `opacity` | `{ normal: 0.88, dimmed: 0.26, hover: 0.98, locked: 1 }` | 各状态透明度。 | +| `stateScale` | `{ hover: 1, locked: 1, dimmed: 1 }` | 各状态尺寸倍率。 | +| `pulse` | `{}` | locked 态可选呼吸缩放，支持 `enabled`、`speed`、`amplitude`。 | +| `avoidance` | `{ enabled: true, precision: 4, radius: 1.1, step: 0.35 }` | 跨 Interactable 的同坐标避让配置。 | +| `icon` | 必填 | 图标来源，支持 canvas draw、SVG / 图片 asset、状态 asset、锚点和后处理。 | +| `getPosition(item)` | 必填 | 返回 `{ latitude, longitude }` 或 `THREE.Vector3`。 | +| `getKind(item)` | `item.type || "default"` | 返回业务类型，用于颜色和 texture 分桶。 | +| `getRotationBin(marker)` | `0` | 返回旋转分桶，例如船只按航向分 32 桶。 | +| `getBucketKey(marker)` | `String(getRotationBin(marker))` | 返回 texture / geometry 分桶 key。 | +| `getPointSizeMultiplier(marker)` | `1` | 单 marker 尺寸倍率。BGP 事件按严重级别、观测站按活跃度使用它。 | +| `getUserData(item)` | `item` | 写入 marker 的业务字段。 | + +## Icon 配置 + +`icon.anchor` 可选，默认 `{ x: 0.5, y: 0.5 }`，表示纹理中心对齐 marker 坐标。它只适合小范围的视觉锚点偏移；如果图标主体很大、且需要在地球边缘完整显示，例如登陆点曾使用过的 pin 类图标，不应强行走 `THREE.Points + depthTest`，否则图标主体会被地球深度裁切。 + +### Canvas 图标 + +canvas 图标适合船只、BGP 事件这类需要按状态或旋转动态绘制的符号： + +```javascript +const vesselIconLayer = createInteractableLayer({ + id: "vessels", + objectType: "vessel", + pointSize: 34, + icon: { + draw(context, { marker, rotationBin = 0, glow = false, color = "#ffffff" }) { + if (!marker.userData.anchored) { + context.rotate((rotationBin / 32) * Math.PI * 2); + } + context.fillStyle = color; + context.shadowColor = color; + context.shadowBlur = glow ? 14 : 0; + context.beginPath(); + context.moveTo(0, -37); + context.lineTo(28, 32); + context.lineTo(0, 17); + context.lineTo(-28, 32); + context.closePath(); + context.fill(); + }, + }, + getRotationBin: getCourseBin, + getBucketKey: (marker) => `${marker.userData.anchored ? "anchored" : "moving"}:${getCourseBin(marker)}`, +}); +``` + +当 `icon.coordinates !== "canvas"` 时，`Interactable` 会先把 context 平移到 atlas 中心；船只这类自己使用中心坐标绘制的图标不需要声明 `coordinates`。 + +### SVG / 图片 Asset 图标 + +asset 图标适合算力中心、BGP 观测站这类已有 SVG 的设施图标： + +```javascript +const computeCenterIconLayer = createInteractableLayer({ + id: "computeCenters", + objectType: "compute_center", + pointSize: 36, + atlasCellSize: 128, + icon: { + coordinates: "canvas", + colorable: false, + fitSize: 60, + glowBlur: 16, + getSource({ marker, item }) { + const siteType = marker?.userData?.site_type || item?.site_type || "gpu_cluster"; + return COMPUTE_CENTER_ICON_SOURCES[siteType]; + }, + afterDraw(context, { marker, item }) { + if (marker?.userData?.is_estimated ?? item?.is_estimated) { + drawComputeCenterEstimatedBadge(context, true); + } + }, + }, +}); +``` + +使用 asset 时有几个约定： + +- SVG / 图片文件放在 `frontend/public/earth/assets/icons/`，以 `/earth/assets/icons/name.svg` 引用。 +- 原始 SVG 应尽量保留标准 `viewBox` 和路径，不要为了显示大小写死 transform。 +- 显示尺寸由 `icon.fitSize` 控制；它可以是数字、`{ width, height }`，也可以是函数。 +- `icon.colorable !== false` 且提供状态颜色时，公共层会先把 asset 画到临时 canvas，再用 `source-in` tint 成目标颜色。 +- 多色图片或不希望被 tint 的 SVG 应设置 `colorable: false`。 + +## 生命周期 + +常规加载流程： + +```javascript +export async function loadExampleLayer(_scene, earth) { + clearExampleData(earth); + + const markerData = await fetchExampleData(); + await layer.preloadAssets(markerData); + layer.setData(markerData); + layer.attach(earth); + layer.setVisible(showExampleLayer); + + return { totalCount: layer.getCount() }; +} +``` + +各方法职责： + +| 方法 | 说明 | +| --- | --- | +| `preloadAssets(items)` | 收集 normal / hover / locked 可能用到的 asset source，并用浏览器 `Image` 预加载。canvas draw 图标可跳过。 | +| `setData(items)` | 清理旧 points，生成 marker，注册避让，按 bucket 重建 `THREE.Points`。 | +| `attach(parent)` | 将图层 group 挂到 Earth root。 | +| `setVisible(next)` | 控制 group、points 和 overlay 可见性。 | +| `setMarkerState(marker, state)` | 设置 `normal` / `hover` 等状态并触发视觉状态失效。 | +| `updateVisualState(focusType, focusObject, camera)` | 更新普通态 opacity / size，并刷新 hover / locked overlay。 | +| `getPointerIntersections(options)` | 屏幕空间拾取，返回按像素距离排序的命中结果。 | +| `clearData(parent)` | 注销避让、释放 geometry / material、清空 marker 并从 parent 移除 group。 | + +## Picking 接入 + +`Interactable` 不依赖 Three.js 对 `Points` 的默认 raycast。主交互层只要把 Earth、camera、pointer 和命中半径传入： + +```javascript +const intersects = getVesselPointerIntersections({ + earth, + camera, + pointer, + radiusPx: 22, + width: window.innerWidth, + height: window.innerHeight, +}); +``` + +公共层会做这些事： + +1. 把相机位置转到 Earth local 坐标。 +2. 跳过背面 marker。 +3. 把 marker world position 投影到屏幕坐标。 +4. 用 `radiusPx` 做像素距离命中。 +5. 返回最近的候选对象。 + +拖动地球、惯性旋转、hover 节流这些策略仍属于 `main.js`，因为它们和全局输入状态有关。 + +## 同坐标避让 + +避让默认开启，作用范围是所有通过 `createInteractableLayer()` 创建的图层。公共层会按经纬度或 `THREE.Vector3` 生成 `icon_avoidance_key`，同 key 的 marker 会沿地表切平面排成小圈。 + +关键点： + +- `icon_base_position` 保留业务原始位置。 +- 避让只改渲染位置和 picking 位置，不改业务经纬度。 +- 单个 marker 回到原始位置时会直接使用 `altitudeOffset` 计算出的业务贴地位置。 +- 多个 marker 同坐标时，第一圈用 `avoidance.radius`，后续每圈加 `avoidance.step`。 + +如果某个业务图层需要严格压在原始点位，可以显式关闭： + +```javascript +createInteractableLayer({ + id: "strict-layer", + avoidance: { enabled: false }, +}); +``` + +## 业务动画边界 + +`Interactable` 当前只负责图标本体和通用 hover / locked overlay。复杂动画仍放在业务模块里，但要跟随 Interactable marker 的位置： + +- BGP 事件扩散圈由 `bgp.js` 创建独立 ring sprite，并在每帧 `position.copy(marker.position)`。 +- BGP 观测站 halo、status core、coverage halo 和覆盖扇形由 `bgp.js` 管理，图标本体由 Interactable 管理。 +- 船只轨迹线仍由 `vessels.js` 管理，因为它依赖点击后额外加载的轨迹数据。 + +这个边界能避免通用接口过早承载所有动画类型。后续如果多个图层复用同一类动画，再把它收进 Interactable 的 `animations` 扩展。 + +## 新图层接入清单 + +1. 在业务文件中准备 marker data，并保留必要的业务字段。 +2. 选择 icon 类型：canvas draw、SVG / 图片 asset，或 `getSource()` 动态选择。 +3. 配置 `pointSize`、`icon.fitSize`、`colors`、`opacity`、`stateScale`。 +4. 如果需要业务尺寸差异，提供 `getPointSizeMultiplier()`。 +5. 如果有旋转，提供 `getRotationBin()` 和稳定的 `getBucketKey()`。 +6. 加载时先 `preloadAssets()`，再 `setData()`、`attach()`、`setVisible()`。 +7. 在 `main.js` 接入 `getPointerIntersections()`，并复用现有 hover / locked 状态更新流程。 +8. 在图层样式索引和渲染顺序文档中记录 altitude、renderOrder、pointSize 和动画层级。 diff --git a/docs/technical/zh/earth-layer-style-reference.md b/docs/technical/zh/earth-layer-style-reference.md new file mode 100644 index 00000000..05bb27a6 --- /dev/null +++ b/docs/technical/zh/earth-layer-style-reference.md @@ -0,0 +1,273 @@ +# Earth 图层样式属性索引 + +本文记录当前 Earth 前端各图层的材质、颜色、透明度、线宽、半径偏移和 +`renderOrder` 等样式属性。层级关系请配合 +[Earth 渲染图层顺序](/home/ray/dev/linkong/planet/docs/technical/zh/earth-render-layer-order.md) +查看。 + +## 命名约定 + +| 类别 | 约定 | 示例 | +| --- | --- | --- | +| 全局配置对象 | `*_CONFIG` | `COUNTRY_BOUNDARY_CONFIG` | +| 图层半径偏移 | `*AltitudeOffset` / `radiusOffset` | `lineAltitudeOffset`, `GRID_CONFIG.radiusOffset` | +| 透明度 | `*Opacity` | `hoverLineOpacity` | +| 渲染顺序 | `*RenderOrder` | `textureOverlayRenderOrder` | +| 颜色 | `*Color`，十六进制数字或 CSS 色值 | `lineColor`, `colors.supercomputer` | +| 线宽 | `lineWidth` / `*LineWidth` | `GRID_CONFIG.lineWidth` | + +## Earth 基座与高清材质 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| Earth 基座半径 | `CONFIG.earthRadius` | `100` | `earth.js:createEarth()` | +| Earth 基座颜色 | `EARTH_MATERIAL_CONFIG.color` | `0x010609` | `MeshPhongMaterial.color` | +| Earth 基座 emissive | `EARTH_MATERIAL_CONFIG.emissive` | `0x010609` | `MeshPhongMaterial.emissive` | +| Earth 基座 specular | `EARTH_MATERIAL_CONFIG.specular` | `0x1a2d45` | `MeshPhongMaterial.specular` | +| Earth 基座 shininess | `EARTH_MATERIAL_CONFIG.shininess` | `12` | `MeshPhongMaterial.shininess` | +| Earth 基座 opacity | `EARTH_MATERIAL_CONFIG.opacity` | `1` | `MeshPhongMaterial.opacity` | +| 高清材质半径偏移 | `EARTH_MATERIAL_CONFIG.textureOverlayAltitudeOffset` | `0.1` | 独立高清材质球半径 | +| 高清材质透明度 | `EARTH_MATERIAL_CONFIG.textureOverlayOpacity` | `0.88` | 高清材质 `MeshPhongMaterial.opacity` | +| 高清材质 renderOrder | `EARTH_MATERIAL_CONFIG.textureOverlayRenderOrder` | `0.96` | `_earthTextureOverlay.renderOrder` | +| 高清材质 specular | `EARTH_MATERIAL_CONFIG.textureOverlaySpecular` | `0x05080d` | 降低直射区域镜面高光，避免贴图死白 | +| 高清材质 shininess | `EARTH_MATERIAL_CONFIG.textureOverlayShininess` | `4` | 降低高光集中度 | +| 高清材质颜色乘色 | inline | `0xffffff` | `_earthTextureOverlayMaterial.color` | + +## Earth 遮挡与昼夜 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| 遮挡球半径系数 | `EARTH_MATERIAL_CONFIG.occluderRadiusFactor` | `0.999` | 深度遮挡球半径 | +| 遮挡球分段 | `EARTH_MATERIAL_CONFIG.occluderSegments` | `48` | 遮挡球几何分段 | +| 遮挡球 renderOrder | inline | `-1` | `occluder.renderOrder` | +| 昼夜太阳方向 | `EARTH_MATERIAL_CONFIG.dayNight.sunDirection` | `{ x: 1, y: 0.2, z: 0.4 }` | 自定义 day/night shader | +| 夜侧最低亮度 | `EARTH_MATERIAL_CONFIG.dayNight.nightFloor` | `0.24` | shader uniform | +| 日侧增强 | `EARTH_MATERIAL_CONFIG.dayNight.dayBoost` | `1.12` | shader uniform | +| 暮光宽度 | `EARTH_MATERIAL_CONFIG.dayNight.twilightWidth` | `0.2` | shader uniform | +| 暮光强度 | `EARTH_MATERIAL_CONFIG.dayNight.twilightIntensity` | `0.14` | shader uniform | +| 暮光颜色 | `EARTH_MATERIAL_CONFIG.dayNight.twilightColor` | `0x4ea0ff` | shader uniform | +| 夜侧 tint 颜色 | `EARTH_MATERIAL_CONFIG.dayNight.nightTintColor` | `0x0b1830` | shader uniform | +| 夜侧 tint 强度 | `EARTH_MATERIAL_CONFIG.dayNight.nightTintIntensity` | `0.08` | shader uniform | + +## 大气辉光与云图 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| 内层大气半径系数 | `EARTH_MATERIAL_CONFIG.atmosInnerRadiusFactor` | `1.01` | `atmosInnerGeo` | +| 内层大气分段 | `EARTH_MATERIAL_CONFIG.atmosInnerSegments` | `64` | `atmosInnerGeo` | +| 内层大气颜色 | `EARTH_MATERIAL_CONFIG.atmosInnerColor` | `[0.25, 0.62, 1.0]` | shader RGB | +| 内层大气 rim power | `EARTH_MATERIAL_CONFIG.atmosInnerRimPower` | `3.2` | shader rim | +| 内层大气强度 | `EARTH_MATERIAL_CONFIG.atmosInnerIntensity` | `0.18` | shader alpha multiplier | +| 外层大气半径系数 | `EARTH_MATERIAL_CONFIG.atmosOuterRadiusFactor` | `1.0025` | `atmosOuterGeo` | +| 外层大气分段 | `EARTH_MATERIAL_CONFIG.atmosOuterSegments` | `48` | `atmosOuterGeo` | +| 外层大气颜色 | `EARTH_MATERIAL_CONFIG.atmosOuterColor` | `[0.18, 0.45, 0.9]` | shader RGB | +| 外层大气 rim power | `EARTH_MATERIAL_CONFIG.atmosOuterRimPower` | `9.0` | shader rim | +| 外层大气强度 | `EARTH_MATERIAL_CONFIG.atmosOuterIntensity` | `0.0025` | shader alpha multiplier | +| 大气辉光 blending | inline | `THREE.AdditiveBlending` | `ShaderMaterial.blending` | +| 大气辉光 renderOrder | inline | `1` | `atmosInner/Outer.renderOrder` | +| 无高清材质边缘光颜色 | `EARTH_MATERIAL_CONFIG.rimGlowColor` | `[0.42, 0.72, 1.0]` | 高清材质隐藏或不可用时的 Fresnel shell RGB | +| 无高清材质边缘光半径系数 | `EARTH_MATERIAL_CONFIG.rimGlowRadiusFactor` | `1.0035` | `earth-rim-glow` 外扩球壳半径 | +| 无高清材质边缘光 rim power | `EARTH_MATERIAL_CONFIG.rimGlowPower` | `3.4` | shader rim 衰减；值越大边缘越窄 | +| 无高清材质边缘光强度 | `EARTH_MATERIAL_CONFIG.rimGlowIntensity` | `0.24` | shader alpha multiplier | +| 无高清材质边缘光分段 | `EARTH_MATERIAL_CONFIG.rimGlowSegments` | `96` | `earth-rim-glow` 几何分段 | +| 无高清材质边缘光 renderOrder | `EARTH_MATERIAL_CONFIG.rimGlowRenderOrder` | `1.08` | `_earthRimGlow.renderOrder` | +| 无高清材质边缘光 depthTest | inline | `false` | 避免被海陆基座或地表填充遮住 | +| 云图半径偏移 | `CLOUD_LAYER_CONFIG.radiusOffset` | `3` | 云层球半径 | +| 云图分段 | `CLOUD_LAYER_CONFIG.widthSegments / heightSegments` | `64 / 64` | 云层球几何分段 | +| 云图透明度 | `CLOUD_LAYER_CONFIG.opacity` | `0.15` | `MeshPhongMaterial.opacity` | +| 云图贴图 | `CLOUD_LAYER_CONFIG.textureUrl` | `"./assets/earth_clouds_1024.png"` | 云层贴图 | +| 云图 blending | inline | `THREE.AdditiveBlending` | `MeshPhongMaterial.blending` | + +## 海陆基座与国界 + +海陆基座是 Earth 的底图资产，随启动预加载；图层面板里的“国界线”只控制普通国界线、hover 线和可交互 hover。 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| 国界数据路径 | `COUNTRY_BOUNDARY_CONFIG.dataPath` | `"/earth/data/countries-admin0.min.geojson"` | GeoJSON 输入 | +| 海洋填充色 | local `OCEAN_HEX` | `0x010609` | 海陆基座 canvas 背景 | +| 陆地填充色 | `COUNTRY_BOUNDARY_CONFIG.landColor` | `0x080f1b` | 海陆基座 canvas 陆地 | +| 海陆基座透明度 | `COUNTRY_BOUNDARY_CONFIG.landOpacity` | `1.0` | `MeshBasicMaterial.opacity` | +| 海陆基座半径偏移 | `COUNTRY_BOUNDARY_CONFIG.landAltitudeOffset` | `0.08` | `country-land-ocean` 半径 | +| 海陆基座 renderOrder | `COUNTRY_BOUNDARY_CONFIG.landRenderOrder` | `0.86` | `country-land-ocean.renderOrder` | +| 海陆 mask 尺寸 | `landMaskWidth / landMaskHeight` | `2048 / 1024` | canvas / DataTexture 尺寸 | +| 国界 tint 颜色 | `COUNTRY_BOUNDARY_CONFIG.tintColor` | `0x0b1830` | 高清材质关闭时 tint | +| 国界 tint 半径偏移 | `COUNTRY_BOUNDARY_CONFIG.tintAltitudeOffset` | `0.04` | `country-tint` 半径 | +| 国界 tint renderOrder | `COUNTRY_BOUNDARY_CONFIG.tintRenderOrder` | `0.2` | `country-tint.renderOrder` | +| 国界线颜色 | `COUNTRY_BOUNDARY_CONFIG.lineColor` | `0x7fc7ff` | 普通国界线 | +| 国界线透明度 | `COUNTRY_BOUNDARY_CONFIG.lineOpacity` | `0.58` | 普通国界线 opacity | +| 国界线 hover 时压暗透明度 | `COUNTRY_BOUNDARY_CONFIG.dimmedLineOpacity` | `0.18` | hover 时普通国界线 opacity | +| 国界线半径偏移 | `COUNTRY_BOUNDARY_CONFIG.lineAltitudeOffset` | `0.115` | 普通国界线半径；略高于高清材质 `0.10`，低于地形基准 `0.16`，减少悬浮感 | +| 国界线 renderOrder | `COUNTRY_BOUNDARY_CONFIG.lineRenderOrder` | `2.2` | 普通国界线层级 | +| 国界 hover 颜色 | `COUNTRY_BOUNDARY_CONFIG.hoverLineColor` | `0xff3b1f` | 霓虹红橘 | +| 国界 hover 透明度 | `COUNTRY_BOUNDARY_CONFIG.hoverLineOpacity` | `1.0` | hover 实线 opacity | +| 国界 hover 半径偏移 | `COUNTRY_BOUNDARY_CONFIG.hoverAltitudeOffset` | `0.14` | hover 实线半径；贴近地表但高于普通国界线 | +| 国界 hover renderOrder | `COUNTRY_BOUNDARY_CONFIG.hoverLineRenderOrder` | `2.3` | hover 实线层级 | +| 国界 hover glow 透明度 | `COUNTRY_BOUNDARY_CONFIG.hoverGlowOpacity` | `0.38` | glow 线 opacity | +| 国界 hover glow 线宽 | `COUNTRY_BOUNDARY_CONFIG.hoverGlowLineWidth` | `3` | glow `LineBasicMaterial.linewidth` | +| 国界 hover glow 层级偏移 | `COUNTRY_BOUNDARY_CONFIG.hoverGlowRenderOrderOffset` | `0.01` | glow renderOrder = `2.29` | +| 国界 hover glow 半径偏移 | `COUNTRY_BOUNDARY_CONFIG.hoverGlowRadiusOffset` | `0.04` | glow 半径 = hover 半径 + 0.04 | + +## 真实地形 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| 地形 tile size | `TERRAIN_CONFIG.tileSize` | `256` | Terrarium tile 读取 | +| 地形 base zoom | `TERRAIN_CONFIG.baseZoom` | `4` | 地形采样 zoom | +| 地形几何分段 | `geometryWidthSegments / geometryHeightSegments` | `320 / 320` | 地形球几何 | +| 地形基准半径偏移 | `TERRAIN_CONFIG.baseRadiusOffset` | `0.16` | 地形压过高清材质 | +| 地形夸张系数 | `TERRAIN_CONFIG.exaggeration` | `34` | 海拔转世界单位 | +| 地形陆地淡入高度 | `TERRAIN_CONFIG.landRevealFadeMeters` | `220` | 顶点 alpha | +| 地形透明度 | `TERRAIN_CONFIG.opacity` | `0.68` | `MeshPhongMaterial.opacity` | +| 地形颜色 | `TERRAIN_CONFIG.color` | `0x8aa884` | `MeshPhongMaterial.color` | +| 地形 emissive | `TERRAIN_CONFIG.emissive` | `0x030704` | 降低自发光，恢复地形明暗层次 | +| 地形 specular | `TERRAIN_CONFIG.specular` | `0x344438` | 给地形局部光泽，不抬高高清贴图直射亮度 | +| 地形 shininess | `TERRAIN_CONFIG.shininess` | `16` | 收紧地形高光，增强起伏辨识 | +| 地形 renderOrder | inline | `1.2` | `terrain.renderOrder` | +| 地形 polygonOffset | inline | `factor -1`, `units -1` | 降低贴近球面时的闪烁 | + +## 经纬线 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| 经纬线半径偏移 | `GRID_CONFIG.radiusOffset` | `0.14` | 经纬线球面半径 | +| 经纬线颜色 | `GRID_CONFIG.color` | `0xc0e0ff` | `LineBasicMaterial.color` | +| 经纬线透明度 | `GRID_CONFIG.opacity` | `0.08` | `LineBasicMaterial.opacity` | +| 经纬线线宽 | `GRID_CONFIG.lineWidth` | `1` | `LineBasicMaterial.linewidth` | +| 经纬线 renderOrder | `GRID_CONFIG.renderOrder` | `2.05` | 经纬线层级 | +| 纬线间隔 | `GRID_CONFIG.latitudeStep` | `15` | 纬线生成步长 | +| 经线间隔 | `GRID_CONFIG.longitudeStep` | `30` | 经线生成步长 | +| 线段采样步长 | `GRID_CONFIG.segmentStep` | `5` | 经纬线采样步长 | + +## 海缆与登陆点 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| 默认海缆颜色 | `CABLE_COLORS.default` | `0xffff44` | 无数据颜色时使用 | +| 海缆半径偏移 | `CABLE_CONFIG.line.altitudeOffset` | `0.2` | 海缆线半径 | +| 海缆线宽 | `CABLE_CONFIG.line.lineWidth` | `1` | `LineBasicMaterial.linewidth` | +| 海缆透明度 | `CABLE_CONFIG.line.opacity` | `1.0` | 海缆线 opacity | +| 海缆 renderOrder | `CABLE_CONFIG.line.renderOrder` | `1` | 海缆线层级 | +| 登陆点半径偏移 | `CABLE_CONFIG.landingPoint.altitudeOffset` | `0.2` | 与海缆线同层贴地，避免凌空 | +| 登陆点 sprite 高度 | local `LANDING_POINT_SPRITE_HEIGHT` | `3` | `THREE.Sprite` 基准高度 | +| 登陆点缩放参考 FOV | local `LANDING_POINT_SIZE_REFERENCE_FOV` | `75` | 与当前 Earth 相机 FOV 一致 | +| 登陆点缩放下限 | local `LANDING_POINT_SIZE_SCALE_MIN` | `0.16` | 地球放到 200% 之后的最小倍率，限制高倍 zoom 下的屏幕占比；`3 * 0.16 = 0.48` | +| 登陆点缩放上限 | local `LANDING_POINT_SIZE_SCALE_MAX` | `3` | 远距离时的最大倍率；当前最小缩放约只能到 `2.50` | +| 登陆点 atlas 尺寸 | local `LANDING_POINT_ATLAS_CELL_SIZE` | `128` | canvas 扁平立体球纹理尺寸 | +| 登陆点颜色 | `CABLE_CONFIG.landingPoint.color` | `0xffaa00` | `SpriteMaterial.color` | +| 登陆点透明度 | `CABLE_CONFIG.landingPoint.opacity` | `1.0` | `SpriteMaterial.opacity` | +| 登陆点 renderOrder | `CABLE_CONFIG.landingPoint.renderOrder` | `1` | 与海缆线同层；`depthTest: false` 保持球体完整，背面通过相机到球心的球体遮挡判断隐藏 | +| 登陆点 dim 亮度系数 | `landingPointVisual.dimBrightness` | `0.62` | dim 状态颜色乘数 | +| 相关登陆点高亮 opacity | `landingPointVisual.related.opacityBase / opacityPulse` | `0.8 / 0.2` | 高亮脉冲 | +| 非相关登陆点颜色 | `landingPointVisual.dimmed.colorRGB` | `{ r: 180, g: 116, b: 28 }` | dim 状态颜色，避免黑色基座透出成暗洞 | +| 非相关登陆点 opacity | `landingPointVisual.dimmed.opacity` | `0.78` | dim 状态透明度，不再用低 alpha 混黑底 | + +## 卫星、轨迹和 footprint + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| 卫星显示半径偏移 | `SATELLITE_CONFIG.displayAltitudeOffset` | `8` | 卫星点位置 | +| 卫星点基础像素大小 | `SATELLITE_CONFIG.dotBaseSize` | `2.8` | 点 shader size | +| 卫星背景点缩放 | `SATELLITE_CONFIG.dotBackdropScale` | `1.28` | 背景点大小 | +| 卫星点透明度范围 | `dotOpacityMin / dotOpacityMax` | `0.7 / 1.0` | 呼吸动画 | +| 卫星点呼吸速度 | `SATELLITE_CONFIG.dotBreathingSpeed` | `0.12` | 点 opacity 动画 | +| 卫星背景点颜色 | inline | `0x0b1626` | backdrop point baseColor | +| 卫星背景点透明度 | inline | `0.42` | backdrop point opacity | +| 卫星点透明度 | inline | `0.9` | point material opacity | +| 卫星背景点 renderOrder | inline | `5` | `satelliteBackdropPoints.renderOrder` | +| 卫星点 renderOrder | inline | `6` | `satellitePoints.renderOrder` | +| 卫星轨迹长度 | `SATELLITE_CONFIG.trailLength` | `10` | trail buffer | +| 卫星轨迹线宽 | `SATELLITE_CONFIG.trailLineWidth` | `3` | ribbon shader uniform | +| 选中 ring 大小 | `SATELLITE_CONFIG.ringSize` | `0.07` | hover / locked ring sprite | +| 卫星覆盖层 renderOrder | `SATELLITE_CONFIG.overlayRenderOrder` | `12` | locked ring / halo / orbit | +| 自发光选中点颜色 | inline default | `"#ffd25a"` | `showSelfGlowStyle()` | +| 自发光选中点透明度 | inline | `0.96` | locked dot material | +| footprint renderOrder | local `GROUND_FOOTPRINT_RENDER_ORDER` | `3` | Starlink footprint fill 和 Iridium coverage ring；必须高于地表 land / texture / terrain 层 | +| footprint group renderOrder | inline | `0` | 避免 Group 排序盖过卫星点 | + +## AIS 船只 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| 船只半径偏移 | `VESSEL_CONFIG.altitudeOffset` | `0.2` | 普通 marker 位置，贴近真实地形基础层 | +| 船只轨迹半径偏移 | `VESSEL_CONFIG.track.altitudeOffset` | `0.2` | 选中船只轨迹线，与船只 marker 同一半径；前端会把轨迹末端锚到当前 marker 位置 | +| 船只 renderOrder | local `VESSEL_RENDER_ORDER` | `4.4` | 普通 marker 和交互 overlay | +| 船只轨迹 renderOrder | `VESSEL_RENDER_ORDER - 0.1` | `4.3` | 低于船只 marker | +| 船只点像素尺寸 | local `VESSEL_POINT_SIZE` | `34` | 普通 marker 与 hover / locked overlay 共享尺寸 | +| 船只默认渲染上限 | `VESSEL_CONFIG.maxRenderedMarkers` | `0` | `0` 表示不在前端默认裁剪；正数才会给接口传 `limit` 并裁剪 marker | +| 船只纹理画布尺寸 | local `VESSEL_ATLAS_CELL_SIZE` | `128` | canvas 点纹理 | +| 航向分桶数 | local `VESSEL_COURSE_BINS` | `32` | moving 船只按 COG 分桶，降低 draw call 同时保留方向 | +| 船只 hover 拾取节流 | local `VESSEL_HOVER_PICK_INTERVAL_MS` | `100` | `main.js` hover picking | +| 船只屏幕命中半径 | local `VESSEL_POINTER_RADIUS_PX` | `22` | `main.js` 屏幕空间 picking | +| 普通船只透明度 | `VESSEL_CONFIG.marker.baseOpacity` | `0.88` | 普通 `PointsMaterial.opacity` | +| dimmed 船只透明度 | `VESSEL_CONFIG.marker.dimmedOpacity` | `0.26` | 锁定某艘船后其他批次透明度 | +| hover 船只透明度 | inline | `0.98` | hover overlay | +| locked 船只透明度 | inline | `1` | locked overlay | +| 船型颜色 | `VESSEL_CONFIG.colors.*` | cargo / tanker / passenger / fishing / military / other | `PointsMaterial.vertexColors` 和 overlay texture | + +AIS 船只普通态使用批量 `THREE.Points`，不是逐船 `THREE.Sprite`。航行船只保持三角形，停泊或低速船只保持圆点；普通态不带 glow，hover / locked 时在同一屏幕尺寸上叠加带 glow 的单点 overlay。AIS 航向按 `course / cog` 从正北顺时针解释，普通态和交互态必须使用同一套 canvas 旋转规则。 + +船型颜色和详情卡船型文本必须来自同一套归一化结果：`vessels.js` 同时读取后端 `vessel_type_name` 和 AIS 数字 `vessel_type`，先得到颜色用的 `type`，再生成 `vessel_type_display` 给详情卡、hover 和搜索使用。 + +## 算力中心 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| 算力中心半径偏移 | `COMPUTE_CENTER_CONFIG.altitudeOffset` | `0.48` | marker 位置 | +| 算力中心点像素尺寸 | local `COMPUTE_CENTER_POINT_SIZE` | `36` | `Interactable` 普通 marker 与 hover / locked overlay 共享基准尺寸 | +| 算力中心 asset fit size | local `COMPUTE_CENTER_ICON_FIT_SIZE` | `60` | SVG asset 在 `128x128` atlas canvas 内的最大绘制尺寸，由 `icon.fitSize` 控制 | +| 算力中心基础透明度 | `COMPUTE_CENTER_CONFIG.marker.baseOpacity` | `0.88` | 普通 `PointsMaterial.opacity` | +| 超算 marker 缩放 | `COMPUTE_CENTER_CONFIG.marker.supercomputerScale` | `12` | 旧 Sprite 缩放参数；当前 Interactable 路径不再直接使用 | +| GPU 集群 marker 缩放 | `COMPUTE_CENTER_CONFIG.marker.gpuClusterScale` | `12` | 旧 Sprite 缩放参数；当前 Interactable 路径不再直接使用 | +| hover 缩放 | `COMPUTE_CENTER_CONFIG.marker.hoverScale` | `1.16` | hover overlay 尺寸倍率 | +| locked 缩放 | `COMPUTE_CENTER_CONFIG.marker.lockedScale` | `1.22` | locked overlay 尺寸倍率，并叠加 pulse | +| dimmed 缩放 / 透明度 | `dimmedScale / dimmedOpacity` | `0.82 / 0.34` | dim 状态 | +| 超算颜色 | `COMPUTE_CENTER_CONFIG.colors.supercomputer` | `"#38bdf8"` | marker texture | +| GPU 集群颜色 | `COMPUTE_CENTER_CONFIG.colors.gpu_cluster` | `"#2dd4bf"` | marker texture | +| 关联颜色 | `COMPUTE_CENTER_CONFIG.colors.linked` | `"#f8fafc"` | 关联态 | +| 算力中心 renderOrder | local `COMPUTE_CENTER_RENDER_ORDER` | `4.5` | 地表设施低于卫星点 | + +## BGP 观测 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| BGP 事件半径偏移 | `BGP_CONFIG.altitudeOffset` | `0.48` | BGP 事件 Interactable marker | +| BGP collector 半径偏移 | `BGP_CONFIG.collectorAltitudeOffset` | `0.2` | BGP 观测站 Interactable marker，与船只同层贴地 | +| BGP 事件点像素尺寸 | local `BGP_EVENT_POINT_SIZE` | `34` | 事件 icon 的 Interactable 基准尺寸，按严重级别通过 `getPointSizeMultiplier()` 调整 | +| BGP 事件符号绘制尺寸 | local `BGP_EVENT_SYMBOL_SIZE` | `60` | 事件 canvas 符号在 `128x128` atlas 中的绘制尺寸 | +| BGP collector 点像素尺寸 | local `BGP_COLLECTOR_POINT_SIZE` | `36` | 观测站 Interactable 基准尺寸，按活跃度通过 `getPointSizeMultiplier()` 调整 | +| BGP collector asset fit size | local `BGP_COLLECTOR_ICON_FIT_SIZE` | `60` | `bgp-broadcast-pin.svg` 在 atlas canvas 内的最大绘制尺寸 | +| 事件基础缩放 | `BGP_CONFIG.marker.eventBaseScale` | `6.2` | 事件扩散圈锚点 | +| collector 基础缩放 | `BGP_CONFIG.marker.collectorBaseScale` | `7.4` | 观测站 halo / 覆盖动画锚点 | +| hover / dim 缩放 | `hoverScale / dimmedScale` | `1.16 / 0.92` | 交互状态 | +| 普通事件透明度 | `BGP_CONFIG.opacity.normal` | `0.78` | BGP 事件 Interactable 普通态 | +| hover 透明度 | `BGP_CONFIG.opacity.hover` | `1.0` | hover 状态 | +| dimmed 透明度 | `BGP_CONFIG.opacity.dimmed` | `0.24` | dim 状态 | +| collector 透明度 | `BGP_CONFIG.opacity.collector` | `0.62` | collector 状态 | +| critical 颜色 | `BGP_CONFIG.severityColors.critical` | `0xff4d4f` | 严重事件 | +| high 颜色 | `BGP_CONFIG.severityColors.high` | `0xff9f43` | 高危事件 | +| medium 颜色 | `BGP_CONFIG.severityColors.medium` | `0xffd166` | 中危事件 | +| low 颜色 | `BGP_CONFIG.severityColors.low` | `0x4dabf7` | 低危事件 | +| collector 基础色 | `BGP_CONFIG.collectorColor` | `0x6db7ff` | collector 默认色 | +| region 色 | `BGP_CONFIG.regionColor` | `0x2dd4bf` | 区域覆盖 | +| BGP ring 缩放 | `BGP_CONFIG.ring.scaleA / scaleB` | `2.5 / 3.4` | anomaly ring | +| BGP ring 透明度 | `BGP_CONFIG.ring.opacity` | `0.5` | anomaly ring | +| collector marker renderOrder | local `BGP_COLLECTOR_RENDER_ORDER` | `4.4` | 观测站主图标，与船只同层 | +| anomaly marker renderOrder | local `BGP_EVENT_RENDER_ORDER` | `4.5` | BGP 事件主图标，与算力中心同层 | + +## 天体与星空 + +| 正式名称 | 变量名 | 当前值 | 使用位置 / 说明 | +| --- | --- | --- | --- | +| 天球半径 | `CELESTIAL_CONFIG.skyRadius` | `2600` | 天体背景 | +| 天球透明度 | `CELESTIAL_CONFIG.skyOpacity` | `1` | 背景材质 | +| 太阳距离 / 缩放 | `sunDistance / sunScale` | `2150 / 78` | 太阳 sprite | +| 月亮距离 / 缩放 | `moonDistance / moonScale` | `2050 / 38` | 月亮 sprite | +| 太阳 halo 缩放 | `CELESTIAL_CONFIG.sunHaloScale` | `136` | 太阳 halo | +| 月亮 halo 缩放 | `CELESTIAL_CONFIG.moonHaloScale` | `62` | 月亮 halo | +| 太阳光颜色 / 强度 | `sunLightColor / sunLightIntensity` | `0xfff4df / 1.02` | scene light | +| 背光颜色 / 强度 | `backLightColor / backLightIntensity` | `0x2b4c78 / 0.3` | scene light | +| 星空点数量 | `STARFIELD_CONFIG.count` | `8000` | `createStars()` | +| 星空半径范围 | `minRadius + radiusJitter` | `800 + 200` | 随机分布 | +| 星空点颜色 | `STARFIELD_CONFIG.color` | `0xffffff` | `PointsMaterial.color` | +| 星空点大小 | `STARFIELD_CONFIG.size` | `0.5` | `PointsMaterial.size` | diff --git a/docs/technical/earth-news-live-streams-collector-format.md b/docs/technical/zh/earth-news-live-streams-collector-format.md similarity index 97% rename from docs/technical/earth-news-live-streams-collector-format.md rename to docs/technical/zh/earth-news-live-streams-collector-format.md index 2883e4d7..fbfae585 100644 --- a/docs/technical/earth-news-live-streams-collector-format.md +++ b/docs/technical/zh/earth-news-live-streams-collector-format.md @@ -1,4 +1,4 @@ -# News Live Streams Collector Format +# 新闻直播采集格式 `news_live_streams` 采集器面向“频道目录 JSON”输入，而不是直接抓网页。 @@ -103,7 +103,7 @@ ## 采集器配置方式 -`news_live_streams` 不需要单独新页面，直接复用现有数据源配置： +`news_live_streams` 不需要单独新页面，直接复用控制台 `/settings` 的“采集器设置”： - `endpoint` - 频道目录 JSON API 地址 diff --git a/docs/technical/zh/earth-render-layer-order.md b/docs/technical/zh/earth-render-layer-order.md new file mode 100644 index 00000000..e669c9b8 --- /dev/null +++ b/docs/technical/zh/earth-render-layer-order.md @@ -0,0 +1,60 @@ +# Earth 渲染图层顺序 + +本文记录当前 Earth 渲染器的图层顺序和每层意图。后续调整 +`renderOrder`、半径偏移、深度策略或指针交互时，需要同步更新这里。 + +注意：图层控制面板顺序和注册 / 启动加载顺序是两套语义。 + +| 顺序类型 | 当前顺序 | 说明 | +| --- | --- | --- | +| 控制面板顺序 | 海缆 → 轨迹 → 卫星 → 算力中心 → 船只 → BGP → 地形 → 高清材质 → 大气云图 → 国界线 → 经纬线 | 由 `displayOrder` 控制，按操作关注度排列。 | +| 注册 / 启动加载顺序 | 经纬线 → 国界线 / 海陆基座 → 高清材质 → 大气云图 → 海缆 → 算力中心 → 船只 → BGP → 卫星 | 由注册顺序和 `startupPriority` 控制，按地表到天空排列；启动队列会先读取保存的图层可见状态，明确关闭的普通图层不预加载，高清材质关闭时不下载贴图；国界线图层例外，海陆基座始终预加载，保存状态只控制可交互国界线和 hover；轨迹和地形是依赖/可选显示层，不参与常规启动数据加载。 | + +## 地表图层栈 + +| 顺序 | 图层 | 来源 | 渲染 / 半径策略 | 深度 / 交互策略 | 备注 | +| --- | --- | --- | --- | --- | --- | +| -1000 | 天体背景 mesh | `celestial.js` | 背景球 | 不参与地表拾取 | 位于所有 Earth 内容之后。 | +| -1 | Earth 遮挡球 | `earth.js` | 地球内侧不可见球 | 写入深度 | 遮挡地球背面的对象。 | +| 0 | Earth 基座球 | `earth.js` | `CONFIG.earthRadius` | 地表拾取兜底目标 | 深色基座，所有可选地图层关闭时仍可见。 | +| 0.2 | 国界暗色 tint | `country-boundaries.js` | `tintAltitudeOffset` | 禁用 raycast | 高清材质关闭时使用。 | +| 0.86 | 海陆基座填充 | `country-boundaries.js` | `landAltitudeOffset`; 海洋 `#010609`，陆地 `#080f1b` | 禁用 raycast | 即使国界线关闭，基座地图仍保持可用。 | +| 0.96 | 高清 Earth 材质 | `earth.js` | `textureOverlayAltitudeOffset` | 可见时作为地表拾取目标 | 高清材质始终压过海陆基座填充。 | +| 1 | 大气辉光和云图 | `earth.js` | 大气 / 云层球 | 不走普通对象选择路径 | 云图由“大气云图”图层开关控制。 | +| 1 | 海缆 / 登陆点 | `cables.js` | 海缆线和登陆点都使用 `renderOrder = 1`；半径偏移都为 `0.2`；登陆点是专用 `THREE.Sprite` 黄色扁平球 | 海缆走海缆拾取路径；登陆点 `depthTest: false` 保持球体完整，并用相机到球心的球体遮挡判断避免背面穿透 | 登陆点和海缆同层贴地，避免地表设施层的凌空感。 | +| 1.2 | 真实地形 | `earth.js`, `terrain.js` | `TERRAIN_CONFIG.baseRadiusOffset` 加地形位移 | 禁用 raycast | 地形压过高清材质；高清材质关闭时临时隐藏，重新开启后恢复原状态。 | +| 2.05 | 经纬线 | `earth.js` | `CONFIG.earthRadius + 0.14` | 禁用 raycast | 低透明度显示在高清材质上。 | +| 2.2 | 国界线 | `country-boundaries.js` | `lineAltitudeOffset = 0.115` | `depthTest: true`，禁用 raycast | 略高于高清材质 `0.10`，低于地形基准 `0.16`，减少悬浮感；地形 `depthWrite: false`，所以地形开启时仍可见。 | +| 2.29 | 国界 hover 光晕 | `country-boundaries.js` | hover 半径加 glow 偏移 | `depthTest: false`，禁用 raycast | 用 additive 光晕增强交界边和地形开启时的 hover 可见性。 | +| 2.3 | 国界 hover 实线 | `country-boundaries.js` | `hoverAltitudeOffset = 0.14` | `depthTest: false`，禁用 raycast | 霓虹红橘 hover 线；中国和中国（台湾）共享高亮组。 | +| 3 | 卫星 footprint 填充 / Iridium coverage ring | `satellites.js`, `iridium-footprint-adapter.js` | `GROUND_FOOTPRINT_RENDER_ORDER` | depth-tested；Iridium adapter 的 fill / ring 也使用同一 renderOrder | Footprint 在 land / texture / terrain 和国界线之上，但在算力中心和卫星之下。 | +| 3-4.5 | BGP 观测站、事件扩散圈和事件 marker | `bgp.js`, `interactable.js` | BGP 观测站和事件 marker 均使用 `Interactable` 批量 `THREE.Points`；事件 marker 使用 `BGP_EVENT_RENDER_ORDER = 4.5`；观测站主图标使用 `BGP_COLLECTOR_RENDER_ORDER = 4.4` 和 `BGP_CONFIG.collectorAltitudeOffset = 0.2`；事件 overlay 进入 `bgp-event-overlay-layer`；观测站 halo 和覆盖扇形进入 `bgp-collector-radar-layer` | BGP 事件和观测站都通过 `Interactable` 屏幕空间 picking，并参与同坐标避让 | BGP 观测站主图标与船只同层；BGP 事件与算力中心同层；向外扩散圈、观测站雷达/覆盖动画继续由 BGP 业务逻辑驱动。 | +| 4.3 | AIS 船只轨迹线 | `vessels.js` | `VESSEL_RENDER_ORDER - 0.1`；`CONFIG.earthRadius + VESSEL_CONFIG.track.altitudeOffset` | 跟随船只显隐，不单独参与拾取 | 选中船只后显示最近轨迹，低于船只 marker。 | +| 4.4 | AIS 船只 marker | `vessels.js`, `interactable.js` | `VESSEL_RENDER_ORDER`；业务高度为 `CONFIG.earthRadius + VESSEL_CONFIG.altitudeOffset`；普通 marker 为分桶 `THREE.Points`，hover / locked 为单点 `THREE.Points` overlay | `depthTest: true`；`main.js` 使用屏幕空间 picking，只取正面 marker；参与 Interactable 同坐标避让 | 航行船只用三角点纹理，停泊/低速用圆点；普通态无 glow，交互态叠加同尺寸 glow；低于算力中心 `4.5`。 | +| 4.5 | 算力中心 | `compute-centers.js`, `interactable.js` | 使用 `COMPUTE_CENTER_RENDER_ORDER` 并由 `Interactable` 绘制 | 通过 `Interactable` 屏幕空间 picking，参与同坐标避让 | 地表设施层，保持在卫星下方。登陆点已下沉到海缆层。 | +| 5 | 卫星背景点 | `satellites.js` | 固定 renderOrder | 屏幕空间卫星拾取 | 位于卫星点下方。 | +| 6 | 卫星点 | `satellites.js` | 固定 renderOrder | 屏幕空间卫星拾取 | 卫星点压过 footprint 和算力中心。 | +| 12+ | 卫星锁定 ring、halo、预测轨道 | `satellites.js` | `SATELLITE_CONFIG.overlayRenderOrder` 及偏移 | 卫星覆盖层路径 | 用于选中 / 锁定卫星强调。 | +| 98-100 | 太阳 / 月亮 halo 和 sprite | `celestial.js` | 固定 renderOrder | 天体拾取禁用 | 前景天体 sprite。 | + +## 开关联动 + +| 开关 | 行为 | +| --- | --- | +| 高清材质 off | 隐藏高清材质，启用国界 tint / 基座表面，禁用地形和昼夜开关交互，并记住地形和昼夜之前状态。 | +| 高清材质 on | 恢复高清材质，并恢复记住的地形 / 昼夜状态。 | +| 地形 on | 显示在高清材质之上，但低于国界 hover、footprint、卫星等强调层。 | +| 大气云图 | 只控制云图 mesh 显隐。 | +| 国界线 off | 只隐藏可交互国界线和 hover，高亮状态会清除；海陆基座填充仍作为 Earth 底图保留。 | + +## 交互规则 + +| 交互 | 当前规则 | +| --- | --- | +| Earth 坐标 hover | 高清材质可见时使用高清材质 overlay 作为地表拾取目标，否则使用 Earth 基座球。 | +| 国界 hover | 先把地表拾取坐标转成经纬度，再用 GeoJSON 点面判断；国界 hover 线本身不接收 raycast。 | +| 国界 hover 视觉 | hover 时压暗普通国界线，并绘制无深度测试的光晕和实线。 | +| 中国 / 台湾 hover | `CHN` 和 `TWN` 被归到同一个 hover 高亮组；tooltip 仍显示鼠标实际命中的 feature。 | +| 地形 | 只作为视觉层参与，`terrain.raycast` 已禁用。 | +| 卫星 | 使用屏幕空间卫星拾取，避免 footprint 或地表层挡住卫星点击。 | +| 船只 | 不使用对象级 sprite raycast。`main.js` 会在拖动 / 惯性期间跳过 hover picking，平时将正面船只投影到屏幕坐标，用像素半径命中最近船只；点击后可加载轨迹线。 | diff --git a/docs/technical/zh/earth-satellite-footprint-policy.md b/docs/technical/zh/earth-satellite-footprint-policy.md new file mode 100644 index 00000000..6cd28837 --- /dev/null +++ b/docs/technical/zh/earth-satellite-footprint-policy.md @@ -0,0 +1,190 @@ +# Earth 卫星覆盖策略 + +本文件记录 Earth 卫星图层当前关于 `footprint` 的产品边界、资料依据和已落地实现，目标是避免把 Starlink 这套专用地表覆盖模型误用到其它星座上。 + +相关上下文： + +- [Earth 前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/earth-frontend-context.md) +- [数据采集系统](/home/ray/dev/linkong/planet/docs/technical/zh/backend-collectors.md) +- [backend/app/services/collectors/celestrak.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/celestrak.py) +- [frontend/public/earth/js/satellites.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/satellites.js) + +## 当前目标 + +- 明确哪些非 Starlink 卫星不该显示贴地 footprint +- 明确哪些星座未来可以有独立 footprint，但不能复用 Starlink bowtie / GSO-gap 模型 +- 把这条策略沉淀成可执行实现边界，而不是继续散落在视觉参数里 + +## 本地实际类别 + +当前 CelesTrak 卫星分组在 [backend/app/services/collectors/celestrak.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/celestrak.py) 中包括： + +- `starlink` +- `gps-ops` +- `galileo` +- `glonass` +- `beidou` +- `leo` +- `geo` +- `iridium-next` + +其中非 Starlink 类别是： + +- `gps-ops` +- `galileo` +- `glonass` +- `beidou` +- `leo` +- `geo` +- `iridium-next` + +## 资料结论 + +### 1. GNSS / RNSS: `gps-ops`, `galileo`, `glonass`, `beidou` + +默认不要画局部地表 footprint。 + +原因： + +- 公开资料强调的是 `Earth-pointing`、`Earth coverage`、`continuous global coverage` +- 这类系统的公开语义是全球导航 / 授时覆盖，不是 Starlink 那种面向终端业务的局部 spot footprint + +更合适的表示： + +- 默认只显示卫星本体和轨道 +- 如果后续要强调“服务可达性”，只能做很弱的 global coverage 语义，不应画贴地局部光斑 + +资料： + +- [GPS III EC Antenna Patterns](https://www.navcen.uscg.gov/sites/default/files/pdf/gps/GPS_ZIP/GPS_III_EC_Antenna_Patterns_SVN_74_75_76_77_78.pdf) +- [ESA Galileo satellites](https://www.esa.int/Applications/Satellite_navigation/Galileo/Galileo_satellites) +- [Navipedia Galileo General Introduction](https://gssc.esa.int/navipedia/index.php/Galileo_General_Introduction) +- [BeiDou official overview](https://www.beidou.gov.cn/xt/gfxz/201812/P020190117356387956569.pdf) +- [GPS.gov GNSS overview](https://www.gps.gov/systems/gnss/) + +### 2. `iridium-next` + +可以有 footprint，但不能复用 Starlink 的单一 bowtie footprint。 + +原因： + +- Iridium NEXT 公开资料强调的是固定多 spot beam 体系 +- 公开示例里常见的是 `48 fixed spot beams in 4 tiers` +- 这和 Starlink 当前这套“单星、单主 footprint、带 GSO 缺口”的业务可视化不是同一个问题 + +更合适的表示： + +- 默认：仍然不画 Starlink 式地表 footprint +- 后续如果要做：单独接入 Iridium 多波束适配层 +- 在视觉上更接近多束 cluster / 蜂窝 / 分层束，而不是单个 bowtie 光斑 + +资料： + +- [Iridium Satellite Spot Beam Coverage on the US](https://www.mathworks.com/help/phased/ug/iridium-satellite-spot-beam-coverage-on-the-us-1.html) + +### 3. `geo` + +默认不要画统一 footprint。 + +原因： + +- GEO 通信星公开上可能是 global beam、zone beam、spot beam、steerable spot beam +- 没有 operator / payload / beam contour 元数据时，统一画一个 footprint 很容易错 + +更合适的表示： + +- 默认只显示 GEO belt 和卫星驻点语义 +- 只有拿到 beam contour / operator metadata 时才允许画 footprint + +资料： + +- [ITU Handbook on Satellite](https://www.itu.int/dms_pub/itu-r/opb/hdb/R-HDB-42-2002-PDF-E.pdf) + +### 4. `leo`（generic） + +默认不要画 footprint。 + +原因： + +- `leo` 组过于混杂，可能同时包含通信、遥感、试验、观测等不同任务 +- 没有 mission / payload / antenna pattern 元数据时，无法判断是否存在可视化意义上的服务覆盖面 + +更合适的表示： + +- 默认只显示卫星和轨道 +- 后续如果按 operator / mission subtype 细分，再决定是否引入独立 coverage mode + +## 产品策略 + +当前统一策略如下： + +- `Starlink` + - 保留当前专用 `ground_footprint` 逻辑 +- `Iridium NEXT` + - 预留独立适配层 + - 当前不复用 Starlink footprint +- `GPS / Galileo / GLONASS / BeiDou` + - 不显示贴地 footprint +- `GEO` + - 无 beam metadata 不显示 footprint +- `generic LEO` + - 无 mission metadata 不显示 footprint + +## 已落地实现 + +本次实现只做最小可执行版本，不改现有 Starlink 视觉参数： + +1. 后端把星座分组和 footprint 策略提示透给前端 + +- CelesTrak collector 会把 `GROUP` 记入 `metadata.constellation_group` +- Visualization API 会输出： + - `properties.constellation_group` + - `properties.footprint_policy` + +当前策略值： + +- `starlink_ground_footprint` +- `iridium_coverage_ring` +- `none` + +对应代码： + +- [backend/app/services/collectors/celestrak.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/celestrak.py) +- [backend/app/api/v1/visualization.py](/home/ray/dev/linkong/planet/backend/app/api/v1/visualization.py) + +2. 前端把 footprint 变成 capability-gated renderer + +- `ground_footprint` 只有在 `footprint_policy === starlink_ground_footprint` 时才真正启用 +- `iridium-next` 不再回退成占位分支，而是走独立的 Iridium coverage ring adapter +- 其它非 Starlink 即使用户全局选择了 `ground_footprint`，也会自动回退到 `self_glow` + +对应代码： + +- [frontend/public/earth/js/satellites.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/satellites.js) +- [frontend/public/earth/js/iridium-footprint-adapter.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/iridium-footprint-adapter.js) + +3. 卫星信息卡显示 capability，而不是只显示轨道参数 + +- 卫星详情现在会明确显示： + - `星座/分组` + - `覆盖能力` + - `当前显示` + - `覆盖模型` +- 这样用户能直接看到： + - 当前卫星是否支持 footprint + - 当前显示是不是因为 capability gating 被回退 + - Iridium 和 Starlink 使用的不是同一种模型 + +对应代码： + +- [frontend/public/earth/js/main.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/main.js) +- [frontend/public/earth/js/info-card.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/info-card.js) + +## 当前实现边界 + +这条边界需要继续保持： + +- `Starlink` 的 footprint 参数和 shader 逻辑只服务于 Starlink +- 非 Starlink 的能力判断属于“策略层 / 适配层” +- 不要把不同星座的覆盖模型再混写进同一套参数里 +- `iridium-next` 已经切成独立 adapter，应继续沿这条边界演进，而不是给现有 Starlink bowtie 增加更多 if/else diff --git a/docs/technical/zh/earth-toolbar-overlay-coordination.md b/docs/technical/zh/earth-toolbar-overlay-coordination.md new file mode 100644 index 00000000..59618cb0 --- /dev/null +++ b/docs/technical/zh/earth-toolbar-overlay-coordination.md @@ -0,0 +1,94 @@ +# Earth 工具栏与浮层协同 + +本文件描述 Earth 大屏右侧工具栏按钮，以及搜索面板、设置弹窗、新闻直播面板、图层面板这几个浮层之间当前的协同规则。改交互、加按钮、调整面板时按这个表对齐，避免出现「点 A 把不该关的 B 也关了」之类的协同冲突。 + +相关入口： + +- [Earth 前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/earth-frontend-context.md) +- [前端布局指南](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-layout-guidelines.md) + +## 工具栏按钮目录 + +工具栏在 [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) 中以 `.earth-toolbar-btn` 标识，按钮列表： + +| ID | 标题 | 类型 | 触发的浮层/动作 | +|----|------|------|------------------| +| `layer-action` | 图层 | 浮层切换 | HUD 面板 `layer-toggles`（桌面）/ 移动端抽屉 `layers` 卡 | +| `search-action` | 搜索 | 浮层切换 | 搜索面板（桌面）/ 移动端抽屉 `search` 卡 | +| `rotate-toggle` | 自动旋转 | 独立开关 | 不打开任何浮层 | +| `toggle-tv` | 新闻直播 | 浮层切换 | 媒体面板 `media-panel`（含 TV/News 两个 tab） | +| `reload-data` | 重新加载数据 | 独立动作 | 不打开任何浮层 | +| `zoom-trigger` | 缩放控制 | 浮动菜单 | 缩放 floating menu | +| `settings-trigger` | 设置 | 浮层切换 | 设置弹窗（桌面）/ 移动端抽屉 `settings` 卡 | +| `reset-view` | 重置视角 | 独立动作 | 不打开任何浮层 | +| `layout-toggle` | 最大化布局 | 独立开关 | 不打开任何浮层 | + +## 浮层协同的统一入口 + +[controls.js::closeTransientMobileOverlays](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) 是「打开 X 时该关谁」的统一协调函数。 + +调用约定：每个会进入 fullscreen-style 浮层的开启路径调用 `closeTransientMobileOverlays({ except })`，告诉协调函数「除了我这一类，其他互斥浮层一律关掉」。 + +```js +closeTransientMobileOverlays({ except: "search" }); // 搜索打开 +closeTransientMobileOverlays({ except: "settings" }); // 设置打开 +closeTransientMobileOverlays({ except: "media" }); // 新闻直播打开 +closeTransientMobileOverlays({ except: "layer-toggles" }); // 图层抽屉（移动端） +``` + +`except` 当前可取的值：`"search"`、`"settings"`、`"media"`、`"layer-toggles"`，或省略表示「全部关闭」。 + +## 关闭矩阵 + +下表描述「打开 X」时其它浮层的命运。`✓` = 关闭，`—` = 保留。 + +| 触发动作 → | 关搜索 | 关设置 | 关图层抽屉（移动端） | 关新闻/直播 | +|-----------|:------:|:------:|:--------------------:|:-----------:| +| 打开搜索 (`except: "search"`) | （自身）| ✓ | ✓ | — | +| 打开设置 (`except: "settings"`) | ✓ | （自身）| ✓ | — | +| 打开新闻/直播 (`except: "media"`) | ✓ | ✓ | ✓ | （自身）| +| 打开图层抽屉 (`except: "layer-toggles"`) | ✓ | ✓ | （自身）| ✓ | +| 全部关闭 (`except: null`) | ✓ | ✓ | ✓ | ✓ | + +读法举例： + +- 点工具栏「设置」，搜索面板和图层抽屉会被关掉，新闻/直播面板保持原状。 +- 点工具栏「图层」（移动端打开 `layers` 抽屉），搜索 / 设置 / 新闻 全关。 +- 点工具栏「新闻直播」，搜索 / 设置 / 图层抽屉全关，新闻面板自身切换为打开。 + +## 设计原则 + +下面是当前矩阵背后的几条不变量。新增浮层或调整规则时按它们对齐： + +1. **`zoom-trigger` 等浮动菜单不属于浮层。** 它们走 `bindFloatingMenu`，由 `closeFloatingMenus()` 单独管理；任何浮层打开都会先调一次 `closeFloatingMenus()`。 +2. **桌面 `layer-toggles` 是常驻 HUD 面板，不是浮层。** `closeTransientMobileOverlays` 中只有 `activeMobileDrawerId === "layer-toggles"`（移动端抽屉态）才会被关掉。所以桌面打开搜索/设置/新闻不会动图层面板，符合「桌面屏幕大、可共存」的预期。 +3. **新闻/直播面板独立于设置。** 用户切到设置改采集器时，常常想边看新闻边改配置，所以打开设置时不关新闻面板。这条是 2026-05 的协同补丁后建立的不变量；改设置打开路径时不要再去主动关 `media-panel`。 +4. **搜索和新闻面板视为「主信息浮层」，互相独立。** 搜索打开不关新闻、新闻打开不关搜索：两者面向不同任务（搜索定位 / 浏览态势新闻），允许同屏共存。如果未来 UX 上希望它们互斥，要在 `closeTransientMobileOverlays` 中**同时**改两边的规则，避免单边修改导致非对称的关闭逻辑。 +5. **移动端抽屉是 fullscreen 级别的状态。** 一旦进入移动端抽屉，无论是 `layers` / `search` / `settings` 哪一类，都会通过 `setMobileDrawerState` 关闭其它浮层。这是 mobile 单一焦点 UX 的要求。 +6. **`Escape` 键有固定的关闭顺序。** 见 [controls.js::setupKeyboardControls](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js)：搜索 → 设置 → 移动端抽屉 → 浮动菜单 → 工具栏 hub → 锁定对象。新增浮层要决定它在这个顺序中的位置。 + +## 新加按钮 / 浮层时怎么接 + +按下面的清单走，规则就不会乱： + +1. 按钮加在 [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) 的 `.earth-toolbar` 容器里，class 跟齐 `floating-btn liquid-glass-surface earth-toolbar-btn`。 +2. 决定它属于哪一类： + - **独立动作**（reload / reset / rotate / layout）：直接 `bindListener`，不调任何 `closeTransientMobileOverlays`。 + - **浮动菜单**（zoom 这种 dropdown）：用 `bindFloatingMenu`，不进协同矩阵。 + - **互斥浮层**：进矩阵。 +3. 互斥浮层要做两件事： + - 在打开路径调用 `closeTransientMobileOverlays({ except: "" })`，让其他浮层主动让位。 + - 在 `closeTransientMobileOverlays` 函数体内补一条 `if (except !== "" && isYourPanelVisible()) closeYourPanel();` 让别的浮层打开时关掉自己。 +4. 如果新浮层和某个现有浮层（例如新闻面板）应当共存，参考第 3 条规则：在自己的关闭判断里 `&& except !== ""` 把对方排除掉。**不要**只单边改一处，否则关闭逻辑会非对称。 +5. 新浮层应该有 `Escape` 关闭路径，加在 `setupKeyboardControls` 中合适的位置。 +6. 移动端如果应进入抽屉态，使用 `setMobileDrawerState({ open: true, card: "" })` 而不是直接 toggle 面板。 + +## 当前实现位置 + +- 协调入口：[controls.js::closeTransientMobileOverlays](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- 设置浮层：[controls.js::openSettingsModal / closeSettingsModal](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- 搜索浮层：[controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js)（导入自 search 模块） +- 新闻/直播浮层：[tv.js::setTVPanelVisible](/home/ray/dev/linkong/planet/frontend/public/earth/js/tv.js)、新闻 tab 在 [news.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/news.js) +- 图层抽屉（移动端）：[controls.js::setMobileDrawerState](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- 浮动菜单：[controls.js::bindFloatingMenu](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- 工具栏 DOM：[index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) diff --git a/docs/technical/zh/frontend-admin-frontend-context.md b/docs/technical/zh/frontend-admin-frontend-context.md new file mode 100644 index 00000000..1a4445be --- /dev/null +++ b/docs/technical/zh/frontend-admin-frontend-context.md @@ -0,0 +1,325 @@ +# 控制台前端结构 + +本文件描述当前控制台前端的真实结构，目标是帮助后续页面开发、表格改造、布局治理和状态收口时快速找到正确入口。 + +相关规则建议一起参考： + +- [项目规则](/home/ray/dev/linkong/planet/rules.md) +- [前端布局指南](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-layout-guidelines.md) + +## 当前目标 + +控制台前端承担的是后台工作台，而不是展示型大屏。当前约束是： + +- 页面默认遵循单屏工作区 +- 主交互在内部模块滚动，而不是依赖整页无限变长 +- 列表、表格、分析页优先保证主工作区可见 +- 通用布局、滚动条、表格滚动行为尽量复用，不要每页各写一套 + +## 当前路由入口 + +主入口在： + +- [App.tsx](/home/ray/dev/linkong/planet/frontend/src/App.tsx) + +当前后台相关路由包括： + +- `/admin` +- `/users` +- `/datasources` +- `/data` +- `/alerts/system` +- `/alerts/bgp` +- `/alerts/situational` +- `/bgp` +- `/playground` +- `/settings` + +`/earth` 是独立展示页，不属于控制台骨架。 + +## 当前页面骨架 + +控制台公共壳层在： + +- [AppLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/components/AppLayout/AppLayout.tsx) + +职责： + +- 左侧导航 +- 折叠与展开 +- 当前账号/版本信息 +- 内容区高度闭合 +- 全站统一侧边栏滚动条 + +当前结构是： + +```tsx + + ... + + +
{children}
+ + + +``` + +后续控制台页面应优先适配这套壳层，而不是重新定义全页高度语义。 + +## 当前共享组件 + +### 1. `Scrollbar` + +文件： + +- [Scrollbar.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/Scrollbar.tsx) + +用途： + +- 控制台侧边栏这类普通内容容器 +- 组件内部管理可见性、thumb 尺寸、拖拽和双轴 overflow 判定 + +当前约束： + +- 滚动条必须是浮层，不参与布局 +- 无 overflow 时不应留下可见痕迹 +- 真实滚动仍交给原生容器，只替换可见层和交互层 + +### 2. `ScrollbarOverlay` + +文件： + +- [ScrollbarOverlay.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/ScrollbarOverlay.tsx) + +用途： + +- Ant Table 这类内部已有滚动容器的区域 +- 不接管滚动语义，只叠加新的滚动条可见层 + +当前使用场景： + +- 数据源 +- 采集数据 +- 用户管理 +- 设置页 +- 告警页 +- BGP 页面 + +### 3. `TableScrollRegion` + +文件： + +- [TableScrollRegion.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/TableScrollRegion.tsx) + +用途： + +- 为表格滚动区提供统一包裹层 +- 后续新表格页优先复用，不要重复写“表格区域 + overlay scrollbar”样板 + +### 4. `SegmentedControl` + +文件： + +- [SegmentedControl.tsx](/home/ray/dev/linkong/planet/frontend/src/components/SegmentedControl/SegmentedControl.tsx) +- [SegmentedControl.css](/home/ray/dev/linkong/planet/frontend/src/components/SegmentedControl/SegmentedControl.css) + +用途： + +- 语言切换、主题切换、模式切换这类 2 到 3 项的分段控制器 +- 需要保留滑块动画、激活态和紧凑按钮布局的设置项 +- 当前 `/docs` 页底部语言切换与主题切换已经复用它 + +接口语义： + +- `options`：每个选项包含 `value`、`label`，可选 `icon`、`title` +- `value`：当前激活值 +- `onChange`：切换选项时回调 +- `ariaLabel`：控制器可访问名称 +- `className`：业务页面用于覆盖尺寸或局部样式 + +当前约束： + +- 组件自身负责滑块数量、位置和弹性动画 +- 业务页面只传选项和状态，不要重复写私有 slider DOM +- 颜色优先通过 CSS 变量覆盖，避免在业务组件里硬编码主题色 +- 适合少量互斥选项，不适合用作长列表、导航菜单或表单下拉 + +### 5. `MarkdownRenderer` + +文件： + +- [MarkdownRenderer.tsx](/home/ray/dev/linkong/planet/frontend/src/components/MarkdownRenderer/MarkdownRenderer.tsx) + +用途： + +- 渲染 `/docs` 的 Markdown 正文 +- 支持标题、列表、引用、代码块、表格和基础行内格式 +- 代码块和表格内部复用 `Scrollbar`，避免横向内容撑爆文档页 +- Docs 正文由后端 `/api/v1/docs/...` 按 Gatekeeper 权限返回；前端只渲染当前用户可见内容 + +当前约束： + +- 它不是完整 GitHub Markdown 引擎，只覆盖项目文档当前需要的语法 +- 文档内部链接应通过 `transformLink` 转成 `/docs/:slug` +- 标题锚点由 `getHeadingId` 注入，避免渲染器自己理解路由状态 + +### 6. `TableActions` + +文件： + +- [TableActions.tsx](/home/ray/dev/linkong/planet/frontend/src/components/TableActions/TableActions.tsx) + +用途： + +- 表格操作列的统一操作入口 +- 展开状态下直接展示按钮 +- 收起状态下用更多菜单承载操作 + +配套导出： + +- `actionCellProps`：用于操作列 `onCell`，防止操作按钮被省略号截断或换行 + +## 当前状态来源 + +### 1. 认证状态 + +文件： + +- [auth.ts](/home/ray/dev/linkong/planet/frontend/src/stores/auth.ts) + +职责： + +- token +- 当前用户 +- Gatekeeper 权限组 +- 登录/退出 + +`App.tsx` 用它判断是否进入登录页。`/docs` 仍是公开路由，但目录和正文由后端按 token 决定；未登录时只返回公开文档。 + +### 2. 业务数据网关 + +目前 AI / 态势感知相关服务集中在： + +- [http-gateway.ts](/home/ray/dev/linkong/planet/frontend/src/services/situational-awareness/http-gateway.ts) +- [port.ts](/home/ray/dev/linkong/planet/frontend/src/services/situational-awareness/port.ts) +- [types.ts](/home/ray/dev/linkong/planet/frontend/src/services/situational-awareness/types.ts) + +约束： + +- 页面不要直接散落拼 URL +- 先通过 port/types 定义边界 +- 再由 http/mock gateway 实现 + +## 当前页面分层 + +### 1. 仪表盘和摘要型页面 + +例如： + +- [Dashboard.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Dashboard/Dashboard.tsx) + +优先目标： + +- 页头稳定 +- 摘要卡片先紧凑化 +- 主工作区占据主要高度 + +### 2. 表格型页面 + +例如： + +- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx) +- [DataList.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataList/DataList.tsx) +- [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Users/Users.tsx) +- [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx) + +约束： + +- 优先内部滚动 +- 不要让表格撑爆整页 +- 新表格区域优先复用 `TableScrollRegion` / `ScrollbarOverlay` + +### 数据源目录页 + +[DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx) 当前不再承担配置编辑职责，而是数据源目录和采集操作页。 + +当前页面边界： + +- 内置数据源和自定义数据源合并为 `UnifiedDataSource`。 +- 列表展示类型、状态、最近运行、采集进度和操作。 +- 点击名称打开只读抽屉。 +- 抽屉中明确显示“内置数据源”或“自定义数据源”。 +- endpoint、headers、config 只展示，不在这里编辑。 +- 需要凭证的采集器提示用户到“设置 -> 采集器设置”维护。 + +这个边界很重要：后续不要把自定义数据源编辑、内置 endpoint 覆盖或凭证表单再塞回 `/datasources`。这些配置入口统一放在 `/settings?tab=collector_credentials`。 + +页面顶部的总进度区域新增 `采集中 N` 标签： + +- 仅在存在运行中采集任务时显示。 +- 样式定义在 [index.css](/home/ray/dev/linkong/planet/frontend/src/index.css) 的 `data-source-bulk-toolbar__running-pill`。 +- 点击后打开 `采集中任务` Modal。 +- Modal 内展示每个运行任务的阶段、进度、已处理/总数。 + +这个标签和其他状态标签同排，但通过 hover、蓝色描边和箭头表示可交互，不应改成普通 Tag。 + +### 采集器设置页 + +[Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx) 中的 `collector_credentials` tab 当前显示为“采集器设置”。 + +当前页面边界： + +- 下拉框选择所有内置采集器。 +- 下拉框右侧只有一个插头图标按钮，用于健康检查。 +- 状态标签在选择器下方展示 `未检查` / `可用` / `不可用`。 +- 需要凭证的采集器将凭证卡片放在基础配置上方。 +- 不需要凭证的采集器只显示基础配置：endpoint、默认 endpoint、请求头、timeout、retry。 +- `BarentsWatch AIS` 使用专用凭证表单。 + +连接图标使用内联 `PlugConnectIcon`，视觉语义来自 Tabler `plug-connected`。后续如果控制台重写图标体系，应迁移到 Tabler Icons，而不是继续使用 Ant Design 刷新图标表达连接。 + +`Client Secret` 的表单语义： + +- 已配置时，输入框显示脱敏 preview。 +- 聚焦且当前值等于 preview 时清空，方便输入新 secret。 +- 保存时如果值仍等于 preview，提交空值表示保留原 secret。 +- 不再提供单独的“清除当前 secret”复选框。 + +凭证教程 Modal： + +- `GET /api/v1/settings/credential-guides/{provider}` 读取教程。 +- `POST /generate` 调用 AI Provider 重新生成教程。 +- `POST /reset` 恢复默认教程。 +- Modal 使用 `MarkdownRenderer` 渲染教程正文。 + +相关后端设计见： + +- [数据源、采集器设置与连接验证](/home/ray/dev/linkong/planet/docs/technical/zh/datasource-collector-settings-connectivity.md) + +### 3. 复杂工作区页面 + +例如： + +- [BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx) +- [Playground.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Playground/Playground.tsx) + +约束： + +- Tabs 里的内容不能套同一套高度逻辑 +- 表格 tab、Markdown tab、配置 tab 要各自定义滚动责任 +- AI 结果区、长文本区优先保证最小可读高度 + +## 当前布局约束 + +这些原则已经在项目里反复验证过： + +1. 父容器高度链要闭合 +2. `min-height: 0` 不能漏 +3. overflow 责任必须明确 +4. 不要用 `overflow: hidden` 掩盖结构问题 +5. 不要为了摘要卡完整显示去压缩主工作区 +6. 自定义滚动条必须是浮层，不得挤压内容宽度 + +详细经验见： + +- [前端布局指南](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-layout-guidelines.md) diff --git a/docs/technical/frontend-layout-guidelines.md b/docs/technical/zh/frontend-layout-guidelines.md similarity index 99% rename from docs/technical/frontend-layout-guidelines.md rename to docs/technical/zh/frontend-layout-guidelines.md index 81b5c96d..a158bec8 100644 --- a/docs/technical/frontend-layout-guidelines.md +++ b/docs/technical/zh/frontend-layout-guidelines.md @@ -1,4 +1,4 @@ -# Frontend Layout Guidelines +# 前端布局指南 本项目后台页面默认遵循“单屏工作区”布局规范。目标不是让页面永远不溢出，而是确保在常见桌面视口下： diff --git a/docs/technical/zh/location-pipeline-development.md b/docs/technical/zh/location-pipeline-development.md new file mode 100644 index 00000000..d7cce8c3 --- /dev/null +++ b/docs/technical/zh/location-pipeline-development.md @@ -0,0 +1,200 @@ +# 通用位置估算管线开发说明 + +`backend/app/services/location/` 是所有“给定一条记录，决定它的 lat/lon”业务的共享抽象。算力中心、BGP 观测站、BGP 事件目前都跑在这条管线上。未来需要位置估算的实体，例如卫星地面站、用户认领点位、IXP 设施，也应接入这里，而不是各自再写地理解析逻辑。 + +用户侧流程见 [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md)。 + +## 设计目标 + +历史上算力中心有自己的 4 层链路，BGP 观测站使用写死字典，BGP 事件继承 collector。三套实现互不复用，新算法也没有稳定挂入点。 + +重构后的原则： + +- 共享 `LocationResolver` 协议和 `LocationPipeline` 编排器。 +- 各领域只负责构造 `LocationQuery` 和选择 resolver 顺序。 +- 新算法通过新增 resolver 类接入，不改 ingestion、API 和前端 envelope。 +- 只有达到城市级或更高精度的位置能渲染到 Earth。 +- 本地 JSON registry 不作为算力中心或 BGP 观测站的运行时候选来源；持久事实写入数据库维表。 + +## 核心接口 + +```python +@dataclass(frozen=True) +class LocationQuery: + name: str | None + aliases: tuple[str, ...] + city: str | None + country: str | None + region: str | None + source_latitude: float | None + source_longitude: float | None + extra: Mapping[str, Any] +``` + +```python +@dataclass(frozen=True) +class LocationCandidate: + latitude: float + longitude: float + display_name: str + precision: str + confidence: float + source: str + needs_confirmation: bool + matched_fields: tuple[str, ...] + suggested_registry_entry: dict | None +``` + +```python +class LocationResolver(Protocol): + name: str + def resolve(self, query: LocationQuery) -> ResolverOutput: ... +``` + +`LocationPipeline.collect_candidates()` 返回排序后的候选和 `attempted_queries`；`resolve_best()` 返回最佳候选及诊断信息。默认排序按 source rank、precision rank、confidence，且对同 source 和同坐标候选去重。 + +## 内置 resolver + +| Resolver | 文件 | 职责 | +| --- | --- | --- | +| `SourceCoordinatesResolver` | `resolvers/source_coordinates.py` | 源记录已有 lat/lon 时直接产出 `precision="precise"` | +| `RegistryResolver` | `resolvers/registry.py` | 遗留通用 resolver；当前算力中心和 BGP 运行时链路不使用它生成候选 | +| `NominatimResolver` | `resolvers/nominatim.py` | 按领域 query plan 调 Nominatim，带 LRU 缓存和速率限制 | +| `InheritFromAnotherEntityResolver` | `resolvers/inherit.py` | 把外部实体的已解析位置包装为候选 | + +`RegistryResolver` 仍保留给后续可能的受控导入场景，但它不应被重新接入算力中心或 BGP 作为“硬编码 hint”候选源。过去仅凭 `operator`、`city` 等通用字段匹配 registry 容易把多个实体落到同一个点，这是这次下线 registry 候选链路的主要原因。 + +## 当前领域管线 + +### 算力中心 + +入口文件： + +- [compute_center_locations.py](/home/ray/dev/linkong/planet/backend/app/services/compute_center_locations.py) + +管线顺序： + +```python +SourceCoordinatesResolver() +StoredComputeCenterLocationResolver() +``` + +主地图启动链路只做“源坐标优先，其次数据库维表坐标”。数据库表为 `compute_center_locations`，唯一键是 `(source, source_id)`，用于保存人工确认或从源记录真实坐标迁入的位置。`init_db()` 只幂等迁入源记录里已有的真实经纬度，不迁入旧硬编码 hint，不在启动期批量调用 ROR、Nominatim 或 LLM。 + +手动候选采集链路和渲染链路分开。`collect_location_candidates()` 使用源字段构造 ROR 和 Nominatim/OpenStreetMap 查询，但不会把 `compute_center_locations` 当前坐标当候选返回。用户在前端确认某个候选后，通过保存接口写入维表；之后地图刷新时由 `StoredComputeCenterLocationResolver` 渲染。 + +`resolve_compute_center_location()`、`resolve_compute_center_location_full()` 和 `collect_location_candidates()` 保留为领域 API。`visualization.py` 只消费领域 API，不再持有坐标提示常量、国家质心兜底或 Nominatim 细节。 + +GeoJSON 输出只包含 `RENDERABLE_PRECISIONS` 内的位置。未解析记录进入 `unresolved`，并带上 `failure_reason`、`attempted_queries`、`source_id`、`record_id` 等诊断字段。 + +### BGP 观测站 + +入口文件： + +- [bgp_collector_locations.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_collector_locations.py) +- [bgp_collector_location.py](/home/ray/dev/linkong/planet/backend/app/models/bgp_collector_location.py) + +管线顺序： + +```python +SourceCoordinatesResolver() +StoredCollectorLocationResolver() +NominatimResolver(_bgp_collector_query_plan) +``` + +23 个 RIPE RIS collector 坐标从旧表迁入 `bgp_collector_locations` 维表，默认 `source=legacy_seed`、`needs_confirmation=true`。旧字典仍由 DB-backed cache 维护，保证下游接口兼容；手动候选采集不会把这份维表坐标当作候选，只用它补齐 site/city/country 查询上下文。 + +### BGP 事件 + +入口文件： + +- [bgp_event_locations.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_event_locations.py) + +管线顺序： + +```python +SourceCoordinatesResolver() +InheritFromAnotherEntityResolver(_inherit_from_owning_collector) +``` + +事件继承使用所属 collector 的严格查找，不跑完整 collector registry 模糊匹配。后续 ASN 设施、PrefixGeo 或 PeeringDB resolver 可以挂在继承 resolver 之后。 + +## API envelope + +```http +POST /api/v1/visualization/compute-centers/{source_id}/collect-location +POST /api/v1/visualization/compute-centers/{source_id}/location +POST /api/v1/bgp/collectors/{collector_id}/collect-location +``` + +`collect-location` 返回统一 envelope： + +```json +{ + "success": true, + "candidates": [], + "best_candidate": {}, + "attempted_queries": [], + "context": {} +} +``` + +`POST /api/v1/visualization/compute-centers/{source_id}/location` 把前端选中的候选 upsert 到 `compute_center_locations`。人工保存默认 `needs_confirmation=false`、`verification_status="verified"` 并写入 `verified_at`；如果后续接入自动暂存，也可以显式传 `needs_confirmation=true`。 + +前端 [info-card.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/info-card.js) 使用通用候选列表和预览事件渲染对象详情卡。算力中心图层按钮左上角会显示 `unresolved` 数量；点击角标打开待定位列表。列表中的 `采集` 只拉候选，`一键采用` 会逐条调用候选采集接口，选择最高置信且有有效经纬度的候选保存。保存成功一条就从列表移除并重新编号，同时通过 `earth:compute-center-unresolved-count-change` 同步角标；批量结束后再触发 `earth:compute-center-location-saved` 刷新真实图层。 + +如果剩余记录没有任何 city-level 候选，批量采用不会伪造坐标。前端会保留这些记录并展示后端返回的 `failure_reason` 和已尝试查询。 + +## 新增 resolver + +resolver 只需要实现 `name` 和 `resolve()`，返回 `ResolverOutput`。 + +```python +class PeeringDBFacilityResolver: + name = "peeringdb_facility" + + def __init__(self, client): + self._client = client + + def resolve(self, query): + asn = query.extra.get("origin_asn") + if not asn: + return ResolverOutput() + return ResolverOutput(candidates=tuple( + LocationCandidate( + latitude=f.latitude, + longitude=f.longitude, + display_name=f.name, + precision="site", + confidence=0.78, + query=f"peeringdb::{asn}", + source=self.name, + source_note=f"PeeringDB facility for AS{asn}", + matched_fields=("origin_asn",), + needs_confirmation=False, + city=f.city, + country=f.country, + ) + for f in self._client.facilities_for_asn(asn) + )) +``` + +挂入： + +```python +BGP_EVENT_PIPELINE = LocationPipeline([ + SourceCoordinatesResolver(), + InheritFromAnotherEntityResolver(source_lookup=...), + PeeringDBFacilityResolver(client=peeringdb_client), +]) +``` + +## 测试覆盖 + +相关测试： + +- [test_location_pipeline.py](/home/ray/dev/linkong/planet/backend/tests/test_location_pipeline.py) +- [test_bgp_collector_locations.py](/home/ray/dev/linkong/planet/backend/tests/test_bgp_collector_locations.py) +- [test_visualization_compute_centers.py](/home/ray/dev/linkong/planet/backend/tests/test_visualization_compute_centers.py) + +测试重点包括 resolver 可插拔性、注册表 alias 约束、BGP collector 兼容字典、算力中心公共 API 兼容、不可渲染位置进入 `unresolved`。 diff --git a/docs/technical/zh/location-pipeline-user.md b/docs/technical/zh/location-pipeline-user.md new file mode 100644 index 00000000..5ff948f5 --- /dev/null +++ b/docs/technical/zh/location-pipeline-user.md @@ -0,0 +1,127 @@ +# Earth 位置候选采集使用手册 + +位置候选采集用于给 Earth 上的算力中心和 BGP 观测站补齐或核验经纬度。它不会要求用户手工输入坐标，而是把源数据、开放组织注册 API 和在线地理编码结果整理成候选列表，供用户预览和后续认领。 + +## 适用对象 + +当前支持： + +- 算力中心：TOP500 超算、Epoch AI GPU 集群。 +- BGP 观测站：RIPE RIS `rrcXX` collector。 + +BGP 事件的位置默认继承所属 collector。事件本身暂不提供单独按钮；后续 ASN 设施、Prefix 地理位置或 PeeringDB 算法接入后会继续走同一条管线。 + +## 用户能看到什么 + +在 Earth 上点击算力中心或 BGP 观测站后，详情卡会展示位置相关字段： + +| 字段 | 含义 | +| --- | --- | +| 位置精度 | `精确坐标`、`站点级位置`、`城市级位置` 或 `位置未确认` | +| 位置来源 | 源数据坐标、ROR 组织注册 API、Nominatim 在线搜索，或已存储的 BGP collector 维表位置 | +| 位置置信度 | 后端 resolver 给出的相对置信度百分比 | +| 核验状态 | 已确认、估算位置或在线检索结果待确认 | +| 解析依据 | 为什么选择这个位置，例如匹配了哪个站点或城市 | +| 匹配的位置名称 | 开放来源、在线结果或已存储 collector 位置中的规范名称 | +| 位置核验时间 | 已确认位置的核验日期，在线候选通常为空 | + +算力中心 GeoJSON 不再渲染国家质心、未知位置或 `[0, 0]` 占位点。无法达到城市级精度的数据会进入接口的 `unresolved` 列表，并在图层开关左上角显示待定位数量。点击这个通知气泡会打开待定位列表。 + +地图上带 `?` 的算力中心不是 `unresolved`。它们已经有坐标，只是 `needs_confirmation=true` 或来自在线地理编码，仍需人工核验。真正 `unresolved` 的记录没有可信经纬度，因此不会出现在地球上。 + +## 自动采集候选 + +1. 打开 `http://localhost:3000/earth`。 +2. 打开 `算力中心` 或 `BGP 观测` 图层。 +3. 点击目标对象打开详情卡。 +4. 点击 `自动采集坐标候选` 或 `重新自动采集坐标`。 +5. 等待详情卡列出最多 5 个候选位置。 +6. 点击候选行里的 `预览`，Earth 会飞到该候选经纬度附近。 + +候选列表会显示： + +- 候选名称。 +- 精度：精确、站点或城市。 +- 来源 resolver。 +- 置信度。 +- 经纬度。 + +点击候选行里的 `保存` 会把所选候选写入算力中心位置维表。保存成功后，算力中心图层会刷新；如果该记录原本在待定位列表中，待定位数量也会减少。 + +## 待定位列表和一键采用 + +算力中心图层按钮左上角的通知气泡显示当前 `unresolved` 数量。点击后会在图层面板右侧打开固定列表： + +1. 列表只包含没有可信经纬度的算力中心。 +2. 单条 `采集` 会调用候选接口，并展示最多 5 个候选供预览和保存。 +3. 顶部 `一键采用` 会从上到下逐条采集候选，选择置信度最高且有有效经纬度的候选保存。 +4. 成功保存一条后，该行会立即从列表中移除，下面的序号自动上移，通知气泡数量同步减少。 +5. 批量结束后，前端会刷新算力中心图层，确保 UI 和后端真实状态一致。 + +如果某条记录没有任何可保存候选，系统不会用国家中心点、厂商总部或硬编码 hint 伪造位置。该记录会留在列表中，并显示后端返回的失败原因和已尝试查询，等待人工补充更可靠的地址或坐标证据。 + +## 后端接口 + +前端按钮调用的接口如下： + +```http +POST /api/v1/visualization/compute-centers/{source_id}/collect-location +POST /api/v1/visualization/compute-centers/{source_id}/location +POST /api/v1/bgp/collectors/{collector_id}/collect-location +``` + +两个 `collect-location` 接口返回相同结构： + +```json +{ + "success": true, + "candidates": [], + "best_candidate": {}, + "attempted_queries": [], + "context": {} +} +``` + +当没有候选达到城市级精度时，`success` 为 `false`，响应会包含 `failure_reason` 和已尝试的查询文本，便于判断是源数据字段不足、开放来源缺项，还是在线地理编码没有命中。 + +## 数据维护建议 + +算力中心和 BGP 观测站都不再维护本地候选注册表。算力中心的人工确认位置保存在 `compute_center_locations` 数据库维表中，唯一键是 `(source, source_id)`；BGP 观测站的当前位置保存在 `bgp_collector_locations` 数据库维表中，旧 RIPE RIS 城市级坐标只作为初始化 seed 写入，默认仍需人工核验。 + +维护算力中心时优先补齐： + +- `source` / `source_id`：例如 `top500` + `top500_50`。 +- `name` / `operator` / `site`。 +- `city` / `country`。 +- `latitude` / `longitude`。 +- `precision`：`precise`、`site` 或 `city`。 +- `confidence`：0 到 1 的置信度。 +- `location_source` / `source_url` / `source_note` / `raw_payload`：证据来源。 +- `needs_confirmation` / `verification_status` / `verified_at`：人工核验状态和日期。 + +维护 BGP 观测站时优先补齐： + +- `collector_id`：例如 `rrc12`。 +- `site` / `operator`：站点和运营方。 +- `city` / `country` / `region`。 +- `latitude` / `longitude`。 +- `precision`：`precise`、`site` 或 `city`。 +- `confidence`：0 到 1 的置信度。 +- `source` / `source_url` / `raw_payload`：证据来源。 +- `verification_status` / `verified_at`：人工核验状态和日期。 + +如果只是知道城市，不知道设施坐标，应使用城市级精度，不要填一个看似精确但无法核验的点位。 + +## 常见问题 + +### 为什么有些算力中心不显示在 Earth 上 + +Earth 只渲染达到城市级或更高精度的坐标。源数据没有坐标、已验证位置没有命中、在线搜索也没有城市级结果时，记录会进入 `unresolved`，避免在地图上出现误导性的国家中心点或 `[0, 0]`。 + +### 为什么在线搜索结果显示“待确认” + +Nominatim/OpenStreetMap 结果来自在线地理编码，可能匹配到同名城市、机构或园区。它可以用于快速定位和预览，但在写入已验证位置前应人工确认。 + +### 为什么 BGP 事件没有全部落到 Amsterdam + +旧逻辑中，事件可能因为 `operator="RIPE NCC"` 这种通用字段误匹配到 `rrc00`。当前 BGP 事件继承只按所属 collector 在 DB-backed cache 中严格查找，不再用 registry 模糊匹配。 diff --git a/docs/technical/zh/manual.md b/docs/technical/zh/manual.md new file mode 100644 index 00000000..0677dd7d --- /dev/null +++ b/docs/technical/zh/manual.md @@ -0,0 +1,672 @@ +# Planet 使用手册 + +这份手册面向日常使用、演示、开发联调和本地运维。它覆盖四个核心入口： + +- `planet.sh`：本地启动、停止、重启、健康检查和日志入口 +- Earth：公开 3D 地球态势页面 +- 控制台：登录后的管理后台 +- Docs：后端 Gatekeeper 受控的文档站，基础使用文档公开，开发/运维文档按权限组开放 + +快速启动路径见 [快速开始](/home/ray/dev/linkong/planet/docs/technical/zh/quickstart.md)。 + +## 入口总览 + +默认启动后，常用地址如下： + +| 名称 | 地址 | 是否需要登录 | 说明 | +| --- | --- | --- | --- | +| Earth | `http://localhost:3000/earth` | 否 | 3D 地球、图层、BGP、卫星、海缆、新闻态势 | +| Docs | `http://localhost:3000/docs` | 部分需要 | 使用手册公开；开发、后端、运维文档按 Gatekeeper 权限组开放 | +| 控制台 | `http://localhost:3000/admin` | 是 | 数据、配置、告警、日志和专题观测 | +| AI Playground | `http://localhost:3000/playground` | 是 | AI Provider 状态和调试 | +| 后端 API 文档 | `http://localhost:8000/docs` | 视接口而定 | FastAPI / OpenAPI 文档 | + +## planet.sh + +`planet.sh` 是本地开发和演示的主控脚本。优先使用它管理服务，而不是手动分别启动前端、后端、数据库和 AI Provider。 + +### 启动 + +```bash +./planet.sh start +``` + +默认行为： + +- 启动 PostgreSQL 和 Redis +- 启动 AI Provider +- 启动后端 API +- 启动前端 Vite dev server +- 输出 Earth、控制台、Playground 和后端 API 文档入口 + +可指定端口： + +```bash +./planet.sh start -b 8001 -f 3001 -a 8101 +``` + +参数含义： + +| 参数 | 含义 | +| --- | --- | +| `-b ` | 后端端口 | +| `-f ` | 前端端口 | +| `-a ` | AI Provider 端口 | +| `--allow-lan` | 允许局域网访问 | +| `--verbose` | 在执行过程中显示更多命令输出 | + +### AI Provider 环境变量和构建 + +AI Provider 的运行期配置可以放在两处： + +| 位置 | 适合内容 | 说明 | +| --- | --- | --- | +| `aiprovider/.env` | 团队约定的本地默认配置 | Docker Compose 会作为 `env_file` 读取 | +| `~/.zshrc` | 个人机器上的 provider、模型、密钥和代理变量 | `planet.sh` 启动时会读取常见的 `AI_*`、`SERVICE_*`、`PYTHON_IMAGE`、`UV_IMAGE`、代理变量 | + +推荐写法： + +```bash +export AI_PROVIDER=minimax +export AI_PROVIDER_API=anthropic-messages +export AI_BASE_URL=https://api.example.com/anthropic +export AI_API_KEY=sk-change-me +export AI_MODEL=MiniMax-M2.7 +export AI_PROVIDER_SERVICE_TOKEN=change_me +``` + +默认情况下，`planet.sh` 只静态解析 `~/.zshrc` 中简单的 `export KEY=value` 或 `KEY=value` 行，避免 shell 主题、插件或交互初始化拖慢启动。如果变量依赖复杂 shell 展开，可以显式启用 source 模式： + +```bash +PLANET_LOAD_ZSHRC_ENV=source ./planet.sh start -a +``` + +如需完全忽略 `~/.zshrc`： + +```bash +PLANET_LOAD_ZSHRC_ENV=0 ./planet.sh start -a +``` + +AI Provider 镜像只在代码、Dockerfile、Compose 配置或相关 Python 依赖变化时重建。修改 `aiprovider/.env` 或 `~/.zshrc` 中的模型、密钥、Base URL 不会触发镜像重建；重启 AI Provider 即可让容器读取新配置： + +```bash +./planet.sh restart -a +``` + +构建较慢时，优先判断当前卡在哪一层： + +| 现象 | 常见原因 | 处理方式 | +| --- | --- | --- | +| `transferring context` 很大 | Docker build context 包含前端资源、PDF、数据目录等无关文件 | 当前仓库通过 `.dockerignore` 只发送 AI Provider 必需文件 | +| `uv sync` 下载依赖较慢 | 首次构建或缓存为空，网络访问 Python 包较慢 | 等待首次构建完成；后续会复用 BuildKit 的 uv 下载缓存 | +| 改密钥后仍显示旧配置 | 容器尚未重启 | 执行 `./planet.sh restart -a` | + +### 停止 + +```bash +./planet.sh stop +``` + +会停止： + +- 后端 +- AI Provider +- 前端 +- PostgreSQL +- Redis + +### 重启 + +全量重启： + +```bash +./planet.sh restart +``` + +按模块重启： + +```bash +./planet.sh restart -b +./planet.sh restart -f +./planet.sh restart -a +./planet.sh restart -d +``` + +| 参数 | 作用 | +| --- | --- | +| `-b` | 只重启后端 | +| `-f` | 只重启前端 | +| `-a` | 只重启 AI Provider | +| `-d` | 只重启数据库 | + +按模块重启适合日常开发，能避免无关服务被打断。 + +### 创建用户 + +```bash +./planet.sh createuser +``` + +用于首次进入控制台前创建登录账号。脚本会交互式提示用户名、密码和角色。 + +### 健康检查 + +```bash +./planet.sh health +``` + +会检查： + +- `planet_*` 容器状态 +- 后端 `/health` +- AI Provider `/health` +- 前端页面可达性 + +如果某项显示 offline，优先查看对应日志。 + +### 日志 + +最近日志： + +```bash +./planet.sh log +``` + +持续跟随日志： + +```bash +./planet.sh log -f +./planet.sh log -b +./planet.sh log -a +``` + +| 参数 | 日志来源 | +| --- | --- | +| `-f` / `--frontend` | `/tmp/planet_frontend.log` | +| `-b` / `--backend` | `/tmp/planet_backend.log` | +| `-a` / `--ai-provider` | `planet_aiprovider` 容器日志 | + +### 局域网访问 + +```bash +./planet.sh start --allow-lan +``` + +适合： + +- WSL 中启动，Windows 浏览器访问 +- 手机或平板演示 Earth +- 局域网其他机器访问同一个开发实例 + +`--allow-lan` 只负责让前端和后端监听 `0.0.0.0`。如果服务运行在 WSL 中，Windows 本机通常可以通过 `localhost` 访问，但手机或其他电脑访问 `http://:3000` 还依赖 Windows 端口转发和防火墙放行。 + +推荐按顺序判断： + +```bash +# 在 WSL 或运行 Planet 的 shell 中 +curl http://localhost:3000 +curl http://localhost:8000/health +ss -ltnp | grep -E ':3000|:8000' +``` + +如果这里能看到 `0.0.0.0:3000` 和 `0.0.0.0:8000`，但局域网 IP 访问失败，请在管理员 PowerShell 中配置： + +```powershell +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 + +New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 +New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 +``` + +## Earth + +Earth 是公开的 3D 态势页面，入口： + +```text +http://localhost:3000/earth +``` + +它是独立前端，实际页面位于： + +- `frontend/public/earth/index.html` +- `frontend/public/earth/js/` +- `frontend/public/earth/css/` + +React 路由中的 `/earth` 只是用 iframe 承载它。 + +### 主要用途 + +Earth 用于在一个地球视图中观察： + +- BGP 事件、异常和观测态势 +- 卫星和轨迹 +- 海缆与登陆点 +- 算力中心 +- 国界线、经纬线、高清材质、云图、地形 +- 新闻直播和态势新闻 +- 搜索和聚焦对象详情 + +### 图层控制 + +右侧图层面板用于打开或关闭可视图层。 + +常见图层包括： + +- 经纬线 +- 国界线 +- 高清材质 +- 大气云图 +- 海缆 +- 算力中心 +- BGP 观测 +- 卫星 +- AIS 船只 +- 轨迹 +- 地形 + +部分图层存在依赖关系： + +- 地形依赖高清材质 +- 轨迹依赖卫星 +- 高清材质关闭时，地球会显示基座地图和边缘识别效果 + +### 图例 + +左下角图例会跟随当前聚焦或启用的图层切换。 + +当前已覆盖： + +- 海缆 +- 卫星 +- 国界线 +- 算力中心 +- BGP +- AIS 船只 + +AIS 船只图例按船型显示颜色： + +- 货轮 +- 油轮 +- 客船 +- 渔船 +- 军舰 +- 停泊/低速 +- 其他船只 + +船只图例中的三角形对应地图上的航行船只标记，圆点对应停泊或低速状态。 + +### 搜索 + +Earth 搜索支持查找当前地球对象，例如： + +- 海缆 +- 登陆点 +- 卫星 +- 算力中心 +- BGP 事件 +- BGP 观测站 + +搜索结果可以用于快速定位对象，并打开对应详情。 + +### 位置候选采集 + +算力中心和 BGP 观测站详情卡支持自动采集坐标候选。点击对象后，使用详情卡中的 `自动采集坐标候选` 或 `重新自动采集坐标` 按钮，后端会从源坐标、开放组织注册 API 和在线地理编码中整理候选位置。BGP 观测站的已存储位置只用于补齐查询上下文，不会作为候选直接返回。 + +候选可以直接在 Earth 上预览。算力中心候选点击 `保存` 后会写入 `compute_center_locations` 维表，并立即刷新图层。算力中心图层左上角的通知气泡显示无法渲染的待定位数量；点击后可查看列表，单条采集候选，或用 `一键采用` 从上到下保存最高置信候选。没有可用候选的记录会留在列表中，不会被国家中心点或硬编码 hint 伪造位置。详细流程见 [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md)。 + +### 设置 + +设置面板包含： + +- 旋转模式 / 巡航模式 +- 巡航模块：BGP、新闻 +- 卫星显示风格：自身发光、真实地表覆盖 +- 日夜模式 +- 面板显示开关 +- 地球默认大小 +- 地形透明度 +- 重置设置 + +这些设置会保存在浏览器本地存储中。换浏览器或清理站点数据后会恢复默认值。 + +### 视角控制 + +Earth 支持鼠标、触控板和触屏操作。 + +常用控制方式： + +| 操作 | 作用 | +| --- | --- | +| 鼠标左键拖动 | 旋转地球 | +| 手指单指拖动 | 在触屏设备上旋转地球 | +| 鼠标滚轮 | 放大或缩小视角 | +| 双指捏合 | 在触屏设备上放大或缩小视角 | +| 缩放按钮 | 按固定步长调整缩放 | +| 点击缩放百分比 | 重置到默认缩放 | + +缩放时，顶部胶囊会短暂显示当前缩放比例，例如 `缩放 180%`。这个提示只表示当前视角缩放，不代表数据加载进度；如果页面正在加载数据，加载提示优先显示，缩放提示不会打断加载状态。 + +拖动灵敏度会根据当前缩放自动调整。默认视角附近保持常规旋转速度；放大后拖动会逐步变细，适合检查某个区域、船只、卫星或 BGP 事件；缩小后拖动会略快，方便快速浏览全球态势。 + +### 巡航模式 + +巡航模式会让 Earth 自动轮播聚焦目标。 + +当前巡航模块包括： + +- BGP +- 新闻 + +适合演示、监控大屏或无人值守展示。 + +### 移动端 + +Earth 有移动端抽屉布局。小屏下： + +- 图层控制进入移动抽屉 +- 搜索、设置、详情会使用移动端面板 +- 主要交互仍围绕地球对象点击、搜索和图层开关 + +### 常见问题 + +#### Earth 打不开 + +先检查前端是否在线： + +```bash +./planet.sh health +./planet.sh log -f +``` + +如果前端端口不是 `3000`，使用启动时输出的实际端口。 + +#### 图层没有数据 + +检查后端和数据源： + +```bash +./planet.sh health +./planet.sh log -b +``` + +然后进入控制台查看： + +- `/datasources` +- `/data` +- `/bgp` + +#### 卫星、BGP 或海缆加载慢 + +这些图层可能依赖后端接口、外部数据源或首次加载任务。先等待启动任务完成，再查看日志和控制台数据源状态。 + +## 控制台 + +控制台入口： + +```text +http://localhost:3000/admin +``` + +控制台需要登录。首次使用先创建用户： + +```bash +./planet.sh createuser +``` + +### 页面结构 + +控制台使用 React + Ant Design，左侧菜单按工作域组织。 + +常见入口： + +| 页面 | 路由 | 用途 | +| --- | --- | --- | +| 仪表盘 | `/admin` | 系统概览 | +| Earth | `/earth` | 打开公开 Earth 页面 | +| 数据源 | `/datasources` | 查看数据源和触发采集 | +| 采集数据 | `/data` | 查看采集后的数据 | +| BGP 观测 | `/bgp` | 查看 BGP 专题数据 | +| 系统告警 | `/alerts/system` | 系统级告警 | +| BGP 告警 | `/alerts/bgp` | BGP 相关告警 | +| 态势告警 | `/alerts/situational` | 态势研判告警 | +| AI Playground | `/playground` | AI Provider 调试 | +| 系统日志 | `/logs` | 查看系统日志，通常仅 super admin 可见 | +| 用户管理 | `/users` | 管理用户 | +| 系统配置 | `/settings` | 系统配置和电视直播源等设置 | + +### 数据源 + +`/datasources` 用于查看采集来源和触发采集。当前页面是“数据源目录”，会把内置数据源和自定义数据源放在同一张列表里展示。 + +常见操作： + +- 查看数据源状态 +- 触发采集 +- 查看最近采集任务 +- 打开详情抽屉查看 endpoint、请求头、基础配置和是否为内置数据源 + +如果 Earth 上某类对象缺失，通常先到这里确认数据源是否可用。 + +数据源列表中的名称点击后只打开信息抽屉，不再承担编辑入口。接口地址、凭证、请求头和自定义数据源配置统一到 `/settings` 的“采集器设置”里维护。 + +当有采集任务正在运行时，总体进度下方会出现 `采集中 N` 标签。这个标签和其他状态标签放在同一排，但带有可点击样式；点击后会弹出当前采集中任务列表，显示每个任务的阶段、进度和处理数量。 + +### 采集数据 + +`/data` 用于查看采集后的数据表。 + +适合排查： + +- 数据是否已经进入系统 +- 数据更新时间是否符合预期 +- 某个数据源是否产出了有效记录 + +### BGP 观测 + +`/bgp` 是 BGP 专题页面。 + +它和 Earth 的 BGP 图层互补： + +- Earth 强调空间态势和可视聚焦 +- 控制台 BGP 页面强调列表、状态、详情和研判 + +### 告警 + +告警入口包括： + +- `/alerts/system` +- `/alerts/bgp` +- `/alerts/situational` + +用于查看系统、网络和态势相关告警。 + +### 系统配置 + +`/settings` 用于管理系统级配置。 + +当前常见用途包括： + +- 系统设置 +- 电视直播源配置 +- 采集器设置 +- 外部集成和 AI Provider 配置 + +具体可用配置取决于当前登录用户权限。 + +#### 采集器设置 + +`/settings?tab=collector_credentials` 当前显示为“采集器设置”。这里统一维护所有采集器的连接配置，而不是只维护凭证。 + +使用方式： + +1. 在下拉框选择采集器。 +2. 查看状态标签： + - `无需凭证` / `需要凭证` + - 所属模块 + - `启用` / `禁用` + - `未检查` / `可用` / `不可用` +3. 点击下拉框右侧的插头图标执行健康检查。 +4. 如果检查通过，状态会变为 `可用`。 +5. 修改 endpoint、请求头、超时或重试次数后保存。 + +对于免费且不需要凭证的采集器，连接检查会直接请求对应 endpoint。对于需要凭证的采集器，连接检查会走对应凭证链路；如果凭证或 endpoint 相比上次验证成功时发生变化，需要重新点击连接。 + +系统判断“已连接”的条件是： + +- 当前配置已经成功采集过数据；或 +- 当前配置已经点击过连接按钮并验证成功。 + +#### BarentsWatch AIS 凭证 + +`BarentsWatch AIS` 是需要凭证的内置采集器。选择该采集器后，凭证区域会显示在基础配置上方。 + +配置项： + +- `Client ID` +- `Client Secret` +- `Endpoint` + +如果已经配置过 secret，输入框会显示脱敏预览。保存时如果保持这个脱敏预览不变，系统会保留原 secret；只有输入新的 secret 才会替换。 + +BarentsWatch AIS 支持从以下位置读取凭证： + +1. 控制台采集器设置中保存的凭证。 +2. 后端环境变量： + - `BARENTSWATCH_CLIENT_ID` + - `BARENTSWATCH_CLIENT_SECRET` + - 兼容历史拼写：`BARRENTSWATCH_CLIENT_ID`、`BARRENTSWATCH_CLIENT_SECRET` +3. `~/.zshrc` 中的同名 `export`。 + +如果连接失败，页面会弹出凭证获取教程。教程支持： + +- 查看默认教程。 +- 点击“教程不好用”让 AI Provider 根据默认 prompt 重新生成教程。 +- 点击“重置”恢复默认教程。 + +默认教程以 BarentsWatch 官方 tutorial 为准，并提醒 Live AIS 应选择 `AIS - API`，不是普通 `BarentsWatch - API`。 + +### 系统日志 + +`/logs` 用于查看系统日志。若菜单中不可见，通常是当前用户角色没有权限。 + +排查问题时常用组合： + +```bash +./planet.sh health +./planet.sh log +``` + +再进入 `/logs` 查看更结构化的运行信息。 + +## Docs + +文档站入口： + +```text +http://localhost:3000/docs +``` + +Docs 正文由后端 API 按权限读取，不再把全部 Markdown 直接打进前端构建产物。当前文档源文件仍位于： + +```text +docs/technical/zh/*.md +docs/technical/en/*.md +``` + +未登录访客默认只能看到 `public` 文档，例如首页、快速开始和使用手册。登录用户如果被分配 Gatekeeper 权限组，可以看到更多技术文档： + +- `docs_user`：用户操作类文档。 +- `docs_developer`：Earth、前端、后端、采集器和 AI Provider 等开发文档。 +- `docs_admin`：服务控制、运维、环境变量和敏感操作文档。 + +`admin` 默认拥有管理文档权限，`super_admin` 拥有全部 Docs 权限。Gatekeeper 权限组在控制台“用户管理”中配置。 + +Docs 支持： + +- 分类导航 +- Markdown 渲染 +- 表格和代码块 +- 文档内目录 +- 对当前可见文档搜索 +- technical 文档之间的内部链接跳转 + +如果新增 technical 文档，应同步检查： + +- 是否有清晰的一级标题 +- 是否需要加入后端 Docs metadata 的人工分类和排序 +- 应归入 `public`、`docs_user`、`docs_developer` 还是 `docs_admin` + +## 开发命令约定 + +前端命令必须使用 Bun： + +```bash +cd frontend +bun install +bun run dev +bun run build +``` + +不要使用 `npm run ...`。项目在 WSL / Windows 混合环境中优先依赖 Bun，避免 Node/npm 路径差异带来的兼容问题。 + +验证前端构建： + +```bash +source ~/.zshrc && bun run build +``` + +## 故障排查顺序 + +遇到问题时，建议按这个顺序排查： + +1. 看服务状态： + +```bash +./planet.sh health +``` + +2. 看最近日志： + +```bash +./planet.sh log +``` + +3. 按模块查看日志： + +```bash +./planet.sh log -f +./planet.sh log -b +./planet.sh log -a +``` + +4. 只重启有问题的模块： + +```bash +./planet.sh restart -f +./planet.sh restart -b +./planet.sh restart -a +``` + +5. 如果数据库或缓存异常，再重启数据库： + +```bash +./planet.sh restart -d +``` + +6. 仍无法恢复时，执行全量重启： + +```bash +./planet.sh restart +``` + +## 相关文档 + +- [快速开始](/home/ray/dev/linkong/planet/docs/technical/zh/quickstart.md) +- [控制台前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-admin-frontend-context.md) +- [Earth 前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/earth-frontend-context.md) +- [Earth 图层样式属性索引](/home/ray/dev/linkong/planet/docs/technical/zh/earth-layer-style-reference.md) +- [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md) +- [系统服务控制](/home/ray/dev/linkong/planet/docs/technical/zh/backend-system-service-control.md) +- [数据采集系统](/home/ray/dev/linkong/planet/docs/technical/zh/backend-collectors.md) +- [数据源、采集器设置与连接验证](/home/ray/dev/linkong/planet/docs/technical/zh/datasource-collector-settings-connectivity.md) diff --git a/docs/technical/ops-docker-compose-buildx-upgrade.md b/docs/technical/zh/ops-docker-compose-buildx-upgrade.md similarity index 100% rename from docs/technical/ops-docker-compose-buildx-upgrade.md rename to docs/technical/zh/ops-docker-compose-buildx-upgrade.md diff --git a/docs/technical/zh/ops-planet-sh-startup.md b/docs/technical/zh/ops-planet-sh-startup.md new file mode 100644 index 00000000..e0f5bc75 --- /dev/null +++ b/docs/technical/zh/ops-planet-sh-startup.md @@ -0,0 +1,218 @@ +# planet.sh 启动性能优化 + +## 背景 + +`planet.sh` 管理所有服务的启动/停止/重启。原有实现存在以下问题： + +1. AI Provider 每次都重新构建（即使代码未变） +2. 杀端口速度极慢（最长等 45 秒） +3. 端口绑定检测用 Python 子进程（每次 ~300ms） +4. 无参 `restart` 与 `restart -b` 行为不一致 + +## 问题一：AI Provider 每次重建 + +### 根因 + +构建戳文件存放在 `/tmp/`，WSL/Linux 重启后 `/tmp` 被清空，导致三个条件中的"戳文件非空"这一条始终不满足，进而判定需要重建： + +```bash +# 三个条件必须同时成立才跳过重建 +image_exists AND stamp_non_empty AND fingerprint_match +``` + +### 修复 + +将戳文件路径从 `/tmp/` 改到持久路径： + +```bash +AI_PROVIDER_BUILD_STAMP_FILE="$HOME/.cache/planet/aiprovider_build.sha256" +``` + +写入时确保目录存在： + +```bash +write_ai_provider_build_stamp() { + mkdir -p "$(dirname "$AI_PROVIDER_BUILD_STAMP_FILE")" + compute_ai_provider_build_fingerprint > "$AI_PROVIDER_BUILD_STAMP_FILE" +} +``` + +### fingerprint 计算提速 + +原实现对整个 `aiprovider/` 打 tar 包再算 SHA，大目录下耗时可达数秒。改为 `find + stat`（只读文件元信息，不读内容）： + +```bash +compute_ai_provider_build_fingerprint() { + find aiprovider \ + -type f \ + ! -path '*/__pycache__/*' \ + ! -name '.env' \ + ! -name '.env.*' \ + ! -name '*.pyc' \ + ! -name '*.pyo' \ + | LC_ALL=C sort \ + | xargs -r stat --format="%Y %s %n" 2>/dev/null + sha256sum docker-compose.yml docker-compose.simple.yml 2>/dev/null + python3 "$SCRIPT_DIR/scripts/compute_aiprovider_dependency_fingerprint.py" 2>/dev/null +} +``` + +速度提升约 10 倍（大量小文件场景），误报率相同（mtime+size 变化 ≡ 文件被修改）。 + +`.env` 和 `.env.*` 被排除在 fingerprint 外。它们属于运行期配置，不应该因为修改模型、密钥或 Base URL 触发镜像重建。 + +### Docker build context 收敛 + +AI Provider 镜像只需要根目录的 `pyproject.toml`、`uv.lock` 和 `aiprovider/` 代码。仓库中还包含前端静态大图、PDF、历史数据和 Unreal 资料，如果 build context 使用整个仓库，`transferring context` 会浪费大量时间。 + +当前通过根目录 `.dockerignore` 收敛上下文： + +```dockerignore +** + +!pyproject.toml +!uv.lock +!aiprovider/ +!aiprovider/** + +aiprovider/.env +aiprovider/.env.* +!aiprovider/.env.example +``` + +Dockerfile 也从全仓复制改为只复制 AI Provider 代码： + +```dockerfile +COPY pyproject.toml uv.lock /app/ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + +COPY aiprovider /app/aiprovider +``` + +`uv sync` 使用 BuildKit cache mount 后，首次构建仍可能受网络影响；后续构建会复用 `/root/.cache/uv`，依赖下载不再重复从零开始。 + +### 运行期配置来源 + +`planet.sh` 启动 AI Provider 前会生成临时 env-file，并把它传给 Compose 或手动 `docker run` fallback。配置优先来自： + +1. `aiprovider/.env` +2. `~/.zshrc` 中简单的 `export AI_...=...` 或 `AI_...=...` 行 + +默认解析是静态的，只覆盖 AI Provider、镜像、代理相关变量，避免执行交互 shell 初始化。如果确实需要复杂 shell 展开，可以显式启用： + +```bash +PLANET_LOAD_ZSHRC_ENV=source ./planet.sh start -a +``` + +如果排查时需要忽略个人 shell 配置： + +```bash +PLANET_LOAD_ZSHRC_ENV=0 ./planet.sh start -a +``` + +### 跳过重建的原理 + +fingerprint 一致时不执行 `docker compose build`，而是： + +```bash +docker start planet_aiprovider # 启动已存在的容器，几秒内完成 +``` + +`docker stop` 停容器，不删镜像；`cleanup_exit_containers` 删已退出容器，不删镜像。下次 `docker start` 会从现有镜像直接创建并启动容器。 + +## 问题二：杀端口速度慢 + +### 原因 + +`wait_for_port_release` 默认最多等 45 秒（15 次 × 3 秒）。 + +### 修复 + +将后台进程清理场景的超时缩短至 3 秒（TERM→1.5s→KILL→1.5s）： + +```bash +PORT_RELEASE_ATTEMPTS=15 +PORT_RELEASE_INTERVAL=0.2 # 每次等 0.2s，总计 3s + +# cleanup_backend_processes / kill_port_if_requested +wait_for_port_release "$port" 15 0.2 +``` + +`wait_for_port_release` 增加可选参数，允许不同场景使用不同超时： + +```bash +wait_for_port_release() { + local port="$1" + local max_attempts="${2:-$PORT_RELEASE_ATTEMPTS}" + local interval="${3:-$PORT_RELEASE_INTERVAL}" + ... +} +``` + +当前启动前端时还有一层预清理重试： + +- `PORT_PRESTART_RETRIES`：默认 3 次。 +- `PORT_PRESTART_RETRY_INTERVAL`：默认 2 秒。 + +`kill_port_if_requested()` 只在当前环境能找到监听 PID 时主动杀进程；如果没有 PID 但端口暂时不可绑定，它会记录诊断并把最终确认交给服务启动流程。`start_frontend_with_retry()` 也只在发现监听 PID 时进入预清理重试，避免在宿主机或外部 network namespace 尚未释放端口时做无意义的“空杀重试”。这意味着第一次重启时看到“未发现监听进程但端口仍不可绑定”通常是外部环境仍在释放端口；脚本不会再把这种情况当成立刻失败的本地进程清理问题。 + +## 问题三：端口检测用 Python + +### 原因 + +`can_bind_port` 用 `python3 -c "import socket..."` 检测端口，每次调用约 300ms。 + +### 修复 + +优先使用系统工具（~10ms），Python 作为兜底： + +```bash +can_bind_port() { + local port="$1" + if command -v ss >/dev/null 2>&1; then + ! ss -tlnH 2>/dev/null | awk '{print $4}' | grep -qE ":${port}$" + return + fi + if command -v lsof >/dev/null 2>&1; then + [ -z "$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null)" ] + return + fi + python3 - "$port" <<'PY' +import sys, socket +p = int(sys.argv[1]) +s = socket.socket() +s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +try: + s.bind(("", p)); s.close(); sys.exit(0) +except OSError: + sys.exit(1) +PY +} +``` + +## 问题四：restart 行为不一致 + +### 现象 + +- `restart -b`：停全部服务 → 检查 AI Provider fingerprint → 按需重建 → 启动 +- `restart`（无参）：停全部服务 → AI Provider 总是判定需要重建（因戳文件在 /tmp） + +### 修复 + +修复戳文件路径后，无参 `restart` 同样使用 `stop + start`，fingerprint 检查正常生效，行为与 `restart -b` 完全一致。无需额外代码变更。 + +## 其他：移除不必要的 sleep + +启动链路中两处 `sleep 3` 在实际已有健康检查覆盖的情况下多余，已移除： + +- `start_backend_service`：数据库健康检查通过后的 `sleep 3` +- `restart_database_service`：重启后的等待 `sleep 3` + +## 相关文件 + +- `planet.sh` — 全量修改 +- `.dockerignore` — 收敛 AI Provider Docker build context +- `aiprovider/Dockerfile` — 只复制 AI Provider 代码，并为 `uv sync` 启用 BuildKit cache mount +- `docker-compose.yml` / `docker-compose.simple.yml` — 读取 `planet.sh` 生成的运行期 env-file +- `scripts/compute_aiprovider_dependency_fingerprint.py` — 依赖 fingerprint（未改动） diff --git a/docs/technical/zh/quickstart.md b/docs/technical/zh/quickstart.md new file mode 100644 index 00000000..9590a8fd --- /dev/null +++ b/docs/technical/zh/quickstart.md @@ -0,0 +1,231 @@ +# 快速开始 + +这份快速开始面向第一次启动 Planet 的开发者或演示操作者。目标是用最短路径把服务跑起来，并知道应该打开哪些入口。 + +## 前置条件 + +推荐在 WSL / Linux shell 中运行。 + +需要具备： + +- Docker / Docker Compose 可用 +- 当前 shell 能访问 `uv` 和 `bun` +- 仓库已 clone 到本机 + +如果是新机器，优先执行仓库自带初始化脚本： + +```bash +./scripts/bootstrap-dev.sh +``` + +这个脚本会检查并同步常用依赖，并在缺少时生成： + +- `backend/.env` +- `aiprovider/.env` +- `frontend/.env.local` + +AI Provider 的个人配置也可以放在 `~/.zshrc`。`planet.sh` 会读取简单的 `export AI_...=...` 或 `AI_...=...` 行，并在启动 AI Provider 时传给容器。修改模型、密钥或 Base URL 后，通常只需要重启 AI Provider： + +```bash +./planet.sh restart -a +``` + +AISStream、BarentsWatch 等采集器凭证也可以先写在 `~/.zshrc` 里供连接验证读取，例如： + +```bash +export AISSTREAM_API_KEY="..." +export BARENTSWATCH_CLIENT_ID="..." +export BARENTSWATCH_CLIENT_SECRET="..." +``` + +正式采集更推荐在控制台 `设置 -> 采集器设置` 保存凭证，尤其是 AISStream 这类长连接 WebSocket collector。这样连接验证、后端采集任务和 Earth 实时船只聚合会使用同一份配置。 + +## 1. 启动服务 + +在仓库根目录执行： + +```bash +./planet.sh start +``` + +启动完成后，常用入口是： + +| 入口 | 默认地址 | 用途 | +| --- | --- | --- | +| Earth | `http://localhost:3000/earth` | 公开 3D Earth 可视化页面 | +| 控制台 | `http://localhost:3000/admin` | 登录后的管理后台 | +| 文档站 | `http://localhost:3000/docs` | 使用手册公开；开发/运维文档按 Gatekeeper 权限组开放 | +| AI Playground | `http://localhost:3000/playground` | 登录后的 AI 调试入口 | +| 后端 API 文档 | `http://localhost:8000/docs` | FastAPI / OpenAPI 接口文档 | + +如果默认端口被占用，可以指定端口： + +```bash +./planet.sh start -f 3001 -b 8001 -a 8101 +``` + +## 2. 创建登录用户 + +控制台需要登录。首次使用可以执行： + +```bash +./planet.sh createuser +``` + +按提示输入用户名、密码和角色。 + +如果需要阅读开发或运维文档，用 `super_admin` 登录控制台后，在“用户管理”里给目标用户分配 Gatekeeper 权限组：`docs_developer` 用于开发文档，`docs_admin` 用于服务控制和运维文档。 + +## 3. 打开 Earth + +访问： + +```text +http://localhost:3000/earth +``` + +Earth 是公开页面，不需要登录。 + +进入后可以先确认： + +- 地球正常显示 +- 右侧图层控制可打开/关闭图层 +- 搜索可以查找海缆、卫星、算力中心、BGP 事件 +- 算力中心和 BGP 观测站详情卡可以自动采集并预览坐标候选；算力中心待定位气泡可以打开列表并保存候选 +- 鼠标拖动、滚轮缩放和缩放百分比提示正常工作 +- 设置面板可以切换巡航模式、日夜模式、卫星显示风格 + +## 4. 打开控制台 + +访问： + +```text +http://localhost:3000/admin +``` + +控制台用于数据源、采集数据、专题观测、告警、系统日志和配置管理。 + +首次排查建议查看： + +- `/datasources`：数据源目录和采集触发；接口、请求头和凭证配置在 `/settings` 的“采集器设置” +- `/data`：已采集数据 +- `/bgp`：BGP 专题观测 +- `/alerts/system`：系统告警 +- `/settings`：系统配置 + +## 5. 查看运行状态 + +```bash +./planet.sh health +``` + +这个命令会显示容器状态，并检查： + +- 后端 +- AI Provider +- 前端 + +## 6. 查看日志 + +最近日志： + +```bash +./planet.sh log +``` + +持续查看某个服务： + +```bash +./planet.sh log -f +./planet.sh log -b +./planet.sh log -a +``` + +含义： + +- `-f`：前端日志 +- `-b`：后端日志 +- `-a`：AI Provider 日志 + +## 7. 常用重启 + +只重启前端： + +```bash +./planet.sh restart -f +``` + +只重启后端： + +```bash +./planet.sh restart -b +``` + +只重启 AI Provider： + +```bash +./planet.sh restart -a +``` + +只重启数据库： + +```bash +./planet.sh restart -d +``` + +全量重启： + +```bash +./planet.sh restart +``` + +## 8. 局域网访问 + +如果希望 Windows 浏览器、手机或同一局域网的其他设备访问： + +```bash +./planet.sh start --allow-lan +``` + +这会让前端和后端监听局域网可访问地址。 + +注意：`--allow-lan` 只负责让 Planet 服务监听 `0.0.0.0`，不等于自动把 WSL 服务暴露到 Windows 局域网 IP。常见情况是： + +- WSL 内 `localhost:3000` / `localhost:8000` 能访问 +- Windows 本机 `localhost:3000` / `localhost:8000` 能访问 +- 但手机或其他电脑访问 `http://:3000` 失败 + +这通常说明 Windows 端还缺少端口转发或防火墙放行。 + +如果访问失败，先在运行 Planet 的 shell 中检查： + +```bash +curl http://localhost:3000 +curl http://localhost:8000/health +ss -ltnp | grep -E ':3000|:8000' +``` + +如果确认 WSL 中已监听 `0.0.0.0:3000` 和 `0.0.0.0:8000`，但局域网 IP 仍不能访问，请在管理员 PowerShell 中配置 Windows 端转发和防火墙： + +```powershell +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 + +New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 +New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 +``` + +## 9. 停止服务 + +```bash +./planet.sh stop +``` + +停止后会关闭前端、后端、AI Provider、PostgreSQL 和 Redis。 + +## 下一步 + +- 完整操作说明见 [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md) +- 控制台结构见 [控制台前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-admin-frontend-context.md) +- Earth 结构见 [Earth 前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/earth-frontend-context.md) +- 后端采集器见 [数据采集系统](/home/ray/dev/linkong/planet/docs/technical/zh/backend-collectors.md) diff --git a/docs/version-history.md b/docs/version-history.md index 33b21828..6976a24b 100644 --- a/docs/version-history.md +++ b/docs/version-history.md @@ -16,12 +16,46 @@ ## Current Version - `main` 当前主线历史推导到：`0.16.5` -- `dev` 当前开发分支历史推导到：`0.33.0` +- `dev` 当前开发分支历史推导到：`0.49.0` ## Timeline | Version | Type | Branch | Commit | Summary | | --- | --- | --- | --- | --- | +| `0.49.0` | feature | `dev` | `pending` | 新增位置解析 Pipeline、BGP/算力中心地理定位、Docs Gatekeeper、Earth 新闻栏与 Mobile 国家高亮 | +| `0.48.0` | feature | `dev` | `pending` | 新增自定义源 REST/WebSocket 实时 mock 链路，完善 AIS 多源聚合/船舶 enrichment，并将 Earth 全球态势统计改为轻量 SQL 聚合 | +| `0.47.0` | feature | `dev` | `pending` | 新增 AISStream WebSocket 船只采集器、多源 AIS 原始观测聚合、采集器状态配置、船型显示修正和文档规则解耦 | +| `0.46.3` | bugfix | `dev` | `pending` | 优化 Starlink footprint 拖拽性能，避免旋转地球时重复重建覆盖网格，并恢复线缆点击呼吸动画 | +| `0.46.2` | bugfix | `dev` | `pending` | 修复 Earth 启动加载顺序、图层 localStorage 恢复、国界线底图语义、媒体面板、船只轨迹和 Iridium footprint 显示问题，并补充 AIS 聚合计划 | +| `0.46.1` | bugfix | `dev` | `pending` | 修复新增 Docs 技术文档未进前端白名单导致页面不可访问的问题，补齐英文文档并固化白名单/双语/裸文件标题检查 | +| `0.46.0` | feature | `dev` | `pending` | Earth 新增通用 Interactable 图标层，统一船只、算力中心、BGP 事件/观测站交互图标，并优化登陆点与 toolbar 初始渲染 | +| `0.45.0` | feature | `dev` | `pending` | 新增采集任务 fetching 阶段量化进度，收敛 AI Provider 运行期环境注入和 Docker build context | +| `0.44.2` | bugfix | `dev` | `pending` | 补充 Earth 船只批量渲染、屏幕拾取、图层顺序、样式参考和性能计划状态文档 | +| `0.44.1` | bugfix | `dev` | `pending` | 优化 Earth 船只批量渲染性能，修复拖动卡顿、拾取错位、交互态方向/尺寸和地表压盖问题 | +| `0.44.0` | feature | `dev` | `pending` | 重构数据源目录与采集器设置，新增 BarentsWatch AIS 连接教程、Earth 船只/缩放体验优化和仪表盘前端重启 | +| `0.43.1` | bugfix | `dev` | `pending` | 修正全量 restart 后 AI Provider 启动提示语义，避免把预期未就绪描述成异常 | +| `0.43.0` | feature | `dev` | `pending` | 新增 Earth 船舶追踪、自定义数据源映射、外部集成配置中心、Markdown 渲染器增强，并整理规则/技能文档加载约束 | +| `0.42.2` | bugfix | `dev` | `pending` | Docs 中文模式补齐分组和文档标题翻译，并更新文档站品牌文案 | +| `0.42.1` | bugfix | `dev` | `pending` | 修正 release skill 的 feature 版本计算规则，minor 进位时重置 patch 为 0 | +| `0.42.0` | feature | `dev` | `pending` | 新增公开 `/docs` 文档站、中英文技术/使用文档、搜索与主题切换，并补充公共组件复用和 Earth 无高清材质边缘提示 | +| `0.41.2` | improvement | `dev` | `pending` | 启动脚本新增 verbose 滚动日志与端口占用诊断，Docker 构建支持镜像源覆盖，并修复 Earth 登陆点遮挡判断 | +| `0.41.1` | improvement | `dev` | `pending` | 修复新闻直播持久化失效、pin 边缘遮挡；图标抽取为 SVG 并建立规范 | +| `0.41.0` | feature | `dev` | `pending` | Earth 图层顺序拆分、基座海陆色块、国界交互、高清材质/云图/地形层级与样式文档落地 | +| `0.40.5` | improvement | `dev` | `pending` | 卫星 ribbon 拖尾、Iridium 覆盖球面投影填充+外圈、搜索自动聚焦修复 | +| `0.40.4` | bugfix | `dev` | `pending` | 修复页面后台恢复后卫星轨迹跳变与位置错位，统一轨迹重置路径 | +| `0.40.3` | improvement | `dev` | `pending` | 卫星点云升级 ShaderMaterial，修复锁定环 depthTest 与位置漂移，新增悬停态缩放 | +| `0.40.2` | improvement | `dev` | `pending` | 卫星点大小随镜头缩放动态调整，调小默认基础尺寸 | +| `0.40.1` | improvement | `dev` | `pending` | 卫星选中标记配色跟随图例，修复 footprint 遮蔽卫星渲染问题，修复选中海缆误触发卫星高亮 | +| `0.40.0` | feature | `dev` | `pending` | Earth 卫星 footprint 按星座能力分层，Iridium 独立 coverage ring 落地，卫星详情卡补齐覆盖能力与当前显示说明 | +| `0.39.0` | feature | `dev` | `pending` | 后端统一结构化日志地基落地，系统日志页重构为紧凑日志工作台，并修复 Earth 移动端态势抽屉与新闻详情同步问题 | +| `0.38.0` | feature | `dev` | `pending` | Earth 新闻接入通用巡航与专用卡片链路，系统日志页升级为结构化时间/级别过滤与真正字符串检索 | +| `0.37.2` | bugfix | `dev` | `pending` | Earth 图层系统新增经纬线开关，并将经纬线接入统一 layer registry、移动端抽屉与设置持久化流 | +| `0.37.1` | bugfix | `dev` | `pending` | 修复 `planet.sh` 在 `uvicorn --reload` 场景下未清理旧 worker 的问题，避免后端重启后仍停留旧实例并导致算力中心聚合接口 404 | +| `0.37.0` | feature | `dev` | `pending` | Earth 连线系统从巡航语义中完全解耦为通用 callout connector，统一桌面/移动端对象级锚点、临界区锚点滑动与稳定巡航展示链路 | +| `0.36.0` | feature | `dev` | `pending` | Earth 新增统一算力中心图层与估算位置展示，继续收口拖拽交互，并补充 AI Provider 指纹与 WSL 局域网访问支撑 | +| `0.35.1` | bugfix | `dev` | `pending` | 收口 Earth 桌面 HUD 与移动端抽屉的统一统计绑定机制，修复态势统计在图层切换后的同步遗漏 | +| `0.35.0` | feature | `dev` | `pending` | Earth 移动端抽屉系统与悬浮卡片全面上线：手势驱动抽屉、点击物件弹出可拖动详情卡、单指旋转双指缩放地球 | +| `0.34.0` | feature | `dev` | `pending` | Earth 搜索面板正式接入，`planet.sh --allow-lan` 打通 Bun + Vite 局域网开放链路，并自动输出推荐访问地址与健康检查地址 | | `0.33.0` | feature | `dev` | `pending` | `news_live_streams` 默认接入 iptv-org 频道目录，内置数据源支持直接编辑 override，并修复 TV 合并采集源后默认频道消失的问题 | | `0.32.0` | feature | `dev` | `pending` | Earth 设置新增默认地球大小真源，并继续收口卫星焦点层次、toolbar/scrollbar 性能与 HUD 设置面板细节 | | `0.31.3` | bugfix | `dev` | `pending` | 收口 Earth 图层注册表与启动任务框架，修复旋转/巡航切换、卫星地形遮挡与日夜关闭照明回归 | diff --git a/frontend/package.json b/frontend/package.json index 0d2d4392..deda0ea2 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "planet-frontend", - "version": "0.33.0", + "version": "0.49.0", "private": true, "packageManager": "bun@1", "dependencies": { @@ -25,8 +25,8 @@ "vite": "^5.0.10" }, "scripts": { - "dev": "vite", - "build": "tsc && vite build", - "preview": "vite preview" + "dev": "bun ./node_modules/vite/bin/vite.js", + "build": "bun x tsc && bun ./node_modules/vite/bin/vite.js build", + "preview": "bun ./node_modules/vite/bin/vite.js preview" } } diff --git a/frontend/public/earth/assets/icons/bgp-broadcast-pin.svg b/frontend/public/earth/assets/icons/bgp-broadcast-pin.svg new file mode 100644 index 00000000..df937883 --- /dev/null +++ b/frontend/public/earth/assets/icons/bgp-broadcast-pin.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/public/earth/assets/icons/bgp-collector.svg b/frontend/public/earth/assets/icons/bgp-collector.svg new file mode 100644 index 00000000..708e5218 --- /dev/null +++ b/frontend/public/earth/assets/icons/bgp-collector.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + diff --git a/frontend/public/earth/assets/icons/bgp-event-burst.svg b/frontend/public/earth/assets/icons/bgp-event-burst.svg new file mode 100644 index 00000000..1c8e2cf1 --- /dev/null +++ b/frontend/public/earth/assets/icons/bgp-event-burst.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/frontend/public/earth/assets/icons/bgp-event-dot.svg b/frontend/public/earth/assets/icons/bgp-event-dot.svg new file mode 100644 index 00000000..b74bce5a --- /dev/null +++ b/frontend/public/earth/assets/icons/bgp-event-dot.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/public/earth/assets/icons/bgp-event-exclamation.svg b/frontend/public/earth/assets/icons/bgp-event-exclamation.svg new file mode 100644 index 00000000..9bc60bea --- /dev/null +++ b/frontend/public/earth/assets/icons/bgp-event-exclamation.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/frontend/public/earth/assets/icons/bgp-event-leak.svg b/frontend/public/earth/assets/icons/bgp-event-leak.svg new file mode 100644 index 00000000..4b3927e4 --- /dev/null +++ b/frontend/public/earth/assets/icons/bgp-event-leak.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/frontend/public/earth/assets/icons/bgp-event-ring.svg b/frontend/public/earth/assets/icons/bgp-event-ring.svg new file mode 100644 index 00000000..538e6fa7 --- /dev/null +++ b/frontend/public/earth/assets/icons/bgp-event-ring.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/public/earth/assets/icons/bgp-event-triangle.svg b/frontend/public/earth/assets/icons/bgp-event-triangle.svg new file mode 100644 index 00000000..da9c9c0b --- /dev/null +++ b/frontend/public/earth/assets/icons/bgp-event-triangle.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/public/earth/assets/icons/bgp-event-wave.svg b/frontend/public/earth/assets/icons/bgp-event-wave.svg new file mode 100644 index 00000000..d84634ad --- /dev/null +++ b/frontend/public/earth/assets/icons/bgp-event-wave.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/public/earth/assets/icons/bgp-glow-dot.svg b/frontend/public/earth/assets/icons/bgp-glow-dot.svg new file mode 100644 index 00000000..1ee04289 --- /dev/null +++ b/frontend/public/earth/assets/icons/bgp-glow-dot.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/frontend/public/earth/assets/icons/compute-gpu-cluster.svg b/frontend/public/earth/assets/icons/compute-gpu-cluster.svg new file mode 100644 index 00000000..49f9913e --- /dev/null +++ b/frontend/public/earth/assets/icons/compute-gpu-cluster.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/frontend/public/earth/assets/icons/compute-hdd-network.svg b/frontend/public/earth/assets/icons/compute-hdd-network.svg new file mode 100644 index 00000000..63daedf0 --- /dev/null +++ b/frontend/public/earth/assets/icons/compute-hdd-network.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/public/earth/assets/icons/compute-supercomputer.svg b/frontend/public/earth/assets/icons/compute-supercomputer.svg new file mode 100644 index 00000000..1269f989 --- /dev/null +++ b/frontend/public/earth/assets/icons/compute-supercomputer.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/public/earth/assets/icons/landing-point-geo-alt-fill.svg b/frontend/public/earth/assets/icons/landing-point-geo-alt-fill.svg new file mode 100755 index 00000000..a53f2bdf --- /dev/null +++ b/frontend/public/earth/assets/icons/landing-point-geo-alt-fill.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/public/earth/assets/icons/marker-landing-point.svg b/frontend/public/earth/assets/icons/marker-landing-point.svg new file mode 100644 index 00000000..387ae548 --- /dev/null +++ b/frontend/public/earth/assets/icons/marker-landing-point.svg @@ -0,0 +1,10 @@ + + + + + diff --git a/frontend/public/earth/css/base.css b/frontend/public/earth/css/base.css index 1243c65e..3330f6f2 100644 --- a/frontend/public/earth/css/base.css +++ b/frontend/public/earth/css/base.css @@ -8,6 +8,10 @@ :root { --hud-scale: 1; + --safe-top: env(safe-area-inset-top, 0px); + --safe-right: env(safe-area-inset-right, 0px); + --safe-bottom: env(safe-area-inset-bottom, 0px); + --safe-left: env(safe-area-inset-left, 0px); --hud-offset: calc(20px * var(--hud-scale)); --hud-radius: calc(22px * var(--hud-scale)); --hud-panel-padding: calc(18px * var(--hud-scale)); @@ -66,12 +70,23 @@ body.earth-page { overflow: hidden; } +html.is-globe-dragging, +body.earth-page.is-globe-dragging, +body.earth-page.is-globe-dragging * { + user-select: none !important; + -webkit-user-select: none !important; +} + .earth-app { position: relative; width: 100vw; height: 100vh; } +.earth-app canvas { + touch-action: none; +} + .earth-app.dragging { cursor: grabbing; } diff --git a/frontend/public/earth/css/earth-stats.css b/frontend/public/earth/css/earth-stats.css index e09874a8..ed35e38d 100644 --- a/frontend/public/earth/css/earth-stats.css +++ b/frontend/public/earth/css/earth-stats.css @@ -133,3 +133,20 @@ right: var(--hud-offset); transform: translate(calc(100% - var(--hud-offset)), calc(-100% + var(--hud-offset))); } + +.layout-mode-mobile .hud-panel-stats { + position: fixed; + top: calc(8px + var(--safe-top)); + right: 8px; + width: min(180px, calc(100vw - 16px)); + z-index: 205; +} + +.layout-mode-mobile.earth-search-open .hud-panel-stats, +.layout-mode-mobile.earth-settings-open .hud-panel-stats, +.layout-mode-mobile.earth-media-open .hud-panel-stats, +.layout-mode-mobile.earth-info-open .hud-panel-stats { + opacity: 0; + pointer-events: none; + transform: translateY(-12px); +} diff --git a/frontend/public/earth/css/hud.css b/frontend/public/earth/css/hud.css index f75badf0..385ab44b 100644 --- a/frontend/public/earth/css/hud.css +++ b/frontend/public/earth/css/hud.css @@ -117,6 +117,7 @@ .hud-panel-drag-handle { cursor: grab; user-select: none; + touch-action: none; } .hud-panel-drag-handle:active { @@ -260,6 +261,1352 @@ display: none !important; } +/* ── Mobile popup card ───────────────────────────────────────── */ + +.earth-mobile-popup { + display: none; +} + +.layout-mode-mobile .earth-mobile-popup { + display: flex; + position: fixed; + z-index: 260; + overflow: visible; + align-items: center; + gap: 10px; + padding: 10px 12px 10px 12px; + max-width: 220px; + min-width: 120px; + background: linear-gradient(145deg, rgba(14, 25, 42, 0.97), rgba(7, 15, 28, 0.97)); + border: 1px solid rgba(200, 224, 255, 0.13); + border-radius: 20px; + box-shadow: 0 10px 36px rgba(0, 0, 0, 0.48), 0 0 0 1px rgba(140, 190, 255, 0.05); + backdrop-filter: blur(20px); + -webkit-backdrop-filter: blur(20px); + pointer-events: auto; + opacity: 0; + transform: scale(0.86) translateY(4px); + transition: opacity 0.17s ease, transform 0.17s cubic-bezier(0.22, 1, 0.36, 1); + cursor: grab; + user-select: none; + touch-action: none; +} + +.layout-mode-mobile .earth-mobile-popup:active { + cursor: grabbing; +} + +.layout-mode-mobile .earth-mobile-popup[hidden] { + display: none; +} + +.layout-mode-mobile .earth-mobile-popup.is-visible { + opacity: 1; + transform: scale(1) translateY(0); +} + +.layout-mode-mobile .earth-mobile-popup.earth-mobile-popup--anchor-stable { + transform: none; + transition: opacity 0.17s ease; +} + +.layout-mode-mobile .earth-mobile-popup.earth-mobile-popup--anchor-stable.is-visible { + transform: none; +} + +.earth-mobile-popup-icon { + font-size: 1.25rem; + flex-shrink: 0; + line-height: 1; +} + +.earth-mobile-popup-body { + flex: 1; + min-width: 0; +} + +.earth-mobile-popup-title { + color: var(--hud-text); + font-size: 0.84rem; + font-weight: 700; + line-height: 1.25; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + letter-spacing: 0.01em; +} + +.earth-mobile-popup-sub { + color: var(--hud-text-muted); + font-size: 0.70rem; + margin-top: 2px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + letter-spacing: 0.03em; +} + +.earth-mobile-popup-chevron { + color: rgba(160, 200, 255, 0.5); + font-size: 1rem; + flex-shrink: 0; +} + +.earth-mobile-popup-dock { + --earth-mobile-popup-dock-size: 14px; + --earth-mobile-popup-dock-offset: calc(var(--earth-mobile-popup-dock-size) * -0.5); + position: absolute; + top: 32%; + width: var(--earth-mobile-popup-dock-size); + height: var(--earth-mobile-popup-dock-size); + border-radius: 999px; + background: transparent; + border: 0; + box-shadow: none; + transform: translateY(-50%); + pointer-events: none; +} + +.earth-mobile-popup[data-dock-side="left"] .earth-mobile-popup-dock { + left: var(--earth-mobile-popup-dock-offset); +} + +.earth-mobile-popup[data-dock-side="right"] .earth-mobile-popup-dock { + right: var(--earth-mobile-popup-dock-offset); +} + +.earth-mobile-popup[data-dock-side="top"] .earth-mobile-popup-dock { + top: var(--earth-mobile-popup-dock-offset); + left: 50%; + transform: translateX(-50%); +} + +.earth-mobile-popup[data-dock-side="bottom"] .earth-mobile-popup-dock { + top: auto; + bottom: var(--earth-mobile-popup-dock-offset); + left: 50%; + transform: translateX(-50%); +} + +/* ── Mobile drawer ───────────────────────────────────────────── */ + +.earth-mobile-drawer-overlay, +.earth-mobile-drawer-shell { + display: none; +} + +.layout-mode-mobile .earth-mobile-drawer-overlay, +.layout-mode-mobile .earth-mobile-drawer-shell { + display: block; +} + +.earth-mobile-drawer-overlay { + position: fixed; + inset: 0; + background: linear-gradient(180deg, rgba(3, 8, 16, 0.06), rgba(3, 8, 16, 0.54)); + opacity: 0; + pointer-events: none; + transition: opacity 0.22s ease; + z-index: 245; +} + +.earth-mobile-drawer-shell { + position: fixed; + left: 0; + right: 0; + bottom: 0; + z-index: 250; + pointer-events: none; +} + +.earth-mobile-drawer-sheet { + position: relative; + display: flex; + flex-direction: column; + min-height: calc(240px + var(--safe-bottom)); + height: min(78vh, calc(100vh - 72px - var(--safe-top))); + padding: 10px 14px calc(14px + var(--safe-bottom)) 14px; + border-top-left-radius: 28px; + border-top-right-radius: 28px; + background: + linear-gradient(180deg, rgba(15, 27, 43, 0.98), rgba(7, 14, 24, 0.98)), + rgba(5, 10, 19, 0.92); + border-top: 1px solid rgba(220, 238, 255, 0.1); + box-shadow: 0 -26px 60px rgba(0, 0, 0, 0.38); + transform: translateY(calc(100% - 36px - var(--safe-bottom))); + transition: transform 0.28s cubic-bezier(0.22, 1, 0.36, 1); + pointer-events: none; + touch-action: none; +} + +.earth-mobile-drawer-header { + order: 1; + flex: 0 0 auto; + padding: 10px 4px 8px; + cursor: ns-resize; + user-select: none; + pointer-events: auto; +} + +.earth-mobile-drawer-grabber { + width: 54px; + height: 5px; + margin: 0 auto; + border-radius: 999px; + background: rgba(225, 239, 255, 0.18); +} + +.earth-mobile-drawer-nav { + order: 3; + flex: 0 0 auto; + display: grid; + gap: 8px; + margin-top: 8px; + padding-top: 10px; + border-top: 1px solid rgba(214, 231, 247, 0.08); +} + +.earth-mobile-drawer-nav-copy { + display: none; +} + +.earth-mobile-drawer-nav-kicker { + color: rgba(167, 194, 223, 0.58); + font-size: 0.65rem; + font-weight: 700; + letter-spacing: 0.16em; + text-transform: uppercase; +} + +.earth-mobile-drawer-nav-title { + color: var(--hud-title); + font-size: 0.92rem; + font-weight: 700; + letter-spacing: 0.01em; +} + +.earth-mobile-drawer-tabs-shell { + position: relative; + padding: 8px 8px 2px; + border: 1px solid rgba(207, 226, 245, 0.06); + border-radius: 22px; + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.035), rgba(255, 255, 255, 0.01)), + rgba(6, 16, 31, 0.32); + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.04), + 0 10px 22px rgba(2, 8, 18, 0.12); + overflow: hidden; +} + +.earth-mobile-drawer-tabs { + display: flex; + align-items: stretch; + gap: 10px; + padding: 2px; + overflow-x: auto; + overflow-y: hidden; + scrollbar-width: none; + -webkit-overflow-scrolling: touch; + scroll-snap-type: x proximity; +} + +.earth-mobile-drawer-tabs-fade { + position: absolute; + top: 8px; + bottom: 8px; + width: 22px; + z-index: 2; + pointer-events: none; +} + +.earth-mobile-drawer-tabs-fade--left { + left: 0; + background: linear-gradient(90deg, rgba(6, 16, 31, 0.92), rgba(6, 16, 31, 0)); +} + +.earth-mobile-drawer-tabs-fade--right { + right: 0; + background: linear-gradient(270deg, rgba(6, 16, 31, 0.92), rgba(6, 16, 31, 0)); +} + +.earth-mobile-drawer-tab { + flex: 0 0 auto; + min-width: 68px; + min-height: 58px; + padding: 8px 11px 9px; + display: inline-flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 5px; + border: 1px solid rgba(201, 225, 247, 0.06); + border-radius: 16px; + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.06), rgba(255, 255, 255, 0.015)), + rgba(7, 18, 34, 0.38); + color: var(--hud-text-muted); + cursor: pointer; + scroll-snap-align: start; + position: relative; + z-index: 1; + transition: + background 0.18s ease, + border-color 0.18s ease, + color 0.18s ease, + transform 0.18s ease, + box-shadow 0.18s ease; +} + +.earth-mobile-drawer-tab.is-active { + color: var(--hud-title); + background: + linear-gradient(180deg, rgba(111, 174, 255, 0.18), rgba(74, 126, 210, 0.08)), + rgba(10, 26, 52, 0.58); + border-color: rgba(122, 180, 255, 0.22); + box-shadow: + 0 8px 18px rgba(11, 22, 40, 0.16), + inset 0 1px 0 rgba(255, 255, 255, 0.1); + transform: translateY(-1px); +} + +.earth-mobile-drawer-tab:hover { + color: var(--hud-text); + border-color: rgba(201, 225, 247, 0.14); +} + +.earth-mobile-drawer-tabs::-webkit-scrollbar { + display: none; +} + +.earth-mobile-drawer-tab-icon { + width: 28px; + height: 28px; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 999px; + font-size: 17px; + color: rgba(221, 235, 248, 0.88); + background: rgba(255, 255, 255, 0.06); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.06); +} + +.earth-mobile-drawer-tab.is-active .earth-mobile-drawer-tab-icon { + color: #f6fbff; + background: + linear-gradient(180deg, rgba(161, 207, 255, 0.26), rgba(95, 154, 237, 0.18)), + rgba(255, 255, 255, 0.08); +} + +.earth-mobile-drawer-tab-label { + font-size: 0.68rem; + font-weight: 700; + letter-spacing: 0.04em; + white-space: nowrap; +} + +.earth-mobile-drawer-content { + order: 2; + flex: 1 1 auto; + position: relative; + min-height: 0; + max-height: none; + overflow: hidden; +} + +.earth-mobile-drawer-slot { + display: none; + min-height: 0; + height: 100%; + max-height: none; + overflow: auto; + overscroll-behavior: contain; + padding-bottom: 6px; +} + +.earth-mobile-drawer-slot.is-active { + display: block; +} + +.earth-mobile-drawer-slot--situation { + display: none; + gap: 12px; +} + +.earth-mobile-drawer-slot--situation, +.earth-mobile-page, +.earth-mobile-stats-grid, +.earth-mobile-situation-card, +.earth-mobile-situation-legend-list { + width: 100%; + min-width: 0; +} + +.earth-mobile-drawer-slot--situation.is-active { + display: grid; +} + +@keyframes earth-drawer-hint { + 0% { transform: translateY(calc(100% - 36px - var(--safe-bottom))); } + 14% { transform: translateY(calc(100% - 36px - var(--safe-bottom) - 22px)); } + 26% { transform: translateY(calc(100% - 36px - var(--safe-bottom) - 4px)); } + 38% { transform: translateY(calc(100% - 36px - var(--safe-bottom) - 14px)); } + 50% { transform: translateY(calc(100% - 36px - var(--safe-bottom) - 1px)); } + 62% { transform: translateY(calc(100% - 36px - var(--safe-bottom) - 7px)); } + 76% { transform: translateY(calc(100% - 36px - var(--safe-bottom))); } + 100% { transform: translateY(calc(100% - 36px - var(--safe-bottom))); } +} + +.layout-mode-mobile .earth-mobile-drawer-sheet.is-hinting { + animation: earth-drawer-hint 0.85s cubic-bezier(0.22, 1, 0.36, 1) forwards; +} + +.layout-mode-mobile.earth-mobile-drawer-open .earth-mobile-drawer-overlay { + opacity: 1; + pointer-events: auto; +} + +.layout-mode-mobile.earth-mobile-drawer-open .earth-mobile-drawer-sheet { + transform: translateY(0); + pointer-events: auto; +} + +.layout-mode-mobile .earth-left-column { + pointer-events: none; +} + +.layout-mode-mobile #brand-panel { + pointer-events: auto; +} + +.layout-mode-mobile #search-modal, +.layout-mode-mobile #settings-modal { + display: block; + position: static; + inset: auto; + pointer-events: none; +} + +.layout-mode-mobile #search-backdrop, +.layout-mode-mobile #settings-backdrop { + display: none; +} + +.layout-mode-mobile .earth-mobile-drawer-slot > .hud-panel, +.layout-mode-mobile .earth-mobile-drawer-slot > .earth-search-sheet, +.layout-mode-mobile .earth-mobile-drawer-slot > .earth-settings-sheet { + position: relative !important; + inset: auto !important; + left: auto !important; + right: auto !important; + top: auto !important; + bottom: auto !important; + transform: none !important; + width: 100% !important; + max-width: none !important; + min-width: 0 !important; + max-height: none !important; + height: auto !important; + margin: 0; + pointer-events: auto; + opacity: 1 !important; + filter: none !important; + visibility: visible !important; +} + +.layout-mode-mobile .earth-mobile-drawer-slot > .hud-panel { + box-shadow: none; +} + +.layout-mode-mobile .earth-mobile-drawer-slot > .earth-search-sheet { + display: flex !important; +} + +.layout-mode-mobile .earth-mobile-drawer-slot > .earth-settings-sheet { + display: flex !important; +} + +.layout-mode-mobile .earth-mobile-drawer-slot .hud-panel-drag-handle { + cursor: default; +} + +.layout-mode-mobile .earth-mobile-drawer-slot .tv-panel-edge { + display: none; +} + +.layout-mode-mobile .earth-mobile-drawer-slot .tv-panel-player { + min-height: 220px; +} + +.layout-mode-mobile #layer-toggles, +.layout-mode-mobile #legend, +.layout-mode-mobile #earth-stats, +.layout-mode-mobile #media-panel, +.layout-mode-mobile #search-modal, +.layout-mode-mobile #settings-modal, +.layout-mode-mobile #info-panel { + display: none !important; +} + +.earth-mobile-page { + display: flex; + flex-direction: column; + gap: 14px; + min-height: 0; +} + +.earth-mobile-page--tv { + gap: 8px; +} + +.earth-mobile-page--tv .earth-mobile-page-intro { + gap: 2px; +} + +.earth-mobile-page-intro { + display: flex; + flex-direction: column; + gap: 4px; +} + +.earth-mobile-page-kicker { + color: var(--hud-text-muted); + font-size: 0.68rem; + letter-spacing: 0.12em; + text-transform: uppercase; +} + +.earth-mobile-page-summary { + color: var(--hud-text-soft); + font-size: 0.84rem; + line-height: 1.4; +} + +.earth-mobile-page--tv .earth-mobile-page-summary { + font-size: 0.72rem; + line-height: 1.3; +} + +.earth-mobile-layer-list, +.earth-mobile-news-board-list, +.earth-mobile-search-results { + display: flex; + flex-direction: column; + gap: 10px; +} + +.earth-mobile-layer-card, +.earth-mobile-action-btn, +.earth-mobile-settings-pill, +.earth-mobile-search-clear { + appearance: none; + -webkit-appearance: none; +} + +.earth-mobile-layer-card { + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + gap: 12px; + align-items: center; + width: 100%; + border: 1px solid rgba(212, 227, 244, 0.08); + border-radius: 18px; + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.05), transparent), + rgba(255, 255, 255, 0.03); + color: var(--hud-text); + padding: 14px 16px; + text-align: left; +} + +.earth-mobile-layer-card.is-active { + border-color: rgba(122, 180, 255, 0.24); + background: + linear-gradient(180deg, rgba(122, 180, 255, 0.12), transparent), + rgba(255, 255, 255, 0.04); +} + +.earth-mobile-layer-card.is-disabled, +.earth-mobile-layer-card:disabled { + cursor: not-allowed; + opacity: 0.46; +} + +.earth-mobile-layer-card-icon { + font-size: 22px; + color: var(--hud-accent-strong); +} + +.earth-mobile-layer-card-copy { + display: flex; + flex-direction: column; + gap: 3px; + min-width: 0; +} + +.earth-mobile-layer-card-title, +.earth-mobile-detail-title, +.earth-mobile-tv-title, +.earth-mobile-news-focus-label, +.earth-mobile-settings-label { + color: var(--hud-title); + font-weight: 600; +} + +.earth-mobile-layer-card-subtitle, +.earth-mobile-detail-type, +.earth-mobile-tv-subtitle, +.earth-mobile-settings-subtitle, +.earth-mobile-news-focus-coords, +.earth-mobile-tv-notes, +.earth-mobile-tv-catalog, +.earth-mobile-news-board-status, +.earth-mobile-search-meta, +.earth-mobile-search-empty { + color: var(--hud-text-soft); + font-size: 0.76rem; + line-height: 1.45; +} + +.earth-mobile-layer-card-status { + color: var(--hud-text-muted); + font-size: 0.72rem; + letter-spacing: 0.08em; +} + +.earth-mobile-search-shell { + display: flex; + align-items: center; + gap: 10px; + min-height: 52px; + padding: 0 14px; + border-radius: 16px; + border: 1px solid rgba(214, 230, 247, 0.12); + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.05), transparent), + rgba(255, 255, 255, 0.03); +} + +.earth-mobile-search-input { + flex: 1 1 auto; + min-width: 0; + border: 0; + background: transparent; + color: var(--hud-title); + font: inherit; + outline: 0; +} + +.earth-mobile-search-input::placeholder { + color: rgba(190, 208, 227, 0.46); +} + +.earth-mobile-search-icon { + color: var(--hud-text-muted); +} + +.earth-mobile-search-clear { + border: 0; + background: transparent; + color: var(--hud-text-muted); + display: inline-flex; + align-items: center; + justify-content: center; +} + +.earth-mobile-search-clear[hidden] { + display: none; +} + +.earth-mobile-stats-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 10px; +} + +.earth-mobile-page--situation .earth-mobile-stats-grid > * { + min-width: 0; +} + +.earth-mobile-stat-card, +.earth-mobile-situation-card, +.earth-mobile-news-focus, +.earth-mobile-settings-card, +.earth-mobile-detail-card, +.earth-mobile-tv-meta { + border-radius: 18px; + border: 1px solid rgba(212, 227, 244, 0.08); + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.05), transparent), + rgba(255, 255, 255, 0.03); + padding: 14px 16px; +} + +.earth-mobile-stat-card { + display: flex; + flex-direction: column; + gap: 6px; +} + +.earth-mobile-page--situation .earth-mobile-stat-card { + min-width: 0; + padding: 12px 12px; + gap: 4px; +} + +.earth-mobile-stat-num { + color: var(--hud-title); + font-size: 1.26rem; + font-weight: 700; +} + +.earth-mobile-page--situation .earth-mobile-stat-num { + font-size: 1.08rem; + line-height: 1.1; +} + +.earth-mobile-stat-label, +.earth-mobile-situation-card-subtitle, +.earth-mobile-news-focus-kicker, +.earth-mobile-settings-title, +.earth-mobile-tv-status { + color: var(--hud-text-muted); + font-size: 0.7rem; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.earth-mobile-page--situation .earth-mobile-stat-label { + font-size: 0.64rem; + letter-spacing: 0.04em; + line-height: 1.25; + white-space: normal; + word-break: break-word; +} + +@media (orientation: landscape) and (max-height: 540px) { + .earth-mobile-page--situation { + gap: 8px; + } + + .earth-mobile-page--situation .earth-mobile-page-intro { + gap: 1px; + } + + .earth-mobile-page--situation .earth-mobile-page-kicker { + font-size: 0.6rem; + } + + .earth-mobile-page--situation .earth-mobile-page-summary { + font-size: 0.68rem; + line-height: 1.2; + } + + .earth-mobile-page--situation .earth-mobile-stats-grid { + gap: 8px; + } + + .earth-mobile-page--situation .earth-mobile-stat-card { + padding: 10px 10px; + gap: 3px; + border-radius: 15px; + } + + .earth-mobile-page--situation .earth-mobile-stat-num { + font-size: 0.98rem; + } + + .earth-mobile-page--situation .earth-mobile-stat-label { + font-size: 0.6rem; + line-height: 1.15; + } +} + +.earth-mobile-situation-card { + display: flex; + flex-direction: column; + gap: 10px; +} + +.earth-mobile-situation-card-title { + color: var(--hud-title); + font-size: 0.92rem; + font-weight: 600; +} + +.earth-mobile-situation-legend-list { + display: flex; + flex-direction: column; + gap: 8px; + overflow-x: hidden; +} + +.earth-mobile-situation-status { + color: var(--hud-text); + line-height: 1.5; + min-width: 0; + overflow-wrap: anywhere; + word-break: break-word; +} + +.earth-mobile-situation-legend-list .legend-item, +.earth-mobile-situation-legend-list .legend-label { + min-width: 0; +} + +.earth-mobile-situation-legend-list .legend-label { + white-space: normal; + overflow: visible; + text-overflow: clip; + overflow-wrap: anywhere; + word-break: break-word; +} + +.earth-mobile-news-focus, +.earth-mobile-tv-actions, +.earth-mobile-news-actions, +.earth-mobile-settings-actions { + display: flex; + gap: 10px; +} + +.earth-mobile-news-focus { + justify-content: space-between; + align-items: flex-start; +} + +.earth-mobile-news-source-count { + color: var(--hud-accent-strong); + font-size: 0.74rem; + font-weight: 600; +} + +.earth-mobile-news-board-list .news-story-card { + margin: 0; +} + +.earth-mobile-news-board-empty[hidden] { + display: none; +} + +.earth-mobile-tv-select, +.earth-mobile-settings-slider { + width: 100%; +} + +.earth-mobile-tv-overview { + border-radius: 14px; + border: 1px solid rgba(212, 227, 244, 0.08); + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.05), transparent), + rgba(255, 255, 255, 0.03); + padding: 8px 10px; +} + +.earth-mobile-tv-overview-bar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + cursor: pointer; +} + +.earth-mobile-tv-overview-copy { + display: flex; + flex-direction: column; + gap: 2px; + min-width: 0; +} + +.earth-mobile-tv-overview-kicker { + color: var(--hud-text-muted); + font-size: 0.58rem; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.earth-mobile-tv-overview-headline { + color: var(--hud-title); + font-size: 0.84rem; + font-weight: 600; + line-height: 1.2; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.earth-mobile-tv-overview-summary { + color: var(--hud-text-soft); + font-size: 0.66rem; + line-height: 1.2; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.earth-mobile-tv-overview-tags { + display: flex; + flex-wrap: nowrap; + gap: 4px; + margin-top: 1px; + overflow: hidden; +} + +.earth-mobile-tv-overview-tag { + display: inline-flex; + align-items: center; + min-height: 20px; + padding: 0 6px; + border-radius: 999px; + border: 1px solid rgba(212, 227, 244, 0.12); + background: rgba(255, 255, 255, 0.06); + color: var(--hud-text); + font-size: 0.62rem; + line-height: 1; + white-space: nowrap; +} + +.earth-mobile-tv-overview-tag--status { + color: var(--hud-accent-strong); + border-color: rgba(122, 180, 255, 0.2); + background: rgba(122, 180, 255, 0.12); +} + +.earth-mobile-tv-overview-actions { + display: inline-flex; + align-items: center; + gap: 4px; + flex: 0 0 auto; +} + +.earth-mobile-tv-overview-bar:focus-visible { + outline: 2px solid rgba(122, 180, 255, 0.5); + outline-offset: 4px; + border-radius: 12px; +} + +.earth-mobile-tv-meta-wrap { + overflow: hidden; + max-height: 240px; + opacity: 1; + margin-top: 10px; + transition: max-height 0.22s ease, opacity 0.18s ease, margin 0.22s ease; +} + +.earth-mobile-tv-meta-wrap.is-collapsed { + max-height: 0; + opacity: 0; + pointer-events: none; + margin-top: 0; +} + +.earth-mobile-tv-select { + border: 1px solid rgba(201, 225, 247, 0.14); + border-radius: 12px; + background: rgba(255, 255, 255, 0.04); + color: var(--hud-text); + padding: 10px 12px; +} + +.earth-mobile-tv-player { + position: relative; + aspect-ratio: 16 / 9; + min-height: 0; + border-radius: 18px; + overflow: hidden; + border: 1px solid rgba(201, 225, 247, 0.1); + background: linear-gradient(180deg, rgba(9, 18, 32, 0.94), rgba(5, 11, 22, 0.94)); +} + +.earth-mobile-tv-empty, +.earth-mobile-tv-iframe, +.earth-mobile-tv-video { + position: absolute; + inset: 0; + width: 100%; + height: 100%; +} + +.earth-mobile-tv-empty { + display: grid; + place-items: center; + padding: 16px; + text-align: center; + color: var(--hud-text-muted); +} + +.earth-mobile-tv-iframe, +.earth-mobile-tv-video { + border: 0; + background: #050a14; +} + +.earth-mobile-action-btn { + display: inline-flex; + align-items: center; + justify-content: center; + min-height: 42px; + padding: 0 14px; + border-radius: 12px; + border: 1px solid rgba(122, 180, 255, 0.22); + background: rgba(122, 180, 255, 0.12); + color: var(--hud-title); + text-decoration: none; + font-size: 0.82rem; + font-weight: 600; +} + +.earth-mobile-action-btn:disabled { + opacity: 0.42; + cursor: default; +} + +.earth-mobile-action-btn--compact { + min-width: 32px; + min-height: 32px; + padding: 0; + border-radius: 10px; + background: rgba(255, 255, 255, 0.06); + border-color: rgba(212, 227, 244, 0.14); + color: var(--hud-text); + flex: 0 0 auto; +} + +.earth-mobile-action-btn--compact .material-symbols-rounded { + font-size: 0.92rem; +} + + + +.earth-mobile-action-btn--ghost { + background: rgba(255, 255, 255, 0.04); + border-color: rgba(212, 227, 244, 0.1); +} + +.earth-mobile-settings-group { + display: flex; + flex-direction: column; + gap: 10px; +} + +.earth-mobile-settings-card { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.earth-mobile-settings-card--stacked { + flex-direction: column; + align-items: stretch; +} + +.earth-mobile-settings-copy { + display: flex; + flex-direction: column; + gap: 4px; +} + +.earth-mobile-settings-segmented { + display: inline-flex; + flex-wrap: wrap; + gap: 8px; +} + +.earth-mobile-settings-pill { + border: 1px solid rgba(212, 227, 244, 0.1); + border-radius: 999px; + background: rgba(255, 255, 255, 0.04); + color: var(--hud-text-soft); + padding: 10px 14px; +} + +.earth-mobile-settings-pill.is-active { + color: var(--hud-title); + border-color: rgba(122, 180, 255, 0.24); + background: rgba(122, 180, 255, 0.14); +} + +.earth-mobile-settings-chip-group { + display: inline-flex; + flex-wrap: wrap; + gap: 8px; +} + +.earth-mobile-settings-chip { + border: 1px solid rgba(212, 227, 244, 0.12); + border-radius: 999px; + background: rgba(255, 255, 255, 0.04); + color: var(--hud-text-soft); + padding: 9px 14px; + font: inherit; + font-size: 0.82rem; + font-weight: 600; + letter-spacing: 0.02em; + cursor: pointer; + transition: + background 0.18s ease, + border-color 0.18s ease, + color 0.18s ease, + transform 0.18s ease; +} + +.earth-mobile-settings-chip:hover { + color: var(--hud-text); + transform: translateY(-1px); +} + +.earth-mobile-settings-chip.is-active { + color: var(--hud-title); + border-color: rgba(122, 180, 255, 0.24); + background: + radial-gradient(circle at 35% 22%, rgba(255, 255, 255, 0.18), transparent 58%), + linear-gradient(180deg, rgba(122, 180, 255, 0.16), rgba(82, 123, 186, 0.22)); +} + +.earth-mobile-settings-switch { + position: relative; + display: inline-flex; + width: 44px; + height: 28px; + flex-shrink: 0; +} + +.earth-mobile-settings-switch input { + position: absolute; + inset: 0; + opacity: 0; +} + +.earth-mobile-settings-switch-track { + width: 100%; + height: 100%; + border-radius: 999px; + background: rgba(255, 255, 255, 0.1); + position: relative; +} + +.earth-mobile-settings-switch-track::after { + content: ""; + position: absolute; + top: 4px; + left: 4px; + width: 20px; + height: 20px; + border-radius: 50%; + background: #fff; + transition: transform 0.18s ease; +} + +.earth-mobile-settings-switch input:checked + .earth-mobile-settings-switch-track { + background: rgba(122, 180, 255, 0.34); +} + +.earth-mobile-settings-switch input:checked + .earth-mobile-settings-switch-track::after { + transform: translateX(16px); +} + +label.is-disabled.earth-mobile-settings-card { + opacity: 0.38; + cursor: not-allowed; + pointer-events: none; +} + +.earth-mobile-settings-slider-row { + display: flex; + align-items: center; + gap: 12px; +} + +.earth-mobile-settings-slider-value { + color: var(--hud-title); + font-weight: 600; +} + +.earth-mobile-detail-card { + display: flex; + flex-direction: column; + gap: 14px; +} + +.earth-mobile-detail-header { + display: flex; + gap: 12px; + align-items: center; +} + +.earth-mobile-detail-icon { + font-size: 1.5rem; +} + +.earth-mobile-detail-heading { + display: flex; + flex-direction: column; + gap: 4px; +} + +.earth-mobile-detail-content { + display: flex; + flex-direction: column; + gap: 10px; +} + +.earth-mobile-news-detail { + display: grid; + gap: 12px; +} + +.earth-mobile-news-detail-kicker { + color: rgba(255, 215, 122, 0.82); + font-size: 0.66rem; + font-weight: 700; + letter-spacing: 0.18em; + text-transform: uppercase; +} + +.earth-mobile-news-detail-title { + color: var(--hud-title); + font-size: 1rem; + line-height: 1.5; + font-weight: 700; +} + +.earth-mobile-news-detail-summary-shell { + position: relative; + padding: 12px 13px; + border: 1px solid rgba(255, 215, 122, 0.12); + background: + linear-gradient(180deg, rgba(255, 215, 122, 0.05), rgba(255, 255, 255, 0.02)), + radial-gradient(circle at top left, rgba(120, 180, 255, 0.08), transparent 56%), + rgba(7, 15, 29, 0.42); + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.05), + 0 10px 28px rgba(0, 0, 0, 0.16); + overflow: hidden; +} + +.earth-mobile-news-detail-summary-shell::before { + content: ""; + position: absolute; + inset: 0; + background: linear-gradient(90deg, transparent 0%, rgba(122, 180, 255, 0.12) 50%, transparent 100%); + opacity: 0.42; + transform: translateX(-100%); + animation: infoCardNewsScan 3.2s linear infinite; + pointer-events: none; +} + +.earth-mobile-news-detail-summary-label { + color: rgba(188, 212, 238, 0.72); + font-size: 0.64rem; + font-weight: 700; + letter-spacing: 0.16em; + text-transform: uppercase; + margin-bottom: 7px; +} + +.earth-mobile-news-detail-summary { + color: #d7e6f7; + font-size: 0.9rem; + line-height: 1.72; + min-height: 5.2em; + white-space: pre-wrap; + word-break: break-word; +} + +.earth-mobile-news-detail-summary.is-typing::after { + content: ""; + display: inline-block; + width: 0.58em; + height: 1.05em; + margin-left: 0.16em; + vertical-align: -0.14em; + background: linear-gradient(180deg, rgba(255, 215, 122, 0.96), rgba(122, 180, 255, 0.78)); + box-shadow: 0 0 10px rgba(255, 215, 122, 0.28); + animation: infoCardNewsCaret 0.9s steps(1, end) infinite; +} + +.earth-mobile-detail-row { + display: flex; + flex-direction: column; + gap: 4px; + padding-bottom: 10px; + border-bottom: 1px solid rgba(255, 255, 255, 0.06); +} + +.earth-mobile-detail-row-label { + color: var(--hud-text-muted); + font-size: 0.72rem; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.earth-mobile-detail-row-value, +.earth-mobile-detail-empty { + color: var(--hud-text); + line-height: 1.5; +} + +.layout-mode-mobile .earth-status-message, +.layout-mode-mobile .earth-error-message { + position: fixed; + top: calc(var(--safe-top) + 10px); + left: auto; + right: calc(10px + var(--safe-right)); + transform: translate(12px, 0); + min-width: 0; + max-width: min(220px, 46vw); + border-radius: 16px; + font-size: 0.74rem; + line-height: 1.28; + padding: 7px 12px 7px 10px; + gap: 8px; + border-color: rgba(214, 230, 247, 0.1); + border-left-color: transparent; + background: + linear-gradient(135deg, rgba(255, 255, 255, 0.06), transparent 52%), + linear-gradient(180deg, rgba(17, 29, 46, 0.92), rgba(7, 14, 24, 0.9)); + box-shadow: + 0 12px 28px rgba(0, 0, 0, 0.22), + 0 0 0 1px rgba(255, 255, 255, 0.03); + backdrop-filter: blur(14px); + -webkit-backdrop-filter: blur(14px); +} + +.layout-mode-mobile .earth-status-message.visible, +.layout-mode-mobile .earth-error-message.visible { + transform: translate(0, 0); +} + +.layout-mode-mobile .earth-error-message { + top: calc(var(--safe-top) + 60px); +} + +.layout-mode-mobile .earth-status-indicator { + gap: 4px; +} + +.layout-mode-mobile .earth-status-dot { + width: 6px; + height: 6px; + box-shadow: + 0 0 6px rgba(145, 186, 255, 0.48), + 0 0 14px rgba(145, 186, 255, 0.16); +} + +.layout-mode-mobile .earth-status-text { + font-weight: 600; +} + +.layout-mode-mobile .earth-status-message.loading { + max-width: min(240px, 52vw); +} + +.layout-mode-mobile .earth-status-message.loading .earth-status-text { + color: rgba(232, 242, 252, 0.92); +} + .hud-panel-row { display: flex; justify-content: space-between; @@ -321,6 +1668,12 @@ opacity: 1; } +.earth-status-message.gesture { + min-width: 0; + padding-right: calc(16px * var(--hud-scale)); + color: #dcecff; +} + .earth-error-message { top: calc(62px * var(--hud-scale)); z-index: 211; @@ -453,6 +1806,246 @@ user-select: none; } +.earth-search-modal { + position: fixed; + inset: 0; + z-index: 255; + visibility: hidden; + opacity: 0; + pointer-events: none; + transition: opacity 0.2s ease, visibility 0.2s ease; +} + +.earth-search-modal.is-open { + visibility: visible; + opacity: 1; + pointer-events: auto; +} + +.earth-search-backdrop { + position: fixed; + inset: 0; + background: rgba(2, 8, 20, 0.38); + backdrop-filter: blur(10px); + -webkit-backdrop-filter: blur(10px); + opacity: 0; + transition: opacity 0.2s ease; +} + +.earth-search-modal.is-open .earth-search-backdrop { + opacity: 1; +} + +.earth-search-sheet { + position: fixed; + top: max(calc(28px * var(--hud-scale)), 8vh); + left: 50%; + width: min(calc(680px * var(--hud-scale)), calc(100vw - (32px * var(--hud-scale)))); + max-height: min(calc(720px * var(--hud-scale)), calc(100vh - (56px * var(--hud-scale)))); + transform: translateX(-50%) scale(0.98); + transform-origin: top center; + padding: calc(var(--hud-panel-padding) * var(--hud-scale)); + display: flex; + flex-direction: column; + gap: calc(var(--hud-gap-md) * var(--hud-scale)); + overflow: hidden; + opacity: 0; + filter: blur(10px); + transition: + transform 0.22s cubic-bezier(0.2, 0.8, 0.2, 1), + opacity 0.22s ease, + filter 0.22s ease; +} + +.earth-search-modal.is-open .earth-search-sheet { + transform: translateX(-50%) scale(1); + opacity: 1; + filter: blur(0); +} + +.earth-search-header, +.earth-search-content { + position: relative; + z-index: 1; +} + +.earth-search-kicker { + color: var(--hud-text-soft); + font-size: calc(0.72rem * var(--hud-scale)); + letter-spacing: 0.16em; + text-transform: uppercase; +} + +.earth-search-content { + display: flex; + flex-direction: column; + gap: calc(var(--hud-gap-sm) * var(--hud-scale)); + min-height: 0; +} + +.earth-search-input-shell { + display: flex; + align-items: center; + gap: calc(8px * var(--hud-scale)); + min-height: calc(48px * var(--hud-scale)); + padding: calc(8px * var(--hud-scale)) calc(12px * var(--hud-scale)); + border: 1px solid rgba(214, 230, 247, 0.12); + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.05), transparent), + rgba(255, 255, 255, 0.03); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.04); +} + +.earth-search-input-shell:focus-within { + border-color: rgba(215, 230, 249, 0.22); + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.08), + 0 0 0 1px rgba(126, 174, 236, 0.1); +} + +.earth-search-input-icon { + color: var(--hud-text-muted); + font-size: calc(20px * var(--hud-scale)); +} + +.earth-search-input { + flex: 1 1 auto; + min-width: 0; + border: 0; + outline: 0; + background: transparent; + color: var(--hud-title); + font: inherit; + font-size: calc(0.9rem * var(--hud-scale)); + letter-spacing: 0.01em; +} + +.earth-search-input::placeholder { + color: rgba(190, 208, 227, 0.46); +} + +.earth-search-clear[hidden] { + display: none; +} + +.earth-search-meta { + min-height: calc(18px * var(--hud-scale)); + color: var(--hud-text-muted); + font-size: calc(0.7rem * var(--hud-scale)); + letter-spacing: 0.02em; +} + +.earth-search-results { + display: flex; + flex-direction: column; + gap: calc(8px * var(--hud-scale)); + min-height: 0; + overflow-y: auto; + padding-right: calc(4px * var(--hud-scale)); + scrollbar-width: thin; + scrollbar-color: rgba(160, 186, 216, 0.34) transparent; +} + +.earth-search-results::-webkit-scrollbar { + width: 6px; +} + +.earth-search-results::-webkit-scrollbar-track { + background: transparent; +} + +.earth-search-results::-webkit-scrollbar-thumb { + background: linear-gradient(180deg, rgba(210, 225, 242, 0.2), rgba(126, 154, 185, 0.28)); + border-radius: 999px; +} + +.earth-search-empty { + color: var(--hud-text-muted); + font-size: calc(0.76rem * var(--hud-scale)); + line-height: 1.55; + padding: calc(8px * var(--hud-scale)) calc(2px * var(--hud-scale)); +} + +.earth-search-result { + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + align-items: center; + gap: calc(12px * var(--hud-scale)); + width: 100%; + padding: calc(12px * var(--hud-scale)) calc(14px * var(--hud-scale)); + border: 1px solid rgba(212, 227, 244, 0.09); + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.05), transparent), + rgba(255, 255, 255, 0.025); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.04); + color: inherit; + text-align: left; + cursor: pointer; + transition: + background 0.18s ease, + border-color 0.18s ease, + transform 0.18s ease; +} + +.earth-search-result:hover, +.earth-search-result.is-active { + border-color: rgba(224, 236, 249, 0.16); + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.06), transparent), + rgba(255, 255, 255, 0.04); + transform: translateY(-1px); +} + +.earth-search-result-icon { + display: inline-flex; + align-items: center; + justify-content: center; + width: calc(32px * var(--hud-scale)); + height: calc(32px * var(--hud-scale)); + border-radius: calc(10px * var(--hud-scale)); + background: rgba(255, 255, 255, 0.06); + border: 1px solid rgba(214, 230, 247, 0.1); + color: var(--hud-accent-strong); +} + +.earth-search-result-icon .material-symbols-rounded { + font-size: calc(18px * var(--hud-scale)); +} + +.earth-search-result-copy { + min-width: 0; + display: flex; + flex-direction: column; + gap: 3px; +} + +.earth-search-result-title { + color: var(--hud-title); + font-size: calc(0.86rem * var(--hud-scale)); + font-weight: 600; + letter-spacing: 0.01em; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.earth-search-result-subtitle { + color: var(--hud-text-soft); + font-size: calc(0.72rem * var(--hud-scale)); + line-height: 1.45; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.earth-search-result-type { + color: var(--hud-text-muted); + font-size: calc(0.68rem * var(--hud-scale)); + letter-spacing: 0.08em; + text-transform: uppercase; + white-space: nowrap; +} + .earth-settings-modal { position: fixed; inset: 0; @@ -694,6 +2287,50 @@ 0 8px 18px rgba(0, 0, 0, 0.2); } +.earth-settings-chip-group { + display: inline-flex; + flex-wrap: wrap; + gap: 8px; + align-self: flex-start; +} + +.earth-settings-chip { + border: 1px solid rgba(212, 227, 244, 0.1); + border-radius: 999px; + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.04), transparent), + rgba(255, 255, 255, 0.025); + color: var(--hud-text-soft); + padding: calc(6px * var(--hud-scale)) calc(12px * var(--hud-scale)); + font: inherit; + font-size: calc(0.7rem * var(--hud-scale)); + font-weight: 600; + letter-spacing: 0.02em; + cursor: pointer; + transition: + background 0.18s ease, + border-color 0.18s ease, + color 0.18s ease, + box-shadow 0.18s ease, + transform 0.18s ease; +} + +.earth-settings-chip:hover { + color: var(--hud-text); + transform: translateY(-1px); +} + +.earth-settings-chip.is-active { + color: var(--hud-title); + border-color: rgba(122, 180, 255, 0.24); + background: + radial-gradient(circle at 35% 22%, rgba(255, 255, 255, 0.22), transparent 58%), + linear-gradient(180deg, rgba(121, 159, 207, 0.2), rgba(72, 101, 139, 0.26)); + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.08), + 0 8px 18px rgba(0, 0, 0, 0.16); +} + .earth-settings-slider { flex: 1 1 auto; width: 100%; @@ -859,6 +2496,12 @@ transform: translateX(calc(16px * var(--hud-scale))); } +.earth-settings-item.is-disabled { + opacity: 0.38; + cursor: not-allowed; + pointer-events: none; +} + @media (max-width: 960px) { .earth-settings-sheet { top: 24px; @@ -870,6 +2513,30 @@ } } +.layout-mode-mobile .earth-search-sheet { + top: calc(8px + var(--safe-top)); + left: 8px; + right: 8px; + width: auto; + max-height: calc(100vh - var(--safe-top) - var(--safe-bottom) - 16px); + transform: translateX(0) translateY(8px) scale(1); +} + +.layout-mode-mobile .earth-search-modal.is-open .earth-search-sheet { + transform: translateX(0) translateY(0) scale(1); +} + +.layout-mode-mobile .earth-settings-sheet { + top: auto; + left: 8px; + right: 8px; + bottom: calc(8px + var(--safe-bottom)); + width: auto; + max-width: none; + max-height: min(78vh, 720px); + border-radius: 24px 24px 18px 18px; +} + /* .earth-left-column layout-expanded rule lives in info-panel.css */ /* .hud-panel-legend layout-expanded rule lives in legend.css */ /* .hud-panel-stats layout-expanded rule lives in earth-stats.css */ diff --git a/frontend/public/earth/css/info-panel.css b/frontend/public/earth/css/info-panel.css index f415dc0d..79f88fbb 100644 --- a/frontend/public/earth/css/info-panel.css +++ b/frontend/public/earth/css/info-panel.css @@ -153,15 +153,28 @@ transition: opacity 0.22s ease, transform 0.22s ease; + visibility: hidden; + user-select: none; + -webkit-user-select: none; } .hud-panel-info.is-visible { opacity: 1; transform: scale(1) translateY(0); pointer-events: auto; + visibility: visible; } -.info-card-cruise-link { +.hud-panel-info.hud-panel-info--anchor-stable { + transform: none; + transition: opacity 0.22s ease; +} + +.hud-panel-info.hud-panel-info--anchor-stable.is-visible { + transform: none; +} + +.callout-connector { position: absolute; inset: 0; width: 100%; @@ -173,7 +186,7 @@ z-index: 49; } -.info-card-cruise-link polyline { +.callout-connector polyline { fill: none; stroke: rgba(255, 255, 255, 0.98); stroke-width: 2.15; @@ -185,7 +198,7 @@ drop-shadow(0 0 6px rgba(8, 20, 36, 0.1)); } -.info-card-cruise-link circle { +.callout-connector circle { fill: rgba(255, 255, 255, 0.98); stroke: rgba(7, 16, 32, 0.72); stroke-width: 1.0; @@ -197,29 +210,29 @@ transform-origin: center; } -.info-card-cruise-link.is-visible { +.callout-connector.is-visible { opacity: 1; } -.info-card-cruise-link.is-animating polyline { - animation: cruiseConnectorDraw 0.42s cubic-bezier(0.22, 1, 0.36, 1) forwards; +.callout-connector.is-animating polyline { + animation: calloutConnectorDraw 0.42s cubic-bezier(0.22, 1, 0.36, 1) forwards; } -.info-card-cruise-link.is-animating circle { +.callout-connector.is-animating circle { opacity: 0; } -.info-card-cruise-link.is-animating circle:first-of-type { - animation: cruiseConnectorNodeIn 0.14s ease forwards; +.callout-connector.is-animating circle:first-of-type { + animation: calloutConnectorNodeIn 0.14s ease forwards; animation-delay: 0.02s; } -.info-card-cruise-link.is-animating circle:last-of-type { - animation: cruiseConnectorNodeIn 0.16s ease forwards; +.callout-connector.is-animating circle:last-of-type { + animation: calloutConnectorNodeIn 0.16s ease forwards; animation-delay: 0.34s; } -@keyframes cruiseConnectorDraw { +@keyframes calloutConnectorDraw { from { stroke-dashoffset: var(--connector-length, 0px); } @@ -228,7 +241,7 @@ } } -@keyframes cruiseConnectorNodeIn { +@keyframes calloutConnectorNodeIn { from { opacity: 0; transform: scale(0.72); @@ -290,6 +303,12 @@ scrollbar-width: thin; scrollbar-color: rgba(160, 186, 216, 0.34) transparent; pointer-events: auto; + user-select: none; + -webkit-user-select: none; +} + +.info-card.compute_unresolved .info-card-content { + max-height: min(calc(330px * var(--hud-scale)), calc(100vh - 180px)); } .info-card-content::-webkit-scrollbar { @@ -327,6 +346,8 @@ cursor: pointer; flex-shrink: 0; transition: color 0.18s ease; + user-select: none; + -webkit-user-select: none; } .info-card-label:hover { @@ -341,6 +362,8 @@ text-align: right; max-width: calc(180px * var(--hud-scale)); word-break: break-word; + user-select: none; + -webkit-user-select: none; } /* Type-specific header accent colors */ @@ -362,8 +385,328 @@ } .info-card.bgp .info-card-header h3 { color: var(--hud-accent-strong); } +.info-card.news .info-card-header { + background: rgba(255, 196, 92, 0.12); + border-bottom-color: rgba(255, 196, 92, 0.16); +} +.info-card.news .info-card-header h3 { color: #ffd77a; } + +.info-card.news .info-card-content { + padding-top: calc(10px * var(--hud-scale)); + padding-bottom: calc(12px * var(--hud-scale)); +} + +.info-card-news-layout { + display: grid; + gap: calc(10px * var(--hud-scale)); +} + +.info-card-news-kicker { + color: rgba(255, 215, 122, 0.82); + font-size: calc(0.62rem * var(--hud-scale)); + font-weight: 700; + letter-spacing: 0.18em; + text-transform: uppercase; +} + +.info-card-news-title { + color: var(--hud-title); + font-size: calc(0.96rem * var(--hud-scale)); + line-height: 1.45; + font-weight: 700; + text-wrap: balance; +} + +.info-card-news-summary-shell { + position: relative; + padding: calc(10px * var(--hud-scale)) calc(12px * var(--hud-scale)); + border: 1px solid rgba(255, 215, 122, 0.12); + background: + linear-gradient(180deg, rgba(255, 215, 122, 0.05), rgba(255, 255, 255, 0.02)), + radial-gradient(circle at top left, rgba(120, 180, 255, 0.08), transparent 56%), + rgba(7, 15, 29, 0.42); + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.05), + 0 10px 28px rgba(0, 0, 0, 0.16); + overflow: hidden; +} + +.info-card-news-summary-shell::before { + content: ""; + position: absolute; + inset: 0; + background: + linear-gradient(90deg, transparent 0%, rgba(122, 180, 255, 0.12) 50%, transparent 100%); + opacity: 0.42; + transform: translateX(-100%); + animation: infoCardNewsScan 3.2s linear infinite; + pointer-events: none; +} + +.info-card-news-summary-label { + color: rgba(188, 212, 238, 0.72); + font-size: calc(0.6rem * var(--hud-scale)); + font-weight: 700; + letter-spacing: 0.16em; + text-transform: uppercase; + margin-bottom: calc(6px * var(--hud-scale)); +} + +.info-card-news-summary { + position: relative; + color: #d7e6f7; + font-size: calc(0.8rem * var(--hud-scale)); + line-height: 1.65; + min-height: calc(4.8em * var(--hud-scale)); + white-space: pre-wrap; + word-break: break-word; +} + +.info-card-news-summary.is-typing::after { + content: ""; + display: inline-block; + width: 0.58em; + height: 1.05em; + margin-left: 0.16em; + vertical-align: -0.14em; + background: linear-gradient(180deg, rgba(255, 215, 122, 0.96), rgba(122, 180, 255, 0.78)); + box-shadow: 0 0 10px rgba(255, 215, 122, 0.28); + animation: infoCardNewsCaret 0.9s steps(1, end) infinite; +} + +@keyframes infoCardNewsScan { + from { + transform: translateX(-100%); + } + to { + transform: translateX(100%); + } +} + +@keyframes infoCardNewsCaret { + 0%, 49% { + opacity: 1; + } + 50%, 100% { + opacity: 0; + } +} + /* ── Layout-expanded: slide left column off-screen ────────────── */ .earth-app.layout-expanded .earth-left-column { transform: translate(calc(-100% + var(--hud-offset)), 0); } + +.layout-mode-mobile .earth-left-column { + top: calc(8px + var(--safe-top)); + left: 8px; + max-width: min(300px, calc(100vw - 16px)); +} + +.layout-mode-mobile .hud-panel-info { + position: fixed; + left: 8px !important; + right: 8px !important; + top: auto !important; + bottom: calc(84px + var(--safe-bottom)) !important; + width: auto; + max-width: none; + max-height: min(58vh, 520px); + z-index: 240; +} + +.layout-mode-mobile .info-card-header { + cursor: default; +} + +.layout-mode-mobile .info-card-content { + max-height: min(46vh, 420px); +} + +.layout-mode-mobile .info-card-property { + flex-direction: column; + align-items: stretch; +} + +.layout-mode-mobile .info-card-value { + max-width: none; + text-align: left; +} + +.info-card-compute-collect { + margin-top: calc(8px * var(--hud-scale)); + padding-top: calc(8px * var(--hud-scale)); + border-top: 1px solid rgba(214, 229, 245, 0.06); + pointer-events: auto; +} + +.info-card-compute-collect-button { + display: inline-flex; + align-items: center; + gap: 4px; + padding: calc(3px * var(--hud-scale)) calc(8px * var(--hud-scale)); + background: transparent; + color: var(--hud-text-soft); + border: 1px solid rgba(214, 229, 245, 0.18); + border-radius: 4px; + cursor: pointer; + font-size: calc(0.7rem * var(--hud-scale)); + letter-spacing: 0.04em; + transition: color 0.15s ease, border-color 0.15s ease, background 0.15s ease; +} + +.info-card-compute-collect-button:hover:not(:disabled) { + color: #c9dcff; + background: rgba(72, 138, 255, 0.14); + border-color: rgba(72, 138, 255, 0.45); +} + +.info-card-compute-collect-button:disabled { + opacity: 0.55; + cursor: progress; +} + +.info-card-compute-collect-button .material-symbols-rounded { + font-size: calc(13px * var(--hud-scale)); +} + +.info-card-compute-collect-status { + margin-top: calc(8px * var(--hud-scale)); + color: var(--hud-text-soft); + font-size: calc(0.7rem * var(--hud-scale)); +} + +.info-card-compute-collect-candidates { + margin-top: calc(6px * var(--hud-scale)); + display: flex; + flex-direction: column; + gap: calc(6px * var(--hud-scale)); +} + +.info-card-compute-candidate { + background: rgba(214, 229, 245, 0.04); + border: 1px solid rgba(214, 229, 245, 0.08); + border-radius: 6px; + padding: calc(6px * var(--hud-scale)) calc(8px * var(--hud-scale)); + font-size: calc(0.7rem * var(--hud-scale)); +} + +.info-card-compute-candidate.is-best { + border-color: rgba(72, 138, 255, 0.5); + background: rgba(72, 138, 255, 0.1); +} + +.info-card-compute-candidate-line { + display: flex; + justify-content: space-between; + gap: 8px; + align-items: center; +} + +.info-card-compute-candidate-precision { + color: #cfe1ff; + font-weight: 600; +} + +.info-card-compute-candidate-preview { + background: transparent; + color: #c9dcff; + border: 1px solid rgba(214, 229, 245, 0.18); + border-radius: 4px; + cursor: pointer; + padding: 2px 6px; + font-size: calc(0.68rem * var(--hud-scale)); +} + +.info-card-compute-candidate-preview:hover { + background: rgba(72, 138, 255, 0.18); +} + +.info-card-unresolved-summary { + display: flex; + align-items: center; + justify-content: space-between; + gap: calc(8px * var(--hud-scale)); + padding: calc(4px * var(--hud-scale)) 0 calc(8px * var(--hud-scale)); + color: var(--hud-text-soft); + font-size: calc(0.72rem * var(--hud-scale)); + line-height: 1.35; +} + +.info-card-unresolved-summary > span { + min-width: 0; +} + +.info-card-unresolved-list { + display: flex; + flex-direction: column; + gap: calc(7px * var(--hud-scale)); +} + +.info-card-unresolved-item { + padding: calc(7px * var(--hud-scale)) calc(8px * var(--hud-scale)); + border: 1px solid rgba(214, 229, 245, 0.08); + border-radius: 6px; + background: rgba(214, 229, 245, 0.035); + pointer-events: auto; +} + +.info-card-unresolved-main { + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + align-items: center; + gap: calc(8px * var(--hud-scale)); +} + +.info-card-unresolved-index { + display: inline-flex; + align-items: center; + justify-content: center; + width: calc(18px * var(--hud-scale)); + height: calc(18px * var(--hud-scale)); + border-radius: 999px; + background: rgba(255, 171, 81, 0.14); + color: #ffd59b; + font-size: calc(0.62rem * var(--hud-scale)); + font-weight: 700; + font-variant-numeric: tabular-nums; +} + +.info-card-unresolved-copy { + min-width: 0; +} + +.info-card-unresolved-name { + overflow: hidden; + color: var(--hud-title); + font-size: calc(0.78rem * var(--hud-scale)); + font-weight: 600; + line-height: 1.3; + text-overflow: ellipsis; + white-space: nowrap; +} + +.info-card-unresolved-meta { + overflow: hidden; + color: var(--hud-text-soft); + font-size: calc(0.64rem * var(--hud-scale)); + line-height: 1.3; + text-overflow: ellipsis; + white-space: nowrap; +} + +.info-card-unresolved-adopt { + color: #ffe0aa; + border-color: rgba(255, 171, 81, 0.32); +} + +.info-card-unresolved-adopt:hover { + background: rgba(255, 171, 81, 0.16); +} + +.info-card-unresolved-empty { + padding: calc(10px * var(--hud-scale)) 0; + color: var(--hud-text-soft); + font-size: calc(0.74rem * var(--hud-scale)); +} diff --git a/frontend/public/earth/css/layer-panel.css b/frontend/public/earth/css/layer-panel.css index 2c9511fe..ef3dc873 100644 --- a/frontend/public/earth/css/layer-panel.css +++ b/frontend/public/earth/css/layer-panel.css @@ -160,15 +160,34 @@ .layer-panel-list { display: flex; flex-direction: column; + max-height: calc(5 * (56px * var(--hud-scale))); + overflow-y: auto; + scrollbar-width: thin; + scrollbar-color: rgba(160, 186, 216, 0.34) transparent; +} + +.layer-panel-list::-webkit-scrollbar { + width: 4px; +} + +.layer-panel-list::-webkit-scrollbar-track { + background: transparent; +} + +.layer-panel-list::-webkit-scrollbar-thumb { + background: linear-gradient(180deg, rgba(210, 225, 242, 0.2), rgba(126, 154, 185, 0.28)); + border-radius: 999px; } .layer-row { + position: relative; display: flex; align-items: center; gap: calc(8px * var(--hud-scale)); padding: calc(9px * var(--hud-scale)) calc(10px * var(--hud-scale)); border-bottom: 1px solid var(--hud-line); transition: background 0.14s ease; + min-height: calc(56px * var(--hud-scale)); } .layer-row:last-child { @@ -253,6 +272,16 @@ opacity: 1; } +.layer-row-toggle.is-disabled { + cursor: not-allowed; + opacity: 0.35; +} + +.layer-row:has(.layer-row-toggle.is-disabled) .layer-row-label, +.layer-row:has(.layer-row-toggle.is-disabled) .layer-row-icon { + opacity: 0.4; +} + /* Thumb */ .layer-row-toggle-track::after { content: ""; @@ -296,6 +325,42 @@ transform: translateX(calc(14px * var(--hud-scale))); } +.layer-row-notification-badge { + appearance: none; + position: absolute; + top: calc(4px * var(--hud-scale)); + left: calc(19px * var(--hud-scale)); + z-index: 2; + display: inline-flex; + align-items: center; + justify-content: center; + min-width: calc(16px * var(--hud-scale)); + height: calc(16px * var(--hud-scale)); + padding: 0 calc(4px * var(--hud-scale)); + border: 1px solid rgba(255, 226, 186, 0.62); + border-radius: 999px; + background: linear-gradient(180deg, rgba(255, 171, 81, 0.96), rgba(213, 78, 54, 0.96)); + box-shadow: + 0 calc(2px * var(--hud-scale)) calc(6px * var(--hud-scale)) rgba(2, 8, 20, 0.42), + 0 0 calc(10px * var(--hud-scale)) rgba(255, 123, 67, 0.34); + color: #fff8e8; + font-size: calc(0.5rem * var(--hud-scale)); + font-weight: 700; + font-variant-numeric: tabular-nums; + line-height: 1; + cursor: pointer; + transition: + filter 0.16s ease, + transform 0.16s ease, + border-color 0.16s ease; +} + +.layer-row-notification-badge:hover { + filter: brightness(1.08); + transform: translateY(calc(-1px * var(--hud-scale))); + border-color: rgba(255, 238, 205, 0.78); +} + @keyframes layer-toggle-loading-track { 0% { background-position: 0% 50%; @@ -317,3 +382,34 @@ /* Layout-expanded: layer panel slides off with .earth-left-column — no individual rule needed since the whole column translates together. */ + +.layout-mode-mobile .hud-panel-layers { + position: fixed; + left: 12px; + right: 12px; + bottom: calc(88px + var(--safe-bottom)); + width: auto; + max-height: min(60vh, 520px); + margin-top: 0; + z-index: 220; + transform: translateY(calc(100% + 28px)); + opacity: 0; + pointer-events: none; + transition: transform 0.24s ease, opacity 0.2s ease; +} + +.layout-mode-mobile .hud-panel-layers.is-mobile-open { + transform: translateY(0); + opacity: 1; + pointer-events: auto; +} + +.layout-mode-mobile .layer-panel-body { + max-height: min(52vh, 460px); + overflow: auto; +} + +.layout-mode-mobile .layer-panel-list { + max-height: none; + overflow-y: visible; +} diff --git a/frontend/public/earth/css/legend.css b/frontend/public/earth/css/legend.css index 8eca724e..07e5730d 100644 --- a/frontend/public/earth/css/legend.css +++ b/frontend/public/earth/css/legend.css @@ -121,6 +121,19 @@ box-shadow: 0 0 4px currentColor; } +.legend-dot--vessel { + width: 0; + height: 0; + border-radius: 0; + background: transparent !important; + border-left: calc(5px * var(--hud-scale)) solid transparent; + border-right: calc(5px * var(--hud-scale)) solid transparent; + border-bottom: calc(13px * var(--hud-scale)) solid currentColor; + color: inherit; + box-shadow: none; + transform: rotate(45deg); +} + .legend-label { color: var(--hud-text); font-size: calc(0.78rem * var(--hud-scale)); @@ -138,3 +151,24 @@ bottom: var(--hud-offset); transform: translate(calc(-100% + var(--hud-offset)), calc(100% - var(--hud-offset))); } + +.layout-mode-mobile .hud-panel-legend { + position: fixed; + left: 8px; + bottom: calc(84px + var(--safe-bottom)); + width: min(172px, calc(100vw - 16px)); + z-index: 205; +} + +.layout-mode-mobile .legend-list { + max-height: min(20vh, 180px); +} + +.layout-mode-mobile.earth-search-open .hud-panel-legend, +.layout-mode-mobile.earth-settings-open .hud-panel-legend, +.layout-mode-mobile.earth-media-open .hud-panel-legend, +.layout-mode-mobile.earth-info-open .hud-panel-legend { + opacity: 0; + pointer-events: none; + transform: translateY(12px); +} diff --git a/frontend/public/earth/css/news-panel.css b/frontend/public/earth/css/news-panel.css index 23941ce3..10189f5f 100644 --- a/frontend/public/earth/css/news-panel.css +++ b/frontend/public/earth/css/news-panel.css @@ -135,6 +135,15 @@ box-shadow: 0 0 0 1px rgba(122, 214, 255, 0.08) inset; } +.news-story-card--cruise { + border-color: rgba(255, 213, 128, 0.42); + background: + linear-gradient(180deg, rgba(255, 248, 220, 0.1), rgba(255, 184, 77, 0.08)); + box-shadow: + 0 0 0 1px rgba(255, 213, 128, 0.18) inset, + 0 0 18px rgba(255, 184, 77, 0.12); +} + .news-story-meta, .news-story-tags { display: flex; diff --git a/frontend/public/earth/css/toolbar.css b/frontend/public/earth/css/toolbar.css index 353be6b4..4e1fbb69 100644 --- a/frontend/public/earth/css/toolbar.css +++ b/frontend/public/earth/css/toolbar.css @@ -24,16 +24,16 @@ } .earth-toolbar { - --toolbar-scale: 1; - --toolbar-orb-size: calc(46px * var(--toolbar-scale)); - --toolbar-hub-size: calc(58px * var(--toolbar-scale)); - --toolbar-arc-width: calc(420px * var(--toolbar-scale)); - --toolbar-arc-height: calc(160px * var(--toolbar-scale)); - --toolbar-inner-arc-width: calc(260px * var(--toolbar-scale)); - --toolbar-inner-arc-height: calc(56px * var(--toolbar-scale)); + --toolbar-scale: var(--initial-toolbar-scale, 1); + --toolbar-orb-size: var(--initial-toolbar-orb-size, calc(46px * var(--toolbar-scale))); + --toolbar-hub-size: var(--initial-toolbar-hub-size, calc(58px * var(--toolbar-scale))); + --toolbar-arc-width: var(--initial-toolbar-arc-width, calc(420px * var(--toolbar-scale))); + --toolbar-arc-height: var(--initial-toolbar-arc-height, calc(160px * var(--toolbar-scale))); + --toolbar-inner-arc-width: var(--initial-toolbar-inner-arc-width, calc(260px * var(--toolbar-scale))); + --toolbar-inner-arc-height: var(--initial-toolbar-inner-arc-height, calc(56px * var(--toolbar-scale))); position: relative; width: min(620px, calc(100vw - 40px)); - height: calc(200px * var(--toolbar-scale)); + height: var(--initial-toolbar-height, calc(200px * var(--toolbar-scale))); display: flex; align-items: center; justify-content: center; @@ -105,6 +105,14 @@ pointer-events: auto; } +.earth-toolbar-orb:has(#layer-action) { + display: none; +} + +.layout-mode-mobile .earth-toolbar-group { + display: none; +} + .earth-toolbar-cluster.is-collapsed .earth-toolbar-orb > * { pointer-events: none; } @@ -147,7 +155,8 @@ height: var(--toolbar-orb-size); min-width: var(--toolbar-orb-size); min-height: var(--toolbar-orb-size); - border-radius: 50%; + aspect-ratio: 1 / 1; + border-radius: 9999px; overflow: hidden; } @@ -156,7 +165,8 @@ height: var(--toolbar-hub-size); min-width: var(--toolbar-hub-size); min-height: var(--toolbar-hub-size); - border-radius: 50%; + aspect-ratio: 1 / 1; + border-radius: 9999px; overflow: hidden; color: var(--hud-title); } diff --git a/frontend/public/earth/css/tv-panel.css b/frontend/public/earth/css/tv-panel.css index 23b5708e..4c26f2d0 100644 --- a/frontend/public/earth/css/tv-panel.css +++ b/frontend/public/earth/css/tv-panel.css @@ -285,6 +285,27 @@ cursor: nesw-resize; } +.layout-mode-mobile .hud-panel-media { + position: fixed; + left: 8px; + right: 8px; + top: calc(8px + var(--safe-top)); + bottom: calc(84px + var(--safe-bottom)); + width: auto; + max-width: none; + max-height: none; + min-width: 0; + z-index: 230; +} + +.layout-mode-mobile .tv-panel-player { + min-height: min(42vh, 360px); +} + +.layout-mode-mobile .tv-panel-edge { + display: none; +} + /* 右下角视觉标记 */ .tv-panel-edge[data-edge="br"]::before { content: ""; diff --git a/frontend/public/earth/data/countries-admin0.min.geojson b/frontend/public/earth/data/countries-admin0.min.geojson new file mode 100644 index 00000000..e8273b89 --- /dev/null +++ b/frontend/public/earth/data/countries-admin0.min.geojson @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"ADMIN":"Fiji","NAME_EN":"Fiji","CONTINENT":"Oceania","ADM0_A3":"FJI","ISO_A3":"FJI","NAME":"Fiji","ISO_A2":"FJ","NAME_ZH":"斐济"},"geometry":{"type":"MultiPolygon","coordinates":[[[[180,-16.067133],[180,-16.555217],[179.364143,-16.801354],[178.725059,-17.012042],[178.596839,-16.63915],[179.096609,-16.433984],[179.413509,-16.379054],[180,-16.067133]]],[[[178.12557,-17.50481],[178.3736,-17.33992],[178.71806,-17.62846],[178.55271,-18.15059],[177.93266,-18.28799],[177.38146,-18.16432],[177.28504,-17.72465],[177.67087,-17.38114],[178.12557,-17.50481]]],[[[-179.79332,-16.020882],[-179.917369,-16.501783],[-180,-16.555217],[-180,-16.067133],[-179.79332,-16.020882]]]]}},{"type":"Feature","properties":{"ADMIN":"United Republic of Tanzania","NAME_EN":"Tanzania","CONTINENT":"Africa","ADM0_A3":"TZA","ISO_A3":"TZA","NAME":"Tanzania","ISO_A2":"TZ","NAME_ZH":"坦桑尼亚"},"geometry":{"type":"Polygon","coordinates":[[[33.903711,-0.95],[34.07262,-1.05982],[37.69869,-3.09699],[37.7669,-3.67712],[39.20222,-4.67677],[38.74054,-5.90895],[38.79977,-6.47566],[39.44,-6.84],[39.47,-7.1],[39.19469,-7.7039],[39.25203,-8.00781],[39.18652,-8.48551],[39.53574,-9.11237],[39.9496,-10.0984],[40.316586,-10.317098],[40.31659,-10.3171],[39.521,-10.89688],[38.427557,-11.285202],[37.82764,-11.26879],[37.47129,-11.56876],[36.775151,-11.594537],[36.514082,-11.720938],[35.312398,-11.439146],[34.559989,-11.52002],[34.28,-10.16],[33.940838,-9.693674],[33.73972,-9.41715],[32.759375,-9.230599],[32.191865,-8.930359],[31.556348,-8.762049],[31.157751,-8.594579],[30.74001,-8.340006],[30.740015,-8.340007],[30.199997,-7.079981],[29.620032,-6.520015],[29.419993,-5.939999],[29.519987,-5.419979],[29.339998,-4.499983],[29.753512,-4.452389],[30.11632,-4.09012],[30.50554,-3.56858],[30.75224,-3.35931],[30.74301,-3.03431],[30.52766,-2.80762],[30.469674,-2.413855],[30.46967,-2.41383],[30.758309,-2.28725],[30.816135,-1.698914],[30.419105,-1.134659],[30.76986,-1.01455],[31.86617,-1.02736],[33.903711,-0.95]]]}},{"type":"Feature","properties":{"ADMIN":"Western Sahara","NAME_EN":"Western Sahara","CONTINENT":"Africa","ADM0_A3":"SAH","ISO_A3":"ESH","NAME":"W. Sahara","ISO_A2":"EH","NAME_ZH":"西撒哈拉"},"geometry":{"type":"Polygon","coordinates":[[[-8.66559,27.656426],[-8.665124,27.589479],[-8.6844,27.395744],[-8.687294,25.881056],[-11.969419,25.933353],[-11.937224,23.374594],[-12.874222,23.284832],[-13.118754,22.77122],[-12.929102,21.327071],[-16.845194,21.333323],[-17.063423,20.999752],[-17.020428,21.42231],[-17.002962,21.420734],[-14.750955,21.5006],[-14.630833,21.86094],[-14.221168,22.310163],[-13.89111,23.691009],[-12.500963,24.770116],[-12.030759,26.030866],[-11.71822,26.104092],[-11.392555,26.883424],[-10.551263,26.990808],[-10.189424,26.860945],[-9.735343,26.860945],[-9.413037,27.088476],[-8.794884,27.120696],[-8.817828,27.656426],[-8.66559,27.656426]]]}},{"type":"Feature","properties":{"ADMIN":"Canada","NAME_EN":"Canada","CONTINENT":"North America","ADM0_A3":"CAN","ISO_A3":"CAN","NAME":"Canada","ISO_A2":"CA","NAME_ZH":"加拿大"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-122.84,49],[-122.97421,49.002538],[-124.91024,49.98456],[-125.62461,50.41656],[-127.43561,50.83061],[-127.99276,51.71583],[-127.85032,52.32961],[-129.12979,52.75538],[-129.30523,53.56159],[-130.51497,54.28757],[-130.536109,54.802754],[-130.53611,54.80278],[-129.98,55.285],[-130.00778,55.91583],[-131.70781,56.55212],[-132.73042,57.69289],[-133.35556,58.41028],[-134.27111,58.86111],[-134.945,59.27056],[-135.47583,59.78778],[-136.47972,59.46389],[-137.4525,58.905],[-138.34089,59.56211],[-139.039,60],[-140.013,60.27682],[-140.99778,60.30639],[-140.9925,66.00003],[-140.986,69.712],[-140.985988,69.711998],[-139.12052,69.47102],[-137.54636,68.99002],[-136.50358,68.89804],[-135.62576,69.31512],[-134.41464,69.62743],[-132.92925,69.50534],[-131.43136,69.94451],[-129.79471,70.19369],[-129.10773,69.77927],[-128.36156,70.01286],[-128.13817,70.48384],[-127.44712,70.37721],[-125.75632,69.48058],[-124.42483,70.1584],[-124.28968,69.39969],[-123.06108,69.56372],[-122.6835,69.85553],[-121.47226,69.79778],[-119.94288,69.37786],[-117.60268,69.01128],[-116.22643,68.84151],[-115.2469,68.90591],[-113.89794,68.3989],[-115.30489,67.90261],[-113.49727,67.68815],[-110.798,67.80612],[-109.94619,67.98104],[-108.8802,67.38144],[-107.79239,67.88736],[-108.81299,68.31164],[-108.16721,68.65392],[-106.95,68.7],[-106.15,68.8],[-105.34282,68.56122],[-104.33791,68.018],[-103.22115,68.09775],[-101.45433,67.64689],[-99.90195,67.80566],[-98.4432,67.78165],[-98.5586,68.40394],[-97.66948,68.57864],[-96.11991,68.23939],[-96.12588,67.29338],[-95.48943,68.0907],[-94.685,68.06383],[-94.23282,69.06903],[-95.30408,69.68571],[-96.47131,70.08976],[-96.39115,71.19482],[-95.2088,71.92053],[-93.88997,71.76015],[-92.87818,71.31869],[-91.51964,70.19129],[-92.40692,69.69997],[-90.5471,69.49766],[-90.55151,68.47499],[-89.21515,69.25873],[-88.01966,68.61508],[-88.31749,67.87338],[-87.35017,67.19872],[-86.30607,67.92146],[-85.57664,68.78456],[-85.52197,69.88211],[-84.10081,69.80539],[-82.62258,69.65826],[-81.28043,69.16202],[-81.2202,68.66567],[-81.96436,68.13253],[-81.25928,67.59716],[-81.38653,67.11078],[-83.34456,66.41154],[-84.73542,66.2573],[-85.76943,66.55833],[-86.0676,66.05625],[-87.03143,65.21297],[-87.32324,64.77563],[-88.48296,64.09897],[-89.91444,64.03273],[-90.70398,63.61017],[-90.77004,62.96021],[-91.93342,62.83508],[-93.15698,62.02469],[-94.24153,60.89865],[-94.62931,60.11021],[-94.6846,58.94882],[-93.21502,58.78212],[-92.76462,57.84571],[-92.29703,57.08709],[-90.89769,57.28468],[-89.03953,56.85172],[-88.03978,56.47162],[-87.32421,55.99914],[-86.07121,55.72383],[-85.01181,55.3026],[-83.36055,55.24489],[-82.27285,55.14832],[-82.4362,54.28227],[-82.12502,53.27703],[-81.40075,52.15788],[-79.91289,51.20842],[-79.14301,51.53393],[-78.60191,52.56208],[-79.12421,54.14145],[-79.82958,54.66772],[-78.22874,55.13645],[-77.0956,55.83741],[-76.54137,56.53423],[-76.62319,57.20263],[-77.30226,58.05209],[-78.51688,58.80458],[-77.33676,59.85261],[-77.77272,60.75788],[-78.10687,62.31964],[-77.41067,62.55053],[-75.69621,62.2784],[-74.6682,62.18111],[-73.83988,62.4438],[-72.90853,62.10507],[-71.67708,61.52535],[-71.37369,61.13717],[-69.59042,61.06141],[-69.62033,60.22125],[-69.2879,58.95736],[-68.37455,58.80106],[-67.64976,58.21206],[-66.20178,58.76731],[-65.24517,59.87071],[-64.58352,60.33558],[-63.80475,59.4426],[-62.50236,58.16708],[-61.39655,56.96745],[-61.79866,56.33945],[-60.46853,55.77548],[-59.56962,55.20407],[-57.97508,54.94549],[-57.3332,54.6265],[-56.93689,53.78032],[-56.15811,53.64749],[-55.75632,53.27036],[-55.68338,52.14664],[-56.40916,51.7707],[-57.12691,51.41972],[-58.77482,51.0643],[-60.03309,50.24277],[-61.72366,50.08046],[-63.86251,50.29099],[-65.36331,50.2982],[-66.39905,50.22897],[-67.23631,49.51156],[-68.51114,49.06836],[-69.95362,47.74488],[-71.10458,46.82171],[-70.25522,46.98606],[-68.65,48.3],[-66.55243,49.1331],[-65.05626,49.23278],[-64.17099,48.74248],[-65.11545,48.07085],[-64.79854,46.99297],[-64.47219,46.23849],[-63.17329,45.73902],[-61.52072,45.88377],[-60.51815,47.00793],[-60.4486,46.28264],[-59.80287,45.9204],[-61.03988,45.26525],[-63.25471,44.67014],[-64.24656,44.26553],[-65.36406,43.54523],[-66.1234,43.61867],[-66.16173,44.46512],[-64.42549,45.29204],[-66.02605,45.25931],[-67.13741,45.13753],[-67.79134,45.70281],[-67.79046,47.06636],[-68.23444,47.35486],[-68.905,47.185],[-69.237216,47.447781],[-69.99997,46.69307],[-70.305,45.915],[-70.66,45.46],[-71.08482,45.30524],[-71.405,45.255],[-71.50506,45.0082],[-73.34783,45.00738],[-74.867,45.00048],[-75.31821,44.81645],[-76.375,44.09631],[-76.5,44.018459],[-76.820034,43.628784],[-77.737885,43.629056],[-78.72028,43.625089],[-79.171674,43.466339],[-79.01,43.27],[-78.92,42.965],[-78.939362,42.863611],[-80.247448,42.3662],[-81.277747,42.209026],[-82.439278,41.675105],[-82.690089,41.675105],[-83.02981,41.832796],[-83.142,41.975681],[-83.12,42.08],[-82.9,42.43],[-82.43,42.98],[-82.137642,43.571088],[-82.337763,44.44],[-82.550925,45.347517],[-83.592851,45.816894],[-83.469551,45.994686],[-83.616131,46.116927],[-83.890765,46.116927],[-84.091851,46.275419],[-84.14212,46.512226],[-84.3367,46.40877],[-84.6049,46.4396],[-84.543749,46.538684],[-84.779238,46.637102],[-84.87608,46.900083],[-85.652363,47.220219],[-86.461991,47.553338],[-87.439793,47.94],[-88.378114,48.302918],[-89.272917,48.019808],[-89.6,48.01],[-90.83,48.27],[-91.64,48.14],[-92.61,48.45],[-93.63087,48.60926],[-94.32914,48.67074],[-94.64,48.84],[-94.81758,49.38905],[-95.15609,49.38425],[-95.15907,49],[-97.22872,49.0007],[-100.65,49],[-104.04826,48.99986],[-107.05,49],[-110.05,49],[-113,49],[-116.04818,49],[-117.03121,49],[-120,49],[-122.84,49]]],[[[-83.99367,62.4528],[-83.25048,62.91409],[-81.87699,62.90458],[-81.89825,62.7108],[-83.06857,62.15922],[-83.77462,62.18231],[-83.99367,62.4528]]],[[[-79.775833,72.802902],[-80.876099,73.333183],[-80.833885,73.693184],[-80.353058,73.75972],[-78.064438,73.651932],[-76.34,73.102685],[-76.251404,72.826385],[-77.314438,72.855545],[-78.39167,72.876656],[-79.486252,72.742203],[-79.775833,72.802902]]],[[[-80.315395,62.085565],[-79.92939,62.3856],[-79.52002,62.36371],[-79.26582,62.158675],[-79.65752,61.63308],[-80.09956,61.7181],[-80.36215,62.01649],[-80.315395,62.085565]]],[[[-93.612756,74.979997],[-94.156909,74.592347],[-95.608681,74.666864],[-96.820932,74.927623],[-96.288587,75.377828],[-94.85082,75.647218],[-93.977747,75.29649],[-93.612756,74.979997]]],[[[-93.840003,77.519997],[-94.295608,77.491343],[-96.169654,77.555111],[-96.436304,77.834629],[-94.422577,77.820005],[-93.720656,77.634331],[-93.840003,77.519997]]],[[[-96.754399,78.765813],[-95.559278,78.418315],[-95.830295,78.056941],[-97.309843,77.850597],[-98.124289,78.082857],[-98.552868,78.458105],[-98.631984,78.87193],[-97.337231,78.831984],[-96.754399,78.765813]]],[[[-88.15035,74.392307],[-89.764722,74.515555],[-92.422441,74.837758],[-92.768285,75.38682],[-92.889906,75.882655],[-93.893824,76.319244],[-95.962457,76.441381],[-97.121379,76.751078],[-96.745123,77.161389],[-94.684086,77.097878],[-93.573921,76.776296],[-91.605023,76.778518],[-90.741846,76.449597],[-90.969661,76.074013],[-89.822238,75.847774],[-89.187083,75.610166],[-87.838276,75.566189],[-86.379192,75.482421],[-84.789625,75.699204],[-82.753445,75.784315],[-81.128531,75.713983],[-80.057511,75.336849],[-79.833933,74.923127],[-80.457771,74.657304],[-81.948843,74.442459],[-83.228894,74.564028],[-86.097452,74.410032],[-88.15035,74.392307]]],[[[-111.264443,78.152956],[-109.854452,77.996325],[-110.186938,77.697015],[-112.051191,77.409229],[-113.534279,77.732207],[-112.724587,78.05105],[-111.264443,78.152956]]],[[[-110.963661,78.804441],[-109.663146,78.601973],[-110.881314,78.40692],[-112.542091,78.407902],[-112.525891,78.550555],[-111.50001,78.849994],[-110.963661,78.804441]]],[[[-55.600218,51.317075],[-56.134036,50.68701],[-56.795882,49.812309],[-56.143105,50.150117],[-55.471492,49.935815],[-55.822401,49.587129],[-54.935143,49.313011],[-54.473775,49.556691],[-53.476549,49.249139],[-53.786014,48.516781],[-53.086134,48.687804],[-52.958648,48.157164],[-52.648099,47.535548],[-53.069158,46.655499],[-53.521456,46.618292],[-54.178936,46.807066],[-53.961869,47.625207],[-54.240482,47.752279],[-55.400773,46.884994],[-55.997481,46.91972],[-55.291219,47.389562],[-56.250799,47.632545],[-57.325229,47.572807],[-59.266015,47.603348],[-59.419494,47.899454],[-58.796586,48.251525],[-59.231625,48.523188],[-58.391805,49.125581],[-57.35869,50.718274],[-56.73865,51.287438],[-55.870977,51.632094],[-55.406974,51.588273],[-55.600218,51.317075]]],[[[-83.882626,65.109618],[-82.787577,64.766693],[-81.642014,64.455136],[-81.55344,63.979609],[-80.817361,64.057486],[-80.103451,63.725981],[-80.99102,63.411246],[-82.547178,63.651722],[-83.108798,64.101876],[-84.100417,63.569712],[-85.523405,63.052379],[-85.866769,63.637253],[-87.221983,63.541238],[-86.35276,64.035833],[-86.224886,64.822917],[-85.883848,65.738778],[-85.161308,65.657285],[-84.975764,65.217518],[-84.464012,65.371772],[-83.882626,65.109618]]],[[[-78.770639,72.352173],[-77.824624,72.749617],[-75.605845,72.243678],[-74.228616,71.767144],[-74.099141,71.33084],[-72.242226,71.556925],[-71.200015,70.920013],[-68.786054,70.525024],[-67.91497,70.121948],[-66.969033,69.186087],[-68.805123,68.720198],[-66.449866,68.067163],[-64.862314,67.847539],[-63.424934,66.928473],[-61.851981,66.862121],[-62.163177,66.160251],[-63.918444,64.998669],[-65.14886,65.426033],[-66.721219,66.388041],[-68.015016,66.262726],[-68.141287,65.689789],[-67.089646,65.108455],[-65.73208,64.648406],[-65.320168,64.382737],[-64.669406,63.392927],[-65.013804,62.674185],[-66.275045,62.945099],[-68.783186,63.74567],[-67.369681,62.883966],[-66.328297,62.280075],[-66.165568,61.930897],[-68.877367,62.330149],[-71.023437,62.910708],[-72.235379,63.397836],[-71.886278,63.679989],[-73.378306,64.193963],[-74.834419,64.679076],[-74.818503,64.389093],[-77.70998,64.229542],[-78.555949,64.572906],[-77.897281,65.309192],[-76.018274,65.326969],[-73.959795,65.454765],[-74.293883,65.811771],[-73.944912,66.310578],[-72.651167,67.284576],[-72.92606,67.726926],[-73.311618,68.069437],[-74.843307,68.554627],[-76.869101,68.894736],[-76.228649,69.147769],[-77.28737,69.76954],[-78.168634,69.826488],[-78.957242,70.16688],[-79.492455,69.871808],[-81.305471,69.743185],[-84.944706,69.966634],[-87.060003,70.260001],[-88.681713,70.410741],[-89.51342,70.762038],[-88.467721,71.218186],[-89.888151,71.222552],[-90.20516,72.235074],[-89.436577,73.129464],[-88.408242,73.537889],[-85.826151,73.803816],[-86.562179,73.157447],[-85.774371,72.534126],[-84.850112,73.340278],[-82.31559,73.750951],[-80.600088,72.716544],[-80.748942,72.061907],[-78.770639,72.352173]]],[[[-94.503658,74.134907],[-92.420012,74.100025],[-90.509793,73.856732],[-92.003965,72.966244],[-93.196296,72.771992],[-94.269047,72.024596],[-95.409856,72.061881],[-96.033745,72.940277],[-96.018268,73.43743],[-95.495793,73.862417],[-94.503658,74.134907]]],[[[-122.854924,76.116543],[-122.854925,76.116543],[-121.157535,76.864508],[-119.103939,77.51222],[-117.570131,77.498319],[-116.198587,77.645287],[-116.335813,76.876962],[-117.106051,76.530032],[-118.040412,76.481172],[-119.899318,76.053213],[-121.499995,75.900019],[-122.854924,76.116543]]],[[[-132.710008,54.040009],[-131.74999,54.120004],[-132.04948,52.984621],[-131.179043,52.180433],[-131.57783,52.182371],[-132.180428,52.639707],[-132.549992,53.100015],[-133.054611,53.411469],[-133.239664,53.85108],[-133.180004,54.169975],[-132.710008,54.040009]]],[[[-105.492289,79.301594],[-103.529282,79.165349],[-100.825158,78.800462],[-100.060192,78.324754],[-99.670939,77.907545],[-101.30394,78.018985],[-102.949809,78.343229],[-105.176133,78.380332],[-104.210429,78.67742],[-105.41958,78.918336],[-105.492289,79.301594]]],[[[-123.510002,48.510011],[-124.012891,48.370846],[-125.655013,48.825005],[-125.954994,49.179996],[-126.850004,49.53],[-127.029993,49.814996],[-128.059336,49.994959],[-128.444584,50.539138],[-128.358414,50.770648],[-127.308581,50.552574],[-126.695001,50.400903],[-125.755007,50.295018],[-125.415002,49.950001],[-124.920768,49.475275],[-123.922509,49.062484],[-123.510002,48.510011]]],[[[-121.53788,74.44893],[-120.10978,74.24135],[-117.55564,74.18577],[-116.58442,73.89607],[-115.51081,73.47519],[-116.76794,73.22292],[-119.22,72.52],[-120.46,71.82],[-120.46,71.383602],[-123.09219,70.90164],[-123.62,71.34],[-125.928949,71.868688],[-125.5,72.292261],[-124.80729,73.02256],[-123.94,73.68],[-124.91775,74.29275],[-121.53788,74.44893]]],[[[-107.81943,75.84552],[-106.92893,76.01282],[-105.881,75.9694],[-105.70498,75.47951],[-106.31347,75.00527],[-109.7,74.85],[-112.22307,74.41696],[-113.74381,74.39427],[-113.87135,74.72029],[-111.79421,75.1625],[-116.31221,75.04343],[-117.7104,75.2222],[-116.34602,76.19903],[-115.40487,76.47887],[-112.59056,76.14134],[-110.81422,75.54919],[-109.0671,75.47321],[-110.49726,76.42982],[-109.5811,76.79417],[-108.54859,76.67832],[-108.21141,76.20168],[-107.81943,75.84552]]],[[[-106.52259,73.07601],[-105.40246,72.67259],[-104.77484,71.6984],[-104.46476,70.99297],[-102.78537,70.49776],[-100.98078,70.02432],[-101.08929,69.58447],[-102.73116,69.50402],[-102.09329,69.11962],[-102.43024,68.75282],[-104.24,68.91],[-105.96,69.18],[-107.12254,69.11922],[-109,68.78],[-111.534149,68.630059],[-113.3132,68.53554],[-113.85496,69.00744],[-115.22,69.28],[-116.10794,69.16821],[-117.34,69.96],[-116.67473,70.06655],[-115.13112,70.2373],[-113.72141,70.19237],[-112.4161,70.36638],[-114.35,70.6],[-116.48684,70.52045],[-117.9048,70.54056],[-118.43238,70.9092],[-116.11311,71.30918],[-117.65568,71.2952],[-119.40199,71.55859],[-118.56267,72.30785],[-117.86642,72.70594],[-115.18909,73.31459],[-114.16717,73.12145],[-114.66634,72.65277],[-112.44102,72.9554],[-111.05039,72.4504],[-109.92035,72.96113],[-109.00654,72.63335],[-108.18835,71.65089],[-107.68599,72.06548],[-108.39639,73.08953],[-107.51645,73.23598],[-106.52259,73.07601]]],[[[-100.43836,72.70588],[-101.54,73.36],[-100.35642,73.84389],[-99.16387,73.63339],[-97.38,73.76],[-97.12,73.47],[-98.05359,72.99052],[-96.54,72.56],[-96.72,71.66],[-98.35966,71.27285],[-99.32286,71.35639],[-100.01482,71.73827],[-102.5,72.51],[-102.48,72.83],[-100.43836,72.70588]]],[[[-106.6,73.6],[-105.26,73.64],[-104.5,73.42],[-105.38,72.76],[-106.94,73.46],[-106.6,73.6]]],[[[-98.5,76.72],[-97.735585,76.25656],[-97.704415,75.74344],[-98.16,75],[-99.80874,74.89744],[-100.88366,75.05736],[-100.86292,75.64075],[-102.50209,75.5638],[-102.56552,76.3366],[-101.48973,76.30537],[-99.98349,76.64634],[-98.57699,76.58859],[-98.5,76.72]]],[[[-96.01644,80.60233],[-95.32345,80.90729],[-94.29843,80.97727],[-94.73542,81.20646],[-92.40984,81.25739],[-91.13289,80.72345],[-89.45,80.509322],[-87.81,80.32],[-87.02,79.66],[-85.81435,79.3369],[-87.18756,79.0393],[-89.03535,78.28723],[-90.80436,78.21533],[-92.87669,78.34333],[-93.95116,78.75099],[-93.93574,79.11373],[-93.14524,79.3801],[-94.974,79.37248],[-96.07614,79.70502],[-96.70972,80.15777],[-96.01644,80.60233]]],[[[-91.58702,81.89429],[-90.1,82.085],[-88.93227,82.11751],[-86.97024,82.27961],[-85.5,82.652273],[-84.260005,82.6],[-83.18,82.32],[-82.42,82.86],[-81.1,83.02],[-79.30664,83.13056],[-76.25,83.172059],[-75.71878,83.06404],[-72.83153,83.23324],[-70.665765,83.169781],[-68.5,83.106322],[-65.82735,83.02801],[-63.68,82.9],[-61.85,82.6286],[-61.89388,82.36165],[-64.334,81.92775],[-66.75342,81.72527],[-67.65755,81.50141],[-65.48031,81.50657],[-67.84,80.9],[-69.4697,80.61683],[-71.18,79.8],[-73.2428,79.63415],[-73.88,79.430162],[-76.90773,79.32309],[-75.52924,79.19766],[-76.22046,79.01907],[-75.39345,78.52581],[-76.34354,78.18296],[-77.88851,77.89991],[-78.36269,77.50859],[-79.75951,77.20968],[-79.61965,76.98336],[-77.91089,77.022045],[-77.88911,76.777955],[-80.56125,76.17812],[-83.17439,76.45403],[-86.11184,76.29901],[-87.6,76.42],[-89.49068,76.47239],[-89.6161,76.95213],[-87.76739,77.17833],[-88.26,77.9],[-87.65,77.970222],[-84.97634,77.53873],[-86.34,78.18],[-87.96192,78.37181],[-87.15198,78.75867],[-85.37868,78.9969],[-85.09495,79.34543],[-86.50734,79.73624],[-86.93179,80.25145],[-84.19844,80.20836],[-83.408696,80.1],[-81.84823,80.46442],[-84.1,80.58],[-87.59895,80.51627],[-89.36663,80.85569],[-90.2,81.26],[-91.36786,81.5531],[-91.58702,81.89429]]],[[[-75.21597,67.44425],[-75.86588,67.14886],[-76.98687,67.09873],[-77.2364,67.58809],[-76.81166,68.14856],[-75.89521,68.28721],[-75.1145,68.01036],[-75.10333,67.58202],[-75.21597,67.44425]]],[[[-96.257401,69.49003],[-95.647681,69.10769],[-96.269521,68.75704],[-97.617401,69.06003],[-98.431801,68.9507],[-99.797401,69.40003],[-98.917401,69.71003],[-98.218261,70.14354],[-97.157401,69.86003],[-96.557401,69.68003],[-96.257401,69.49003]]],[[[-64.51912,49.87304],[-64.17322,49.95718],[-62.85829,49.70641],[-61.835585,49.28855],[-61.806305,49.10506],[-62.29318,49.08717],[-63.58926,49.40069],[-64.51912,49.87304]]],[[[-64.01486,47.03601],[-63.6645,46.55001],[-62.9393,46.41587],[-62.01208,46.44314],[-62.50391,46.03339],[-62.87433,45.96818],[-64.1428,46.39265],[-64.39261,46.72747],[-64.01486,47.03601]]]]}},{"type":"Feature","properties":{"ADMIN":"United States of America","NAME_EN":"United States of America","CONTINENT":"North America","ADM0_A3":"USA","ISO_A3":"USA","NAME":"United States of America","ISO_A2":"US","NAME_ZH":"美国"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-122.84,49],[-120,49],[-117.03121,49],[-116.04818,49],[-113,49],[-110.05,49],[-107.05,49],[-104.04826,48.99986],[-100.65,49],[-97.22872,49.0007],[-95.15907,49],[-95.15609,49.38425],[-94.81758,49.38905],[-94.64,48.84],[-94.32914,48.67074],[-93.63087,48.60926],[-92.61,48.45],[-91.64,48.14],[-90.83,48.27],[-89.6,48.01],[-89.272917,48.019808],[-88.378114,48.302918],[-87.439793,47.94],[-86.461991,47.553338],[-85.652363,47.220219],[-84.87608,46.900083],[-84.779238,46.637102],[-84.543749,46.538684],[-84.6049,46.4396],[-84.3367,46.40877],[-84.14212,46.512226],[-84.091851,46.275419],[-83.890765,46.116927],[-83.616131,46.116927],[-83.469551,45.994686],[-83.592851,45.816894],[-82.550925,45.347517],[-82.337763,44.44],[-82.137642,43.571088],[-82.43,42.98],[-82.9,42.43],[-83.12,42.08],[-83.142,41.975681],[-83.02981,41.832796],[-82.690089,41.675105],[-82.439278,41.675105],[-81.277747,42.209026],[-80.247448,42.3662],[-78.939362,42.863611],[-78.92,42.965],[-79.01,43.27],[-79.171674,43.466339],[-78.72028,43.625089],[-77.737885,43.629056],[-76.820034,43.628784],[-76.5,44.018459],[-76.375,44.09631],[-75.31821,44.81645],[-74.867,45.00048],[-73.34783,45.00738],[-71.50506,45.0082],[-71.405,45.255],[-71.08482,45.30524],[-70.66,45.46],[-70.305,45.915],[-69.99997,46.69307],[-69.237216,47.447781],[-68.905,47.185],[-68.23444,47.35486],[-67.79046,47.06636],[-67.79134,45.70281],[-67.13741,45.13753],[-66.96466,44.8097],[-68.03252,44.3252],[-69.06,43.98],[-70.11617,43.68405],[-70.645476,43.090238],[-70.81489,42.8653],[-70.825,42.335],[-70.495,41.805],[-70.08,41.78],[-70.185,42.145],[-69.88497,41.92283],[-69.96503,41.63717],[-70.64,41.475],[-71.12039,41.49445],[-71.86,41.32],[-72.295,41.27],[-72.87643,41.22065],[-73.71,40.931102],[-72.24126,41.11948],[-71.945,40.93],[-73.345,40.63],[-73.982,40.628],[-73.952325,40.75075],[-74.25671,40.47351],[-73.96244,40.42763],[-74.17838,39.70926],[-74.90604,38.93954],[-74.98041,39.1964],[-75.20002,39.24845],[-75.52805,39.4985],[-75.32,38.96],[-75.071835,38.782032],[-75.05673,38.40412],[-75.37747,38.01551],[-75.94023,37.21689],[-76.03127,37.2566],[-75.72205,37.93705],[-76.23287,38.319215],[-76.35,39.15],[-76.542725,38.717615],[-76.32933,38.08326],[-76.989998,38.239992],[-76.30162,37.917945],[-76.25874,36.9664],[-75.9718,36.89726],[-75.86804,36.55125],[-75.72749,35.55074],[-76.36318,34.80854],[-77.397635,34.51201],[-78.05496,33.92547],[-78.55435,33.86133],[-79.06067,33.49395],[-79.20357,33.15839],[-80.301325,32.509355],[-80.86498,32.0333],[-81.33629,31.44049],[-81.49042,30.72999],[-81.31371,30.03552],[-80.98,29.18],[-80.535585,28.47213],[-80.53,28.04],[-80.056539,26.88],[-80.088015,26.205765],[-80.13156,25.816775],[-80.38103,25.20616],[-80.68,25.08],[-81.17213,25.20126],[-81.33,25.64],[-81.71,25.87],[-82.24,26.73],[-82.70515,27.49504],[-82.85526,27.88624],[-82.65,28.55],[-82.93,29.1],[-83.70959,29.93656],[-84.1,30.09],[-85.10882,29.63615],[-85.28784,29.68612],[-85.7731,30.15261],[-86.4,30.4],[-87.53036,30.27433],[-88.41782,30.3849],[-89.18049,30.31598],[-89.593831,30.159994],[-89.413735,29.89419],[-89.43,29.48864],[-89.21767,29.29108],[-89.40823,29.15961],[-89.77928,29.30714],[-90.15463,29.11743],[-90.880225,29.148535],[-91.626785,29.677],[-92.49906,29.5523],[-93.22637,29.78375],[-93.84842,29.71363],[-94.69,29.48],[-95.60026,28.73863],[-96.59404,28.30748],[-97.14,27.83],[-97.37,27.38],[-97.38,26.69],[-97.33,26.21],[-97.14,25.87],[-97.53,25.84],[-98.24,26.06],[-99.02,26.37],[-99.3,26.84],[-99.52,27.54],[-100.11,28.11],[-100.45584,28.69612],[-100.9576,29.38071],[-101.6624,29.7793],[-102.48,29.76],[-103.11,28.97],[-103.94,29.27],[-104.45697,29.57196],[-104.70575,30.12173],[-105.03737,30.64402],[-105.63159,31.08383],[-106.1429,31.39995],[-106.50759,31.75452],[-108.24,31.754854],[-108.24194,31.34222],[-109.035,31.34194],[-111.02361,31.33472],[-113.30498,32.03914],[-114.815,32.52528],[-114.72139,32.72083],[-115.99135,32.61239],[-117.12776,32.53534],[-117.295938,33.046225],[-117.944,33.621236],[-118.410602,33.740909],[-118.519895,34.027782],[-119.081,34.078],[-119.438841,34.348477],[-120.36778,34.44711],[-120.62286,34.60855],[-120.74433,35.15686],[-121.71457,36.16153],[-122.54747,37.55176],[-122.51201,37.78339],[-122.95319,38.11371],[-123.7272,38.95166],[-123.86517,39.76699],[-124.39807,40.3132],[-124.17886,41.14202],[-124.2137,41.99964],[-124.53284,42.76599],[-124.14214,43.70838],[-124.020535,44.615895],[-123.89893,45.52341],[-124.079635,46.86475],[-124.39567,47.72017],[-124.68721,48.184433],[-124.566101,48.379715],[-123.12,48.04],[-122.58736,47.096],[-122.34,47.36],[-122.5,48.18],[-122.84,49]]],[[[-155.40214,20.07975],[-155.22452,19.99302],[-155.06226,19.8591],[-154.80741,19.50871],[-154.83147,19.45328],[-155.22217,19.23972],[-155.54211,19.08348],[-155.68817,18.91619],[-155.93665,19.05939],[-155.90806,19.33888],[-156.07347,19.70294],[-156.02368,19.81422],[-155.85008,19.97729],[-155.91907,20.17395],[-155.86108,20.26721],[-155.78505,20.2487],[-155.40214,20.07975]]],[[[-155.99566,20.76404],[-156.07926,20.64397],[-156.41445,20.57241],[-156.58673,20.783],[-156.70167,20.8643],[-156.71055,20.92676],[-156.61258,21.01249],[-156.25711,20.91745],[-155.99566,20.76404]]],[[[-156.75824,21.17684],[-156.78933,21.06873],[-157.32521,21.09777],[-157.25027,21.21958],[-156.75824,21.17684]]],[[[-158.0252,21.71696],[-157.94161,21.65272],[-157.65283,21.32217],[-157.70703,21.26442],[-157.7786,21.27729],[-158.12667,21.31244],[-158.2538,21.53919],[-158.29265,21.57912],[-158.0252,21.71696]]],[[[-159.36569,22.21494],[-159.34512,21.982],[-159.46372,21.88299],[-159.80051,22.06533],[-159.74877,22.1382],[-159.5962,22.23618],[-159.36569,22.21494]]],[[[-166.467792,60.38417],[-165.67443,60.293607],[-165.579164,59.909987],[-166.19277,59.754441],[-166.848337,59.941406],[-167.455277,60.213069],[-166.467792,60.38417]]],[[[-153.228729,57.968968],[-152.564791,57.901427],[-152.141147,57.591059],[-153.006314,57.115842],[-154.00509,56.734677],[-154.516403,56.992749],[-154.670993,57.461196],[-153.76278,57.816575],[-153.228729,57.968968]]],[[[-140.985988,69.711998],[-140.986,69.712],[-140.9925,66.00003],[-140.99778,60.30639],[-140.013,60.27682],[-139.039,60],[-138.34089,59.56211],[-137.4525,58.905],[-136.47972,59.46389],[-135.47583,59.78778],[-134.945,59.27056],[-134.27111,58.86111],[-133.35556,58.41028],[-132.73042,57.69289],[-131.70781,56.55212],[-130.00778,55.91583],[-129.98,55.285],[-130.53611,54.80278],[-130.536109,54.802754],[-130.53611,54.802753],[-131.085818,55.178906],[-131.967211,55.497776],[-132.250011,56.369996],[-133.539181,57.178887],[-134.078063,58.123068],[-135.038211,58.187715],[-136.628062,58.212209],[-137.800006,58.499995],[-139.867787,59.537762],[-140.825274,59.727517],[-142.574444,60.084447],[-143.958881,59.99918],[-145.925557,60.45861],[-147.114374,60.884656],[-148.224306,60.672989],[-148.018066,59.978329],[-148.570823,59.914173],[-149.727858,59.705658],[-150.608243,59.368211],[-151.716393,59.155821],[-151.859433,59.744984],[-151.409719,60.725803],[-150.346941,61.033588],[-150.621111,61.284425],[-151.895839,60.727198],[-152.57833,60.061657],[-154.019172,59.350279],[-153.287511,58.864728],[-154.232492,58.146374],[-155.307491,57.727795],[-156.308335,57.422774],[-156.556097,56.979985],[-158.117217,56.463608],[-158.433321,55.994154],[-159.603327,55.566686],[-160.28972,55.643581],[-161.223048,55.364735],[-162.237766,55.024187],[-163.069447,54.689737],[-164.785569,54.404173],[-164.942226,54.572225],[-163.84834,55.039431],[-162.870001,55.348043],[-161.804175,55.894986],[-160.563605,56.008055],[-160.07056,56.418055],[-158.684443,57.016675],[-158.461097,57.216921],[-157.72277,57.570001],[-157.550274,58.328326],[-157.041675,58.918885],[-158.194731,58.615802],[-158.517218,58.787781],[-159.058606,58.424186],[-159.711667,58.93139],[-159.981289,58.572549],[-160.355271,59.071123],[-161.355003,58.670838],[-161.968894,58.671665],[-162.054987,59.266925],[-161.874171,59.633621],[-162.518059,59.989724],[-163.818341,59.798056],[-164.662218,60.267484],[-165.346388,60.507496],[-165.350832,61.073895],[-166.121379,61.500019],[-165.734452,62.074997],[-164.919179,62.633076],[-164.562508,63.146378],[-163.753332,63.219449],[-163.067224,63.059459],[-162.260555,63.541936],[-161.53445,63.455817],[-160.772507,63.766108],[-160.958335,64.222799],[-161.518068,64.402788],[-160.777778,64.788604],[-161.391926,64.777235],[-162.45305,64.559445],[-162.757786,64.338605],[-163.546394,64.55916],[-164.96083,64.446945],[-166.425288,64.686672],[-166.845004,65.088896],[-168.11056,65.669997],[-166.705271,66.088318],[-164.47471,66.57666],[-163.652512,66.57666],[-163.788602,66.077207],[-161.677774,66.11612],[-162.489715,66.735565],[-163.719717,67.116395],[-164.430991,67.616338],[-165.390287,68.042772],[-166.764441,68.358877],[-166.204707,68.883031],[-164.430811,68.915535],[-163.168614,69.371115],[-162.930566,69.858062],[-161.908897,70.33333],[-160.934797,70.44769],[-159.039176,70.891642],[-158.119723,70.824721],[-156.580825,71.357764],[-155.06779,71.147776],[-154.344165,70.696409],[-153.900006,70.889989],[-152.210006,70.829992],[-152.270002,70.600006],[-150.739992,70.430017],[-149.720003,70.53001],[-147.613362,70.214035],[-145.68999,70.12001],[-144.920011,69.989992],[-143.589446,70.152514],[-142.07251,69.851938],[-140.985988,69.711998],[-140.985988,69.711998]]],[[[-171.731657,63.782515],[-171.114434,63.592191],[-170.491112,63.694975],[-169.682505,63.431116],[-168.689439,63.297506],[-168.771941,63.188598],[-169.52944,62.976931],[-170.290556,63.194438],[-170.671386,63.375822],[-171.553063,63.317789],[-171.791111,63.405846],[-171.731657,63.782515]]]]}},{"type":"Feature","properties":{"ADMIN":"Kazakhstan","NAME_EN":"Kazakhstan","CONTINENT":"Asia","ADM0_A3":"KAZ","ISO_A3":"KAZ","NAME":"Kazakhstan","ISO_A2":"KZ","NAME_ZH":"哈萨克斯坦"},"geometry":{"type":"Polygon","coordinates":[[[87.35997,49.214981],[86.598776,48.549182],[85.768233,48.455751],[85.720484,47.452969],[85.16429,47.000956],[83.180484,47.330031],[82.458926,45.53965],[81.947071,45.317027],[79.966106,44.917517],[80.866206,43.180362],[80.18015,42.920068],[80.25999,42.349999],[79.643645,42.496683],[79.142177,42.856092],[77.658392,42.960686],[76.000354,42.988022],[75.636965,42.8779],[74.212866,43.298339],[73.645304,43.091272],[73.489758,42.500894],[71.844638,42.845395],[71.186281,42.704293],[70.962315,42.266154],[70.388965,42.081308],[69.070027,41.384244],[68.632483,40.668681],[68.259896,40.662325],[67.985856,41.135991],[66.714047,41.168444],[66.510649,41.987644],[66.023392,41.994646],[66.098012,42.99766],[64.900824,43.728081],[63.185787,43.650075],[62.0133,43.504477],[61.05832,44.405817],[60.239972,44.784037],[58.689989,45.500014],[58.503127,45.586804],[55.928917,44.995858],[55.968191,41.308642],[55.455251,41.259859],[54.755345,42.043971],[54.079418,42.324109],[52.944293,42.116034],[52.50246,41.783316],[52.446339,42.027151],[52.692112,42.443895],[52.501426,42.792298],[51.342427,43.132975],[50.891292,44.031034],[50.339129,44.284016],[50.305643,44.609836],[51.278503,44.514854],[51.316899,45.245998],[52.16739,45.408391],[53.040876,45.259047],[53.220866,46.234646],[53.042737,46.853006],[52.042023,46.804637],[51.191945,47.048705],[50.034083,46.60899],[49.10116,46.39933],[48.59325,46.56104],[48.694734,47.075628],[48.05725,47.74377],[47.31524,47.71585],[46.466446,48.394152],[47.043672,49.152039],[46.751596,49.356006],[47.54948,50.454698],[48.577841,49.87476],[48.702382,50.605128],[50.766648,51.692762],[52.328724,51.718652],[54.532878,51.02624],[55.71694,50.62171],[56.77798,51.04355],[58.36332,51.06364],[59.642282,50.545442],[59.932807,50.842194],[61.337424,50.79907],[61.588003,51.272659],[59.967534,51.96042],[60.927269,52.447548],[60.739993,52.719986],[61.699986,52.979996],[60.978066,53.664993],[61.4366,54.00625],[65.178534,54.354228],[65.66687,54.60125],[68.1691,54.970392],[69.068167,55.38525],[70.865267,55.169734],[71.180131,54.133285],[72.22415,54.376655],[73.508516,54.035617],[73.425679,53.48981],[74.38482,53.54685],[76.8911,54.490524],[76.525179,54.177003],[77.800916,53.404415],[80.03556,50.864751],[80.568447,51.388336],[81.945986,50.812196],[83.383004,51.069183],[83.935115,50.889246],[84.416377,50.3114],[85.11556,50.117303],[85.54127,49.692859],[86.829357,49.826675],[87.35997,49.214981]]]}},{"type":"Feature","properties":{"ADMIN":"Uzbekistan","NAME_EN":"Uzbekistan","CONTINENT":"Asia","ADM0_A3":"UZB","ISO_A3":"UZB","NAME":"Uzbekistan","ISO_A2":"UZ","NAME_ZH":"乌兹别克斯坦"},"geometry":{"type":"Polygon","coordinates":[[[55.968191,41.308642],[55.928917,44.995858],[58.503127,45.586804],[58.689989,45.500014],[60.239972,44.784037],[61.05832,44.405817],[62.0133,43.504477],[63.185787,43.650075],[64.900824,43.728081],[66.098012,42.99766],[66.023392,41.994646],[66.510649,41.987644],[66.714047,41.168444],[67.985856,41.135991],[68.259896,40.662325],[68.632483,40.668681],[69.070027,41.384244],[70.388965,42.081308],[70.962315,42.266154],[71.259248,42.167711],[70.420022,41.519998],[71.157859,41.143587],[71.870115,41.3929],[73.055417,40.866033],[71.774875,40.145844],[71.014198,40.244366],[70.601407,40.218527],[70.45816,40.496495],[70.666622,40.960213],[69.329495,40.727824],[69.011633,40.086158],[68.536416,39.533453],[67.701429,39.580478],[67.44222,39.140144],[68.176025,38.901553],[68.392033,38.157025],[67.83,37.144994],[67.075782,37.356144],[66.518607,37.362784],[66.54615,37.974685],[65.215999,38.402695],[64.170223,38.892407],[63.518015,39.363257],[62.37426,40.053886],[61.882714,41.084857],[61.547179,41.26637],[60.465953,41.220327],[60.083341,41.425146],[59.976422,42.223082],[58.629011,42.751551],[57.78653,42.170553],[56.932215,41.826026],[57.096391,41.32231],[55.968191,41.308642]]]}},{"type":"Feature","properties":{"ADMIN":"Papua New Guinea","NAME_EN":"Papua New Guinea","CONTINENT":"Oceania","ADM0_A3":"PNG","ISO_A3":"PNG","NAME":"Papua New Guinea","ISO_A2":"PG","NAME_ZH":"巴布亚新几内亚"},"geometry":{"type":"MultiPolygon","coordinates":[[[[141.00021,-2.600151],[142.735247,-3.289153],[144.583971,-3.861418],[145.27318,-4.373738],[145.829786,-4.876498],[145.981922,-5.465609],[147.648073,-6.083659],[147.891108,-6.614015],[146.970905,-6.721657],[147.191874,-7.388024],[148.084636,-8.044108],[148.734105,-9.104664],[149.306835,-9.071436],[149.266631,-9.514406],[150.038728,-9.684318],[149.738798,-9.872937],[150.801628,-10.293687],[150.690575,-10.582713],[150.028393,-10.652476],[149.78231,-10.393267],[148.923138,-10.280923],[147.913018,-10.130441],[147.135443,-9.492444],[146.567881,-8.942555],[146.048481,-8.067414],[144.744168,-7.630128],[143.897088,-7.91533],[143.286376,-8.245491],[143.413913,-8.983069],[142.628431,-9.326821],[142.068259,-9.159596],[141.033852,-9.117893],[141.017057,-5.859022],[141.00021,-2.600151]]],[[[152.640017,-3.659983],[153.019994,-3.980015],[153.140038,-4.499983],[152.827292,-4.766427],[152.638673,-4.176127],[152.406026,-3.789743],[151.953237,-3.462062],[151.384279,-3.035422],[150.66205,-2.741486],[150.939965,-2.500002],[151.479984,-2.779985],[151.820015,-2.999972],[152.239989,-3.240009],[152.640017,-3.659983]]],[[[151.30139,-5.840728],[150.754447,-6.083763],[150.241197,-6.317754],[149.709963,-6.316513],[148.890065,-6.02604],[148.318937,-5.747142],[148.401826,-5.437756],[149.298412,-5.583742],[149.845562,-5.505503],[149.99625,-5.026101],[150.139756,-5.001348],[150.236908,-5.53222],[150.807467,-5.455842],[151.089672,-5.113693],[151.647881,-4.757074],[151.537862,-4.167807],[152.136792,-4.14879],[152.338743,-4.312966],[152.318693,-4.867661],[151.982796,-5.478063],[151.459107,-5.56028],[151.30139,-5.840728]]],[[[154.759991,-5.339984],[155.062918,-5.566792],[155.547746,-6.200655],[156.019965,-6.540014],[155.880026,-6.819997],[155.599991,-6.919991],[155.166994,-6.535931],[154.729192,-5.900828],[154.514114,-5.139118],[154.652504,-5.042431],[154.759991,-5.339984]]]]}},{"type":"Feature","properties":{"ADMIN":"Indonesia","NAME_EN":"Indonesia","CONTINENT":"Asia","ADM0_A3":"IDN","ISO_A3":"IDN","NAME":"Indonesia","ISO_A2":"ID","NAME_ZH":"印度尼西亚"},"geometry":{"type":"MultiPolygon","coordinates":[[[[141.00021,-2.600151],[141.017057,-5.859022],[141.033852,-9.117893],[140.143415,-8.297168],[139.127767,-8.096043],[138.881477,-8.380935],[137.614474,-8.411683],[138.039099,-7.597882],[138.668621,-7.320225],[138.407914,-6.232849],[137.92784,-5.393366],[135.98925,-4.546544],[135.164598,-4.462931],[133.66288,-3.538853],[133.367705,-4.024819],[132.983956,-4.112979],[132.756941,-3.746283],[132.753789,-3.311787],[131.989804,-2.820551],[133.066845,-2.460418],[133.780031,-2.479848],[133.696212,-2.214542],[132.232373,-2.212526],[131.836222,-1.617162],[130.94284,-1.432522],[130.519558,-0.93772],[131.867538,-0.695461],[132.380116,-0.369538],[133.985548,-0.78021],[134.143368,-1.151867],[134.422627,-2.769185],[135.457603,-3.367753],[136.293314,-2.307042],[137.440738,-1.703513],[138.329727,-1.702686],[139.184921,-2.051296],[139.926684,-2.409052],[141.00021,-2.600151]]],[[[124.968682,-8.89279],[125.07002,-9.089987],[125.08852,-9.393173],[124.43595,-10.140001],[123.579982,-10.359987],[123.459989,-10.239995],[123.550009,-9.900016],[123.980009,-9.290027],[124.968682,-8.89279]]],[[[134.210134,-6.895238],[134.112776,-6.142467],[134.290336,-5.783058],[134.499625,-5.445042],[134.727002,-5.737582],[134.724624,-6.214401],[134.210134,-6.895238]]],[[[117.882035,4.137551],[117.313232,3.234428],[118.04833,2.28769],[117.875627,1.827641],[118.996747,0.902219],[117.811858,0.784242],[117.478339,0.102475],[117.521644,-0.803723],[116.560048,-1.487661],[116.533797,-2.483517],[116.148084,-4.012726],[116.000858,-3.657037],[114.864803,-4.106984],[114.468652,-3.495704],[113.755672,-3.43917],[113.256994,-3.118776],[112.068126,-3.478392],[111.703291,-2.994442],[111.04824,-3.049426],[110.223846,-2.934032],[110.070936,-1.592874],[109.571948,-1.314907],[109.091874,-0.459507],[108.952658,0.415375],[109.069136,1.341934],[109.66326,2.006467],[109.830227,1.338136],[110.514061,0.773131],[111.159138,0.976478],[111.797548,0.904441],[112.380252,1.410121],[112.859809,1.49779],[113.80585,1.217549],[114.621355,1.430688],[115.134037,2.821482],[115.519078,3.169238],[115.865517,4.306559],[117.015214,4.306094],[117.882035,4.137551]]],[[[129.370998,-2.802154],[130.471344,-3.093764],[130.834836,-3.858472],[129.990547,-3.446301],[129.155249,-3.362637],[128.590684,-3.428679],[127.898891,-3.393436],[128.135879,-2.84365],[129.370998,-2.802154]]],[[[126.874923,-3.790983],[126.183802,-3.607376],[125.989034,-3.177273],[127.000651,-3.129318],[127.249215,-3.459065],[126.874923,-3.790983]]],[[[127.932378,2.174596],[128.004156,1.628531],[128.594559,1.540811],[128.688249,1.132386],[128.635952,0.258486],[128.12017,0.356413],[127.968034,-0.252077],[128.379999,-0.780004],[128.100016,-0.899996],[127.696475,-0.266598],[127.39949,1.011722],[127.600512,1.810691],[127.932378,2.174596]]],[[[122.927567,0.875192],[124.077522,0.917102],[125.065989,1.643259],[125.240501,1.419836],[124.437035,0.427881],[123.685505,0.235593],[122.723083,0.431137],[121.056725,0.381217],[120.183083,0.237247],[120.04087,-0.519658],[120.935905,-1.408906],[121.475821,-0.955962],[123.340565,-0.615673],[123.258399,-1.076213],[122.822715,-0.930951],[122.38853,-1.516858],[121.508274,-1.904483],[122.454572,-3.186058],[122.271896,-3.5295],[123.170963,-4.683693],[123.162333,-5.340604],[122.628515,-5.634591],[122.236394,-5.282933],[122.719569,-4.464172],[121.738234,-4.851331],[121.489463,-4.574553],[121.619171,-4.188478],[120.898182,-3.602105],[120.972389,-2.627643],[120.305453,-2.931604],[120.390047,-4.097579],[120.430717,-5.528241],[119.796543,-5.6734],[119.366906,-5.379878],[119.653606,-4.459417],[119.498835,-3.494412],[119.078344,-3.487022],[118.767769,-2.801999],[119.180974,-2.147104],[119.323394,-1.353147],[119.825999,0.154254],[120.035702,0.566477],[120.885779,1.309223],[121.666817,1.013944],[122.927567,0.875192]]],[[[120.295014,-10.25865],[118.967808,-9.557969],[119.90031,-9.36134],[120.425756,-9.665921],[120.775502,-9.969675],[120.715609,-10.239581],[120.295014,-10.25865]]],[[[121.341669,-8.53674],[122.007365,-8.46062],[122.903537,-8.094234],[122.756983,-8.649808],[121.254491,-8.933666],[119.924391,-8.810418],[119.920929,-8.444859],[120.715092,-8.236965],[121.341669,-8.53674]]],[[[118.260616,-8.362383],[118.87846,-8.280683],[119.126507,-8.705825],[117.970402,-8.906639],[117.277731,-9.040895],[116.740141,-9.032937],[117.083737,-8.457158],[117.632024,-8.449303],[117.900018,-8.095681],[118.260616,-8.362383]]],[[[108.486846,-6.421985],[108.623479,-6.777674],[110.539227,-6.877358],[110.759576,-6.465186],[112.614811,-6.946036],[112.978768,-7.594213],[114.478935,-7.776528],[115.705527,-8.370807],[114.564511,-8.751817],[113.464734,-8.348947],[112.559672,-8.376181],[111.522061,-8.302129],[110.58615,-8.122605],[109.427667,-7.740664],[108.693655,-7.6416],[108.277763,-7.766657],[106.454102,-7.3549],[106.280624,-6.9249],[105.365486,-6.851416],[106.051646,-5.895919],[107.265009,-5.954985],[108.072091,-6.345762],[108.486846,-6.421985]]],[[[104.369991,-1.084843],[104.53949,-1.782372],[104.887893,-2.340425],[105.622111,-2.428844],[106.108593,-3.061777],[105.857446,-4.305525],[105.817655,-5.852356],[104.710384,-5.873285],[103.868213,-5.037315],[102.584261,-4.220259],[102.156173,-3.614146],[101.399113,-2.799777],[100.902503,-2.050262],[100.141981,-0.650348],[99.26374,0.183142],[98.970011,1.042882],[98.601351,1.823507],[97.699598,2.453184],[97.176942,3.308791],[96.424017,3.86886],[95.380876,4.970782],[95.293026,5.479821],[95.936863,5.439513],[97.484882,5.246321],[98.369169,4.26837],[99.142559,3.59035],[99.693998,3.174329],[100.641434,2.099381],[101.658012,2.083697],[102.498271,1.3987],[103.07684,0.561361],[103.838396,0.104542],[103.437645,-0.711946],[104.010789,-1.059212],[104.369991,-1.084843]]]]}},{"type":"Feature","properties":{"ADMIN":"Argentina","NAME_EN":"Argentina","CONTINENT":"South America","ADM0_A3":"ARG","ISO_A3":"ARG","NAME":"Argentina","ISO_A2":"AR","NAME_ZH":"阿根廷"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-68.63401,-52.63637],[-68.25,-53.1],[-67.75,-53.85],[-66.45,-54.45],[-65.05,-54.7],[-65.5,-55.2],[-66.45,-55.25],[-66.95992,-54.89681],[-67.56244,-54.87001],[-68.63335,-54.8695],[-68.63401,-52.63637]]],[[[-57.625133,-30.216295],[-57.874937,-31.016556],[-58.14244,-32.044504],[-58.132648,-33.040567],[-58.349611,-33.263189],[-58.427074,-33.909454],[-58.495442,-34.43149],[-57.22583,-35.288027],[-57.362359,-35.97739],[-56.737487,-36.413126],[-56.788285,-36.901572],[-57.749157,-38.183871],[-59.231857,-38.72022],[-61.237445,-38.928425],[-62.335957,-38.827707],[-62.125763,-39.424105],[-62.330531,-40.172586],[-62.145994,-40.676897],[-62.745803,-41.028761],[-63.770495,-41.166789],[-64.73209,-40.802677],[-65.118035,-41.064315],[-64.978561,-42.058001],[-64.303408,-42.359016],[-63.755948,-42.043687],[-63.458059,-42.563138],[-64.378804,-42.873558],[-65.181804,-43.495381],[-65.328823,-44.501366],[-65.565269,-45.036786],[-66.509966,-45.039628],[-67.293794,-45.551896],[-67.580546,-46.301773],[-66.597066,-47.033925],[-65.641027,-47.236135],[-65.985088,-48.133289],[-67.166179,-48.697337],[-67.816088,-49.869669],[-68.728745,-50.264218],[-69.138539,-50.73251],[-68.815561,-51.771104],[-68.149995,-52.349983],[-68.571545,-52.299444],[-69.498362,-52.142761],[-71.914804,-52.009022],[-72.329404,-51.425956],[-72.309974,-50.67701],[-72.975747,-50.74145],[-73.328051,-50.378785],[-73.415436,-49.318436],[-72.648247,-48.878618],[-72.331161,-48.244238],[-72.447355,-47.738533],[-71.917258,-46.884838],[-71.552009,-45.560733],[-71.659316,-44.973689],[-71.222779,-44.784243],[-71.329801,-44.407522],[-71.793623,-44.207172],[-71.464056,-43.787611],[-71.915424,-43.408565],[-72.148898,-42.254888],[-71.746804,-42.051386],[-71.915734,-40.832339],[-71.680761,-39.808164],[-71.413517,-38.916022],[-70.814664,-38.552995],[-71.118625,-37.576827],[-71.121881,-36.658124],[-70.364769,-36.005089],[-70.388049,-35.169688],[-69.817309,-34.193571],[-69.814777,-33.273886],[-70.074399,-33.09121],[-70.535069,-31.36501],[-69.919008,-30.336339],[-70.01355,-29.367923],[-69.65613,-28.459141],[-69.001235,-27.521214],[-68.295542,-26.89934],[-68.5948,-26.506909],[-68.386001,-26.185016],[-68.417653,-24.518555],[-67.328443,-24.025303],[-66.985234,-22.986349],[-67.106674,-22.735925],[-66.273339,-21.83231],[-64.964892,-22.075862],[-64.377021,-22.798091],[-63.986838,-21.993644],[-62.846468,-22.034985],[-62.685057,-22.249029],[-60.846565,-23.880713],[-60.028966,-24.032796],[-58.807128,-24.771459],[-57.777217,-25.16234],[-57.63366,-25.603657],[-58.618174,-27.123719],[-57.60976,-27.395899],[-56.486702,-27.548499],[-55.695846,-27.387837],[-54.788795,-26.621786],[-54.625291,-25.739255],[-54.13005,-25.547639],[-53.628349,-26.124865],[-53.648735,-26.923473],[-54.490725,-27.474757],[-55.162286,-27.881915],[-56.2909,-28.852761],[-57.625133,-30.216295]]]]}},{"type":"Feature","properties":{"ADMIN":"Chile","NAME_EN":"Chile","CONTINENT":"South America","ADM0_A3":"CHL","ISO_A3":"CHL","NAME":"Chile","ISO_A2":"CL","NAME_ZH":"智利"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-68.63401,-52.63637],[-68.63335,-54.8695],[-67.56244,-54.87001],[-66.95992,-54.89681],[-67.29103,-55.30124],[-68.14863,-55.61183],[-68.639991,-55.580018],[-69.2321,-55.49906],[-69.95809,-55.19843],[-71.00568,-55.05383],[-72.2639,-54.49514],[-73.2852,-53.95752],[-74.66253,-52.83749],[-73.8381,-53.04743],[-72.43418,-53.7154],[-71.10773,-54.07433],[-70.59178,-53.61583],[-70.26748,-52.93123],[-69.34565,-52.5183],[-68.63401,-52.63637]]],[[[-69.590424,-17.580012],[-69.100247,-18.260125],[-68.966818,-18.981683],[-68.442225,-19.405068],[-68.757167,-20.372658],[-68.219913,-21.494347],[-67.82818,-22.872919],[-67.106674,-22.735925],[-66.985234,-22.986349],[-67.328443,-24.025303],[-68.417653,-24.518555],[-68.386001,-26.185016],[-68.5948,-26.506909],[-68.295542,-26.89934],[-69.001235,-27.521214],[-69.65613,-28.459141],[-70.01355,-29.367923],[-69.919008,-30.336339],[-70.535069,-31.36501],[-70.074399,-33.09121],[-69.814777,-33.273886],[-69.817309,-34.193571],[-70.388049,-35.169688],[-70.364769,-36.005089],[-71.121881,-36.658124],[-71.118625,-37.576827],[-70.814664,-38.552995],[-71.413517,-38.916022],[-71.680761,-39.808164],[-71.915734,-40.832339],[-71.746804,-42.051386],[-72.148898,-42.254888],[-71.915424,-43.408565],[-71.464056,-43.787611],[-71.793623,-44.207172],[-71.329801,-44.407522],[-71.222779,-44.784243],[-71.659316,-44.973689],[-71.552009,-45.560733],[-71.917258,-46.884838],[-72.447355,-47.738533],[-72.331161,-48.244238],[-72.648247,-48.878618],[-73.415436,-49.318436],[-73.328051,-50.378785],[-72.975747,-50.74145],[-72.309974,-50.67701],[-72.329404,-51.425956],[-71.914804,-52.009022],[-69.498362,-52.142761],[-68.571545,-52.299444],[-69.461284,-52.291951],[-69.94278,-52.537931],[-70.845102,-52.899201],[-71.006332,-53.833252],[-71.429795,-53.856455],[-72.557943,-53.53141],[-73.702757,-52.835069],[-73.702757,-52.83507],[-74.946763,-52.262754],[-75.260026,-51.629355],[-74.976632,-51.043396],[-75.479754,-50.378372],[-75.608015,-48.673773],[-75.18277,-47.711919],[-74.126581,-46.939253],[-75.644395,-46.647643],[-74.692154,-45.763976],[-74.351709,-44.103044],[-73.240356,-44.454961],[-72.717804,-42.383356],[-73.3889,-42.117532],[-73.701336,-43.365776],[-74.331943,-43.224958],[-74.017957,-41.794813],[-73.677099,-39.942213],[-73.217593,-39.258689],[-73.505559,-38.282883],[-73.588061,-37.156285],[-73.166717,-37.12378],[-72.553137,-35.50884],[-71.861732,-33.909093],[-71.43845,-32.418899],[-71.668721,-30.920645],[-71.370083,-30.095682],[-71.489894,-28.861442],[-70.905124,-27.64038],[-70.724954,-25.705924],[-70.403966,-23.628997],[-70.091246,-21.393319],[-70.16442,-19.756468],[-70.372572,-18.347975],[-69.858444,-18.092694],[-69.590424,-17.580012]]]]}},{"type":"Feature","properties":{"ADMIN":"Democratic Republic of the Congo","NAME_EN":"Democratic Republic of the Congo","CONTINENT":"Africa","ADM0_A3":"COD","ISO_A3":"COD","NAME":"Dem. Rep. Congo","ISO_A2":"CD","NAME_ZH":"刚果民主共和国"},"geometry":{"type":"Polygon","coordinates":[[[29.339998,-4.499983],[29.519987,-5.419979],[29.419993,-5.939999],[29.620032,-6.520015],[30.199997,-7.079981],[30.740015,-8.340007],[30.74001,-8.340006],[30.346086,-8.238257],[29.002912,-8.407032],[28.734867,-8.526559],[28.449871,-9.164918],[28.673682,-9.605925],[28.49607,-10.789884],[28.372253,-11.793647],[28.642417,-11.971569],[29.341548,-12.360744],[29.616001,-12.178895],[29.699614,-13.257227],[28.934286,-13.248958],[28.523562,-12.698604],[28.155109,-12.272481],[27.388799,-12.132747],[27.16442,-11.608748],[26.553088,-11.92444],[25.75231,-11.784965],[25.418118,-11.330936],[24.78317,-11.238694],[24.314516,-11.262826],[24.257155,-10.951993],[23.912215,-10.926826],[23.456791,-10.867863],[22.837345,-11.017622],[22.402798,-10.993075],[22.155268,-11.084801],[22.208753,-9.894796],[21.875182,-9.523708],[21.801801,-8.908707],[21.949131,-8.305901],[21.746456,-7.920085],[21.728111,-7.290872],[20.514748,-7.299606],[20.601823,-6.939318],[20.091622,-6.94309],[20.037723,-7.116361],[19.417502,-7.155429],[19.166613,-7.738184],[19.016752,-7.988246],[18.464176,-7.847014],[18.134222,-7.987678],[17.47297,-8.068551],[17.089996,-7.545689],[16.860191,-7.222298],[16.57318,-6.622645],[16.326528,-5.87747],[13.375597,-5.864241],[13.024869,-5.984389],[12.735171,-5.965682],[12.322432,-6.100092],[12.182337,-5.789931],[12.436688,-5.684304],[12.468004,-5.248362],[12.631612,-4.991271],[12.995517,-4.781103],[13.25824,-4.882957],[13.600235,-4.500138],[14.144956,-4.510009],[14.209035,-4.793092],[14.582604,-4.970239],[15.170992,-4.343507],[15.75354,-3.855165],[16.00629,-3.535133],[15.972803,-2.712392],[16.407092,-1.740927],[16.865307,-1.225816],[17.523716,-0.74383],[17.638645,-0.424832],[17.663553,-0.058084],[17.82654,0.288923],[17.774192,0.855659],[17.898835,1.741832],[18.094276,2.365722],[18.393792,2.900443],[18.453065,3.504386],[18.542982,4.201785],[18.932312,4.709506],[19.467784,5.031528],[20.290679,4.691678],[20.927591,4.322786],[21.659123,4.224342],[22.405124,4.02916],[22.704124,4.633051],[22.84148,4.710126],[23.297214,4.609693],[24.410531,5.108784],[24.805029,4.897247],[25.128833,4.927245],[25.278798,5.170408],[25.650455,5.256088],[26.402761,5.150875],[27.044065,5.127853],[27.374226,5.233944],[27.979977,4.408413],[28.428994,4.287155],[28.696678,4.455077],[29.159078,4.389267],[29.715995,4.600805],[29.9535,4.173699],[30.833852,3.509172],[30.83386,3.509166],[30.773347,2.339883],[31.174149,2.204465],[30.85267,1.849396],[30.468508,1.583805],[30.086154,1.062313],[29.875779,0.59738],[29.819503,-0.20531],[29.587838,-0.587406],[29.579466,-1.341313],[29.291887,-1.620056],[29.254835,-2.21511],[29.117479,-2.292211],[29.024926,-2.839258],[29.276384,-3.293907],[29.339998,-4.499983]]]}},{"type":"Feature","properties":{"ADMIN":"Somalia","NAME_EN":"Somalia","CONTINENT":"Africa","ADM0_A3":"SOM","ISO_A3":"SOM","NAME":"Somalia","ISO_A2":"SO","NAME_ZH":"索马里"},"geometry":{"type":"Polygon","coordinates":[[[41.58513,-1.68325],[40.993,-0.85829],[40.98105,2.78452],[41.855083,3.918912],[42.12861,4.23413],[42.76967,4.25259],[43.66087,4.95755],[44.9636,5.00162],[47.78942,8.003],[48.486736,8.837626],[48.93813,9.451749],[48.938233,9.9735],[48.938491,10.982327],[48.942005,11.394266],[48.948205,11.410617],[48.948205,11.410617],[49.26776,11.43033],[49.72862,11.5789],[50.25878,11.67957],[50.73202,12.0219],[51.1112,12.02464],[51.13387,11.74815],[51.04153,11.16651],[51.04531,10.6409],[50.83418,10.27972],[50.55239,9.19874],[50.07092,8.08173],[49.4527,6.80466],[48.59455,5.33911],[47.74079,4.2194],[46.56476,2.85529],[45.56399,2.04576],[44.06815,1.05283],[43.13597,0.2922],[42.04157,-0.91916],[41.81095,-1.44647],[41.58513,-1.68325]]]}},{"type":"Feature","properties":{"ADMIN":"Kenya","NAME_EN":"Kenya","CONTINENT":"Africa","ADM0_A3":"KEN","ISO_A3":"KEN","NAME":"Kenya","ISO_A2":"KE","NAME_ZH":"肯尼亚"},"geometry":{"type":"Polygon","coordinates":[[[39.20222,-4.67677],[37.7669,-3.67712],[37.69869,-3.09699],[34.07262,-1.05982],[33.903711,-0.95],[33.893569,0.109814],[34.18,0.515],[34.6721,1.17694],[35.03599,1.90584],[34.59607,3.05374],[34.47913,3.5556],[34.005,4.249885],[34.620196,4.847123],[35.298007,5.506],[35.817448,5.338232],[35.817448,4.776966],[36.159079,4.447864],[36.855093,4.447864],[38.120915,3.598605],[38.43697,3.58851],[38.67114,3.61607],[38.89251,3.50074],[39.559384,3.42206],[39.85494,3.83879],[40.76848,4.25702],[41.1718,3.91909],[41.855083,3.918912],[40.98105,2.78452],[40.993,-0.85829],[41.58513,-1.68325],[40.88477,-2.08255],[40.63785,-2.49979],[40.26304,-2.57309],[40.12119,-3.27768],[39.80006,-3.68116],[39.60489,-4.34653],[39.20222,-4.67677]]]}},{"type":"Feature","properties":{"ADMIN":"Sudan","NAME_EN":"Sudan","CONTINENT":"Africa","ADM0_A3":"SDN","ISO_A3":"SDN","NAME":"Sudan","ISO_A2":"SD","NAME_ZH":"苏丹"},"geometry":{"type":"Polygon","coordinates":[[[24.567369,8.229188],[23.805813,8.666319],[23.459013,8.954286],[23.394779,9.265068],[23.55725,9.681218],[23.554304,10.089255],[22.977544,10.714463],[22.864165,11.142395],[22.87622,11.38461],[22.50869,11.67936],[22.49762,12.26024],[22.28801,12.64605],[21.93681,12.58818],[22.03759,12.95546],[22.29658,13.37232],[22.18329,13.78648],[22.51202,14.09318],[22.30351,14.32682],[22.56795,14.94429],[23.02459,15.68072],[23.88689,15.61084],[23.83766,19.58047],[23.85,20],[25,20.00304],[25,22],[29.02,22],[32.9,22],[36.86623,22],[37.18872,21.01885],[36.96941,20.83744],[37.1147,19.80796],[37.48179,18.61409],[37.86276,18.36786],[38.41009,17.998307],[37.904,17.42754],[37.16747,17.26314],[36.85253,16.95655],[36.75389,16.29186],[36.32322,14.82249],[36.42951,14.42211],[36.27022,13.56333],[35.86363,12.57828],[35.26049,12.08286],[34.83163,11.31896],[34.73115,10.91017],[34.25745,10.63009],[33.96162,9.58358],[33.97498,8.68456],[33.963393,9.464285],[33.824963,9.484061],[33.842131,9.981915],[33.721959,10.325262],[33.206938,10.720112],[33.086766,11.441141],[33.206938,12.179338],[32.743419,12.248008],[32.67475,12.024832],[32.073892,11.97333],[32.314235,11.681484],[32.400072,11.080626],[31.850716,10.531271],[31.352862,9.810241],[30.837841,9.707237],[29.996639,10.290927],[29.618957,10.084919],[29.515953,9.793074],[29.000932,9.604232],[28.966597,9.398224],[27.97089,9.398224],[27.833551,9.604232],[27.112521,9.638567],[26.752006,9.466893],[26.477328,9.55273],[25.962307,10.136421],[25.790633,10.411099],[25.069604,10.27376],[24.794926,9.810241],[24.537415,8.917538],[24.194068,8.728696],[23.88698,8.61973],[24.567369,8.229188]]]}},{"type":"Feature","properties":{"ADMIN":"Chad","NAME_EN":"Chad","CONTINENT":"Africa","ADM0_A3":"TCD","ISO_A3":"TCD","NAME":"Chad","ISO_A2":"TD","NAME_ZH":"乍得"},"geometry":{"type":"Polygon","coordinates":[[[23.83766,19.58047],[23.88689,15.61084],[23.02459,15.68072],[22.56795,14.94429],[22.30351,14.32682],[22.51202,14.09318],[22.18329,13.78648],[22.29658,13.37232],[22.03759,12.95546],[21.93681,12.58818],[22.28801,12.64605],[22.49762,12.26024],[22.50869,11.67936],[22.87622,11.38461],[22.864165,11.142395],[22.231129,10.971889],[21.723822,10.567056],[21.000868,9.475985],[20.059685,9.012706],[19.094008,9.074847],[18.81201,8.982915],[18.911022,8.630895],[18.389555,8.281304],[17.96493,7.890914],[16.705988,7.508328],[16.456185,7.734774],[16.290562,7.754307],[16.106232,7.497088],[15.27946,7.421925],[15.436092,7.692812],[15.120866,8.38215],[14.979996,8.796104],[14.544467,8.965861],[13.954218,9.549495],[14.171466,10.021378],[14.627201,9.920919],[14.909354,9.992129],[15.467873,9.982337],[14.923565,10.891325],[14.960152,11.555574],[14.89336,12.21905],[14.495787,12.859396],[14.595781,13.330427],[13.954477,13.353449],[13.956699,13.996691],[13.540394,14.367134],[13.97217,15.68437],[15.247731,16.627306],[15.300441,17.92795],[15.685741,19.95718],[15.903247,20.387619],[15.487148,20.730415],[15.47106,21.04845],[15.096888,21.308519],[14.8513,22.86295],[15.86085,23.40972],[19.84926,21.49509],[23.83766,19.58047]]]}},{"type":"Feature","properties":{"ADMIN":"Haiti","NAME_EN":"Haiti","CONTINENT":"North America","ADM0_A3":"HTI","ISO_A3":"HTI","NAME":"Haiti","ISO_A2":"HT","NAME_ZH":"海地"},"geometry":{"type":"Polygon","coordinates":[[[-71.712361,19.714456],[-71.624873,19.169838],[-71.701303,18.785417],[-71.945112,18.6169],[-71.687738,18.31666],[-71.708305,18.044997],[-72.372476,18.214961],[-72.844411,18.145611],[-73.454555,18.217906],[-73.922433,18.030993],[-74.458034,18.34255],[-74.369925,18.664908],[-73.449542,18.526053],[-72.694937,18.445799],[-72.334882,18.668422],[-72.79165,19.101625],[-72.784105,19.483591],[-73.415022,19.639551],[-73.189791,19.915684],[-72.579673,19.871501],[-71.712361,19.714456]]]}},{"type":"Feature","properties":{"ADMIN":"Dominican Republic","NAME_EN":"Dominican Republic","CONTINENT":"North America","ADM0_A3":"DOM","ISO_A3":"DOM","NAME":"Dominican Rep.","ISO_A2":"DO","NAME_ZH":"多米尼加"},"geometry":{"type":"Polygon","coordinates":[[[-71.708305,18.044997],[-71.687738,18.31666],[-71.945112,18.6169],[-71.701303,18.785417],[-71.624873,19.169838],[-71.712361,19.714456],[-71.587304,19.884911],[-70.806706,19.880286],[-70.214365,19.622885],[-69.950815,19.648],[-69.76925,19.293267],[-69.222126,19.313214],[-69.254346,19.015196],[-68.809412,18.979074],[-68.317943,18.612198],[-68.689316,18.205142],[-69.164946,18.422648],[-69.623988,18.380713],[-69.952934,18.428307],[-70.133233,18.245915],[-70.517137,18.184291],[-70.669298,18.426886],[-70.99995,18.283329],[-71.40021,17.598564],[-71.657662,17.757573],[-71.708305,18.044997]]]}},{"type":"Feature","properties":{"ADMIN":"Russia","NAME_EN":"Russia","CONTINENT":"Europe","ADM0_A3":"RUS","ISO_A3":"RUS","NAME":"Russia","ISO_A2":"RU","NAME_ZH":"俄罗斯"},"geometry":{"type":"MultiPolygon","coordinates":[[[[178.7253,71.0988],[180,71.515714],[180,70.832199],[178.903425,70.78114],[178.7253,71.0988]]],[[[49.10116,46.39933],[48.64541,45.80629],[47.67591,45.64149],[46.68201,44.6092],[47.59094,43.66016],[47.49252,42.98658],[48.58437,41.80888],[48.584353,41.808869],[47.987283,41.405819],[47.815666,41.151416],[47.373315,41.219732],[46.686071,41.827137],[46.404951,41.860675],[45.7764,42.09244],[45.470279,42.502781],[44.537623,42.711993],[43.93121,42.55496],[43.75599,42.74083],[42.3944,43.2203],[40.92219,43.38215],[40.076965,43.553104],[39.955009,43.434998],[38.68,44.28],[37.53912,44.65721],[36.67546,45.24469],[37.40317,45.40451],[38.23295,46.24087],[37.67372,46.63657],[39.14767,47.04475],[39.1212,47.26336],[38.223538,47.10219],[38.255112,47.5464],[38.77057,47.82562],[39.738278,47.898937],[39.89562,48.23241],[39.67465,48.78382],[40.080789,49.30743],[40.06904,49.60105],[38.594988,49.926462],[38.010631,49.915662],[37.39346,50.383953],[36.626168,50.225591],[35.356116,50.577197],[35.37791,50.77394],[35.022183,51.207572],[34.224816,51.255993],[34.141978,51.566413],[34.391731,51.768882],[33.7527,52.335075],[32.715761,52.238465],[32.412058,52.288695],[32.15944,52.06125],[31.785992,52.101678],[31.78597,52.10168],[31.540018,52.742052],[31.305201,53.073996],[31.49764,53.16743],[32.304519,53.132726],[32.693643,53.351421],[32.405599,53.618045],[31.731273,53.794029],[31.791424,53.974639],[31.384472,54.157056],[30.757534,54.811771],[30.971836,55.081548],[30.873909,55.550976],[29.896294,55.789463],[29.371572,55.670091],[29.229513,55.918344],[28.176709,56.16913],[27.855282,56.759326],[27.770016,57.244258],[27.288185,57.474528],[27.716686,57.791899],[27.42015,58.72457],[28.131699,59.300825],[27.98112,59.47537],[27.981127,59.475373],[29.1177,60.02805],[28.070002,60.503519],[28.07,60.50352],[30.211107,61.780028],[31.139991,62.357693],[31.516092,62.867687],[30.035872,63.552814],[30.444685,64.204453],[29.54443,64.948672],[30.21765,65.80598],[29.054589,66.944286],[29.977426,67.698297],[28.445944,68.364613],[28.59193,69.064777],[29.39955,69.15692],[31.101042,69.558101],[31.10108,69.55811],[32.13272,69.90595],[33.77547,69.30142],[36.51396,69.06342],[40.29234,67.9324],[41.05987,67.45713],[41.12595,66.79158],[40.01583,66.26618],[38.38295,65.99953],[33.91871,66.75961],[33.18444,66.63253],[34.81477,65.90015],[34.878574,65.436213],[34.94391,64.41437],[36.23129,64.10945],[37.01273,63.84983],[37.14197,64.33471],[36.539579,64.76446],[37.17604,65.14322],[39.59345,64.52079],[40.4356,64.76446],[39.7626,65.49682],[42.09309,66.47623],[43.01604,66.41858],[43.94975,66.06908],[44.53226,66.75634],[43.69839,67.35245],[44.18795,67.95051],[43.45282,68.57079],[46.25,68.25],[46.82134,67.68997],[45.55517,67.56652],[45.56202,67.01005],[46.34915,66.66767],[47.89416,66.88455],[48.13876,67.52238],[50.22766,67.99867],[53.71743,68.85738],[54.47171,68.80815],[53.48582,68.20131],[54.72628,68.09702],[55.44268,68.43866],[57.31702,68.46628],[58.802,68.88082],[59.94142,68.27844],[61.07784,68.94069],[60.03,69.52],[60.55,69.85],[63.504,69.54739],[64.888115,69.234835],[68.51216,68.09233],[69.18068,68.61563],[68.16444,69.14436],[68.13522,69.35649],[66.93008,69.45461],[67.25976,69.92873],[66.72492,70.70889],[66.69466,71.02897],[68.54006,71.9345],[69.19636,72.84336],[69.94,73.04],[72.58754,72.77629],[72.79603,72.22006],[71.84811,71.40898],[72.47011,71.09019],[72.79188,70.39114],[72.5647,69.02085],[73.66787,68.4079],[73.2387,67.7404],[71.28,66.32],[72.42301,66.17267],[72.82077,66.53267],[73.92099,66.78946],[74.18651,67.28429],[75.052,67.76047],[74.46926,68.32899],[74.93584,68.98918],[73.84236,69.07146],[73.60187,69.62763],[74.3998,70.63175],[73.1011,71.44717],[74.89082,72.12119],[74.65926,72.83227],[75.15801,72.85497],[75.68351,72.30056],[75.28898,71.33556],[76.35911,71.15287],[75.90313,71.87401],[77.57665,72.26717],[79.65202,72.32011],[81.5,71.75],[80.61071,72.58285],[80.51109,73.6482],[82.25,73.85],[84.65526,73.80591],[86.8223,73.93688],[86.00956,74.45967],[87.16682,75.11643],[88.31571,75.14393],[90.26,75.64],[92.90058,75.77333],[93.23421,76.0472],[95.86,76.14],[96.67821,75.91548],[98.92254,76.44689],[100.75967,76.43028],[101.03532,76.86189],[101.99084,77.28754],[104.3516,77.69792],[106.06664,77.37389],[104.705,77.1274],[106.97013,76.97419],[107.24,76.48],[108.1538,76.72335],[111.07726,76.71],[113.33151,76.22224],[114.13417,75.84764],[113.88539,75.32779],[112.77918,75.03186],[110.15125,74.47673],[109.4,74.18],[110.64,74.04],[112.11919,73.78774],[113.01954,73.97693],[113.52958,73.33505],[113.96881,73.59488],[115.56782,73.75285],[118.77633,73.58772],[119.02,73.12],[123.20066,72.97122],[123.25777,73.73503],[125.38,73.56],[126.97644,73.56549],[128.59126,73.03871],[129.05157,72.39872],[128.46,71.98],[129.71599,71.19304],[131.28858,70.78699],[132.2535,71.8363],[133.85766,71.38642],[135.56193,71.65525],[137.49755,71.34763],[138.23409,71.62803],[139.86983,71.48783],[139.14791,72.41619],[140.46817,72.84941],[149.5,72.2],[150.35118,71.60643],[152.9689,70.84222],[157.00688,71.03141],[158.99779,70.86672],[159.83031,70.45324],[159.70866,69.72198],[160.94053,69.43728],[162.27907,69.64204],[164.05248,69.66823],[165.94037,69.47199],[167.83567,69.58269],[169.57763,68.6938],[170.81688,69.01363],[170.0082,69.65276],[170.45345,70.09703],[173.64391,69.81743],[175.72403,69.87725],[178.6,69.4],[180,68.963636],[180,64.979709],[179.99281,64.97433],[178.7072,64.53493],[177.41128,64.60821],[178.313,64.07593],[178.90825,63.25197],[179.37034,62.98262],[179.48636,62.56894],[179.22825,62.3041],[177.3643,62.5219],[174.56929,61.76915],[173.68013,61.65261],[172.15,60.95],[170.6985,60.33618],[170.33085,59.88177],[168.90046,60.57355],[166.29498,59.78855],[165.84,60.16],[164.87674,59.7316],[163.53929,59.86871],[163.21711,59.21101],[162.01733,58.24328],[162.05297,57.83912],[163.19191,57.61503],[163.05794,56.15924],[162.12958,56.12219],[161.70146,55.28568],[162.11749,54.85514],[160.36877,54.34433],[160.02173,53.20257],[158.53094,52.95868],[158.23118,51.94269],[156.78979,51.01105],[156.42,51.7],[155.99182,53.15895],[155.43366,55.38103],[155.91442,56.76792],[156.75815,57.3647],[156.81035,57.83204],[158.36433,58.05575],[160.15064,59.31477],[161.87204,60.343],[163.66969,61.1409],[164.47355,62.55061],[163.25842,62.46627],[162.65791,61.6425],[160.12148,60.54423],[159.30232,61.77396],[156.72068,61.43442],[154.21806,59.75818],[155.04375,59.14495],[152.81185,58.88385],[151.26573,58.78089],[151.33815,59.50396],[149.78371,59.65573],[148.54481,59.16448],[145.48722,59.33637],[142.19782,59.03998],[138.95848,57.08805],[135.12619,54.72959],[136.70171,54.60355],[137.19342,53.97732],[138.1647,53.75501],[138.80463,54.25455],[139.90151,54.18968],[141.34531,53.08957],[141.37923,52.23877],[140.59742,51.23967],[140.51308,50.04553],[140.06193,48.44671],[138.55472,46.99965],[138.21971,46.30795],[136.86232,45.1435],[135.51535,43.989],[134.86939,43.39821],[133.53687,42.81147],[132.90627,42.79849],[132.27807,43.28456],[130.93587,42.55274],[130.780005,42.22001],[130.780004,42.220008],[130.78,42.22],[130.779992,42.22001],[130.64,42.395],[130.64,42.395024],[130.633866,42.903015],[131.144688,42.92999],[131.288555,44.11152],[131.02519,44.96796],[131.883454,45.321162],[133.09712,45.14409],[133.769644,46.116927],[134.11235,47.21248],[134.50081,47.57845],[135.026311,48.47823],[133.373596,48.183442],[132.50669,47.78896],[130.98726,47.79013],[130.582293,48.729687],[129.397818,49.4406],[127.6574,49.76027],[127.287456,50.739797],[126.939157,51.353894],[126.564399,51.784255],[125.946349,52.792799],[125.068211,53.161045],[123.57147,53.4588],[122.245748,53.431726],[121.003085,53.251401],[120.177089,52.753886],[120.725789,52.516226],[120.7382,51.96411],[120.18208,51.64355],[119.27939,50.58292],[119.288461,50.142883],[117.879244,49.510983],[116.678801,49.888531],[115.485695,49.805177],[114.96211,50.140247],[114.362456,50.248303],[112.89774,49.543565],[111.581231,49.377968],[110.662011,49.130128],[109.402449,49.292961],[108.475167,49.282548],[107.868176,49.793705],[106.888804,50.274296],[105.886591,50.406019],[104.62158,50.27532],[103.676545,50.089966],[102.25589,50.51056],[102.06521,51.25991],[100.88948,51.516856],[99.981732,51.634006],[98.861491,52.047366],[97.82574,51.010995],[98.231762,50.422401],[97.25976,49.72605],[95.81402,49.97746],[94.815949,50.013433],[94.147566,50.480537],[93.10421,50.49529],[92.234712,50.802171],[90.713667,50.331812],[88.805567,49.470521],[87.751264,49.297198],[87.35997,49.214981],[86.829357,49.826675],[85.54127,49.692859],[85.11556,50.117303],[84.416377,50.3114],[83.935115,50.889246],[83.383004,51.069183],[81.945986,50.812196],[80.568447,51.388336],[80.03556,50.864751],[77.800916,53.404415],[76.525179,54.177003],[76.8911,54.490524],[74.38482,53.54685],[73.425679,53.48981],[73.508516,54.035617],[72.22415,54.376655],[71.180131,54.133285],[70.865267,55.169734],[69.068167,55.38525],[68.1691,54.970392],[65.66687,54.60125],[65.178534,54.354228],[61.4366,54.00625],[60.978066,53.664993],[61.699986,52.979996],[60.739993,52.719986],[60.927269,52.447548],[59.967534,51.96042],[61.588003,51.272659],[61.337424,50.79907],[59.932807,50.842194],[59.642282,50.545442],[58.36332,51.06364],[56.77798,51.04355],[55.71694,50.62171],[54.532878,51.02624],[52.328724,51.718652],[50.766648,51.692762],[48.702382,50.605128],[48.577841,49.87476],[47.54948,50.454698],[46.751596,49.356006],[47.043672,49.152039],[46.466446,48.394152],[47.31524,47.71585],[48.05725,47.74377],[48.694734,47.075628],[48.59325,46.56104],[49.10116,46.39933]]],[[[93.77766,81.0246],[95.940895,81.2504],[97.88385,80.746975],[100.186655,79.780135],[99.93976,78.88094],[97.75794,78.7562],[94.97259,79.044745],[93.31288,79.4265],[92.5454,80.14379],[91.18107,80.34146],[93.77766,81.0246]]],[[[102.837815,79.28129],[105.37243,78.71334],[105.07547,78.30689],[99.43814,77.921],[101.2649,79.23399],[102.08635,79.34641],[102.837815,79.28129]]],[[[138.831075,76.13676],[141.471615,76.09289],[145.086285,75.562625],[144.3,74.82],[140.61381,74.84768],[138.95544,74.61148],[136.97439,75.26167],[137.51176,75.94917],[138.831075,76.13676]]],[[[148.22223,75.345845],[150.73167,75.08406],[149.575925,74.68892],[147.977465,74.778355],[146.11919,75.17298],[146.358485,75.49682],[148.22223,75.345845]]],[[[139.86312,73.36983],[140.81171,73.76506],[142.06207,73.85758],[143.48283,73.47525],[143.60385,73.21244],[142.08763,73.20544],[140.038155,73.31692],[139.86312,73.36983]]],[[[44.846958,80.58981],[46.799139,80.771918],[48.318477,80.78401],[48.522806,80.514569],[49.09719,80.753986],[50.039768,80.918885],[51.522933,80.699726],[51.136187,80.54728],[49.793685,80.415428],[48.894411,80.339567],[48.754937,80.175468],[47.586119,80.010181],[46.502826,80.247247],[47.072455,80.559424],[44.846958,80.58981]]],[[[22.731099,54.327537],[20.892245,54.312525],[19.66064,54.426084],[19.888481,54.86616],[21.268449,55.190482],[22.315724,55.015299],[22.757764,54.856574],[22.651052,54.582741],[22.731099,54.327537]]],[[[53.50829,73.749814],[55.902459,74.627486],[55.631933,75.081412],[57.868644,75.60939],[61.170044,76.251883],[64.498368,76.439055],[66.210977,76.809782],[68.15706,76.939697],[68.852211,76.544811],[68.180573,76.233642],[64.637326,75.737755],[61.583508,75.260885],[58.477082,74.309056],[56.986786,73.333044],[55.419336,72.371268],[55.622838,71.540595],[57.535693,70.720464],[56.944979,70.632743],[53.677375,70.762658],[53.412017,71.206662],[51.601895,71.474759],[51.455754,72.014881],[52.478275,72.229442],[52.444169,72.774731],[54.427614,73.627548],[53.50829,73.749814]]],[[[142.914616,53.704578],[143.260848,52.74076],[143.235268,51.75666],[143.648007,50.7476],[144.654148,48.976391],[143.173928,49.306551],[142.558668,47.861575],[143.533492,46.836728],[143.505277,46.137908],[142.747701,46.740765],[142.09203,45.966755],[141.906925,46.805929],[142.018443,47.780133],[141.904445,48.859189],[142.1358,49.615163],[142.179983,50.952342],[141.594076,51.935435],[141.682546,53.301966],[142.606934,53.762145],[142.209749,54.225476],[142.654786,54.365881],[142.914616,53.704578]]],[[[-174.92825,67.20589],[-175.01425,66.58435],[-174.33983,66.33556],[-174.57182,67.06219],[-171.85731,66.91308],[-169.89958,65.97724],[-170.89107,65.54139],[-172.53025,65.43791],[-172.555,64.46079],[-172.95533,64.25269],[-173.89184,64.2826],[-174.65392,64.63125],[-175.98353,64.92288],[-176.20716,65.35667],[-177.22266,65.52024],[-178.35993,65.39052],[-178.90332,65.74044],[-178.68611,66.11211],[-179.88377,65.87456],[-179.43268,65.40411],[-180,64.979709],[-180,68.963636],[-177.55,68.2],[-174.92825,67.20589]]],[[[-178.69378,70.89302],[-180,70.832199],[-180,71.515714],[-179.871875,71.55762],[-179.02433,71.55553],[-177.577945,71.26948],[-177.663575,71.13277],[-178.69378,70.89302]]],[[[33.435988,45.971917],[33.699462,46.219573],[34.410402,46.005162],[34.732017,45.965666],[34.861792,45.768182],[35.012659,45.737725],[35.020788,45.651219],[35.510009,45.409993],[36.529998,45.46999],[36.334713,45.113216],[35.239999,44.939996],[33.882511,44.361479],[33.326421,44.564877],[33.546924,45.034771],[32.454174,45.327466],[32.630804,45.519186],[33.588162,45.851569],[33.435988,45.971917]]]]}},{"type":"Feature","properties":{"ADMIN":"The Bahamas","NAME_EN":"The Bahamas","CONTINENT":"North America","ADM0_A3":"BHS","ISO_A3":"BHS","NAME":"Bahamas","ISO_A2":"BS","NAME_ZH":"巴哈马"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-78.98,26.79],[-78.51,26.87],[-77.85,26.84],[-77.82,26.58],[-78.91,26.42],[-78.98,26.79]]],[[[-77.79,27.04],[-77,26.59],[-77.17255,25.87918],[-77.35641,26.00735],[-77.34,26.53],[-77.78802,26.92516],[-77.79,27.04]]],[[[-78.19087,25.2103],[-77.89,25.17],[-77.54,24.34],[-77.53466,23.75975],[-77.78,23.71],[-78.03405,24.28615],[-78.40848,24.57564],[-78.19087,25.2103]]]]}},{"type":"Feature","properties":{"ADMIN":"Falkland Islands","NAME_EN":"Falkland Islands","CONTINENT":"South America","ADM0_A3":"FLK","ISO_A3":"FLK","NAME":"Falkland Is.","ISO_A2":"FK","NAME_ZH":"福克兰群岛"},"geometry":{"type":"Polygon","coordinates":[[[-61.2,-51.85],[-60,-51.25],[-59.15,-51.5],[-58.55,-51.1],[-57.75,-51.55],[-58.05,-51.9],[-59.4,-52.2],[-59.85,-51.85],[-60.7,-52.3],[-61.2,-51.85]]]}},{"type":"Feature","properties":{"ADMIN":"Norway","NAME_EN":"Norway","CONTINENT":"Europe","ADM0_A3":"NOR","ISO_A3":"-99","NAME":"Norway","ISO_A2":"-99","NAME_ZH":"挪威"},"geometry":{"type":"MultiPolygon","coordinates":[[[[15.14282,79.67431],[15.52255,80.01608],[16.99085,80.05086],[18.25183,79.70175],[21.54383,78.95611],[19.02737,78.5626],[18.47172,77.82669],[17.59441,77.63796],[17.1182,76.80941],[15.91315,76.77045],[13.76259,77.38035],[14.66956,77.73565],[13.1706,78.02493],[11.22231,78.8693],[10.44453,79.65239],[13.17077,80.01046],[13.71852,79.66039],[15.14282,79.67431]]],[[[31.101042,69.558101],[29.39955,69.15692],[28.59193,69.064777],[29.015573,69.766491],[27.732292,70.164193],[26.179622,69.825299],[25.689213,69.092114],[24.735679,68.649557],[23.66205,68.891247],[22.356238,68.841741],[21.244936,69.370443],[20.645593,69.106247],[20.025269,69.065139],[19.87856,68.407194],[17.993868,68.567391],[17.729182,68.010552],[16.768879,68.013937],[16.108712,67.302456],[15.108411,66.193867],[13.55569,64.787028],[13.919905,64.445421],[13.571916,64.049114],[12.579935,64.066219],[11.930569,63.128318],[11.992064,61.800362],[12.631147,61.293572],[12.300366,60.117933],[11.468272,59.432393],[11.027369,58.856149],[10.356557,59.469807],[8.382,58.313288],[7.048748,58.078884],[5.665835,58.588155],[5.308234,59.663232],[4.992078,61.970998],[5.9129,62.614473],[8.553411,63.454008],[10.527709,64.486038],[12.358347,65.879726],[14.761146,67.810642],[16.435927,68.563205],[19.184028,69.817444],[21.378416,70.255169],[23.023742,70.202072],[24.546543,71.030497],[26.37005,70.986262],[28.165547,71.185474],[31.293418,70.453788],[30.005435,70.186259],[31.101042,69.558101]]],[[[27.407506,80.056406],[25.924651,79.517834],[23.024466,79.400012],[20.075188,79.566823],[19.897266,79.842362],[18.462264,79.85988],[17.368015,80.318896],[20.455992,80.598156],[21.907945,80.357679],[22.919253,80.657144],[25.447625,80.40734],[27.407506,80.056406]]],[[[24.72412,77.85385],[22.49032,77.44493],[20.72601,77.67704],[21.41611,77.93504],[20.8119,78.25463],[22.88426,78.45494],[23.28134,78.07954],[24.72412,77.85385]]]]}},{"type":"Feature","properties":{"ADMIN":"Greenland","NAME_EN":"Greenland","CONTINENT":"North America","ADM0_A3":"GRL","ISO_A3":"GRL","NAME":"Greenland","ISO_A2":"GL","NAME_ZH":"格陵兰"},"geometry":{"type":"Polygon","coordinates":[[[-46.76379,82.62796],[-43.40644,83.22516],[-39.89753,83.18018],[-38.62214,83.54905],[-35.08787,83.64513],[-27.10046,83.51966],[-20.84539,82.72669],[-22.69182,82.34165],[-26.51753,82.29765],[-31.9,82.2],[-31.39646,82.02154],[-27.85666,82.13178],[-24.84448,81.78697],[-22.90328,82.09317],[-22.07175,81.73449],[-23.16961,81.15271],[-20.62363,81.52462],[-15.76818,81.91245],[-12.77018,81.71885],[-12.20855,81.29154],[-16.28533,80.58004],[-16.85,80.35],[-20.04624,80.17708],[-17.73035,80.12912],[-18.9,79.4],[-19.70499,78.75128],[-19.67353,77.63859],[-18.47285,76.98565],[-20.03503,76.94434],[-21.67944,76.62795],[-19.83407,76.09808],[-19.59896,75.24838],[-20.66818,75.15585],[-19.37281,74.29561],[-21.59422,74.22382],[-20.43454,73.81713],[-20.76234,73.46436],[-22.17221,73.30955],[-23.56593,73.30663],[-22.31311,72.62928],[-22.29954,72.18409],[-24.27834,72.59788],[-24.79296,72.3302],[-23.44296,72.08016],[-22.13281,71.46898],[-21.75356,70.66369],[-23.53603,70.471],[-24.30702,70.85649],[-25.54341,71.43094],[-25.20135,70.75226],[-26.36276,70.22646],[-23.72742,70.18401],[-22.34902,70.12946],[-25.02927,69.2588],[-27.74737,68.47046],[-30.67371,68.12503],[-31.77665,68.12078],[-32.81105,67.73547],[-34.20196,66.67974],[-36.35284,65.9789],[-37.04378,65.93768],[-38.37505,65.69213],[-39.81222,65.45848],[-40.66899,64.83997],[-40.68281,64.13902],[-41.1887,63.48246],[-42.81938,62.68233],[-42.41666,61.90093],[-42.86619,61.07404],[-43.3784,60.09772],[-44.7875,60.03676],[-46.26364,60.85328],[-48.26294,60.85843],[-49.23308,61.40681],[-49.90039,62.38336],[-51.63325,63.62691],[-52.14014,64.27842],[-52.27659,65.1767],[-53.66166,66.09957],[-53.30161,66.8365],[-53.96911,67.18899],[-52.9804,68.35759],[-51.47536,68.72958],[-51.08041,69.14781],[-50.87122,69.9291],[-52.013585,69.574925],[-52.55792,69.42616],[-53.45629,69.283625],[-54.68336,69.61003],[-54.75001,70.28932],[-54.35884,70.821315],[-53.431315,70.835755],[-51.39014,70.56978],[-53.10937,71.20485],[-54.00422,71.54719],[-55,71.406537],[-55.83468,71.65444],[-54.71819,72.58625],[-55.32634,72.95861],[-56.12003,73.64977],[-57.32363,74.71026],[-58.59679,75.09861],[-58.58516,75.51727],[-61.26861,76.10238],[-63.39165,76.1752],[-66.06427,76.13486],[-68.50438,76.06141],[-69.66485,76.37975],[-71.40257,77.00857],[-68.77671,77.32312],[-66.76397,77.37595],[-71.04293,77.63595],[-73.297,78.04419],[-73.15938,78.43271],[-69.37345,78.91388],[-65.7107,79.39436],[-65.3239,79.75814],[-68.02298,80.11721],[-67.15129,80.51582],[-63.68925,81.21396],[-62.23444,81.3211],[-62.65116,81.77042],[-60.28249,82.03363],[-57.20744,82.19074],[-54.13442,82.19962],[-53.04328,81.88833],[-50.39061,82.43883],[-48.00386,82.06481],[-46.59984,81.985945],[-44.523,81.6607],[-46.9007,82.19979],[-46.76379,82.62796]]]}},{"type":"Feature","properties":{"ADMIN":"French Southern and Antarctic Lands","NAME_EN":"French Southern and Antarctic Lands","CONTINENT":"Seven seas (open ocean)","ADM0_A3":"ATF","ISO_A3":"ATF","NAME":"Fr. S. Antarctic Lands","ISO_A2":"TF","NAME_ZH":"法属南部和南极领地"},"geometry":{"type":"Polygon","coordinates":[[[68.935,-48.625],[69.58,-48.94],[70.525,-49.065],[70.56,-49.255],[70.28,-49.71],[68.745,-49.775],[68.72,-49.2425],[68.8675,-48.83],[68.935,-48.625]]]}},{"type":"Feature","properties":{"ADMIN":"East Timor","NAME_EN":"East Timor","CONTINENT":"Asia","ADM0_A3":"TLS","ISO_A3":"TLS","NAME":"Timor-Leste","ISO_A2":"TL","NAME_ZH":"东帝汶"},"geometry":{"type":"Polygon","coordinates":[[[124.968682,-8.89279],[125.086246,-8.656887],[125.947072,-8.432095],[126.644704,-8.398247],[126.957243,-8.273345],[127.335928,-8.397317],[126.967992,-8.668256],[125.925885,-9.106007],[125.08852,-9.393173],[125.07002,-9.089987],[124.968682,-8.89279]]]}},{"type":"Feature","properties":{"ADMIN":"South Africa","NAME_EN":"South Africa","CONTINENT":"Africa","ADM0_A3":"ZAF","ISO_A3":"ZAF","NAME":"South Africa","ISO_A2":"ZA","NAME_ZH":"南非"},"geometry":{"type":"Polygon","coordinates":[[[16.344977,-28.576705],[16.824017,-28.082162],[17.218929,-28.355943],[17.387497,-28.783514],[17.836152,-28.856378],[18.464899,-29.045462],[19.002127,-28.972443],[19.894734,-28.461105],[19.895768,-24.76779],[20.165726,-24.917962],[20.758609,-25.868136],[20.66647,-26.477453],[20.889609,-26.828543],[21.605896,-26.726534],[22.105969,-26.280256],[22.579532,-25.979448],[22.824271,-25.500459],[23.312097,-25.26869],[23.73357,-25.390129],[24.211267,-25.670216],[25.025171,-25.71967],[25.664666,-25.486816],[25.765849,-25.174845],[25.941652,-24.696373],[26.485753,-24.616327],[26.786407,-24.240691],[27.11941,-23.574323],[28.017236,-22.827754],[29.432188,-22.091313],[29.839037,-22.102216],[30.322883,-22.271612],[30.659865,-22.151567],[31.191409,-22.25151],[31.670398,-23.658969],[31.930589,-24.369417],[31.752408,-25.484284],[31.837778,-25.843332],[31.333158,-25.660191],[31.04408,-25.731452],[30.949667,-26.022649],[30.676609,-26.398078],[30.685962,-26.743845],[31.282773,-27.285879],[31.86806,-27.177927],[32.071665,-26.73382],[32.83012,-26.742192],[32.580265,-27.470158],[32.462133,-28.301011],[32.203389,-28.752405],[31.521001,-29.257387],[31.325561,-29.401978],[30.901763,-29.909957],[30.622813,-30.423776],[30.055716,-31.140269],[28.925553,-32.172041],[28.219756,-32.771953],[27.464608,-33.226964],[26.419452,-33.61495],[25.909664,-33.66704],[25.780628,-33.944646],[25.172862,-33.796851],[24.677853,-33.987176],[23.594043,-33.794474],[22.988189,-33.916431],[22.574157,-33.864083],[21.542799,-34.258839],[20.689053,-34.417175],[20.071261,-34.795137],[19.616405,-34.819166],[19.193278,-34.462599],[18.855315,-34.444306],[18.424643,-33.997873],[18.377411,-34.136521],[18.244499,-33.867752],[18.25008,-33.281431],[17.92519,-32.611291],[18.24791,-32.429131],[18.221762,-31.661633],[17.566918,-30.725721],[17.064416,-29.878641],[17.062918,-29.875954],[16.344977,-28.576705]],[[28.978263,-28.955597],[28.5417,-28.647502],[28.074338,-28.851469],[27.532511,-29.242711],[26.999262,-29.875954],[27.749397,-30.645106],[28.107205,-30.545732],[28.291069,-30.226217],[28.8484,-30.070051],[29.018415,-29.743766],[29.325166,-29.257387],[28.978263,-28.955597]]]}},{"type":"Feature","properties":{"ADMIN":"Lesotho","NAME_EN":"Lesotho","CONTINENT":"Africa","ADM0_A3":"LSO","ISO_A3":"LSO","NAME":"Lesotho","ISO_A2":"LS","NAME_ZH":"莱索托"},"geometry":{"type":"Polygon","coordinates":[[[28.978263,-28.955597],[29.325166,-29.257387],[29.018415,-29.743766],[28.8484,-30.070051],[28.291069,-30.226217],[28.107205,-30.545732],[27.749397,-30.645106],[26.999262,-29.875954],[27.532511,-29.242711],[28.074338,-28.851469],[28.5417,-28.647502],[28.978263,-28.955597]]]}},{"type":"Feature","properties":{"ADMIN":"Mexico","NAME_EN":"Mexico","CONTINENT":"North America","ADM0_A3":"MEX","ISO_A3":"MEX","NAME":"Mexico","ISO_A2":"MX","NAME_ZH":"墨西哥"},"geometry":{"type":"Polygon","coordinates":[[[-117.12776,32.53534],[-115.99135,32.61239],[-114.72139,32.72083],[-114.815,32.52528],[-113.30498,32.03914],[-111.02361,31.33472],[-109.035,31.34194],[-108.24194,31.34222],[-108.24,31.754854],[-106.50759,31.75452],[-106.1429,31.39995],[-105.63159,31.08383],[-105.03737,30.64402],[-104.70575,30.12173],[-104.45697,29.57196],[-103.94,29.27],[-103.11,28.97],[-102.48,29.76],[-101.6624,29.7793],[-100.9576,29.38071],[-100.45584,28.69612],[-100.11,28.11],[-99.52,27.54],[-99.3,26.84],[-99.02,26.37],[-98.24,26.06],[-97.53,25.84],[-97.140008,25.869997],[-97.528072,24.992144],[-97.702946,24.272343],[-97.776042,22.93258],[-97.872367,22.444212],[-97.699044,21.898689],[-97.38896,21.411019],[-97.189333,20.635433],[-96.525576,19.890931],[-96.292127,19.320371],[-95.900885,18.828024],[-94.839063,18.562717],[-94.42573,18.144371],[-93.548651,18.423837],[-92.786114,18.524839],[-92.037348,18.704569],[-91.407903,18.876083],[-90.77187,19.28412],[-90.53359,19.867418],[-90.451476,20.707522],[-90.278618,20.999855],[-89.601321,21.261726],[-88.543866,21.493675],[-87.658417,21.458846],[-87.05189,21.543543],[-86.811982,21.331515],[-86.845908,20.849865],[-87.383291,20.255405],[-87.621054,19.646553],[-87.43675,19.472403],[-87.58656,19.04013],[-87.837191,18.259816],[-88.090664,18.516648],[-88.300031,18.499982],[-88.490123,18.486831],[-88.848344,17.883198],[-89.029857,18.001511],[-89.150909,17.955468],[-89.14308,17.808319],[-90.067934,17.819326],[-91.00152,17.817595],[-91.002269,17.254658],[-91.453921,17.252177],[-91.08167,16.918477],[-90.711822,16.687483],[-90.600847,16.470778],[-90.438867,16.41011],[-90.464473,16.069562],[-91.74796,16.066565],[-92.229249,15.251447],[-92.087216,15.064585],[-92.20323,14.830103],[-92.22775,14.538829],[-93.359464,15.61543],[-93.875169,15.940164],[-94.691656,16.200975],[-95.250227,16.128318],[-96.053382,15.752088],[-96.557434,15.653515],[-97.263592,15.917065],[-98.01303,16.107312],[-98.947676,16.566043],[-99.697397,16.706164],[-100.829499,17.171071],[-101.666089,17.649026],[-101.918528,17.91609],[-102.478132,17.975751],[-103.50099,18.292295],[-103.917527,18.748572],[-104.99201,19.316134],[-105.493038,19.946767],[-105.731396,20.434102],[-105.397773,20.531719],[-105.500661,20.816895],[-105.270752,21.076285],[-105.265817,21.422104],[-105.603161,21.871146],[-105.693414,22.26908],[-106.028716,22.773752],[-106.90998,23.767774],[-107.915449,24.548915],[-108.401905,25.172314],[-109.260199,25.580609],[-109.444089,25.824884],[-109.291644,26.442934],[-109.801458,26.676176],[-110.391732,27.162115],[-110.641019,27.859876],[-111.178919,27.941241],[-111.759607,28.467953],[-112.228235,28.954409],[-112.271824,29.266844],[-112.809594,30.021114],[-113.163811,30.786881],[-113.148669,31.170966],[-113.871881,31.567608],[-114.205737,31.524045],[-114.776451,31.799532],[-114.9367,31.393485],[-114.771232,30.913617],[-114.673899,30.162681],[-114.330974,29.750432],[-113.588875,29.061611],[-113.424053,28.826174],[-113.271969,28.754783],[-113.140039,28.411289],[-112.962298,28.42519],[-112.761587,27.780217],[-112.457911,27.525814],[-112.244952,27.171727],[-111.616489,26.662817],[-111.284675,25.73259],[-110.987819,25.294606],[-110.710007,24.826004],[-110.655049,24.298595],[-110.172856,24.265548],[-109.771847,23.811183],[-109.409104,23.364672],[-109.433392,23.185588],[-109.854219,22.818272],[-110.031392,22.823078],[-110.295071,23.430973],[-110.949501,24.000964],[-111.670568,24.484423],[-112.182036,24.738413],[-112.148989,25.470125],[-112.300711,26.012004],[-112.777297,26.32196],[-113.464671,26.768186],[-113.59673,26.63946],[-113.848937,26.900064],[-114.465747,27.14209],[-115.055142,27.722727],[-114.982253,27.7982],[-114.570366,27.741485],[-114.199329,28.115003],[-114.162018,28.566112],[-114.931842,29.279479],[-115.518654,29.556362],[-115.887365,30.180794],[-116.25835,30.836464],[-116.721526,31.635744],[-117.12776,32.53534]]]}},{"type":"Feature","properties":{"ADMIN":"Uruguay","NAME_EN":"Uruguay","CONTINENT":"South America","ADM0_A3":"URY","ISO_A3":"URY","NAME":"Uruguay","ISO_A2":"UY","NAME_ZH":"乌拉圭"},"geometry":{"type":"Polygon","coordinates":[[[-57.625133,-30.216295],[-56.976026,-30.109686],[-55.973245,-30.883076],[-55.60151,-30.853879],[-54.572452,-31.494511],[-53.787952,-32.047243],[-53.209589,-32.727666],[-53.650544,-33.202004],[-53.373662,-33.768378],[-53.806426,-34.396815],[-54.935866,-34.952647],[-55.67409,-34.752659],[-56.215297,-34.859836],[-57.139685,-34.430456],[-57.817861,-34.462547],[-58.427074,-33.909454],[-58.349611,-33.263189],[-58.132648,-33.040567],[-58.14244,-32.044504],[-57.874937,-31.016556],[-57.625133,-30.216295]]]}},{"type":"Feature","properties":{"ADMIN":"Brazil","NAME_EN":"Brazil","CONTINENT":"South America","ADM0_A3":"BRA","ISO_A3":"BRA","NAME":"Brazil","ISO_A2":"BR","NAME_ZH":"巴西"},"geometry":{"type":"Polygon","coordinates":[[[-53.373662,-33.768378],[-53.650544,-33.202004],[-53.209589,-32.727666],[-53.787952,-32.047243],[-54.572452,-31.494511],[-55.60151,-30.853879],[-55.973245,-30.883076],[-56.976026,-30.109686],[-57.625133,-30.216295],[-56.2909,-28.852761],[-55.162286,-27.881915],[-54.490725,-27.474757],[-53.648735,-26.923473],[-53.628349,-26.124865],[-54.13005,-25.547639],[-54.625291,-25.739255],[-54.428946,-25.162185],[-54.293476,-24.5708],[-54.29296,-24.021014],[-54.652834,-23.839578],[-55.027902,-24.001274],[-55.400747,-23.956935],[-55.517639,-23.571998],[-55.610683,-22.655619],[-55.797958,-22.35693],[-56.473317,-22.0863],[-56.88151,-22.282154],[-57.937156,-22.090176],[-57.870674,-20.732688],[-58.166392,-20.176701],[-57.853802,-19.969995],[-57.949997,-19.400004],[-57.676009,-18.96184],[-57.498371,-18.174188],[-57.734558,-17.552468],[-58.280804,-17.27171],[-58.388058,-16.877109],[-58.24122,-16.299573],[-60.15839,-16.258284],[-60.542966,-15.09391],[-60.251149,-15.077219],[-60.264326,-14.645979],[-60.459198,-14.354007],[-60.503304,-13.775955],[-61.084121,-13.479384],[-61.713204,-13.489202],[-62.127081,-13.198781],[-62.80306,-13.000653],[-63.196499,-12.627033],[-64.316353,-12.461978],[-65.402281,-11.56627],[-65.321899,-10.895872],[-65.444837,-10.511451],[-65.338435,-9.761988],[-66.646908,-9.931331],[-67.173801,-10.306812],[-68.048192,-10.712059],[-68.271254,-11.014521],[-68.786158,-11.03638],[-69.529678,-10.951734],[-70.093752,-11.123972],[-70.548686,-11.009147],[-70.481894,-9.490118],[-71.302412,-10.079436],[-72.184891,-10.053598],[-72.563033,-9.520194],[-73.226713,-9.462213],[-73.015383,-9.032833],[-73.571059,-8.424447],[-73.987235,-7.52383],[-73.723401,-7.340999],[-73.724487,-6.918595],[-73.120027,-6.629931],[-73.219711,-6.089189],[-72.964507,-5.741251],[-72.891928,-5.274561],[-71.748406,-4.593983],[-70.928843,-4.401591],[-70.794769,-4.251265],[-69.893635,-4.298187],[-69.444102,-1.556287],[-69.420486,-1.122619],[-69.577065,-0.549992],[-70.020656,-0.185156],[-70.015566,0.541414],[-69.452396,0.706159],[-69.252434,0.602651],[-69.218638,0.985677],[-69.804597,1.089081],[-69.816973,1.714805],[-67.868565,1.692455],[-67.53781,2.037163],[-67.259998,1.719999],[-67.065048,1.130112],[-66.876326,1.253361],[-66.325765,0.724452],[-65.548267,0.789254],[-65.354713,1.095282],[-64.611012,1.328731],[-64.199306,1.492855],[-64.083085,1.916369],[-63.368788,2.2009],[-63.422867,2.411068],[-64.269999,2.497006],[-64.408828,3.126786],[-64.368494,3.79721],[-64.816064,4.056445],[-64.628659,4.148481],[-63.888343,4.02053],[-63.093198,3.770571],[-62.804533,4.006965],[-62.08543,4.162124],[-60.966893,4.536468],[-60.601179,4.918098],[-60.733574,5.200277],[-60.213683,5.244486],[-59.980959,5.014061],[-60.111002,4.574967],[-59.767406,4.423503],[-59.53804,3.958803],[-59.815413,3.606499],[-59.974525,2.755233],[-59.718546,2.24963],[-59.646044,1.786894],[-59.030862,1.317698],[-58.540013,1.268088],[-58.429477,1.463942],[-58.11345,1.507195],[-57.660971,1.682585],[-57.335823,1.948538],[-56.782704,1.863711],[-56.539386,1.899523],[-55.995698,1.817667],[-55.9056,2.021996],[-56.073342,2.220795],[-55.973322,2.510364],[-55.569755,2.421506],[-55.097587,2.523748],[-54.524754,2.311849],[-54.088063,2.105557],[-53.778521,2.376703],[-53.554839,2.334897],[-53.418465,2.053389],[-52.939657,2.124858],[-52.556425,2.504705],[-52.249338,3.241094],[-51.657797,4.156232],[-51.317146,4.203491],[-51.069771,3.650398],[-50.508875,1.901564],[-49.974076,1.736483],[-49.947101,1.04619],[-50.699251,0.222984],[-50.388211,-0.078445],[-48.620567,-0.235489],[-48.584497,-1.237805],[-47.824956,-0.581618],[-46.566584,-0.941028],[-44.905703,-1.55174],[-44.417619,-2.13775],[-44.581589,-2.691308],[-43.418791,-2.38311],[-41.472657,-2.912018],[-39.978665,-2.873054],[-38.500383,-3.700652],[-37.223252,-4.820946],[-36.452937,-5.109404],[-35.597796,-5.149504],[-35.235389,-5.464937],[-34.89603,-6.738193],[-34.729993,-7.343221],[-35.128212,-8.996401],[-35.636967,-9.649282],[-37.046519,-11.040721],[-37.683612,-12.171195],[-38.423877,-13.038119],[-38.673887,-13.057652],[-38.953276,-13.79337],[-38.882298,-15.667054],[-39.161092,-17.208407],[-39.267339,-17.867746],[-39.583521,-18.262296],[-39.760823,-19.599113],[-40.774741,-20.904512],[-40.944756,-21.937317],[-41.754164,-22.370676],[-41.988284,-22.97007],[-43.074704,-22.967693],[-44.647812,-23.351959],[-45.352136,-23.796842],[-46.472093,-24.088969],[-47.648972,-24.885199],[-48.495458,-25.877025],[-48.641005,-26.623698],[-48.474736,-27.175912],[-48.66152,-28.186135],[-48.888457,-28.674115],[-49.587329,-29.224469],[-50.696874,-30.984465],[-51.576226,-31.777698],[-52.256081,-32.24537],[-52.7121,-33.196578],[-53.373662,-33.768378]]]}},{"type":"Feature","properties":{"ADMIN":"Bolivia","NAME_EN":"Bolivia","CONTINENT":"South America","ADM0_A3":"BOL","ISO_A3":"BOL","NAME":"Bolivia","ISO_A2":"BO","NAME_ZH":"玻利维亚"},"geometry":{"type":"Polygon","coordinates":[[[-69.529678,-10.951734],[-68.786158,-11.03638],[-68.271254,-11.014521],[-68.048192,-10.712059],[-67.173801,-10.306812],[-66.646908,-9.931331],[-65.338435,-9.761988],[-65.444837,-10.511451],[-65.321899,-10.895872],[-65.402281,-11.56627],[-64.316353,-12.461978],[-63.196499,-12.627033],[-62.80306,-13.000653],[-62.127081,-13.198781],[-61.713204,-13.489202],[-61.084121,-13.479384],[-60.503304,-13.775955],[-60.459198,-14.354007],[-60.264326,-14.645979],[-60.251149,-15.077219],[-60.542966,-15.09391],[-60.15839,-16.258284],[-58.24122,-16.299573],[-58.388058,-16.877109],[-58.280804,-17.27171],[-57.734558,-17.552468],[-57.498371,-18.174188],[-57.676009,-18.96184],[-57.949997,-19.400004],[-57.853802,-19.969995],[-58.166392,-20.176701],[-58.183471,-19.868399],[-59.115042,-19.356906],[-60.043565,-19.342747],[-61.786326,-19.633737],[-62.265961,-20.513735],[-62.291179,-21.051635],[-62.685057,-22.249029],[-62.846468,-22.034985],[-63.986838,-21.993644],[-64.377021,-22.798091],[-64.964892,-22.075862],[-66.273339,-21.83231],[-67.106674,-22.735925],[-67.82818,-22.872919],[-68.219913,-21.494347],[-68.757167,-20.372658],[-68.442225,-19.405068],[-68.966818,-18.981683],[-69.100247,-18.260125],[-69.590424,-17.580012],[-68.959635,-16.500698],[-69.389764,-15.660129],[-69.160347,-15.323974],[-69.339535,-14.953195],[-68.948887,-14.453639],[-68.929224,-13.602684],[-68.88008,-12.899729],[-68.66508,-12.5613],[-69.529678,-10.951734]]]}},{"type":"Feature","properties":{"ADMIN":"Peru","NAME_EN":"Peru","CONTINENT":"South America","ADM0_A3":"PER","ISO_A3":"PER","NAME":"Peru","ISO_A2":"PE","NAME_ZH":"秘鲁"},"geometry":{"type":"Polygon","coordinates":[[[-69.893635,-4.298187],[-70.794769,-4.251265],[-70.928843,-4.401591],[-71.748406,-4.593983],[-72.891928,-5.274561],[-72.964507,-5.741251],[-73.219711,-6.089189],[-73.120027,-6.629931],[-73.724487,-6.918595],[-73.723401,-7.340999],[-73.987235,-7.52383],[-73.571059,-8.424447],[-73.015383,-9.032833],[-73.226713,-9.462213],[-72.563033,-9.520194],[-72.184891,-10.053598],[-71.302412,-10.079436],[-70.481894,-9.490118],[-70.548686,-11.009147],[-70.093752,-11.123972],[-69.529678,-10.951734],[-68.66508,-12.5613],[-68.88008,-12.899729],[-68.929224,-13.602684],[-68.948887,-14.453639],[-69.339535,-14.953195],[-69.160347,-15.323974],[-69.389764,-15.660129],[-68.959635,-16.500698],[-69.590424,-17.580012],[-69.858444,-18.092694],[-70.372572,-18.347975],[-71.37525,-17.773799],[-71.462041,-17.363488],[-73.44453,-16.359363],[-75.237883,-15.265683],[-76.009205,-14.649286],[-76.423469,-13.823187],[-76.259242,-13.535039],[-77.106192,-12.222716],[-78.092153,-10.377712],[-79.036953,-8.386568],[-79.44592,-7.930833],[-79.760578,-7.194341],[-80.537482,-6.541668],[-81.249996,-6.136834],[-80.926347,-5.690557],[-81.410943,-4.736765],[-81.09967,-4.036394],[-80.302561,-3.404856],[-80.184015,-3.821162],[-80.469295,-4.059287],[-80.442242,-4.425724],[-80.028908,-4.346091],[-79.624979,-4.454198],[-79.205289,-4.959129],[-78.639897,-4.547784],[-78.450684,-3.873097],[-77.837905,-3.003021],[-76.635394,-2.608678],[-75.544996,-1.56161],[-75.233723,-0.911417],[-75.373223,-0.152032],[-75.106625,-0.057205],[-74.441601,-0.53082],[-74.122395,-1.002833],[-73.659504,-1.260491],[-73.070392,-2.308954],[-72.325787,-2.434218],[-71.774761,-2.16979],[-71.413646,-2.342802],[-70.813476,-2.256865],[-70.047709,-2.725156],[-70.692682,-3.742872],[-70.394044,-3.766591],[-69.893635,-4.298187]]]}},{"type":"Feature","properties":{"ADMIN":"Colombia","NAME_EN":"Colombia","CONTINENT":"South America","ADM0_A3":"COL","ISO_A3":"COL","NAME":"Colombia","ISO_A2":"CO","NAME_ZH":"哥伦比亚"},"geometry":{"type":"Polygon","coordinates":[[[-66.876326,1.253361],[-67.065048,1.130112],[-67.259998,1.719999],[-67.53781,2.037163],[-67.868565,1.692455],[-69.816973,1.714805],[-69.804597,1.089081],[-69.218638,0.985677],[-69.252434,0.602651],[-69.452396,0.706159],[-70.015566,0.541414],[-70.020656,-0.185156],[-69.577065,-0.549992],[-69.420486,-1.122619],[-69.444102,-1.556287],[-69.893635,-4.298187],[-70.394044,-3.766591],[-70.692682,-3.742872],[-70.047709,-2.725156],[-70.813476,-2.256865],[-71.413646,-2.342802],[-71.774761,-2.16979],[-72.325787,-2.434218],[-73.070392,-2.308954],[-73.659504,-1.260491],[-74.122395,-1.002833],[-74.441601,-0.53082],[-75.106625,-0.057205],[-75.373223,-0.152032],[-75.801466,0.084801],[-76.292314,0.416047],[-76.57638,0.256936],[-77.424984,0.395687],[-77.668613,0.825893],[-77.855061,0.809925],[-78.855259,1.380924],[-78.990935,1.69137],[-78.617831,1.766404],[-78.662118,2.267355],[-78.42761,2.629556],[-77.931543,2.696606],[-77.510431,3.325017],[-77.12769,3.849636],[-77.496272,4.087606],[-77.307601,4.667984],[-77.533221,5.582812],[-77.318815,5.845354],[-77.476661,6.691116],[-77.881571,7.223771],[-77.753414,7.70984],[-77.431108,7.638061],[-77.242566,7.935278],[-77.474723,8.524286],[-77.353361,8.670505],[-76.836674,8.638749],[-76.086384,9.336821],[-75.6746,9.443248],[-75.664704,9.774003],[-75.480426,10.61899],[-74.906895,11.083045],[-74.276753,11.102036],[-74.197223,11.310473],[-73.414764,11.227015],[-72.627835,11.731972],[-72.238195,11.95555],[-71.75409,12.437303],[-71.399822,12.376041],[-71.137461,12.112982],[-71.331584,11.776284],[-71.973922,11.608672],[-72.227575,11.108702],[-72.614658,10.821975],[-72.905286,10.450344],[-73.027604,9.73677],[-73.304952,9.152],[-72.78873,9.085027],[-72.660495,8.625288],[-72.439862,8.405275],[-72.360901,8.002638],[-72.479679,7.632506],[-72.444487,7.423785],[-72.198352,7.340431],[-71.960176,6.991615],[-70.674234,7.087785],[-70.093313,6.960376],[-69.38948,6.099861],[-68.985319,6.206805],[-68.265052,6.153268],[-67.695087,6.267318],[-67.34144,6.095468],[-67.521532,5.55687],[-67.744697,5.221129],[-67.823012,4.503937],[-67.621836,3.839482],[-67.337564,3.542342],[-67.303173,3.318454],[-67.809938,2.820655],[-67.447092,2.600281],[-67.181294,2.250638],[-66.876326,1.253361]]]}},{"type":"Feature","properties":{"ADMIN":"Panama","NAME_EN":"Panama","CONTINENT":"North America","ADM0_A3":"PAN","ISO_A3":"PAN","NAME":"Panama","ISO_A2":"PA","NAME_ZH":"巴拿马"},"geometry":{"type":"Polygon","coordinates":[[[-77.353361,8.670505],[-77.474723,8.524286],[-77.242566,7.935278],[-77.431108,7.638061],[-77.753414,7.70984],[-77.881571,7.223771],[-78.214936,7.512255],[-78.429161,8.052041],[-78.182096,8.319182],[-78.435465,8.387705],[-78.622121,8.718124],[-79.120307,8.996092],[-79.557877,8.932375],[-79.760578,8.584515],[-80.164481,8.333316],[-80.382659,8.298409],[-80.480689,8.090308],[-80.00369,7.547524],[-80.276671,7.419754],[-80.421158,7.271572],[-80.886401,7.220541],[-81.059543,7.817921],[-81.189716,7.647906],[-81.519515,7.70661],[-81.721311,8.108963],[-82.131441,8.175393],[-82.390934,8.292362],[-82.820081,8.290864],[-82.850958,8.073823],[-82.965783,8.225028],[-82.913176,8.423517],[-82.829771,8.626295],[-82.868657,8.807266],[-82.719183,8.925709],[-82.927155,9.07433],[-82.932891,9.476812],[-82.546196,9.566135],[-82.187123,9.207449],[-82.207586,8.995575],[-81.808567,8.950617],[-81.714154,9.031955],[-81.439287,8.786234],[-80.947302,8.858504],[-80.521901,9.111072],[-79.9146,9.312765],[-79.573303,9.61161],[-79.021192,9.552931],[-79.05845,9.454565],[-78.500888,9.420459],[-78.055928,9.24773],[-77.729514,8.946844],[-77.353361,8.670505]]]}},{"type":"Feature","properties":{"ADMIN":"Costa Rica","NAME_EN":"Costa Rica","CONTINENT":"North America","ADM0_A3":"CRI","ISO_A3":"CRI","NAME":"Costa Rica","ISO_A2":"CR","NAME_ZH":"哥斯达黎加"},"geometry":{"type":"Polygon","coordinates":[[[-82.546196,9.566135],[-82.932891,9.476812],[-82.927155,9.07433],[-82.719183,8.925709],[-82.868657,8.807266],[-82.829771,8.626295],[-82.913176,8.423517],[-82.965783,8.225028],[-83.508437,8.446927],[-83.711474,8.656836],[-83.596313,8.830443],[-83.632642,9.051386],[-83.909886,9.290803],[-84.303402,9.487354],[-84.647644,9.615537],[-84.713351,9.908052],[-84.97566,10.086723],[-84.911375,9.795992],[-85.110923,9.55704],[-85.339488,9.834542],[-85.660787,9.933347],[-85.797445,10.134886],[-85.791709,10.439337],[-85.659314,10.754331],[-85.941725,10.895278],[-85.71254,11.088445],[-85.561852,11.217119],[-84.903003,10.952303],[-84.673069,11.082657],[-84.355931,10.999226],[-84.190179,10.79345],[-83.895054,10.726839],[-83.655612,10.938764],[-83.40232,10.395438],[-83.015677,9.992982],[-82.546196,9.566135]]]}},{"type":"Feature","properties":{"ADMIN":"Nicaragua","NAME_EN":"Nicaragua","CONTINENT":"North America","ADM0_A3":"NIC","ISO_A3":"NIC","NAME":"Nicaragua","ISO_A2":"NI","NAME_ZH":"尼加拉瓜"},"geometry":{"type":"Polygon","coordinates":[[[-83.655612,10.938764],[-83.895054,10.726839],[-84.190179,10.79345],[-84.355931,10.999226],[-84.673069,11.082657],[-84.903003,10.952303],[-85.561852,11.217119],[-85.71254,11.088445],[-86.058488,11.403439],[-86.52585,11.806877],[-86.745992,12.143962],[-87.167516,12.458258],[-87.668493,12.90991],[-87.557467,13.064552],[-87.392386,12.914018],[-87.316654,12.984686],[-87.005769,13.025794],[-86.880557,13.254204],[-86.733822,13.263093],[-86.755087,13.754845],[-86.520708,13.778487],[-86.312142,13.771356],[-86.096264,14.038187],[-85.801295,13.836055],[-85.698665,13.960078],[-85.514413,14.079012],[-85.165365,14.35437],[-85.148751,14.560197],[-85.052787,14.551541],[-84.924501,14.790493],[-84.820037,14.819587],[-84.649582,14.666805],[-84.449336,14.621614],[-84.228342,14.748764],[-83.975721,14.749436],[-83.628585,14.880074],[-83.489989,15.016267],[-83.147219,14.995829],[-83.233234,14.899866],[-83.284162,14.676624],[-83.182126,14.310703],[-83.4125,13.970078],[-83.519832,13.567699],[-83.552207,13.127054],[-83.498515,12.869292],[-83.473323,12.419087],[-83.626104,12.32085],[-83.719613,11.893124],[-83.650858,11.629032],[-83.85547,11.373311],[-83.808936,11.103044],[-83.655612,10.938764]]]}},{"type":"Feature","properties":{"ADMIN":"Honduras","NAME_EN":"Honduras","CONTINENT":"North America","ADM0_A3":"HND","ISO_A3":"HND","NAME":"Honduras","ISO_A2":"HN","NAME_ZH":"洪都拉斯"},"geometry":{"type":"Polygon","coordinates":[[[-83.147219,14.995829],[-83.489989,15.016267],[-83.628585,14.880074],[-83.975721,14.749436],[-84.228342,14.748764],[-84.449336,14.621614],[-84.649582,14.666805],[-84.820037,14.819587],[-84.924501,14.790493],[-85.052787,14.551541],[-85.148751,14.560197],[-85.165365,14.35437],[-85.514413,14.079012],[-85.698665,13.960078],[-85.801295,13.836055],[-86.096264,14.038187],[-86.312142,13.771356],[-86.520708,13.778487],[-86.755087,13.754845],[-86.733822,13.263093],[-86.880557,13.254204],[-87.005769,13.025794],[-87.316654,12.984686],[-87.489409,13.297535],[-87.793111,13.38448],[-87.723503,13.78505],[-87.859515,13.893312],[-88.065343,13.964626],[-88.503998,13.845486],[-88.541231,13.980155],[-88.843073,14.140507],[-89.058512,14.340029],[-89.353326,14.424133],[-89.145535,14.678019],[-89.22522,14.874286],[-89.154811,15.066419],[-88.68068,15.346247],[-88.225023,15.727722],[-88.121153,15.688655],[-87.901813,15.864458],[-87.61568,15.878799],[-87.522921,15.797279],[-87.367762,15.84694],[-86.903191,15.756713],[-86.440946,15.782835],[-86.119234,15.893449],[-86.001954,16.005406],[-85.683317,15.953652],[-85.444004,15.885749],[-85.182444,15.909158],[-84.983722,15.995923],[-84.52698,15.857224],[-84.368256,15.835158],[-84.063055,15.648244],[-83.773977,15.424072],[-83.410381,15.270903],[-83.147219,14.995829]]]}},{"type":"Feature","properties":{"ADMIN":"El Salvador","NAME_EN":"El Salvador","CONTINENT":"North America","ADM0_A3":"SLV","ISO_A3":"SLV","NAME":"El Salvador","ISO_A2":"SV","NAME_ZH":"萨尔瓦多"},"geometry":{"type":"Polygon","coordinates":[[[-89.353326,14.424133],[-89.058512,14.340029],[-88.843073,14.140507],[-88.541231,13.980155],[-88.503998,13.845486],[-88.065343,13.964626],[-87.859515,13.893312],[-87.723503,13.78505],[-87.793111,13.38448],[-87.904112,13.149017],[-88.483302,13.163951],[-88.843228,13.259734],[-89.256743,13.458533],[-89.812394,13.520622],[-90.095555,13.735338],[-90.064678,13.88197],[-89.721934,14.134228],[-89.534219,14.244816],[-89.587343,14.362586],[-89.353326,14.424133]]]}},{"type":"Feature","properties":{"ADMIN":"Guatemala","NAME_EN":"Guatemala","CONTINENT":"North America","ADM0_A3":"GTM","ISO_A3":"GTM","NAME":"Guatemala","ISO_A2":"GT","NAME_ZH":"危地马拉"},"geometry":{"type":"Polygon","coordinates":[[[-92.22775,14.538829],[-92.20323,14.830103],[-92.087216,15.064585],[-92.229249,15.251447],[-91.74796,16.066565],[-90.464473,16.069562],[-90.438867,16.41011],[-90.600847,16.470778],[-90.711822,16.687483],[-91.08167,16.918477],[-91.453921,17.252177],[-91.002269,17.254658],[-91.00152,17.817595],[-90.067934,17.819326],[-89.14308,17.808319],[-89.150806,17.015577],[-89.229122,15.886938],[-88.930613,15.887273],[-88.604586,15.70638],[-88.518364,15.855389],[-88.225023,15.727722],[-88.68068,15.346247],[-89.154811,15.066419],[-89.22522,14.874286],[-89.145535,14.678019],[-89.353326,14.424133],[-89.587343,14.362586],[-89.534219,14.244816],[-89.721934,14.134228],[-90.064678,13.88197],[-90.095555,13.735338],[-90.608624,13.909771],[-91.23241,13.927832],[-91.689747,14.126218],[-92.22775,14.538829]]]}},{"type":"Feature","properties":{"ADMIN":"Belize","NAME_EN":"Belize","CONTINENT":"North America","ADM0_A3":"BLZ","ISO_A3":"BLZ","NAME":"Belize","ISO_A2":"BZ","NAME_ZH":"伯利兹"},"geometry":{"type":"Polygon","coordinates":[[[-89.14308,17.808319],[-89.150909,17.955468],[-89.029857,18.001511],[-88.848344,17.883198],[-88.490123,18.486831],[-88.300031,18.499982],[-88.296336,18.353273],[-88.106813,18.348674],[-88.123479,18.076675],[-88.285355,17.644143],[-88.197867,17.489475],[-88.302641,17.131694],[-88.239518,17.036066],[-88.355428,16.530774],[-88.551825,16.265467],[-88.732434,16.233635],[-88.930613,15.887273],[-89.229122,15.886938],[-89.150806,17.015577],[-89.14308,17.808319]]]}},{"type":"Feature","properties":{"ADMIN":"Venezuela","NAME_EN":"Venezuela","CONTINENT":"South America","ADM0_A3":"VEN","ISO_A3":"VEN","NAME":"Venezuela","ISO_A2":"VE","NAME_ZH":"委内瑞拉"},"geometry":{"type":"Polygon","coordinates":[[[-60.733574,5.200277],[-60.601179,4.918098],[-60.966893,4.536468],[-62.08543,4.162124],[-62.804533,4.006965],[-63.093198,3.770571],[-63.888343,4.02053],[-64.628659,4.148481],[-64.816064,4.056445],[-64.368494,3.79721],[-64.408828,3.126786],[-64.269999,2.497006],[-63.422867,2.411068],[-63.368788,2.2009],[-64.083085,1.916369],[-64.199306,1.492855],[-64.611012,1.328731],[-65.354713,1.095282],[-65.548267,0.789254],[-66.325765,0.724452],[-66.876326,1.253361],[-67.181294,2.250638],[-67.447092,2.600281],[-67.809938,2.820655],[-67.303173,3.318454],[-67.337564,3.542342],[-67.621836,3.839482],[-67.823012,4.503937],[-67.744697,5.221129],[-67.521532,5.55687],[-67.34144,6.095468],[-67.695087,6.267318],[-68.265052,6.153268],[-68.985319,6.206805],[-69.38948,6.099861],[-70.093313,6.960376],[-70.674234,7.087785],[-71.960176,6.991615],[-72.198352,7.340431],[-72.444487,7.423785],[-72.479679,7.632506],[-72.360901,8.002638],[-72.439862,8.405275],[-72.660495,8.625288],[-72.78873,9.085027],[-73.304952,9.152],[-73.027604,9.73677],[-72.905286,10.450344],[-72.614658,10.821975],[-72.227575,11.108702],[-71.973922,11.608672],[-71.331584,11.776284],[-71.360006,11.539994],[-71.94705,11.423282],[-71.620868,10.96946],[-71.633064,10.446494],[-72.074174,9.865651],[-71.695644,9.072263],[-71.264559,9.137195],[-71.039999,9.859993],[-71.350084,10.211935],[-71.400623,10.968969],[-70.155299,11.375482],[-70.293843,11.846822],[-69.943245,12.162307],[-69.5843,11.459611],[-68.882999,11.443385],[-68.233271,10.885744],[-68.194127,10.554653],[-67.296249,10.545868],[-66.227864,10.648627],[-65.655238,10.200799],[-64.890452,10.077215],[-64.329479,10.389599],[-64.318007,10.641418],[-63.079322,10.701724],[-61.880946,10.715625],[-62.730119,10.420269],[-62.388512,9.948204],[-61.588767,9.873067],[-60.830597,9.38134],[-60.671252,8.580174],[-60.150096,8.602757],[-59.758285,8.367035],[-60.550588,7.779603],[-60.637973,7.415],[-60.295668,7.043911],[-60.543999,6.856584],[-61.159336,6.696077],[-61.139415,6.234297],[-61.410303,5.959068],[-60.733574,5.200277]]]}},{"type":"Feature","properties":{"ADMIN":"Guyana","NAME_EN":"Guyana","CONTINENT":"South America","ADM0_A3":"GUY","ISO_A3":"GUY","NAME":"Guyana","ISO_A2":"GY","NAME_ZH":"圭亚那"},"geometry":{"type":"Polygon","coordinates":[[[-56.539386,1.899523],[-56.782704,1.863711],[-57.335823,1.948538],[-57.660971,1.682585],[-58.11345,1.507195],[-58.429477,1.463942],[-58.540013,1.268088],[-59.030862,1.317698],[-59.646044,1.786894],[-59.718546,2.24963],[-59.974525,2.755233],[-59.815413,3.606499],[-59.53804,3.958803],[-59.767406,4.423503],[-60.111002,4.574967],[-59.980959,5.014061],[-60.213683,5.244486],[-60.733574,5.200277],[-61.410303,5.959068],[-61.139415,6.234297],[-61.159336,6.696077],[-60.543999,6.856584],[-60.295668,7.043911],[-60.637973,7.415],[-60.550588,7.779603],[-59.758285,8.367035],[-59.101684,7.999202],[-58.482962,7.347691],[-58.454876,6.832787],[-58.078103,6.809094],[-57.542219,6.321268],[-57.147436,5.97315],[-57.307246,5.073567],[-57.914289,4.812626],[-57.86021,4.576801],[-58.044694,4.060864],[-57.601569,3.334655],[-57.281433,3.333492],[-57.150098,2.768927],[-56.539386,1.899523]]]}},{"type":"Feature","properties":{"ADMIN":"Suriname","NAME_EN":"Suriname","CONTINENT":"South America","ADM0_A3":"SUR","ISO_A3":"SUR","NAME":"Suriname","ISO_A2":"SR","NAME_ZH":"苏里南"},"geometry":{"type":"Polygon","coordinates":[[[-54.524754,2.311849],[-55.097587,2.523748],[-55.569755,2.421506],[-55.973322,2.510364],[-56.073342,2.220795],[-55.9056,2.021996],[-55.995698,1.817667],[-56.539386,1.899523],[-57.150098,2.768927],[-57.281433,3.333492],[-57.601569,3.334655],[-58.044694,4.060864],[-57.86021,4.576801],[-57.914289,4.812626],[-57.307246,5.073567],[-57.147436,5.97315],[-55.949318,5.772878],[-55.84178,5.953125],[-55.03325,6.025291],[-53.958045,5.756548],[-54.478633,4.896756],[-54.399542,4.212611],[-54.006931,3.620038],[-54.181726,3.18978],[-54.269705,2.732392],[-54.524754,2.311849]]]}},{"type":"Feature","properties":{"ADMIN":"France","NAME_EN":"France","CONTINENT":"Europe","ADM0_A3":"FRA","ISO_A3":"-99","NAME":"France","ISO_A2":"-99","NAME_ZH":"法国"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-51.657797,4.156232],[-52.249338,3.241094],[-52.556425,2.504705],[-52.939657,2.124858],[-53.418465,2.053389],[-53.554839,2.334897],[-53.778521,2.376703],[-54.088063,2.105557],[-54.524754,2.311849],[-54.269705,2.732392],[-54.181726,3.18978],[-54.006931,3.620038],[-54.399542,4.212611],[-54.478633,4.896756],[-53.958045,5.756548],[-53.618453,5.646529],[-52.882141,5.409851],[-51.823343,4.565768],[-51.657797,4.156232]]],[[[6.18632,49.463803],[6.65823,49.201958],[8.099279,49.017784],[7.593676,48.333019],[7.466759,47.620582],[7.192202,47.449766],[6.736571,47.541801],[6.768714,47.287708],[6.037389,46.725779],[6.022609,46.27299],[6.5001,46.429673],[6.843593,45.991147],[6.802355,45.70858],[7.096652,45.333099],[6.749955,45.028518],[7.007562,44.254767],[7.549596,44.127901],[7.435185,43.693845],[6.529245,43.128892],[4.556963,43.399651],[3.100411,43.075201],[2.985999,42.473015],[1.826793,42.343385],[0.701591,42.795734],[0.338047,42.579546],[-1.502771,43.034014],[-1.901351,43.422802],[-1.384225,44.02261],[-1.193798,46.014918],[-2.225724,47.064363],[-2.963276,47.570327],[-4.491555,47.954954],[-4.59235,48.68416],[-3.295814,48.901692],[-1.616511,48.644421],[-1.933494,49.776342],[-0.989469,49.347376],[1.338761,50.127173],[1.639001,50.946606],[2.513573,51.148506],[2.658422,50.796848],[3.123252,50.780363],[3.588184,50.378992],[4.286023,49.907497],[4.799222,49.985373],[5.674052,49.529484],[5.897759,49.442667],[6.18632,49.463803]]],[[[8.746009,42.628122],[9.390001,43.009985],[9.560016,42.152492],[9.229752,41.380007],[8.775723,41.583612],[8.544213,42.256517],[8.746009,42.628122]]]]}},{"type":"Feature","properties":{"ADMIN":"Ecuador","NAME_EN":"Ecuador","CONTINENT":"South America","ADM0_A3":"ECU","ISO_A3":"ECU","NAME":"Ecuador","ISO_A2":"EC","NAME_ZH":"厄瓜多尔"},"geometry":{"type":"Polygon","coordinates":[[[-75.373223,-0.152032],[-75.233723,-0.911417],[-75.544996,-1.56161],[-76.635394,-2.608678],[-77.837905,-3.003021],[-78.450684,-3.873097],[-78.639897,-4.547784],[-79.205289,-4.959129],[-79.624979,-4.454198],[-80.028908,-4.346091],[-80.442242,-4.425724],[-80.469295,-4.059287],[-80.184015,-3.821162],[-80.302561,-3.404856],[-79.770293,-2.657512],[-79.986559,-2.220794],[-80.368784,-2.685159],[-80.967765,-2.246943],[-80.764806,-1.965048],[-80.933659,-1.057455],[-80.58337,-0.906663],[-80.399325,-0.283703],[-80.020898,0.36034],[-80.09061,0.768429],[-79.542762,0.982938],[-78.855259,1.380924],[-77.855061,0.809925],[-77.668613,0.825893],[-77.424984,0.395687],[-76.57638,0.256936],[-76.292314,0.416047],[-75.801466,0.084801],[-75.373223,-0.152032]]]}},{"type":"Feature","properties":{"ADMIN":"Puerto Rico","NAME_EN":"Puerto Rico","CONTINENT":"North America","ADM0_A3":"PRI","ISO_A3":"PRI","NAME":"Puerto Rico","ISO_A2":"PR","NAME_ZH":"波多黎各"},"geometry":{"type":"Polygon","coordinates":[[[-66.282434,18.514762],[-65.771303,18.426679],[-65.591004,18.228035],[-65.847164,17.975906],[-66.599934,17.981823],[-67.184162,17.946553],[-67.242428,18.37446],[-67.100679,18.520601],[-66.282434,18.514762]]]}},{"type":"Feature","properties":{"ADMIN":"Jamaica","NAME_EN":"Jamaica","CONTINENT":"North America","ADM0_A3":"JAM","ISO_A3":"JAM","NAME":"Jamaica","ISO_A2":"JM","NAME_ZH":"牙买加"},"geometry":{"type":"Polygon","coordinates":[[[-77.569601,18.490525],[-76.896619,18.400867],[-76.365359,18.160701],[-76.199659,17.886867],[-76.902561,17.868238],[-77.206341,17.701116],[-77.766023,17.861597],[-78.337719,18.225968],[-78.217727,18.454533],[-77.797365,18.524218],[-77.569601,18.490525]]]}},{"type":"Feature","properties":{"ADMIN":"Cuba","NAME_EN":"Cuba","CONTINENT":"North America","ADM0_A3":"CUB","ISO_A3":"CUB","NAME":"Cuba","ISO_A2":"CU","NAME_ZH":"古巴"},"geometry":{"type":"Polygon","coordinates":[[[-82.268151,23.188611],[-81.404457,23.117271],[-80.618769,23.10598],[-79.679524,22.765303],[-79.281486,22.399202],[-78.347434,22.512166],[-77.993296,22.277194],[-77.146422,21.657851],[-76.523825,21.20682],[-76.19462,21.220565],[-75.598222,21.016624],[-75.67106,20.735091],[-74.933896,20.693905],[-74.178025,20.284628],[-74.296648,20.050379],[-74.961595,19.923435],[-75.63468,19.873774],[-76.323656,19.952891],[-77.755481,19.855481],[-77.085108,20.413354],[-77.492655,20.673105],[-78.137292,20.739949],[-78.482827,21.028613],[-78.719867,21.598114],[-79.285,21.559175],[-80.217475,21.827324],[-80.517535,22.037079],[-81.820943,22.192057],[-82.169992,22.387109],[-81.795002,22.636965],[-82.775898,22.68815],[-83.494459,22.168518],[-83.9088,22.154565],[-84.052151,21.910575],[-84.54703,21.801228],[-84.974911,21.896028],[-84.447062,22.20495],[-84.230357,22.565755],[-83.77824,22.788118],[-83.267548,22.983042],[-82.510436,23.078747],[-82.268151,23.188611]]]}},{"type":"Feature","properties":{"ADMIN":"Zimbabwe","NAME_EN":"Zimbabwe","CONTINENT":"Africa","ADM0_A3":"ZWE","ISO_A3":"ZWE","NAME":"Zimbabwe","ISO_A2":"ZW","NAME_ZH":"津巴布韦"},"geometry":{"type":"Polygon","coordinates":[[[31.191409,-22.25151],[30.659865,-22.151567],[30.322883,-22.271612],[29.839037,-22.102216],[29.432188,-22.091313],[28.794656,-21.639454],[28.02137,-21.485975],[27.727228,-20.851802],[27.724747,-20.499059],[27.296505,-20.39152],[26.164791,-19.293086],[25.850391,-18.714413],[25.649163,-18.536026],[25.264226,-17.73654],[26.381935,-17.846042],[26.706773,-17.961229],[27.044427,-17.938026],[27.598243,-17.290831],[28.467906,-16.4684],[28.825869,-16.389749],[28.947463,-16.043051],[29.516834,-15.644678],[30.274256,-15.507787],[30.338955,-15.880839],[31.173064,-15.860944],[31.636498,-16.07199],[31.852041,-16.319417],[32.328239,-16.392074],[32.847639,-16.713398],[32.849861,-17.979057],[32.654886,-18.67209],[32.611994,-19.419383],[32.772708,-19.715592],[32.659743,-20.30429],[32.508693,-20.395292],[32.244988,-21.116489],[31.191409,-22.25151]]]}},{"type":"Feature","properties":{"ADMIN":"Botswana","NAME_EN":"Botswana","CONTINENT":"Africa","ADM0_A3":"BWA","ISO_A3":"BWA","NAME":"Botswana","ISO_A2":"BW","NAME_ZH":"博茨瓦纳"},"geometry":{"type":"Polygon","coordinates":[[[29.432188,-22.091313],[28.017236,-22.827754],[27.11941,-23.574323],[26.786407,-24.240691],[26.485753,-24.616327],[25.941652,-24.696373],[25.765849,-25.174845],[25.664666,-25.486816],[25.025171,-25.71967],[24.211267,-25.670216],[23.73357,-25.390129],[23.312097,-25.26869],[22.824271,-25.500459],[22.579532,-25.979448],[22.105969,-26.280256],[21.605896,-26.726534],[20.889609,-26.828543],[20.66647,-26.477453],[20.758609,-25.868136],[20.165726,-24.917962],[19.895768,-24.76779],[19.895458,-21.849157],[20.881134,-21.814327],[20.910641,-18.252219],[21.65504,-18.219146],[23.196858,-17.869038],[23.579006,-18.281261],[24.217365,-17.889347],[24.520705,-17.887125],[25.084443,-17.661816],[25.264226,-17.73654],[25.649163,-18.536026],[25.850391,-18.714413],[26.164791,-19.293086],[27.296505,-20.39152],[27.724747,-20.499059],[27.727228,-20.851802],[28.02137,-21.485975],[28.794656,-21.639454],[29.432188,-22.091313]]]}},{"type":"Feature","properties":{"ADMIN":"Namibia","NAME_EN":"Namibia","CONTINENT":"Africa","ADM0_A3":"NAM","ISO_A3":"NAM","NAME":"Namibia","ISO_A2":"NA","NAME_ZH":"纳米比亚"},"geometry":{"type":"Polygon","coordinates":[[[19.895768,-24.76779],[19.894734,-28.461105],[19.002127,-28.972443],[18.464899,-29.045462],[17.836152,-28.856378],[17.387497,-28.783514],[17.218929,-28.355943],[16.824017,-28.082162],[16.344977,-28.576705],[15.601818,-27.821247],[15.210472,-27.090956],[14.989711,-26.117372],[14.743214,-25.39292],[14.408144,-23.853014],[14.385717,-22.656653],[14.257714,-22.111208],[13.868642,-21.699037],[13.352498,-20.872834],[12.826845,-19.673166],[12.608564,-19.045349],[11.794919,-18.069129],[11.734199,-17.301889],[12.215461,-17.111668],[12.814081,-16.941343],[13.462362,-16.971212],[14.058501,-17.423381],[14.209707,-17.353101],[18.263309,-17.309951],[18.956187,-17.789095],[21.377176,-17.930636],[23.215048,-17.523116],[24.033862,-17.295843],[24.682349,-17.353411],[25.07695,-17.578823],[25.084443,-17.661816],[24.520705,-17.887125],[24.217365,-17.889347],[23.579006,-18.281261],[23.196858,-17.869038],[21.65504,-18.219146],[20.910641,-18.252219],[20.881134,-21.814327],[19.895458,-21.849157],[19.895768,-24.76779]]]}},{"type":"Feature","properties":{"ADMIN":"Senegal","NAME_EN":"Senegal","CONTINENT":"Africa","ADM0_A3":"SEN","ISO_A3":"SEN","NAME":"Senegal","ISO_A2":"SN","NAME_ZH":"塞内加尔"},"geometry":{"type":"Polygon","coordinates":[[[-16.713729,13.594959],[-17.126107,14.373516],[-17.625043,14.729541],[-17.185173,14.919477],[-16.700706,15.621527],[-16.463098,16.135036],[-16.12069,16.455663],[-15.623666,16.369337],[-15.135737,16.587282],[-14.577348,16.598264],[-14.099521,16.304302],[-13.435738,16.039383],[-12.830658,15.303692],[-12.17075,14.616834],[-12.124887,13.994727],[-11.927716,13.422075],[-11.553398,13.141214],[-11.467899,12.754519],[-11.513943,12.442988],[-11.658301,12.386583],[-12.203565,12.465648],[-12.278599,12.35444],[-12.499051,12.33209],[-13.217818,12.575874],[-13.700476,12.586183],[-15.548477,12.62817],[-15.816574,12.515567],[-16.147717,12.547762],[-16.677452,12.384852],[-16.841525,13.151394],[-15.931296,13.130284],[-15.691001,13.270353],[-15.511813,13.27857],[-15.141163,13.509512],[-14.712197,13.298207],[-14.277702,13.280585],[-13.844963,13.505042],[-14.046992,13.794068],[-14.376714,13.62568],[-14.687031,13.630357],[-15.081735,13.876492],[-15.39877,13.860369],[-15.624596,13.623587],[-16.713729,13.594959]]]}},{"type":"Feature","properties":{"ADMIN":"Mali","NAME_EN":"Mali","CONTINENT":"Africa","ADM0_A3":"MLI","ISO_A3":"MLI","NAME":"Mali","ISO_A2":"ML","NAME_ZH":"马里"},"geometry":{"type":"Polygon","coordinates":[[[-11.513943,12.442988],[-11.467899,12.754519],[-11.553398,13.141214],[-11.927716,13.422075],[-12.124887,13.994727],[-12.17075,14.616834],[-11.834208,14.799097],[-11.666078,15.388208],[-11.349095,15.411256],[-10.650791,15.132746],[-10.086846,15.330486],[-9.700255,15.264107],[-9.550238,15.486497],[-5.537744,15.50169],[-5.315277,16.201854],[-5.488523,16.325102],[-5.971129,20.640833],[-6.453787,24.956591],[-4.923337,24.974574],[-1.550055,22.792666],[1.823228,20.610809],[2.060991,20.142233],[2.683588,19.85623],[3.146661,19.693579],[3.158133,19.057364],[4.267419,19.155265],[4.27021,16.852227],[3.723422,16.184284],[3.638259,15.56812],[2.749993,15.409525],[1.385528,15.323561],[1.015783,14.968182],[0.374892,14.928908],[-0.266257,14.924309],[-0.515854,15.116158],[-1.066363,14.973815],[-2.001035,14.559008],[-2.191825,14.246418],[-2.967694,13.79815],[-3.103707,13.541267],[-3.522803,13.337662],[-4.006391,13.472485],[-4.280405,13.228444],[-4.427166,12.542646],[-5.220942,11.713859],[-5.197843,11.375146],[-5.470565,10.95127],[-5.404342,10.370737],[-5.816926,10.222555],[-6.050452,10.096361],[-6.205223,10.524061],[-6.493965,10.411303],[-6.666461,10.430811],[-6.850507,10.138994],[-7.622759,10.147236],[-7.89959,10.297382],[-8.029944,10.206535],[-8.335377,10.494812],[-8.282357,10.792597],[-8.407311,10.909257],[-8.620321,10.810891],[-8.581305,11.136246],[-8.376305,11.393646],[-8.786099,11.812561],[-8.905265,12.088358],[-9.127474,12.30806],[-9.327616,12.334286],[-9.567912,12.194243],[-9.890993,12.060479],[-10.165214,11.844084],[-10.593224,11.923975],[-10.87083,12.177887],[-11.036556,12.211245],[-11.297574,12.077971],[-11.456169,12.076834],[-11.513943,12.442988]]]}},{"type":"Feature","properties":{"ADMIN":"Mauritania","NAME_EN":"Mauritania","CONTINENT":"Africa","ADM0_A3":"MRT","ISO_A3":"MRT","NAME":"Mauritania","ISO_A2":"MR","NAME_ZH":"毛里塔尼亚"},"geometry":{"type":"Polygon","coordinates":[[[-17.063423,20.999752],[-16.845194,21.333323],[-12.929102,21.327071],[-13.118754,22.77122],[-12.874222,23.284832],[-11.937224,23.374594],[-11.969419,25.933353],[-8.687294,25.881056],[-8.6844,27.395744],[-4.923337,24.974574],[-6.453787,24.956591],[-5.971129,20.640833],[-5.488523,16.325102],[-5.315277,16.201854],[-5.537744,15.50169],[-9.550238,15.486497],[-9.700255,15.264107],[-10.086846,15.330486],[-10.650791,15.132746],[-11.349095,15.411256],[-11.666078,15.388208],[-11.834208,14.799097],[-12.17075,14.616834],[-12.830658,15.303692],[-13.435738,16.039383],[-14.099521,16.304302],[-14.577348,16.598264],[-15.135737,16.587282],[-15.623666,16.369337],[-16.12069,16.455663],[-16.463098,16.135036],[-16.549708,16.673892],[-16.270552,17.166963],[-16.146347,18.108482],[-16.256883,19.096716],[-16.377651,19.593817],[-16.277838,20.092521],[-16.536324,20.567866],[-17.063423,20.999752]]]}},{"type":"Feature","properties":{"ADMIN":"Benin","NAME_EN":"Benin","CONTINENT":"Africa","ADM0_A3":"BEN","ISO_A3":"BEN","NAME":"Benin","ISO_A2":"BJ","NAME_ZH":"贝宁"},"geometry":{"type":"Polygon","coordinates":[[[2.691702,6.258817],[1.865241,6.142158],[1.618951,6.832038],[1.664478,9.12859],[1.463043,9.334624],[1.425061,9.825395],[1.077795,10.175607],[0.772336,10.470808],[0.899563,10.997339],[1.24347,11.110511],[1.447178,11.547719],[1.935986,11.64115],[2.154474,11.94015],[2.490164,12.233052],[2.848643,12.235636],[3.61118,11.660167],[3.572216,11.327939],[3.797112,10.734746],[3.60007,10.332186],[3.705438,10.06321],[3.220352,9.444153],[2.912308,9.137608],[2.723793,8.506845],[2.749063,7.870734],[2.691702,6.258817]]]}},{"type":"Feature","properties":{"ADMIN":"Niger","NAME_EN":"Niger","CONTINENT":"Africa","ADM0_A3":"NER","ISO_A3":"NER","NAME":"Niger","ISO_A2":"NE","NAME_ZH":"尼日尔"},"geometry":{"type":"Polygon","coordinates":[[[14.8513,22.86295],[15.096888,21.308519],[15.47106,21.04845],[15.487148,20.730415],[15.903247,20.387619],[15.685741,19.95718],[15.300441,17.92795],[15.247731,16.627306],[13.97217,15.68437],[13.540394,14.367134],[13.956699,13.996691],[13.954477,13.353449],[14.595781,13.330427],[14.495787,12.859396],[14.213531,12.802035],[14.181336,12.483657],[13.995353,12.461565],[13.318702,13.556356],[13.083987,13.596147],[12.302071,13.037189],[11.527803,13.32898],[10.989593,13.387323],[10.701032,13.246918],[10.114814,13.277252],[9.524928,12.851102],[9.014933,12.826659],[7.804671,13.343527],[7.330747,13.098038],[6.820442,13.115091],[6.445426,13.492768],[5.443058,13.865924],[4.368344,13.747482],[4.107946,13.531216],[3.967283,12.956109],[3.680634,12.552903],[3.61118,11.660167],[2.848643,12.235636],[2.490164,12.233052],[2.154474,11.94015],[2.177108,12.625018],[1.024103,12.851826],[0.993046,13.33575],[0.429928,13.988733],[0.295646,14.444235],[0.374892,14.928908],[1.015783,14.968182],[1.385528,15.323561],[2.749993,15.409525],[3.638259,15.56812],[3.723422,16.184284],[4.27021,16.852227],[4.267419,19.155265],[5.677566,19.601207],[8.572893,21.565661],[11.999506,23.471668],[13.581425,23.040506],[14.143871,22.491289],[14.8513,22.86295]]]}},{"type":"Feature","properties":{"ADMIN":"Nigeria","NAME_EN":"Nigeria","CONTINENT":"Africa","ADM0_A3":"NGA","ISO_A3":"NGA","NAME":"Nigeria","ISO_A2":"NG","NAME_ZH":"尼日利亚"},"geometry":{"type":"Polygon","coordinates":[[[2.691702,6.258817],[2.749063,7.870734],[2.723793,8.506845],[2.912308,9.137608],[3.220352,9.444153],[3.705438,10.06321],[3.60007,10.332186],[3.797112,10.734746],[3.572216,11.327939],[3.61118,11.660167],[3.680634,12.552903],[3.967283,12.956109],[4.107946,13.531216],[4.368344,13.747482],[5.443058,13.865924],[6.445426,13.492768],[6.820442,13.115091],[7.330747,13.098038],[7.804671,13.343527],[9.014933,12.826659],[9.524928,12.851102],[10.114814,13.277252],[10.701032,13.246918],[10.989593,13.387323],[11.527803,13.32898],[12.302071,13.037189],[13.083987,13.596147],[13.318702,13.556356],[13.995353,12.461565],[14.181336,12.483657],[14.577178,12.085361],[14.468192,11.904752],[14.415379,11.572369],[13.57295,10.798566],[13.308676,10.160362],[13.1676,9.640626],[12.955468,9.417772],[12.753672,8.717763],[12.218872,8.305824],[12.063946,7.799808],[11.839309,7.397042],[11.745774,6.981383],[11.058788,6.644427],[10.497375,7.055358],[10.118277,7.03877],[9.522706,6.453482],[9.233163,6.444491],[8.757533,5.479666],[8.500288,4.771983],[7.462108,4.412108],[7.082596,4.464689],[6.698072,4.240594],[5.898173,4.262453],[5.362805,4.887971],[5.033574,5.611802],[4.325607,6.270651],[3.57418,6.2583],[2.691702,6.258817]]]}},{"type":"Feature","properties":{"ADMIN":"Cameroon","NAME_EN":"Cameroon","CONTINENT":"Africa","ADM0_A3":"CMR","ISO_A3":"CMR","NAME":"Cameroon","ISO_A2":"CM","NAME_ZH":"喀麦隆"},"geometry":{"type":"Polygon","coordinates":[[[14.495787,12.859396],[14.89336,12.21905],[14.960152,11.555574],[14.923565,10.891325],[15.467873,9.982337],[14.909354,9.992129],[14.627201,9.920919],[14.171466,10.021378],[13.954218,9.549495],[14.544467,8.965861],[14.979996,8.796104],[15.120866,8.38215],[15.436092,7.692812],[15.27946,7.421925],[14.776545,6.408498],[14.53656,6.226959],[14.459407,5.451761],[14.558936,5.030598],[14.478372,4.732605],[14.950953,4.210389],[15.03622,3.851367],[15.405396,3.335301],[15.862732,3.013537],[15.907381,2.557389],[16.012852,2.26764],[15.940919,1.727673],[15.146342,1.964015],[14.337813,2.227875],[13.075822,2.267097],[12.951334,2.321616],[12.35938,2.192812],[11.751665,2.326758],[11.276449,2.261051],[9.649158,2.283866],[9.795196,3.073404],[9.404367,3.734527],[8.948116,3.904129],[8.744924,4.352215],[8.488816,4.495617],[8.500288,4.771983],[8.757533,5.479666],[9.233163,6.444491],[9.522706,6.453482],[10.118277,7.03877],[10.497375,7.055358],[11.058788,6.644427],[11.745774,6.981383],[11.839309,7.397042],[12.063946,7.799808],[12.218872,8.305824],[12.753672,8.717763],[12.955468,9.417772],[13.1676,9.640626],[13.308676,10.160362],[13.57295,10.798566],[14.415379,11.572369],[14.468192,11.904752],[14.577178,12.085361],[14.181336,12.483657],[14.213531,12.802035],[14.495787,12.859396]]]}},{"type":"Feature","properties":{"ADMIN":"Togo","NAME_EN":"Togo","CONTINENT":"Africa","ADM0_A3":"TGO","ISO_A3":"TGO","NAME":"Togo","ISO_A2":"TG","NAME_ZH":"多哥"},"geometry":{"type":"Polygon","coordinates":[[[0.899563,10.997339],[0.772336,10.470808],[1.077795,10.175607],[1.425061,9.825395],[1.463043,9.334624],[1.664478,9.12859],[1.618951,6.832038],[1.865241,6.142158],[1.060122,5.928837],[0.836931,6.279979],[0.570384,6.914359],[0.490957,7.411744],[0.712029,8.312465],[0.461192,8.677223],[0.365901,9.465004],[0.36758,10.191213],[-0.049785,10.706918],[0.023803,11.018682],[0.899563,10.997339]]]}},{"type":"Feature","properties":{"ADMIN":"Ghana","NAME_EN":"Ghana","CONTINENT":"Africa","ADM0_A3":"GHA","ISO_A3":"GHA","NAME":"Ghana","ISO_A2":"GH","NAME_ZH":"加纳"},"geometry":{"type":"Polygon","coordinates":[[[0.023803,11.018682],[-0.049785,10.706918],[0.36758,10.191213],[0.365901,9.465004],[0.461192,8.677223],[0.712029,8.312465],[0.490957,7.411744],[0.570384,6.914359],[0.836931,6.279979],[1.060122,5.928837],[-0.507638,5.343473],[-1.063625,5.000548],[-1.964707,4.710462],[-2.856125,4.994476],[-2.810701,5.389051],[-3.24437,6.250472],[-2.983585,7.379705],[-2.56219,8.219628],[-2.827496,9.642461],[-2.963896,10.395335],[-2.940409,10.96269],[-1.203358,11.009819],[-0.761576,10.93693],[-0.438702,11.098341],[0.023803,11.018682]]]}},{"type":"Feature","properties":{"ADMIN":"Ivory Coast","NAME_EN":"Ivory Coast","CONTINENT":"Africa","ADM0_A3":"CIV","ISO_A3":"CIV","NAME":"Côte d'Ivoire","ISO_A2":"CI","NAME_ZH":"科特迪瓦"},"geometry":{"type":"Polygon","coordinates":[[[-8.029944,10.206535],[-7.89959,10.297382],[-7.622759,10.147236],[-6.850507,10.138994],[-6.666461,10.430811],[-6.493965,10.411303],[-6.205223,10.524061],[-6.050452,10.096361],[-5.816926,10.222555],[-5.404342,10.370737],[-4.954653,10.152714],[-4.779884,9.821985],[-4.330247,9.610835],[-3.980449,9.862344],[-3.511899,9.900326],[-2.827496,9.642461],[-2.56219,8.219628],[-2.983585,7.379705],[-3.24437,6.250472],[-2.810701,5.389051],[-2.856125,4.994476],[-3.311084,4.984296],[-4.00882,5.179813],[-4.649917,5.168264],[-5.834496,4.993701],[-6.528769,4.705088],[-7.518941,4.338288],[-7.712159,4.364566],[-7.635368,5.188159],[-7.539715,5.313345],[-7.570153,5.707352],[-7.993693,6.12619],[-8.311348,6.193033],[-8.60288,6.467564],[-8.385452,6.911801],[-8.485446,7.395208],[-8.439298,7.686043],[-8.280703,7.68718],[-8.221792,8.123329],[-8.299049,8.316444],[-8.203499,8.455453],[-7.8321,8.575704],[-8.079114,9.376224],[-8.309616,9.789532],[-8.229337,10.12902],[-8.029944,10.206535]]]}},{"type":"Feature","properties":{"ADMIN":"Guinea","NAME_EN":"Guinea","CONTINENT":"Africa","ADM0_A3":"GIN","ISO_A3":"GIN","NAME":"Guinea","ISO_A2":"GN","NAME_ZH":"几内亚"},"geometry":{"type":"Polygon","coordinates":[[[-13.700476,12.586183],[-13.217818,12.575874],[-12.499051,12.33209],[-12.278599,12.35444],[-12.203565,12.465648],[-11.658301,12.386583],[-11.513943,12.442988],[-11.456169,12.076834],[-11.297574,12.077971],[-11.036556,12.211245],[-10.87083,12.177887],[-10.593224,11.923975],[-10.165214,11.844084],[-9.890993,12.060479],[-9.567912,12.194243],[-9.327616,12.334286],[-9.127474,12.30806],[-8.905265,12.088358],[-8.786099,11.812561],[-8.376305,11.393646],[-8.581305,11.136246],[-8.620321,10.810891],[-8.407311,10.909257],[-8.282357,10.792597],[-8.335377,10.494812],[-8.029944,10.206535],[-8.229337,10.12902],[-8.309616,9.789532],[-8.079114,9.376224],[-7.8321,8.575704],[-8.203499,8.455453],[-8.299049,8.316444],[-8.221792,8.123329],[-8.280703,7.68718],[-8.439298,7.686043],[-8.722124,7.711674],[-8.926065,7.309037],[-9.208786,7.313921],[-9.403348,7.526905],[-9.33728,7.928534],[-9.755342,8.541055],[-10.016567,8.428504],[-10.230094,8.406206],[-10.505477,8.348896],[-10.494315,8.715541],[-10.65477,8.977178],[-10.622395,9.26791],[-10.839152,9.688246],[-11.117481,10.045873],[-11.917277,10.046984],[-12.150338,9.858572],[-12.425929,9.835834],[-12.596719,9.620188],[-12.711958,9.342712],[-13.24655,8.903049],[-13.685154,9.494744],[-14.074045,9.886167],[-14.330076,10.01572],[-14.579699,10.214467],[-14.693232,10.656301],[-14.839554,10.876572],[-15.130311,11.040412],[-14.685687,11.527824],[-14.382192,11.509272],[-14.121406,11.677117],[-13.9008,11.678719],[-13.743161,11.811269],[-13.828272,12.142644],[-13.718744,12.247186],[-13.700476,12.586183]]]}},{"type":"Feature","properties":{"ADMIN":"Guinea-Bissau","NAME_EN":"Guinea-Bissau","CONTINENT":"Africa","ADM0_A3":"GNB","ISO_A3":"GNB","NAME":"Guinea-Bissau","ISO_A2":"GW","NAME_ZH":"几内亚比绍"},"geometry":{"type":"Polygon","coordinates":[[[-16.677452,12.384852],[-16.147717,12.547762],[-15.816574,12.515567],[-15.548477,12.62817],[-13.700476,12.586183],[-13.718744,12.247186],[-13.828272,12.142644],[-13.743161,11.811269],[-13.9008,11.678719],[-14.121406,11.677117],[-14.382192,11.509272],[-14.685687,11.527824],[-15.130311,11.040412],[-15.66418,11.458474],[-16.085214,11.524594],[-16.314787,11.806515],[-16.308947,11.958702],[-16.613838,12.170911],[-16.677452,12.384852]]]}},{"type":"Feature","properties":{"ADMIN":"Liberia","NAME_EN":"Liberia","CONTINENT":"Africa","ADM0_A3":"LBR","ISO_A3":"LBR","NAME":"Liberia","ISO_A2":"LR","NAME_ZH":"利比里亚"},"geometry":{"type":"Polygon","coordinates":[[[-8.439298,7.686043],[-8.485446,7.395208],[-8.385452,6.911801],[-8.60288,6.467564],[-8.311348,6.193033],[-7.993693,6.12619],[-7.570153,5.707352],[-7.539715,5.313345],[-7.635368,5.188159],[-7.712159,4.364566],[-7.974107,4.355755],[-9.004794,4.832419],[-9.91342,5.593561],[-10.765384,6.140711],[-11.438779,6.785917],[-11.199802,7.105846],[-11.146704,7.396706],[-10.695595,7.939464],[-10.230094,8.406206],[-10.016567,8.428504],[-9.755342,8.541055],[-9.33728,7.928534],[-9.403348,7.526905],[-9.208786,7.313921],[-8.926065,7.309037],[-8.722124,7.711674],[-8.439298,7.686043]]]}},{"type":"Feature","properties":{"ADMIN":"Sierra Leone","NAME_EN":"Sierra Leone","CONTINENT":"Africa","ADM0_A3":"SLE","ISO_A3":"SLE","NAME":"Sierra Leone","ISO_A2":"SL","NAME_ZH":"塞拉利昂"},"geometry":{"type":"Polygon","coordinates":[[[-13.24655,8.903049],[-12.711958,9.342712],[-12.596719,9.620188],[-12.425929,9.835834],[-12.150338,9.858572],[-11.917277,10.046984],[-11.117481,10.045873],[-10.839152,9.688246],[-10.622395,9.26791],[-10.65477,8.977178],[-10.494315,8.715541],[-10.505477,8.348896],[-10.230094,8.406206],[-10.695595,7.939464],[-11.146704,7.396706],[-11.199802,7.105846],[-11.438779,6.785917],[-11.708195,6.860098],[-12.428099,7.262942],[-12.949049,7.798646],[-13.124025,8.163946],[-13.24655,8.903049]]]}},{"type":"Feature","properties":{"ADMIN":"Burkina Faso","NAME_EN":"Burkina Faso","CONTINENT":"Africa","ADM0_A3":"BFA","ISO_A3":"BFA","NAME":"Burkina Faso","ISO_A2":"BF","NAME_ZH":"布基纳法索"},"geometry":{"type":"Polygon","coordinates":[[[-5.404342,10.370737],[-5.470565,10.95127],[-5.197843,11.375146],[-5.220942,11.713859],[-4.427166,12.542646],[-4.280405,13.228444],[-4.006391,13.472485],[-3.522803,13.337662],[-3.103707,13.541267],[-2.967694,13.79815],[-2.191825,14.246418],[-2.001035,14.559008],[-1.066363,14.973815],[-0.515854,15.116158],[-0.266257,14.924309],[0.374892,14.928908],[0.295646,14.444235],[0.429928,13.988733],[0.993046,13.33575],[1.024103,12.851826],[2.177108,12.625018],[2.154474,11.94015],[1.935986,11.64115],[1.447178,11.547719],[1.24347,11.110511],[0.899563,10.997339],[0.023803,11.018682],[-0.438702,11.098341],[-0.761576,10.93693],[-1.203358,11.009819],[-2.940409,10.96269],[-2.963896,10.395335],[-2.827496,9.642461],[-3.511899,9.900326],[-3.980449,9.862344],[-4.330247,9.610835],[-4.779884,9.821985],[-4.954653,10.152714],[-5.404342,10.370737]]]}},{"type":"Feature","properties":{"ADMIN":"Central African Republic","NAME_EN":"Central African Republic","CONTINENT":"Africa","ADM0_A3":"CAF","ISO_A3":"CAF","NAME":"Central African Rep.","ISO_A2":"CF","NAME_ZH":"中非共和国"},"geometry":{"type":"Polygon","coordinates":[[[27.374226,5.233944],[27.044065,5.127853],[26.402761,5.150875],[25.650455,5.256088],[25.278798,5.170408],[25.128833,4.927245],[24.805029,4.897247],[24.410531,5.108784],[23.297214,4.609693],[22.84148,4.710126],[22.704124,4.633051],[22.405124,4.02916],[21.659123,4.224342],[20.927591,4.322786],[20.290679,4.691678],[19.467784,5.031528],[18.932312,4.709506],[18.542982,4.201785],[18.453065,3.504386],[17.8099,3.560196],[17.133042,3.728197],[16.537058,3.198255],[16.012852,2.26764],[15.907381,2.557389],[15.862732,3.013537],[15.405396,3.335301],[15.03622,3.851367],[14.950953,4.210389],[14.478372,4.732605],[14.558936,5.030598],[14.459407,5.451761],[14.53656,6.226959],[14.776545,6.408498],[15.27946,7.421925],[16.106232,7.497088],[16.290562,7.754307],[16.456185,7.734774],[16.705988,7.508328],[17.96493,7.890914],[18.389555,8.281304],[18.911022,8.630895],[18.81201,8.982915],[19.094008,9.074847],[20.059685,9.012706],[21.000868,9.475985],[21.723822,10.567056],[22.231129,10.971889],[22.864165,11.142395],[22.977544,10.714463],[23.554304,10.089255],[23.55725,9.681218],[23.394779,9.265068],[23.459013,8.954286],[23.805813,8.666319],[24.567369,8.229188],[25.114932,7.825104],[25.124131,7.500085],[25.796648,6.979316],[26.213418,6.546603],[26.465909,5.946717],[27.213409,5.550953],[27.374226,5.233944]]]}},{"type":"Feature","properties":{"ADMIN":"Republic of the Congo","NAME_EN":"Republic of the Congo","CONTINENT":"Africa","ADM0_A3":"COG","ISO_A3":"COG","NAME":"Congo","ISO_A2":"CG","NAME_ZH":"刚果共和国"},"geometry":{"type":"Polygon","coordinates":[[[18.453065,3.504386],[18.393792,2.900443],[18.094276,2.365722],[17.898835,1.741832],[17.774192,0.855659],[17.82654,0.288923],[17.663553,-0.058084],[17.638645,-0.424832],[17.523716,-0.74383],[16.865307,-1.225816],[16.407092,-1.740927],[15.972803,-2.712392],[16.00629,-3.535133],[15.75354,-3.855165],[15.170992,-4.343507],[14.582604,-4.970239],[14.209035,-4.793092],[14.144956,-4.510009],[13.600235,-4.500138],[13.25824,-4.882957],[12.995517,-4.781103],[12.62076,-4.438023],[12.318608,-4.60623],[11.914963,-5.037987],[11.093773,-3.978827],[11.855122,-3.426871],[11.478039,-2.765619],[11.820964,-2.514161],[12.495703,-2.391688],[12.575284,-1.948511],[13.109619,-2.42874],[13.992407,-2.470805],[14.29921,-1.998276],[14.425456,-1.333407],[14.316418,-0.552627],[13.843321,0.038758],[14.276266,1.19693],[14.026669,1.395677],[13.282631,1.314184],[13.003114,1.830896],[13.075822,2.267097],[14.337813,2.227875],[15.146342,1.964015],[15.940919,1.727673],[16.012852,2.26764],[16.537058,3.198255],[17.133042,3.728197],[17.8099,3.560196],[18.453065,3.504386]]]}},{"type":"Feature","properties":{"ADMIN":"Gabon","NAME_EN":"Gabon","CONTINENT":"Africa","ADM0_A3":"GAB","ISO_A3":"GAB","NAME":"Gabon","ISO_A2":"GA","NAME_ZH":"加蓬"},"geometry":{"type":"Polygon","coordinates":[[[11.276449,2.261051],[11.751665,2.326758],[12.35938,2.192812],[12.951334,2.321616],[13.075822,2.267097],[13.003114,1.830896],[13.282631,1.314184],[14.026669,1.395677],[14.276266,1.19693],[13.843321,0.038758],[14.316418,-0.552627],[14.425456,-1.333407],[14.29921,-1.998276],[13.992407,-2.470805],[13.109619,-2.42874],[12.575284,-1.948511],[12.495703,-2.391688],[11.820964,-2.514161],[11.478039,-2.765619],[11.855122,-3.426871],[11.093773,-3.978827],[10.066135,-2.969483],[9.405245,-2.144313],[8.797996,-1.111301],[8.830087,-0.779074],[9.04842,-0.459351],[9.291351,0.268666],[9.492889,1.01012],[9.830284,1.067894],[11.285079,1.057662],[11.276449,2.261051]]]}},{"type":"Feature","properties":{"ADMIN":"Equatorial Guinea","NAME_EN":"Equatorial Guinea","CONTINENT":"Africa","ADM0_A3":"GNQ","ISO_A3":"GNQ","NAME":"Eq. Guinea","ISO_A2":"GQ","NAME_ZH":"赤道几内亚"},"geometry":{"type":"Polygon","coordinates":[[[9.649158,2.283866],[11.276449,2.261051],[11.285079,1.057662],[9.830284,1.067894],[9.492889,1.01012],[9.305613,1.160911],[9.649158,2.283866]]]}},{"type":"Feature","properties":{"ADMIN":"Zambia","NAME_EN":"Zambia","CONTINENT":"Africa","ADM0_A3":"ZMB","ISO_A3":"ZMB","NAME":"Zambia","ISO_A2":"ZM","NAME_ZH":"赞比亚"},"geometry":{"type":"Polygon","coordinates":[[[30.74001,-8.340006],[31.157751,-8.594579],[31.556348,-8.762049],[32.191865,-8.930359],[32.759375,-9.230599],[33.231388,-9.676722],[33.485688,-10.525559],[33.31531,-10.79655],[33.114289,-11.607198],[33.306422,-12.435778],[32.991764,-12.783871],[32.688165,-13.712858],[33.214025,-13.97186],[30.179481,-14.796099],[30.274256,-15.507787],[29.516834,-15.644678],[28.947463,-16.043051],[28.825869,-16.389749],[28.467906,-16.4684],[27.598243,-17.290831],[27.044427,-17.938026],[26.706773,-17.961229],[26.381935,-17.846042],[25.264226,-17.73654],[25.084443,-17.661816],[25.07695,-17.578823],[24.682349,-17.353411],[24.033862,-17.295843],[23.215048,-17.523116],[22.562478,-16.898451],[21.887843,-16.08031],[21.933886,-12.898437],[24.016137,-12.911046],[23.930922,-12.565848],[24.079905,-12.191297],[23.904154,-11.722282],[24.017894,-11.237298],[23.912215,-10.926826],[24.257155,-10.951993],[24.314516,-11.262826],[24.78317,-11.238694],[25.418118,-11.330936],[25.75231,-11.784965],[26.553088,-11.92444],[27.16442,-11.608748],[27.388799,-12.132747],[28.155109,-12.272481],[28.523562,-12.698604],[28.934286,-13.248958],[29.699614,-13.257227],[29.616001,-12.178895],[29.341548,-12.360744],[28.642417,-11.971569],[28.372253,-11.793647],[28.49607,-10.789884],[28.673682,-9.605925],[28.449871,-9.164918],[28.734867,-8.526559],[29.002912,-8.407032],[30.346086,-8.238257],[30.74001,-8.340006]]]}},{"type":"Feature","properties":{"ADMIN":"Malawi","NAME_EN":"Malawi","CONTINENT":"Africa","ADM0_A3":"MWI","ISO_A3":"MWI","NAME":"Malawi","ISO_A2":"MW","NAME_ZH":"马拉维"},"geometry":{"type":"Polygon","coordinates":[[[32.759375,-9.230599],[33.73972,-9.41715],[33.940838,-9.693674],[34.28,-10.16],[34.559989,-11.52002],[34.280006,-12.280025],[34.559989,-13.579998],[34.907151,-13.565425],[35.267956,-13.887834],[35.686845,-14.611046],[35.771905,-15.896859],[35.339063,-16.10744],[35.03381,-16.8013],[34.381292,-16.18356],[34.307291,-15.478641],[34.517666,-15.013709],[34.459633,-14.61301],[34.064825,-14.35995],[33.7897,-14.451831],[33.214025,-13.97186],[32.688165,-13.712858],[32.991764,-12.783871],[33.306422,-12.435778],[33.114289,-11.607198],[33.31531,-10.79655],[33.485688,-10.525559],[33.231388,-9.676722],[32.759375,-9.230599]]]}},{"type":"Feature","properties":{"ADMIN":"Mozambique","NAME_EN":"Mozambique","CONTINENT":"Africa","ADM0_A3":"MOZ","ISO_A3":"MOZ","NAME":"Mozambique","ISO_A2":"MZ","NAME_ZH":"莫桑比克"},"geometry":{"type":"Polygon","coordinates":[[[34.559989,-11.52002],[35.312398,-11.439146],[36.514082,-11.720938],[36.775151,-11.594537],[37.47129,-11.56876],[37.82764,-11.26879],[38.427557,-11.285202],[39.521,-10.89688],[40.31659,-10.3171],[40.316586,-10.317098],[40.316589,-10.317096],[40.478387,-10.765441],[40.437253,-11.761711],[40.560811,-12.639177],[40.59962,-14.201975],[40.775475,-14.691764],[40.477251,-15.406294],[40.089264,-16.100774],[39.452559,-16.720891],[38.538351,-17.101023],[37.411133,-17.586368],[36.281279,-18.659688],[35.896497,-18.84226],[35.1984,-19.552811],[34.786383,-19.784012],[34.701893,-20.497043],[35.176127,-21.254361],[35.373428,-21.840837],[35.385848,-22.14],[35.562546,-22.09],[35.533935,-23.070788],[35.371774,-23.535359],[35.60747,-23.706563],[35.458746,-24.12261],[35.040735,-24.478351],[34.215824,-24.816314],[33.01321,-25.357573],[32.574632,-25.727318],[32.660363,-26.148584],[32.915955,-26.215867],[32.83012,-26.742192],[32.071665,-26.73382],[31.985779,-26.29178],[31.837778,-25.843332],[31.752408,-25.484284],[31.930589,-24.369417],[31.670398,-23.658969],[31.191409,-22.25151],[32.244988,-21.116489],[32.508693,-20.395292],[32.659743,-20.30429],[32.772708,-19.715592],[32.611994,-19.419383],[32.654886,-18.67209],[32.849861,-17.979057],[32.847639,-16.713398],[32.328239,-16.392074],[31.852041,-16.319417],[31.636498,-16.07199],[31.173064,-15.860944],[30.338955,-15.880839],[30.274256,-15.507787],[30.179481,-14.796099],[33.214025,-13.97186],[33.7897,-14.451831],[34.064825,-14.35995],[34.459633,-14.61301],[34.517666,-15.013709],[34.307291,-15.478641],[34.381292,-16.18356],[35.03381,-16.8013],[35.339063,-16.10744],[35.771905,-15.896859],[35.686845,-14.611046],[35.267956,-13.887834],[34.907151,-13.565425],[34.559989,-13.579998],[34.280006,-12.280025],[34.559989,-11.52002]]]}},{"type":"Feature","properties":{"ADMIN":"eSwatini","NAME_EN":"Eswatini","CONTINENT":"Africa","ADM0_A3":"SWZ","ISO_A3":"SWZ","NAME":"eSwatini","ISO_A2":"SZ","NAME_ZH":"斯威士兰"},"geometry":{"type":"Polygon","coordinates":[[[32.071665,-26.73382],[31.86806,-27.177927],[31.282773,-27.285879],[30.685962,-26.743845],[30.676609,-26.398078],[30.949667,-26.022649],[31.04408,-25.731452],[31.333158,-25.660191],[31.837778,-25.843332],[31.985779,-26.29178],[32.071665,-26.73382]]]}},{"type":"Feature","properties":{"ADMIN":"Angola","NAME_EN":"Angola","CONTINENT":"Africa","ADM0_A3":"AGO","ISO_A3":"AGO","NAME":"Angola","ISO_A2":"AO","NAME_ZH":"安哥拉"},"geometry":{"type":"MultiPolygon","coordinates":[[[[12.995517,-4.781103],[12.631612,-4.991271],[12.468004,-5.248362],[12.436688,-5.684304],[12.182337,-5.789931],[11.914963,-5.037987],[12.318608,-4.60623],[12.62076,-4.438023],[12.995517,-4.781103]]],[[[12.322432,-6.100092],[12.735171,-5.965682],[13.024869,-5.984389],[13.375597,-5.864241],[16.326528,-5.87747],[16.57318,-6.622645],[16.860191,-7.222298],[17.089996,-7.545689],[17.47297,-8.068551],[18.134222,-7.987678],[18.464176,-7.847014],[19.016752,-7.988246],[19.166613,-7.738184],[19.417502,-7.155429],[20.037723,-7.116361],[20.091622,-6.94309],[20.601823,-6.939318],[20.514748,-7.299606],[21.728111,-7.290872],[21.746456,-7.920085],[21.949131,-8.305901],[21.801801,-8.908707],[21.875182,-9.523708],[22.208753,-9.894796],[22.155268,-11.084801],[22.402798,-10.993075],[22.837345,-11.017622],[23.456791,-10.867863],[23.912215,-10.926826],[24.017894,-11.237298],[23.904154,-11.722282],[24.079905,-12.191297],[23.930922,-12.565848],[24.016137,-12.911046],[21.933886,-12.898437],[21.887843,-16.08031],[22.562478,-16.898451],[23.215048,-17.523116],[21.377176,-17.930636],[18.956187,-17.789095],[18.263309,-17.309951],[14.209707,-17.353101],[14.058501,-17.423381],[13.462362,-16.971212],[12.814081,-16.941343],[12.215461,-17.111668],[11.734199,-17.301889],[11.640096,-16.673142],[11.778537,-15.793816],[12.123581,-14.878316],[12.175619,-14.449144],[12.500095,-13.5477],[12.738479,-13.137906],[13.312914,-12.48363],[13.633721,-12.038645],[13.738728,-11.297863],[13.686379,-10.731076],[13.387328,-10.373578],[13.120988,-9.766897],[12.87537,-9.166934],[12.929061,-8.959091],[13.236433,-8.562629],[12.93304,-7.596539],[12.728298,-6.927122],[12.227347,-6.294448],[12.322432,-6.100092]]]]}},{"type":"Feature","properties":{"ADMIN":"Burundi","NAME_EN":"Burundi","CONTINENT":"Africa","ADM0_A3":"BDI","ISO_A3":"BDI","NAME":"Burundi","ISO_A2":"BI","NAME_ZH":"布隆迪"},"geometry":{"type":"Polygon","coordinates":[[[30.469674,-2.413855],[30.52766,-2.80762],[30.74301,-3.03431],[30.75224,-3.35931],[30.50554,-3.56858],[30.11632,-4.09012],[29.753512,-4.452389],[29.339998,-4.499983],[29.276384,-3.293907],[29.024926,-2.839258],[29.632176,-2.917858],[29.938359,-2.348487],[30.469674,-2.413855]]]}},{"type":"Feature","properties":{"ADMIN":"Palestine","NAME_EN":"Palestine","CONTINENT":"Asia","ADM0_A3":"PSX","ISO_A3":"PSE","NAME":"Palestine","ISO_A2":"PS","NAME_ZH":"巴勒斯坦"},"geometry":{"type":"MultiPolygon","coordinates":[[[[35.397561,31.489086],[34.927408,31.353435],[34.970507,31.616778],[35.225892,31.754341],[34.974641,31.866582],[35.18393,32.532511],[35.545665,32.393992],[35.545252,31.782505],[35.397561,31.489086]]],[[[34.2296,31.2208],[34.222,31.274],[34.222,31.38],[34.222,31.48],[34.232,31.535],[34.259,31.578],[34.308,31.595],[34.488,31.595],[34.556,31.575],[34.56,31.543],[34.55,31.396],[34.54,31.242],[34.456,31.221],[34.2296,31.2208]]]]}},{"type":"Feature","properties":{"ADMIN":"Israel","NAME_EN":"Israel","CONTINENT":"Asia","ADM0_A3":"ISR","ISO_A3":"ISR","NAME":"Israel","ISO_A2":"IL","NAME_ZH":"以色列"},"geometry":{"type":"Polygon","coordinates":[[[35.719918,32.709192],[35.545665,32.393992],[35.18393,32.532511],[34.974641,31.866582],[35.225892,31.754341],[34.970507,31.616778],[34.927408,31.353435],[35.397561,31.489086],[35.420918,31.100066],[34.922603,29.501326],[34.823243,29.761081],[34.26544,31.21936],[34.265435,31.219357],[34.265433,31.219361],[34.556372,31.548824],[34.488107,31.605539],[34.752587,32.072926],[34.955417,32.827376],[35.098457,33.080539],[35.126053,33.0909],[35.460709,33.08904],[35.552797,33.264275],[35.821101,33.277426],[35.836397,32.868123],[35.700798,32.716014],[35.719918,32.709192]]]}},{"type":"Feature","properties":{"ADMIN":"Lebanon","NAME_EN":"Lebanon","CONTINENT":"Asia","ADM0_A3":"LBN","ISO_A3":"LBN","NAME":"Lebanon","ISO_A2":"LB","NAME_ZH":"黎巴嫩"},"geometry":{"type":"Polygon","coordinates":[[[35.821101,33.277426],[35.552797,33.264275],[35.460709,33.08904],[35.126053,33.0909],[35.482207,33.90545],[35.979592,34.610058],[35.998403,34.644914],[36.448194,34.593935],[36.61175,34.201789],[36.06646,33.824912],[35.821101,33.277426]]]}},{"type":"Feature","properties":{"ADMIN":"Madagascar","NAME_EN":"Madagascar","CONTINENT":"Africa","ADM0_A3":"MDG","ISO_A3":"MDG","NAME":"Madagascar","ISO_A2":"MG","NAME_ZH":"马达加斯加"},"geometry":{"type":"Polygon","coordinates":[[[49.543519,-12.469833],[49.808981,-12.895285],[50.056511,-13.555761],[50.217431,-14.758789],[50.476537,-15.226512],[50.377111,-15.706069],[50.200275,-16.000263],[49.860606,-15.414253],[49.672607,-15.710204],[49.863344,-16.451037],[49.774564,-16.875042],[49.498612,-17.106036],[49.435619,-17.953064],[49.041792,-19.118781],[48.548541,-20.496888],[47.930749,-22.391501],[47.547723,-23.781959],[47.095761,-24.94163],[46.282478,-25.178463],[45.409508,-25.601434],[44.833574,-25.346101],[44.03972,-24.988345],[43.763768,-24.460677],[43.697778,-23.574116],[43.345654,-22.776904],[43.254187,-22.057413],[43.433298,-21.336475],[43.893683,-21.163307],[43.89637,-20.830459],[44.374325,-20.072366],[44.464397,-19.435454],[44.232422,-18.961995],[44.042976,-18.331387],[43.963084,-17.409945],[44.312469,-16.850496],[44.446517,-16.216219],[44.944937,-16.179374],[45.502732,-15.974373],[45.872994,-15.793454],[46.312243,-15.780018],[46.882183,-15.210182],[47.70513,-14.594303],[48.005215,-14.091233],[47.869047,-13.663869],[48.293828,-13.784068],[48.84506,-13.089175],[48.863509,-12.487868],[49.194651,-12.040557],[49.543519,-12.469833]]]}},{"type":"Feature","properties":{"ADMIN":"Gambia","NAME_EN":"The Gambia","CONTINENT":"Africa","ADM0_A3":"GMB","ISO_A3":"GMB","NAME":"Gambia","ISO_A2":"GM","NAME_ZH":"冈比亚"},"geometry":{"type":"Polygon","coordinates":[[[-16.713729,13.594959],[-15.624596,13.623587],[-15.39877,13.860369],[-15.081735,13.876492],[-14.687031,13.630357],[-14.376714,13.62568],[-14.046992,13.794068],[-13.844963,13.505042],[-14.277702,13.280585],[-14.712197,13.298207],[-15.141163,13.509512],[-15.511813,13.27857],[-15.691001,13.270353],[-15.931296,13.130284],[-16.841525,13.151394],[-16.713729,13.594959]]]}},{"type":"Feature","properties":{"ADMIN":"Tunisia","NAME_EN":"Tunisia","CONTINENT":"Africa","ADM0_A3":"TUN","ISO_A3":"TUN","NAME":"Tunisia","ISO_A2":"TN","NAME_ZH":"突尼斯"},"geometry":{"type":"Polygon","coordinates":[[[9.48214,30.307556],[9.055603,32.102692],[8.439103,32.506285],[8.430473,32.748337],[7.612642,33.344115],[7.524482,34.097376],[8.140981,34.655146],[8.376368,35.479876],[8.217824,36.433177],[8.420964,36.946427],[9.509994,37.349994],[10.210002,37.230002],[10.18065,36.724038],[11.028867,37.092103],[11.100026,36.899996],[10.600005,36.41],[10.593287,35.947444],[10.939519,35.698984],[10.807847,34.833507],[10.149593,34.330773],[10.339659,33.785742],[10.856836,33.76874],[11.108501,33.293343],[11.488787,33.136996],[11.432253,32.368903],[10.94479,32.081815],[10.636901,31.761421],[9.950225,31.37607],[10.056575,30.961831],[9.970017,30.539325],[9.48214,30.307556]]]}},{"type":"Feature","properties":{"ADMIN":"Algeria","NAME_EN":"Algeria","CONTINENT":"Africa","ADM0_A3":"DZA","ISO_A3":"DZA","NAME":"Algeria","ISO_A2":"DZ","NAME_ZH":"阿尔及利亚"},"geometry":{"type":"Polygon","coordinates":[[[-8.6844,27.395744],[-8.665124,27.589479],[-8.66559,27.656426],[-8.674116,28.841289],[-7.059228,29.579228],[-6.060632,29.7317],[-5.242129,30.000443],[-4.859646,30.501188],[-3.690441,30.896952],[-3.647498,31.637294],[-3.06898,31.724498],[-2.616605,32.094346],[-1.307899,32.262889],[-1.124551,32.651522],[-1.388049,32.864015],[-1.733455,33.919713],[-1.792986,34.527919],[-2.169914,35.168396],[-1.208603,35.714849],[-0.127454,35.888662],[0.503877,36.301273],[1.466919,36.605647],[3.161699,36.783905],[4.815758,36.865037],[5.32012,36.716519],[6.26182,37.110655],[7.330385,37.118381],[7.737078,36.885708],[8.420964,36.946427],[8.217824,36.433177],[8.376368,35.479876],[8.140981,34.655146],[7.524482,34.097376],[7.612642,33.344115],[8.430473,32.748337],[8.439103,32.506285],[9.055603,32.102692],[9.48214,30.307556],[9.805634,29.424638],[9.859998,28.95999],[9.683885,28.144174],[9.756128,27.688259],[9.629056,27.140953],[9.716286,26.512206],[9.319411,26.094325],[9.910693,25.365455],[9.948261,24.936954],[10.303847,24.379313],[10.771364,24.562532],[11.560669,24.097909],[11.999506,23.471668],[8.572893,21.565661],[5.677566,19.601207],[4.267419,19.155265],[3.158133,19.057364],[3.146661,19.693579],[2.683588,19.85623],[2.060991,20.142233],[1.823228,20.610809],[-1.550055,22.792666],[-4.923337,24.974574],[-8.6844,27.395744]]]}},{"type":"Feature","properties":{"ADMIN":"Jordan","NAME_EN":"Jordan","CONTINENT":"Asia","ADM0_A3":"JOR","ISO_A3":"JOR","NAME":"Jordan","ISO_A2":"JO","NAME_ZH":"约旦"},"geometry":{"type":"Polygon","coordinates":[[[35.545665,32.393992],[35.719918,32.709192],[36.834062,32.312938],[38.792341,33.378686],[39.195468,32.161009],[39.004886,32.010217],[37.002166,31.508413],[37.998849,30.5085],[37.66812,30.338665],[37.503582,30.003776],[36.740528,29.865283],[36.501214,29.505254],[36.068941,29.197495],[34.956037,29.356555],[34.922603,29.501326],[35.420918,31.100066],[35.397561,31.489086],[35.545252,31.782505],[35.545665,32.393992]]]}},{"type":"Feature","properties":{"ADMIN":"United Arab Emirates","NAME_EN":"United Arab Emirates","CONTINENT":"Asia","ADM0_A3":"ARE","ISO_A3":"ARE","NAME":"United Arab Emirates","ISO_A2":"AE","NAME_ZH":"阿拉伯联合酋长国"},"geometry":{"type":"Polygon","coordinates":[[[51.579519,24.245497],[51.757441,24.294073],[51.794389,24.019826],[52.577081,24.177439],[53.404007,24.151317],[54.008001,24.121758],[54.693024,24.797892],[55.439025,25.439145],[56.070821,26.055464],[56.261042,25.714606],[56.396847,24.924732],[55.886233,24.920831],[55.804119,24.269604],[55.981214,24.130543],[55.528632,23.933604],[55.525841,23.524869],[55.234489,23.110993],[55.208341,22.70833],[55.006803,22.496948],[52.000733,23.001154],[51.617708,24.014219],[51.579519,24.245497]]]}},{"type":"Feature","properties":{"ADMIN":"Qatar","NAME_EN":"Qatar","CONTINENT":"Asia","ADM0_A3":"QAT","ISO_A3":"QAT","NAME":"Qatar","ISO_A2":"QA","NAME_ZH":"卡塔尔"},"geometry":{"type":"Polygon","coordinates":[[[50.810108,24.754743],[50.743911,25.482424],[51.013352,26.006992],[51.286462,26.114582],[51.589079,25.801113],[51.6067,25.21567],[51.389608,24.627386],[51.112415,24.556331],[50.810108,24.754743]]]}},{"type":"Feature","properties":{"ADMIN":"Kuwait","NAME_EN":"Kuwait","CONTINENT":"Asia","ADM0_A3":"KWT","ISO_A3":"KWT","NAME":"Kuwait","ISO_A2":"KW","NAME_ZH":"科威特"},"geometry":{"type":"Polygon","coordinates":[[[47.974519,29.975819],[48.183189,29.534477],[48.093943,29.306299],[48.416094,28.552004],[47.708851,28.526063],[47.459822,29.002519],[46.568713,29.099025],[47.302622,30.05907],[47.974519,29.975819]]]}},{"type":"Feature","properties":{"ADMIN":"Iraq","NAME_EN":"Iraq","CONTINENT":"Asia","ADM0_A3":"IRQ","ISO_A3":"IRQ","NAME":"Iraq","ISO_A2":"IQ","NAME_ZH":"伊拉克"},"geometry":{"type":"Polygon","coordinates":[[[39.195468,32.161009],[38.792341,33.378686],[41.006159,34.419372],[41.383965,35.628317],[41.289707,36.358815],[41.837064,36.605854],[42.349591,37.229873],[42.779126,37.385264],[43.942259,37.256228],[44.293452,37.001514],[44.772677,37.170437],[45.420618,35.977546],[46.07634,35.677383],[46.151788,35.093259],[45.64846,34.748138],[45.416691,33.967798],[46.109362,33.017287],[47.334661,32.469155],[47.849204,31.709176],[47.685286,30.984853],[48.004698,30.985137],[48.014568,30.452457],[48.567971,29.926778],[47.974519,29.975819],[47.302622,30.05907],[46.568713,29.099025],[44.709499,29.178891],[41.889981,31.190009],[40.399994,31.889992],[39.195468,32.161009]]]}},{"type":"Feature","properties":{"ADMIN":"Oman","NAME_EN":"Oman","CONTINENT":"Asia","ADM0_A3":"OMN","ISO_A3":"OMN","NAME":"Oman","ISO_A2":"OM","NAME_ZH":"阿曼"},"geometry":{"type":"MultiPolygon","coordinates":[[[[55.208341,22.70833],[55.234489,23.110993],[55.525841,23.524869],[55.528632,23.933604],[55.981214,24.130543],[55.804119,24.269604],[55.886233,24.920831],[56.396847,24.924732],[56.84514,24.241673],[57.403453,23.878594],[58.136948,23.747931],[58.729211,23.565668],[59.180502,22.992395],[59.450098,22.660271],[59.80806,22.533612],[59.806148,22.310525],[59.442191,21.714541],[59.282408,21.433886],[58.861141,21.114035],[58.487986,20.428986],[58.034318,20.481437],[57.826373,20.243002],[57.665762,19.736005],[57.7887,19.06757],[57.694391,18.94471],[57.234264,18.947991],[56.609651,18.574267],[56.512189,18.087113],[56.283521,17.876067],[55.661492,17.884128],[55.269939,17.632309],[55.2749,17.228354],[54.791002,16.950697],[54.239253,17.044981],[53.570508,16.707663],[53.108573,16.651051],[52.782184,17.349742],[52.00001,19.000003],[54.999982,19.999994],[55.666659,22.000001],[55.208341,22.70833]]],[[[56.261042,25.714606],[56.070821,26.055464],[56.362017,26.395934],[56.485679,26.309118],[56.391421,25.895991],[56.261042,25.714606]]]]}},{"type":"Feature","properties":{"ADMIN":"Vanuatu","NAME_EN":"Vanuatu","CONTINENT":"Oceania","ADM0_A3":"VUT","ISO_A3":"VUT","NAME":"Vanuatu","ISO_A2":"VU","NAME_ZH":"瓦努阿图"},"geometry":{"type":"MultiPolygon","coordinates":[[[[167.216801,-15.891846],[167.844877,-16.466333],[167.515181,-16.59785],[167.180008,-16.159995],[167.216801,-15.891846]]],[[[166.793158,-15.668811],[166.649859,-15.392704],[166.629137,-14.626497],[167.107712,-14.93392],[167.270028,-15.740021],[167.001207,-15.614602],[166.793158,-15.668811]]]]}},{"type":"Feature","properties":{"ADMIN":"Cambodia","NAME_EN":"Cambodia","CONTINENT":"Asia","ADM0_A3":"KHM","ISO_A3":"KHM","NAME":"Cambodia","ISO_A2":"KH","NAME_ZH":"柬埔寨"},"geometry":{"type":"Polygon","coordinates":[[[102.584932,12.186595],[102.348099,13.394247],[102.988422,14.225721],[104.281418,14.416743],[105.218777,14.273212],[106.043946,13.881091],[106.496373,14.570584],[107.382727,14.202441],[107.614548,13.535531],[107.491403,12.337206],[105.810524,11.567615],[106.24967,10.961812],[105.199915,10.88931],[104.334335,10.486544],[103.49728,10.632555],[103.09069,11.153661],[102.584932,12.186595]]]}},{"type":"Feature","properties":{"ADMIN":"Thailand","NAME_EN":"Thailand","CONTINENT":"Asia","ADM0_A3":"THA","ISO_A3":"THA","NAME":"Thailand","ISO_A2":"TH","NAME_ZH":"泰国"},"geometry":{"type":"Polygon","coordinates":[[[105.218777,14.273212],[104.281418,14.416743],[102.988422,14.225721],[102.348099,13.394247],[102.584932,12.186595],[101.687158,12.64574],[100.83181,12.627085],[100.978467,13.412722],[100.097797,13.406856],[100.018733,12.307001],[99.478921,10.846367],[99.153772,9.963061],[99.222399,9.239255],[99.873832,9.207862],[100.279647,8.295153],[100.459274,7.429573],[101.017328,6.856869],[101.623079,6.740622],[102.141187,6.221636],[101.814282,5.810808],[101.154219,5.691384],[101.075516,6.204867],[100.259596,6.642825],[100.085757,6.464489],[99.690691,6.848213],[99.519642,7.343454],[98.988253,7.907993],[98.503786,8.382305],[98.339662,7.794512],[98.150009,8.350007],[98.25915,8.973923],[98.553551,9.93296],[99.038121,10.960546],[99.587286,11.892763],[99.196354,12.804748],[99.212012,13.269294],[99.097755,13.827503],[98.430819,14.622028],[98.192074,15.123703],[98.537376,15.308497],[98.903348,16.177824],[98.493761,16.837836],[97.859123,17.567946],[97.375896,18.445438],[97.797783,18.62708],[98.253724,19.708203],[98.959676,19.752981],[99.543309,20.186598],[100.115988,20.41785],[100.548881,20.109238],[100.606294,19.508344],[101.282015,19.462585],[101.035931,18.408928],[101.059548,17.512497],[102.113592,18.109102],[102.413005,17.932782],[102.998706,17.961695],[103.200192,18.309632],[103.956477,18.240954],[104.716947,17.428859],[104.779321,16.441865],[105.589039,15.570316],[105.544338,14.723934],[105.218777,14.273212]]]}},{"type":"Feature","properties":{"ADMIN":"Laos","NAME_EN":"Laos","CONTINENT":"Asia","ADM0_A3":"LAO","ISO_A3":"LAO","NAME":"Laos","ISO_A2":"LA","NAME_ZH":"老挝"},"geometry":{"type":"Polygon","coordinates":[[[107.382727,14.202441],[106.496373,14.570584],[106.043946,13.881091],[105.218777,14.273212],[105.544338,14.723934],[105.589039,15.570316],[104.779321,16.441865],[104.716947,17.428859],[103.956477,18.240954],[103.200192,18.309632],[102.998706,17.961695],[102.413005,17.932782],[102.113592,18.109102],[101.059548,17.512497],[101.035931,18.408928],[101.282015,19.462585],[100.606294,19.508344],[100.548881,20.109238],[100.115988,20.41785],[100.329101,20.786122],[101.180005,21.436573],[101.270026,21.201652],[101.80312,21.174367],[101.652018,22.318199],[102.170436,22.464753],[102.754896,21.675137],[103.203861,20.766562],[104.435,20.758733],[104.822574,19.886642],[104.183388,19.624668],[103.896532,19.265181],[105.094598,18.666975],[105.925762,17.485315],[106.556008,16.604284],[107.312706,15.908538],[107.564525,15.202173],[107.382727,14.202441]]]}},{"type":"Feature","properties":{"ADMIN":"Myanmar","NAME_EN":"Myanmar","CONTINENT":"Asia","ADM0_A3":"MMR","ISO_A3":"MMR","NAME":"Myanmar","ISO_A2":"MM","NAME_ZH":"缅甸"},"geometry":{"type":"Polygon","coordinates":[[[100.115988,20.41785],[99.543309,20.186598],[98.959676,19.752981],[98.253724,19.708203],[97.797783,18.62708],[97.375896,18.445438],[97.859123,17.567946],[98.493761,16.837836],[98.903348,16.177824],[98.537376,15.308497],[98.192074,15.123703],[98.430819,14.622028],[99.097755,13.827503],[99.212012,13.269294],[99.196354,12.804748],[99.587286,11.892763],[99.038121,10.960546],[98.553551,9.93296],[98.457174,10.675266],[98.764546,11.441292],[98.428339,12.032987],[98.509574,13.122378],[98.103604,13.64046],[97.777732,14.837286],[97.597072,16.100568],[97.16454,16.928734],[96.505769,16.427241],[95.369352,15.71439],[94.808405,15.803454],[94.188804,16.037936],[94.533486,17.27724],[94.324817,18.213514],[93.540988,19.366493],[93.663255,19.726962],[93.078278,19.855145],[92.368554,20.670883],[92.303234,21.475485],[92.652257,21.324048],[92.672721,22.041239],[93.166128,22.27846],[93.060294,22.703111],[93.286327,23.043658],[93.325188,24.078556],[94.106742,23.850741],[94.552658,24.675238],[94.603249,25.162495],[95.155153,26.001307],[95.124768,26.573572],[96.419366,27.264589],[97.133999,27.083774],[97.051989,27.699059],[97.402561,27.882536],[97.327114,28.261583],[97.911988,28.335945],[98.246231,27.747221],[98.68269,27.508812],[98.712094,26.743536],[98.671838,25.918703],[97.724609,25.083637],[97.60472,23.897405],[98.660262,24.063286],[98.898749,23.142722],[99.531992,22.949039],[99.240899,22.118314],[99.983489,21.742937],[100.416538,21.558839],[101.150033,21.849984],[101.180005,21.436573],[100.329101,20.786122],[100.115988,20.41785]]]}},{"type":"Feature","properties":{"ADMIN":"Vietnam","NAME_EN":"Vietnam","CONTINENT":"Asia","ADM0_A3":"VNM","ISO_A3":"VNM","NAME":"Vietnam","ISO_A2":"VN","NAME_ZH":"越南"},"geometry":{"type":"Polygon","coordinates":[[[104.334335,10.486544],[105.199915,10.88931],[106.24967,10.961812],[105.810524,11.567615],[107.491403,12.337206],[107.614548,13.535531],[107.382727,14.202441],[107.564525,15.202173],[107.312706,15.908538],[106.556008,16.604284],[105.925762,17.485315],[105.094598,18.666975],[103.896532,19.265181],[104.183388,19.624668],[104.822574,19.886642],[104.435,20.758733],[103.203861,20.766562],[102.754896,21.675137],[102.170436,22.464753],[102.706992,22.708795],[103.504515,22.703757],[104.476858,22.81915],[105.329209,23.352063],[105.811247,22.976892],[106.725403,22.794268],[106.567273,22.218205],[107.04342,21.811899],[108.05018,21.55238],[106.715068,20.696851],[105.881682,19.75205],[105.662006,19.058165],[106.426817,18.004121],[107.361954,16.697457],[108.269495,16.079742],[108.877107,15.276691],[109.33527,13.426028],[109.200136,11.666859],[108.36613,11.008321],[107.220929,10.364484],[106.405113,9.53084],[105.158264,8.59976],[104.795185,9.241038],[105.076202,9.918491],[104.334335,10.486544]]]}},{"type":"Feature","properties":{"ADMIN":"North Korea","NAME_EN":"North Korea","CONTINENT":"Asia","ADM0_A3":"PRK","ISO_A3":"PRK","NAME":"North Korea","ISO_A2":"KP","NAME_ZH":"朝鲜民主主义人民共和国"},"geometry":{"type":"MultiPolygon","coordinates":[[[[130.780004,42.220008],[130.780005,42.22001],[130.780007,42.220007],[130.780004,42.220008]]],[[[130.64,42.395024],[130.64,42.395],[130.779992,42.22001],[130.400031,42.280004],[129.965949,41.941368],[129.667362,41.601104],[129.705189,40.882828],[129.188115,40.661808],[129.0104,40.485436],[128.633368,40.189847],[127.967414,40.025413],[127.533436,39.75685],[127.50212,39.323931],[127.385434,39.213472],[127.783343,39.050898],[128.349716,38.612243],[128.205746,38.370397],[127.780035,38.304536],[127.073309,38.256115],[126.68372,37.804773],[126.237339,37.840378],[126.174759,37.749686],[125.689104,37.94001],[125.568439,37.752089],[125.27533,37.669071],[125.240087,37.857224],[124.981033,37.948821],[124.712161,38.108346],[124.985994,38.548474],[125.221949,38.665857],[125.132859,38.848559],[125.38659,39.387958],[125.321116,39.551385],[124.737482,39.660344],[124.265625,39.928493],[125.079942,40.569824],[126.182045,41.107336],[126.869083,41.816569],[127.343783,41.503152],[128.208433,41.466772],[128.052215,41.994285],[129.596669,42.424982],[129.994267,42.985387],[130.64,42.395024]]]]}},{"type":"Feature","properties":{"ADMIN":"South Korea","NAME_EN":"South Korea","CONTINENT":"Asia","ADM0_A3":"KOR","ISO_A3":"KOR","NAME":"South Korea","ISO_A2":"KR","NAME_ZH":"大韩民国"},"geometry":{"type":"Polygon","coordinates":[[[126.174759,37.749686],[126.237339,37.840378],[126.68372,37.804773],[127.073309,38.256115],[127.780035,38.304536],[128.205746,38.370397],[128.349716,38.612243],[129.21292,37.432392],[129.46045,36.784189],[129.468304,35.632141],[129.091377,35.082484],[128.18585,34.890377],[127.386519,34.475674],[126.485748,34.390046],[126.37392,34.93456],[126.559231,35.684541],[126.117398,36.725485],[126.860143,36.893924],[126.174759,37.749686]]]}},{"type":"Feature","properties":{"ADMIN":"Mongolia","NAME_EN":"Mongolia","CONTINENT":"Asia","ADM0_A3":"MNG","ISO_A3":"MNG","NAME":"Mongolia","ISO_A2":"MN","NAME_ZH":"蒙古国"},"geometry":{"type":"Polygon","coordinates":[[[87.751264,49.297198],[88.805567,49.470521],[90.713667,50.331812],[92.234712,50.802171],[93.10421,50.49529],[94.147566,50.480537],[94.815949,50.013433],[95.81402,49.97746],[97.25976,49.72605],[98.231762,50.422401],[97.82574,51.010995],[98.861491,52.047366],[99.981732,51.634006],[100.88948,51.516856],[102.06521,51.25991],[102.25589,50.51056],[103.676545,50.089966],[104.62158,50.27532],[105.886591,50.406019],[106.888804,50.274296],[107.868176,49.793705],[108.475167,49.282548],[109.402449,49.292961],[110.662011,49.130128],[111.581231,49.377968],[112.89774,49.543565],[114.362456,50.248303],[114.96211,50.140247],[115.485695,49.805177],[116.678801,49.888531],[116.191802,49.134598],[115.485282,48.135383],[115.742837,47.726545],[116.308953,47.85341],[117.295507,47.697709],[118.064143,48.06673],[118.866574,47.74706],[119.772824,47.048059],[119.66327,46.69268],[118.874326,46.805412],[117.421701,46.672733],[116.717868,46.388202],[115.985096,45.727235],[114.460332,45.339817],[113.463907,44.808893],[112.436062,45.011646],[111.873306,45.102079],[111.348377,44.457442],[111.667737,44.073176],[111.829588,43.743118],[111.129682,43.406834],[110.412103,42.871234],[109.243596,42.519446],[107.744773,42.481516],[106.129316,42.134328],[104.964994,41.59741],[104.522282,41.908347],[103.312278,41.907468],[101.83304,42.514873],[100.845866,42.663804],[99.515817,42.524691],[97.451757,42.74889],[96.349396,42.725635],[95.762455,43.319449],[95.306875,44.241331],[94.688929,44.352332],[93.480734,44.975472],[92.133891,45.115076],[90.94554,45.286073],[90.585768,45.719716],[90.970809,46.888146],[90.280826,47.693549],[88.854298,48.069082],[88.013832,48.599463],[87.751264,49.297198]]]}},{"type":"Feature","properties":{"ADMIN":"India","NAME_EN":"India","CONTINENT":"Asia","ADM0_A3":"IND","ISO_A3":"IND","NAME":"India","ISO_A2":"IN","NAME_ZH":"印度"},"geometry":{"type":"Polygon","coordinates":[[[97.327114,28.261583],[97.402561,27.882536],[97.051989,27.699059],[97.133999,27.083774],[96.419366,27.264589],[95.124768,26.573572],[95.155153,26.001307],[94.603249,25.162495],[94.552658,24.675238],[94.106742,23.850741],[93.325188,24.078556],[93.286327,23.043658],[93.060294,22.703111],[93.166128,22.27846],[92.672721,22.041239],[92.146035,23.627499],[91.869928,23.624346],[91.706475,22.985264],[91.158963,23.503527],[91.46773,24.072639],[91.915093,24.130414],[92.376202,24.976693],[91.799596,25.147432],[90.872211,25.132601],[89.920693,25.26975],[89.832481,25.965082],[89.355094,26.014407],[88.563049,26.446526],[88.209789,25.768066],[88.931554,25.238692],[88.306373,24.866079],[88.084422,24.501657],[88.69994,24.233715],[88.52977,23.631142],[88.876312,22.879146],[89.031961,22.055708],[88.888766,21.690588],[88.208497,21.703172],[86.975704,21.495562],[87.033169,20.743308],[86.499351,20.151638],[85.060266,19.478579],[83.941006,18.30201],[83.189217,17.671221],[82.192792,17.016636],[82.191242,16.556664],[81.692719,16.310219],[80.791999,15.951972],[80.324896,15.899185],[80.025069,15.136415],[80.233274,13.835771],[80.286294,13.006261],[79.862547,12.056215],[79.857999,10.357275],[79.340512,10.308854],[78.885345,9.546136],[79.18972,9.216544],[78.277941,8.933047],[77.941165,8.252959],[77.539898,7.965535],[76.592979,8.899276],[76.130061,10.29963],[75.746467,11.308251],[75.396101,11.781245],[74.864816,12.741936],[74.616717,13.992583],[74.443859,14.617222],[73.534199,15.990652],[73.119909,17.92857],[72.820909,19.208234],[72.824475,20.419503],[72.630533,21.356009],[71.175273,20.757441],[70.470459,20.877331],[69.16413,22.089298],[69.644928,22.450775],[69.349597,22.84318],[68.176645,23.691965],[68.842599,24.359134],[71.04324,24.356524],[70.844699,25.215102],[70.282873,25.722229],[70.168927,26.491872],[69.514393,26.940966],[70.616496,27.989196],[71.777666,27.91318],[72.823752,28.961592],[73.450638,29.976413],[74.42138,30.979815],[74.405929,31.692639],[75.258642,32.271105],[74.451559,32.7649],[74.104294,33.441473],[73.749948,34.317699],[74.240203,34.748887],[75.757061,34.504923],[76.871722,34.653544],[77.837451,35.49401],[78.912269,34.321936],[78.811086,33.506198],[79.208892,32.994395],[79.176129,32.48378],[78.458446,32.618164],[78.738894,31.515906],[79.721367,30.882715],[81.111256,30.183481],[80.476721,29.729865],[80.088425,28.79447],[81.057203,28.416095],[81.999987,27.925479],[83.304249,27.364506],[84.675018,27.234901],[85.251779,26.726198],[86.024393,26.630985],[87.227472,26.397898],[88.060238,26.414615],[88.174804,26.810405],[88.043133,27.445819],[88.120441,27.876542],[88.730326,28.086865],[88.814248,27.299316],[88.835643,27.098966],[89.744528,26.719403],[90.373275,26.875724],[91.217513,26.808648],[92.033484,26.83831],[92.103712,27.452614],[91.696657,27.771742],[92.503119,27.896876],[93.413348,28.640629],[94.56599,29.277438],[95.404802,29.031717],[96.117679,29.452802],[96.586591,28.83098],[96.248833,28.411031],[97.327114,28.261583]]]}},{"type":"Feature","properties":{"ADMIN":"Bangladesh","NAME_EN":"Bangladesh","CONTINENT":"Asia","ADM0_A3":"BGD","ISO_A3":"BGD","NAME":"Bangladesh","ISO_A2":"BD","NAME_ZH":"孟加拉国"},"geometry":{"type":"Polygon","coordinates":[[[92.672721,22.041239],[92.652257,21.324048],[92.303234,21.475485],[92.368554,20.670883],[92.082886,21.192195],[92.025215,21.70157],[91.834891,22.182936],[91.417087,22.765019],[90.496006,22.805017],[90.586957,22.392794],[90.272971,21.836368],[89.847467,22.039146],[89.70205,21.857116],[89.418863,21.966179],[89.031961,22.055708],[88.876312,22.879146],[88.52977,23.631142],[88.69994,24.233715],[88.084422,24.501657],[88.306373,24.866079],[88.931554,25.238692],[88.209789,25.768066],[88.563049,26.446526],[89.355094,26.014407],[89.832481,25.965082],[89.920693,25.26975],[90.872211,25.132601],[91.799596,25.147432],[92.376202,24.976693],[91.915093,24.130414],[91.46773,24.072639],[91.158963,23.503527],[91.706475,22.985264],[91.869928,23.624346],[92.146035,23.627499],[92.672721,22.041239]]]}},{"type":"Feature","properties":{"ADMIN":"Bhutan","NAME_EN":"Bhutan","CONTINENT":"Asia","ADM0_A3":"BTN","ISO_A3":"BTN","NAME":"Bhutan","ISO_A2":"BT","NAME_ZH":"不丹"},"geometry":{"type":"Polygon","coordinates":[[[91.696657,27.771742],[92.103712,27.452614],[92.033484,26.83831],[91.217513,26.808648],[90.373275,26.875724],[89.744528,26.719403],[88.835643,27.098966],[88.814248,27.299316],[89.47581,28.042759],[90.015829,28.296439],[90.730514,28.064954],[91.258854,28.040614],[91.696657,27.771742]]]}},{"type":"Feature","properties":{"ADMIN":"Nepal","NAME_EN":"Nepal","CONTINENT":"Asia","ADM0_A3":"NPL","ISO_A3":"NPL","NAME":"Nepal","ISO_A2":"NP","NAME_ZH":"尼泊尔"},"geometry":{"type":"Polygon","coordinates":[[[88.120441,27.876542],[88.043133,27.445819],[88.174804,26.810405],[88.060238,26.414615],[87.227472,26.397898],[86.024393,26.630985],[85.251779,26.726198],[84.675018,27.234901],[83.304249,27.364506],[81.999987,27.925479],[81.057203,28.416095],[80.088425,28.79447],[80.476721,29.729865],[81.111256,30.183481],[81.525804,30.422717],[82.327513,30.115268],[83.337115,29.463732],[83.898993,29.320226],[84.23458,28.839894],[85.011638,28.642774],[85.82332,28.203576],[86.954517,27.974262],[88.120441,27.876542]]]}},{"type":"Feature","properties":{"ADMIN":"Pakistan","NAME_EN":"Pakistan","CONTINENT":"Asia","ADM0_A3":"PAK","ISO_A3":"PAK","NAME":"Pakistan","ISO_A2":"PK","NAME_ZH":"巴基斯坦"},"geometry":{"type":"Polygon","coordinates":[[[77.837451,35.49401],[76.871722,34.653544],[75.757061,34.504923],[74.240203,34.748887],[73.749948,34.317699],[74.104294,33.441473],[74.451559,32.7649],[75.258642,32.271105],[74.405929,31.692639],[74.42138,30.979815],[73.450638,29.976413],[72.823752,28.961592],[71.777666,27.91318],[70.616496,27.989196],[69.514393,26.940966],[70.168927,26.491872],[70.282873,25.722229],[70.844699,25.215102],[71.04324,24.356524],[68.842599,24.359134],[68.176645,23.691965],[67.443667,23.944844],[67.145442,24.663611],[66.372828,25.425141],[64.530408,25.237039],[62.905701,25.218409],[61.497363,25.078237],[61.874187,26.239975],[63.316632,26.756532],[63.233898,27.217047],[62.755426,27.378923],[62.72783,28.259645],[61.771868,28.699334],[61.369309,29.303276],[60.874248,29.829239],[62.549857,29.318572],[63.550261,29.468331],[64.148002,29.340819],[64.350419,29.560031],[65.046862,29.472181],[66.346473,29.887943],[66.381458,30.738899],[66.938891,31.304911],[67.683394,31.303154],[67.792689,31.58293],[68.556932,31.71331],[68.926677,31.620189],[69.317764,31.901412],[69.262522,32.501944],[69.687147,33.105499],[70.323594,33.358533],[69.930543,34.02012],[70.881803,33.988856],[71.156773,34.348911],[71.115019,34.733126],[71.613076,35.153203],[71.498768,35.650563],[71.262348,36.074388],[71.846292,36.509942],[72.920025,36.720007],[74.067552,36.836176],[74.575893,37.020841],[75.158028,37.133031],[75.896897,36.666806],[76.192848,35.898403],[77.837451,35.49401]]]}},{"type":"Feature","properties":{"ADMIN":"Afghanistan","NAME_EN":"Afghanistan","CONTINENT":"Asia","ADM0_A3":"AFG","ISO_A3":"AFG","NAME":"Afghanistan","ISO_A2":"AF","NAME_ZH":"阿富汗"},"geometry":{"type":"Polygon","coordinates":[[[66.518607,37.362784],[67.075782,37.356144],[67.83,37.144994],[68.135562,37.023115],[68.859446,37.344336],[69.196273,37.151144],[69.518785,37.608997],[70.116578,37.588223],[70.270574,37.735165],[70.376304,38.138396],[70.806821,38.486282],[71.348131,38.258905],[71.239404,37.953265],[71.541918,37.905774],[71.448693,37.065645],[71.844638,36.738171],[72.193041,36.948288],[72.63689,37.047558],[73.260056,37.495257],[73.948696,37.421566],[74.980002,37.41999],[75.158028,37.133031],[74.575893,37.020841],[74.067552,36.836176],[72.920025,36.720007],[71.846292,36.509942],[71.262348,36.074388],[71.498768,35.650563],[71.613076,35.153203],[71.115019,34.733126],[71.156773,34.348911],[70.881803,33.988856],[69.930543,34.02012],[70.323594,33.358533],[69.687147,33.105499],[69.262522,32.501944],[69.317764,31.901412],[68.926677,31.620189],[68.556932,31.71331],[67.792689,31.58293],[67.683394,31.303154],[66.938891,31.304911],[66.381458,30.738899],[66.346473,29.887943],[65.046862,29.472181],[64.350419,29.560031],[64.148002,29.340819],[63.550261,29.468331],[62.549857,29.318572],[60.874248,29.829239],[61.781222,30.73585],[61.699314,31.379506],[60.941945,31.548075],[60.863655,32.18292],[60.536078,32.981269],[60.9637,33.528832],[60.52843,33.676446],[60.803193,34.404102],[61.210817,35.650072],[62.230651,35.270664],[62.984662,35.404041],[63.193538,35.857166],[63.982896,36.007957],[64.546479,36.312073],[64.746105,37.111818],[65.588948,37.305217],[65.745631,37.661164],[66.217385,37.39379],[66.518607,37.362784]]]}},{"type":"Feature","properties":{"ADMIN":"Tajikistan","NAME_EN":"Tajikistan","CONTINENT":"Asia","ADM0_A3":"TJK","ISO_A3":"TJK","NAME":"Tajikistan","ISO_A2":"TJ","NAME_ZH":"塔吉克斯坦"},"geometry":{"type":"Polygon","coordinates":[[[67.83,37.144994],[68.392033,38.157025],[68.176025,38.901553],[67.44222,39.140144],[67.701429,39.580478],[68.536416,39.533453],[69.011633,40.086158],[69.329495,40.727824],[70.666622,40.960213],[70.45816,40.496495],[70.601407,40.218527],[71.014198,40.244366],[70.648019,39.935754],[69.55961,40.103211],[69.464887,39.526683],[70.549162,39.604198],[71.784694,39.279463],[73.675379,39.431237],[73.928852,38.505815],[74.257514,38.606507],[74.864816,38.378846],[74.829986,37.990007],[74.980002,37.41999],[73.948696,37.421566],[73.260056,37.495257],[72.63689,37.047558],[72.193041,36.948288],[71.844638,36.738171],[71.448693,37.065645],[71.541918,37.905774],[71.239404,37.953265],[71.348131,38.258905],[70.806821,38.486282],[70.376304,38.138396],[70.270574,37.735165],[70.116578,37.588223],[69.518785,37.608997],[69.196273,37.151144],[68.859446,37.344336],[68.135562,37.023115],[67.83,37.144994]]]}},{"type":"Feature","properties":{"ADMIN":"Kyrgyzstan","NAME_EN":"Kyrgyzstan","CONTINENT":"Asia","ADM0_A3":"KGZ","ISO_A3":"KGZ","NAME":"Kyrgyzstan","ISO_A2":"KG","NAME_ZH":"吉尔吉斯斯坦"},"geometry":{"type":"Polygon","coordinates":[[[70.962315,42.266154],[71.186281,42.704293],[71.844638,42.845395],[73.489758,42.500894],[73.645304,43.091272],[74.212866,43.298339],[75.636965,42.8779],[76.000354,42.988022],[77.658392,42.960686],[79.142177,42.856092],[79.643645,42.496683],[80.25999,42.349999],[80.11943,42.123941],[78.543661,41.582243],[78.187197,41.185316],[76.904484,41.066486],[76.526368,40.427946],[75.467828,40.562072],[74.776862,40.366425],[73.822244,39.893973],[73.960013,39.660008],[73.675379,39.431237],[71.784694,39.279463],[70.549162,39.604198],[69.464887,39.526683],[69.55961,40.103211],[70.648019,39.935754],[71.014198,40.244366],[71.774875,40.145844],[73.055417,40.866033],[71.870115,41.3929],[71.157859,41.143587],[70.420022,41.519998],[71.259248,42.167711],[70.962315,42.266154]]]}},{"type":"Feature","properties":{"ADMIN":"Turkmenistan","NAME_EN":"Turkmenistan","CONTINENT":"Asia","ADM0_A3":"TKM","ISO_A3":"TKM","NAME":"Turkmenistan","ISO_A2":"TM","NAME_ZH":"土库曼斯坦"},"geometry":{"type":"Polygon","coordinates":[[[52.50246,41.783316],[52.944293,42.116034],[54.079418,42.324109],[54.755345,42.043971],[55.455251,41.259859],[55.968191,41.308642],[57.096391,41.32231],[56.932215,41.826026],[57.78653,42.170553],[58.629011,42.751551],[59.976422,42.223082],[60.083341,41.425146],[60.465953,41.220327],[61.547179,41.26637],[61.882714,41.084857],[62.37426,40.053886],[63.518015,39.363257],[64.170223,38.892407],[65.215999,38.402695],[66.54615,37.974685],[66.518607,37.362784],[66.217385,37.39379],[65.745631,37.661164],[65.588948,37.305217],[64.746105,37.111818],[64.546479,36.312073],[63.982896,36.007957],[63.193538,35.857166],[62.984662,35.404041],[62.230651,35.270664],[61.210817,35.650072],[61.123071,36.491597],[60.377638,36.527383],[59.234762,37.412988],[58.436154,37.522309],[57.330434,38.029229],[56.619366,38.121394],[56.180375,37.935127],[55.511578,37.964117],[54.800304,37.392421],[53.921598,37.198918],[53.735511,37.906136],[53.880929,38.952093],[53.101028,39.290574],[53.357808,39.975286],[52.693973,40.033629],[52.915251,40.876523],[53.858139,40.631034],[54.736845,40.951015],[54.008311,41.551211],[53.721713,42.123191],[52.91675,41.868117],[52.814689,41.135371],[52.50246,41.783316]]]}},{"type":"Feature","properties":{"ADMIN":"Iran","NAME_EN":"Iran","CONTINENT":"Asia","ADM0_A3":"IRN","ISO_A3":"IRN","NAME":"Iran","ISO_A2":"IR","NAME_ZH":"伊朗"},"geometry":{"type":"Polygon","coordinates":[[[48.567971,29.926778],[48.014568,30.452457],[48.004698,30.985137],[47.685286,30.984853],[47.849204,31.709176],[47.334661,32.469155],[46.109362,33.017287],[45.416691,33.967798],[45.64846,34.748138],[46.151788,35.093259],[46.07634,35.677383],[45.420618,35.977546],[44.772677,37.170437],[44.77267,37.17045],[44.225756,37.971584],[44.421403,38.281281],[44.109225,39.428136],[44.79399,39.713003],[44.952688,39.335765],[45.457722,38.874139],[46.143623,38.741201],[46.50572,38.770605],[47.685079,39.508364],[48.060095,39.582235],[48.355529,39.288765],[48.010744,38.794015],[48.634375,38.270378],[48.883249,38.320245],[49.199612,37.582874],[50.147771,37.374567],[50.842354,36.872814],[52.264025,36.700422],[53.82579,36.965031],[53.921598,37.198918],[54.800304,37.392421],[55.511578,37.964117],[56.180375,37.935127],[56.619366,38.121394],[57.330434,38.029229],[58.436154,37.522309],[59.234762,37.412988],[60.377638,36.527383],[61.123071,36.491597],[61.210817,35.650072],[60.803193,34.404102],[60.52843,33.676446],[60.9637,33.528832],[60.536078,32.981269],[60.863655,32.18292],[60.941945,31.548075],[61.699314,31.379506],[61.781222,30.73585],[60.874248,29.829239],[61.369309,29.303276],[61.771868,28.699334],[62.72783,28.259645],[62.755426,27.378923],[63.233898,27.217047],[63.316632,26.756532],[61.874187,26.239975],[61.497363,25.078237],[59.616134,25.380157],[58.525761,25.609962],[57.397251,25.739902],[56.970766,26.966106],[56.492139,27.143305],[55.72371,26.964633],[54.71509,26.480658],[53.493097,26.812369],[52.483598,27.580849],[51.520763,27.86569],[50.852948,28.814521],[50.115009,30.147773],[49.57685,29.985715],[48.941333,30.31709],[48.567971,29.926778]]]}},{"type":"Feature","properties":{"ADMIN":"Syria","NAME_EN":"Syria","CONTINENT":"Asia","ADM0_A3":"SYR","ISO_A3":"SYR","NAME":"Syria","ISO_A2":"SY","NAME_ZH":"叙利亚"},"geometry":{"type":"Polygon","coordinates":[[[35.719918,32.709192],[35.700798,32.716014],[35.836397,32.868123],[35.821101,33.277426],[36.06646,33.824912],[36.61175,34.201789],[36.448194,34.593935],[35.998403,34.644914],[35.905023,35.410009],[36.149763,35.821535],[36.41755,36.040617],[36.685389,36.259699],[36.739494,36.81752],[37.066761,36.623036],[38.167727,36.90121],[38.699891,36.712927],[39.52258,36.716054],[40.673259,37.091276],[41.212089,37.074352],[42.349591,37.229873],[41.837064,36.605854],[41.289707,36.358815],[41.383965,35.628317],[41.006159,34.419372],[38.792341,33.378686],[36.834062,32.312938],[35.719918,32.709192]]]}},{"type":"Feature","properties":{"ADMIN":"Armenia","NAME_EN":"Armenia","CONTINENT":"Asia","ADM0_A3":"ARM","ISO_A3":"ARM","NAME":"Armenia","ISO_A2":"AM","NAME_ZH":"亚美尼亚"},"geometry":{"type":"Polygon","coordinates":[[[46.50572,38.770605],[46.143623,38.741201],[45.735379,39.319719],[45.739978,39.473999],[45.298145,39.471751],[45.001987,39.740004],[44.79399,39.713003],[44.400009,40.005],[43.656436,40.253564],[43.752658,40.740201],[43.582746,41.092143],[44.97248,41.248129],[45.179496,40.985354],[45.560351,40.81229],[45.359175,40.561504],[45.891907,40.218476],[45.610012,39.899994],[46.034534,39.628021],[46.483499,39.464155],[46.50572,38.770605]]]}},{"type":"Feature","properties":{"ADMIN":"Sweden","NAME_EN":"Sweden","CONTINENT":"Europe","ADM0_A3":"SWE","ISO_A3":"SWE","NAME":"Sweden","ISO_A2":"SE","NAME_ZH":"瑞典"},"geometry":{"type":"Polygon","coordinates":[[[11.027369,58.856149],[11.468272,59.432393],[12.300366,60.117933],[12.631147,61.293572],[11.992064,61.800362],[11.930569,63.128318],[12.579935,64.066219],[13.571916,64.049114],[13.919905,64.445421],[13.55569,64.787028],[15.108411,66.193867],[16.108712,67.302456],[16.768879,68.013937],[17.729182,68.010552],[17.993868,68.567391],[19.87856,68.407194],[20.025269,69.065139],[20.645593,69.106247],[21.978535,68.616846],[23.539473,67.936009],[23.56588,66.396051],[23.903379,66.006927],[22.183173,65.723741],[21.213517,65.026005],[21.369631,64.413588],[19.778876,63.609554],[17.847779,62.7494],[17.119555,61.341166],[17.831346,60.636583],[18.787722,60.081914],[17.869225,58.953766],[16.829185,58.719827],[16.44771,57.041118],[15.879786,56.104302],[14.666681,56.200885],[14.100721,55.407781],[12.942911,55.361737],[12.625101,56.30708],[11.787942,57.441817],[11.027369,58.856149]]]}},{"type":"Feature","properties":{"ADMIN":"Belarus","NAME_EN":"Belarus","CONTINENT":"Europe","ADM0_A3":"BLR","ISO_A3":"BLR","NAME":"Belarus","ISO_A2":"BY","NAME_ZH":"白俄罗斯"},"geometry":{"type":"Polygon","coordinates":[[[28.176709,56.16913],[29.229513,55.918344],[29.371572,55.670091],[29.896294,55.789463],[30.873909,55.550976],[30.971836,55.081548],[30.757534,54.811771],[31.384472,54.157056],[31.791424,53.974639],[31.731273,53.794029],[32.405599,53.618045],[32.693643,53.351421],[32.304519,53.132726],[31.49764,53.16743],[31.305201,53.073996],[31.540018,52.742052],[31.78597,52.10168],[31.785992,52.101678],[30.927549,52.042353],[30.619454,51.822806],[30.555117,51.319503],[30.157364,51.416138],[29.254938,51.368234],[28.992835,51.602044],[28.617613,51.427714],[28.241615,51.572227],[27.454066,51.592303],[26.337959,51.832289],[25.327788,51.910656],[24.553106,51.888461],[24.005078,51.617444],[23.527071,51.578454],[23.508002,52.023647],[23.199494,52.486977],[23.799199,52.691099],[23.804935,53.089731],[23.527536,53.470122],[23.484128,53.912498],[24.450684,53.905702],[25.536354,54.282423],[25.768433,54.846963],[26.588279,55.167176],[26.494331,55.615107],[27.10246,55.783314],[28.176709,56.16913]]]}},{"type":"Feature","properties":{"ADMIN":"Ukraine","NAME_EN":"Ukraine","CONTINENT":"Europe","ADM0_A3":"UKR","ISO_A3":"UKR","NAME":"Ukraine","ISO_A2":"UA","NAME_ZH":"乌克兰"},"geometry":{"type":"Polygon","coordinates":[[[31.785992,52.101678],[32.15944,52.06125],[32.412058,52.288695],[32.715761,52.238465],[33.7527,52.335075],[34.391731,51.768882],[34.141978,51.566413],[34.224816,51.255993],[35.022183,51.207572],[35.37791,50.77394],[35.356116,50.577197],[36.626168,50.225591],[37.39346,50.383953],[38.010631,49.915662],[38.594988,49.926462],[40.06904,49.60105],[40.080789,49.30743],[39.67465,48.78382],[39.89562,48.23241],[39.738278,47.898937],[38.77057,47.82562],[38.255112,47.5464],[38.223538,47.10219],[37.425137,47.022221],[36.759855,46.6987],[35.823685,46.645964],[34.962342,46.273197],[35.012659,45.737725],[34.861792,45.768182],[34.732017,45.965666],[34.410402,46.005162],[33.699462,46.219573],[33.435988,45.971917],[33.298567,46.080598],[31.74414,46.333348],[31.675307,46.706245],[30.748749,46.5831],[30.377609,46.03241],[29.603289,45.293308],[29.149725,45.464925],[28.679779,45.304031],[28.233554,45.488283],[28.485269,45.596907],[28.659987,45.939987],[28.933717,46.25883],[28.862972,46.437889],[29.072107,46.517678],[29.170654,46.379262],[29.759972,46.349988],[30.024659,46.423937],[29.83821,46.525326],[29.908852,46.674361],[29.559674,46.928583],[29.415135,47.346645],[29.050868,47.510227],[29.122698,47.849095],[28.670891,48.118149],[28.259547,48.155562],[27.522537,48.467119],[26.857824,48.368211],[26.619337,48.220726],[26.19745,48.220881],[25.945941,47.987149],[25.207743,47.891056],[24.866317,47.737526],[24.402056,47.981878],[23.760958,47.985598],[23.142236,48.096341],[22.710531,47.882194],[22.64082,48.15024],[22.085608,48.422264],[22.280842,48.825392],[22.558138,49.085738],[22.776419,49.027395],[22.51845,49.476774],[23.426508,50.308506],[23.922757,50.424881],[24.029986,50.705407],[23.527071,51.578454],[24.005078,51.617444],[24.553106,51.888461],[25.327788,51.910656],[26.337959,51.832289],[27.454066,51.592303],[28.241615,51.572227],[28.617613,51.427714],[28.992835,51.602044],[29.254938,51.368234],[30.157364,51.416138],[30.555117,51.319503],[30.619454,51.822806],[30.927549,52.042353],[31.785992,52.101678]]]}},{"type":"Feature","properties":{"ADMIN":"Poland","NAME_EN":"Poland","CONTINENT":"Europe","ADM0_A3":"POL","ISO_A3":"POL","NAME":"Poland","ISO_A2":"PL","NAME_ZH":"波兰"},"geometry":{"type":"Polygon","coordinates":[[[23.484128,53.912498],[23.527536,53.470122],[23.804935,53.089731],[23.799199,52.691099],[23.199494,52.486977],[23.508002,52.023647],[23.527071,51.578454],[24.029986,50.705407],[23.922757,50.424881],[23.426508,50.308506],[22.51845,49.476774],[22.776419,49.027395],[22.558138,49.085738],[21.607808,49.470107],[20.887955,49.328772],[20.415839,49.431453],[19.825023,49.217125],[19.320713,49.571574],[18.909575,49.435846],[18.853144,49.49623],[18.392914,49.988629],[17.649445,50.049038],[17.554567,50.362146],[16.868769,50.473974],[16.719476,50.215747],[16.176253,50.422607],[16.238627,50.697733],[15.490972,50.78473],[15.016996,51.106674],[14.607098,51.745188],[14.685026,52.089947],[14.4376,52.62485],[14.074521,52.981263],[14.353315,53.248171],[14.119686,53.757029],[14.8029,54.050706],[16.363477,54.513159],[17.622832,54.851536],[18.620859,54.682606],[18.696255,54.438719],[19.66064,54.426084],[20.892245,54.312525],[22.731099,54.327537],[23.243987,54.220567],[23.484128,53.912498]]]}},{"type":"Feature","properties":{"ADMIN":"Austria","NAME_EN":"Austria","CONTINENT":"Europe","ADM0_A3":"AUT","ISO_A3":"AUT","NAME":"Austria","ISO_A2":"AT","NAME_ZH":"奥地利"},"geometry":{"type":"Polygon","coordinates":[[[16.979667,48.123497],[16.903754,47.714866],[16.340584,47.712902],[16.534268,47.496171],[16.202298,46.852386],[16.011664,46.683611],[15.137092,46.658703],[14.632472,46.431817],[13.806475,46.509306],[12.376485,46.767559],[12.153088,47.115393],[11.164828,46.941579],[11.048556,46.751359],[10.442701,46.893546],[9.932448,46.920728],[9.47997,47.10281],[9.632932,47.347601],[9.594226,47.525058],[9.896068,47.580197],[10.402084,47.302488],[10.544504,47.566399],[11.426414,47.523766],[12.141357,47.703083],[12.62076,47.672388],[12.932627,47.467646],[13.025851,47.637584],[12.884103,48.289146],[13.243357,48.416115],[13.595946,48.877172],[14.338898,48.555305],[14.901447,48.964402],[15.253416,49.039074],[16.029647,48.733899],[16.499283,48.785808],[16.960288,48.596982],[16.879983,48.470013],[16.979667,48.123497]]]}},{"type":"Feature","properties":{"ADMIN":"Hungary","NAME_EN":"Hungary","CONTINENT":"Europe","ADM0_A3":"HUN","ISO_A3":"HUN","NAME":"Hungary","ISO_A2":"HU","NAME_ZH":"匈牙利"},"geometry":{"type":"Polygon","coordinates":[[[22.085608,48.422264],[22.64082,48.15024],[22.710531,47.882194],[22.099768,47.672439],[21.626515,46.994238],[21.021952,46.316088],[20.220192,46.127469],[19.596045,46.17173],[18.829838,45.908878],[18.829825,45.908872],[18.456062,45.759481],[17.630066,45.951769],[16.882515,46.380632],[16.564808,46.503751],[16.370505,46.841327],[16.202298,46.852386],[16.534268,47.496171],[16.340584,47.712902],[16.903754,47.714866],[16.979667,48.123497],[17.488473,47.867466],[17.857133,47.758429],[18.696513,47.880954],[18.777025,48.081768],[19.174365,48.111379],[19.661364,48.266615],[19.769471,48.202691],[20.239054,48.327567],[20.473562,48.56285],[20.801294,48.623854],[21.872236,48.319971],[22.085608,48.422264]]]}},{"type":"Feature","properties":{"ADMIN":"Moldova","NAME_EN":"Moldova","CONTINENT":"Europe","ADM0_A3":"MDA","ISO_A3":"MDA","NAME":"Moldova","ISO_A2":"MD","NAME_ZH":"摩尔多瓦"},"geometry":{"type":"Polygon","coordinates":[[[26.619337,48.220726],[26.857824,48.368211],[27.522537,48.467119],[28.259547,48.155562],[28.670891,48.118149],[29.122698,47.849095],[29.050868,47.510227],[29.415135,47.346645],[29.559674,46.928583],[29.908852,46.674361],[29.83821,46.525326],[30.024659,46.423937],[29.759972,46.349988],[29.170654,46.379262],[29.072107,46.517678],[28.862972,46.437889],[28.933717,46.25883],[28.659987,45.939987],[28.485269,45.596907],[28.233554,45.488283],[28.054443,45.944586],[28.160018,46.371563],[28.12803,46.810476],[27.551166,47.405117],[27.233873,47.826771],[26.924176,48.123264],[26.619337,48.220726]]]}},{"type":"Feature","properties":{"ADMIN":"Romania","NAME_EN":"Romania","CONTINENT":"Europe","ADM0_A3":"ROU","ISO_A3":"ROU","NAME":"Romania","ISO_A2":"RO","NAME_ZH":"罗马尼亚"},"geometry":{"type":"Polygon","coordinates":[[[28.233554,45.488283],[28.679779,45.304031],[29.149725,45.464925],[29.603289,45.293308],[29.626543,45.035391],[29.141612,44.82021],[28.837858,44.913874],[28.558081,43.707462],[27.970107,43.812468],[27.2424,44.175986],[26.065159,43.943494],[25.569272,43.688445],[24.100679,43.741051],[23.332302,43.897011],[22.944832,43.823785],[22.65715,44.234923],[22.474008,44.409228],[22.705726,44.578003],[22.459022,44.702517],[22.145088,44.478422],[21.562023,44.768947],[21.483526,45.18117],[20.874313,45.416375],[20.762175,45.734573],[20.220192,46.127469],[21.021952,46.316088],[21.626515,46.994238],[22.099768,47.672439],[22.710531,47.882194],[23.142236,48.096341],[23.760958,47.985598],[24.402056,47.981878],[24.866317,47.737526],[25.207743,47.891056],[25.945941,47.987149],[26.19745,48.220881],[26.619337,48.220726],[26.924176,48.123264],[27.233873,47.826771],[27.551166,47.405117],[28.12803,46.810476],[28.160018,46.371563],[28.054443,45.944586],[28.233554,45.488283]]]}},{"type":"Feature","properties":{"ADMIN":"Lithuania","NAME_EN":"Lithuania","CONTINENT":"Europe","ADM0_A3":"LTU","ISO_A3":"LTU","NAME":"Lithuania","ISO_A2":"LT","NAME_ZH":"立陶宛"},"geometry":{"type":"Polygon","coordinates":[[[26.494331,55.615107],[26.588279,55.167176],[25.768433,54.846963],[25.536354,54.282423],[24.450684,53.905702],[23.484128,53.912498],[23.243987,54.220567],[22.731099,54.327537],[22.651052,54.582741],[22.757764,54.856574],[22.315724,55.015299],[21.268449,55.190482],[21.0558,56.031076],[22.201157,56.337802],[23.878264,56.273671],[24.860684,56.372528],[25.000934,56.164531],[25.533047,56.100297],[26.494331,55.615107]]]}},{"type":"Feature","properties":{"ADMIN":"Latvia","NAME_EN":"Latvia","CONTINENT":"Europe","ADM0_A3":"LVA","ISO_A3":"LVA","NAME":"Latvia","ISO_A2":"LV","NAME_ZH":"拉脱维亚"},"geometry":{"type":"Polygon","coordinates":[[[27.288185,57.474528],[27.770016,57.244258],[27.855282,56.759326],[28.176709,56.16913],[27.10246,55.783314],[26.494331,55.615107],[25.533047,56.100297],[25.000934,56.164531],[24.860684,56.372528],[23.878264,56.273671],[22.201157,56.337802],[21.0558,56.031076],[21.090424,56.783873],[21.581866,57.411871],[22.524341,57.753374],[23.318453,57.006236],[24.12073,57.025693],[24.312863,57.793424],[25.164594,57.970157],[25.60281,57.847529],[26.463532,57.476389],[27.288185,57.474528]]]}},{"type":"Feature","properties":{"ADMIN":"Estonia","NAME_EN":"Estonia","CONTINENT":"Europe","ADM0_A3":"EST","ISO_A3":"EST","NAME":"Estonia","ISO_A2":"EE","NAME_ZH":"爱沙尼亚"},"geometry":{"type":"Polygon","coordinates":[[[27.981127,59.475373],[27.98112,59.47537],[28.131699,59.300825],[27.42015,58.72457],[27.716686,57.791899],[27.288185,57.474528],[26.463532,57.476389],[25.60281,57.847529],[25.164594,57.970157],[24.312863,57.793424],[24.428928,58.383413],[24.061198,58.257375],[23.42656,58.612753],[23.339795,59.18724],[24.604214,59.465854],[25.864189,59.61109],[26.949136,59.445803],[27.981114,59.475388],[27.981127,59.475373]]]}},{"type":"Feature","properties":{"ADMIN":"Germany","NAME_EN":"Germany","CONTINENT":"Europe","ADM0_A3":"DEU","ISO_A3":"DEU","NAME":"Germany","ISO_A2":"DE","NAME_ZH":"德国"},"geometry":{"type":"Polygon","coordinates":[[[14.119686,53.757029],[14.353315,53.248171],[14.074521,52.981263],[14.4376,52.62485],[14.685026,52.089947],[14.607098,51.745188],[15.016996,51.106674],[14.570718,51.002339],[14.307013,51.117268],[14.056228,50.926918],[13.338132,50.733234],[12.966837,50.484076],[12.240111,50.266338],[12.415191,49.969121],[12.521024,49.547415],[13.031329,49.307068],[13.595946,48.877172],[13.243357,48.416115],[12.884103,48.289146],[13.025851,47.637584],[12.932627,47.467646],[12.62076,47.672388],[12.141357,47.703083],[11.426414,47.523766],[10.544504,47.566399],[10.402084,47.302488],[9.896068,47.580197],[9.594226,47.525058],[8.522612,47.830828],[8.317301,47.61358],[7.466759,47.620582],[7.593676,48.333019],[8.099279,49.017784],[6.65823,49.201958],[6.18632,49.463803],[6.242751,49.902226],[6.043073,50.128052],[6.156658,50.803721],[5.988658,51.851616],[6.589397,51.852029],[6.84287,52.22844],[7.092053,53.144043],[6.90514,53.482162],[7.100425,53.693932],[7.936239,53.748296],[8.121706,53.527792],[8.800734,54.020786],[8.572118,54.395646],[8.526229,54.962744],[9.282049,54.830865],[9.921906,54.983104],[9.93958,54.596642],[10.950112,54.363607],[10.939467,54.008693],[11.956252,54.196486],[12.51844,54.470371],[13.647467,54.075511],[14.119686,53.757029]]]}},{"type":"Feature","properties":{"ADMIN":"Bulgaria","NAME_EN":"Bulgaria","CONTINENT":"Europe","ADM0_A3":"BGR","ISO_A3":"BGR","NAME":"Bulgaria","ISO_A2":"BG","NAME_ZH":"保加利亚"},"geometry":{"type":"Polygon","coordinates":[[[22.65715,44.234923],[22.944832,43.823785],[23.332302,43.897011],[24.100679,43.741051],[25.569272,43.688445],[26.065159,43.943494],[27.2424,44.175986],[27.970107,43.812468],[28.558081,43.707462],[28.039095,43.293172],[27.673898,42.577892],[27.99672,42.007359],[27.135739,42.141485],[26.117042,41.826905],[26.106138,41.328899],[25.197201,41.234486],[24.492645,41.583896],[23.692074,41.309081],[22.952377,41.337994],[22.881374,41.999297],[22.380526,42.32026],[22.545012,42.461362],[22.436595,42.580321],[22.604801,42.898519],[22.986019,43.211161],[22.500157,43.642814],[22.410446,44.008063],[22.65715,44.234923]]]}},{"type":"Feature","properties":{"ADMIN":"Greece","NAME_EN":"Greece","CONTINENT":"Europe","ADM0_A3":"GRC","ISO_A3":"GRC","NAME":"Greece","ISO_A2":"GR","NAME_ZH":"希腊"},"geometry":{"type":"MultiPolygon","coordinates":[[[[26.290003,35.29999],[26.164998,35.004995],[24.724982,34.919988],[24.735007,35.084991],[23.514978,35.279992],[23.69998,35.705004],[24.246665,35.368022],[25.025015,35.424996],[25.769208,35.354018],[25.745023,35.179998],[26.290003,35.29999]]],[[[22.952377,41.337994],[23.692074,41.309081],[24.492645,41.583896],[25.197201,41.234486],[26.106138,41.328899],[26.117042,41.826905],[26.604196,41.562115],[26.294602,40.936261],[26.056942,40.824123],[25.447677,40.852545],[24.925848,40.947062],[23.714811,40.687129],[24.407999,40.124993],[23.899968,39.962006],[23.342999,39.960998],[22.813988,40.476005],[22.626299,40.256561],[22.849748,39.659311],[23.350027,39.190011],[22.973099,38.970903],[23.530016,38.510001],[24.025025,38.219993],[24.040011,37.655015],[23.115003,37.920011],[23.409972,37.409991],[22.774972,37.30501],[23.154225,36.422506],[22.490028,36.41],[21.670026,36.844986],[21.295011,37.644989],[21.120034,38.310323],[20.730032,38.769985],[20.217712,39.340235],[20.150016,39.624998],[20.615,40.110007],[20.674997,40.435],[20.99999,40.580004],[21.02004,40.842727],[21.674161,40.931275],[22.055378,41.149866],[22.597308,41.130487],[22.76177,41.3048],[22.952377,41.337994]]]]}},{"type":"Feature","properties":{"ADMIN":"Turkey","NAME_EN":"Turkey","CONTINENT":"Asia","ADM0_A3":"TUR","ISO_A3":"TUR","NAME":"Turkey","ISO_A2":"TR","NAME_ZH":"土耳其"},"geometry":{"type":"MultiPolygon","coordinates":[[[[44.772677,37.170437],[44.293452,37.001514],[43.942259,37.256228],[42.779126,37.385264],[42.349591,37.229873],[41.212089,37.074352],[40.673259,37.091276],[39.52258,36.716054],[38.699891,36.712927],[38.167727,36.90121],[37.066761,36.623036],[36.739494,36.81752],[36.685389,36.259699],[36.41755,36.040617],[36.149763,35.821535],[35.782085,36.274995],[36.160822,36.650606],[35.550936,36.565443],[34.714553,36.795532],[34.026895,36.21996],[32.509158,36.107564],[31.699595,36.644275],[30.621625,36.677865],[30.391096,36.262981],[29.699976,36.144357],[28.732903,36.676831],[27.641187,36.658822],[27.048768,37.653361],[26.318218,38.208133],[26.8047,38.98576],[26.170785,39.463612],[27.28002,40.420014],[28.819978,40.460011],[29.240004,41.219991],[31.145934,41.087622],[32.347979,41.736264],[33.513283,42.01896],[35.167704,42.040225],[36.913127,41.335358],[38.347665,40.948586],[39.512607,41.102763],[40.373433,41.013673],[41.554084,41.535656],[42.619549,41.583173],[43.582746,41.092143],[43.752658,40.740201],[43.656436,40.253564],[44.400009,40.005],[44.79399,39.713003],[44.109225,39.428136],[44.421403,38.281281],[44.225756,37.971584],[44.77267,37.17045],[44.772677,37.170437]]],[[[26.117042,41.826905],[27.135739,42.141485],[27.99672,42.007359],[28.115525,41.622886],[28.988443,41.299934],[28.806438,41.054962],[27.619017,40.999823],[27.192377,40.690566],[26.358009,40.151994],[26.043351,40.617754],[26.056942,40.824123],[26.294602,40.936261],[26.604196,41.562115],[26.117042,41.826905]]]]}},{"type":"Feature","properties":{"ADMIN":"Albania","NAME_EN":"Albania","CONTINENT":"Europe","ADM0_A3":"ALB","ISO_A3":"ALB","NAME":"Albania","ISO_A2":"AL","NAME_ZH":"阿尔巴尼亚"},"geometry":{"type":"Polygon","coordinates":[[[21.02004,40.842727],[20.99999,40.580004],[20.674997,40.435],[20.615,40.110007],[20.150016,39.624998],[19.98,39.694993],[19.960002,39.915006],[19.406082,40.250773],[19.319059,40.72723],[19.40355,41.409566],[19.540027,41.719986],[19.371769,41.877548],[19.371768,41.877551],[19.304486,42.195745],[19.738051,42.688247],[19.801613,42.500093],[20.0707,42.58863],[20.283755,42.32026],[20.52295,42.21787],[20.590247,41.855409],[20.590247,41.855404],[20.463175,41.515089],[20.605182,41.086226],[21.02004,40.842727]]]}},{"type":"Feature","properties":{"ADMIN":"Croatia","NAME_EN":"Croatia","CONTINENT":"Europe","ADM0_A3":"HRV","ISO_A3":"HRV","NAME":"Croatia","ISO_A2":"HR","NAME_ZH":"克罗地亚"},"geometry":{"type":"Polygon","coordinates":[[[16.564808,46.503751],[16.882515,46.380632],[17.630066,45.951769],[18.456062,45.759481],[18.829825,45.908872],[19.072769,45.521511],[19.390476,45.236516],[19.005485,44.860234],[18.553214,45.08159],[17.861783,45.06774],[17.002146,45.233777],[16.534939,45.211608],[16.318157,45.004127],[15.959367,45.233777],[15.750026,44.818712],[16.23966,44.351143],[16.456443,44.04124],[16.916156,43.667722],[17.297373,43.446341],[17.674922,43.028563],[18.56,42.65],[18.450017,42.479992],[18.450016,42.479991],[17.50997,42.849995],[16.930006,43.209998],[16.015385,43.507215],[15.174454,44.243191],[15.37625,44.317915],[14.920309,44.738484],[14.901602,45.07606],[14.258748,45.233777],[13.952255,44.802124],[13.656976,45.136935],[13.679403,45.484149],[13.71506,45.500324],[14.411968,45.466166],[14.595109,45.634941],[14.935244,45.471695],[15.327675,45.452316],[15.323954,45.731783],[15.67153,45.834154],[15.768733,46.238108],[16.564808,46.503751]]]}},{"type":"Feature","properties":{"ADMIN":"Switzerland","NAME_EN":"Switzerland","CONTINENT":"Europe","ADM0_A3":"CHE","ISO_A3":"CHE","NAME":"Switzerland","ISO_A2":"CH","NAME_ZH":"瑞士"},"geometry":{"type":"Polygon","coordinates":[[[9.594226,47.525058],[9.632932,47.347601],[9.47997,47.10281],[9.932448,46.920728],[10.442701,46.893546],[10.363378,46.483571],[9.922837,46.314899],[9.182882,46.440215],[8.966306,46.036932],[8.489952,46.005151],[8.31663,46.163642],[7.755992,45.82449],[7.273851,45.776948],[6.843593,45.991147],[6.5001,46.429673],[6.022609,46.27299],[6.037389,46.725779],[6.768714,47.287708],[6.736571,47.541801],[7.192202,47.449766],[7.466759,47.620582],[8.317301,47.61358],[8.522612,47.830828],[9.594226,47.525058]]]}},{"type":"Feature","properties":{"ADMIN":"Luxembourg","NAME_EN":"Luxembourg","CONTINENT":"Europe","ADM0_A3":"LUX","ISO_A3":"LUX","NAME":"Luxembourg","ISO_A2":"LU","NAME_ZH":"卢森堡"},"geometry":{"type":"Polygon","coordinates":[[[6.043073,50.128052],[6.242751,49.902226],[6.18632,49.463803],[5.897759,49.442667],[5.674052,49.529484],[5.782417,50.090328],[6.043073,50.128052]]]}},{"type":"Feature","properties":{"ADMIN":"Belgium","NAME_EN":"Belgium","CONTINENT":"Europe","ADM0_A3":"BEL","ISO_A3":"BEL","NAME":"Belgium","ISO_A2":"BE","NAME_ZH":"比利时"},"geometry":{"type":"Polygon","coordinates":[[[6.156658,50.803721],[6.043073,50.128052],[5.782417,50.090328],[5.674052,49.529484],[4.799222,49.985373],[4.286023,49.907497],[3.588184,50.378992],[3.123252,50.780363],[2.658422,50.796848],[2.513573,51.148506],[3.314971,51.345781],[3.315011,51.345777],[3.314971,51.345755],[4.047071,51.267259],[4.973991,51.475024],[5.606976,51.037298],[6.156658,50.803721]]]}},{"type":"Feature","properties":{"ADMIN":"Netherlands","NAME_EN":"Netherlands","CONTINENT":"Europe","ADM0_A3":"NLD","ISO_A3":"NLD","NAME":"Netherlands","ISO_A2":"NL","NAME_ZH":"荷兰"},"geometry":{"type":"Polygon","coordinates":[[[6.90514,53.482162],[7.092053,53.144043],[6.84287,52.22844],[6.589397,51.852029],[5.988658,51.851616],[6.156658,50.803721],[5.606976,51.037298],[4.973991,51.475024],[4.047071,51.267259],[3.314971,51.345755],[3.315011,51.345777],[3.830289,51.620545],[4.705997,53.091798],[6.074183,53.510403],[6.90514,53.482162]]]}},{"type":"Feature","properties":{"ADMIN":"Portugal","NAME_EN":"Portugal","CONTINENT":"Europe","ADM0_A3":"PRT","ISO_A3":"PRT","NAME":"Portugal","ISO_A2":"PT","NAME_ZH":"葡萄牙"},"geometry":{"type":"Polygon","coordinates":[[[-9.034818,41.880571],[-8.671946,42.134689],[-8.263857,42.280469],[-8.013175,41.790886],[-7.422513,41.792075],[-7.251309,41.918346],[-6.668606,41.883387],[-6.389088,41.381815],[-6.851127,41.111083],[-6.86402,40.330872],[-7.026413,40.184524],[-7.066592,39.711892],[-7.498632,39.629571],[-7.098037,39.030073],[-7.374092,38.373059],[-7.029281,38.075764],[-7.166508,37.803894],[-7.537105,37.428904],[-7.453726,37.097788],[-7.855613,36.838269],[-8.382816,36.97888],[-8.898857,36.868809],[-8.746101,37.651346],[-8.839998,38.266243],[-9.287464,38.358486],[-9.526571,38.737429],[-9.446989,39.392066],[-9.048305,39.755093],[-8.977353,40.159306],[-8.768684,40.760639],[-8.790853,41.184334],[-8.990789,41.543459],[-9.034818,41.880571]]]}},{"type":"Feature","properties":{"ADMIN":"Spain","NAME_EN":"Spain","CONTINENT":"Europe","ADM0_A3":"ESP","ISO_A3":"ESP","NAME":"Spain","ISO_A2":"ES","NAME_ZH":"西班牙"},"geometry":{"type":"Polygon","coordinates":[[[-7.453726,37.097788],[-7.537105,37.428904],[-7.166508,37.803894],[-7.029281,38.075764],[-7.374092,38.373059],[-7.098037,39.030073],[-7.498632,39.629571],[-7.066592,39.711892],[-7.026413,40.184524],[-6.86402,40.330872],[-6.851127,41.111083],[-6.389088,41.381815],[-6.668606,41.883387],[-7.251309,41.918346],[-7.422513,41.792075],[-8.013175,41.790886],[-8.263857,42.280469],[-8.671946,42.134689],[-9.034818,41.880571],[-8.984433,42.592775],[-9.392884,43.026625],[-7.97819,43.748338],[-6.754492,43.567909],[-5.411886,43.57424],[-4.347843,43.403449],[-3.517532,43.455901],[-1.901351,43.422802],[-1.502771,43.034014],[0.338047,42.579546],[0.701591,42.795734],[1.826793,42.343385],[2.985999,42.473015],[3.039484,41.89212],[2.091842,41.226089],[0.810525,41.014732],[0.721331,40.678318],[0.106692,40.123934],[-0.278711,39.309978],[0.111291,38.738514],[-0.467124,38.292366],[-0.683389,37.642354],[-1.438382,37.443064],[-2.146453,36.674144],[-3.415781,36.6589],[-4.368901,36.677839],[-4.995219,36.324708],[-5.37716,35.94685],[-5.866432,36.029817],[-6.236694,36.367677],[-6.520191,36.942913],[-7.453726,37.097788]]]}},{"type":"Feature","properties":{"ADMIN":"Ireland","NAME_EN":"Ireland","CONTINENT":"Europe","ADM0_A3":"IRL","ISO_A3":"IRL","NAME":"Ireland","ISO_A2":"IE","NAME_ZH":"爱尔兰"},"geometry":{"type":"Polygon","coordinates":[[[-6.197885,53.867565],[-6.032985,53.153164],[-6.788857,52.260118],[-8.561617,51.669301],[-9.977086,51.820455],[-9.166283,52.864629],[-9.688525,53.881363],[-8.327987,54.664519],[-7.572168,55.131622],[-7.366031,54.595841],[-7.572168,54.059956],[-6.95373,54.073702],[-6.197885,53.867565]]]}},{"type":"Feature","properties":{"ADMIN":"New Caledonia","NAME_EN":"New Caledonia","CONTINENT":"Oceania","ADM0_A3":"NCL","ISO_A3":"NCL","NAME":"New Caledonia","ISO_A2":"NC","NAME_ZH":"新喀里多尼亚"},"geometry":{"type":"Polygon","coordinates":[[[165.77999,-21.080005],[166.599991,-21.700019],[167.120011,-22.159991],[166.740035,-22.399976],[166.189732,-22.129708],[165.474375,-21.679607],[164.829815,-21.14982],[164.167995,-20.444747],[164.029606,-20.105646],[164.459967,-20.120012],[165.020036,-20.459991],[165.460009,-20.800022],[165.77999,-21.080005]]]}},{"type":"Feature","properties":{"ADMIN":"Solomon Islands","NAME_EN":"Solomon Islands","CONTINENT":"Oceania","ADM0_A3":"SLB","ISO_A3":"SLB","NAME":"Solomon Is.","ISO_A2":"SB","NAME_ZH":"所罗门群岛"},"geometry":{"type":"MultiPolygon","coordinates":[[[[162.119025,-10.482719],[162.398646,-10.826367],[161.700032,-10.820011],[161.319797,-10.204751],[161.917383,-10.446701],[162.119025,-10.482719]]],[[[161.679982,-9.599982],[161.529397,-9.784312],[160.788253,-8.917543],[160.579997,-8.320009],[160.920028,-8.320009],[161.280006,-9.120011],[161.679982,-9.599982]]],[[[160.852229,-9.872937],[160.462588,-9.89521],[159.849447,-9.794027],[159.640003,-9.63998],[159.702945,-9.24295],[160.362956,-9.400304],[160.688518,-9.610162],[160.852229,-9.872937]]],[[[159.640003,-8.020027],[159.875027,-8.33732],[159.917402,-8.53829],[159.133677,-8.114181],[158.586114,-7.754824],[158.21115,-7.421872],[158.359978,-7.320018],[158.820001,-7.560003],[159.640003,-8.020027]]],[[[157.14,-7.021638],[157.538426,-7.34782],[157.33942,-7.404767],[156.90203,-7.176874],[156.491358,-6.765943],[156.542828,-6.599338],[157.14,-7.021638]]]]}},{"type":"Feature","properties":{"ADMIN":"New Zealand","NAME_EN":"New Zealand","CONTINENT":"Oceania","ADM0_A3":"NZL","ISO_A3":"NZL","NAME":"New Zealand","ISO_A2":"NZ","NAME_ZH":"新西兰"},"geometry":{"type":"MultiPolygon","coordinates":[[[[176.885824,-40.065978],[176.508017,-40.604808],[176.01244,-41.289624],[175.239567,-41.688308],[175.067898,-41.425895],[174.650973,-41.281821],[175.22763,-40.459236],[174.900157,-39.908933],[173.824047,-39.508854],[173.852262,-39.146602],[174.574802,-38.797683],[174.743474,-38.027808],[174.697017,-37.381129],[174.292028,-36.711092],[174.319004,-36.534824],[173.840997,-36.121981],[173.054171,-35.237125],[172.636005,-34.529107],[173.007042,-34.450662],[173.551298,-35.006183],[174.32939,-35.265496],[174.612009,-36.156397],[175.336616,-37.209098],[175.357596,-36.526194],[175.808887,-36.798942],[175.95849,-37.555382],[176.763195,-37.881253],[177.438813,-37.961248],[178.010354,-37.579825],[178.517094,-37.695373],[178.274731,-38.582813],[177.97046,-39.166343],[177.206993,-39.145776],[176.939981,-39.449736],[177.032946,-39.879943],[176.885824,-40.065978]]],[[[169.667815,-43.555326],[170.52492,-43.031688],[171.12509,-42.512754],[171.569714,-41.767424],[171.948709,-41.514417],[172.097227,-40.956104],[172.79858,-40.493962],[173.020375,-40.919052],[173.247234,-41.331999],[173.958405,-40.926701],[174.247587,-41.349155],[174.248517,-41.770008],[173.876447,-42.233184],[173.22274,-42.970038],[172.711246,-43.372288],[173.080113,-43.853344],[172.308584,-43.865694],[171.452925,-44.242519],[171.185138,-44.897104],[170.616697,-45.908929],[169.831422,-46.355775],[169.332331,-46.641235],[168.411354,-46.619945],[167.763745,-46.290197],[166.676886,-46.219917],[166.509144,-45.852705],[167.046424,-45.110941],[168.303763,-44.123973],[168.949409,-43.935819],[169.667815,-43.555326]]]]}},{"type":"Feature","properties":{"ADMIN":"Australia","NAME_EN":"Australia","CONTINENT":"Oceania","ADM0_A3":"AUS","ISO_A3":"AUS","NAME":"Australia","ISO_A2":"AU","NAME_ZH":"澳大利亚"},"geometry":{"type":"MultiPolygon","coordinates":[[[[147.689259,-40.808258],[148.289068,-40.875438],[148.359865,-42.062445],[148.017301,-42.407024],[147.914052,-43.211522],[147.564564,-42.937689],[146.870343,-43.634597],[146.663327,-43.580854],[146.048378,-43.549745],[145.43193,-42.693776],[145.29509,-42.03361],[144.718071,-41.162552],[144.743755,-40.703975],[145.397978,-40.792549],[146.364121,-41.137695],[146.908584,-41.000546],[147.689259,-40.808258]]],[[[126.148714,-32.215966],[125.088623,-32.728751],[124.221648,-32.959487],[124.028947,-33.483847],[123.659667,-33.890179],[122.811036,-33.914467],[122.183064,-34.003402],[121.299191,-33.821036],[120.580268,-33.930177],[119.893695,-33.976065],[119.298899,-34.509366],[119.007341,-34.464149],[118.505718,-34.746819],[118.024972,-35.064733],[117.295507,-35.025459],[116.625109,-35.025097],[115.564347,-34.386428],[115.026809,-34.196517],[115.048616,-33.623425],[115.545123,-33.487258],[115.714674,-33.259572],[115.679379,-32.900369],[115.801645,-32.205062],[115.689611,-31.612437],[115.160909,-30.601594],[114.997043,-30.030725],[115.040038,-29.461095],[114.641974,-28.810231],[114.616498,-28.516399],[114.173579,-28.118077],[114.048884,-27.334765],[113.477498,-26.543134],[113.338953,-26.116545],[113.778358,-26.549025],[113.440962,-25.621278],[113.936901,-25.911235],[114.232852,-26.298446],[114.216161,-25.786281],[113.721255,-24.998939],[113.625344,-24.683971],[113.393523,-24.384764],[113.502044,-23.80635],[113.706993,-23.560215],[113.843418,-23.059987],[113.736552,-22.475475],[114.149756,-21.755881],[114.225307,-22.517488],[114.647762,-21.82952],[115.460167,-21.495173],[115.947373,-21.068688],[116.711615,-20.701682],[117.166316,-20.623599],[117.441545,-20.746899],[118.229559,-20.374208],[118.836085,-20.263311],[118.987807,-20.044203],[119.252494,-19.952942],[119.805225,-19.976506],[120.85622,-19.683708],[121.399856,-19.239756],[121.655138,-18.705318],[122.241665,-18.197649],[122.286624,-17.798603],[122.312772,-17.254967],[123.012574,-16.4052],[123.433789,-17.268558],[123.859345,-17.069035],[123.503242,-16.596506],[123.817073,-16.111316],[124.258287,-16.327944],[124.379726,-15.56706],[124.926153,-15.0751],[125.167275,-14.680396],[125.670087,-14.51007],[125.685796,-14.230656],[126.125149,-14.347341],[126.142823,-14.095987],[126.582589,-13.952791],[127.065867,-13.817968],[127.804633,-14.276906],[128.35969,-14.86917],[128.985543,-14.875991],[129.621473,-14.969784],[129.4096,-14.42067],[129.888641,-13.618703],[130.339466,-13.357376],[130.183506,-13.10752],[130.617795,-12.536392],[131.223495,-12.183649],[131.735091,-12.302453],[132.575298,-12.114041],[132.557212,-11.603012],[131.824698,-11.273782],[132.357224,-11.128519],[133.019561,-11.376411],[133.550846,-11.786515],[134.393068,-12.042365],[134.678632,-11.941183],[135.298491,-12.248606],[135.882693,-11.962267],[136.258381,-12.049342],[136.492475,-11.857209],[136.95162,-12.351959],[136.685125,-12.887223],[136.305407,-13.29123],[135.961758,-13.324509],[136.077617,-13.724278],[135.783836,-14.223989],[135.428664,-14.715432],[135.500184,-14.997741],[136.295175,-15.550265],[137.06536,-15.870762],[137.580471,-16.215082],[138.303217,-16.807604],[138.585164,-16.806622],[139.108543,-17.062679],[139.260575,-17.371601],[140.215245,-17.710805],[140.875463,-17.369069],[141.07111,-16.832047],[141.274095,-16.38887],[141.398222,-15.840532],[141.702183,-15.044921],[141.56338,-14.561333],[141.63552,-14.270395],[141.519869,-13.698078],[141.65092,-12.944688],[141.842691,-12.741548],[141.68699,-12.407614],[141.928629,-11.877466],[142.118488,-11.328042],[142.143706,-11.042737],[142.51526,-10.668186],[142.79731,-11.157355],[142.866763,-11.784707],[143.115947,-11.90563],[143.158632,-12.325656],[143.522124,-12.834358],[143.597158,-13.400422],[143.561811,-13.763656],[143.922099,-14.548311],[144.563714,-14.171176],[144.894908,-14.594458],[145.374724,-14.984976],[145.271991,-15.428205],[145.48526,-16.285672],[145.637033,-16.784918],[145.888904,-16.906926],[146.160309,-17.761655],[146.063674,-18.280073],[146.387478,-18.958274],[147.471082,-19.480723],[148.177602,-19.955939],[148.848414,-20.39121],[148.717465,-20.633469],[149.28942,-21.260511],[149.678337,-22.342512],[150.077382,-22.122784],[150.482939,-22.556142],[150.727265,-22.402405],[150.899554,-23.462237],[151.609175,-24.076256],[152.07354,-24.457887],[152.855197,-25.267501],[153.136162,-26.071173],[153.161949,-26.641319],[153.092909,-27.2603],[153.569469,-28.110067],[153.512108,-28.995077],[153.339095,-29.458202],[153.069241,-30.35024],[153.089602,-30.923642],[152.891578,-31.640446],[152.450002,-32.550003],[151.709117,-33.041342],[151.343972,-33.816023],[151.010555,-34.31036],[150.714139,-35.17346],[150.32822,-35.671879],[150.075212,-36.420206],[149.946124,-37.109052],[149.997284,-37.425261],[149.423882,-37.772681],[148.304622,-37.809061],[147.381733,-38.219217],[146.922123,-38.606532],[146.317922,-39.035757],[145.489652,-38.593768],[144.876976,-38.417448],[145.032212,-37.896188],[144.485682,-38.085324],[143.609974,-38.809465],[142.745427,-38.538268],[142.17833,-38.380034],[141.606582,-38.308514],[140.638579,-38.019333],[139.992158,-37.402936],[139.806588,-36.643603],[139.574148,-36.138362],[139.082808,-35.732754],[138.120748,-35.612296],[138.449462,-35.127261],[138.207564,-34.384723],[137.71917,-35.076825],[136.829406,-35.260535],[137.352371,-34.707339],[137.503886,-34.130268],[137.890116,-33.640479],[137.810328,-32.900007],[136.996837,-33.752771],[136.372069,-34.094766],[135.989043,-34.890118],[135.208213,-34.47867],[135.239218,-33.947953],[134.613417,-33.222778],[134.085904,-32.848072],[134.273903,-32.617234],[132.990777,-32.011224],[132.288081,-31.982647],[131.326331,-31.495803],[129.535794,-31.590423],[128.240938,-31.948489],[127.102867,-32.282267],[126.148714,-32.215966]]]]}},{"type":"Feature","properties":{"ADMIN":"Sri Lanka","NAME_EN":"Sri Lanka","CONTINENT":"Asia","ADM0_A3":"LKA","ISO_A3":"LKA","NAME":"Sri Lanka","ISO_A2":"LK","NAME_ZH":"斯里兰卡"},"geometry":{"type":"Polygon","coordinates":[[[81.787959,7.523055],[81.637322,6.481775],[81.21802,6.197141],[80.348357,5.96837],[79.872469,6.763463],[79.695167,8.200843],[80.147801,9.824078],[80.838818,9.268427],[81.304319,8.564206],[81.787959,7.523055]]]}},{"type":"Feature","properties":{"ADMIN":"China","NAME_EN":"People's Republic of China","CONTINENT":"Asia","ADM0_A3":"CHN","ISO_A3":"CHN","NAME":"China","ISO_A2":"CN","NAME_ZH":"中国"},"geometry":{"type":"MultiPolygon","coordinates":[[[[109.47521,18.197701],[108.655208,18.507682],[108.626217,19.367888],[109.119056,19.821039],[110.211599,20.101254],[110.786551,20.077534],[111.010051,19.69593],[110.570647,19.255879],[110.339188,18.678395],[109.47521,18.197701]]],[[[80.25999,42.349999],[80.18015,42.920068],[80.866206,43.180362],[79.966106,44.917517],[81.947071,45.317027],[82.458926,45.53965],[83.180484,47.330031],[85.16429,47.000956],[85.720484,47.452969],[85.768233,48.455751],[86.598776,48.549182],[87.35997,49.214981],[87.751264,49.297198],[88.013832,48.599463],[88.854298,48.069082],[90.280826,47.693549],[90.970809,46.888146],[90.585768,45.719716],[90.94554,45.286073],[92.133891,45.115076],[93.480734,44.975472],[94.688929,44.352332],[95.306875,44.241331],[95.762455,43.319449],[96.349396,42.725635],[97.451757,42.74889],[99.515817,42.524691],[100.845866,42.663804],[101.83304,42.514873],[103.312278,41.907468],[104.522282,41.908347],[104.964994,41.59741],[106.129316,42.134328],[107.744773,42.481516],[109.243596,42.519446],[110.412103,42.871234],[111.129682,43.406834],[111.829588,43.743118],[111.667737,44.073176],[111.348377,44.457442],[111.873306,45.102079],[112.436062,45.011646],[113.463907,44.808893],[114.460332,45.339817],[115.985096,45.727235],[116.717868,46.388202],[117.421701,46.672733],[118.874326,46.805412],[119.66327,46.69268],[119.772824,47.048059],[118.866574,47.74706],[118.064143,48.06673],[117.295507,47.697709],[116.308953,47.85341],[115.742837,47.726545],[115.485282,48.135383],[116.191802,49.134598],[116.678801,49.888531],[117.879244,49.510983],[119.288461,50.142883],[119.27939,50.58292],[120.18208,51.64355],[120.7382,51.96411],[120.725789,52.516226],[120.177089,52.753886],[121.003085,53.251401],[122.245748,53.431726],[123.57147,53.4588],[125.068211,53.161045],[125.946349,52.792799],[126.564399,51.784255],[126.939157,51.353894],[127.287456,50.739797],[127.6574,49.76027],[129.397818,49.4406],[130.582293,48.729687],[130.98726,47.79013],[132.50669,47.78896],[133.373596,48.183442],[135.026311,48.47823],[134.50081,47.57845],[134.11235,47.21248],[133.769644,46.116927],[133.09712,45.14409],[131.883454,45.321162],[131.02519,44.96796],[131.288555,44.11152],[131.144688,42.92999],[130.633866,42.903015],[130.64,42.395024],[129.994267,42.985387],[129.596669,42.424982],[128.052215,41.994285],[128.208433,41.466772],[127.343783,41.503152],[126.869083,41.816569],[126.182045,41.107336],[125.079942,40.569824],[124.265625,39.928493],[122.86757,39.637788],[122.131388,39.170452],[121.054554,38.897471],[121.585995,39.360854],[121.376757,39.750261],[122.168595,40.422443],[121.640359,40.94639],[120.768629,40.593388],[119.639602,39.898056],[119.023464,39.252333],[118.042749,39.204274],[117.532702,38.737636],[118.059699,38.061476],[118.87815,37.897325],[118.911636,37.448464],[119.702802,37.156389],[120.823457,37.870428],[121.711259,37.481123],[122.357937,37.454484],[122.519995,36.930614],[121.104164,36.651329],[120.637009,36.11144],[119.664562,35.609791],[119.151208,34.909859],[120.227525,34.360332],[120.620369,33.376723],[121.229014,32.460319],[121.908146,31.692174],[121.891919,30.949352],[121.264257,30.676267],[121.503519,30.142915],[122.092114,29.83252],[121.938428,29.018022],[121.684439,28.225513],[121.125661,28.135673],[120.395473,27.053207],[119.585497,25.740781],[118.656871,24.547391],[117.281606,23.624501],[115.890735,22.782873],[114.763827,22.668074],[114.152547,22.22376],[113.80678,22.54834],[113.241078,22.051367],[111.843592,21.550494],[110.785466,21.397144],[110.444039,20.341033],[109.889861,20.282457],[109.627655,21.008227],[109.864488,21.395051],[108.522813,21.715212],[108.05018,21.55238],[107.04342,21.811899],[106.567273,22.218205],[106.725403,22.794268],[105.811247,22.976892],[105.329209,23.352063],[104.476858,22.81915],[103.504515,22.703757],[102.706992,22.708795],[102.170436,22.464753],[101.652018,22.318199],[101.80312,21.174367],[101.270026,21.201652],[101.180005,21.436573],[101.150033,21.849984],[100.416538,21.558839],[99.983489,21.742937],[99.240899,22.118314],[99.531992,22.949039],[98.898749,23.142722],[98.660262,24.063286],[97.60472,23.897405],[97.724609,25.083637],[98.671838,25.918703],[98.712094,26.743536],[98.68269,27.508812],[98.246231,27.747221],[97.911988,28.335945],[97.327114,28.261583],[96.248833,28.411031],[96.586591,28.83098],[96.117679,29.452802],[95.404802,29.031717],[94.56599,29.277438],[93.413348,28.640629],[92.503119,27.896876],[91.696657,27.771742],[91.258854,28.040614],[90.730514,28.064954],[90.015829,28.296439],[89.47581,28.042759],[88.814248,27.299316],[88.730326,28.086865],[88.120441,27.876542],[86.954517,27.974262],[85.82332,28.203576],[85.011638,28.642774],[84.23458,28.839894],[83.898993,29.320226],[83.337115,29.463732],[82.327513,30.115268],[81.525804,30.422717],[81.111256,30.183481],[79.721367,30.882715],[78.738894,31.515906],[78.458446,32.618164],[79.176129,32.48378],[79.208892,32.994395],[78.811086,33.506198],[78.912269,34.321936],[77.837451,35.49401],[76.192848,35.898403],[75.896897,36.666806],[75.158028,37.133031],[74.980002,37.41999],[74.829986,37.990007],[74.864816,38.378846],[74.257514,38.606507],[73.928852,38.505815],[73.675379,39.431237],[73.960013,39.660008],[73.822244,39.893973],[74.776862,40.366425],[75.467828,40.562072],[76.526368,40.427946],[76.904484,41.066486],[78.187197,41.185316],[78.543661,41.582243],[80.11943,42.123941],[80.25999,42.349999]]]]}},{"type":"Feature","properties":{"ADMIN":"Taiwan, China","NAME_EN":"Taiwan, China","CONTINENT":"Asia","ADM0_A3":"TWN","ISO_A3":"TWN","NAME":"Taiwan, China","ISO_A2":"TW","NAME_ZH":"中国（台湾）"},"geometry":{"type":"Polygon","coordinates":[[[121.777818,24.394274],[121.175632,22.790857],[120.74708,21.970571],[120.220083,22.814861],[120.106189,23.556263],[120.69468,24.538451],[121.495044,25.295459],[121.951244,24.997596],[121.777818,24.394274]]]}},{"type":"Feature","properties":{"ADMIN":"Italy","NAME_EN":"Italy","CONTINENT":"Europe","ADM0_A3":"ITA","ISO_A3":"ITA","NAME":"Italy","ISO_A2":"IT","NAME_ZH":"意大利"},"geometry":{"type":"MultiPolygon","coordinates":[[[[10.442701,46.893546],[11.048556,46.751359],[11.164828,46.941579],[12.153088,47.115393],[12.376485,46.767559],[13.806475,46.509306],[13.69811,46.016778],[13.93763,45.591016],[13.141606,45.736692],[12.328581,45.381778],[12.383875,44.885374],[12.261453,44.600482],[12.589237,44.091366],[13.526906,43.587727],[14.029821,42.761008],[15.14257,41.95514],[15.926191,41.961315],[16.169897,41.740295],[15.889346,41.541082],[16.785002,41.179606],[17.519169,40.877143],[18.376687,40.355625],[18.480247,40.168866],[18.293385,39.810774],[17.73838,40.277671],[16.869596,40.442235],[16.448743,39.795401],[17.17149,39.4247],[17.052841,38.902871],[16.635088,38.843572],[16.100961,37.985899],[15.684087,37.908849],[15.687963,38.214593],[15.891981,38.750942],[16.109332,38.964547],[15.718814,39.544072],[15.413613,40.048357],[14.998496,40.172949],[14.703268,40.60455],[14.060672,40.786348],[13.627985,41.188287],[12.888082,41.25309],[12.106683,41.704535],[11.191906,42.355425],[10.511948,42.931463],[10.200029,43.920007],[9.702488,44.036279],[8.888946,44.366336],[8.428561,44.231228],[7.850767,43.767148],[7.435185,43.693845],[7.549596,44.127901],[7.007562,44.254767],[6.749955,45.028518],[7.096652,45.333099],[6.802355,45.70858],[6.843593,45.991147],[7.273851,45.776948],[7.755992,45.82449],[8.31663,46.163642],[8.489952,46.005151],[8.966306,46.036932],[9.182882,46.440215],[9.922837,46.314899],[10.363378,46.483571],[10.442701,46.893546]]],[[[14.761249,38.143874],[15.520376,38.231155],[15.160243,37.444046],[15.309898,37.134219],[15.099988,36.619987],[14.335229,36.996631],[13.826733,37.104531],[12.431004,37.61295],[12.570944,38.126381],[13.741156,38.034966],[14.761249,38.143874]]],[[[8.709991,40.899984],[9.210012,41.209991],[9.809975,40.500009],[9.669519,39.177376],[9.214818,39.240473],[8.806936,38.906618],[8.428302,39.171847],[8.388253,40.378311],[8.159998,40.950007],[8.709991,40.899984]]]]}},{"type":"Feature","properties":{"ADMIN":"Denmark","NAME_EN":"Denmark","CONTINENT":"Europe","ADM0_A3":"DNK","ISO_A3":"DNK","NAME":"Denmark","ISO_A2":"DK","NAME_ZH":"丹麦"},"geometry":{"type":"MultiPolygon","coordinates":[[[[9.921906,54.983104],[9.282049,54.830865],[8.526229,54.962744],[8.120311,55.517723],[8.089977,56.540012],[8.256582,56.809969],[8.543438,57.110003],[9.424469,57.172066],[9.775559,57.447941],[10.580006,57.730017],[10.546106,57.215733],[10.25,56.890016],[10.369993,56.609982],[10.912182,56.458621],[10.667804,56.081383],[10.369993,56.190007],[9.649985,55.469999],[9.921906,54.983104]]],[[[12.370904,56.111407],[12.690006,55.609991],[12.089991,54.800015],[11.043543,55.364864],[10.903914,55.779955],[12.370904,56.111407]]]]}},{"type":"Feature","properties":{"ADMIN":"United Kingdom","NAME_EN":"United Kingdom","CONTINENT":"Europe","ADM0_A3":"GBR","ISO_A3":"GBR","NAME":"United Kingdom","ISO_A2":"GB","NAME_ZH":"英国"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-6.197885,53.867565],[-6.95373,54.073702],[-7.572168,54.059956],[-7.366031,54.595841],[-7.572168,55.131622],[-6.733847,55.17286],[-5.661949,54.554603],[-6.197885,53.867565]]],[[[-3.093831,53.404547],[-3.09208,53.404441],[-2.945009,53.985],[-3.614701,54.600937],[-3.630005,54.615013],[-4.844169,54.790971],[-5.082527,55.061601],[-4.719112,55.508473],[-5.047981,55.783986],[-5.586398,55.311146],[-5.644999,56.275015],[-6.149981,56.78501],[-5.786825,57.818848],[-5.009999,58.630013],[-4.211495,58.550845],[-3.005005,58.635],[-4.073828,57.553025],[-3.055002,57.690019],[-1.959281,57.6848],[-2.219988,56.870017],[-3.119003,55.973793],[-2.085009,55.909998],[-2.005676,55.804903],[-1.114991,54.624986],[-0.430485,54.464376],[0.184981,53.325014],[0.469977,52.929999],[1.681531,52.73952],[1.559988,52.099998],[1.050562,51.806761],[1.449865,51.289428],[0.550334,50.765739],[-0.787517,50.774989],[-2.489998,50.500019],[-2.956274,50.69688],[-3.617448,50.228356],[-4.542508,50.341837],[-5.245023,49.96],[-5.776567,50.159678],[-4.30999,51.210001],[-3.414851,51.426009],[-3.422719,51.426848],[-4.984367,51.593466],[-5.267296,51.9914],[-4.222347,52.301356],[-4.770013,52.840005],[-4.579999,53.495004],[-3.093831,53.404547]]]]}},{"type":"Feature","properties":{"ADMIN":"Iceland","NAME_EN":"Iceland","CONTINENT":"Europe","ADM0_A3":"ISL","ISO_A3":"ISL","NAME":"Iceland","ISO_A2":"IS","NAME_ZH":"冰岛"},"geometry":{"type":"Polygon","coordinates":[[[-14.508695,66.455892],[-14.739637,65.808748],[-13.609732,65.126671],[-14.909834,64.364082],[-17.794438,63.678749],[-18.656246,63.496383],[-19.972755,63.643635],[-22.762972,63.960179],[-21.778484,64.402116],[-23.955044,64.89113],[-22.184403,65.084968],[-22.227423,65.378594],[-24.326184,65.611189],[-23.650515,66.262519],[-22.134922,66.410469],[-20.576284,65.732112],[-19.056842,66.276601],[-17.798624,65.993853],[-16.167819,66.526792],[-14.508695,66.455892]]]}},{"type":"Feature","properties":{"ADMIN":"Azerbaijan","NAME_EN":"Azerbaijan","CONTINENT":"Asia","ADM0_A3":"AZE","ISO_A3":"AZE","NAME":"Azerbaijan","ISO_A2":"AZ","NAME_ZH":"阿塞拜疆"},"geometry":{"type":"MultiPolygon","coordinates":[[[[46.404951,41.860675],[46.686071,41.827137],[47.373315,41.219732],[47.815666,41.151416],[47.987283,41.405819],[48.584353,41.808869],[49.110264,41.282287],[49.618915,40.572924],[50.08483,40.526157],[50.392821,40.256561],[49.569202,40.176101],[49.395259,39.399482],[49.223228,39.049219],[48.856532,38.815486],[48.883249,38.320245],[48.634375,38.270378],[48.010744,38.794015],[48.355529,39.288765],[48.060095,39.582235],[47.685079,39.508364],[46.50572,38.770605],[46.483499,39.464155],[46.034534,39.628021],[45.610012,39.899994],[45.891907,40.218476],[45.359175,40.561504],[45.560351,40.81229],[45.179496,40.985354],[44.97248,41.248129],[45.217426,41.411452],[45.962601,41.123873],[46.501637,41.064445],[46.637908,41.181673],[46.145432,41.722802],[46.404951,41.860675]]],[[[46.143623,38.741201],[45.457722,38.874139],[44.952688,39.335765],[44.79399,39.713003],[45.001987,39.740004],[45.298145,39.471751],[45.739978,39.473999],[45.735379,39.319719],[46.143623,38.741201]]]]}},{"type":"Feature","properties":{"ADMIN":"Georgia","NAME_EN":"Georgia","CONTINENT":"Asia","ADM0_A3":"GEO","ISO_A3":"GEO","NAME":"Georgia","ISO_A2":"GE","NAME_ZH":"格鲁吉亚"},"geometry":{"type":"Polygon","coordinates":[[[39.955009,43.434998],[40.076965,43.553104],[40.92219,43.38215],[42.3944,43.2203],[43.75599,42.74083],[43.93121,42.55496],[44.537623,42.711993],[45.470279,42.502781],[45.7764,42.09244],[46.404951,41.860675],[46.145432,41.722802],[46.637908,41.181673],[46.501637,41.064445],[45.962601,41.123873],[45.217426,41.411452],[44.97248,41.248129],[43.582746,41.092143],[42.619549,41.583173],[41.554084,41.535656],[41.703171,41.962943],[41.45347,42.645123],[40.875469,43.013628],[40.321394,43.128634],[39.955009,43.434998]]]}},{"type":"Feature","properties":{"ADMIN":"Philippines","NAME_EN":"Philippines","CONTINENT":"Asia","ADM0_A3":"PHL","ISO_A3":"PHL","NAME":"Philippines","ISO_A2":"PH","NAME_ZH":"菲律宾"},"geometry":{"type":"MultiPolygon","coordinates":[[[[120.833896,12.704496],[120.323436,13.466413],[121.180128,13.429697],[121.527394,13.06959],[121.26219,12.20556],[120.833896,12.704496]]],[[[122.586089,9.981045],[122.837081,10.261157],[122.947411,10.881868],[123.49885,10.940624],[123.337774,10.267384],[124.077936,11.232726],[123.982438,10.278779],[123.623183,9.950091],[123.309921,9.318269],[122.995883,9.022189],[122.380055,9.713361],[122.586089,9.981045]]],[[[126.376814,8.414706],[126.478513,7.750354],[126.537424,7.189381],[126.196773,6.274294],[125.831421,7.293715],[125.363852,6.786485],[125.683161,6.049657],[125.396512,5.581003],[124.219788,6.161355],[123.93872,6.885136],[124.243662,7.36061],[123.610212,7.833527],[123.296071,7.418876],[122.825506,7.457375],[122.085499,6.899424],[121.919928,7.192119],[122.312359,8.034962],[122.942398,8.316237],[123.487688,8.69301],[123.841154,8.240324],[124.60147,8.514158],[124.764612,8.960409],[125.471391,8.986997],[125.412118,9.760335],[126.222714,9.286074],[126.306637,8.782487],[126.376814,8.414706]]],[[[118.504581,9.316383],[117.174275,8.3675],[117.664477,9.066889],[118.386914,9.6845],[118.987342,10.376292],[119.511496,11.369668],[119.689677,10.554291],[119.029458,10.003653],[118.504581,9.316383]]],[[[122.336957,18.224883],[122.174279,17.810283],[122.515654,17.093505],[122.252311,16.262444],[121.662786,15.931018],[121.50507,15.124814],[121.728829,14.328376],[122.258925,14.218202],[122.701276,14.336541],[123.950295,13.782131],[123.855107,13.237771],[124.181289,12.997527],[124.077419,12.536677],[123.298035,13.027526],[122.928652,13.55292],[122.671355,13.185836],[122.03465,13.784482],[121.126385,13.636687],[120.628637,13.857656],[120.679384,14.271016],[120.991819,14.525393],[120.693336,14.756671],[120.564145,14.396279],[120.070429,14.970869],[119.920929,15.406347],[119.883773,16.363704],[120.286488,16.034629],[120.390047,17.599081],[120.715867,18.505227],[121.321308,18.504065],[121.937601,18.218552],[122.246006,18.47895],[122.336957,18.224883]]],[[[122.03837,11.415841],[121.883548,11.891755],[122.483821,11.582187],[123.120217,11.58366],[123.100838,11.165934],[122.637714,10.741308],[122.00261,10.441017],[121.967367,10.905691],[122.03837,11.415841]]],[[[125.502552,12.162695],[125.783465,11.046122],[125.011884,11.311455],[125.032761,10.975816],[125.277449,10.358722],[124.801819,10.134679],[124.760168,10.837995],[124.459101,10.88993],[124.302522,11.495371],[124.891013,11.415583],[124.87799,11.79419],[124.266762,12.557761],[125.227116,12.535721],[125.502552,12.162695]]]]}},{"type":"Feature","properties":{"ADMIN":"Malaysia","NAME_EN":"Malaysia","CONTINENT":"Asia","ADM0_A3":"MYS","ISO_A3":"MYS","NAME":"Malaysia","ISO_A2":"MY","NAME_ZH":"马来西亚"},"geometry":{"type":"MultiPolygon","coordinates":[[[[100.085757,6.464489],[100.259596,6.642825],[101.075516,6.204867],[101.154219,5.691384],[101.814282,5.810808],[102.141187,6.221636],[102.371147,6.128205],[102.961705,5.524495],[103.381215,4.855001],[103.438575,4.181606],[103.332122,3.726698],[103.429429,3.382869],[103.502448,2.791019],[103.854674,2.515454],[104.247932,1.631141],[104.228811,1.293048],[103.519707,1.226334],[102.573615,1.967115],[101.390638,2.760814],[101.27354,3.270292],[100.695435,3.93914],[100.557408,4.76728],[100.196706,5.312493],[100.30626,6.040562],[100.085757,6.464489]]],[[[117.882035,4.137551],[117.015214,4.306094],[115.865517,4.306559],[115.519078,3.169238],[115.134037,2.821482],[114.621355,1.430688],[113.80585,1.217549],[112.859809,1.49779],[112.380252,1.410121],[111.797548,0.904441],[111.159138,0.976478],[110.514061,0.773131],[109.830227,1.338136],[109.66326,2.006467],[110.396135,1.663775],[111.168853,1.850637],[111.370081,2.697303],[111.796928,2.885897],[112.995615,3.102395],[113.712935,3.893509],[114.204017,4.525874],[114.659596,4.007637],[114.869557,4.348314],[115.347461,4.316636],[115.4057,4.955228],[115.45071,5.44773],[116.220741,6.143191],[116.725103,6.924771],[117.129626,6.928053],[117.643393,6.422166],[117.689075,5.98749],[118.347691,5.708696],[119.181904,5.407836],[119.110694,5.016128],[118.439727,4.966519],[118.618321,4.478202],[117.882035,4.137551]]]]}},{"type":"Feature","properties":{"ADMIN":"Brunei","NAME_EN":"Brunei","CONTINENT":"Asia","ADM0_A3":"BRN","ISO_A3":"BRN","NAME":"Brunei","ISO_A2":"BN","NAME_ZH":"文莱"},"geometry":{"type":"Polygon","coordinates":[[[115.45071,5.44773],[115.4057,4.955228],[115.347461,4.316636],[114.869557,4.348314],[114.659596,4.007637],[114.204017,4.525874],[114.599961,4.900011],[115.45071,5.44773]]]}},{"type":"Feature","properties":{"ADMIN":"Slovenia","NAME_EN":"Slovenia","CONTINENT":"Europe","ADM0_A3":"SVN","ISO_A3":"SVN","NAME":"Slovenia","ISO_A2":"SI","NAME_ZH":"斯洛文尼亚"},"geometry":{"type":"Polygon","coordinates":[[[13.806475,46.509306],[14.632472,46.431817],[15.137092,46.658703],[16.011664,46.683611],[16.202298,46.852386],[16.370505,46.841327],[16.564808,46.503751],[15.768733,46.238108],[15.67153,45.834154],[15.323954,45.731783],[15.327675,45.452316],[14.935244,45.471695],[14.595109,45.634941],[14.411968,45.466166],[13.71506,45.500324],[13.93763,45.591016],[13.69811,46.016778],[13.806475,46.509306]]]}},{"type":"Feature","properties":{"ADMIN":"Finland","NAME_EN":"Finland","CONTINENT":"Europe","ADM0_A3":"FIN","ISO_A3":"FIN","NAME":"Finland","ISO_A2":"FI","NAME_ZH":"芬兰"},"geometry":{"type":"Polygon","coordinates":[[[28.59193,69.064777],[28.445944,68.364613],[29.977426,67.698297],[29.054589,66.944286],[30.21765,65.80598],[29.54443,64.948672],[30.444685,64.204453],[30.035872,63.552814],[31.516092,62.867687],[31.139991,62.357693],[30.211107,61.780028],[28.07,60.50352],[28.070002,60.503519],[28.069998,60.503517],[26.255173,60.423961],[24.496624,60.057316],[22.869695,59.846373],[22.290764,60.391921],[21.322244,60.72017],[21.544866,61.705329],[21.059211,62.607393],[21.536029,63.189735],[22.442744,63.81781],[24.730512,64.902344],[25.398068,65.111427],[25.294043,65.534346],[23.903379,66.006927],[23.56588,66.396051],[23.539473,67.936009],[21.978535,68.616846],[20.645593,69.106247],[21.244936,69.370443],[22.356238,68.841741],[23.66205,68.891247],[24.735679,68.649557],[25.689213,69.092114],[26.179622,69.825299],[27.732292,70.164193],[29.015573,69.766491],[28.59193,69.064777]]]}},{"type":"Feature","properties":{"ADMIN":"Slovakia","NAME_EN":"Slovakia","CONTINENT":"Europe","ADM0_A3":"SVK","ISO_A3":"SVK","NAME":"Slovakia","ISO_A2":"SK","NAME_ZH":"斯洛伐克"},"geometry":{"type":"Polygon","coordinates":[[[22.558138,49.085738],[22.280842,48.825392],[22.085608,48.422264],[21.872236,48.319971],[20.801294,48.623854],[20.473562,48.56285],[20.239054,48.327567],[19.769471,48.202691],[19.661364,48.266615],[19.174365,48.111379],[18.777025,48.081768],[18.696513,47.880954],[17.857133,47.758429],[17.488473,47.867466],[16.979667,48.123497],[16.879983,48.470013],[16.960288,48.596982],[17.101985,48.816969],[17.545007,48.800019],[17.886485,48.903475],[17.913512,48.996493],[18.104973,49.043983],[18.170498,49.271515],[18.399994,49.315001],[18.554971,49.495015],[18.853144,49.49623],[18.909575,49.435846],[19.320713,49.571574],[19.825023,49.217125],[20.415839,49.431453],[20.887955,49.328772],[21.607808,49.470107],[22.558138,49.085738]]]}},{"type":"Feature","properties":{"ADMIN":"Czechia","NAME_EN":"Czech Republic","CONTINENT":"Europe","ADM0_A3":"CZE","ISO_A3":"CZE","NAME":"Czechia","ISO_A2":"CZ","NAME_ZH":"捷克"},"geometry":{"type":"Polygon","coordinates":[[[15.016996,51.106674],[15.490972,50.78473],[16.238627,50.697733],[16.176253,50.422607],[16.719476,50.215747],[16.868769,50.473974],[17.554567,50.362146],[17.649445,50.049038],[18.392914,49.988629],[18.853144,49.49623],[18.554971,49.495015],[18.399994,49.315001],[18.170498,49.271515],[18.104973,49.043983],[17.913512,48.996493],[17.886485,48.903475],[17.545007,48.800019],[17.101985,48.816969],[16.960288,48.596982],[16.499283,48.785808],[16.029647,48.733899],[15.253416,49.039074],[14.901447,48.964402],[14.338898,48.555305],[13.595946,48.877172],[13.031329,49.307068],[12.521024,49.547415],[12.415191,49.969121],[12.240111,50.266338],[12.966837,50.484076],[13.338132,50.733234],[14.056228,50.926918],[14.307013,51.117268],[14.570718,51.002339],[15.016996,51.106674]]]}},{"type":"Feature","properties":{"ADMIN":"Eritrea","NAME_EN":"Eritrea","CONTINENT":"Africa","ADM0_A3":"ERI","ISO_A3":"ERI","NAME":"Eritrea","ISO_A2":"ER","NAME_ZH":"厄立特里亚"},"geometry":{"type":"Polygon","coordinates":[[[36.42951,14.42211],[36.32322,14.82249],[36.75389,16.29186],[36.85253,16.95655],[37.16747,17.26314],[37.904,17.42754],[38.41009,17.998307],[38.990623,16.840626],[39.26611,15.922723],[39.814294,15.435647],[41.179275,14.49108],[41.734952,13.921037],[42.276831,13.343992],[42.589576,13.000421],[43.081226,12.699639],[42.779642,12.455416],[42.35156,12.54223],[42.00975,12.86582],[41.59856,13.45209],[41.1552,13.77333],[40.8966,14.11864],[40.02625,14.51959],[39.34061,14.53155],[39.0994,14.74064],[38.51295,14.50547],[37.90607,14.95943],[37.59377,14.2131],[36.42951,14.42211]]]}},{"type":"Feature","properties":{"ADMIN":"Japan","NAME_EN":"Japan","CONTINENT":"Asia","ADM0_A3":"JPN","ISO_A3":"JPN","NAME":"Japan","ISO_A2":"JP","NAME_ZH":"日本"},"geometry":{"type":"MultiPolygon","coordinates":[[[[141.884601,39.180865],[140.959489,38.174001],[140.976388,37.142074],[140.59977,36.343983],[140.774074,35.842877],[140.253279,35.138114],[138.975528,34.6676],[137.217599,34.606286],[135.792983,33.464805],[135.120983,33.849071],[135.079435,34.596545],[133.340316,34.375938],[132.156771,33.904933],[130.986145,33.885761],[132.000036,33.149992],[131.33279,31.450355],[130.686318,31.029579],[130.20242,31.418238],[130.447676,32.319475],[129.814692,32.61031],[129.408463,33.296056],[130.353935,33.604151],[130.878451,34.232743],[131.884229,34.749714],[132.617673,35.433393],[134.608301,35.731618],[135.677538,35.527134],[136.723831,37.304984],[137.390612,36.827391],[138.857602,37.827485],[139.426405,38.215962],[140.05479,39.438807],[139.883379,40.563312],[140.305783,41.195005],[141.368973,41.37856],[141.914263,39.991616],[141.884601,39.180865]]],[[[144.613427,43.960883],[145.320825,44.384733],[145.543137,43.262088],[144.059662,42.988358],[143.18385,41.995215],[141.611491,42.678791],[141.067286,41.584594],[139.955106,41.569556],[139.817544,42.563759],[140.312087,43.333273],[141.380549,43.388825],[141.671952,44.772125],[141.967645,45.551483],[143.14287,44.510358],[143.910162,44.1741],[144.613427,43.960883]]],[[[132.371176,33.463642],[132.924373,34.060299],[133.492968,33.944621],[133.904106,34.364931],[134.638428,34.149234],[134.766379,33.806335],[134.203416,33.201178],[133.79295,33.521985],[133.280268,33.28957],[133.014858,32.704567],[132.363115,32.989382],[132.371176,33.463642]]]]}},{"type":"Feature","properties":{"ADMIN":"Paraguay","NAME_EN":"Paraguay","CONTINENT":"South America","ADM0_A3":"PRY","ISO_A3":"PRY","NAME":"Paraguay","ISO_A2":"PY","NAME_ZH":"巴拉圭"},"geometry":{"type":"Polygon","coordinates":[[[-58.166392,-20.176701],[-57.870674,-20.732688],[-57.937156,-22.090176],[-56.88151,-22.282154],[-56.473317,-22.0863],[-55.797958,-22.35693],[-55.610683,-22.655619],[-55.517639,-23.571998],[-55.400747,-23.956935],[-55.027902,-24.001274],[-54.652834,-23.839578],[-54.29296,-24.021014],[-54.293476,-24.5708],[-54.428946,-25.162185],[-54.625291,-25.739255],[-54.788795,-26.621786],[-55.695846,-27.387837],[-56.486702,-27.548499],[-57.60976,-27.395899],[-58.618174,-27.123719],[-57.63366,-25.603657],[-57.777217,-25.16234],[-58.807128,-24.771459],[-60.028966,-24.032796],[-60.846565,-23.880713],[-62.685057,-22.249029],[-62.291179,-21.051635],[-62.265961,-20.513735],[-61.786326,-19.633737],[-60.043565,-19.342747],[-59.115042,-19.356906],[-58.183471,-19.868399],[-58.166392,-20.176701]]]}},{"type":"Feature","properties":{"ADMIN":"Yemen","NAME_EN":"Yemen","CONTINENT":"Asia","ADM0_A3":"YEM","ISO_A3":"YEM","NAME":"Yemen","ISO_A2":"YE","NAME_ZH":"也门"},"geometry":{"type":"Polygon","coordinates":[[[52.00001,19.000003],[52.782184,17.349742],[53.108573,16.651051],[52.385206,16.382411],[52.191729,15.938433],[52.168165,15.59742],[51.172515,15.17525],[49.574576,14.708767],[48.679231,14.003202],[48.238947,13.94809],[47.938914,14.007233],[47.354454,13.59222],[46.717076,13.399699],[45.877593,13.347764],[45.62505,13.290946],[45.406459,13.026905],[45.144356,12.953938],[44.989533,12.699587],[44.494576,12.721653],[44.175113,12.58595],[43.482959,12.6368],[43.222871,13.22095],[43.251448,13.767584],[43.087944,14.06263],[42.892245,14.802249],[42.604873,15.213335],[42.805015,15.261963],[42.702438,15.718886],[42.823671,15.911742],[42.779332,16.347891],[43.218375,16.66689],[43.115798,17.08844],[43.380794,17.579987],[43.791519,17.319977],[44.062613,17.410359],[45.216651,17.433329],[45.399999,17.333335],[46.366659,17.233315],[46.749994,17.283338],[47.000005,16.949999],[47.466695,17.116682],[48.183344,18.166669],[49.116672,18.616668],[52.00001,19.000003]]]}},{"type":"Feature","properties":{"ADMIN":"Saudi Arabia","NAME_EN":"Saudi Arabia","CONTINENT":"Asia","ADM0_A3":"SAU","ISO_A3":"SAU","NAME":"Saudi Arabia","ISO_A2":"SA","NAME_ZH":"沙特阿拉伯"},"geometry":{"type":"Polygon","coordinates":[[[34.956037,29.356555],[36.068941,29.197495],[36.501214,29.505254],[36.740528,29.865283],[37.503582,30.003776],[37.66812,30.338665],[37.998849,30.5085],[37.002166,31.508413],[39.004886,32.010217],[39.195468,32.161009],[40.399994,31.889992],[41.889981,31.190009],[44.709499,29.178891],[46.568713,29.099025],[47.459822,29.002519],[47.708851,28.526063],[48.416094,28.552004],[48.807595,27.689628],[49.299554,27.461218],[49.470914,27.109999],[50.152422,26.689663],[50.212935,26.277027],[50.113303,25.943972],[50.239859,25.60805],[50.527387,25.327808],[50.660557,24.999896],[50.810108,24.754743],[51.112415,24.556331],[51.389608,24.627386],[51.579519,24.245497],[51.617708,24.014219],[52.000733,23.001154],[55.006803,22.496948],[55.208341,22.70833],[55.666659,22.000001],[54.999982,19.999994],[52.00001,19.000003],[49.116672,18.616668],[48.183344,18.166669],[47.466695,17.116682],[47.000005,16.949999],[46.749994,17.283338],[46.366659,17.233315],[45.399999,17.333335],[45.216651,17.433329],[44.062613,17.410359],[43.791519,17.319977],[43.380794,17.579987],[43.115798,17.08844],[43.218375,16.66689],[42.779332,16.347891],[42.649573,16.774635],[42.347989,17.075806],[42.270888,17.474722],[41.754382,17.833046],[41.221391,18.6716],[40.939341,19.486485],[40.247652,20.174635],[39.801685,20.338862],[39.139399,21.291905],[39.023696,21.986875],[39.066329,22.579656],[38.492772,23.688451],[38.02386,24.078686],[37.483635,24.285495],[37.154818,24.858483],[37.209491,25.084542],[36.931627,25.602959],[36.639604,25.826228],[36.249137,26.570136],[35.640182,27.37652],[35.130187,28.063352],[34.632336,28.058546],[34.787779,28.607427],[34.83222,28.957483],[34.956037,29.356555]]]}},{"type":"Feature","properties":{"ADMIN":"Antarctica","NAME_EN":"Antarctica","CONTINENT":"Antarctica","ADM0_A3":"ATA","ISO_A3":"ATA","NAME":"Antarctica","ISO_A2":"AQ","NAME_ZH":"南极洲"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-48.660616,-78.047019],[-48.151396,-78.04707],[-46.662857,-77.831476],[-45.154758,-78.04707],[-43.920828,-78.478103],[-43.48995,-79.08556],[-43.372438,-79.516645],[-43.333267,-80.026123],[-44.880537,-80.339644],[-46.506174,-80.594357],[-48.386421,-80.829485],[-50.482107,-81.025442],[-52.851988,-80.966685],[-54.164259,-80.633528],[-53.987991,-80.222028],[-51.853134,-79.94773],[-50.991326,-79.614623],[-50.364595,-79.183487],[-49.914131,-78.811209],[-49.306959,-78.458569],[-48.660616,-78.047018],[-48.660616,-78.047019]]],[[[-66.290031,-80.255773],[-64.037688,-80.294944],[-61.883246,-80.39287],[-61.138976,-79.981371],[-60.610119,-79.628679],[-59.572095,-80.040179],[-59.865849,-80.549657],[-60.159656,-81.000327],[-62.255393,-80.863178],[-64.488125,-80.921934],[-65.741666,-80.588827],[-65.741666,-80.549657],[-66.290031,-80.255773]]],[[[-73.915819,-71.269345],[-73.915819,-71.269344],[-73.230331,-71.15178],[-72.074717,-71.190951],[-71.780962,-70.681473],[-71.72218,-70.309196],[-71.741791,-69.505782],[-71.173815,-69.035475],[-70.253252,-68.87874],[-69.724447,-69.251017],[-69.489422,-69.623346],[-69.058518,-70.074016],[-68.725541,-70.505153],[-68.451346,-70.955823],[-68.333834,-71.406493],[-68.510128,-71.798407],[-68.784297,-72.170736],[-69.959471,-72.307885],[-71.075889,-72.503842],[-72.388134,-72.484257],[-71.8985,-72.092343],[-73.073622,-72.229492],[-74.19004,-72.366693],[-74.953895,-72.072757],[-75.012625,-71.661258],[-73.915819,-71.269345]]],[[[-102.330725,-71.894164],[-102.330725,-71.894164],[-101.703967,-71.717792],[-100.430919,-71.854993],[-98.98155,-71.933334],[-97.884743,-72.070535],[-96.787937,-71.952971],[-96.20035,-72.521205],[-96.983765,-72.442864],[-98.198083,-72.482035],[-99.432013,-72.442864],[-100.783455,-72.50162],[-101.801868,-72.305663],[-102.330725,-71.894164]]],[[[-122.621735,-73.657778],[-122.621735,-73.657777],[-122.406245,-73.324619],[-121.211511,-73.50099],[-119.918851,-73.657725],[-118.724143,-73.481353],[-119.292119,-73.834097],[-120.232217,-74.08881],[-121.62283,-74.010468],[-122.621735,-73.657778]]],[[[-127.28313,-73.461769],[-127.28313,-73.461768],[-126.558472,-73.246226],[-125.559566,-73.481353],[-124.031882,-73.873268],[-124.619469,-73.834097],[-125.912181,-73.736118],[-127.28313,-73.461769]]],[[[-163.712896,-78.595667],[-163.712896,-78.595667],[-163.105801,-78.223338],[-161.245113,-78.380176],[-160.246208,-78.693645],[-159.482405,-79.046338],[-159.208184,-79.497059],[-161.127601,-79.634209],[-162.439847,-79.281465],[-163.027408,-78.928774],[-163.066604,-78.869966],[-163.712896,-78.595667]]],[[[180,-84.71338],[180,-90],[-180,-90],[-180,-84.71338],[-179.942499,-84.721443],[-179.058677,-84.139412],[-177.256772,-84.452933],[-177.140807,-84.417941],[-176.084673,-84.099259],[-175.947235,-84.110449],[-175.829882,-84.117914],[-174.382503,-84.534323],[-173.116559,-84.117914],[-172.889106,-84.061019],[-169.951223,-83.884647],[-168.999989,-84.117914],[-168.530199,-84.23739],[-167.022099,-84.570497],[-164.182144,-84.82521],[-161.929775,-85.138731],[-158.07138,-85.37391],[-155.192253,-85.09956],[-150.942099,-85.295517],[-148.533073,-85.609038],[-145.888918,-85.315102],[-143.107718,-85.040752],[-142.892279,-84.570497],[-146.829068,-84.531274],[-150.060732,-84.296146],[-150.902928,-83.904232],[-153.586201,-83.68869],[-153.409907,-83.23802],[-153.037759,-82.82652],[-152.665637,-82.454192],[-152.861517,-82.042692],[-154.526299,-81.768394],[-155.29018,-81.41565],[-156.83745,-81.102129],[-154.408787,-81.160937],[-152.097662,-81.004151],[-150.648293,-81.337309],[-148.865998,-81.043373],[-147.22075,-80.671045],[-146.417749,-80.337938],[-146.770286,-79.926439],[-148.062947,-79.652089],[-149.531901,-79.358205],[-151.588416,-79.299397],[-153.390322,-79.162248],[-155.329376,-79.064269],[-155.975668,-78.69194],[-157.268302,-78.378419],[-158.051768,-78.025676],[-158.365134,-76.889207],[-157.875474,-76.987238],[-156.974573,-77.300759],[-155.329376,-77.202728],[-153.742832,-77.065579],[-152.920247,-77.496664],[-151.33378,-77.398737],[-150.00195,-77.183143],[-148.748486,-76.908845],[-147.612483,-76.575738],[-146.104409,-76.47776],[-146.143528,-76.105431],[-146.496091,-75.733154],[-146.20231,-75.380411],[-144.909624,-75.204039],[-144.322037,-75.537197],[-142.794353,-75.34124],[-141.638764,-75.086475],[-140.209007,-75.06689],[-138.85759,-74.968911],[-137.5062,-74.733783],[-136.428901,-74.518241],[-135.214583,-74.302699],[-134.431194,-74.361455],[-133.745654,-74.439848],[-132.257168,-74.302699],[-130.925311,-74.479019],[-129.554284,-74.459433],[-128.242038,-74.322284],[-126.890622,-74.420263],[-125.402082,-74.518241],[-124.011496,-74.479019],[-122.562152,-74.498604],[-121.073613,-74.518241],[-119.70256,-74.479019],[-118.684145,-74.185083],[-117.469801,-74.028348],[-116.216312,-74.243891],[-115.021552,-74.067519],[-113.944331,-73.714828],[-113.297988,-74.028348],[-112.945452,-74.38104],[-112.299083,-74.714198],[-111.261059,-74.420263],[-110.066325,-74.79254],[-108.714909,-74.910103],[-107.559346,-75.184454],[-106.149148,-75.125698],[-104.876074,-74.949326],[-103.367949,-74.988497],[-102.016507,-75.125698],[-100.645531,-75.302018],[-100.1167,-74.870933],[-100.763043,-74.537826],[-101.252703,-74.185083],[-102.545337,-74.106742],[-103.113313,-73.734413],[-103.328752,-73.362084],[-103.681289,-72.61753],[-102.917485,-72.754679],[-101.60524,-72.813436],[-100.312528,-72.754679],[-99.13738,-72.911414],[-98.118889,-73.20535],[-97.688037,-73.558041],[-96.336595,-73.616849],[-95.043961,-73.4797],[-93.672907,-73.283743],[-92.439003,-73.166179],[-91.420564,-73.401307],[-90.088733,-73.322914],[-89.226951,-72.558722],[-88.423951,-73.009393],[-87.268337,-73.185764],[-86.014822,-73.087786],[-85.192236,-73.4797],[-83.879991,-73.518871],[-82.665646,-73.636434],[-81.470913,-73.851977],[-80.687447,-73.4797],[-80.295791,-73.126956],[-79.296886,-73.518871],[-77.925858,-73.420892],[-76.907367,-73.636434],[-76.221879,-73.969541],[-74.890049,-73.871614],[-73.852024,-73.65602],[-72.833533,-73.401307],[-71.619215,-73.264157],[-70.209042,-73.146542],[-68.935916,-73.009393],[-67.956622,-72.79385],[-67.369061,-72.480329],[-67.134036,-72.049244],[-67.251548,-71.637745],[-67.56494,-71.245831],[-67.917477,-70.853917],[-68.230843,-70.462055],[-68.485452,-70.109311],[-68.544209,-69.717397],[-68.446282,-69.325535],[-67.976233,-68.953206],[-67.5845,-68.541707],[-67.427843,-68.149844],[-67.62367,-67.718759],[-67.741183,-67.326845],[-67.251548,-66.876175],[-66.703184,-66.58224],[-66.056815,-66.209963],[-65.371327,-65.89639],[-64.568276,-65.602506],[-64.176542,-65.171423],[-63.628152,-64.897073],[-63.001394,-64.642308],[-62.041686,-64.583552],[-61.414928,-64.270031],[-60.709855,-64.074074],[-59.887269,-63.95651],[-59.162585,-63.701745],[-58.594557,-63.388224],[-57.811143,-63.27066],[-57.223582,-63.525425],[-57.59573,-63.858532],[-58.614143,-64.152467],[-59.045073,-64.36801],[-59.789342,-64.211223],[-60.611928,-64.309202],[-61.297416,-64.54433],[-62.0221,-64.799094],[-62.51176,-65.09303],[-62.648858,-65.484942],[-62.590128,-65.857219],[-62.120079,-66.190326],[-62.805567,-66.425505],[-63.74569,-66.503847],[-64.294106,-66.837004],[-64.881693,-67.150474],[-65.508425,-67.58161],[-65.665082,-67.953887],[-65.312545,-68.365335],[-64.783715,-68.678908],[-63.961103,-68.913984],[-63.1973,-69.227556],[-62.785955,-69.619419],[-62.570516,-69.991747],[-62.276736,-70.383661],[-61.806661,-70.716768],[-61.512906,-71.089045],[-61.375809,-72.010074],[-61.081977,-72.382351],[-61.003661,-72.774265],[-60.690269,-73.166179],[-60.827367,-73.695242],[-61.375809,-74.106742],[-61.96337,-74.439848],[-63.295201,-74.576997],[-63.74569,-74.92974],[-64.352836,-75.262847],[-65.860987,-75.635124],[-67.192818,-75.79191],[-68.446282,-76.007452],[-69.797724,-76.222995],[-70.600724,-76.634494],[-72.206776,-76.673665],[-73.969536,-76.634494],[-75.555977,-76.712887],[-77.24037,-76.712887],[-76.926979,-77.104802],[-75.399294,-77.28107],[-74.282876,-77.55542],[-73.656119,-77.908112],[-74.772536,-78.221633],[-76.4961,-78.123654],[-77.925858,-78.378419],[-77.984666,-78.789918],[-78.023785,-79.181833],[-76.848637,-79.514939],[-76.633224,-79.887216],[-75.360097,-80.259545],[-73.244852,-80.416331],[-71.442946,-80.69063],[-70.013163,-81.004151],[-68.191646,-81.317672],[-65.704279,-81.474458],[-63.25603,-81.748757],[-61.552026,-82.042692],[-59.691416,-82.37585],[-58.712121,-82.846106],[-58.222487,-83.218434],[-57.008117,-82.865691],[-55.362894,-82.571755],[-53.619771,-82.258235],[-51.543644,-82.003521],[-49.76135,-81.729171],[-47.273931,-81.709586],[-44.825708,-81.846735],[-42.808363,-82.081915],[-42.16202,-81.65083],[-40.771433,-81.356894],[-38.244818,-81.337309],[-36.26667,-81.121715],[-34.386397,-80.906172],[-32.310296,-80.769023],[-30.097098,-80.592651],[-28.549802,-80.337938],[-29.254901,-79.985195],[-29.685805,-79.632503],[-29.685805,-79.260226],[-31.624808,-79.299397],[-33.681324,-79.456132],[-35.639912,-79.456132],[-35.914107,-79.083855],[-35.77701,-78.339248],[-35.326546,-78.123654],[-33.896763,-77.888526],[-32.212369,-77.65345],[-30.998051,-77.359515],[-29.783732,-77.065579],[-28.882779,-76.673665],[-27.511752,-76.497345],[-26.160336,-76.360144],[-25.474822,-76.281803],[-23.927552,-76.24258],[-22.458598,-76.105431],[-21.224694,-75.909474],[-20.010375,-75.674346],[-18.913543,-75.439218],[-17.522982,-75.125698],[-16.641589,-74.79254],[-15.701491,-74.498604],[-15.40771,-74.106742],[-16.46532,-73.871614],[-16.112784,-73.460114],[-15.446855,-73.146542],[-14.408805,-72.950585],[-13.311973,-72.715457],[-12.293508,-72.401936],[-11.510067,-72.010074],[-11.020433,-71.539767],[-10.295774,-71.265416],[-9.101015,-71.324224],[-8.611381,-71.65733],[-7.416622,-71.696501],[-7.377451,-71.324224],[-6.868232,-70.93231],[-5.790985,-71.030289],[-5.536375,-71.402617],[-4.341667,-71.461373],[-3.048981,-71.285053],[-1.795492,-71.167438],[-0.659489,-71.226246],[-0.228637,-71.637745],[0.868195,-71.304639],[1.886686,-71.128267],[3.022638,-70.991118],[4.139055,-70.853917],[5.157546,-70.618789],[6.273912,-70.462055],[7.13572,-70.246512],[7.742866,-69.893769],[8.48711,-70.148534],[9.525135,-70.011333],[10.249845,-70.48164],[10.817821,-70.834332],[11.953824,-70.638375],[12.404287,-70.246512],[13.422778,-69.972162],[14.734998,-70.030918],[15.126757,-70.403247],[15.949342,-70.030918],[17.026589,-69.913354],[18.201711,-69.874183],[19.259373,-69.893769],[20.375739,-70.011333],[21.452985,-70.07014],[21.923034,-70.403247],[22.569403,-70.697182],[23.666184,-70.520811],[24.841357,-70.48164],[25.977309,-70.48164],[27.093726,-70.462055],[28.09258,-70.324854],[29.150242,-70.20729],[30.031583,-69.93294],[30.971733,-69.75662],[31.990172,-69.658641],[32.754053,-69.384291],[33.302443,-68.835642],[33.870419,-68.502588],[34.908495,-68.659271],[35.300202,-69.012014],[36.16201,-69.247142],[37.200035,-69.168748],[37.905108,-69.52144],[38.649404,-69.776205],[39.667894,-69.541077],[40.020431,-69.109941],[40.921358,-68.933621],[41.959434,-68.600514],[42.938702,-68.463313],[44.113876,-68.267408],[44.897291,-68.051866],[45.719928,-67.816738],[46.503343,-67.601196],[47.44344,-67.718759],[48.344419,-67.366068],[48.990736,-67.091718],[49.930885,-67.111303],[50.753471,-66.876175],[50.949325,-66.523484],[51.791547,-66.249133],[52.614133,-66.053176],[53.613038,-65.89639],[54.53355,-65.818049],[55.414943,-65.876805],[56.355041,-65.974783],[57.158093,-66.249133],[57.255968,-66.680218],[58.137361,-67.013324],[58.744508,-67.287675],[59.939318,-67.405239],[60.605221,-67.679589],[61.427806,-67.953887],[62.387489,-68.012695],[63.19049,-67.816738],[64.052349,-67.405239],[64.992447,-67.620729],[65.971715,-67.738345],[66.911864,-67.855909],[67.891133,-67.934302],[68.890038,-67.934302],[69.712624,-68.972791],[69.673453,-69.227556],[69.555941,-69.678226],[68.596258,-69.93294],[67.81274,-70.305268],[67.949889,-70.697182],[69.066307,-70.677545],[68.929157,-71.069459],[68.419989,-71.441788],[67.949889,-71.853287],[68.71377,-72.166808],[69.869307,-72.264787],[71.024895,-72.088415],[71.573285,-71.696501],[71.906288,-71.324224],[72.454627,-71.010703],[73.08141,-70.716768],[73.33602,-70.364024],[73.864877,-69.874183],[74.491557,-69.776205],[75.62756,-69.737034],[76.626465,-69.619419],[77.644904,-69.462684],[78.134539,-69.07077],[78.428371,-68.698441],[79.113859,-68.326216],[80.093127,-68.071503],[80.93535,-67.875546],[81.483792,-67.542388],[82.051767,-67.366068],[82.776426,-67.209282],[83.775331,-67.30726],[84.676206,-67.209282],[85.655527,-67.091718],[86.752359,-67.150474],[87.477017,-66.876175],[87.986289,-66.209911],[88.358411,-66.484261],[88.828408,-66.954568],[89.67063,-67.150474],[90.630365,-67.228867],[91.5901,-67.111303],[92.608539,-67.189696],[93.548637,-67.209282],[94.17542,-67.111303],[95.017591,-67.170111],[95.781472,-67.385653],[96.682399,-67.248504],[97.759646,-67.248504],[98.68021,-67.111303],[99.718182,-67.248504],[100.384188,-66.915346],[100.893356,-66.58224],[101.578896,-66.30789],[102.832411,-65.563284],[103.478676,-65.700485],[104.242557,-65.974783],[104.90846,-66.327527],[106.181561,-66.934931],[107.160881,-66.954568],[108.081393,-66.954568],[109.15864,-66.837004],[110.235835,-66.699804],[111.058472,-66.425505],[111.74396,-66.13157],[112.860378,-66.092347],[113.604673,-65.876805],[114.388088,-66.072762],[114.897308,-66.386283],[115.602381,-66.699804],[116.699161,-66.660633],[117.384701,-66.915346],[118.57946,-67.170111],[119.832924,-67.268089],[120.871,-67.189696],[121.654415,-66.876175],[122.320369,-66.562654],[123.221296,-66.484261],[124.122274,-66.621462],[125.160247,-66.719389],[126.100396,-66.562654],[127.001427,-66.562654],[127.882768,-66.660633],[128.80328,-66.758611],[129.704259,-66.58224],[130.781454,-66.425505],[131.799945,-66.386283],[132.935896,-66.386283],[133.85646,-66.288304],[134.757387,-66.209963],[135.031582,-65.72007],[135.070753,-65.308571],[135.697485,-65.582869],[135.873805,-66.033591],[136.206705,-66.44509],[136.618049,-66.778197],[137.460271,-66.954568],[138.596223,-66.895761],[139.908442,-66.876175],[140.809421,-66.817367],[142.121692,-66.817367],[143.061842,-66.797782],[144.374061,-66.837004],[145.490427,-66.915346],[146.195552,-67.228867],[145.999699,-67.601196],[146.646067,-67.895131],[147.723263,-68.130259],[148.839629,-68.385024],[150.132314,-68.561292],[151.483705,-68.71813],[152.502247,-68.874813],[153.638199,-68.894502],[154.284567,-68.561292],[155.165857,-68.835642],[155.92979,-69.149215],[156.811132,-69.384291],[158.025528,-69.482269],[159.181013,-69.599833],[159.670699,-69.991747],[160.80665,-70.226875],[161.570479,-70.579618],[162.686897,-70.736353],[163.842434,-70.716768],[164.919681,-70.775524],[166.11444,-70.755938],[167.309095,-70.834332],[168.425616,-70.971481],[169.463589,-71.20666],[170.501665,-71.402617],[171.20679,-71.696501],[171.089227,-72.088415],[170.560422,-72.441159],[170.109958,-72.891829],[169.75737,-73.24452],[169.287321,-73.65602],[167.975101,-73.812806],[167.387489,-74.165498],[166.094803,-74.38104],[165.644391,-74.772954],[164.958851,-75.145283],[164.234193,-75.458804],[163.822797,-75.870303],[163.568239,-76.24258],[163.47026,-76.693302],[163.489897,-77.065579],[164.057873,-77.457442],[164.273363,-77.82977],[164.743464,-78.182514],[166.604126,-78.319611],[166.995781,-78.750748],[165.193876,-78.907483],[163.666217,-79.123025],[161.766385,-79.162248],[160.924162,-79.730482],[160.747894,-80.200737],[160.316964,-80.573066],[159.788211,-80.945395],[161.120016,-81.278501],[161.629287,-81.690001],[162.490992,-82.062278],[163.705336,-82.395435],[165.095949,-82.708956],[166.604126,-83.022477],[168.895665,-83.335998],[169.404782,-83.825891],[172.283934,-84.041433],[172.477049,-84.117914],[173.224083,-84.41371],[175.985672,-84.158997],[178.277212,-84.472518],[180,-84.71338]]]]}},{"type":"Feature","properties":{"ADMIN":"Northern Cyprus","NAME_EN":"Turkish Republic of Northern Cyprus","CONTINENT":"Asia","ADM0_A3":"CYN","ISO_A3":"-99","NAME":"N. Cyprus","ISO_A2":"-99","NAME_ZH":"北塞浦路斯土耳其共和国"},"geometry":{"type":"Polygon","coordinates":[[[32.73178,35.140026],[32.802474,35.145504],[32.946961,35.386703],[33.667227,35.373216],[34.576474,35.671596],[33.900804,35.245756],[33.973617,35.058506],[33.86644,35.093595],[33.675392,35.017863],[33.525685,35.038688],[33.475817,35.000345],[33.455922,35.101424],[33.383833,35.162712],[33.190977,35.173125],[32.919572,35.087833],[32.73178,35.140026]]]}},{"type":"Feature","properties":{"ADMIN":"Cyprus","NAME_EN":"Cyprus","CONTINENT":"Asia","ADM0_A3":"CYP","ISO_A3":"CYP","NAME":"Cyprus","ISO_A2":"CY","NAME_ZH":"塞浦路斯"},"geometry":{"type":"Polygon","coordinates":[[[32.73178,35.140026],[32.919572,35.087833],[33.190977,35.173125],[33.383833,35.162712],[33.455922,35.101424],[33.475817,35.000345],[33.525685,35.038688],[33.675392,35.017863],[33.86644,35.093595],[33.973617,35.058506],[34.004881,34.978098],[32.979827,34.571869],[32.490296,34.701655],[32.256667,35.103232],[32.73178,35.140026]]]}},{"type":"Feature","properties":{"ADMIN":"Morocco","NAME_EN":"Morocco","CONTINENT":"Africa","ADM0_A3":"MAR","ISO_A3":"MAR","NAME":"Morocco","ISO_A2":"MA","NAME_ZH":"摩洛哥"},"geometry":{"type":"Polygon","coordinates":[[[-2.169914,35.168396],[-1.792986,34.527919],[-1.733455,33.919713],[-1.388049,32.864015],[-1.124551,32.651522],[-1.307899,32.262889],[-2.616605,32.094346],[-3.06898,31.724498],[-3.647498,31.637294],[-3.690441,30.896952],[-4.859646,30.501188],[-5.242129,30.000443],[-6.060632,29.7317],[-7.059228,29.579228],[-8.674116,28.841289],[-8.66559,27.656426],[-8.817828,27.656426],[-8.794884,27.120696],[-9.413037,27.088476],[-9.735343,26.860945],[-10.189424,26.860945],[-10.551263,26.990808],[-11.392555,26.883424],[-11.71822,26.104092],[-12.030759,26.030866],[-12.500963,24.770116],[-13.89111,23.691009],[-14.221168,22.310163],[-14.630833,21.86094],[-14.750955,21.5006],[-17.002962,21.420734],[-17.020428,21.42231],[-16.973248,21.885745],[-16.589137,22.158234],[-16.261922,22.67934],[-16.326414,23.017768],[-15.982611,23.723358],[-15.426004,24.359134],[-15.089332,24.520261],[-14.824645,25.103533],[-14.800926,25.636265],[-14.43994,26.254418],[-13.773805,26.618892],[-13.139942,27.640148],[-13.121613,27.654148],[-12.618837,28.038186],[-11.688919,28.148644],[-10.900957,28.832142],[-10.399592,29.098586],[-9.564811,29.933574],[-9.814718,31.177736],[-9.434793,32.038096],[-9.300693,32.564679],[-8.657476,33.240245],[-7.654178,33.697065],[-6.912544,34.110476],[-6.244342,35.145865],[-5.929994,35.759988],[-5.193863,35.755182],[-4.591006,35.330712],[-3.640057,35.399855],[-2.604306,35.179093],[-2.169914,35.168396]]]}},{"type":"Feature","properties":{"ADMIN":"Egypt","NAME_EN":"Egypt","CONTINENT":"Africa","ADM0_A3":"EGY","ISO_A3":"EGY","NAME":"Egypt","ISO_A2":"EG","NAME_ZH":"埃及"},"geometry":{"type":"Polygon","coordinates":[[[36.86623,22],[32.9,22],[29.02,22],[25,22],[25,25.6825],[25,29.238655],[24.70007,30.04419],[24.95762,30.6616],[24.80287,31.08929],[25.16482,31.56915],[26.49533,31.58568],[27.45762,31.32126],[28.45048,31.02577],[28.91353,30.87005],[29.68342,31.18686],[30.09503,31.4734],[30.97693,31.55586],[31.68796,31.4296],[31.96041,30.9336],[32.19247,31.26034],[32.99392,31.02407],[33.7734,30.96746],[34.265435,31.219357],[34.26544,31.21936],[34.823243,29.761081],[34.9226,29.50133],[34.64174,29.09942],[34.42655,28.34399],[34.15451,27.8233],[33.92136,27.6487],[33.58811,27.97136],[33.13676,28.41765],[32.42323,29.85108],[32.32046,29.76043],[32.73482,28.70523],[33.34876,27.69989],[34.10455,26.14227],[34.47387,25.59856],[34.79507,25.03375],[35.69241,23.92671],[35.49372,23.75237],[35.52598,23.10244],[36.69069,22.20485],[36.86623,22]]]}},{"type":"Feature","properties":{"ADMIN":"Libya","NAME_EN":"Libya","CONTINENT":"Africa","ADM0_A3":"LBY","ISO_A3":"LBY","NAME":"Libya","ISO_A2":"LY","NAME_ZH":"利比亚"},"geometry":{"type":"Polygon","coordinates":[[[25,22],[25,20.00304],[23.85,20],[23.83766,19.58047],[19.84926,21.49509],[15.86085,23.40972],[14.8513,22.86295],[14.143871,22.491289],[13.581425,23.040506],[11.999506,23.471668],[11.560669,24.097909],[10.771364,24.562532],[10.303847,24.379313],[9.948261,24.936954],[9.910693,25.365455],[9.319411,26.094325],[9.716286,26.512206],[9.629056,27.140953],[9.756128,27.688259],[9.683885,28.144174],[9.859998,28.95999],[9.805634,29.424638],[9.48214,30.307556],[9.970017,30.539325],[10.056575,30.961831],[9.950225,31.37607],[10.636901,31.761421],[10.94479,32.081815],[11.432253,32.368903],[11.488787,33.136996],[12.66331,32.79278],[13.08326,32.87882],[13.91868,32.71196],[15.24563,32.26508],[15.71394,31.37626],[16.61162,31.18218],[18.02109,30.76357],[19.08641,30.26639],[19.57404,30.52582],[20.05335,30.98576],[19.82033,31.75179],[20.13397,32.2382],[20.85452,32.7068],[21.54298,32.8432],[22.89576,32.63858],[23.2368,32.19149],[23.60913,32.18726],[23.9275,32.01667],[24.92114,31.89936],[25.16482,31.56915],[24.80287,31.08929],[24.95762,30.6616],[24.70007,30.04419],[25,29.238655],[25,25.6825],[25,22]]]}},{"type":"Feature","properties":{"ADMIN":"Ethiopia","NAME_EN":"Ethiopia","CONTINENT":"Africa","ADM0_A3":"ETH","ISO_A3":"ETH","NAME":"Ethiopia","ISO_A2":"ET","NAME_ZH":"埃塞俄比亚"},"geometry":{"type":"Polygon","coordinates":[[[47.78942,8.003],[44.9636,5.00162],[43.66087,4.95755],[42.76967,4.25259],[42.12861,4.23413],[41.855083,3.918912],[41.1718,3.91909],[40.76848,4.25702],[39.85494,3.83879],[39.559384,3.42206],[38.89251,3.50074],[38.67114,3.61607],[38.43697,3.58851],[38.120915,3.598605],[36.855093,4.447864],[36.159079,4.447864],[35.817448,4.776966],[35.817448,5.338232],[35.298007,5.506],[34.70702,6.59422],[34.25032,6.82607],[34.0751,7.22595],[33.56829,7.71334],[32.95418,7.78497],[33.2948,8.35458],[33.8255,8.37916],[33.97498,8.68456],[33.96162,9.58358],[34.25745,10.63009],[34.73115,10.91017],[34.83163,11.31896],[35.26049,12.08286],[35.86363,12.57828],[36.27022,13.56333],[36.42951,14.42211],[37.59377,14.2131],[37.90607,14.95943],[38.51295,14.50547],[39.0994,14.74064],[39.34061,14.53155],[40.02625,14.51959],[40.8966,14.11864],[41.1552,13.77333],[41.59856,13.45209],[42.00975,12.86582],[42.35156,12.54223],[42,12.1],[41.66176,11.6312],[41.73959,11.35511],[41.75557,11.05091],[42.31414,11.0342],[42.55493,11.10511],[42.776852,10.926879],[42.55876,10.57258],[42.92812,10.02194],[43.29699,9.54048],[43.67875,9.18358],[46.94834,7.99688],[47.78942,8.003]]]}},{"type":"Feature","properties":{"ADMIN":"Djibouti","NAME_EN":"Djibouti","CONTINENT":"Africa","ADM0_A3":"DJI","ISO_A3":"DJI","NAME":"Djibouti","ISO_A2":"DJ","NAME_ZH":"吉布提"},"geometry":{"type":"Polygon","coordinates":[[[42.35156,12.54223],[42.779642,12.455416],[43.081226,12.699639],[43.317852,12.390148],[43.286381,11.974928],[42.715874,11.735641],[43.145305,11.46204],[42.776852,10.926879],[42.55493,11.10511],[42.31414,11.0342],[41.75557,11.05091],[41.73959,11.35511],[41.66176,11.6312],[42,12.1],[42.35156,12.54223]]]}},{"type":"Feature","properties":{"ADMIN":"Somaliland","NAME_EN":"Somaliland","CONTINENT":"Africa","ADM0_A3":"SOL","ISO_A3":"-99","NAME":"Somaliland","ISO_A2":"-99","NAME_ZH":"索马里兰"},"geometry":{"type":"Polygon","coordinates":[[[48.948205,11.410617],[48.948205,11.410617],[48.942005,11.394266],[48.938491,10.982327],[48.938233,9.9735],[48.93813,9.451749],[48.486736,8.837626],[47.78942,8.003],[46.94834,7.99688],[43.67875,9.18358],[43.29699,9.54048],[42.92812,10.02194],[42.55876,10.57258],[42.776852,10.926879],[43.145305,11.46204],[43.47066,11.27771],[43.666668,10.864169],[44.117804,10.445538],[44.614259,10.442205],[45.556941,10.698029],[46.645401,10.816549],[47.525658,11.127228],[48.021596,11.193064],[48.378784,11.375482],[48.948206,11.410622],[48.948205,11.410617]]]}},{"type":"Feature","properties":{"ADMIN":"Uganda","NAME_EN":"Uganda","CONTINENT":"Africa","ADM0_A3":"UGA","ISO_A3":"UGA","NAME":"Uganda","ISO_A2":"UG","NAME_ZH":"乌干达"},"geometry":{"type":"Polygon","coordinates":[[[33.903711,-0.95],[31.86617,-1.02736],[30.76986,-1.01455],[30.419105,-1.134659],[29.821519,-1.443322],[29.579466,-1.341313],[29.587838,-0.587406],[29.819503,-0.20531],[29.875779,0.59738],[30.086154,1.062313],[30.468508,1.583805],[30.85267,1.849396],[31.174149,2.204465],[30.773347,2.339883],[30.83386,3.509166],[30.833852,3.509172],[31.24556,3.7819],[31.88145,3.55827],[32.68642,3.79232],[33.39,3.79],[34.005,4.249885],[34.47913,3.5556],[34.59607,3.05374],[35.03599,1.90584],[34.6721,1.17694],[34.18,0.515],[33.893569,0.109814],[33.903711,-0.95]]]}},{"type":"Feature","properties":{"ADMIN":"Rwanda","NAME_EN":"Rwanda","CONTINENT":"Africa","ADM0_A3":"RWA","ISO_A3":"RWA","NAME":"Rwanda","ISO_A2":"RW","NAME_ZH":"卢旺达"},"geometry":{"type":"Polygon","coordinates":[[[30.419105,-1.134659],[30.816135,-1.698914],[30.758309,-2.28725],[30.46967,-2.41383],[30.469674,-2.413855],[29.938359,-2.348487],[29.632176,-2.917858],[29.024926,-2.839258],[29.117479,-2.292211],[29.254835,-2.21511],[29.291887,-1.620056],[29.579466,-1.341313],[29.821519,-1.443322],[30.419105,-1.134659]]]}},{"type":"Feature","properties":{"ADMIN":"Bosnia and Herzegovina","NAME_EN":"Bosnia and Herzegovina","CONTINENT":"Europe","ADM0_A3":"BIH","ISO_A3":"BIH","NAME":"Bosnia and Herz.","ISO_A2":"BA","NAME_ZH":"波斯尼亚和黑塞哥维那"},"geometry":{"type":"Polygon","coordinates":[[[18.56,42.65],[17.674922,43.028563],[17.297373,43.446341],[16.916156,43.667722],[16.456443,44.04124],[16.23966,44.351143],[15.750026,44.818712],[15.959367,45.233777],[16.318157,45.004127],[16.534939,45.211608],[17.002146,45.233777],[17.861783,45.06774],[18.553214,45.08159],[19.005485,44.860234],[19.00548,44.86023],[19.36803,44.863],[19.11761,44.42307],[19.59976,44.03847],[19.454,43.5681],[19.21852,43.52384],[19.03165,43.43253],[18.70648,43.20011],[18.56,42.65]]]}},{"type":"Feature","properties":{"ADMIN":"North Macedonia","NAME_EN":"North Macedonia","CONTINENT":"Europe","ADM0_A3":"MKD","ISO_A3":"MKD","NAME":"North Macedonia","ISO_A2":"MK","NAME_ZH":"北马其顿"},"geometry":{"type":"Polygon","coordinates":[[[22.380526,42.32026],[22.881374,41.999297],[22.952377,41.337994],[22.76177,41.3048],[22.597308,41.130487],[22.055378,41.149866],[21.674161,40.931275],[21.02004,40.842727],[20.605182,41.086226],[20.463175,41.515089],[20.590247,41.855404],[20.590247,41.855409],[20.71731,41.84711],[20.76216,42.05186],[21.3527,42.2068],[21.576636,42.245224],[21.91708,42.30364],[22.380526,42.32026]]]}},{"type":"Feature","properties":{"ADMIN":"Republic of Serbia","NAME_EN":"Serbia","CONTINENT":"Europe","ADM0_A3":"SRB","ISO_A3":"SRB","NAME":"Serbia","ISO_A2":"RS","NAME_ZH":"塞尔维亚"},"geometry":{"type":"MultiPolygon","coordinates":[[[[18.829825,45.908872],[18.829838,45.908878],[19.596045,46.17173],[20.220192,46.127469],[20.762175,45.734573],[20.874313,45.416375],[21.483526,45.18117],[21.562023,44.768947],[22.145088,44.478422],[22.459022,44.702517],[22.705726,44.578003],[22.474008,44.409228],[22.65715,44.234923],[22.410446,44.008063],[22.500157,43.642814],[22.986019,43.211161],[22.604801,42.898519],[22.436595,42.580321],[22.545012,42.461362],[22.380526,42.32026],[21.91708,42.30364],[21.576636,42.245224],[21.54332,42.32025],[21.66292,42.43922],[21.77505,42.6827],[21.63302,42.67717],[21.43866,42.86255],[21.27421,42.90959],[21.143395,43.068685],[20.95651,43.13094],[20.81448,43.27205],[20.63508,43.21671],[20.49679,42.88469],[20.25758,42.81275],[20.3398,42.89852],[19.95857,43.10604],[19.63,43.21378],[19.48389,43.35229],[19.21852,43.52384],[19.454,43.5681],[19.59976,44.03847],[19.11761,44.42307],[19.36803,44.863],[19.00548,44.86023],[19.005485,44.860234],[19.390476,45.236516],[19.072769,45.521511],[18.829825,45.908872]]],[[[20.590247,41.855409],[20.52295,42.21787],[20.283755,42.32026],[20.0707,42.58863],[20.25758,42.81275],[20.49679,42.88469],[20.63508,43.21671],[20.81448,43.27205],[20.95651,43.13094],[21.143395,43.068685],[21.27421,42.90959],[21.43866,42.86255],[21.63302,42.67717],[21.77505,42.6827],[21.66292,42.43922],[21.54332,42.32025],[21.576636,42.245224],[21.3527,42.2068],[20.76216,42.05186],[20.71731,41.84711],[20.590247,41.855409]]]]}},{"type":"Feature","properties":{"ADMIN":"Montenegro","NAME_EN":"Montenegro","CONTINENT":"Europe","ADM0_A3":"MNE","ISO_A3":"MNE","NAME":"Montenegro","ISO_A2":"ME","NAME_ZH":"黑山"},"geometry":{"type":"Polygon","coordinates":[[[20.0707,42.58863],[19.801613,42.500093],[19.738051,42.688247],[19.304486,42.195745],[19.371768,41.877551],[19.16246,41.95502],[18.88214,42.28151],[18.450017,42.479992],[18.56,42.65],[18.70648,43.20011],[19.03165,43.43253],[19.21852,43.52384],[19.48389,43.35229],[19.63,43.21378],[19.95857,43.10604],[20.3398,42.89852],[20.25758,42.81275],[20.0707,42.58863]]]}},{"type":"Feature","properties":{"ADMIN":"Trinidad and Tobago","NAME_EN":"Trinidad and Tobago","CONTINENT":"North America","ADM0_A3":"TTO","ISO_A3":"TTO","NAME":"Trinidad and Tobago","ISO_A2":"TT","NAME_ZH":"特立尼达和多巴哥"},"geometry":{"type":"Polygon","coordinates":[[[-61.68,10.76],[-61.105,10.89],[-60.895,10.855],[-60.935,10.11],[-61.77,10],[-61.95,10.09],[-61.66,10.365],[-61.68,10.76]]]}},{"type":"Feature","properties":{"ADMIN":"South Sudan","NAME_EN":"South Sudan","CONTINENT":"Africa","ADM0_A3":"SDS","ISO_A3":"SSD","NAME":"S. Sudan","ISO_A2":"SS","NAME_ZH":"南苏丹"},"geometry":{"type":"Polygon","coordinates":[[[30.833852,3.509172],[29.9535,4.173699],[29.715995,4.600805],[29.159078,4.389267],[28.696678,4.455077],[28.428994,4.287155],[27.979977,4.408413],[27.374226,5.233944],[27.213409,5.550953],[26.465909,5.946717],[26.213418,6.546603],[25.796648,6.979316],[25.124131,7.500085],[25.114932,7.825104],[24.567369,8.229188],[23.88698,8.61973],[24.194068,8.728696],[24.537415,8.917538],[24.794926,9.810241],[25.069604,10.27376],[25.790633,10.411099],[25.962307,10.136421],[26.477328,9.55273],[26.752006,9.466893],[27.112521,9.638567],[27.833551,9.604232],[27.97089,9.398224],[28.966597,9.398224],[29.000932,9.604232],[29.515953,9.793074],[29.618957,10.084919],[29.996639,10.290927],[30.837841,9.707237],[31.352862,9.810241],[31.850716,10.531271],[32.400072,11.080626],[32.314235,11.681484],[32.073892,11.97333],[32.67475,12.024832],[32.743419,12.248008],[33.206938,12.179338],[33.086766,11.441141],[33.206938,10.720112],[33.721959,10.325262],[33.842131,9.981915],[33.824963,9.484061],[33.963393,9.464285],[33.97498,8.68456],[33.8255,8.37916],[33.2948,8.35458],[32.95418,7.78497],[33.56829,7.71334],[34.0751,7.22595],[34.25032,6.82607],[34.70702,6.59422],[35.298007,5.506],[34.620196,4.847123],[34.005,4.249885],[33.39,3.79],[32.68642,3.79232],[31.88145,3.55827],[31.24556,3.7819],[30.833852,3.509172]]]}}]} \ No newline at end of file diff --git a/frontend/public/earth/index.html b/frontend/public/earth/index.html index c4882284..f9e85b54 100644 --- a/frontend/public/earth/index.html +++ b/frontend/public/earth/index.html @@ -16,6 +16,22 @@ } @@ -92,23 +189,13 @@
-
- landscape +
+ cable
- 地形 - Terrain + 海缆 + Subsea Cables
- -
-
- satellite_alt -
- 卫星 - Satellites -
-
@@ -122,13 +209,33 @@
-
- cable +
+ satellite_alt
- 海缆 - Subsea Cables + 卫星 + Satellites
- +
+
+ memory +
+ 算力中心 + Compute Centers +
+ +
+
+ directions_boat +
+ 船只 + AIS Vessels +
+
@@ -142,6 +249,56 @@
+
+ landscape +
+ 地形 + Terrain +
+ +
+
+ globe +
+ 高清材质 + High-Res Texture +
+ +
+
+ cloud +
+ 大气云图 + Cloud Layer +
+ +
+
+ public +
+ 国界 + Country Borders +
+ +
+
+ grid_4x4 +
+ 经纬线 + Graticule +
+ +
@@ -156,14 +313,22 @@
{tableHeaderCells.map((cell, cellIndex) => ( - + ))} @@ -199,28 +418,45 @@ export default function MarkdownRenderer({ markdown, className }: MarkdownRender {bodyRows.map((row, rowIndex) => ( {row.map((cell, cellIndex) => ( - + ))} ))}
{renderInlineMarkdown(cell)}{renderInlineMarkdown(cell, transformLink)}
{renderInlineMarkdown(cell)}{renderInlineMarkdown(cell, transformLink)}
-
, + , ) continue } const paragraphLines: string[] = [] while (index < lines.length && lines[index].trim()) { + if ( + lines[index].trim().startsWith('```') || + lines[index].trim().startsWith('>') || + parseListLine(lines[index]) || + /^#{1,6}\s+/.test(lines[index].trim()) || + /^(-{3,}|\*{3,}|_{3,})$/.test(lines[index].trim()) + ) { + break + } paragraphLines.push(lines[index].trim()) index += 1 } - nodes.push(

{renderInlineMarkdown(paragraphLines.join(' '))}

) + + if (paragraphLines.length === 0) { + index += 1 + continue + } + + nodes.push(

{renderInlineMarkdown(paragraphLines.join(' '), transformLink)}

) } return
{nodes}
} +export default memo(MarkdownRenderer) + function parseTableRow(line: string): string[] | null { if (!line.includes('|')) { return null diff --git a/frontend/src/components/Scrollbar/Scrollbar.tsx b/frontend/src/components/Scrollbar/Scrollbar.tsx index 5fd54519..6a0319ee 100644 --- a/frontend/src/components/Scrollbar/Scrollbar.tsx +++ b/frontend/src/components/Scrollbar/Scrollbar.tsx @@ -1,6 +1,7 @@ import { type PointerEvent as ReactPointerEvent, type ReactNode, + type RefObject, useCallback, useEffect, useRef, @@ -25,6 +26,7 @@ interface ScrollbarProps { children: ReactNode className?: string minThumbSize?: number + viewportRef?: RefObject } interface DragState { @@ -49,8 +51,10 @@ function Scrollbar({ children, className = '', minThumbSize = 28, + viewportRef: externalViewportRef, }: ScrollbarProps) { - const viewportRef = useRef(null) + const internalViewportRef = useRef(null) + const viewportRef = externalViewportRef ?? internalViewportRef const trackXRef = useRef(null) const trackYRef = useRef(null) const dragStateRef = useRef(null) diff --git a/frontend/src/components/SegmentedControl/SegmentedControl.css b/frontend/src/components/SegmentedControl/SegmentedControl.css new file mode 100644 index 00000000..c6bbf0a9 --- /dev/null +++ b/frontend/src/components/SegmentedControl/SegmentedControl.css @@ -0,0 +1,91 @@ +.segmented-control { + position: relative; + display: flex; + align-items: center; + min-width: 0; + height: calc(42px * var(--segmented-control-scale, 1)); + padding: calc(4px * var(--segmented-control-scale, 1)); + border: 1px solid var(--segmented-control-border, var(--d-border, #d9e1ec)); + border-radius: var(--segmented-control-radius, calc(14px * var(--segmented-control-scale, 1))); + background: var(--segmented-control-bg, var(--d-segment-bg, #eef3f9)); + box-shadow: inset 0 1px 2px rgba(15, 23, 42, 0.06); +} + +.segmented-control__slider { + position: absolute; + top: calc(4px * var(--segmented-control-scale, 1)); + left: calc(4px * var(--segmented-control-scale, 1)); + z-index: 1; + width: calc((100% - (8px * var(--segmented-control-scale, 1))) / var(--segmented-control-items, 2)); + height: calc(100% - (8px * var(--segmented-control-scale, 1))); + border-radius: var(--segmented-control-slider-radius, calc(10px * var(--segmented-control-scale, 1))); + background: var(--segmented-control-slider-bg, var(--d-segment-slider, #ffffff)); + box-shadow: var(--segmented-control-slider-shadow, var(--d-segment-shadow, 0 2px 8px rgba(15, 23, 42, 0.12))); + transform: translateX(calc(var(--segmented-control-index, 0) * 100%)); + transition: + transform 0.46s cubic-bezier(0.34, 1.56, 0.64, 1), + background 0.22s ease, + box-shadow 0.22s ease; +} + +.segmented-control__button { + position: relative; + z-index: 2; + flex: 1 1 0; + min-width: 0; + height: 100%; + display: inline-flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: var(--segmented-control-button-gap, calc(2px * var(--segmented-control-scale, 1))); + border: 0; + border-radius: var(--segmented-control-button-radius, calc(10px * var(--segmented-control-scale, 1))); + background: none; + color: var(--segmented-control-color, var(--d-lang-btn, #4a5568)); + font: inherit; + font-size: var(--segmented-control-font-size, calc(10px * var(--segmented-control-scale, 1))); + font-weight: var(--segmented-control-font-weight, 800); + letter-spacing: var(--segmented-control-letter-spacing, 0.04em); + cursor: pointer; + transition: color 0.18s ease, transform 0.18s ease; +} + +.segmented-control__button:hover { + color: var(--segmented-control-hover, var(--d-nav-hover, #0d4f9f)); +} + +.segmented-control__button:active .segmented-control__icon { + transform: scale(0.86); +} + +.segmented-control__button--active { + color: var(--segmented-control-active, var(--d-nav-active, #0b5fc1)); +} + +.segmented-control__icon { + width: var(--segmented-control-icon-size, calc(15px * var(--segmented-control-scale, 1))); + height: var(--segmented-control-icon-size, calc(15px * var(--segmented-control-scale, 1))); + display: inline-flex; + align-items: center; + justify-content: center; + transition: transform 0.2s ease; +} + +.segmented-control__icon svg { + width: var(--segmented-control-icon-size, calc(15px * var(--segmented-control-scale, 1))); + height: var(--segmented-control-icon-size, calc(15px * var(--segmented-control-scale, 1))); + stroke-width: 2.1; +} + +.segmented-control__button--active .segmented-control__icon svg { + stroke-width: 2.45; +} + +.segmented-control__label { + min-width: 0; + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} diff --git a/frontend/src/components/SegmentedControl/SegmentedControl.tsx b/frontend/src/components/SegmentedControl/SegmentedControl.tsx new file mode 100644 index 00000000..9efbcaab --- /dev/null +++ b/frontend/src/components/SegmentedControl/SegmentedControl.tsx @@ -0,0 +1,69 @@ +import type { CSSProperties, ReactNode } from 'react' + +import './SegmentedControl.css' + +export interface SegmentedControlOption { + value: T + label: ReactNode + icon?: ReactNode + title?: string +} + +interface SegmentedControlProps { + ariaLabel: string + className?: string + options: SegmentedControlOption[] + scale?: number + value: T + onChange: (value: T) => void +} + +function SegmentedControl({ + ariaLabel, + className = '', + options, + scale = 1, + value, + onChange, +}: SegmentedControlProps) { + const activeIndex = Math.max(0, options.findIndex((option) => option.value === value)) + const style = { + '--segmented-control-items': options.length, + '--segmented-control-index': activeIndex, + '--segmented-control-scale': scale, + } as CSSProperties + + return ( +
+
+ ) +} + +export default SegmentedControl diff --git a/frontend/src/hooks/useWebSocket.ts b/frontend/src/hooks/useWebSocket.ts index ec96810e..70be063a 100644 --- a/frontend/src/hooks/useWebSocket.ts +++ b/frontend/src/hooks/useWebSocket.ts @@ -54,6 +54,8 @@ interface UseWebSocketOptions { interface UseWebSocketReturn { connected: boolean + connecting: boolean + status: 'connecting' | 'connected' | 'disconnected' lastMessage: WebSocketMessage | null sendMessage: (message: Record) => void subscribe: (channels: string[]) => void @@ -65,6 +67,7 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet const { autoConnect = true, autoSubscribe = [], + heartbeatInterval = 25000, onMessage, onConnect, onDisconnect, @@ -75,6 +78,7 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet const wsRef = useRef(null) const [connected, setConnected] = useState(false) + const [connecting, setConnecting] = useState(false) const [lastMessage, setLastMessage] = useState(null) const reconnectTimeoutRef = useRef | null>(null) const heartbeatTimerRef = useRef | null>(null) @@ -97,17 +101,21 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet const connect = useCallback(() => { if (!token) { + setConnected(false) + setConnecting(false) return } intentionalCloseRef.current = false + setConnected(false) + setConnecting(true) const candidates = buildWebSocketCandidates() let candidateIndex = 0 let opened = false const tryConnect = () => { const baseUrl = candidates[candidateIndex] - const wsUrl = `${baseUrl}?token=${token}` + const wsUrl = `${baseUrl}?token=${encodeURIComponent(token)}` activeWsUrlRef.current = baseUrl const ws = new WebSocket(wsUrl) @@ -119,15 +127,27 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet } opened = true setConnected(true) + setConnecting(false) if (autoSubscribeRef.current.length > 0) { ws.send(JSON.stringify({ type: 'subscribe', data: { channels: autoSubscribeRef.current } })) } + if (heartbeatTimerRef.current) { + clearInterval(heartbeatTimerRef.current) + } + heartbeatTimerRef.current = setInterval(() => { + if (ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify({ type: 'heartbeat' })) + } + }, heartbeatInterval) onConnectRef.current?.() } ws.onmessage = (event) => { try { const message: WebSocketMessage = JSON.parse(event.data) + if (message.type === 'heartbeat' && message.data?.action === 'ping' && ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify({ type: 'heartbeat' })) + } setLastMessage(message) onMessageRef.current?.(message) } catch { @@ -150,10 +170,13 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet if (!opened && candidateIndex < candidates.length - 1) { candidateIndex += 1 + setConnecting(true) tryConnect() return } + setConnecting(false) + if (intentionalCloseRef.current) { return } @@ -169,6 +192,9 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet ws.onerror = (error) => { setConnected(false) + if (opened || candidateIndex >= candidates.length - 1) { + setConnecting(false) + } if (intentionalCloseRef.current || ws.readyState === WebSocket.CLOSING || ws.readyState === WebSocket.CLOSED) { return } @@ -185,6 +211,7 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet tryConnect() } catch (error) { setConnected(false) + setConnecting(false) console.warn('[WebSocket] Failed to initialize connection', { url: activeWsUrlRef.current, error }) if (autoConnect && token) { reconnectTimeoutRef.current = setTimeout(() => { @@ -192,7 +219,7 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet }, 3000) } } - }, [token, autoConnect]) + }, [token, autoConnect, heartbeatInterval]) const disconnect = useCallback(() => { intentionalCloseRef.current = true @@ -213,6 +240,7 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet } } setConnected(false) + setConnecting(false) }, []) const sendMessage = useCallback((message: Record) => { @@ -237,6 +265,8 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet return { connected, + connecting, + status: connected ? 'connected' : connecting ? 'connecting' : 'disconnected', lastMessage, sendMessage, subscribe, diff --git a/frontend/src/index.css b/frontend/src/index.css index 63994175..8b855d10 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1487,6 +1487,49 @@ body { color: #cf1322; } +.data-source-bulk-toolbar__running-pill { + width: fit-content; + display: inline-flex; + align-items: center; + gap: 8px; + padding: 6px 10px; + border: 1px solid #91caff; + border-radius: 999px; + background: #ffffff; + color: #475467; + line-height: 1; + cursor: pointer; + box-shadow: 0 1px 0 rgba(22, 119, 255, 0.08); + transition: border-color 0.16s ease, box-shadow 0.16s ease, transform 0.16s ease; +} + +.data-source-bulk-toolbar__running-pill:hover { + border-color: #1677ff; + box-shadow: 0 6px 14px rgba(22, 119, 255, 0.16); + transform: translateY(-1px); +} + +.data-source-bulk-toolbar__running-pill strong { + color: #0958d9; + font-size: 13px; + font-weight: 700; +} + +.data-source-bulk-toolbar__running-dot { + width: 7px; + height: 7px; + border-radius: 999px; + background: #1677ff; + box-shadow: 0 0 0 4px rgba(22, 119, 255, 0.12); +} + +.data-source-bulk-toolbar__running-arrow { + color: #1677ff; + font-size: 16px; + line-height: 1; + margin-left: 2px; +} + .data-source-bulk-toolbar__progress { display: flex; flex-direction: column; @@ -2080,7 +2123,9 @@ body { .markdown-renderer h1, .markdown-renderer h2, .markdown-renderer h3, -.markdown-renderer h4 { +.markdown-renderer h4, +.markdown-renderer h5, +.markdown-renderer h6 { margin: 1.2em 0 0.5em; color: #111827; font-weight: 600; @@ -2099,11 +2144,20 @@ body { font-size: 16px; } +.markdown-renderer h4 { + font-size: 15px; +} + +.markdown-renderer h5, +.markdown-renderer h6 { + font-size: 14px; +} + .markdown-renderer p, .markdown-renderer ul, .markdown-renderer ol, .markdown-renderer blockquote, -.markdown-renderer pre, +.markdown-renderer__code-block, .markdown-renderer hr, .markdown-renderer__table-wrap { margin: 0 0 0.9em; @@ -2118,6 +2172,34 @@ body { margin-top: 0.25em; } +.markdown-renderer li > ul, +.markdown-renderer li > ol { + margin: 0.3em 0 0; +} + +.markdown-renderer__task-list { + list-style: none; + padding-left: 0; +} + +.markdown-renderer__task-item { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + gap: 8px; +} + +.markdown-renderer__task-item > ul, +.markdown-renderer__task-item > ol { + flex-basis: 100%; +} + +.markdown-renderer__task-checkbox { + flex: 0 0 auto; + margin-top: 0.42em; + accent-color: #1677ff; +} + .markdown-renderer blockquote { padding: 10px 14px; border-left: 3px solid #91caff; @@ -2126,12 +2208,85 @@ body { color: #1f2937; } -.markdown-renderer pre { - overflow: auto; - padding: 12px 14px; +.markdown-renderer__code-block { + overflow: hidden; border-radius: 10px; background: #0f172a; + box-shadow: inset 0 0 0 1px rgba(148, 163, 184, 0.18); +} + +.markdown-renderer__code-toolbar { + display: flex; + align-items: center; + justify-content: space-between; + min-height: 34px; + padding: 6px 8px 6px 12px; + border-bottom: 1px solid rgba(148, 163, 184, 0.18); + background: rgba(15, 23, 42, 0.94); + color: #cbd5e1; + font-size: 12px; +} + +.markdown-renderer__code-language { + overflow: hidden; + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; + text-overflow: ellipsis; + white-space: nowrap; +} + +.markdown-renderer__code-copy { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 6px; + width: 26px; + height: 26px; + padding: 0; + border: 1px solid rgba(148, 163, 184, 0.28); + border-radius: 6px; + background: rgba(30, 41, 59, 0.88); color: #e2e8f0; + cursor: pointer; + font-size: 12px; + line-height: 1; + transition: background 0.15s ease, border-color 0.15s ease, color 0.15s ease; +} + +.markdown-renderer__code-copy:hover { + border-color: rgba(191, 219, 254, 0.55); + background: rgba(51, 65, 85, 0.96); + color: #ffffff; +} + +.markdown-renderer pre { + overflow: visible; + padding: 12px 14px; + border-radius: 0; + background: #0f172a; + color: #e2e8f0; +} + +.markdown-renderer__code-scroll { + max-width: 100%; + border-radius: 0; +} + +.markdown-renderer__code-scroll > .scrollbar__viewport, +.markdown-renderer__table-wrap > .scrollbar__viewport { + padding: 0; +} + +.markdown-renderer__code-scroll pre { + min-width: max-content; + margin: 0; +} + +.markdown-renderer__image { + display: block; + max-width: 100%; + height: auto; + margin: 0.6em 0; + border-radius: 8px; } .markdown-renderer hr { @@ -2140,7 +2295,7 @@ body { } .markdown-renderer__table-wrap { - overflow-x: auto; + max-width: 100%; } .markdown-renderer__table { @@ -3414,8 +3569,365 @@ body { max-height: 180px; } +.system-log-console { + flex: 1 1 auto; + min-height: 0; + height: 100%; + position: relative; + border-radius: 16px; + background: #020617; + border: 1px solid rgba(148, 163, 184, 0.18); + overflow: hidden; +} + +.system-log-console__actions { + position: absolute; + top: 12px; + right: 12px; + z-index: 2; + display: inline-flex; + align-items: center; + gap: 2px; + padding-inline: 6px; +} + +.system-log-console__actions .ant-btn { + color: rgba(226, 232, 240, 0.78) !important; +} + +.system-log-console__actions .ant-btn:hover { + color: #f8fafc !important; + background: rgba(148, 163, 184, 0.16) !important; +} + +.system-log-console__scroll, +.system-log-console__scroll .scrollbar__viewport { + height: 100%; +} + +.system-log-console__scroll, +.system-log-console__scroll .scrollbar__viewport, +.system-log-console__content, +.system-log-console__placeholder { + min-height: 100%; +} + +.system-log-console__content { + margin: 0; + padding: 18px 124px 18px 20px; + color: #e2e8f0; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 12px; + line-height: 1.65; + white-space: pre-wrap; + word-break: break-word; +} + +.system-log-console__placeholder { + display: flex; + align-items: center; + justify-content: center; + padding: 24px; +} + +.logs-page { + --logs-filter-toggle-size: 32px; + height: 100%; + min-height: 0; + gap: 10px; +} + +.logs-page__header-copy { + min-width: 0; +} + +.logs-page .page-shell__header { + gap: 8px; +} + +.logs-page__header-desc { + line-height: 1.4; +} + +.logs-page__card { + flex: 1 1 auto; + min-height: 0; + display: flex; + flex-direction: column; +} + +.logs-page__card .ant-card-body { + flex: 1 1 auto; + min-height: 0; + display: flex; + flex-direction: column; + padding: 14px; +} + +.logs-page__card-body { + flex: 1 1 auto; + min-height: 0; + display: flex; + flex-direction: column; + gap: 12px; + overflow: hidden; +} + +.logs-page__console-shell { + flex: 1 1 auto; + min-height: 0; + display: flex; + flex-direction: column; +} + +.logs-page__toolbar { + flex: 0 0 auto; + display: flex; + flex-direction: column; + gap: 8px; + padding: 10px 12px; + border-radius: 16px; + background: linear-gradient(135deg, rgba(255, 255, 255, 0.98) 0%, rgba(247, 249, 252, 0.98) 100%); + border: 1px solid rgba(5, 5, 5, 0.07); + box-shadow: 0 8px 18px rgba(15, 23, 42, 0.04); +} + +.logs-page__toolbar-row { + display: flex; + align-items: center; + gap: 10px; + min-width: 0; + flex-wrap: wrap; +} + +.logs-page__toolbar-row--primary { + display: grid; + grid-template-columns: minmax(180px, 220px) minmax(220px, 280px) minmax(280px, 1fr) var(--logs-filter-toggle-size); + align-items: center; +} + +.logs-page__source-select { + width: 240px; + min-width: 220px; +} + +.logs-page__search-input { + width: 100%; + min-width: 240px; +} + +.logs-page__level-select { + width: 100%; + min-width: 220px; +} + +.logs-page__date-range { + width: 248px; +} + +.logs-page__line-limit-select { + width: 156px; +} + +.logs-page__filter-toggle { + display: inline-flex; + justify-content: center; + align-items: center; + align-self: center; + width: var(--logs-filter-toggle-size); + height: var(--logs-filter-toggle-size); + padding: 0; + border: 1px solid rgba(5, 5, 5, 0.08); + border-radius: 8px; + background: transparent; + color: rgba(0, 0, 0, 0.65); + cursor: pointer; + transition: color 0.18s ease, border-color 0.18s ease, background 0.18s ease; +} + +.logs-page__filter-toggle:hover { + color: rgba(0, 0, 0, 0.88); + border-color: rgba(5, 5, 5, 0.16); + background: rgba(0, 0, 0, 0.02); +} + +.logs-page__filter-toggle.is-expanded { + color: #1677ff; + border-color: rgba(22, 119, 255, 0.28); + background: rgba(22, 119, 255, 0.06); +} + +.logs-page__filters-panel { + border-top: 1px solid rgba(5, 5, 5, 0.06); + padding-top: 8px; +} + +.logs-page__toolbar-row--secondary { + display: grid; + grid-template-columns: 156px minmax(260px, 320px) minmax(0, 1fr); + align-items: start; +} + +.logs-page__preset-group { + display: flex; + align-self: center; + align-items: center; + flex-wrap: wrap; + justify-content: flex-start; +} + +.logs-page__preset-group .ant-space-item { + display: flex; + align-items: center; +} + +.logs-page__preset-group .ant-btn { + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 999px; +} + +.logs-page__calendar-cell { + position: relative; + width: 100%; + height: 100%; +} + +.logs-page__calendar-cell .ant-picker-cell-inner { + position: relative; + transition: background 0.18s ease, box-shadow 0.18s ease, color 0.18s ease; +} + +.logs-page__calendar-cell--error .ant-picker-cell-inner { + background: rgba(239, 68, 68, 0.12); + box-shadow: inset 0 0 0 1px rgba(239, 68, 68, 0.16); +} + +.logs-page__calendar-cell--warning .ant-picker-cell-inner { + background: rgba(245, 158, 11, 0.12); + box-shadow: inset 0 0 0 1px rgba(245, 158, 11, 0.16); +} + +.logs-page__calendar-cell--info .ant-picker-cell-inner { + background: rgba(34, 197, 94, 0.12); + box-shadow: inset 0 0 0 1px rgba(34, 197, 94, 0.16); +} + +.logs-page__calendar-cell--debug .ant-picker-cell-inner { + background: rgba(148, 163, 184, 0.12); + box-shadow: inset 0 0 0 1px rgba(148, 163, 184, 0.16); +} + +.logs-page__calendar-cell:hover .ant-picker-cell-inner { + filter: saturate(1.04); +} + +.logs-page__line-limit-customizer { + display: flex; + flex-direction: column; + gap: 8px; + padding: 10px 12px 12px; + border-top: 1px solid rgba(5, 5, 5, 0.06); +} + +@media (max-width: 1440px), (max-height: 900px) { + .logs-page { + gap: 8px; + } + + .logs-page__header-desc { + display: none; + } + + .logs-page__card .ant-card-body { + padding: 12px; + } + + .logs-page__toolbar { + padding: 8px 10px; + gap: 6px; + } + + .logs-page__toolbar-row--primary { + grid-template-columns: minmax(160px, 200px) minmax(180px, 220px) minmax(0, 1fr) var(--logs-filter-toggle-size); + grid-template-areas: + "source level search toggle"; + row-gap: 8px; + } + + .logs-page__source-select, + .logs-page__line-limit-select, + .logs-page__level-select, + .logs-page__search-input { + width: 100%; + min-width: 0; + } + + .logs-page__source-select { + grid-area: source; + } + + .logs-page__level-select { + grid-area: level; + } + + .logs-page__search-input { + grid-area: search; + } + + .logs-page__filter-toggle { + grid-area: toggle; + } + + .logs-page__toolbar-row--secondary { + grid-template-columns: 140px minmax(240px, 280px) minmax(0, 1fr); + gap: 8px; + } + + .logs-page__date-range { + width: 100%; + min-width: 0; + } + + .logs-page__console-shell { + min-height: clamp(340px, 58vh, 760px); + } + + .system-log-console__content { + padding: 16px 112px 16px 16px; + } +} + @media (max-width: 768px) { .dashboard-restart-toolbar__meta { grid-template-columns: 1fr; } + + .logs-page__toolbar { + flex-direction: column; + align-items: stretch; + } + + .logs-page__card .ant-card-body { + padding: 10px; + } + + .logs-page__toolbar-row--primary, + .logs-page__toolbar-row--secondary { + grid-template-columns: 1fr; + align-items: stretch; + } + + .logs-page__source-select, + .logs-page__search-input, + .logs-page__level-select, + .logs-page__date-range, + .logs-page__line-limit-select { + width: 100%; + } + + .logs-page__console-shell { + min-height: clamp(280px, 50vh, 620px); + } } diff --git a/frontend/src/pages/Dashboard/Dashboard.tsx b/frontend/src/pages/Dashboard/Dashboard.tsx index 68774a24..740db16f 100644 --- a/frontend/src/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/pages/Dashboard/Dashboard.tsx @@ -9,6 +9,7 @@ import { WifiOutlined, DisconnectOutlined, ReloadOutlined, + LoadingOutlined, } from '@ant-design/icons' import { Link } from 'react-router-dom' import axios from 'axios' @@ -48,7 +49,7 @@ interface RestartTaskLogs { lines: string[] } -type RestartAction = 'restart-backend' | 'restart-ai-provider' | 'restart-database' | 'restart-system' +type RestartAction = 'restart-backend' | 'restart-frontend' | 'restart-ai-provider' | 'restart-database' | 'restart-system' type RestartStage = 'confirming' | 'waiting_for_shutdown' | 'waiting_for_recovery' | 'recovered' | 'failed' | 'timeout' const RESTART_ACTION_OPTIONS: Array<{ value: RestartAction; label: string; description: string; command: string }> = [ @@ -58,6 +59,12 @@ const RESTART_ACTION_OPTIONS: Array<{ value: RestartAction; label: string; descr description: '只重启后端服务，页面通常会短暂失联后自动恢复。', command: './planet.sh restart -b', }, + { + value: 'restart-frontend', + label: '重启前端', + description: '只重启前端开发服务，页面会短暂不可用，恢复后自动刷新。', + command: './planet.sh restart -f', + }, { value: 'restart-ai-provider', label: '重启 AI Provider', @@ -84,6 +91,11 @@ const RESTART_GUIDE_LINES: Record = { '[ctl] handing restart to detached runner', '[ctl] waiting for backend health recovery', ], + 'restart-frontend': [ + '[ctl] preparing frontend restart task', + '[ctl] handing restart to detached runner', + '[ctl] waiting for frontend entrypoint recovery', + ], 'restart-ai-provider': [ '[ctl] preparing ai provider restart task', '[ctl] handing restart to detached runner', @@ -106,6 +118,9 @@ const RESTART_GUIDE_LINES: Record = { let cachedDashboardStats: Stats | null = null function getRestartConfirmMessage(action: RestartAction): string { + if (action === 'restart-frontend') { + return '将重启前端开发服务，页面会短暂不可用，恢复后会自动刷新。' + } if (action === 'restart-ai-provider') { return '将重启 AI Provider 适配服务，页面通常保持在线，但 AI 分析请求会短暂不可用。' } @@ -123,6 +138,7 @@ function Dashboard() { const [stats, setStats] = useState(cachedDashboardStats) const [loading, setLoading] = useState(cachedDashboardStats === null) const [wsConnected, setWsConnected] = useState(false) + const [wsConnecting, setWsConnecting] = useState(false) const [error, setError] = useState(null) const [restartModalOpen, setRestartModalOpen] = useState(false) const [restartSubmitting, setRestartSubmitting] = useState(false) @@ -166,7 +182,7 @@ function Dashboard() { fetchStats() }, [token, clearAuth]) - const { connected: dashboardSocketConnected } = useWebSocket({ + const { connected: dashboardSocketConnected, connecting: dashboardSocketConnecting } = useWebSocket({ autoConnect: true, autoSubscribe: ['dashboard'], onMessage: (message) => { @@ -180,7 +196,8 @@ function Dashboard() { useEffect(() => { setWsConnected(dashboardSocketConnected) - }, [dashboardSocketConnected]) + setWsConnecting(dashboardSocketConnecting) + }, [dashboardSocketConnected, dashboardSocketConnecting]) const handleRetry = () => { window.location.reload() @@ -215,6 +232,8 @@ function Dashboard() { setRestartMessage( restartAction === 'restart-system' ? '已发送完全重启指令，页面可能暂时失联，恢复后会自动刷新。' + : restartAction === 'restart-frontend' + ? '已发送前端重启指令，正在等待页面入口恢复。' : restartAction === 'restart-ai-provider' ? '已发送 AI Provider 重启指令，正在等待 AI 服务恢复。' : '已发送重启指令，正在等待服务进入重启流程。' @@ -290,14 +309,18 @@ function Dashboard() { // Backend may be temporarily down during restart; handled by health polling below. } - if (restartAction === 'restart-system') { + if (restartAction === 'restart-system' || restartAction === 'restart-frontend') { try { const rootRes = await fetch(`/?restart_probe=${Date.now()}`, { cache: 'no-store' }) if (rootRes.ok) { frontendHealthyStreak += 1 if (sawUnhealthy && frontendHealthyStreak >= 2) { setRestartStage('recovered') - setRestartMessage('系统已恢复，正在刷新页面。') + setRestartMessage( + restartAction === 'restart-frontend' + ? '前端已恢复，正在刷新页面。' + : '系统已恢复，正在刷新页面。' + ) appendLog('[ctl] frontend entrypoint reachable again') window.setTimeout(() => window.location.reload(), 600) return @@ -306,14 +329,22 @@ function Dashboard() { sawUnhealthy = true frontendHealthyStreak = 0 setRestartStage('waiting_for_recovery') - setRestartMessage('系统正在完全重启，正在等待前端恢复访问。') + setRestartMessage( + restartAction === 'restart-frontend' + ? '前端正在重启，正在等待页面入口恢复访问。' + : '系统正在完全重启，正在等待前端恢复访问。' + ) appendLog('[ctl] frontend is temporarily unavailable') } } catch { sawUnhealthy = true frontendHealthyStreak = 0 setRestartStage('waiting_for_recovery') - setRestartMessage('系统正在完全重启，正在等待前端恢复访问。') + setRestartMessage( + restartAction === 'restart-frontend' + ? '前端正在重启，正在等待页面入口恢复访问。' + : '系统正在完全重启，正在等待前端恢复访问。' + ) appendLog('[ctl] frontend is temporarily unavailable') } @@ -378,6 +409,8 @@ function Dashboard() { {wsConnected ? ( } color="success">实时连接 + ) : wsConnecting ? ( + } color="processing">正在连接 ) : ( } color="default">离线 )} diff --git a/frontend/src/pages/DataSources/DataSources.tsx b/frontend/src/pages/DataSources/DataSources.tsx index 4b1812c1..2a38d5b5 100644 --- a/frontend/src/pages/DataSources/DataSources.tsx +++ b/frontend/src/pages/DataSources/DataSources.tsx @@ -1,26 +1,46 @@ -import { useCallback, useEffect, useRef, useState } from 'react' -import { useCollapsedActions } from '../../hooks' -import { TableActions, actionCellProps } from '../../components/TableActions/TableActions' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { - Table, Tag, Space, Button, Form, Input, Select, Progress, Checkbox, message, Modal, - Drawer, Tabs, Empty, Tooltip, Popconfirm, Collapse, InputNumber, Row, Col, Card + Button, + Card, + Checkbox, + Col, + Drawer, + Alert, + Form, + Input, + Modal, + Progress, + Row, + Space, + Table, + Tag, + Tooltip, + Typography, + message, } from 'antd' import { - PlayCircleOutlined, PauseCircleOutlined, PlusOutlined, - EditOutlined, DeleteOutlined, ApiOutlined, - CheckCircleOutlined, CloseCircleOutlined, ExperimentOutlined, - SyncOutlined, ClearOutlined, CopyOutlined + CopyOutlined, + InfoCircleOutlined, + PauseCircleOutlined, + PlayCircleOutlined, + SyncOutlined, } from '@ant-design/icons' import axios, { type AxiosResponse } from 'axios' +import { useNavigate } from 'react-router-dom' + import AppLayout from '../../components/AppLayout/AppLayout' import ScrollbarOverlay from '../../components/Scrollbar/ScrollbarOverlay' import { formatDateTimeZhCN } from '../../utils/datetime' -import { useWebSocket } from '../../hooks/useWebSocket' +import { formatPhaseMetric, getPhaseDisplay, getPhaseSummary } from '../../utils/phaseProgress' + +const { Text } = Typography +const COLLECTION_REFRESH_DELAY_MS = 800 interface BuiltInDataSource { id: number source: string name: string + display_name?: string module: string priority: string frequency: string @@ -30,146 +50,24 @@ interface BuiltInDataSource { last_run: string | null last_run_at?: string | null last_status?: string | null - last_records_processed?: number | null - data_count?: number is_running: boolean task_id: number | null progress: number | null phase?: string | null + phase_progress?: number | null + phase_message?: string | null + phase_current?: number | null + phase_total?: number | null + phase_unit?: string | null records_processed: number | null total_records: number | null + is_free?: boolean + requires_credentials?: boolean + credential_provider?: string | null + credential_status?: string } -interface TaskTrackerState { - task_id: number | null - is_running: boolean - progress: number - phase: string | null - status?: string | null - records_processed?: number | null - total_records?: number | null - error_message?: string | null -} - -interface BulkProgressItem { - task_id: number | null - progress: number - status: string | null - phase: string | null - is_running: boolean -} - -interface BulkProgressBatch { - sourceIds: number[] - items: Record -} - -type TriggerDatasourceConflict = { - reason?: string - message?: string - progress?: number | null - phase?: string | null - records_processed?: number | null - total_records?: number | null -} - -type TriggerDatasourceResult = - | { ok: true; response: AxiosResponse } - | { ok: false; status: number; detail?: string | TriggerDatasourceConflict } - -type DatasourceTaskStatus = { - is_running: boolean - task_id?: number | null - progress?: number | null - phase?: string | null - records_processed?: number | null - total_records?: number | null - status?: string | null -} - -const phaseProgressRanges: Record = { - queued: [0, 0], - fetching: [0, 30], - transforming: [30, 50], - saving: [50, 100], - completed: [100, 100], - success: [100, 100], - failed: [0, 0], - cancelled: [0, 0], -} - -function mapPhaseProgressToOverall( - phase?: string | null, - progress?: number | null, - status?: string | null, -): number { - if (status && status !== 'running') { - return status === 'success' ? 100 : 0 - } - - const normalizedPhase = phase || 'queued' - const [start, end] = phaseProgressRanges[normalizedPhase] ?? [0, 100] - if (start === end) return start - - const boundedProgress = Math.max(0, Math.min(100, progress ?? 0)) - return start + ((end - start) * boundedProgress) / 100 -} - -function finalizeBulkProgressBatch(batch: BulkProgressBatch | null): BulkProgressBatch | null { - if (!batch || batch.sourceIds.length === 0) { - return null - } - return batch -} - -function resolveTerminalBatchItem( - sourceId: number, - batch: BulkProgressBatch, - builtInSources: BuiltInDataSource[], - taskProgress: Record, -): BulkProgressItem | null { - const currentItem = batch.items[sourceId] - const source = builtInSources.find((item) => item.id === sourceId) - const trackedTask = taskProgress[sourceId] - - const isRunning = trackedTask?.is_running ?? source?.is_running ?? currentItem?.is_running ?? false - if (isRunning) { - return null - } - - const status = trackedTask?.status ?? source?.last_status ?? currentItem?.status ?? null - if (!status || status === 'running') { - return null - } - - return { - task_id: trackedTask?.task_id ?? currentItem?.task_id ?? source?.task_id ?? null, - progress: - status === 'success' - ? 100 - : trackedTask?.progress ?? source?.progress ?? currentItem?.progress ?? 0, - is_running: false, - phase: trackedTask?.phase ?? source?.phase ?? currentItem?.phase ?? null, - status, - } -} - -interface WebSocketTaskMessage { - type: string - channel?: string - payload?: { - datasource_id?: number - task_id?: number | null - progress?: number | null - phase?: string | null - status?: string | null - records_processed?: number | null - total_records?: number | null - error_message?: string | null - } -} - -interface CustomDataSource { +interface CustomDataSourceOverride { id: number name: string description: string | null @@ -194,464 +92,206 @@ interface EditableDataSourceConfig { is_active?: boolean } -interface ViewDataSource { +interface UnifiedDataSource { + key: string id: number name: string - description: string | null - source_type: string - endpoint: string - auth_type: string + display_name: string + source: string + module?: string + priority?: string + frequency?: string + endpoint?: string + is_active: boolean + collector_class?: string + source_type?: string + auth_type?: string + last_run_at?: string | null + last_status?: string | null + is_running?: boolean + progress?: number | null + phase?: string | null + phase_progress?: number | null + phase_message?: string | null + phase_current?: number | null + phase_total?: number | null + phase_unit?: string | null + task_id?: number | null + created_at?: string + updated_at?: string | null + description?: string | null + headers?: Record + config?: Record + is_free?: boolean + requires_credentials?: boolean + credential_provider?: string | null + credential_status?: string +} + +interface ViewDataSource extends UnifiedDataSource { headers: Record config: Record - collector_class: string - module: string - priority: string - frequency: string +} + +type TriggerDatasourceConflict = { + reason?: string + message?: string + progress?: number | null + phase?: string | null + phase_progress?: number | null + phase_message?: string | null + phase_current?: number | null + phase_total?: number | null + phase_unit?: string | null + records_processed?: number | null + total_records?: number | null +} + +type TriggerDatasourceResult = + | { ok: true; response: AxiosResponse } + | { ok: false; status: number; detail?: string | TriggerDatasourceConflict } + +type DatasourceTaskStatus = { + is_running: boolean + task_id?: number | null + progress?: number | null + phase?: string | null + phase_progress?: number | null + phase_message?: string | null + phase_current?: number | null + phase_total?: number | null + phase_unit?: string | null + records_processed?: number | null + total_records?: number | null + status?: string | null +} + +function normalizeBuiltin(source: BuiltInDataSource): UnifiedDataSource { + return { + key: `builtin:${source.id}`, + id: source.id, + name: source.name, + display_name: source.display_name || source.name, + source: source.source, + module: source.module, + priority: source.priority, + frequency: source.frequency, + endpoint: source.endpoint, + is_active: source.is_active, + collector_class: source.collector_class, + last_run_at: source.last_run_at || source.last_run, + last_status: source.last_status, + is_running: source.is_running, + progress: source.progress, + phase: source.phase, + phase_progress: source.phase_progress, + phase_message: source.phase_message, + phase_current: source.phase_current, + phase_total: source.phase_total, + phase_unit: source.phase_unit, + task_id: source.task_id, + is_free: source.is_free, + requires_credentials: source.requires_credentials, + credential_provider: source.credential_provider, + credential_status: source.credential_status, + headers: {}, + config: {}, + } } function DataSources() { const [messageApi, contextHolder] = message.useMessage() + const navigate = useNavigate() const [modal, modalContextHolder] = Modal.useModal() - const [activeTab, setActiveTab] = useState('builtin') const [builtInSources, setBuiltInSources] = useState([]) - const [customSources, setCustomSources] = useState([]) + const [customOverrides, setCustomOverrides] = useState([]) const [loading, setLoading] = useState(false) - const [drawerVisible, setDrawerVisible] = useState(false) - const [viewDrawerVisible, setViewDrawerVisible] = useState(false) - const [editingConfig, setEditingConfig] = useState(null) - const [builtinEditingSource, setBuiltinEditingSource] = useState(null) - const [viewingSource, setViewingSource] = useState(null) - const [recordCount, setRecordCount] = useState(0) - const [testing, setTesting] = useState(false) const [triggerAllLoading, setTriggerAllLoading] = useState(false) const [forceTriggerAll, setForceTriggerAll] = useState(false) - const [testResult, setTestResult] = useState(null) - const builtinTableRegionRef = useRef(null) - const customTableRegionRef = useRef(null) - const [builtinTableHeight, setBuiltinTableHeight] = useState(360) - const [customTableHeight, setCustomTableHeight] = useState(360) - const [builtinActionsCollapsed, builtinContainerRef] = useCollapsedActions() - const [customActionsCollapsed, customContainerRef] = useCollapsedActions() - const [form] = Form.useForm() + const [viewDrawerVisible, setViewDrawerVisible] = useState(false) + const [runningTasksVisible, setRunningTasksVisible] = useState(false) + const [viewingSource, setViewingSource] = useState(null) + const [recordCount, setRecordCount] = useState(null) + const [tableHeight, setTableHeight] = useState(360) + const tableRegionRef = useRef(null) - const headersMapToList = useCallback((headers?: Record | null) => { - return Object.entries(headers || {}) - .filter(([key, value]) => key && value !== undefined && value !== null && String(value).trim() !== '') - .map(([key, value]) => ({ key, value })) - }, []) + const allSources = useMemo(() => builtInSources.map(normalizeBuiltin), [builtInSources]) - const headersListToMap = useCallback((headers?: Array<{ key?: string; value?: string }> | Record) => { - if (!headers) return {} - if (!Array.isArray(headers)) return headers - - return headers.reduce>((acc, item) => { - const key = item?.key?.trim() - const value = item?.value?.trim() - if (!key || value === undefined) return acc - acc[key] = value - return acc - }, {}) - }, []) - - const applyConfigToForm = useCallback((config?: Partial | null) => { - form.setFieldsValue({ - name: config?.name || '', - description: config?.description || '', - source_type: config?.source_type || 'http', - endpoint: config?.endpoint || '', - auth_type: config?.auth_type || 'none', - auth_config: config?.auth_config || {}, - headers: headersMapToList(config?.headers || {}), - config: config?.config || { timeout: 30, retry: 3 }, - }) - }, [form, headersMapToList]) - - const loadConfigDetail = useCallback(async (configId: number) => { - const res = await axios.get(`/api/v1/datasources/configs/${configId}`) - return res.data - }, []) - - const createDefaultConfigDraft = useCallback((overrides?: Partial) => ({ - source_type: 'http', - auth_type: 'none', - headers: {}, - config: { timeout: 30, retry: 3 }, - ...overrides, - }), []) - - const getBuiltinOverrideDescription = useCallback( - (source?: Pick | null) => - source ? `Built-in datasource override for ${source.name}` : undefined, - [], - ) - - const createFormPayload = useCallback((values: any) => ({ - ...values, - name: builtinEditingSource ? builtinEditingSource.source : values.name, - description: - values.description || - getBuiltinOverrideDescription(builtinEditingSource), - source_type: builtinEditingSource ? 'http' : values.source_type, - headers: headersListToMap(values.headers), - }), [builtinEditingSource, getBuiltinOverrideDescription, headersListToMap]) - - const closeDrawerAfterLoadError = useCallback(( - errorMessage: string, - options?: { clearBuiltin?: boolean; clearEditingConfig?: boolean }, - ) => { - messageApi.error(errorMessage) - setDrawerVisible(false) - if (options?.clearBuiltin) { - setBuiltinEditingSource(null) - } - if (options?.clearEditingConfig) { - setEditingConfig(null) - } - }, [messageApi]) + const activeBuiltInCount = builtInSources.filter((source) => source.is_active).length + const runningBuiltInSources = builtInSources.filter((source) => source.is_running) + const runningBuiltInCount = runningBuiltInSources.length + const aggregateProgress = runningBuiltInCount > 0 + ? Math.round( + builtInSources + .filter((source) => source.is_running) + .reduce((sum, source) => sum + (source.progress || 0), 0) / runningBuiltInCount, + ) + : 0 const fetchData = useCallback(async () => { setLoading(true) try { const [builtinRes, customRes] = await Promise.all([ axios.get('/api/v1/datasources'), - axios.get('/api/v1/datasources/configs') + axios.get('/api/v1/datasources/configs'), ]) setBuiltInSources(builtinRes.data.data || []) - setCustomSources(customRes.data.data || []) + setCustomOverrides(customRes.data.data || []) } catch (error) { console.error('Failed to fetch data:', error) + messageApi.error('获取数据源列表失败') } finally { setLoading(false) } - }, []) - - const [taskProgress, setTaskProgress] = useState>({}) - const [bulkProgressBatch, setBulkProgressBatch] = useState(null) - const activeBuiltInCount = builtInSources.filter((source) => source.is_active).length - const runningBuiltInCount = builtInSources.filter((source) => { - const trackedTask = taskProgress[source.id] - return trackedTask?.is_running || source.is_running - }).length - const runningBuiltInSources = builtInSources.filter((source) => { - const trackedTask = taskProgress[source.id] - return trackedTask?.is_running || source.is_running - }) - const aggregateProgress = bulkProgressBatch && bulkProgressBatch.sourceIds.length > 0 - ? Math.round( - bulkProgressBatch.sourceIds.reduce((sum, sourceId) => { - const item = bulkProgressBatch.items[sourceId] - if (!item) return sum - return sum + mapPhaseProgressToOverall(item.phase, item.progress, item.status) - }, 0) / bulkProgressBatch.sourceIds.length - ) - : runningBuiltInSources.length > 0 - ? Math.round( - runningBuiltInSources.reduce((sum, source) => { - const trackedTask = taskProgress[source.id] - return sum + mapPhaseProgressToOverall( - trackedTask?.phase ?? source.phase ?? 'queued', - trackedTask?.progress ?? source.progress ?? 0, - trackedTask?.status ?? (trackedTask?.is_running || source.is_running ? 'running' : source.last_status ?? null), - ) - }, 0) / runningBuiltInSources.length - ) - : 0 - - const bulkBatchRunningCount = bulkProgressBatch - ? bulkProgressBatch.sourceIds.filter((sourceId) => bulkProgressBatch.items[sourceId]?.is_running).length - : 0 - - const bulkBatchSuccessCount = bulkProgressBatch - ? bulkProgressBatch.sourceIds.filter((sourceId) => { - const status = bulkProgressBatch.items[sourceId]?.status - return status === 'success' - }).length - : 0 - - const bulkBatchFailedCount = bulkProgressBatch - ? bulkProgressBatch.sourceIds.filter((sourceId) => { - const status = bulkProgressBatch.items[sourceId]?.status - return Boolean(status && status !== 'running' && status !== 'success') - }).length - : 0 - - const handleTaskSocketMessage = useCallback((message: WebSocketTaskMessage) => { - if (message.type !== 'data_frame' || message.channel !== 'datasource_tasks' || !message.payload?.datasource_id) { - return - } - - const payload = message.payload - const sourceId = payload.datasource_id - if (typeof sourceId !== 'number') { - return - } - const nextState: TaskTrackerState = { - task_id: payload.task_id ?? null, - progress: payload.progress ?? 0, - is_running: payload.status === 'running', - phase: payload.phase ?? null, - status: payload.status ?? null, - records_processed: payload.records_processed ?? null, - total_records: payload.total_records ?? null, - error_message: payload.error_message ?? null, - } - - setTaskProgress((prev) => { - const next = { - ...prev, - [sourceId]: nextState, - } - - if (!nextState.is_running && nextState.status !== 'running') { - delete next[sourceId] - } - - return next - }) - - setBulkProgressBatch((prev) => { - if (!prev || !prev.sourceIds.includes(sourceId)) { - return prev - } - - const nextItems = { - ...prev.items, - [sourceId]: { - task_id: payload.task_id ?? prev.items[sourceId]?.task_id ?? null, - progress: payload.progress ?? prev.items[sourceId]?.progress ?? 0, - is_running: payload.status === 'running', - phase: payload.phase ?? prev.items[sourceId]?.phase ?? null, - status: payload.status ?? prev.items[sourceId]?.status ?? null, - }, - } - - return finalizeBulkProgressBatch({ - ...prev, - items: nextItems, - }) - }) - - if (payload.status && payload.status !== 'running') { - void fetchData() - } - }, [fetchData]) - - const { connected: taskSocketConnected } = useWebSocket({ - autoConnect: true, - autoSubscribe: ['datasource_tasks'], - onMessage: handleTaskSocketMessage, - }) + }, [messageApi]) useEffect(() => { - fetchData() + void fetchData() }, [fetchData]) useEffect(() => { - const updateHeights = () => { - const builtinRegionHeight = builtinTableRegionRef.current?.offsetHeight || 0 - const customRegionHeight = customTableRegionRef.current?.offsetHeight || 0 - - setBuiltinTableHeight(Math.max(220, builtinRegionHeight - 56)) - setCustomTableHeight(Math.max(220, customRegionHeight - 56)) + const updateHeight = () => { + setTableHeight(Math.max(260, (tableRegionRef.current?.offsetHeight || 0) - 56)) } - - updateHeights() - - if (typeof ResizeObserver === 'undefined') { - return undefined - } - - const observer = new ResizeObserver(updateHeights) - if (builtinTableRegionRef.current) observer.observe(builtinTableRegionRef.current) - if (customTableRegionRef.current) observer.observe(customTableRegionRef.current) - + updateHeight() + if (typeof ResizeObserver === 'undefined') return undefined + const observer = new ResizeObserver(updateHeight) + if (tableRegionRef.current) observer.observe(tableRegionRef.current) return () => observer.disconnect() - }, [activeTab, builtInSources.length, customSources.length]) - - useEffect(() => { - if (taskSocketConnected) return - - const trackedSources = builtInSources.filter((source) => { - const trackedTask = taskProgress[source.id] - return Boolean((trackedTask?.task_id ?? source.task_id) && (trackedTask?.is_running ?? source.is_running)) - }) - - if (trackedSources.length === 0) return - - const interval = setInterval(async () => { - const updates: Record = {} - - await Promise.all( - trackedSources.map(async (source) => { - const trackedTaskId = taskProgress[source.id]?.task_id ?? source.task_id - if (!trackedTaskId) return - - try { - const res = await axios.get(`/api/v1/datasources/${source.id}/task-status`, { - params: { task_id: trackedTaskId }, - }) - updates[source.id] = { - task_id: res.data.task_id ?? trackedTaskId, - progress: res.data.progress || 0, - is_running: !!res.data.is_running, - phase: res.data.phase || null, - status: res.data.status || null, - records_processed: res.data.records_processed, - total_records: res.data.total_records, - } - } catch { - updates[source.id] = { - task_id: trackedTaskId, - progress: 0, - is_running: false, - phase: 'failed', - status: 'failed', - } - } - }) - ) - - setTaskProgress((prev) => { - const next = { ...prev, ...updates } - for (const [sourceId, state] of Object.entries(updates)) { - if (!state.is_running && state.status !== 'running') { - delete next[Number(sourceId)] - } - } - return next - }) - - setBulkProgressBatch((prev) => { - if (!prev) return prev - - const nextItems = { ...prev.items } - for (const [sourceId, state] of Object.entries(updates)) { - const numericSourceId = Number(sourceId) - if (!prev.sourceIds.includes(numericSourceId)) continue - nextItems[numericSourceId] = { - task_id: state.task_id, - progress: state.progress, - is_running: state.is_running, - phase: state.phase ?? null, - status: state.status ?? null, - } - } - - return finalizeBulkProgressBatch({ - ...prev, - items: nextItems, - }) - }) - - if (Object.values(updates).some((state) => !state.is_running)) { - fetchData() - } - }, 2000) - - return () => clearInterval(interval) - }, [builtInSources, taskProgress, taskSocketConnected, fetchData]) - - useEffect(() => { - if (!bulkProgressBatch) return - - let changed = false - const nextItems = { ...bulkProgressBatch.items } - - for (const sourceId of bulkProgressBatch.sourceIds) { - const nextItem = resolveTerminalBatchItem(sourceId, bulkProgressBatch, builtInSources, taskProgress) - if (!nextItem) continue - - const previousItem = bulkProgressBatch.items[sourceId] - if ( - previousItem?.status === nextItem.status && - previousItem?.is_running === nextItem.is_running && - previousItem?.progress === nextItem.progress && - previousItem?.phase === nextItem.phase - ) { - continue - } - - nextItems[sourceId] = nextItem - changed = true - } - - if (!changed) return - - setBulkProgressBatch((prev) => { - if (!prev) return prev - return finalizeBulkProgressBatch({ - ...prev, - items: nextItems, - }) - }) - }, [bulkProgressBatch, builtInSources, taskProgress]) - - const triggerDatasource = async (id: number, options?: { force?: boolean }) => { - const force = options?.force ?? false - const res = await axios.post(`/api/v1/datasources/${id}/trigger`, null, { - params: { force }, - validateStatus: (status) => status < 500, - }) - - if (res.status >= 400) { - return { - ok: false, - status: res.status, - detail: res.data?.detail, - } satisfies TriggerDatasourceResult - } - - if (res.data.task_id) { - setTaskProgress(prev => ({ - ...prev, - [id]: { - task_id: res.data.task_id, - progress: 0, - is_running: true, - phase: 'queued', - status: 'running', - }, - })) - } else { - window.setTimeout(() => { - fetchData() - }, 800) - } - - fetchData() - return { - ok: true, - response: res, - } satisfies TriggerDatasourceResult - } + }, [allSources.length]) const fetchDatasourceTaskStatus = async (id: number) => { const res = await axios.get(`/api/v1/datasources/${id}/task-status`) return res.data } - const confirmForceTrigger = (id: number, taskInfo?: { - progress?: number | null - phase?: string | null - records_processed?: number | null - total_records?: number | null - message?: string - }) => { - const progressText = typeof taskInfo?.progress === 'number' ? `${Math.round(taskInfo.progress)}%` : '未知' - const phaseText = taskInfo?.phase || 'running' - const processedText = typeof taskInfo?.records_processed === 'number' - ? `${taskInfo.records_processed}${typeof taskInfo?.total_records === 'number' && taskInfo.total_records > 0 ? ` / ${taskInfo.total_records}` : ''}` - : '未知' + const triggerDatasource = async (id: number, options?: { force?: boolean }) => { + const res = await axios.post(`/api/v1/datasources/${id}/trigger`, null, { + params: { force: options?.force ?? false }, + validateStatus: (status) => status < 500, + }) + if (res.status >= 400) { + return { ok: false, status: res.status, detail: res.data?.detail } satisfies TriggerDatasourceResult + } + + if (res.data.task_id) { + void fetchData() + } else { + window.setTimeout(fetchData, COLLECTION_REFRESH_DELAY_MS) + } + return { ok: true, response: res } satisfies TriggerDatasourceResult + } + + const confirmForceTrigger = (id: number, taskInfo?: TriggerDatasourceConflict) => { modal.confirm({ title: '当前任务未完成', content: (

{taskInfo?.message || '当前采集任务仍在运行，重新触发会丢失本次未完成进度。'}

-

当前阶段: {phaseText}

-

当前进度: {progressText}

-

已处理记录: {processedText}

-

确认后会强制取消当前采集，并回滚未完成写入，然后重新开始采集。

+

当前阶段: {getPhaseDisplay(taskInfo || {})}

+

当前进度: {typeof taskInfo?.progress === 'number' ? `${Math.round(taskInfo.progress)}%` : '未知'}

+

确认后会强制取消当前采集，并重新开始采集。

), okText: '强制重新采集', @@ -660,56 +300,33 @@ function DataSources() { onOk: async () => { const result = await triggerDatasource(id, { force: true }) if (result.ok) { - messageApi.success('已强制重新触发，未完成采集将回滚') + messageApi.success('已强制重新触发') return } - const detail = result.detail messageApi.error(typeof detail === 'string' ? detail : detail?.message || '强制重新采集失败') }, }) } - const triggerDatasourceWithPrecheck = async ( - id: number, - options?: { - successMessage: string - onSuccess?: () => void - errorMessage?: string - }, - ) => { + const triggerDatasourceWithPrecheck = async (id: number) => { const taskStatus = await fetchDatasourceTaskStatus(id) if (taskStatus.is_running) { - confirmForceTrigger(id, { - message: '当前采集任务尚未完成，重新触发会丢失本次未完成进度。是否强制重新采集？', - progress: taskStatus.progress ?? null, - phase: taskStatus.phase ?? null, - records_processed: taskStatus.records_processed ?? null, - total_records: taskStatus.total_records ?? null, - }) + confirmForceTrigger(id, taskStatus) return } const result = await triggerDatasource(id) if (result.ok) { - messageApi.success(options?.successMessage || '任务已触发') - options?.onSuccess?.() + messageApi.success('任务已触发') return } - const detail = result.detail if (result.status === 409 && typeof detail === 'object' && detail?.reason === 'running_task_in_progress') { confirmForceTrigger(id, detail) return } - messageApi.error(typeof detail === 'string' ? detail : detail?.message || options?.errorMessage || '触发失败') - } - - const handleTrigger = async (id: number) => { - await triggerDatasourceWithPrecheck(id, { - successMessage: '任务已触发', - errorMessage: '触发失败', - }) + messageApi.error(typeof detail === 'string' ? detail : detail?.message || '触发失败') } const handleTriggerAll = async () => { @@ -721,51 +338,12 @@ function DataSources() { const triggered = res.data.triggered || [] const skipped = res.data.skipped || [] const failed = res.data.failed || [] - const skippedInWindow = skipped.filter((item: { reason?: string }) => item.reason === 'within_frequency_window') - const skippedOther = skipped.filter((item: { reason?: string }) => item.reason !== 'within_frequency_window') - - if (triggered.length > 0) { - setBulkProgressBatch({ - sourceIds: triggered.map((item: { id: number }) => item.id), - items: Object.fromEntries( - triggered.map((item: { id: number; task_id?: number | null }) => [ - item.id, - { - task_id: item.task_id ?? null, - progress: 0, - is_running: true, - phase: 'queued', - status: 'running', - } satisfies BulkProgressItem, - ]) - ), - }) - - setTaskProgress((prev) => { - const next = { ...prev } - for (const item of triggered) { - if (!item.task_id) continue - next[item.id] = { - task_id: item.task_id, - progress: 0, - is_running: true, - phase: 'queued', - status: 'running', - } - } - return next - }) - } - - const summaryParts = [ + messageApi.success([ `已触发 ${triggered.length} 个`, - skippedInWindow.length > 0 ? `周期内跳过 ${skippedInWindow.length} 个` : null, - skippedOther.length > 0 ? `其他跳过 ${skippedOther.length} 个` : null, - failed.length > 0 ? `失败 ${failed.length} 个` : null, - ].filter(Boolean) - - messageApi.success(summaryParts.join('，')) - fetchData() + skipped.length ? `跳过 ${skipped.length} 个` : null, + failed.length ? `失败 ${failed.length} 个` : null, + ].filter(Boolean).join('，')) + void fetchData() } catch (error: unknown) { const err = error as { response?: { data?: { detail?: string } } } messageApi.error(err.response?.data?.detail || '全触发失败') @@ -779,356 +357,124 @@ function DataSources() { try { await axios.post(`/api/v1/datasources/${id}/${endpoint}`) messageApi.success(`${current ? '已禁用' : '已启用'}`) - fetchData() + void fetchData() } catch (error: unknown) { const err = error as { response?: { data?: { detail?: string } } } messageApi.error(err.response?.data?.detail || '操作失败') } } - const handleClearDataFromDrawer = async () => { - if (!viewingSource) return + const handleViewSource = async (source: UnifiedDataSource) => { try { - const res = await axios.delete(`/api/v1/datasources/${viewingSource.id}/data`) - messageApi.success(res.data.message || '数据已删除') - setViewDrawerVisible(false) - fetchData() - } catch (error: unknown) { - const err = error as { response?: { data?: { detail?: string } } } - messageApi.error(err.response?.data?.detail || '删除数据失败') - } - } - - const handleViewSource = async (source: BuiltInDataSource) => { - try { - const existingOverride = customSources.find((item) => item.name === source.source) - const [res, statsRes, overrideDetail] = await Promise.all([ - axios.get(`/api/v1/datasources/${source.id}`), - axios.get(`/api/v1/datasources/${source.id}/stats`), - existingOverride ? loadConfigDetail(existingOverride.id) : Promise.resolve(null), - ]) - const data = res.data - setViewingSource({ - id: data.id, - name: data.name, - description: null, - source_type: data.collector_class, - endpoint: overrideDetail?.endpoint || data.endpoint || '', - auth_type: overrideDetail?.auth_type || 'none', - headers: overrideDetail?.headers || {}, - config: overrideDetail?.config || {}, - collector_class: data.collector_class, - module: data.module, - priority: data.priority, - frequency: data.frequency, - }) - setRecordCount(statsRes.data.total_records || 0) + { + const override = customOverrides.find((item) => item.name === source.source) + const [detailRes, statsRes, overrideDetail] = await Promise.all([ + axios.get(`/api/v1/datasources/${source.id}`), + axios.get(`/api/v1/datasources/${source.id}/stats`), + override ? axios.get(`/api/v1/datasources/configs/${override.id}`).then((res) => res.data) : Promise.resolve(null), + ]) + const data = detailRes.data + setViewingSource({ + ...source, + name: data.name, + display_name: data.display_name || source.display_name, + endpoint: overrideDetail?.endpoint || data.endpoint || source.endpoint || '', + auth_type: overrideDetail?.auth_type || 'none', + headers: overrideDetail?.headers || {}, + config: overrideDetail?.config || {}, + collector_class: data.collector_class, + module: data.module, + priority: data.priority, + frequency: data.frequency, + is_free: data.is_free, + requires_credentials: data.requires_credentials, + credential_provider: data.credential_provider, + credential_status: data.credential_status, + }) + setRecordCount(statsRes.data.total_records || 0) + } setViewDrawerVisible(true) } catch (error) { + console.error(error) messageApi.error('获取数据源信息失败') } } - const handleUpdateSource = async () => { - if (!viewingSource) return - await triggerDatasourceWithPrecheck(viewingSource.id, { - successMessage: '已触发更新', - errorMessage: '更新失败', - onSuccess: () => { - setViewDrawerVisible(false) - }, - }) - } - - const handleTest = async () => { - try { - const values = await form.validateFields() - setTesting(true) - setTestResult(null) - const payload = createFormPayload(values) - const res = await axios.post('/api/v1/datasources/configs/test', payload) - setTestResult(res.data) - if (res.data.success) { - messageApi.success('连接测试成功') - } else { - messageApi.error('连接测试失败') - } - } catch (error: unknown) { - const err = error as { response?: { data?: { detail?: string; message?: string } } } - messageApi.error(err.response?.data?.message || err.response?.data?.detail || '测试失败') - } finally { - setTesting(false) - } - } - - const handleSave = async () => { - try { - const values = await form.validateFields() - const payload = createFormPayload(values) - if (editingConfig) { - await axios.put(`/api/v1/datasources/configs/${editingConfig.id}`, payload) - messageApi.success('配置已更新') - } else { - await axios.post('/api/v1/datasources/configs', payload) - messageApi.success('配置已创建') - } - setDrawerVisible(false) - form.resetFields() - setEditingConfig(null) - setBuiltinEditingSource(null) - setTestResult(null) - fetchData() - } catch (error: unknown) { - const err = error as { response?: { data?: { detail?: string; message?: string } } } - messageApi.error(err.response?.data?.message || err.response?.data?.detail || '保存失败') - } - } - - const handleDelete = async (id: number) => { - try { - await axios.delete(`/api/v1/datasources/configs/${id}`) - messageApi.success('配置已删除') - fetchData() - } catch (error: unknown) { - const err = error as { response?: { data?: { detail?: string } } } - messageApi.error(err.response?.data?.detail || '删除失败') - } - } - - const handleResetBuiltinOverride = async () => { - if (!builtinEditingSource || !editingConfig) return - try { - await axios.delete(`/api/v1/datasources/configs/${editingConfig.id}`) - messageApi.success(`已恢复 ${builtinEditingSource.name} 的默认配置`) - setDrawerVisible(false) - form.resetFields() - setEditingConfig(null) - setBuiltinEditingSource(null) - setTestResult(null) - fetchData() - } catch (error: unknown) { - const err = error as { response?: { data?: { detail?: string } } } - messageApi.error(err.response?.data?.detail || '恢复默认失败') - } - } - - const handleToggleCustom = async (id: number, current: boolean) => { - try { - await axios.put(`/api/v1/datasources/configs/${id}`, { is_active: !current }) - messageApi.success(`${current ? '已禁用' : '已启用'}`) - fetchData() - } catch (error: unknown) { - const err = error as { response?: { data?: { detail?: string } } } - messageApi.error(err.response?.data?.detail || '操作失败') - } - } - - const openDrawer = async (config?: CustomDataSource) => { - setBuiltinEditingSource(null) - setEditingConfig(config || null) - setTestResult(null) - setDrawerVisible(true) - - if (config) { - try { - const detail = await loadConfigDetail(config.id) - applyConfigToForm(detail) - } catch { - closeDrawerAfterLoadError('获取配置详情失败', { clearEditingConfig: true }) - } - return - } - - form.resetFields() - applyConfigToForm(createDefaultConfigDraft()) - } - - const openBuiltinConfigDrawer = async (source: BuiltInDataSource) => { - setBuiltinEditingSource(source) - setTestResult(null) - setDrawerVisible(true) - - const existingOverride = customSources.find((item) => item.name === source.source) - setEditingConfig(existingOverride || null) - - if (existingOverride) { - try { - const detail = await loadConfigDetail(existingOverride.id) - applyConfigToForm(detail) - } catch { - closeDrawerAfterLoadError('获取内置数据源配置失败', { - clearBuiltin: true, - clearEditingConfig: true, - }) - } - return - } - - form.resetFields() - applyConfigToForm(createDefaultConfigDraft({ - name: source.source, - description: getBuiltinOverrideDescription(source), - endpoint: source.endpoint || '', - })) - } - const handleCopyLink = async (value: string, successText: string) => { try { - if (navigator.clipboard?.writeText) { - await navigator.clipboard.writeText(value) - } else { - const textArea = document.createElement('textarea') - textArea.value = value - textArea.style.position = 'fixed' - textArea.style.opacity = '0' - document.body.appendChild(textArea) - textArea.focus() - textArea.select() - document.execCommand('copy') - document.body.removeChild(textArea) - } + await navigator.clipboard.writeText(value) messageApi.success(successText) } catch { messageApi.error('复制失败，请手动复制') } } - const builtinColumns = [ - { title: 'ID', dataIndex: 'id', key: 'id', width: 60, fixed: 'left' as const }, + const columns = [ { title: '名称', - dataIndex: 'name', + dataIndex: 'display_name', key: 'name', - width: 180, + width: 260, ellipsis: true, - render: (name: string, record: BuiltInDataSource) => ( - + render: (_: string, record: UnifiedDataSource) => ( + + + {record.source} + ), }, - { title: '模块', dataIndex: 'module', key: 'module', width: 80 }, { - title: '优先级', - dataIndex: 'priority', - key: 'priority', - width: 80, - render: (p: string) => {p}, + title: '类型', + key: 'kind', + width: 100, + render: () => 内置, + }, + { + title: '层级/类型', + key: 'module', + width: 120, + render: (_: unknown, record: UnifiedDataSource) => {record.module}, + }, + { + title: '频率', + dataIndex: 'frequency', + key: 'frequency', + width: 90, + render: (value: string | undefined) => value || '-', }, - { title: '频率', dataIndex: 'frequency', key: 'frequency', width: 80 }, { title: '最近采集', - dataIndex: 'last_run', - key: 'last_run', + dataIndex: 'last_run_at', + key: 'last_run_at', width: 180, - render: (_: string | null, record: BuiltInDataSource) => { - const label = formatDateTimeZhCN(record.last_run_at || record.last_run) - if (!label || label === '-') return '-' - if ((record.data_count || 0) === 0 && record.last_status === 'success') { - return `${label} (0条)` - } - return label - }, + render: (value: string | null | undefined) => formatDateTimeZhCN(value) || '-', }, { title: '状态', - dataIndex: 'is_active', - key: 'is_active', + key: 'status', width: 180, - render: (_: unknown, record: BuiltInDataSource) => { - const taskState = taskProgress[record.id] - const isTaskRunning = taskState?.is_running || record.is_running - - const phaseLabelMap: Record = { - queued: '排队中', - fetching: '抓取中', - transforming: '处理中', - saving: '保存中', - completed: '已完成', - failed: '失败', - } - - if (isTaskRunning) { - const pct = taskState?.progress ?? record.progress ?? 0 - const phase = taskState?.phase || record.phase || 'queued' + render: (_: unknown, record: UnifiedDataSource) => { + if (record.is_running) { return ( - - - {phaseLabelMap[phase] || phase} - {pct > 0 ? ` ${Math.round(pct)}%` : ''} - - + + {getPhaseSummary(record)} + ) } - const lastStatusColor = - record.last_status === 'success' - ? 'success' - : record.last_status === 'failed' - ? 'error' - : 'default' - - return ( - - {record.last_status ? ( - - {record.last_status === 'success' - ? '采集成功' - : record.last_status === 'failed' - ? '采集失败' - : record.last_status} - - ) : null} - - ) + if (!record.last_status) return 未执行 + return {record.last_status} }, }, { title: '操作', key: 'action', fixed: 'right' as const, - width: builtinActionsCollapsed ? 40 : 228, - onCell: () => actionCellProps, - render: (_: unknown, record: BuiltInDataSource) => ( - , - onClick: () => { void openBuiltinConfigDrawer(record) }, - }, - { - key: 'trigger', - label: '触发', - icon: , - disabled: !record.is_active, - onClick: () => handleTrigger(record.id), - }, - { - key: 'toggle', - label: record.is_active ? '禁用' : '启用', - icon: record.is_active ? : , - danger: record.is_active, - onClick: () => handleToggle(record.id, record.is_active), - }, - ]} - > - - - - ), - }, - ] - - const customColumns = [ - { title: 'ID', dataIndex: 'id', key: 'id', width: 60, fixed: 'left' as const }, - { title: '名称', dataIndex: 'name', key: 'name', width: 150, ellipsis: true }, - { title: '类型', dataIndex: 'source_type', key: 'source_type', width: 100 }, - { - title: 'API链接', - dataIndex: 'endpoint', - key: 'endpoint', - width: 280, - ellipsis: true, - render: (endpoint: string) => ( - endpoint ? ( - - - {endpoint} - - - ) : '-' - ), - }, - { - title: '状态', - dataIndex: 'is_active', - key: 'is_active', - width: 80, - render: (active: boolean) => ( - {active ? '启用' : '禁用'} - ), - }, - { title: '创建时间', dataIndex: 'created_at', key: 'created_at', width: 160 }, - { - title: '操作', - key: 'action', - fixed: 'right' as const, - width: customActionsCollapsed ? 40 : 228, - onCell: () => actionCellProps, - render: (_: unknown, record: CustomDataSource) => ( - , - onClick: () => { void openDrawer(record) }, - }, - { - key: 'toggle', - label: record.is_active ? '禁用' : '启用', - icon: record.is_active ? : , - danger: record.is_active, - onClick: () => handleToggleCustom(record.id, record.is_active), - }, - { type: 'divider' }, - { - key: 'delete', - label: '删除', - icon: , - danger: true, - onClick: () => { - Modal.confirm({ - title: '确定删除此配置?', - onOk: () => handleDelete(record.id), - }) - }, - }, - ]} - > - - - handleDelete(record.id)}> - - - - ), - }, - ] - - const tabItems = [ - { - key: 'builtin', - label: '内置数据源', - children: ( -
-
-
-
采集实时进度
-
-
- 总体进度 - {aggregateProgress}% -
- 0 ? 'active' : 'normal'} - showInfo={false} - strokeColor="#1677ff" - /> -
-
-
- 内置 - {builtInSources.length} -
-
- 已启用 - {activeBuiltInCount} -
-
- 执行中 - {bulkProgressBatch ? bulkBatchRunningCount : runningBuiltInCount} -
- {bulkProgressBatch ? ( -
- 成功 - {bulkBatchSuccessCount}/{bulkProgressBatch.sourceIds.length} -
- ) : null} - {bulkProgressBatch ? ( -
- 失败 - {bulkBatchFailedCount} -
- ) : null} -
-
- - setForceTriggerAll(event.target.checked)} - > - 强制全部采集 - - - -
-
- - - - - ), - }, - { - key: 'custom', - label: ( - - 自定义数据源 - - ), - children: ( -
-
- -
- {customSources.length === 0 ? ( -
- -
- ) : ( -
-
- - - )} - + ), }, ] @@ -1360,267 +498,119 @@ function DataSources() { {modalContextHolder}
-

数据源管理

+

数据源

-
-
- +
+
+
+
采集实时进度
+
+
+ 总体进度 + {aggregateProgress}% +
+ 0 ? 'active' : 'normal'} showInfo={false} strokeColor="#1677ff" /> +
+
+
+ 全部 + {allSources.length} +
+
+ 内置 + {builtInSources.length} +
+
+ 已启用内置 + {activeBuiltInCount} +
+ {runningBuiltInCount > 0 ? ( + + + + ) : null} +
+
+ + + + + setForceTriggerAll(event.target.checked)}> + 强制全部采集 + + + +
+
+
+ - { - setDrawerVisible(false) - form.resetFields() - setEditingConfig(null) - setBuiltinEditingSource(null) - setTestResult(null) - }} - footer={ -
- - {builtinEditingSource && editingConfig ? ( - - - - ) : null} - - - - - - -
- } + setRunningTasksVisible(false)} + footer={} + width={680} > -
- {builtinEditingSource ? ( - - - -
内置数据源
- - - -
Collector Key
- - - - - ) : ( - - - - )} - - - - - - {builtinEditingSource ? null : ( - - - - )} - - - - - - - - - -
- auth_type === 'bearer'}> - {({ getFieldValue }) => { - if (getFieldValue('auth_type') === 'bearer') { - return ( - - - - ) - } - return null - }} - - auth_type === 'api_key'}> - {({ getFieldValue }) => { - if (getFieldValue('auth_type') === 'api_key') { - return ( - <> - - - - - - - - - - - ) - } - return null - }} - - auth_type === 'basic'}> - {({ getFieldValue }) => { - if (getFieldValue('auth_type') === 'basic') { - return ( - <> - - - - - - - - ) - } - return null - }} - -
- - ), - }, - ]} - /> - - - {(fields, { add, remove }) => ( - <> - {fields.map(({ key, name, ...restField }) => ( - - - - - - - - - - ))} - - - )} - - ), - }, - ]} - /> - - - - - - - - - - ), - }, - ]} - /> - - {testResult && ( -
-
- {testResult.success ? ( - - ) : ( - - )} - - {testResult.success ? '连接成功' : '连接失败'} - -
- {testResult.status_code &&
状态码: {testResult.status_code}
} - {testResult.response_time_ms &&
响应时间: {testResult.response_time_ms.toFixed(0)}ms
} - {testResult.error &&
错误: {testResult.error}
} - {testResult.data_preview && ( -
- 预览: {testResult.data_preview} + + {runningBuiltInSources.length ? runningBuiltInSources.map((source) => ( + + +
+ + {source.display_name || source.name} + + {source.source}{source.task_id ? ` · #${source.task_id}` : ''} + + + + {getPhaseSummary(source)} +
- )} -
+ {source.phase_message ? ( + {source.phase_message} + ) : null} + + + {source.phase_unit === 'bytes' + ? `下载 ${formatPhaseMetric(source) || '准备中'}` + : `已处理 ${source.records_processed ?? 0}${source.total_records ? ` / ${source.total_records}` : ''}`} + + + + )) : ( + 当前没有采集中任务。 )} - - + + { setViewDrawerVisible(false) setViewingSource(null) + setRecordCount(null) }} - footer={ -
- - - - - - - - -
- } + footer={
} > {viewingSource && ( +
+ + 内置数据源 + {viewingSource.is_active ? '启用' : '禁用'} + +
名称
- + + + +
标识
+
模块
- +
优先级
- +
频率
- +
数据量
- +
采集器
- + + {viewingSource.requires_credentials ? ( + navigate(`/settings?tab=collector_credentials&collector=${encodeURIComponent(viewingSource.source)}`)} + > + 去配置 + + ) : undefined} + /> + ) : null} +
- + ))} + + )} + + ) + })} + + ))} + + +
+
+ +
+ +
+ +
+
+ + +
+
+
+

+ {activeHeaderEntry ? getDocsGroupLabel(activeHeaderEntry.group, lang) : 'Docs'} +

+

+ {activeHeaderEntry?.title || (lang === 'zh' ? '文档不可用' : 'Document unavailable')} +

+
+ +
+ + { + setSearchQuery(event.target.value) + setIsSearchOpen(Boolean(event.target.value.trim())) + }} + onFocus={() => { + if (searchQuery.trim()) { + setIsSearchOpen(true) + } + }} + placeholder={lang === 'zh' ? '搜索文档...' : 'Search guides, APIs, layers...'} + type="search" + /> + {shouldShowSearchResults && ( +
+ + {searchResults.length > 0 ? ( + searchResults.map((result) => ( + + )) + ) : ( +
+ {lang === 'zh' ? '未找到匹配文档' : 'No matching docs'} +
+ )} + +
+ )} +
+
+ +
+ + {isCatalogLoading || isLoading ? ( +
+ {lang === 'zh' ? '加载中...' : 'Loading document...'} +
+ ) : docError === 'none' ? ( + + ) : ( +
+ {docError === 'unauthenticated' ? ( + <> +

{lang === 'zh' ? '需要登录' : 'Login required'}

+

+ {lang === 'zh' + ? '这份文档需要登录并具备对应 Gatekeeper 权限组后才能阅读。' + : 'This document requires login and the matching Gatekeeper permission group.'} +

+ {lang === 'zh' ? '前往登录' : 'Go to login'} + + ) : docError === 'forbidden' ? ( + <> +

{lang === 'zh' ? '无权访问' : 'Permission required'}

+

+ {lang === 'zh' + ? '当前账号没有阅读这份文档所需的 Gatekeeper 权限组。' + : 'Your account does not have the Gatekeeper permission group required for this document.'} +

+ {lang === 'zh' ? '返回文档首页' : 'Return to docs overview'} + + ) : ( + <> +

{lang === 'zh' ? '文档未找到' : 'Document not found'}

+

+ {lang === 'zh' + ? '请求的文档不存在，或当前语言没有对应内容。' + : 'The requested guide does not exist or is not available in the current language.'} +

+ {lang === 'zh' ? '返回文档首页' : 'Return to docs overview'} + + )} +
+ )} + + + +
+
+ + ) +} diff --git a/frontend/src/pages/Docs/docs-content.ts b/frontend/src/pages/Docs/docs-content.ts new file mode 100644 index 00000000..ed1cbb4a --- /dev/null +++ b/frontend/src/pages/Docs/docs-content.ts @@ -0,0 +1,250 @@ +export type DocsGroup = 'Overview' | 'Manual' | 'Earth' | 'Frontend' | 'Backend' | 'Agents' | 'Ops' | 'Other' +export type DocsLang = 'zh' | 'en' + +export interface DocsEntry { + slug: string + filename: string + title: string + group: DocsGroup + order: number + access: DocsAccess +} + +export type DocsAccess = 'public' | 'docs_user' | 'docs_developer' | 'docs_admin' + +export interface DocsCatalogItem { + slug: string + filename: string + lang: DocsLang + title: string + group: DocsGroup + order: number + access: DocsAccess +} + +export interface DocsHeading { + id: string + level: number + text: string +} + +export interface DocsMetadataEntry { + zh: { title: string; group: DocsGroup; order: number } + en: { title: string; group: DocsGroup; order: number } +} + +const DOCS_GROUP_LABELS: Record> = { + zh: { + Overview: '概览', + Manual: '使用手册', + Earth: '地球可视化', + Frontend: '前端', + Backend: '后端', + Agents: '智能体', + Ops: '运维', + Other: '其他', + }, + en: { + Overview: 'Overview', + Manual: 'Manual', + Earth: 'Earth', + Frontend: 'Frontend', + Backend: 'Backend', + Agents: 'Agents', + Ops: 'Ops', + Other: 'Other', + }, +} + +const DOCS_README_FILENAME = 'README.md' +const MAX_HEADING_ID_LENGTH = 80 +export const defaultDocsSlug = 'overview' + +export const DOCS_METADATA: Record = { + [DOCS_README_FILENAME]: { + zh: { title: '技术文档', group: 'Overview', order: 0 }, + en: { title: 'Technical Docs', group: 'Overview', order: 0 }, + }, + 'quickstart.md': { + zh: { title: '快速开始', group: 'Manual', order: 1 }, + en: { title: 'Quickstart', group: 'Manual', order: 1 }, + }, + 'manual.md': { + zh: { title: 'Planet 使用手册', group: 'Manual', order: 2 }, + en: { title: 'Planet Manual', group: 'Manual', order: 2 }, + }, + 'location-pipeline-user.md': { + zh: { title: 'Earth 位置候选采集使用手册', group: 'Manual', order: 3 }, + en: { title: 'Earth Location Candidate Collection User Guide', group: 'Manual', order: 3 }, + }, + 'earth-frontend-context.md': { + zh: { title: 'Earth 前端结构', group: 'Earth', order: 10 }, + en: { title: 'Earth Frontend Context', group: 'Earth', order: 10 }, + }, + 'earth-layer-style-reference.md': { + zh: { title: 'Earth 图层样式属性索引', group: 'Earth', order: 11 }, + en: { title: 'Earth Layer Style Reference', group: 'Earth', order: 11 }, + }, + 'earth-render-layer-order.md': { + zh: { title: 'Earth 渲染图层顺序', group: 'Earth', order: 12 }, + en: { title: 'Earth Render Layer Order', group: 'Earth', order: 12 }, + }, + 'earth-satellite-footprint-policy.md': { + zh: { title: 'Earth 卫星覆盖策略', group: 'Earth', order: 13 }, + en: { title: 'Earth Satellite Footprint Policy', group: 'Earth', order: 13 }, + }, + 'earth-bgp-context.md': { + zh: { title: 'BGP 态势上下文', group: 'Earth', order: 14 }, + en: { title: 'BGP Context', group: 'Earth', order: 14 }, + }, + 'earth-news-live-streams-collector-format.md': { + zh: { title: '新闻直播采集格式', group: 'Earth', order: 15 }, + en: { title: 'News Live Streams Collector Format', group: 'Earth', order: 15 }, + }, + 'earth-interactable-usage.md': { + zh: { title: 'Earth 可交互图标接入', group: 'Earth', order: 16 }, + en: { title: 'Earth Interactable Usage', group: 'Earth', order: 16 }, + }, + 'earth-toolbar-overlay-coordination.md': { + zh: { title: 'Earth 工具栏与浮层协同', group: 'Earth', order: 17 }, + en: { title: 'Earth Toolbar and Overlay Coordination', group: 'Earth', order: 17 }, + }, + 'frontend-admin-frontend-context.md': { + zh: { title: '控制台前端结构', group: 'Frontend', order: 20 }, + en: { title: 'Admin Frontend Context', group: 'Frontend', order: 20 }, + }, + 'frontend-layout-guidelines.md': { + zh: { title: '前端布局指南', group: 'Frontend', order: 21 }, + en: { title: 'Frontend Layout Guidelines', group: 'Frontend', order: 21 }, + }, + 'docs-gatekeeper-development.md': { + zh: { title: 'Docs Gatekeeper 开发说明', group: 'Frontend', order: 22 }, + en: { title: 'Docs Gatekeeper Development Guide', group: 'Frontend', order: 22 }, + }, + 'backend-collectors.md': { + zh: { title: '数据采集系统', group: 'Backend', order: 30 }, + en: { title: 'Data Collectors', group: 'Backend', order: 30 }, + }, + 'backend-system-service-control.md': { + zh: { title: '系统服务控制', group: 'Backend', order: 31 }, + en: { title: 'System Service Control', group: 'Backend', order: 31 }, + }, + 'datasource-collector-settings-connectivity.md': { + zh: { title: '数据源、采集器设置与连接验证', group: 'Backend', order: 32 }, + en: { title: 'Datasource Collector Settings and Connectivity', group: 'Backend', order: 32 }, + }, + 'backend-datasources-api-performance.md': { + zh: { title: '数据源 API 性能', group: 'Backend', order: 33 }, + en: { title: 'Datasource API Performance', group: 'Backend', order: 33 }, + }, + 'location-pipeline-development.md': { + zh: { title: '通用位置估算管线开发说明', group: 'Backend', order: 34 }, + en: { title: 'Shared Location Resolution Pipeline Development Guide', group: 'Backend', order: 34 }, + }, + 'agents-aiprovider.md': { + zh: { title: 'AI Provider 指南', group: 'Agents', order: 40 }, + en: { title: 'AI Provider Guide', group: 'Agents', order: 40 }, + }, + 'ops-docker-compose-buildx-upgrade.md': { + zh: { title: 'Docker + Compose + Buildx 升级', group: 'Ops', order: 50 }, + en: { title: 'Docker + Compose + Buildx Upgrade', group: 'Ops', order: 50 }, + }, + 'ops-planet-sh-startup.md': { + zh: { title: 'planet.sh 启动机制', group: 'Ops', order: 51 }, + en: { title: 'planet.sh Startup', group: 'Ops', order: 51 }, + }, +} + +const GROUP_ORDER: DocsGroup[] = ['Overview', 'Manual', 'Earth', 'Frontend', 'Backend', 'Agents', 'Ops', 'Other'] + +export function slugFromFilename(filename: string): string { + return filename === DOCS_README_FILENAME ? defaultDocsSlug : filename.replace(/\.md$/, '') +} + +export function getDocsEntries(lang: DocsLang, catalogItems: DocsCatalogItem[]): DocsEntry[] { + return catalogItems + .filter((item) => item.lang === lang) + .map((item) => ({ + slug: item.slug, + filename: item.filename, + title: item.title, + group: item.group, + order: item.order, + access: item.access, + })) + .sort((a, b) => a.order - b.order || a.title.localeCompare(b.title)) +} + +export function getDocsEntry(slug: string | undefined, entries: DocsEntry[]): DocsEntry | undefined { + const normalizedSlug = slug || defaultDocsSlug + return entries.find((entry) => entry.slug === normalizedSlug) +} + +export function groupDocsEntries(entries: DocsEntry[]): Array<{ group: DocsGroup; entries: DocsEntry[] }> { + return GROUP_ORDER.map((group) => ({ + group, + entries: entries.filter((entry) => entry.group === group), + })).filter((group) => group.entries.length > 0) +} + +export function getDocsGroupLabel(group: DocsGroup, lang: DocsLang): string { + return DOCS_GROUP_LABELS[lang][group] +} + +export function createHeadingId(text: string, usedIds: Map): string { + const base = text + .toLowerCase() + .replace(/`([^`]+)`/g, '$1') + .replace(/[^\p{L}\p{N}\s-]/gu, '') + .trim() + .replace(/\s+/g, '-') + .slice(0, MAX_HEADING_ID_LENGTH) || 'section' + const count = usedIds.get(base) || 0 + usedIds.set(base, count + 1) + return count === 0 ? base : `${base}-${count + 1}` +} + +export function extractHeadings(markdown: string): DocsHeading[] { + const usedIds = new Map() + return markdown + .split(/\r?\n/) + .map((line) => line.trim().match(/^(#{1,3})\s+(.+)$/)) + .filter((match): match is RegExpMatchArray => Boolean(match)) + .map((match) => { + const text = match[2].trim() + return { + id: createHeadingId(text, usedIds), + level: match[1].length, + text, + } + }) +} + +// Returns a factory — call factory() inside MarkdownRenderer to get a fresh resolver +// per render. This is necessary because StrictMode double-invokes renders, which +// would exhaust a shared stateful closure and cause heading IDs to become undefined. +export function createHeadingIdResolver(markdown: string): () => (text: string, level: number) => string | undefined { + const headings = extractHeadings(markdown) + + return () => { + const indexByKey = new Map() + return (text: string, level: number) => { + const key = `${level}:${text}` + const currentIndex = indexByKey.get(key) || 0 + indexByKey.set(key, currentIndex + 1) + const matchingHeadings = headings.filter((heading) => heading.level === level && heading.text === text) + return matchingHeadings[currentIndex]?.id + } + } +} + +export function slugFromDocsHref(href: string): string | null { + const normalized = decodeURIComponent(href).split('#')[0].replace(/\\/g, '/') + const filename = normalized.split('/').pop() + + if (!filename?.endsWith('.md')) { + return null + } + + return slugFromFilename(filename) +} diff --git a/frontend/src/pages/Docs/docs-search.ts b/frontend/src/pages/Docs/docs-search.ts new file mode 100644 index 00000000..d6b5ecb9 --- /dev/null +++ b/frontend/src/pages/Docs/docs-search.ts @@ -0,0 +1,105 @@ +import type { DocsEntry, DocsHeading } from './docs-content' +import { extractHeadings } from './docs-content' + +export interface DocsSearchRecord { + entry: DocsEntry + markdown: string + headings: DocsHeading[] + plainText: string +} + +export interface DocsSearchResult { + entry: DocsEntry + score: number + excerpt: string +} + +const DEFAULT_EXCERPT_LENGTH = 160 +const EXCERPT_CONTEXT_BEFORE = 56 +const EXCERPT_CONTEXT_AFTER = 104 +const MAX_SEARCH_RESULTS = 12 +const SEARCH_SCORE = { + title: 80, + slug: 36, + group: 24, + headings: 32, + body: 8, +} + +function stripMarkdown(markdown: string): string { + return markdown + .replace(/```[\s\S]*?```/g, ' ') + .replace(/`([^`]+)`/g, '$1') + .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') + .replace(/[#>*_\-|]/g, ' ') + .replace(/\s+/g, ' ') + .trim() +} + +function createExcerpt(text: string, query: string): string { + const lowerText = text.toLowerCase() + const lowerQuery = query.toLowerCase() + const matchIndex = lowerText.indexOf(lowerQuery) + + if (matchIndex < 0) { + return text.slice(0, DEFAULT_EXCERPT_LENGTH) + } + + const start = Math.max(0, matchIndex - EXCERPT_CONTEXT_BEFORE) + const end = Math.min(text.length, matchIndex + query.length + EXCERPT_CONTEXT_AFTER) + const prefix = start > 0 ? '...' : '' + const suffix = end < text.length ? '...' : '' + return `${prefix}${text.slice(start, end)}${suffix}` +} + +export async function buildDocsSearchRecords( + entries: DocsEntry[], + loadMarkdown: (entry: DocsEntry) => Promise, +): Promise { + const records = await Promise.all( + entries.map(async (entry) => { + const markdown = await loadMarkdown(entry) + return { + entry, + markdown, + headings: extractHeadings(markdown), + plainText: stripMarkdown(markdown), + } + }), + ) + + return records +} + +export function searchDocs(records: DocsSearchRecord[], rawQuery: string): DocsSearchResult[] { + const query = rawQuery.trim().toLowerCase() + + if (!query) { + return [] + } + + return records + .map((record) => { + const title = record.entry.title.toLowerCase() + const slug = record.entry.slug.toLowerCase() + const group = record.entry.group.toLowerCase() + const headings = record.headings.map((heading) => heading.text).join(' ').toLowerCase() + const body = record.plainText.toLowerCase() + let score = 0 + + if (title.includes(query)) score += SEARCH_SCORE.title + if (slug.includes(query)) score += SEARCH_SCORE.slug + if (group.includes(query)) score += SEARCH_SCORE.group + if (headings.includes(query)) score += SEARCH_SCORE.headings + if (body.includes(query)) score += SEARCH_SCORE.body + + return { + entry: record.entry, + score, + excerpt: createExcerpt(record.plainText, rawQuery.trim()), + } + }) + .filter((result) => result.score > 0) + .sort((a, b) => b.score - a.score || a.entry.order - b.entry.order) + .slice(0, MAX_SEARCH_RESULTS) +} diff --git a/frontend/src/pages/Logs/Logs.tsx b/frontend/src/pages/Logs/Logs.tsx new file mode 100644 index 00000000..9da9291c --- /dev/null +++ b/frontend/src/pages/Logs/Logs.tsx @@ -0,0 +1,533 @@ +import { useEffect, useMemo, useState } from 'react' +import { Alert, Button, Card, DatePicker, Empty, Input, InputNumber, Select, Space, Spin, Tag, Tooltip, Typography, message } from 'antd' +import { CopyOutlined, DownOutlined, InfoCircleOutlined, ReloadOutlined, UpOutlined } from '@ant-design/icons' +import axios from 'axios' +import dayjs, { Dayjs } from 'dayjs' +import type { CustomTagProps } from 'rc-select/lib/BaseSelect' +import AppLayout from '../../components/AppLayout/AppLayout' +import Scrollbar from '../../components/Scrollbar/Scrollbar' +import { useAuthStore } from '../../stores/auth' + +const { Paragraph, Text, Title } = Typography +const { RangePicker } = DatePicker +const LOG_FILTER_STORAGE_KEY = 'planet.logs.filters' +const DATE_PRESET_OPTIONS = [ + { key: 'today', label: 'Today', days: 0 }, + { key: 'last3', label: '3 Days', days: 2 }, + { key: 'last7', label: '7 Days', days: 6 }, +] as const + +interface LogSourceSummary { + source_id: string + name: string + kind: string + location: string + description: string + category: string + status: string +} + +interface LogSourcesResponse { + items: LogSourceSummary[] +} + +interface LogSnapshot { + source_id: string + name: string + kind: string + location: string + description: string + category: string + status: string + level: string + selected_levels: string[] + search_query: string + available_levels: string[] + daily_markers: Array<{ + date_token: string + total: number + dominant_level: 'error' | 'warning' | 'info' | 'debug' + }> + line_limit: number + line_count: number + lines: string[] +} + +interface DailyLogMarker { + total: number + dominantLevel: 'error' | 'warning' | 'info' | 'debug' +} + +const LOG_LIMIT_OPTIONS = [100, 200, 400, 800] +const LOG_LEVEL_OPTIONS = [ + { value: 'error', label: 'ERROR' }, + { value: 'warning', label: 'WARNING' }, + { value: 'info', label: 'INFO' }, + { value: 'debug', label: 'DEBUG' }, +] + +function isDayjsValue(value: unknown): value is Dayjs { + return dayjs.isDayjs(value) +} + +function normalizeSelectedLevels(levels: string[] | null | undefined): string[] { + const allowedLevels = new Set(LOG_LEVEL_OPTIONS.map((item) => item.value)) + return Array.from(new Set((levels || []).filter((level) => allowedLevels.has(level)))) +} + +function getLogLevelTagColor(level: string): string { + if (level === 'error') return 'error' + if (level === 'warning') return 'warning' + if (level === 'info') return 'success' + if (level === 'debug') return 'default' + return 'default' +} + +function readStoredFilters() { + if (typeof window === 'undefined') { + return null + } + + try { + const rawValue = window.localStorage.getItem(LOG_FILTER_STORAGE_KEY) + if (!rawValue) return null + const parsed = JSON.parse(rawValue) as { + selectedSource?: string + lineLimit?: number + selectedLevels?: string[] + selectedDateRange?: [string, string] | null + searchQuery?: string + } + return parsed + } catch { + return null + } +} + +function getStatusLabel(status: string): string { + if (status === 'ok') return '可用' + if (status === 'missing') return '暂无日志' + if (status === 'empty') return '暂无上报' + if (status === 'docker_unavailable') return 'Docker 不可用' + if (status === 'source_unavailable') return '日志源不可用' + return status +} + +function getStatusHelp(status: string): string | null { + if (status === 'missing') return '当前日志文件尚未生成，通常需要先启动对应服务。' + if (status === 'empty') return '当前日志源还没有收到任何上报事件。' + if (status === 'docker_unavailable') return '当前环境没有可用的 docker 命令，暂时无法读取容器日志。' + if (status === 'source_unavailable') return '日志源当前不可读取，请检查服务是否已启动。' + return null +} + +function resolvePresetRange(days: number): [Dayjs, Dayjs] { + const end = dayjs().endOf('day') + const start = dayjs().subtract(days, 'day').startOf('day') + return [start, end] +} + +function normalizeDateRange( + range: [Dayjs | null, Dayjs | null] | null, +): [Dayjs | null, Dayjs | null] | null { + if (!range?.[0] || !range?.[1]) return null + return [range[0].startOf('day'), range[1].endOf('day')] +} + +function getActiveDatePreset(range: [Dayjs | null, Dayjs | null] | null): string | null { + if (!range?.[0] || !range?.[1]) return null + + for (const option of DATE_PRESET_OPTIONS) { + const [presetStart, presetEnd] = resolvePresetRange(option.days) + if (range[0].isSame(presetStart, 'day') && range[1].isSame(presetEnd, 'day')) { + return option.key + } + } + + return null +} + +function Logs() { + const storedFilters = readStoredFilters() + const { user } = useAuthStore() + const isSuperAdmin = user?.role === 'super_admin' + const [sources, setSources] = useState([]) + const [selectedSource, setSelectedSource] = useState(storedFilters?.selectedSource || 'backend') + const [lineLimit, setLineLimit] = useState(storedFilters?.lineLimit || 200) + const [selectedLevels, setSelectedLevels] = useState( + normalizeSelectedLevels(storedFilters?.selectedLevels), + ) + const [selectedDateRange, setSelectedDateRange] = useState<[Dayjs | null, Dayjs | null] | null>( + storedFilters?.selectedDateRange + ? normalizeDateRange([dayjs(storedFilters.selectedDateRange[0]), dayjs(storedFilters.selectedDateRange[1])]) + : null, + ) + const [searchQuery, setSearchQuery] = useState(typeof storedFilters?.searchQuery === 'string' ? storedFilters.searchQuery : '') + const [snapshot, setSnapshot] = useState(null) + const [sourcesLoading, setSourcesLoading] = useState(false) + const [logLoading, setLogLoading] = useState(false) + const [errorMessage, setErrorMessage] = useState(null) + const [filtersExpanded, setFiltersExpanded] = useState( + Boolean( + storedFilters?.selectedLevels?.length + || (storedFilters?.selectedDateRange?.[0] && storedFilters?.selectedDateRange?.[1]), + ), + ) + const [messageApi, contextHolder] = message.useMessage() + + const fetchSources = async () => { + setSourcesLoading(true) + try { + const res = await axios.get('/api/v1/system/logs/sources') + setSources(res.data.items) + setErrorMessage(null) + if (res.data.items.length > 0 && !res.data.items.some((item) => item.source_id === selectedSource)) { + setSelectedSource(res.data.items[0].source_id) + } + } catch (error) { + const detail = axios.isAxiosError(error) ? error.response?.data?.detail : null + setErrorMessage(typeof detail === 'string' ? detail : '加载日志源失败') + } finally { + setSourcesLoading(false) + } + } + + const fetchSnapshot = async ( + sourceId: string, + limit: number, + levels: string[], + dateRange: [Dayjs | null, Dayjs | null] | null, + searchValue: string, + ) => { + setLogLoading(true) + try { + const res = await axios.get(`/api/v1/system/logs/${sourceId}`, { + params: { + limit, + level: levels.length === 1 ? levels[0] : 'all', + levels: levels.length > 0 ? levels.join(',') : undefined, + start_date: dateRange?.[0] ? dateRange[0].format('YYYY-MM-DD') : undefined, + end_date: dateRange?.[1] ? dateRange[1].format('YYYY-MM-DD') : undefined, + search: searchValue.trim() || undefined, + }, + }) + setSnapshot(res.data) + setErrorMessage(null) + } catch (error) { + const detail = axios.isAxiosError(error) ? error.response?.data?.detail : null + setSnapshot(null) + setErrorMessage(typeof detail === 'string' ? detail : '加载日志内容失败') + } finally { + setLogLoading(false) + } + } + + useEffect(() => { + if (!isSuperAdmin) return + fetchSources() + }, [isSuperAdmin]) + + useEffect(() => { + if (!isSuperAdmin || !selectedSource) return + fetchSnapshot(selectedSource, lineLimit, selectedLevels, selectedDateRange, searchQuery) + }, [isSuperAdmin, selectedSource, lineLimit, selectedLevels, selectedDateRange, searchQuery]) + + useEffect(() => { + if (typeof window === 'undefined') return + window.localStorage.setItem( + LOG_FILTER_STORAGE_KEY, + JSON.stringify({ + selectedSource, + lineLimit, + selectedLevels, + selectedDateRange: + selectedDateRange?.[0] && selectedDateRange?.[1] + ? [ + selectedDateRange[0].format('YYYY-MM-DD'), + selectedDateRange[1].format('YYYY-MM-DD'), + ] + : null, + searchQuery, + }), + ) + }, [lineLimit, searchQuery, selectedLevels, selectedDateRange, selectedSource]) + + if (!isSuperAdmin) { + return ( + + + + ) + } + + const selectedMeta = sources.find((item) => item.source_id === selectedSource) + const statusHelp = getStatusHelp(snapshot?.status || selectedMeta?.status || '') + const activeDatePreset = getActiveDatePreset(selectedDateRange) + const dailyLogMarkers = useMemo( + () => + new Map( + (snapshot?.daily_markers || []).map((marker) => [ + marker.date_token, + { + total: marker.total, + dominantLevel: marker.dominant_level, + }, + ]), + ), + [snapshot?.daily_markers], + ) + const currentResultLines = snapshot?.lines || [] + const lineCountLabel = currentResultLines.length + const hasDateFilter = Boolean(selectedDateRange?.[0] && selectedDateRange?.[1]) + const hasAdvancedFilters = selectedLevels.length > 0 || hasDateFilter + const effectiveLevelLabels = selectedLevels.length === 0 + ? ['ALL'] + : normalizeSelectedLevels(selectedLevels).map( + (level) => LOG_LEVEL_OPTIONS.find((item) => item.value === level)?.label || level.toUpperCase(), + ) + + const applyDatePreset = (days: number) => { + setSelectedDateRange(resolvePresetRange(days)) + } + + const renderLevelTag = (props: CustomTagProps) => { + const { label, value, closable, onClose } = props + return ( + + {label} + + ) + } + + return ( + + {contextHolder} +
+
+
+ 系统日志 + + 统一查看 Planet 当前关键服务日志，并串联 Earth 浏览器端错误、后端异常与服务输出。 + +
+
+ +
+ +
+ {errorMessage ? : null} + +
+
+ setSelectedLevels(normalizeSelectedLevels(value))} + options={LOG_LEVEL_OPTIONS} + className="logs-page__level-select" + maxTagCount="responsive" + allowClear + tagRender={renderLevelTag} + placeholder="全部级别" + /> + setSearchQuery(event.target.value)} + onSearch={(value) => setSearchQuery(value)} + placeholder="搜索日志内容、模块名、错误关键字" + className="logs-page__search-input" + /> + +
+ + {filtersExpanded ? ( +
+
+ + + + + + + + + + ({ + value: preset.provider, + label: `${preset.label} · ${preset.provider_api}`, + }))} + onChange={(value) => { + const preset = aiProviderPresets.find((item) => item.provider === value) + if (preset) applyAiProviderPreset(preset) + }} + /> + + +
+ + + + + + +
+ + + + + + + + + + + + + + +
+ + + + + + + + + {integrations?.ai_provider.service_token.configured + ? `已配置 ${integrations.ai_provider.service_token.preview}` + : '未配置'} + + 通常不需要改；用于 backend 调本地 aiprovider。 + + + + + + + + 清除当前代理 token + + + + + + + + ), + }, + { + key: 'collector_credentials', + label: '采集器设置', + forceRender: true, + children: ( + + + + + + + + { + const preview = integrations?.barentswatch.client_secret.preview + if (preview && integrationForm.getFieldValue(['barentswatch', 'client_secret']) === preview) { + integrationForm.setFieldValue(['barentswatch', 'client_secret'], '') + } + }} + /> + + + + ) : null} + + {selectedCollector?.source === 'aisstream_vessels' ? ( + AISStream 凭证} + extra={( + + + + + ))} + + + + )} + +
+ + + + + + +
+ {selectedCollector?.source === 'aisstream_vessels' ? ( + <> +
+ + + + + + +
+ + ({ value: preset.value, label: preset.label })), + { value: 'custom', label: '自定义 JSON' }, + ]} + onChange={applyAisstreamBboxPreset} + /> + + + + + + ) : null} + {selectedCollector?.is_custom ? ( + + + + ) : null} + + + + + + {selectedCollector?.is_custom && selectedCollectorConfig?.source_type === 'websocket' ? ( + <> + + + + {customStreamStatus?.running ? 'streaming' : customStreamStatus?.done ? 'stopped' : '未运行'} + + + ) : null} + {selectedCollector?.is_custom ? ( + + ) : null} + + + + ), + }, { key: 'collectors', label: '采集调度', @@ -767,9 +2124,58 @@ function Settings() {
- +
+ setCredentialGuideOpen(false)} + footer={[ + , + , + , + ]} + width={760} + centered + > + + {credentialGuide?.source === 'ai' ? ( + + ) : ( + + )} + + + + {credentialGuide?.prompt ? ( + + 生成提示词：{credentialGuide.prompt} + + ) : null} + + ) } diff --git a/frontend/src/pages/Tasks/Tasks.tsx b/frontend/src/pages/Tasks/Tasks.tsx index f7af6279..341b42ba 100644 --- a/frontend/src/pages/Tasks/Tasks.tsx +++ b/frontend/src/pages/Tasks/Tasks.tsx @@ -1,16 +1,26 @@ import { useEffect, useState } from 'react' -import { Table, Tag, Card, Row, Col, Statistic, Button } from 'antd' +import { Table, Tag, Card, Row, Col, Statistic, Button, Tooltip } from 'antd' import { ReloadOutlined, CheckCircleOutlined, CloseCircleOutlined, SyncOutlined } from '@ant-design/icons' import { useAuthStore } from '../../stores/auth' import AppLayout from '../../components/AppLayout/AppLayout' import TableScrollRegion from '../../components/Scrollbar/TableScrollRegion' import { formatDateTimeZhCN } from '../../utils/datetime' +import { getPhaseDisplay, getPhaseSummary } from '../../utils/phaseProgress' interface Task { id: number - collector: string + collector?: string + datasource_name?: string status: 'success' | 'failed' | 'running' | 'pending' + phase?: string | null + phase_progress?: number | null + phase_message?: string | null + phase_current?: number | null + phase_total?: number | null + phase_unit?: string | null records_processed: number + total_records?: number | null + progress?: number | null started_at: string completed_at: string duration_seconds: number @@ -53,7 +63,20 @@ function Tasks() { title: '收集器', dataIndex: 'collector', key: 'collector', - render: (c: string) => {c}, + render: (_: string, task: Task) => {task.collector || task.datasource_name || '-'}, + }, + { + title: '阶段', + dataIndex: 'phase', + key: 'phase', + render: (_: string, task: Task) => { + const detail = task.phase ? getPhaseDisplay(task) : null + return detail ? ( + + {getPhaseSummary(task)} + + ) : '-' + }, }, { title: '状态', diff --git a/frontend/src/pages/Users/Users.tsx b/frontend/src/pages/Users/Users.tsx index 43f7f6b1..6c655d23 100644 --- a/frontend/src/pages/Users/Users.tsx +++ b/frontend/src/pages/Users/Users.tsx @@ -6,17 +6,20 @@ import { TableActions, actionCellProps } from '../../components/TableActions/Tab import axios from 'axios' import AppLayout from '../../components/AppLayout/AppLayout' import TableScrollRegion from '../../components/Scrollbar/TableScrollRegion' +import { useAuthStore } from '../../stores/auth' interface User { id: number username: string email: string role: string + gatekeeper_groups: string[] is_active: boolean created_at: string } function Users() { + const { user: currentUser } = useAuthStore() const [users, setUsers] = useState([]) const [loading, setLoading] = useState(false) const [modalVisible, setModalVisible] = useState(false) @@ -64,6 +67,9 @@ function Users() { const handleSubmit = async (values: Record) => { try { + if (currentUser?.role !== 'super_admin') { + delete values.gatekeeper_groups + } if (editingUser) { await axios.put(`/api/v1/users/${editingUser.id}`, values) message.success('更新成功') @@ -98,6 +104,21 @@ function Users() { return {role} }, }, + { + title: 'Gatekeeper', + dataIndex: 'gatekeeper_groups', + key: 'gatekeeper_groups', + width: 260, + render: (groups: string[] = []) => ( + <> + {groups.length > 0 ? groups.map((group) => ( + + {group} + + )) : 未配置} + + ), + }, { title: '状态', dataIndex: 'is_active', @@ -177,6 +198,18 @@ function Users() { 只读用户 + +