From fbca3815129898b44036d0577ccb2b543b46d5a7 Mon Sep 17 00:00:00 2001 From: linkong Date: Thu, 21 May 2026 01:37:32 +0800 Subject: [PATCH] release: bump version to 0.62.0 --- .claude/commands/docs.md | 1 + .codex/skills/docs/SKILL.md | 1 + README.md | 7 +- TODO.md | 4 + VERSION | 2 +- aiprovider/main.py | 2 + aiprovider/provider_service.py | 54 +- backend/app/api/v1/datasource_config.py | 168 +- backend/app/api/v1/datasources.py | 215 +- backend/app/api/v1/settings.py | 402 +- backend/app/api/v1/system_control.py | 135 +- backend/app/api/v1/visualization.py | 41 +- backend/app/api/v1/websocket.py | 4 +- backend/app/core/websocket/broadcaster.py | 6 + backend/app/services/ai_client.py | 4 + backend/app/services/collectors/base.py | 60 +- backend/app/services/credential_guides.py | 58 +- .../app/services/datasource_connectivity.py | 34 +- backend/app/services/earth_layer_cache.py | 16 + backend/app/services/earth_news.py | 2 +- backend/app/services/llm_provider_catalog.py | 68 +- backend/app/services/location/llm_fallback.py | 234 +- backend/app/services/system_logs.py | 39 +- backend/app/services/tv_streams.py | 7 +- backend/tests/test_datasources_batch.py | 66 + backend/tests/test_earth_news.py | 55 + backend/tests/test_location_pipeline.py | 71 +- backend/tests/test_settings_ai_provider.py | 212 +- backend/tests/test_web_search_tools.py | 44 + docs/CHANGELOG.md | 17 + docs/plans/README.md | 4 + ...min-next-dual-track-full-migration-plan.md | 128 + .../admin-next-parity-audit-closeout-plan.md | 135 + docs/plans/admin-next-parity-checklist.md | 43 + .../admin-next-soft-glass-goal-driven-plan.md | 217 + ...ai-provider-openclaw-style-routing-plan.md | 182 + .../integration-config-schema-system-plan.md | 349 ++ docs/technical/en/README.md | 2 + docs/technical/en/agents-aiprovider.md | 41 + docs/technical/en/backend-collectors.md | 10 + ...asource-collector-settings-connectivity.md | 55 +- docs/technical/en/earth-frontend-context.md | 9 + .../en/frontend-admin-frontend-context.md | 94 +- docs/technical/en/naming-glossary.md | 99 + docs/technical/en/ops-runbook.md | 18 +- docs/technical/en/tactile-ui-components.md | 205 + docs/technical/zh/README.md | 2 + docs/technical/zh/agents-aiprovider.md | 41 + docs/technical/zh/backend-collectors.md | 13 + ...asource-collector-settings-connectivity.md | 55 +- docs/technical/zh/earth-frontend-context.md | 9 + .../zh/frontend-admin-frontend-context.md | 94 +- docs/technical/zh/naming-glossary.md | 99 + docs/technical/zh/ops-runbook.md | 18 +- docs/technical/zh/tactile-ui-components.md | 205 + docs/version-history.md | 3 +- frontend/bun.lock | 187 +- frontend/components.json | 19 + frontend/package.json | 25 +- frontend/postcss.config.cjs | 5 + .../public/earth/assets/brand/lim-logo.png | Bin 0 -> 52459 bytes frontend/public/earth/css/hud.css | 471 +- frontend/public/earth/css/toolbar.css | 242 +- frontend/public/earth/index.html | 504 +- frontend/public/earth/js/bgp.js | 5 +- frontend/public/earth/js/constants.js | 27 + frontend/public/earth/js/controls.js | 634 +- .../public/earth/js/country-boundaries.js | 4 +- frontend/public/earth/js/info-card.js | 44 +- frontend/public/earth/js/interactable.js | 320 +- frontend/public/earth/js/main.js | 322 +- .../public/earth/js/motion-debug-panel.js | 189 +- .../public/earth/js/news-cruise-adapter.js | 8 +- frontend/public/earth/js/news-locale.js | 10 + frontend/public/earth/js/news.js | 26 +- frontend/public/earth/js/ui.js | 15 +- frontend/public/earth/js/vessels.js | 1 + frontend/scripts/preview-auto.mjs | 161 + frontend/src/App.tsx | 42 +- frontend/src/admin-next/AdminNextRoutes.tsx | 48 + .../components/data-table/DataTable.tsx | 150 + .../components/layout/AdminNextLayout.tsx | 309 + .../src/admin-next/components/ui/badge.tsx | 12 + .../src/admin-next/components/ui/button.tsx | 61 + .../src/admin-next/components/ui/card.tsx | 22 + .../src/admin-next/components/ui/dialog.tsx | 90 + .../src/admin-next/components/ui/input.tsx | 14 + .../src/admin-next/components/ui/select.tsx | 46 + .../src/admin-next/components/ui/switch.tsx | 29 + .../src/admin-next/components/ui/tactile.tsx | 7 + .../src/admin-next/components/ui/toast.tsx | 64 + frontend/src/admin-next/design/theme.tsx | 60 + .../src/admin-next/pages/DashboardNext.tsx | 417 ++ .../src/admin-next/pages/DataListNext.tsx | 550 ++ frontend/src/admin-next/pages/LogsNext.tsx | 305 + .../admin-next/pages/PlainResourcePages.tsx | 5252 +++++++++++++++++ frontend/src/admin-next/pages/UsersNext.tsx | 316 + frontend/src/admin-next/patterns/patterns.tsx | 150 + frontend/src/admin-next/routes/manifest.tsx | 59 + .../admin-next/search/AdminSearchContext.tsx | 92 + frontend/src/admin-next/search/indexers.ts | 384 ++ frontend/src/admin-next/search/types.ts | 26 + frontend/src/admin-next/styles.css | 3635 ++++++++++++ .../src/components/AppLayout/AppLayout.tsx | 28 +- .../MarkdownRenderer/MarkdownRenderer.tsx | 4 +- .../src/components/Scrollbar/Scrollbar.tsx | 306 +- .../components/Scrollbar/ScrollbarOverlay.tsx | 327 +- .../Scrollbar/TableScrollRegion.tsx | 19 +- .../SegmentedControl/SegmentedControl.css | 45 + .../SegmentedControl/SegmentedControl.tsx | 3 +- frontend/src/components/tactile-ui/Button.tsx | 216 + .../components/tactile-ui/ControlGroup.tsx | 45 + .../src/components/tactile-ui/Scrollbar.tsx | 307 + .../tactile-ui/ScrollbarOverlay.tsx | 268 + frontend/src/components/tactile-ui/Switch.tsx | 54 + .../tactile-ui/TableScrollRegion.tsx | 34 + .../src/components/tactile-ui/Tooltip.tsx | 61 + frontend/src/components/tactile-ui/icons.tsx | 88 + frontend/src/components/tactile-ui/index.ts | 12 + frontend/src/components/tactile-ui/styles.css | 378 ++ frontend/src/components/tactile-ui/types.ts | 89 + frontend/src/index.css | 4247 +------------ frontend/src/pages/Auth/AuthShell.tsx | 100 + frontend/src/pages/DataList/DataList.tsx | 43 +- .../src/pages/DataSources/DataSources.tsx | 4 +- frontend/src/pages/Docs/Docs.css | 40 +- frontend/src/pages/Docs/docs-content.ts | 8 + .../pages/ForgotPassword/ForgotPassword.tsx | 138 +- frontend/src/pages/Login/Login.tsx | 83 +- frontend/src/pages/Playground/Playground.tsx | 4 +- frontend/src/pages/Register/Register.tsx | 185 +- frontend/src/pages/Settings/Settings.tsx | 17 +- .../src/pages/VerifyEmail/VerifyEmail.tsx | 91 +- frontend/tailwind.config.ts | 34 + frontend/vite.config.mts | 12 + planet.sh | 212 +- pyproject.toml | 2 +- uv.lock | 2 +- 138 files changed, 21303 insertions(+), 5721 deletions(-) create mode 100644 docs/plans/admin-next-dual-track-full-migration-plan.md create mode 100644 docs/plans/admin-next-parity-audit-closeout-plan.md create mode 100644 docs/plans/admin-next-parity-checklist.md create mode 100644 docs/plans/admin-next-soft-glass-goal-driven-plan.md create mode 100644 docs/plans/ai-provider-openclaw-style-routing-plan.md create mode 100644 docs/plans/integration-config-schema-system-plan.md create mode 100644 docs/technical/en/naming-glossary.md create mode 100644 docs/technical/en/tactile-ui-components.md create mode 100644 docs/technical/zh/naming-glossary.md create mode 100644 docs/technical/zh/tactile-ui-components.md create mode 100644 frontend/components.json create mode 100644 frontend/postcss.config.cjs create mode 100755 frontend/public/earth/assets/brand/lim-logo.png create mode 100644 frontend/scripts/preview-auto.mjs create mode 100644 frontend/src/admin-next/AdminNextRoutes.tsx create mode 100644 frontend/src/admin-next/components/data-table/DataTable.tsx create mode 100644 frontend/src/admin-next/components/layout/AdminNextLayout.tsx create mode 100644 frontend/src/admin-next/components/ui/badge.tsx create mode 100644 frontend/src/admin-next/components/ui/button.tsx create mode 100644 frontend/src/admin-next/components/ui/card.tsx create mode 100644 frontend/src/admin-next/components/ui/dialog.tsx create mode 100644 frontend/src/admin-next/components/ui/input.tsx create mode 100644 frontend/src/admin-next/components/ui/select.tsx create mode 100644 frontend/src/admin-next/components/ui/switch.tsx create mode 100644 frontend/src/admin-next/components/ui/tactile.tsx create mode 100644 frontend/src/admin-next/components/ui/toast.tsx create mode 100644 frontend/src/admin-next/design/theme.tsx create mode 100644 frontend/src/admin-next/pages/DashboardNext.tsx create mode 100644 frontend/src/admin-next/pages/DataListNext.tsx create mode 100644 frontend/src/admin-next/pages/LogsNext.tsx create mode 100644 frontend/src/admin-next/pages/PlainResourcePages.tsx create mode 100644 frontend/src/admin-next/pages/UsersNext.tsx create mode 100644 frontend/src/admin-next/patterns/patterns.tsx create mode 100644 frontend/src/admin-next/routes/manifest.tsx create mode 100644 frontend/src/admin-next/search/AdminSearchContext.tsx create mode 100644 frontend/src/admin-next/search/indexers.ts create mode 100644 frontend/src/admin-next/search/types.ts create mode 100644 frontend/src/admin-next/styles.css create mode 100644 frontend/src/components/tactile-ui/Button.tsx create mode 100644 frontend/src/components/tactile-ui/ControlGroup.tsx create mode 100644 frontend/src/components/tactile-ui/Scrollbar.tsx create mode 100644 frontend/src/components/tactile-ui/ScrollbarOverlay.tsx create mode 100644 frontend/src/components/tactile-ui/Switch.tsx create mode 100644 frontend/src/components/tactile-ui/TableScrollRegion.tsx create mode 100644 frontend/src/components/tactile-ui/Tooltip.tsx create mode 100644 frontend/src/components/tactile-ui/icons.tsx create mode 100644 frontend/src/components/tactile-ui/index.ts create mode 100644 frontend/src/components/tactile-ui/styles.css create mode 100644 frontend/src/components/tactile-ui/types.ts create mode 100644 frontend/src/pages/Auth/AuthShell.tsx create mode 100644 frontend/tailwind.config.ts diff --git a/.claude/commands/docs.md b/.claude/commands/docs.md index 647bddb1..0be584f4 100644 --- a/.claude/commands/docs.md +++ b/.claude/commands/docs.md @@ -73,6 +73,7 @@ Style: - Use fenced code blocks with language tags. - Prefer tables for comparisons or parameter lists. - Keep snippets concise and relevant. +- For UI labels, chart labels, feature names, datasource names, and other terms that may become mixed Chinese/English copy, check `docs/technical/{zh,en}/naming-glossary.md` and use the documented display name. If a confusing term is missing, update the glossary in both languages as part of the docs change. ### Step 4 — Verify diff --git a/.codex/skills/docs/SKILL.md b/.codex/skills/docs/SKILL.md index 9c399c85..ec7d6c56 100644 --- a/.codex/skills/docs/SKILL.md +++ b/.codex/skills/docs/SKILL.md @@ -52,6 +52,7 @@ rg -n "class |def |function |export |router|@router|interface |type " - Keep code snippets short and directly relevant. - List related files only when they help future maintainers navigate. - Use the repository’s existing language, heading style, and naming conventions. +- For UI labels, chart labels, feature names, datasource names, and other terms that may become mixed Chinese/English copy, check `docs/technical/{zh,en}/naming-glossary.md` and use the documented display name. If a confusing term is missing, update the glossary in both languages as part of the docs change. 4. Verify: diff --git a/README.md b/README.md index 001c18e1..08e8f238 100644 --- a/README.md +++ b/README.md @@ -169,10 +169,11 @@ ## 快速启动 ```bash -# 新机器首次初始化 -./scripts/bootstrap-dev.sh -# 会自动安装/检查 uv、bun，并同步 Python/前端依赖 +# 新机器或空项目首次初始化 +./planet.sh init +# 会自动安装/检查 uv、bun，同步 Python/前端依赖 # 会在缺少时生成 backend/.env、aiprovider/.env、frontend/.env.local +# 会启动 PostgreSQL/Redis，并创建表、默认数据源和本地默认用户 # 启动前后端服务 ./planet.sh start diff --git a/TODO.md b/TODO.md index f5388af0..0749ee13 100644 --- a/TODO.md +++ b/TODO.md @@ -35,6 +35,10 @@ This file is the active backlog only. Completed history belongs in `docs/CHANGEL ## AI Provider And Agents +- [ ] Unified integration config schema: implement the shared low-code schema engine for datasource, AI Provider, Web Search, and OCR configuration described in [Integration Config Schema System Plan](/home/ray/dev/linkong/planet/docs/plans/integration-config-schema-system-plan.md). +- [ ] AI provider routing: finish the OpenClaw-style provider/model routing refactor described in [AI Provider OpenClaw-Style Routing Plan](/home/ray/dev/linkong/planet/docs/plans/ai-provider-openclaw-style-routing-plan.md), so model-specific transport rules live in provider metadata rather than runtime hardcoding. +- [ ] AI provider catalog: replace the temporary `model_provider_apis` bridge with structured `models_metadata`, discovery descriptors, and incremental model sync with stale marking. +- [ ] AI provider connectivity: keep the plug action as lightweight network/auth/model-directory validation only, and keep real generation tests inside Playground or explicit “trial run” actions. - [ ] Agent runtime foundation: add auditable agent runs, steps, evidence, proposals, and the Agent operations UI described in [Agent Runtime, Earth LLM Command, And Speech Entry Plan](/home/ray/dev/linkong/planet/docs/plans/agents-earth-command-runtime-plan.md). - [ ] Agent tool protocol: add backend JSON tool-call fallback, optional provider-native tool compatibility, tool whitelist validation, and policy-gated proposal application. - [ ] Speech/ASR integration for agents: add provider-neutral transcription settings and API, defaulting to Whisper-compatible API providers while keeping text commands usable when ASR is unavailable. diff --git a/VERSION b/VERSION index 0b094550..4d74f323 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.61.0 +0.62.0 diff --git a/aiprovider/main.py b/aiprovider/main.py index b35670b9..e649013a 100644 --- a/aiprovider/main.py +++ b/aiprovider/main.py @@ -45,6 +45,7 @@ def get_provider_service( x_ai_model: str | None = Header(default=None), x_ai_max_tokens: str | None = Header(default=None), x_ai_anthropic_version: str | None = Header(default=None), + x_ai_model_provider_apis: str | None = Header(default=None), ) -> ProviderService: overrides = { "provider": x_ai_provider, @@ -53,6 +54,7 @@ def get_provider_service( "api_key": x_ai_api_key, "model": x_ai_model, "anthropic_version": x_ai_anthropic_version, + "model_provider_apis": x_ai_model_provider_apis, } if x_ai_max_tokens: overrides["max_tokens"] = x_ai_max_tokens diff --git a/aiprovider/provider_service.py b/aiprovider/provider_service.py index 35b6f4b5..674a9448 100644 --- a/aiprovider/provider_service.py +++ b/aiprovider/provider_service.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json from typing import Any import httpx @@ -14,7 +15,6 @@ from aiprovider.schemas import ( SituationalAnalysisResponse, ) - def _normalize_provider(value: str) -> str: return (value or "disabled").strip().lower() @@ -62,6 +62,9 @@ class ProviderService: self.anthropic_version = str( overrides.get("anthropic_version") or settings.AI_ANTHROPIC_VERSION ) + self.model_provider_apis = self._parse_model_provider_apis( + overrides.get("model_provider_apis") + ) def get_status(self) -> AIProviderStatusResponse: enabled = self.provider != "disabled" @@ -93,11 +96,13 @@ class ProviderService: prompt = self._build_prompt(payload) - if self.provider_api == "openai-completions": + provider_api = self._resolve_model_provider_api(model) + + if provider_api == "openai-completions": data = await self._request_openai_compatible(model, prompt, payload.system_prompt) content = self._extract_openai_content(data) content_blocks = self._extract_openai_blocks(data) - elif self.provider_api == "anthropic-messages": + elif provider_api == "anthropic-messages": data = await self._request_anthropic_messages( model, prompt, @@ -106,7 +111,7 @@ class ProviderService: ) content = self._extract_anthropic_content(data) content_blocks = self._extract_anthropic_blocks(data) - elif self.provider_api == "ollama-generate": + elif provider_api == "ollama-generate": data = await self._request_ollama(model, prompt, payload.system_prompt) content = self._extract_ollama_content(data) content_blocks = self._extract_ollama_blocks(data) @@ -132,6 +137,26 @@ class ProviderService: def _requires_api_key(self) -> bool: return self.provider_api != "ollama-generate" + def _resolve_model_provider_api(self, model: str) -> str: + return self.model_provider_apis.get(model) or self.provider_api + + def _parse_model_provider_apis(self, value: Any) -> dict[str, str]: + if isinstance(value, dict): + raw = value + elif isinstance(value, str) and value.strip(): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return {} + raw = parsed if isinstance(parsed, dict) else {} + else: + raw = {} + return { + str(model): _normalize_provider_api(str(provider_api)) + for model, provider_api in raw.items() + if model and provider_api + } + def _build_prompt(self, payload: SituationalAnalysisRequest) -> str: sections = [ f"任务标题:\n{payload.title}", @@ -314,13 +339,19 @@ class ProviderService: message = choices[0].get("message") or {} content = message.get("content") if isinstance(content, str): - return content + if content: + return content + reasoning_content = message.get("reasoning_content") + return reasoning_content if isinstance(reasoning_content, str) else "" if isinstance(content, list): return "".join( item.get("text", "") for item in content if isinstance(item, dict) ) + reasoning_content = message.get("reasoning_content") + if isinstance(reasoning_content, str): + return reasoning_content return "" def _extract_openai_blocks(self, payload: dict[str, Any]) -> list[AIContentBlock]: @@ -331,9 +362,14 @@ class ProviderService: message = choices[0].get("message") or {} content = message.get("content") if isinstance(content, str): - return [AIContentBlock(type="text", text=content)] + blocks = [AIContentBlock(type="text", text=content)] if content else [] + reasoning_content = message.get("reasoning_content") + if isinstance(reasoning_content, str) and reasoning_content: + blocks.append(AIContentBlock(type="thinking", thinking=reasoning_content)) + return blocks if not isinstance(content, list): - return [] + reasoning_content = message.get("reasoning_content") + return [AIContentBlock(type="thinking", thinking=reasoning_content)] if isinstance(reasoning_content, str) and reasoning_content else [] blocks: list[AIContentBlock] = [] for item in content: @@ -346,7 +382,11 @@ class ProviderService: metadata={k: v for k, v in item.items() if k not in {"type", "text"}}, ) ) + reasoning_content = message.get("reasoning_content") + if isinstance(reasoning_content, str) and reasoning_content: + blocks.append(AIContentBlock(type="thinking", thinking=reasoning_content)) return blocks + def _extract_anthropic_content(self, payload: dict[str, Any]) -> str: content = payload.get("content") if isinstance(content, str): diff --git a/backend/app/api/v1/datasource_config.py b/backend/app/api/v1/datasource_config.py index cc24cb21..3f7ce76d 100644 --- a/backend/app/api/v1/datasource_config.py +++ b/backend/app/api/v1/datasource_config.py @@ -5,7 +5,7 @@ from datetime import datetime import base64 import json import re -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from sqlalchemy import delete, select, func from sqlalchemy.ext.asyncio import AsyncSession from pydantic import BaseModel, Field @@ -45,14 +45,70 @@ from app.services.custom_datasource_runtime import ( DATASOURCE_MAPPING_PROMPT_KEY = "datasource.mapping" from app.services.datasource_connectivity import ( + _resolve_aisstream_api_key, + _resolve_spacetrack_credentials_with_override, get_builtin_connection_status, save_connectivity_success, strip_connectivity_validation, test_builtin_connectivity, ) +from app.services.barentswatch import resolve_barentswatch_config +from app.services.persistent_logs import record_audit_log router = APIRouter() +SECRET_REVEAL_ROLES = {"admin", "super_admin"} + + +def _user_role_value(user: User) -> str: + role = getattr(user, "role", "") + return str(getattr(role, "value", role) or "").lower() + + +def _user_display_name(user: User) -> str: + return str(getattr(user, "username", None) or getattr(user, "email", None) or getattr(user, "id", "")) + + +async def _record_datasource_secret_reveal( + *, + current_user: User, + request: Request, + target_id: str, + result: str, + details: dict[str, Any], +) -> None: + await record_audit_log( + action="datasource_config.secret.reveal", + actor_id=getattr(current_user, "id", None), + actor_name=_user_display_name(current_user), + target_type="datasource_config_secret", + target_id=target_id, + result=result, + ip=request.client.host if request.client else None, + details=details, + ) + + +async def _ensure_datasource_secret_reveal_allowed( + current_user: User, + request: Request, + target_id: str, + details: dict[str, Any], +) -> None: + if _user_role_value(current_user) in SECRET_REVEAL_ROLES: + return + await _record_datasource_secret_reveal( + current_user=current_user, + request=request, + target_id=target_id, + result="denied", + details={**details, "role": _user_role_value(current_user)}, + ) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Only administrators can reveal datasource credentials", + ) + def _default_builtin_config(name: str) -> dict[str, Any]: return {"timeout": 30, "retry": 3} @@ -389,10 +445,14 @@ async def list_all_datasources( db_config = db_configs.get(name) default_config = _default_builtin_config(name) default_url = yaml_url + db_auth_config = db_config.auth_config or {} if db_config else {} result.append( { "name": name, + "requires_credentials": bool(metadata.get("requires_credentials", False)), + "credential_provider": metadata.get("credential_provider"), + "credential_status": metadata.get("credential_status", "none"), "default_url": default_url, "endpoint": db_config.endpoint if db_config else default_url, "is_overridden": db_config is not None and db_config.endpoint != yaml_url @@ -401,10 +461,20 @@ async def list_all_datasources( "is_active": db_config.is_active if db_config else True, "source_type": db_config.source_type if db_config else _default_builtin_source_type(name), "auth_type": db_config.auth_type if db_config else "none", + "auth_config": { + "client_id": db_auth_config.get("client_id") or "", + "username": db_auth_config.get("username") or "", + "key_name": db_auth_config.get("key_name") or db_auth_config.get("param_name") or "", + "param_name": db_auth_config.get("param_name") or db_auth_config.get("key_name") or "", + "location": db_auth_config.get("location") or db_auth_config.get("in") or "", + "in": db_auth_config.get("in") or db_auth_config.get("location") or "", + }, "auth_configured": { - "api_key": bool((db_config.auth_config or {}).get("api_key")) - if db_config - else False, + "api_key": bool(db_auth_config.get("api_key")), + "client_id": bool(db_auth_config.get("client_id")), + "client_secret": bool(db_auth_config.get("client_secret")), + "username": bool(db_auth_config.get("username")), + "password": bool(db_auth_config.get("password")), }, "headers": db_config.headers if db_config else {}, "config": strip_connectivity_validation(db_config.config if db_config else default_config), @@ -418,6 +488,96 @@ async def list_all_datasources( return {"total": len(result), "data": result} +@router.get("/configs/secrets") +async def reveal_builtin_config_secrets( + request: Request, + name: str = Query(..., min_length=1), + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Reveal configured built-in datasource credentials for admin editing.""" + source = name.strip() + metadata = DEFAULT_DATASOURCES.get(source) + if not metadata or not metadata.get("requires_credentials"): + raise HTTPException(status_code=404, detail="Credentialed datasource config not found") + + provider = str(metadata.get("credential_provider") or "") + target_id = f"datasource_config:{source}" + await _ensure_datasource_secret_reveal_allowed( + current_user, + request, + target_id, + {"source": source, "provider": provider}, + ) + + result = await db.execute(select(DataSourceConfig).where(DataSourceConfig.name == source)) + record = result.scalar_one_or_none() + auth_config = dict(record.auth_config or {}) if record else {} + payload: dict[str, Any] = { + "name": source, + "provider": provider, + } + details: dict[str, Any] = {"source": source, "provider": provider} + + if provider == "barentswatch": + resolved = await resolve_barentswatch_config(db) + client_id = str(auth_config.get("client_id") or resolved.client_id or "") + client_secret = str(auth_config.get("client_secret") or resolved.client_secret or "") + source_label = "datasource_config" if auth_config.get("client_id") or auth_config.get("client_secret") else resolved.credential_source + payload.update( + { + "client_id": client_id, + "client_secret": client_secret, + "client_id_source": source_label if client_id else "missing", + "client_secret_source": source_label if client_secret else "missing", + } + ) + details.update( + { + "client_id_configured": bool(client_id), + "client_secret_configured": bool(client_secret), + "credential_source": source_label, + } + ) + elif provider == "aisstream": + api_key, api_key_source = await _resolve_aisstream_api_key(db) + payload.update({"api_key": api_key, "api_key_source": api_key_source}) + details.update({"api_key_configured": bool(api_key), "api_key_source": api_key_source}) + elif provider == "spacetrack": + if auth_config.get("username") or auth_config.get("password"): + username = str(auth_config.get("username") or "") + password = str(auth_config.get("password") or "") + credential_source = "datasource_config" + else: + username, password, credential_source = _resolve_spacetrack_credentials_with_override() + payload.update( + { + "username": username, + "password": password, + "username_source": credential_source if username else "missing", + "password_source": credential_source if password else "missing", + } + ) + details.update( + { + "username_configured": bool(username), + "password_configured": bool(password), + "credential_source": credential_source, + } + ) + else: + raise HTTPException(status_code=400, detail="Datasource credential provider is not supported") + + await _record_datasource_secret_reveal( + current_user=current_user, + request=request, + target_id=target_id, + result="success", + details=details, + ) + return payload + + @router.get("/configs/{config_id}") async def get_config( config_id: int, diff --git a/backend/app/api/v1/datasources.py b/backend/app/api/v1/datasources.py index c8174f72..99094b3a 100644 --- a/backend/app/api/v1/datasources.py +++ b/backend/app/api/v1/datasources.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, Field from sqlalchemy import func, or_, select, text from sqlalchemy.ext.asyncio import AsyncSession +from app.core.cache import cache from app.core.time import to_iso8601_utc from app.core.security import get_current_user from app.core.data_sources import get_data_sources_config @@ -26,6 +27,7 @@ from app.services.scheduler import ( run_collector_now, sync_datasource_job, ) +from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source router = APIRouter() STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90 @@ -235,6 +237,8 @@ async def _load_datasource_endpoint_overrides( async def _load_datasource_list_context( db: AsyncSession, datasources: list[DataSource], + *, + include_endpoint: bool = True, ) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, str]]: datasource_ids = [datasource.id for datasource in datasources] sources = [datasource.source for datasource in datasources] @@ -260,10 +264,65 @@ async def _load_datasource_list_context( running_tasks = await _load_latest_running_tasks(db, datasource_ids) latest_tasks = await _load_latest_tasks(db, datasource_ids) - endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources) + endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources) if include_endpoint else {} return running_tasks, latest_tasks, endpoint_overrides +def serialize_datasource_row( + datasource: DataSource, + *, + running_tasks: dict[int, CollectionTask], + latest_tasks: dict[int, CollectionTask], + record_counts: dict[str, int], + endpoint_overrides: dict[str, str], + config, + include_endpoint: bool, +) -> dict: + running_task = running_tasks.get(datasource.id) + latest_task = latest_tasks.get(datasource.id) + display_task = running_task or latest_task + endpoint = None + if include_endpoint: + endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source) + last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None) + last_status = datasource.last_status or (latest_task.status if latest_task else None) + collected_records = record_counts.get(datasource.source, 0) + + row = { + "id": datasource.id, + "source": datasource.source, + "name": datasource.name, + **datasource_metadata(datasource.source), + "product": datasource_product_key(datasource), + "module": datasource.module, + "priority": datasource.priority, + "frequency": format_frequency_label(datasource.frequency_minutes), + "frequency_minutes": datasource.frequency_minutes, + "is_active": datasource.is_active, + "collector_class": datasource.collector_class, + "last_run": to_iso8601_utc(last_run_at), + "last_run_at": to_iso8601_utc(last_run_at), + "last_status": last_status, + "is_running": running_task is not None, + "task_id": display_task.id if display_task else None, + "progress": display_task.progress if display_task else None, + "phase": display_task.phase if display_task else None, + "phase_progress": display_task.phase_progress if display_task else None, + "phase_message": display_task.phase_message if display_task else None, + "phase_current": display_task.phase_current if display_task else None, + "phase_total": display_task.phase_total if display_task else None, + "phase_unit": display_task.phase_unit if display_task else None, + "records_processed": display_task.records_processed if display_task else None, + "total_records": display_task.total_records if display_task else None, + "error_message": display_task.error_message if display_task else None, + "collected_records": collected_records, + "has_collected_data": collected_records > 0, + } + if include_endpoint: + row["endpoint"] = endpoint + return row + + def _apply_datasource_query_filters( query, *, @@ -658,6 +717,7 @@ async def list_datasources( collected: Optional[bool] = None, credential_status: Optional[str] = None, q: Optional[str] = None, + include_endpoint: bool = True, current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db), ): @@ -676,7 +736,11 @@ async def list_datasources( collector_list = [] config = get_data_sources_config() - running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources) + running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context( + db, + datasources, + include_endpoint=include_endpoint, + ) record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources]) datasources = _filter_datasources_in_memory( datasources, @@ -689,46 +753,16 @@ async def list_datasources( credential_status=credential_status, ) for datasource in datasources: - running_task = running_tasks.get(datasource.id) - latest_task = latest_tasks.get(datasource.id) - display_task = running_task or latest_task - endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source) - last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None) - last_status = datasource.last_status or (latest_task.status if latest_task else None) - collected_records = record_counts.get(datasource.source, 0) - collector_list.append( - { - "id": datasource.id, - "source": datasource.source, - "name": datasource.name, - **datasource_metadata(datasource.source), - "product": datasource_product_key(datasource), - "module": datasource.module, - "priority": datasource.priority, - "frequency": format_frequency_label(datasource.frequency_minutes), - "frequency_minutes": datasource.frequency_minutes, - "is_active": datasource.is_active, - "collector_class": datasource.collector_class, - "endpoint": endpoint, - "last_run": to_iso8601_utc(last_run_at), - "last_run_at": to_iso8601_utc(last_run_at), - "last_status": last_status, - "is_running": running_task is not None, - "task_id": display_task.id if display_task else None, - "progress": display_task.progress if display_task else None, - "phase": display_task.phase if display_task else None, - "phase_progress": display_task.phase_progress if display_task else None, - "phase_message": display_task.phase_message if display_task else None, - "phase_current": display_task.phase_current if display_task else None, - "phase_total": display_task.phase_total if display_task else None, - "phase_unit": display_task.phase_unit if display_task else None, - "records_processed": display_task.records_processed if display_task else None, - "total_records": display_task.total_records if display_task else None, - "error_message": display_task.error_message if display_task else None, - "collected_records": collected_records, - "has_collected_data": collected_records > 0, - } + serialize_datasource_row( + datasource, + running_tasks=running_tasks, + latest_tasks=latest_tasks, + record_counts=record_counts, + endpoint_overrides=endpoint_overrides, + config=config, + include_endpoint=include_endpoint, + ) ) return {"total": len(collector_list), "data": collector_list} @@ -785,6 +819,51 @@ async def trigger_datasource_batch( return await _trigger_datasource_batch(db, datasources, force=payload.force) +@router.get("/snapshots") +async def list_datasource_snapshots( + source_id: Optional[str] = None, + current_only: Optional[bool] = None, + limit: int = Query(default=100, ge=1, le=500), + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + query = ( + select(DataSnapshot, DataSource.name, DataSource.module) + .outerjoin(DataSource, DataSource.id == DataSnapshot.datasource_id) + .order_by(DataSnapshot.created_at.desc().nullslast(), DataSnapshot.id.desc()) + .limit(limit) + ) + if source_id: + query = query.where(DataSnapshot.source == source_id) + if current_only is not None: + query = query.where(DataSnapshot.is_current.is_(current_only)) + + result = await db.execute(query) + rows = [] + for snapshot, datasource_name, datasource_module in result.all(): + rows.append( + { + "id": snapshot.id, + "datasource_id": snapshot.datasource_id, + "datasource_name": datasource_name, + "module": datasource_module, + "task_id": snapshot.task_id, + "source": snapshot.source, + "snapshot_key": snapshot.snapshot_key, + "reference_date": to_iso8601_utc(snapshot.reference_date), + "started_at": to_iso8601_utc(snapshot.started_at), + "completed_at": to_iso8601_utc(snapshot.completed_at), + "record_count": snapshot.record_count, + "status": snapshot.status, + "is_current": snapshot.is_current, + "parent_snapshot_id": snapshot.parent_snapshot_id, + "summary": snapshot.summary or {}, + "created_at": to_iso8601_utc(snapshot.created_at), + } + ) + return {"total": len(rows), "data": rows} + + @router.get("/{source_id}") async def get_datasource( source_id: str, @@ -813,6 +892,37 @@ async def get_datasource( } +@router.get("/{source_id}/row") +async def get_datasource_row( + source_id: str, + include_endpoint: bool = True, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + datasource = await get_datasource_record(db, source_id) + if not datasource: + raise HTTPException(status_code=404, detail="Data source not found") + + config = get_data_sources_config() + running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context( + db, + [datasource], + include_endpoint=include_endpoint, + ) + record_counts = await _load_collected_record_counts(db, [datasource.source]) + return { + "data": serialize_datasource_row( + datasource, + running_tasks=running_tasks, + latest_tasks=latest_tasks, + record_counts=record_counts, + endpoint_overrides=endpoint_overrides, + config=config, + include_endpoint=include_endpoint, + ) + } + + @router.post("/{source_id}/enable") async def enable_datasource( source_id: str, @@ -960,6 +1070,29 @@ async def clear_datasource_data( } +@router.delete("/{source_id}/cache") +async def clear_datasource_cache( + source_id: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + datasource = await get_datasource_record(db, source_id) + if not datasource: + raise HTTPException(status_code=404, detail="Data source not found") + + earth_deleted_count = invalidate_earth_layer_cache_for_source(datasource.source) + dashboard_deleted_count = int(cache.delete("dashboard:stats")) + int(cache.delete("dashboard:summary")) + deleted_count = earth_deleted_count + dashboard_deleted_count + + return { + "status": "success", + "message": f"Cleared {deleted_count} cache keys for data source '{datasource.name}'", + "deleted_count": deleted_count, + "earth_layer_deleted_count": earth_deleted_count, + "dashboard_deleted_count": dashboard_deleted_count, + } + + @router.get("/{source_id}/task-status") async def get_task_status( source_id: str, diff --git a/backend/app/api/v1/settings.py b/backend/app/api/v1/settings.py index 322162ba..5a8cf3f6 100644 --- a/backend/app/api/v1/settings.py +++ b/backend/app/api/v1/settings.py @@ -4,7 +4,8 @@ import os from pathlib import Path from typing import Optional -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +import httpx from pydantic import BaseModel, EmailStr, Field from dotenv import dotenv_values from sqlalchemy import select @@ -64,10 +65,12 @@ from app.services.llm_provider_catalog import ( ) from app.services.scheduler import sync_datasource_job from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings +from app.services.persistent_logs import record_audit_log router = APIRouter() AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5 AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test" +SECRET_REVEAL_ROLES = {"admin", "super_admin"} DEFAULT_SETTINGS = { "system": { @@ -130,6 +133,70 @@ DEFAULT_SETTINGS = { } +def _user_role_value(user: User) -> str: + role = getattr(user, "role", "") + return role.value if hasattr(role, "value") else str(role or "") + + +def _user_display_name(user: User) -> str | None: + return getattr(user, "username", None) or getattr(user, "email", None) + + +def _request_client_ip(request: Request | None) -> str | None: + if request is None or request.client is None: + return None + return request.client.host + + +def _can_reveal_integration_secrets(user: User) -> bool: + return _user_role_value(user) in SECRET_REVEAL_ROLES + + +async def _record_integration_secret_reveal( + *, + current_user: User, + request: Request | None, + target_id: str, + result: str, + details: dict, +) -> None: + await record_audit_log( + action="settings.integration_secret.reveal", + actor_id=getattr(current_user, "id", None), + actor_name=_user_display_name(current_user), + target_type="integration_secret", + target_id=target_id, + result=result, + ip=_request_client_ip(request), + details=details, + ) + + +async def _ensure_secret_reveal_allowed( + *, + current_user: User, + request: Request | None, + target_id: str, + details: dict | None = None, +) -> None: + if _can_reveal_integration_secrets(current_user): + return + await _record_integration_secret_reveal( + current_user=current_user, + request=request, + target_id=target_id, + result="denied", + details={ + **(details or {}), + "role": _user_role_value(current_user), + }, + ) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Only administrators can reveal integration secrets", + ) + + class SystemSettingsUpdate(BaseModel): system_name: str = "智能星球" refresh_interval: int = Field(default=60, ge=10, le=3600) @@ -353,9 +420,10 @@ def _get_provider_preset(provider: str) -> dict: "provider": provider, "provider_api": "openai-completions", "base_url": "", - "model": "", - "models": [], - "api_key_env": "", + "model": "", + "models": [], + "model_provider_apis": {}, + "api_key_env": "", } @@ -377,6 +445,9 @@ def _resolve_env_secret(*names: str) -> tuple[str, str]: value = env_file_values.get(name) if value: return value, "env_file" + value = os.environ.get(name) + if value: + return value, "env" return "", "" @@ -419,9 +490,16 @@ def _provider_defaults(provider: str) -> dict: 1200 if preset.get("provider_api") == "anthropic-messages" else 4096 ), "anthropic_version": "2023-06-01", + "model_provider_apis": preset.get("model_provider_apis") or {}, } +def _selected_ai_env_provider() -> str: + env_file_values = _read_ai_provider_env_file() + provider = env_file_values.get("AI_PROVIDER") or os.environ.get("AI_PROVIDER") or "minimax" + return _normalize_provider_id(provider) + + def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict: raw = dict(ai_payload or {}) default_provider = _normalize_provider_id(raw.get("default_provider") or raw.get("provider")) @@ -440,6 +518,7 @@ def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict: "api_key", "max_tokens", "anthropic_version", + "model_provider_apis", ) if raw.get(key) not in (None, "") } @@ -477,7 +556,12 @@ def _resolve_provider_api_key(provider: str, provider_config: dict) -> tuple[str return str(saved_key), "runtime" preset = _get_provider_preset(provider) api_key_env = preset.get("api_key_env") or "" - return _resolve_env_secret(api_key_env, "AI_API_KEY") + value, source = _resolve_env_secret(api_key_env) + if value: + return value, source + if _normalize_provider_id(provider) == _selected_ai_env_provider(): + return _resolve_env_secret("AI_API_KEY") + return "", "" def _resolve_service_token(ai_payload: dict) -> tuple[str, str]: @@ -509,7 +593,12 @@ def _is_secret_placeholder(value: Optional[str], current_preview: str = "") -> b def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrationUpdate) -> dict: current_ai = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {}) - provider_id = _normalize_provider_id(update.default_provider or update.provider) + provider_id = _normalize_provider_id(update.provider) + default_provider = ( + _normalize_provider_id(update.default_provider) + if update.default_provider is not None + else current_ai["default_provider"] + ) current_providers = { provider: dict(config or {}) for provider, config in current_ai.get("providers", {}).items() @@ -543,7 +632,7 @@ def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrat "service_url": update.service_url.strip() or app_settings.AI_PROVIDER_SERVICE_URL, "service_token": current_ai.get("service_token") or "", - "default_provider": provider_id, + "default_provider": default_provider, "providers": current_providers, "timeout_seconds": update.timeout_seconds, "retry_attempts": update.retry_attempts, @@ -577,6 +666,8 @@ def _runtime_config_from_ai_payload(ai_payload: dict) -> dict: "api_key": api_key, "max_tokens": int(provider_config.get("max_tokens") or 1200), "anthropic_version": provider_config.get("anthropic_version") or "2023-06-01", + "model_provider_apis": provider_config.get("model_provider_apis") or {}, + "preset_models": _get_provider_preset(default_provider).get("models") or [], }, } @@ -631,6 +722,159 @@ async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict: } +def _join_provider_url(base_url: str, path: str) -> str: + return f"{base_url.rstrip('/')}/{path.lstrip('/')}" + + +def _extract_model_ids(payload: dict) -> list[str]: + data = payload.get("data") if isinstance(payload, dict) else None + if isinstance(data, list): + return [ + str(item.get("id")) + for item in data + if isinstance(item, dict) and item.get("id") + ] + models = payload.get("models") if isinstance(payload, dict) else None + if isinstance(models, list): + return [ + str(item.get("name") or item.get("model") or item.get("id") or item) + for item in models + if item + ] + return [] + + +def _contains_model(model_ids: list[str], model: str) -> bool: + normalized_model = model.strip().lower() + return any(str(item).strip().lower() == normalized_model for item in model_ids) + + +async def _check_ai_provider_lightweight(llm_config: dict, timeout_seconds: int) -> dict: + provider = _normalize_provider_id(llm_config.get("provider") or "") + configured_api = str(llm_config.get("provider_api") or "").strip() or "openai-completions" + model = str(llm_config.get("model") or "").strip() + base_url = str(llm_config.get("base_url") or "").strip().rstrip("/") + api_key = str(llm_config.get("api_key") or "").strip() + provider_api = configured_api + model_provider_apis = llm_config.get("model_provider_apis") + if isinstance(model_provider_apis, dict): + provider_api = str(model_provider_apis.get(model) or provider_api) + preset_models = [ + str(item) + for item in (llm_config.get("preset_models") or []) + if str(item).strip() + ] + + if not provider or not base_url or not model: + return { + "success": False, + "connected": False, + "message": "当前 provider/base_url/model 未完整配置。", + "mode": "lightweight_config", + } + if provider_api != "ollama-generate" and not api_key: + return { + "success": False, + "connected": False, + "message": "当前 provider 未配置 API Key。", + "mode": "lightweight_config", + } + + if provider == "opencode-go": + url = _join_provider_url(base_url, "/models") + headers = {"Authorization": f"Bearer {api_key}"} + elif provider_api == "ollama-generate": + url = _join_provider_url(base_url, "/api/tags") + headers: dict[str, str] = {} + elif provider_api == "openai-completions": + url = _join_provider_url(base_url, "/models") + headers = {"Authorization": f"Bearer {api_key}"} + elif provider_api == "anthropic-messages": + url = _join_provider_url(base_url, "/models") + headers = { + "x-api-key": api_key, + "anthropic-version": str(llm_config.get("anthropic_version") or "2023-06-01"), + } + else: + return { + "success": False, + "connected": False, + "message": f"当前 provider_api 不支持轻量连通性测试: {provider_api}", + "mode": "lightweight_unsupported", + } + + try: + async with httpx.AsyncClient(timeout=min(timeout_seconds, AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS)) as client: + response = await client.get(url, headers=headers) + response.raise_for_status() + payload = response.json() + except httpx.HTTPStatusError as exc: + detail = exc.response.text or exc.response.reason_phrase + if exc.response.status_code == 404 and _contains_model(preset_models, model): + return { + "success": True, + "connected": True, + "message": "轻量连通性测试通过；当前 provider 不提供可用的模型目录，已按内置模型预设确认。", + "mode": "lightweight_preset", + "provider": provider, + "provider_api": provider_api, + "model": model, + "url": url, + } + return { + "success": False, + "connected": False, + "message": f"轻量连通性测试失败: HTTP {exc.response.status_code} {detail}", + "mode": "lightweight_models", + "url": url, + } + except Exception as exc: + return { + "success": False, + "connected": False, + "message": f"轻量连通性测试失败: {exc}", + "mode": "lightweight_models", + "url": url, + } + + model_ids = _extract_model_ids(payload) + if model_ids and not _contains_model(model_ids, model): + if _contains_model(preset_models, model): + return { + "success": True, + "connected": True, + "message": "轻量连通性测试通过；provider 模型目录未返回当前别名，已按内置模型预设确认。", + "mode": "lightweight_models_with_preset_alias", + "provider": provider, + "provider_api": provider_api, + "model": model, + "models_count": len(model_ids), + "url": url, + } + return { + "success": False, + "connected": False, + "message": f"连接可用，但模型目录中没有当前模型: {model}", + "mode": "lightweight_models", + "provider": provider, + "model": model, + "models_count": len(model_ids), + "url": url, + } + + return { + "success": True, + "connected": True, + "message": "轻量连通性测试通过", + "mode": "lightweight_models", + "provider": provider, + "provider_api": provider_api, + "model": model, + "models_count": len(model_ids), + "url": url, + } + + def _web_search_provider_defaults(provider: str) -> dict: return web_search_provider_defaults(provider).model_dump() @@ -691,12 +935,21 @@ def _normalize_web_search_payload(web_search_payload: dict | None) -> dict: } -def _resolve_web_search_api_key(provider: str, provider_config: dict) -> tuple[str, str]: +def _resolve_web_search_api_key( + provider: str, + provider_config: dict, + default_provider: str | None = None, +) -> tuple[str, str]: saved_key = provider_config.get("api_key") or "" if saved_key: return str(saved_key), "runtime" preset = get_web_search_provider_preset(provider) - return _resolve_web_search_env_secret(preset.get("api_key_env") or "", "WEB_SEARCH_API_KEY") + value, source = _resolve_web_search_env_secret(preset.get("api_key_env") or "") + if value: + return value, source + if normalize_web_search_provider(provider) == normalize_web_search_provider(default_provider or "tavily"): + return _resolve_web_search_env_secret("WEB_SEARCH_API_KEY") + return "", "" def _build_web_search_payload( @@ -706,13 +959,22 @@ def _build_web_search_payload( current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {}) if update is None: return current_web_search - provider_id = normalize_web_search_provider(update.default_provider or update.provider) + provider_id = normalize_web_search_provider(update.provider) + default_provider = ( + normalize_web_search_provider(update.default_provider) + if update.default_provider is not None + else current_web_search["default_provider"] + ) current_providers = { provider: dict(config or {}) for provider, config in current_web_search.get("providers", {}).items() } current_provider = current_providers.get(provider_id) or _web_search_provider_defaults(provider_id) - current_key, current_key_source = _resolve_web_search_api_key(provider_id, current_provider) + current_key, current_key_source = _resolve_web_search_api_key( + provider_id, + current_provider, + current_web_search["default_provider"], + ) current_key_preview = _mask_secret(current_key, current_key_source)["preview"] provider_payload = { **_web_search_provider_defaults(provider_id), @@ -745,7 +1007,7 @@ def _build_web_search_payload( current_providers[provider_id] = provider_payload return { "enabled": update.enabled, - "default_provider": provider_id, + "default_provider": default_provider, "providers": current_providers, } @@ -754,12 +1016,12 @@ def _runtime_config_from_web_search_payload(web_search_payload: dict) -> WebSear normalized = _normalize_web_search_payload(web_search_payload) provider_id = normalized["default_provider"] provider_config = normalized["providers"].get(provider_id) or _web_search_provider_defaults(provider_id) - api_key, _source = _resolve_web_search_api_key(provider_id, provider_config) + api_key, _source = _resolve_web_search_api_key(provider_id, provider_config, provider_id) provider_models = { provider: WebSearchProviderConfig(**{ **config, "api_key": ( - api_key if provider == provider_id else _resolve_web_search_api_key(provider, config)[0] + api_key if provider == provider_id else _resolve_web_search_api_key(provider, config, provider_id)[0] ), }) for provider, config in normalized["providers"].items() @@ -893,7 +1155,11 @@ async def serialize_external_integrations(db: AsyncSession) -> dict: normalized_web_search["providers"].get(provider_id) or _web_search_provider_defaults(provider_id) ) - api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config) + api_key, api_key_source = _resolve_web_search_api_key( + provider_id, + provider_config, + normalized_web_search["default_provider"], + ) web_search_providers_payload[provider_id] = { **{ key: value @@ -980,10 +1246,7 @@ async def save_external_integrations_payload( update: ExternalIntegrationsUpdate, ) -> dict: current_payload = await get_setting_payload(db, "external_integrations") - current_ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {}) ai_payload = _build_ai_provider_payload(current_payload, update.ai_provider) - if _ai_provider_runtime_fingerprint(ai_payload) != _ai_provider_runtime_fingerprint(current_ai_payload): - await _validate_ai_provider_full_connection(ai_payload) web_search_payload = _build_web_search_payload(current_payload, update.web_search) ocr_payload = _build_ocr_payload(current_payload, update.ocr) @@ -1331,6 +1594,10 @@ async def connect_ai_provider_integration( db: AsyncSession = Depends(get_db), ): current_payload = await get_setting_payload(db, "external_integrations") + # Connection testing should validate the provider being edited, not the + # currently saved default provider. This is a transient draft only and is + # intentionally not persisted. + payload = payload.model_copy(update={"default_provider": payload.provider}) draft_ai_payload = _build_ai_provider_payload(current_payload, payload) runtime_config = _runtime_config_from_ai_payload(draft_ai_payload) quick_llm_config = { @@ -1354,24 +1621,16 @@ async def connect_ai_provider_integration( "message": "AI Provider 可访问，但当前 provider/model/key 未完整配置。", "status": status_result.model_dump(), } - prompt = await get_effective_prompt(db, AI_CONNECTION_TEST_PROMPT_KEY) - probe_result = await client.analyze( - SituationalAnalysisRequest( - title="快速连接测试", - objective=prompt.prompt, - system_prompt=prompt.system_prompt or None, - observations=[], - constraints=["Output only OK."], - ) + lightweight_result = await _check_ai_provider_lightweight( + quick_llm_config, + timeout_seconds=min( + int(runtime_config["timeout_seconds"] or 60), + AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS, + ), ) return { - "success": True, - "connected": True, - "message": "连接测试通过", + **lightweight_result, "status": status_result.model_dump(), - "provider": probe_result.provider, - "model": probe_result.model, - "mode": "quick_probe", } except HTTPException as exc: return { @@ -1389,16 +1648,38 @@ async def connect_ai_provider_integration( @router.get("/integrations/ai-provider/secrets") async def reveal_ai_provider_secrets( + request: Request, provider: str = Query(default=""), current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db), ): + requested_provider = _normalize_provider_id(provider) if provider else "default" + await _ensure_secret_reveal_allowed( + current_user=current_user, + request=request, + target_id=f"ai_provider:{requested_provider}", + details={"kind": "ai_provider", "provider": requested_provider}, + ) current_payload = await get_setting_payload(db, "external_integrations") ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {}) provider_id = _normalize_provider_id(provider or ai_payload["default_provider"]) provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id) api_key, api_key_source = _resolve_provider_api_key(provider_id, provider_config) service_token, service_token_source = _resolve_service_token(ai_payload) + await _record_integration_secret_reveal( + current_user=current_user, + request=request, + target_id=f"ai_provider:{provider_id}", + result="success", + details={ + "kind": "ai_provider", + "provider": provider_id, + "api_key_configured": bool(api_key), + "api_key_source": api_key_source, + "service_token_configured": bool(service_token), + "service_token_source": service_token_source, + }, + ) return { "provider": provider_id, "api_key": api_key, @@ -1417,10 +1698,18 @@ async def get_web_search_presets( @router.get("/integrations/web-search/secrets") async def reveal_web_search_secrets( + request: Request, provider: str = Query(default=""), current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db), ): + requested_provider = normalize_web_search_provider(provider) if provider else "default" + await _ensure_secret_reveal_allowed( + current_user=current_user, + request=request, + target_id=f"web_search:{requested_provider}", + details={"kind": "web_search", "provider": requested_provider}, + ) current_payload = await get_setting_payload(db, "external_integrations") web_search_payload = _normalize_web_search_payload(current_payload.get("web_search") or {}) provider_id = normalize_web_search_provider(provider or web_search_payload["default_provider"]) @@ -1428,7 +1717,23 @@ async def reveal_web_search_secrets( web_search_payload["providers"].get(provider_id) or _web_search_provider_defaults(provider_id) ) - api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config) + api_key, api_key_source = _resolve_web_search_api_key( + provider_id, + provider_config, + web_search_payload["default_provider"], + ) + await _record_integration_secret_reveal( + current_user=current_user, + request=request, + target_id=f"web_search:{provider_id}", + result="success", + details={ + "kind": "web_search", + "provider": provider_id, + "api_key_configured": bool(api_key), + "api_key_source": api_key_source, + }, + ) return { "provider": provider_id, "api_key": api_key, @@ -1438,12 +1743,31 @@ async def reveal_web_search_secrets( @router.get("/integrations/ocr/secrets") async def reveal_ocr_secrets( + request: Request, current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db), ): + await _ensure_secret_reveal_allowed( + current_user=current_user, + request=request, + target_id="ocr:default", + details={"kind": "ocr", "provider": "default"}, + ) current_payload = await get_setting_payload(db, "external_integrations") ocr_payload = _normalize_ocr_payload(current_payload.get("ocr") or {}) api_key, api_key_source = _resolve_ocr_api_key(ocr_payload) + await _record_integration_secret_reveal( + current_user=current_user, + request=request, + target_id=f"ocr:{ocr_payload['provider']}", + result="success", + details={ + "kind": "ocr", + "provider": ocr_payload["provider"], + "api_key_configured": bool(api_key), + "api_key_source": api_key_source, + }, + ) return { "provider": ocr_payload["provider"], "api_key": api_key, @@ -1547,9 +1871,17 @@ async def get_ai_provider_presets( async def refresh_ai_provider_preset( provider: str, current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), ): try: - return {"data": await refresh_llm_provider_preset(provider)} + provider_id = _normalize_provider_id(provider) + api_key = None + if provider_id == "opencode-go": + current_payload = await get_setting_payload(db, "external_integrations") + ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {}) + provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id) + api_key, _api_key_source = _resolve_provider_api_key(provider_id, provider_config) + return {"data": await refresh_llm_provider_preset(provider_id, api_key=api_key)} except ValueError as exc: raise HTTPException(status_code=404, detail=str(exc)) from exc except Exception as exc: diff --git a/backend/app/api/v1/system_control.py b/backend/app/api/v1/system_control.py index 079b35ad..3cc01032 100644 --- a/backend/app/api/v1/system_control.py +++ b/backend/app/api/v1/system_control.py @@ -8,9 +8,13 @@ from datetime import datetime from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from pydantic import BaseModel +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import ROOT_DIR from app.core.security import get_current_user +from app.db.session import get_db +from app.models.system_log import AuditLog, SystemLog from app.models.user import User from app.services.persistent_logs import record_audit_log, record_system_log from app.services.system_control import ( @@ -310,7 +314,120 @@ async def get_system_log_sources( current_user: User = Depends(get_current_user), ): ensure_super_admin(current_user) - return {"items": list_log_sources()} + return { + "items": [ + *list_log_sources(), + { + "source_id": "system-db", + "name": "系统事件", + "kind": "database", + "location": "table://system_logs", + "description": "后端持久化系统事件、AI 和采集器操作日志。", + "category": "database", + "status": "ok", + }, + { + "source_id": "audit-db", + "name": "审计事件", + "kind": "database", + "location": "table://audit_logs", + "description": "管理员敏感操作和密钥 reveal 审计记录。", + "category": "audit", + "status": "ok", + }, + ] + } + + +async def read_database_log_snapshot( + source_id: str, + *, + limit: int, + level: str, + levels: str | None, + start_date: str | None, + end_date: str | None, + search: str | None, + db: AsyncSession, +) -> dict | None: + selected_levels = set(normalize_log_level(item) for item in (levels or level).split(",") if item.strip()) + selected_levels.discard("all") + search_query = (search or "").strip().lower() + lines: list[str] = [] + + if source_id == "system-db": + query = select(SystemLog).order_by(SystemLog.occurred_at.desc().nullslast(), SystemLog.id.desc()).limit(limit * 5) + result = await db.execute(query) + records = result.scalars().all() + for record in records: + record_level = normalize_log_level(record.level) + if selected_levels and record_level not in selected_levels: + continue + occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else "" + if start_date and occurred_at and occurred_at < start_date: + continue + if end_date and occurred_at and occurred_at > end_date: + continue + line = " ".join( + part + for part in [ + record.occurred_at.isoformat() if record.occurred_at else "", + record_level.upper(), + record.source, + record.event or "", + record.message, + ] + if part + ) + if search_query and search_query not in line.lower(): + continue + lines.append(line) + elif source_id == "audit-db": + query = select(AuditLog).order_by(AuditLog.occurred_at.desc().nullslast(), AuditLog.id.desc()).limit(limit * 5) + result = await db.execute(query) + records = result.scalars().all() + for record in records: + occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else "" + if start_date and occurred_at and occurred_at < start_date: + continue + if end_date and occurred_at and occurred_at > end_date: + continue + line = " ".join( + part + for part in [ + record.occurred_at.isoformat() if record.occurred_at else "", + "INFO", + record.action, + record.target_type or "", + record.target_id or "", + record.result or "", + ] + if part + ) + if search_query and search_query not in line.lower(): + continue + lines.append(line) + else: + return None + + lines = list(reversed(lines[:limit])) + return { + "source_id": source_id, + "name": "系统事件" if source_id == "system-db" else "审计事件", + "kind": "database", + "location": "table://system_logs" if source_id == "system-db" else "table://audit_logs", + "description": "数据库持久化日志", + "category": "database" if source_id == "system-db" else "audit", + "status": "ok" if lines else "empty", + "level": level, + "selected_levels": sorted(selected_levels), + "search_query": search or "", + "available_levels": ["all", "error", "warning", "info", "debug"], + "daily_markers": [], + "line_limit": limit, + "line_count": len(lines), + "lines": lines, + } @router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse) @@ -323,6 +440,7 @@ async def get_system_log_snapshot( end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"), search: str | None = Query(None, description="Case-insensitive substring search"), current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), ): ensure_super_admin(current_user) @@ -345,15 +463,26 @@ async def get_system_log_snapshot( if normalized_start_date and normalized_end_date and normalized_start_date > normalized_end_date: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="start_date must be earlier than or equal to end_date") - snapshot = read_log_snapshot( + snapshot = await read_database_log_snapshot( source_id, - limit, + limit=limit, level=level, levels=levels, start_date=normalized_start_date, end_date=normalized_end_date, search=search, + db=db, ) + if snapshot is None: + snapshot = read_log_snapshot( + source_id, + limit, + level=level, + levels=levels, + start_date=normalized_start_date, + end_date=normalized_end_date, + search=search, + ) if snapshot is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Log source not found") return snapshot diff --git a/backend/app/api/v1/visualization.py b/backend/app/api/v1/visualization.py index 8616ced0..23c369b9 100644 --- a/backend/app/api/v1/visualization.py +++ b/backend/app/api/v1/visualization.py @@ -2159,6 +2159,24 @@ async def collect_compute_center_location( llm_failure_reason = llm_result.failure_reason if not candidates: + logger.warning_event( + "Compute center location collection returned no candidates", + event="visualization.compute_center.location_collect.completed", + context={ + "source_id": source_id, + "record_id": record_id, + "name": name, + "success": False, + "llm_failure_reason": llm_failure_reason, + "attempted_queries": list(attempted_queries), + "context": { + "operator": operator, + "site": site, + "city": city, + "country": country, + }, + }, + ) return { "source_id": source_id, "record_id": record_id, @@ -2180,13 +2198,34 @@ async def collect_compute_center_location( }, } + best_candidate = candidates[0].to_dict() + logger.info_event( + "Compute center location collection returned candidates", + event="visualization.compute_center.location_collect.completed", + context={ + "source_id": source_id, + "record_id": record_id, + "name": name, + "success": True, + "candidate_count": len(candidates), + "best_candidate": best_candidate, + "llm_failure_reason": llm_failure_reason, + "attempted_queries": list(attempted_queries), + "context": { + "operator": operator, + "site": site, + "city": city, + "country": country, + }, + }, + ) return { "source_id": source_id, "record_id": record_id, "name": name, "success": True, "candidates": [candidate.to_dict() for candidate in candidates], - "best_candidate": candidates[0].to_dict(), + "best_candidate": best_candidate, "attempted_queries": list(attempted_queries), "context": { "name": name, diff --git a/backend/app/api/v1/websocket.py b/backend/app/api/v1/websocket.py index 09fdd46b..3e512a85 100644 --- a/backend/app/api/v1/websocket.py +++ b/backend/app/api/v1/websocket.py @@ -14,6 +14,7 @@ from app.core.websocket.manager import manager logger = get_logger(__name__, service="api") router = APIRouter() +EARTH_UPDATES_CHANNEL = "earth_updates" async def authenticate_token(token: str) -> Optional[dict]: @@ -58,7 +59,7 @@ async def websocket_endpoint( is_anonymous = payload is None user_id = str(payload.get("sub")) if payload else f"anonymous:{id(websocket)}" - supported_channels = ["vessels", "earth_news"] if is_anonymous else [ + supported_channels = ["vessels", "earth_news", EARTH_UPDATES_CHANNEL] if is_anonymous else [ "gpu_clusters", "submarine_cables", "ixp_nodes", @@ -67,6 +68,7 @@ async def websocket_endpoint( "datasource_tasks", "vessels", "earth_news", + EARTH_UPDATES_CHANNEL, ] await manager.connect(websocket, user_id) diff --git a/backend/app/core/websocket/broadcaster.py b/backend/app/core/websocket/broadcaster.py index 4e295d5c..a80af079 100644 --- a/backend/app/core/websocket/broadcaster.py +++ b/backend/app/core/websocket/broadcaster.py @@ -8,6 +8,8 @@ from app.core.time import to_iso8601_utc from app.core.websocket.manager import manager +EARTH_UPDATES_CHANNEL = "earth_updates" + class DataBroadcaster: """Periodically broadcasts data to connected WebSocket clients""" @@ -83,6 +85,10 @@ class DataBroadcaster: channel=channel, ) + async def broadcast_earth_update(self, data: Dict[str, Any]): + """Broadcast Earth visualization refresh hints to connected clients.""" + await self.broadcast_custom(EARTH_UPDATES_CHANNEL, data) + def enqueue_vessel_update(self, data: Dict[str, Any]): vessels = data.get("vessels") if isinstance(data, dict) else None if not isinstance(vessels, list): diff --git a/backend/app/services/ai_client.py b/backend/app/services/ai_client.py index ed32bb09..f648af3b 100644 --- a/backend/app/services/ai_client.py +++ b/backend/app/services/ai_client.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json import httpx from fastapi import Depends, HTTPException, status @@ -57,6 +58,9 @@ class AIProviderClient: value = self.llm_config.get(key) if value not in (None, ""): headers[header_name] = str(value) + model_provider_apis = self.llm_config.get("model_provider_apis") + if isinstance(model_provider_apis, dict) and model_provider_apis: + headers["X-AI-Model-Provider-APIs"] = json.dumps(model_provider_apis) return headers async def get_status(self, request_id: str | None = None) -> AIProviderStatusResponse: diff --git a/backend/app/services/collectors/base.py b/backend/app/services/collectors/base.py index b9695d3b..2dcb090d 100644 --- a/backend/app/services/collectors/base.py +++ b/backend/app/services/collectors/base.py @@ -9,12 +9,38 @@ from sqlalchemy import select, text from sqlalchemy.ext.asyncio import AsyncSession from app.core.collected_data_fields import build_dynamic_metadata, get_record_field -from app.core.config import settings from app.core.countries import normalize_country from app.core.time import to_iso8601_utc from app.core.websocket.broadcaster import broadcaster from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source +EARTH_UPDATE_LAYER_HINTS: dict[str, list[str]] = { + "ris_live_bgp": ["bgp"], + "bgpstream_bgp": ["bgp"], + "top500_supercomputers": ["computeCenters"], + "epoch_ai_gpu": ["computeCenters"], + "huggingface_models": ["computeCenters"], + "huggingface_datasets": ["computeCenters"], + "huggingface_spaces": ["computeCenters"], + "telegeography_cables": ["cables"], + "telegeography_landing_points": ["cables"], + "telegeography_cable_systems": ["cables"], + "arcgis_cables": ["cables"], + "fao_landing_points": ["cables"], + "arcgis_landing_points": ["cables"], + "arcgis_cable_landing_relations": ["cables"], + "spacetrack_tle": ["satellites"], + "celestrak_tle": ["satellites"], + "barentswatch_vessels": ["vessels"], + "aisstream_vessels": ["vessels"], + "news_live_streams": ["media"], + "media_news_archive": ["news"], +} + + +def get_earth_update_layers_for_source(source: str) -> list[str]: + return EARTH_UPDATE_LAYER_HINTS.get(source, []) + class BaseCollector(ABC): """Abstract base class for data collectors""" @@ -70,6 +96,29 @@ class BaseCollector(ABC): ) self._last_broadcast_progress = rounded_progress + async def _publish_earth_update( + self, + *, + action: str, + records_processed: int, + task_id: int | None = None, + ) -> None: + layers = get_earth_update_layers_for_source(self.name) + if not layers: + return + await broadcaster.broadcast_earth_update( + { + "action": action, + "source": self.name, + "data_type": self.data_type, + "layers": layers, + "datasource_id": getattr(self, "_datasource_id", None), + "task_id": task_id, + "records_processed": records_processed, + "timestamp": to_iso8601_utc(datetime.now(UTC)), + } + ) + async def update_progress(self, records_processed: int, *, commit: bool = False, force: bool = False): """Update task progress - call this during data processing""" if self._current_task and self._db_session: @@ -187,7 +236,7 @@ class BaseCollector(ABC): result = await db.execute( select(DataSnapshot) - .where(DataSnapshot.source == self.name, DataSnapshot.is_current == True) + .where(DataSnapshot.source == self.name, DataSnapshot.is_current.is_(True)) .order_by(DataSnapshot.completed_at.desc().nullslast(), DataSnapshot.id.desc()) .limit(1) ) @@ -325,6 +374,11 @@ class BaseCollector(ABC): task.completed_at = datetime.now(UTC) await db.commit() await self._publish_task_update(force=True) + await self._publish_earth_update( + action="collector_completed", + records_processed=records_count, + task_id=task_id, + ) return { "status": "success", @@ -406,7 +460,7 @@ class BaseCollector(ABC): select(CollectedData) .where( CollectedData.source == self.name, - CollectedData.is_current == True, + CollectedData.is_current.is_(True), ) .order_by(CollectedData.entity_key.asc(), CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc()) ) diff --git a/backend/app/services/credential_guides.py b/backend/app/services/credential_guides.py index 73b334ba..2d628cad 100644 --- a/backend/app/services/credential_guides.py +++ b/backend/app/services/credential_guides.py @@ -3,9 +3,11 @@ from __future__ import annotations from dataclasses import dataclass +from copy import deepcopy from typing import Any from sqlalchemy import select +from sqlalchemy.orm.attributes import flag_modified from app.ai_tasks.prompts import get_effective_prompt from app.models.system_setting import SystemSetting @@ -135,33 +137,57 @@ DEFAULT_CREDENTIAL_GUIDES = { } +def _normalize_provider(provider: str) -> str: + return provider.strip().lower().replace(" ", "_") + + +def _credential_guide_default(provider: str) -> CredentialGuideDefault: + normalized = _normalize_provider(provider) + known = DEFAULT_CREDENTIAL_GUIDES.get(normalized) + if known is not None: + return known + title = f"{normalized or 'collector'} 凭证配置教程" + return CredentialGuideDefault( + provider=normalized, + title=title, + prompt=( + f"请生成一份中文教程，指导开发者为 Planet 采集器配置 {normalized} 凭证。" + "教程要面向已经有本地开发环境的人，包含官方入口或文档查找方式、" + "获取 API Key / Token / Client credentials 的通用步骤、在 Planet 采集器配置中" + "填写凭证字段、连接测试、保存、常见失败排查。不要编造具体页面按钮文案；" + "如果公开资料不足，必须明确提醒以 provider 官方文档和当前控制台页面为准。" + ), + markdown="", + ) + + async def _get_guide_store(db) -> tuple[SystemSetting | None, dict[str, Any]]: result = await db.execute( select(SystemSetting).where(SystemSetting.category == CREDENTIAL_GUIDES_CATEGORY) ) record = result.scalar_one_or_none() - payload = dict(record.payload or {}) if record and isinstance(record.payload, dict) else {} + payload = deepcopy(record.payload) if record and isinstance(record.payload, dict) else {} return record, payload async def get_credential_guide(db, provider: str) -> dict[str, Any]: - default = DEFAULT_CREDENTIAL_GUIDES.get(provider) - if default is None: - raise ValueError(f"Unsupported credential guide provider: {provider}") + provider = _normalize_provider(provider) + default = _credential_guide_default(provider) _record, store = await _get_guide_store(db) custom = store.get(provider) if isinstance(store.get(provider), dict) else None + has_default_markdown = bool(default.markdown.strip()) return { "provider": provider, "title": custom.get("title") if custom else default.title, "markdown": custom.get("markdown") if custom else default.markdown, "prompt": default.prompt, - "source": "ai" if custom else "default", + "source": "ai" if custom else "default" if has_default_markdown else "missing", "sources": custom.get("sources", []) if custom else [], "verification_status": ( custom.get("verification_status", "verified_with_search_evidence") if custom - else "default_unverified" + else "default_unverified" if has_default_markdown else "missing" ), "verification_error": custom.get("verification_error") if custom else None, } @@ -177,9 +203,8 @@ async def save_credential_guide( verification_status: str = "verified_with_search_evidence", verification_error: str | None = None, ) -> dict[str, Any]: - default = DEFAULT_CREDENTIAL_GUIDES.get(provider) - if default is None: - raise ValueError(f"Unsupported credential guide provider: {provider}") + provider = _normalize_provider(provider) + default = _credential_guide_default(provider) record, store = await _get_guide_store(db) store[provider] = { @@ -192,21 +217,21 @@ async def save_credential_guide( if record is None: db.add(SystemSetting(category=CREDENTIAL_GUIDES_CATEGORY, payload=store)) else: - record.payload = store + record.payload = deepcopy(store) + flag_modified(record, "payload") await db.commit() return await get_credential_guide(db, provider) async def reset_credential_guide(db, provider: str) -> dict[str, Any]: - default = DEFAULT_CREDENTIAL_GUIDES.get(provider) - if default is None: - raise ValueError(f"Unsupported credential guide provider: {provider}") + provider = _normalize_provider(provider) record, store = await _get_guide_store(db) if provider in store: store.pop(provider, None) if record is not None: - record.payload = store + record.payload = deepcopy(store) + flag_modified(record, "payload") await db.commit() return await get_credential_guide(db, provider) @@ -217,9 +242,8 @@ async def generate_credential_guide( ai_client: AIProviderClient, web_search_client: WebSearchClient | None = None, ) -> dict[str, Any]: - default = DEFAULT_CREDENTIAL_GUIDES.get(provider) - if default is None: - raise ValueError(f"Unsupported credential guide provider: {provider}") + provider = _normalize_provider(provider) + default = _credential_guide_default(provider) search_evidence: list[dict[str, Any]] = [] search_error: str | None = None diff --git a/backend/app/services/datasource_connectivity.py b/backend/app/services/datasource_connectivity.py index 9bc89063..f6c7fb3a 100644 --- a/backend/app/services/datasource_connectivity.py +++ b/backend/app/services/datasource_connectivity.py @@ -18,6 +18,7 @@ from app.models.datasource import DataSource from app.models.datasource_config import DataSourceConfig from app.models.system_setting import SystemSetting from app.services.barentswatch import ( + BarentsWatchConfig, _read_zshrc_env, fetch_barentswatch_access_token, resolve_barentswatch_config, @@ -44,6 +45,20 @@ def _resolve_spacetrack_credentials() -> tuple[str, str, str]: return username, password, source or "missing" +def _resolve_spacetrack_credentials_with_override( + credential_override: dict[str, str] | None = None, +) -> tuple[str, str, str]: + if credential_override and ( + credential_override.get("username") or credential_override.get("password") + ): + return ( + str(credential_override.get("username") or ""), + str(credential_override.get("password") or ""), + "draft", + ) + return _resolve_spacetrack_credentials() + + async def _resolve_aisstream_api_key( db=None, credential_override: dict[str, str] | None = None, @@ -126,7 +141,9 @@ async def build_builtin_connectivity_checksum( } ) elif credential_provider == "spacetrack": - username, password, credential_source = _resolve_spacetrack_credentials() + username, password, credential_source = _resolve_spacetrack_credentials_with_override( + credential_override + ) has_credentials = bool(username and password) credential_fingerprint = _sha256_json( { @@ -230,7 +247,16 @@ async def test_builtin_connectivity( try: async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: if credential_context["credential_provider"] == "barentswatch": - barentswatch_config = await resolve_barentswatch_config(db) + if credential_override: + barentswatch_config = BarentsWatchConfig( + endpoint=str(request_endpoint or ""), + client_id=str(credential_override.get("client_id") or ""), + client_secret=str(credential_override.get("client_secret") or ""), + credential_source="draft", + endpoint_source="draft", + ) + else: + barentswatch_config = await resolve_barentswatch_config(db) token = await fetch_barentswatch_access_token(client, barentswatch_config) if not token: return { @@ -243,7 +269,9 @@ async def test_builtin_connectivity( } request_headers["Authorization"] = f"Bearer {token}" elif credential_context["credential_provider"] == "spacetrack": - username, password, _source = _resolve_spacetrack_credentials() + username, password, _source = _resolve_spacetrack_credentials_with_override( + credential_override + ) login_url = "https://www.space-track.org/ajaxauth/login" login_response = await client.post( login_url, diff --git a/backend/app/services/earth_layer_cache.py b/backend/app/services/earth_layer_cache.py index 1d9dc612..913f60c5 100644 --- a/backend/app/services/earth_layer_cache.py +++ b/backend/app/services/earth_layer_cache.py @@ -270,15 +270,31 @@ def apply_payload_budget(payload: dict[str, Any], policy: EarthLayerCachePolicy) def invalidate_earth_layer_cache_for_source(source: str) -> int: source_key = str(source or "").strip() patterns = { + "barentswatch_vessels": ["vessels*", "summary*"], + "aisstream_vessels": ["vessels*", "summary*"], + "telegeography_cables": ["cables*", "landing-points*", "summary*"], + "telegeography_landing": ["landing-points*", "summary*"], + "telegeography_landing_points": ["landing-points*", "summary*"], + "telegeography_systems": ["cables*", "summary*"], + "telegeography_cable_systems": ["cables*", "summary*"], "arcgis_cables": ["cables*", "landing-points*", "summary*"], "arcgis_landing_points": ["landing-points*", "summary*"], "arcgis_cable_landing_relation": ["landing-points*", "summary*"], + "arcgis_cable_landing_relations": ["landing-points*", "summary*"], + "fao_landing_points": ["landing-points*", "summary*"], "celestrak_tle": ["satellites*", "summary*"], "spacetrack_tle": ["satellites*", "summary*"], "top500": ["compute-centers*", "summary*"], + "top500_supercomputers": ["compute-centers*", "summary*"], "epoch_ai_gpu": ["compute-centers*", "summary*"], + "huggingface_models": ["compute-centers*", "summary*"], + "huggingface_datasets": ["compute-centers*", "summary*"], + "huggingface_spaces": ["compute-centers*", "summary*"], "ris_live_bgp": ["bgp*", "summary*"], "bgpstream_bgp": ["bgp*", "summary*"], + "iptoasn_prefix_geo": ["bgp*", "summary*"], + "opengeofeed_prefix_geo": ["bgp*", "summary*"], + "nro_delegated_prefix_geo": ["bgp*", "summary*"], }.get(source_key, []) deleted = 0 for layer_pattern in patterns: diff --git a/backend/app/services/earth_news.py b/backend/app/services/earth_news.py index be79395c..baa1080c 100644 --- a/backend/app/services/earth_news.py +++ b/backend/app/services/earth_news.py @@ -853,7 +853,7 @@ def _parse_feed_entries(xml_text: str, source: NewsFeedSource) -> list[ParsedNew if not clean_title or not link: continue - item_source = _normalize_source_name(clean_title, source.name) + item_source = source.name display_title = clean_title if source.source_type == "aggregated" and " - " in clean_title: parts = clean_title.rsplit(" - ", 1) diff --git a/backend/app/services/llm_provider_catalog.py b/backend/app/services/llm_provider_catalog.py index 89da1a2d..0ef9e06f 100644 --- a/backend/app/services/llm_provider_catalog.py +++ b/backend/app/services/llm_provider_catalog.py @@ -7,6 +7,26 @@ from typing import Any import httpx MODELS_DEV_URL = "https://models.dev/api.json" +OPENCODE_GO_MODELS_URL = "https://opencode.ai/zen/go/v1/models" + +OPENCODE_GO_MODEL_PROVIDER_APIS = { + "minimax-m2.7": "anthropic-messages", + "minimax-m2.5": "anthropic-messages", +} +OPENCODE_GO_FALLBACK_MODELS = [ + "minimax-m2.7", + "minimax-m2.5", + "kimi-k2.6", + "kimi-k2.5", + "glm-5.1", + "glm-5", + "deepseek-v4-pro", + "deepseek-v4-flash", + "qwen3.6-plus", + "qwen3.5-plus", + "mimo-v2.5-pro", + "mimo-v2.5", +] FALLBACK_LLM_PROVIDER_PRESETS: dict[str, dict[str, Any]] = { @@ -80,6 +100,17 @@ FALLBACK_LLM_PROVIDER_PRESETS: dict[str, dict[str, Any]] = { "api_key_env": "OPENROUTER_API_KEY", "source": "fallback", }, + "opencode-go": { + "provider": "opencode-go", + "label": "OpenCode Go", + "provider_api": "openai-completions", + "base_url": "https://opencode.ai/zen/go/v1", + "model": "glm-5.1", + "models": OPENCODE_GO_FALLBACK_MODELS, + "model_provider_apis": OPENCODE_GO_MODEL_PROVIDER_APIS, + "api_key_env": "OPENCODE_GO_API_KEY", + "source": "fallback", + }, "ollama": { "provider": "ollama", "label": "Ollama Local", @@ -114,8 +145,43 @@ def get_fallback_llm_provider_preset(provider: str) -> dict[str, Any]: return dict(FALLBACK_LLM_PROVIDER_PRESETS[key]) -async def refresh_llm_provider_preset(provider: str) -> dict[str, Any]: +def _opencode_go_model_provider_apis(model_ids: list[str]) -> dict[str, str]: + return { + model_id: OPENCODE_GO_MODEL_PROVIDER_APIS.get(model_id, "openai-completions") + for model_id in model_ids + } + + +async def refresh_llm_provider_preset(provider: str, api_key: str | None = None) -> dict[str, Any]: fallback = get_fallback_llm_provider_preset(provider) + if fallback["provider"] == "opencode-go": + headers = {"User-Agent": "Planet/1.0"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client: + response = await client.get( + OPENCODE_GO_MODELS_URL, + headers=headers, + ) + response.raise_for_status() + payload = response.json() + + data = payload.get("data") if isinstance(payload, dict) else [] + model_ids = [ + str(item.get("id")) + for item in data + if isinstance(item, dict) and item.get("id") + ][:120] + if not model_ids: + model_ids = fallback["models"] + return { + **fallback, + "model": fallback["model"] if fallback["model"] in model_ids else model_ids[0], + "models": model_ids, + "model_provider_apis": _opencode_go_model_provider_apis(model_ids), + "source": OPENCODE_GO_MODELS_URL, + } + models_dev_key = MODELS_DEV_PROVIDER_KEYS.get(fallback["provider"]) if not models_dev_key: return fallback diff --git a/backend/app/services/location/llm_fallback.py b/backend/app/services/location/llm_fallback.py index c523d889..00c4e659 100644 --- a/backend/app/services/location/llm_fallback.py +++ b/backend/app/services/location/llm_fallback.py @@ -9,6 +9,7 @@ from typing import Any, Iterable from sqlalchemy.ext.asyncio import AsyncSession +from app.core.logging import get_logger from app.core.countries import COUNTRY_ENTRIES, normalize_country from app.schemas.ai import SituationalAnalysisRequest from app.ai_tasks.prompts import get_effective_prompt @@ -29,6 +30,9 @@ DEFAULT_MIN_CONFIDENCE = 0.55 LOCATION_NORMALIZE_PROMPT_KEY = "location.factcheck.normalize" LOCATION_RESOLVE_PROMPT_KEY = "location.factcheck.resolve" MODEL_CONFIDENCE_WEIGHT = 0.25 +LOG_TEXT_LIMIT = 1200 +LOG_EVIDENCE_LIMIT = 5 +logger = get_logger(__name__, service="location") _geocode_llm_city = build_default_nominatim_geocoder() _LLM_LOCATION_NAME_KEYS = ( "matched_location_name", @@ -97,6 +101,35 @@ class LocationEvidenceScore: summary: str +def _truncate_log_text(value: Any, limit: int = LOG_TEXT_LIMIT) -> str: + text = coerce_str(value) + if len(text) <= limit: + return text + return f"{text[:limit]}…" + + +def _summarize_search_evidence(evidence: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + for item in (evidence or [])[:LOG_EVIDENCE_LIMIT]: + if not isinstance(item, dict): + continue + items.append( + { + "title": _truncate_log_text(item.get("title"), 180), + "source": _truncate_log_text(item.get("source") or item.get("name"), 120), + "url": _truncate_log_text(item.get("url"), 240), + "snippet": _truncate_log_text( + item.get("snippet") + or item.get("content") + or item.get("text") + or item.get("summary"), + 360, + ), + } + ) + return items + + def _first_json_object(text: str) -> dict[str, Any] | None: stripped = text.strip() if not stripped: @@ -162,6 +195,64 @@ def _evidence_label(item: Any) -> str: return coerce_str(item) +def _evidence_text(item: dict[str, Any]) -> str: + return " ".join( + coerce_str(item.get(key)) + for key in ("title", "source", "name", "url", "snippet", "content", "text", "quote", "summary") + if coerce_str(item.get(key)) + ) + + +def _search_evidence_entity_match(item: dict[str, Any], query: LocationQuery) -> bool: + haystack = normalize_text(_evidence_text(item)) + if not haystack: + return False + needles = [ + coerce_str(query.name), + *[coerce_str(alias) for alias in query.aliases], + ] + return any(normalize_text(needle) and normalize_text(needle) in haystack for needle in needles) + + +def _evidence_has_location_assertion(item: dict[str, Any], city: str) -> bool: + normalized_city = normalize_text(city) + text = normalize_text(_evidence_text(item)) + if not normalized_city or normalized_city not in text: + return False + assertion_terms = ( + "located", + "situated", + "built", + "hosted", + "deployed", + "installed", + "facility", + "campus", + "site", + "data center", + "datacenter", + "supercomputer center", + "位于", + "位於", + "坐落", + "建置", + "設置", + "设置", + ) + return any(term in text for term in assertion_terms) + + +def _city_is_unsupported_name_hint(payload: dict[str, Any], query: LocationQuery, evidence_items: list[dict[str, Any]]) -> bool: + city = coerce_str(payload.get("city") or query.city) + if not city: + return False + normalized_city = normalize_text(city) + normalized_name = normalize_text(query.name) + if not normalized_city or not normalized_name or normalized_city not in normalized_name: + return False + return not any(_evidence_has_location_assertion(item, city) for item in evidence_items) + + def _normalize_llm_precision(value: Any) -> str: text = coerce_str(value).lower() return LLM_PRECISION_ALIASES.get(text, text) @@ -588,6 +679,7 @@ def _weak_evidence_penalty( payload: dict[str, Any], evidence_items: list[dict[str, Any]], *, + query: LocationQuery, entity_match: float, geography_match: float, conflict_penalty: float, @@ -598,6 +690,8 @@ def _weak_evidence_penalty( penalty += 0.20 if any(_truthy_evidence_field(item, "ambiguous") for item in evidence_items): penalty += 0.15 + if _city_is_unsupported_name_hint(payload, query, evidence_items): + penalty += 0.10 if conflict_penalty == 0.0 and entity_match > 0 and geography_match >= 0.20: return min(penalty, 0.15) return min(penalty, 0.30) @@ -620,6 +714,7 @@ def _score_llm_location_payload( weak_evidence_penalty = _weak_evidence_penalty( payload, evidence_items, + query=query, entity_match=entity_match, geography_match=geography_match, conflict_penalty=conflict_penalty, @@ -636,6 +731,8 @@ def _score_llm_location_payload( - weak_evidence_penalty ) score = min(max(score, 0.0), 1.0) + if _city_is_unsupported_name_hint(payload, query, evidence_items): + score = min(score, 0.54) summary = ( f"combined={score:.2f}; model={model_confidence:.2f}; " f"source={source_quality:.2f}; entity={entity_match:.2f}; " @@ -847,15 +944,43 @@ async def collect_location_search_evidence( ) -> LocationSearchEvidenceResult: search_query = _location_search_query(query, entity_type) attempt = f"web_search:{entity_type}:{search_query}" + logger.info_event( + "Collecting location search evidence", + event="location.factcheck.web_search.start", + context={ + "entity_type": entity_type, + "search_query": search_query, + "location_query": _query_context(query), + "max_results": max_results, + }, + ) try: evidence = await web_search_client.search(search_query, max_results=max_results) except WebSearchError as exc: + logger.warning_event( + "Location search evidence failed", + event="location.factcheck.web_search.failed", + context={ + "entity_type": entity_type, + "search_query": search_query, + "error": str(exc), + }, + ) return LocationSearchEvidenceResult( evidence=[], attempted_queries=[attempt], failure_reason=f"WebSearch location evidence failed: {exc}", ) except Exception as exc: + logger.warning_event( + "Location search evidence unavailable", + event="location.factcheck.web_search.unavailable", + context={ + "entity_type": entity_type, + "search_query": search_query, + "error": str(exc), + }, + ) return LocationSearchEvidenceResult( evidence=[], attempted_queries=[attempt], @@ -863,11 +988,29 @@ async def collect_location_search_evidence( ) normalized = normalize_search_evidence(evidence, limit=max_results) if not normalized: + logger.warning_event( + "Location search returned no usable evidence", + event="location.factcheck.web_search.empty", + context={ + "entity_type": entity_type, + "search_query": search_query, + }, + ) return LocationSearchEvidenceResult( evidence=[], attempted_queries=[attempt], failure_reason="WebSearch returned no usable location evidence.", ) + logger.info_event( + "Collected location search evidence", + event="location.factcheck.web_search.result", + context={ + "entity_type": entity_type, + "search_query": search_query, + "evidence_count": len(normalized), + "evidence": _summarize_search_evidence(normalized), + }, + ) return LocationSearchEvidenceResult( evidence=normalized, attempted_queries=[attempt], @@ -947,6 +1090,15 @@ async def collect_llm_location_fallback_candidate( """ attempt = f"llm_factcheck:{entity_type}:{coerce_str(query.name) or 'unknown'}" if search_evidence is not None and not search_evidence: + logger.warning_event( + "Skipping LLM location factcheck because search evidence is empty", + event="location.factcheck.llm.skipped_no_evidence", + context={ + "entity_type": entity_type, + "attempt": attempt, + "location_query": _query_context(query), + }, + ) return LocationLLMFallbackResult( candidates=[], attempted_queries=[attempt], @@ -986,20 +1138,66 @@ async def collect_llm_location_fallback_candidate( "Return evidence as objects when possible, including source, url, source_type, and entity_match.", "Include source names or URLs in evidence when known. The backend will recompute the final confidence from model confidence plus evidence quality.", "If search_evidence is provided, use only that evidence as factual support.", + "Do not treat a website footer, office address, publisher address, or contact address as the entity's physical location.", + "If the entity name contains a city name, do not choose that city unless evidence explicitly says the entity/facility/supercomputer is located, hosted, built, deployed, or installed there.", "Prefer the facility/site if known; otherwise use the best supported city.", ], ) + logger.info_event( + "Sending location factcheck request to LLM", + event="location.factcheck.llm.request", + context={ + "entity_type": entity_type, + "attempt": attempt, + "title": request.title, + "objective": request.objective, + "location_query": request.context.get("location_query"), + "observations": request.observations, + "constraints": request.constraints, + "search_evidence_count": len(search_evidence or []), + "search_evidence": _summarize_search_evidence(search_evidence), + }, + ) try: response = await provider_client.analyze(request) except Exception as exc: + logger.warning_event( + "LLM location factcheck failed", + event="location.factcheck.llm.failed", + context={ + "entity_type": entity_type, + "attempt": attempt, + "error": str(exc), + }, + ) return LocationLLMFallbackResult( candidates=[], attempted_queries=[attempt], failure_reason=f"LLM location factcheck failed: {exc}", ) + logger.info_event( + "Received location factcheck response from LLM", + event="location.factcheck.llm.response", + context={ + "entity_type": entity_type, + "attempt": attempt, + "provider": response.provider, + "model": response.model, + "content": _truncate_log_text(response.content, 2000), + }, + ) payload = _first_json_object(response.content) if payload is None: + logger.warning_event( + "LLM location factcheck response was not strict JSON; attempting repair", + event="location.factcheck.llm.non_json", + context={ + "entity_type": entity_type, + "attempt": attempt, + "content": _truncate_log_text(response.content, 1200), + }, + ) payload = await _repair_location_payload_from_text( provider_client=provider_client, raw_text=response.content, @@ -1009,9 +1207,17 @@ async def collect_llm_location_fallback_candidate( ) if payload is None: payload = _payload_from_free_text(response.content, query=query) - if payload is None: + if payload is None and entity_type != "compute_center": payload = _payload_from_query_name_geocode(query) if payload is None: + logger.warning_event( + "LLM location factcheck produced no parseable payload", + event="location.factcheck.llm.unparseable", + context={ + "entity_type": entity_type, + "attempt": attempt, + }, + ) return LocationLLMFallbackResult( candidates=[], attempted_queries=[attempt], @@ -1032,7 +1238,7 @@ async def collect_llm_location_fallback_candidate( "url": item.get("url"), "text": item.get("snippet") or item.get("content"), "source_type": "web_search", - "entity_match": True, + "entity_match": _search_evidence_entity_match(item, query), } for item in search_evidence if isinstance(item, dict) @@ -1054,6 +1260,18 @@ async def collect_llm_location_fallback_candidate( if candidate is None: if city_geocode_failure and rejection_reason == "missing, invalid, or zero latitude/longitude": rejection_reason = f"{rejection_reason}; {city_geocode_failure}" + logger.warning_event( + "Rejected LLM location factcheck candidate", + event="location.factcheck.llm.rejected", + context={ + "entity_type": entity_type, + "attempt": attempt, + "reason": rejection_reason, + "payload": payload, + "search_evidence_count": len(search_evidence or []), + "search_evidence": _summarize_search_evidence(search_evidence), + }, + ) return LocationLLMFallbackResult( candidates=[], attempted_queries=[attempt], @@ -1062,6 +1280,18 @@ async def collect_llm_location_fallback_candidate( + (f": {rejection_reason}." if rejection_reason else ".") ), ) + logger.info_event( + "Accepted LLM location factcheck candidate", + event="location.factcheck.llm.accepted", + context={ + "entity_type": entity_type, + "attempt": attempt, + "candidate": candidate.to_dict(), + "payload": payload, + "search_evidence_count": len(search_evidence or []), + "search_evidence": _summarize_search_evidence(search_evidence), + }, + ) return LocationLLMFallbackResult( candidates=[candidate], attempted_queries=[attempt], diff --git a/backend/app/services/system_logs.py b/backend/app/services/system_logs.py index 136cf373..b1de198a 100644 --- a/backend/app/services/system_logs.py +++ b/backend/app/services/system_logs.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os import re import shutil import subprocess @@ -86,6 +87,7 @@ class LogSource: status: str = "ok" buffer_key: str | None = None container_name: str | None = None + fallback_locations: tuple[str, ...] = () @dataclass @@ -104,22 +106,38 @@ class DailyLogMarker: dominant_level: str +def _planet_state_dir() -> Path: + configured = os.getenv("PLANET_STATE_DIR") + if configured: + return Path(configured).expanduser() + xdg_state = os.getenv("XDG_STATE_HOME") + if xdg_state: + return Path(xdg_state).expanduser() / "planet" + return Path.home() / ".local" / "state" / "planet" + + +def _state_log_path(filename: str) -> str: + return str(_planet_state_dir() / filename) + + LOG_SOURCES: dict[str, LogSource] = { "backend": LogSource( source_id="backend", name="后端服务", kind="file", - location="/tmp/planet_backend.log", + location=_state_log_path("backend.log"), description="FastAPI 后端、调度器和采集任务共享日志。", category="service", + fallback_locations=("/tmp/planet_backend.log",), ), "frontend": LogSource( source_id="frontend", name="前端开发服务", kind="file", - location="/tmp/planet_frontend.log", + location=_state_log_path("frontend.log"), description="控制台与 Earth 前端开发服务输出。", category="service", + fallback_locations=("/tmp/planet_frontend.log",), ), "ai-provider": LogSource( source_id="ai-provider", @@ -164,9 +182,18 @@ def normalize_log_levels(level: str | None = None, levels: str | None = None) -> return tuple(normalized_levels) +def resolve_file_log_path(source: LogSource) -> Path: + primary = Path(source.location).expanduser() + candidates = (primary, *(Path(item).expanduser() for item in source.fallback_locations)) + for candidate in candidates: + if candidate.exists(): + return candidate + return primary + + def get_source_status(source: LogSource) -> str: if source.kind == "file": - path = Path(source.location) + path = resolve_file_log_path(source) if not path.exists(): return "missing" return "ok" if path.stat().st_size > 0 else "empty" @@ -190,7 +217,7 @@ def list_log_sources() -> list[dict[str, str]]: "source_id": source.source_id, "name": source.name, "kind": source.kind, - "location": source.location, + "location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location, "description": source.description, "category": source.category, "status": get_source_status(source), @@ -339,7 +366,7 @@ def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry: def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]: - path = Path(source.location) + path = resolve_file_log_path(source) if not path.exists(): return [] with path.open("r", encoding="utf-8", errors="replace") as handle: @@ -511,7 +538,7 @@ def read_log_snapshot( "source_id": source.source_id, "name": source.name, "kind": source.kind, - "location": source.location, + "location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location, "description": source.description, "category": source.category, "status": get_source_status(source), diff --git a/backend/app/services/tv_streams.py b/backend/app/services/tv_streams.py index a6d83d91..63e97f91 100644 --- a/backend/app/services/tv_streams.py +++ b/backend/app/services/tv_streams.py @@ -385,13 +385,18 @@ def build_public_tv_payload( settings_payload: dict[str, Any], collected_sources: list[dict[str, Any]], ) -> dict[str, Any]: + configured_by_id = { + source["id"]: source + for source in settings_payload["sources"] + if source.get("id") + } configured_sources = [ source for source in settings_payload["sources"] if source["is_enabled"] ] merged_by_id = {source["id"]: source for source in configured_sources} for source in collected_sources: - if source["id"] in merged_by_id or not source["is_enabled"]: + if source["id"] in configured_by_id or not source["is_enabled"]: continue merged_by_id[source["id"]] = source diff --git a/backend/tests/test_datasources_batch.py b/backend/tests/test_datasources_batch.py index 5f1ecf0f..fa8b7b0a 100644 --- a/backend/tests/test_datasources_batch.py +++ b/backend/tests/test_datasources_batch.py @@ -4,6 +4,7 @@ import pytest from app.api.v1 import datasources as datasources_api from app.models.datasource import DataSource +from app.services import earth_layer_cache as earth_cache def make_datasource( @@ -53,6 +54,71 @@ def test_filter_datasources_by_product_status_and_collected_state(): assert filtered == [vessels] +def test_serialize_datasource_row_can_skip_endpoint_resolution(): + datasource = make_datasource(7, "arcgis_cables", last_status="success", module="L2") + + class ExplodingConfig: + def get_yaml_url(self, _source): + raise AssertionError("endpoint resolution should be skipped") + + row = datasources_api.serialize_datasource_row( + datasource, + running_tasks={}, + latest_tasks={}, + record_counts={"arcgis_cables": 343}, + endpoint_overrides={}, + config=ExplodingConfig(), + include_endpoint=False, + ) + + assert row["id"] == 7 + assert row["source"] == "arcgis_cables" + assert row["module"] == "L2" + assert row["last_status"] == "success" + assert row["collected_records"] == 343 + assert row["has_collected_data"] is True + assert "endpoint" not in row + + +def test_serialize_datasource_row_includes_endpoint_when_requested(): + datasource = make_datasource(8, "arcgis_landing_points") + + class Config: + def get_yaml_url(self, source): + return f"https://example.test/{source}" + + row = datasources_api.serialize_datasource_row( + datasource, + running_tasks={}, + latest_tasks={}, + record_counts={}, + endpoint_overrides={}, + config=Config(), + include_endpoint=True, + ) + + assert row["endpoint"] == "https://example.test/arcgis_landing_points" + + +def test_invalidate_earth_layer_cache_for_source_covers_datasource_aliases(monkeypatch): + patterns: list[str] = [] + + def fake_delete_pattern(pattern: str) -> int: + patterns.append(pattern) + return 1 + + monkeypatch.setattr(earth_cache.earth_layer_cache, "delete_pattern", fake_delete_pattern) + + deleted = earth_cache.invalidate_earth_layer_cache_for_source("telegeography_cables") + + assert deleted == 3 + assert patterns == [ + "earth:layer:v1:cables*", + "earth:layer:v1:landing-points*", + "earth:layer:v1:summary*", + ] + + @pytest.mark.asyncio async def test_trigger_datasource_batch_skips_disabled_and_frequency_window(monkeypatch): now = datetime.now(timezone.utc) diff --git a/backend/tests/test_earth_news.py b/backend/tests/test_earth_news.py index 5ca5b8ef..e989c5f9 100644 --- a/backend/tests/test_earth_news.py +++ b/backend/tests/test_earth_news.py @@ -9,6 +9,7 @@ from app.services.earth_news import ( ParsedNewsItem, _enrich_items_with_target_locations, _extract_target_location_from_text, + _parse_feed_entries, _serialize_item, get_earth_news_payload, ) @@ -109,6 +110,60 @@ def test_serialize_item_includes_inferred_target_location(): assert payload["location_meta"]["ai_error"] is None +def test_parse_plain_rss_uses_feed_name_as_source(): + source = NewsFeedSource( + id="bbc-world", + name="BBC World", + region="global", + feed_url="https://feeds.bbci.co.uk/news/world/rss.xml", + homepage_url="https://www.bbc.com/news/world", + ) + xml = """ + + + + This may be the last time you hear my voice: Political executions surge in Iran since start of war + Story summary + https://www.bbc.com/news/example + Fri, 15 May 2026 03:00:00 GMT + + + + """ + + items = _parse_feed_entries(xml, source) + + assert items[0].title == "This may be the last time you hear my voice: Political executions surge in Iran since start of war" + assert items[0].source == "BBC World" + + +def test_parse_aggregated_rss_splits_publisher_from_title(): + source = NewsFeedSource( + id="global-scan", + name="Global Monitor / World", + region="global", + feed_url="https://news.google.com/rss", + homepage_url="https://news.google.com/", + source_type="aggregated", + ) + xml = """ + + + + Example headline - Reuters + Story summary + https://news.google.com/example + + + + """ + + items = _parse_feed_entries(xml, source) + + assert items[0].title == "Example headline" + assert items[0].source == "Reuters" + + @pytest.mark.asyncio async def test_enrich_items_with_target_locations_uses_ai_and_geocode(monkeypatch): item = ParsedNewsItem( diff --git a/backend/tests/test_location_pipeline.py b/backend/tests/test_location_pipeline.py index 88519985..15d4e724 100644 --- a/backend/tests/test_location_pipeline.py +++ b/backend/tests/test_location_pipeline.py @@ -700,7 +700,7 @@ async def test_llm_location_fallback_repairs_non_json_answer(monkeypatch): @pytest.mark.asyncio -async def test_llm_location_fallback_accepts_taipei_name_hint_with_weak_wording(monkeypatch): +async def test_llm_location_fallback_rejects_city_from_name_without_location_evidence(monkeypatch): monkeypatch.setattr( llm_fallback, "_geocode_llm_city", @@ -740,18 +740,60 @@ async def test_llm_location_fallback_accepts_taipei_name_hint_with_weak_wording( entity_type="compute_center", ) - assert result.failure_reason is None - candidate = result.candidates[0] - assert candidate.city == "Taipei" - assert candidate.confidence >= 0.55 - breakdown = candidate.suggested_registry_entry["llm_score_breakdown"] - assert breakdown["weak_evidence_penalty"] <= 0.15 - assert breakdown["conflict_penalty"] == 0 - assert breakdown["name_location_hint"] > 0 + assert result.candidates == [] + assert result.failure_reason is not None + assert "below minimum" in result.failure_reason @pytest.mark.asyncio -async def test_llm_location_fallback_geocodes_city_from_entity_name_when_llm_unparseable(monkeypatch): +async def test_llm_location_fallback_accepts_explicit_facility_location_for_name_city_conflict(monkeypatch): + monkeypatch.setattr( + llm_fallback, + "_geocode_llm_city", + lambda query: { + "lat": "22.6048", + "lon": "120.3000", + "display_name": "Kaohsiung, Taiwan", + "address": {"city": "Kaohsiung", "country": "Taiwan"}, + }, + ) + client = _FakeAIProviderClient( + json.dumps( + { + "latitude": None, + "longitude": None, + "precision": "city", + "confidence": 0.72, + "city": "Kaohsiung", + "country": "Taiwan", + "matched_location_name": "Kaohsiung, Taiwan", + "evidence": [ + { + "source": "Taiwan News", + "source_type": "news", + "entity_match": True, + "text": "Nvidia's first AI supercomputer center, Taipei-1, is located in Kaohsiung.", + } + ], + "reasoning_summary": "Explicit facility location evidence overrides the city-like system name.", + } + ) + ) + + result = await collect_llm_location_fallback_candidate( + provider_client=client, + query=LocationQuery(name="TAIPEI-1", country="Taiwan"), + entity_type="compute_center", + ) + + assert result.failure_reason is None + candidate = result.candidates[0] + assert candidate.city == "Kaohsiung" + assert candidate.confidence >= 0.55 + + +@pytest.mark.asyncio +async def test_llm_location_fallback_rejects_compute_center_city_from_entity_name_when_llm_unparseable(monkeypatch): def _fake_geocode(query): if query != "Taipei, 中国（台湾）": return None @@ -772,12 +814,9 @@ async def test_llm_location_fallback_geocodes_city_from_entity_name_when_llm_unp ) assert client.calls == 2 - assert result.failure_reason is None - candidate = result.candidates[0] - assert candidate.city == "Taipei" - assert candidate.latitude == pytest.approx(25.033) - assert candidate.longitude == pytest.approx(121.5654) - assert "Entity name city hint" in candidate.source_note + assert result.candidates == [] + assert result.failure_reason is not None + assert "parseable city-level location fact" in result.failure_reason @pytest.mark.asyncio diff --git a/backend/tests/test_settings_ai_provider.py b/backend/tests/test_settings_ai_provider.py index 96808a0f..4d5612c4 100644 --- a/backend/tests/test_settings_ai_provider.py +++ b/backend/tests/test_settings_ai_provider.py @@ -5,21 +5,42 @@ import pytest from app.api.v1 import settings as settings_api from app.api.v1.settings import ( AIProviderIntegrationUpdate, + BarentsWatchIntegrationUpdate, + ExternalIntegrationsUpdate, OCRIntegrationUpdate, + WebSearchIntegrationUpdate, _build_ai_provider_payload, _build_ocr_payload, + _can_reveal_integration_secrets, + _ensure_secret_reveal_allowed, _mask_secret, _normalize_ai_provider_payload, _normalize_ocr_payload, + _record_integration_secret_reveal, _resolve_provider_api_key, get_runtime_ai_provider_config, + save_external_integrations_payload, ) +from app.services.llm_provider_catalog import get_fallback_llm_provider_preset @pytest.fixture(autouse=True) def isolated_ai_provider_env_file(monkeypatch, tmp_path): env_file = tmp_path / ".env" monkeypatch.setattr(settings_api, "AI_PROVIDER_ENV_FILE", env_file) + for name in ( + "AI_PROVIDER", + "AI_API_KEY", + "MINIMAX_API_KEY", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "DEEPSEEK_API_KEY", + "DASHSCOPE_API_KEY", + "MOONSHOT_API_KEY", + "OPENROUTER_API_KEY", + "OPENCODE_GO_API_KEY", + ): + monkeypatch.delenv(name, raising=False) return env_file @@ -56,7 +77,7 @@ def test_provider_key_prefers_specific_env_file_key(isolated_ai_provider_env_fil def test_provider_key_falls_back_to_generic_ai_api_key(isolated_ai_provider_env_file): isolated_ai_provider_env_file.write_text( - "AI_API_KEY=generic-env-file-key\n", + "AI_PROVIDER=openai\nAI_API_KEY=generic-env-file-key\n", encoding="utf-8", ) @@ -66,11 +87,101 @@ def test_provider_key_falls_back_to_generic_ai_api_key(isolated_ai_provider_env_ assert source == "env_file" +def test_generic_ai_api_key_only_applies_to_selected_env_provider(isolated_ai_provider_env_file): + isolated_ai_provider_env_file.write_text( + "AI_PROVIDER=minimax\nAI_API_KEY=generic-env-file-key\n", + encoding="utf-8", + ) + + openai_value, openai_source = _resolve_provider_api_key("openai", {"api_key": ""}) + minimax_value, minimax_source = _resolve_provider_api_key("minimax", {"api_key": ""}) + + assert openai_value == "" + assert openai_source == "" + assert minimax_value == "generic-env-file-key" + assert minimax_source == "env_file" + + +def test_opencode_go_provider_preset_is_openai_compatible(): + preset = get_fallback_llm_provider_preset("opencode-go") + + assert preset["label"] == "OpenCode Go" + assert preset["provider_api"] == "openai-completions" + assert preset["base_url"] == "https://opencode.ai/zen/go/v1" + assert preset["model"] == "glm-5.1" + assert "glm-5.1" in preset["models"] + assert "deepseek-v4-flash" in preset["models"] + assert preset["model_provider_apis"]["minimax-m2.7"] == "anthropic-messages" + assert preset["api_key_env"] == "OPENCODE_GO_API_KEY" + + def test_mask_secret_without_prefix_is_fully_masked(): assert _mask_secret("plainsecret")["preview"] == "***********" assert _mask_secret("sk-prefixed")["preview"] == "sk-********" +def test_secret_reveal_permission_is_admin_only(): + assert _can_reveal_integration_secrets(SimpleNamespace(role="super_admin")) is True + assert _can_reveal_integration_secrets(SimpleNamespace(role="admin")) is True + assert _can_reveal_integration_secrets(SimpleNamespace(role="viewer")) is False + + +@pytest.mark.asyncio +async def test_secret_reveal_denial_is_audited(monkeypatch): + events = [] + + async def fake_record_audit_log(**payload): + events.append(payload) + + monkeypatch.setattr(settings_api, "record_audit_log", fake_record_audit_log) + user = SimpleNamespace(id=7, username="viewer", email="viewer@example.test", role="viewer") + + with pytest.raises(settings_api.HTTPException) as exc: + await _ensure_secret_reveal_allowed( + current_user=user, + request=None, + target_id="ai_provider:openai", + details={"kind": "ai_provider", "provider": "openai"}, + ) + + assert exc.value.status_code == 403 + assert events[0]["action"] == "settings.integration_secret.reveal" + assert events[0]["result"] == "denied" + assert events[0]["actor_id"] == 7 + assert events[0]["target_id"] == "ai_provider:openai" + + +@pytest.mark.asyncio +async def test_secret_reveal_audit_does_not_store_plaintext(monkeypatch): + events = [] + + async def fake_record_audit_log(**payload): + events.append(payload) + + monkeypatch.setattr(settings_api, "record_audit_log", fake_record_audit_log) + user = SimpleNamespace(id=1, username="admin", email="admin@example.test", role="admin") + + await _record_integration_secret_reveal( + current_user=user, + request=None, + target_id="ai_provider:minimax", + result="success", + details={ + "kind": "ai_provider", + "provider": "minimax", + "api_key_configured": True, + "api_key_source": "env_file", + "service_token_configured": True, + "service_token_source": "runtime", + }, + ) + + serialized = str(events[0]) + assert "secret-value" not in serialized + assert events[0]["details"]["api_key_source"] == "env_file" + assert events[0]["details"]["service_token_configured"] is True + + def test_build_payload_updates_only_selected_provider_key(): current = { "ai_provider": { @@ -109,6 +220,30 @@ def test_build_payload_updates_only_selected_provider_key(): assert payload["providers"]["minimax"]["api_key"] == "minimax-old-key" +def test_build_payload_saves_provider_without_changing_default(): + current = { + "ai_provider": { + "default_provider": "minimax", + "providers": { + "minimax": {"provider": "minimax", "api_key": "minimax-key"}, + "openai": {"provider": "openai", "api_key": ""}, + }, + } + } + update = AIProviderIntegrationUpdate( + provider="openai", + provider_api="openai-completions", + base_url="https://api.openai.com/v1", + model="gpt-test", + api_key="openai-new-key", + ) + + payload = _build_ai_provider_payload(current, update) + + assert payload["default_provider"] == "minimax" + assert payload["providers"]["openai"]["api_key"] == "openai-new-key" + + def test_build_payload_keeps_saved_key_when_preview_submitted(): current = { "ai_provider": { @@ -207,3 +342,78 @@ async def test_runtime_config_uses_default_provider_specific_key(monkeypatch): assert runtime_config["llm_config"]["provider"] == "minimax" assert runtime_config["llm_config"]["api_key"] == "minimax-key" assert runtime_config["llm_config"]["model"] == "MiniMax-test" + + +@pytest.mark.asyncio +async def test_saving_ai_provider_does_not_run_connection_validation(monkeypatch): + calls = {"validated": 0, "saved": 0} + + async def fake_get_setting_payload(_db, category): + assert category == "external_integrations" + return { + "ai_provider": { + "default_provider": "minimax", + "providers": { + "minimax": { + "provider": "minimax", + "api_key": "minimax-key", + }, + }, + }, + "web_search": {}, + "ocr": {}, + } + + async def fake_validate(_payload): + calls["validated"] += 1 + raise AssertionError("save should not run AI provider connection validation") + + async def fake_save_setting_payload(_db, category, payload): + assert category == "external_integrations" + calls["saved"] += 1 + return payload + + async def fake_get_barentswatch_config_record(_db): + return SimpleNamespace( + endpoint="", + auth_config={}, + ) + + async def fake_serialize_external_integrations(_db): + return {"ai_provider": {"default_provider": "minimax"}} + + monkeypatch.setattr(settings_api, "get_setting_payload", fake_get_setting_payload) + monkeypatch.setattr(settings_api, "_validate_ai_provider_full_connection", fake_validate) + monkeypatch.setattr(settings_api, "save_setting_payload", fake_save_setting_payload) + monkeypatch.setattr(settings_api, "get_barentswatch_config_record", fake_get_barentswatch_config_record) + monkeypatch.setattr(settings_api, "serialize_external_integrations", fake_serialize_external_integrations) + + update = ExternalIntegrationsUpdate( + ai_provider=AIProviderIntegrationUpdate( + provider="minimax", + default_provider="minimax", + provider_api="anthropic-messages", + base_url="https://api.minimaxi.com/anthropic", + model="MiniMax-M2.7", + api_key="sk-new-key", + ), + barentswatch=BarentsWatchIntegrationUpdate(), + web_search=WebSearchIntegrationUpdate(), + ocr=OCRIntegrationUpdate(), + ) + + async def fake_commit(): + return None + + async def fake_refresh(_record): + return None + + db = SimpleNamespace( + add=lambda _record: None, + commit=fake_commit, + refresh=fake_refresh, + ) + + await save_external_integrations_payload(db, update) + + assert calls == {"validated": 0, "saved": 1} diff --git a/backend/tests/test_web_search_tools.py b/backend/tests/test_web_search_tools.py index 1e25c8b3..37bdfd44 100644 --- a/backend/tests/test_web_search_tools.py +++ b/backend/tests/test_web_search_tools.py @@ -46,6 +46,21 @@ def test_web_search_key_prefers_provider_env(isolated_web_search_env_files): assert source == "env_file" +def test_web_search_generic_key_only_applies_to_default_provider(isolated_web_search_env_files): + isolated_web_search_env_files.write_text( + "WEB_SEARCH_API_KEY=generic-search-key\n", + encoding="utf-8", + ) + + default_value, default_source = _resolve_web_search_api_key("tavily", {"api_key": ""}, "tavily") + other_value, other_source = _resolve_web_search_api_key("brave", {"api_key": ""}, "tavily") + + assert default_value == "generic-search-key" + assert default_source == "env_file" + assert other_value == "" + assert other_source == "" + + def test_build_web_search_payload_keeps_saved_key_when_preview_submitted(): current = { "web_search": { @@ -71,6 +86,35 @@ def test_build_web_search_payload_keeps_saved_key_when_preview_submitted(): assert payload["providers"]["tavily"]["api_key"] == "tvly-old-secret" +def test_build_web_search_payload_saves_provider_without_changing_default(): + current = { + "web_search": { + "default_provider": "tavily", + "providers": { + "tavily": { + "provider": "tavily", + "api_key": "tvly-key", + }, + "brave": { + "provider": "brave", + "api_key": "", + }, + }, + } + } + update = WebSearchIntegrationUpdate( + enabled=True, + provider="brave", + base_url="https://api.search.brave.com/res/v1/web/search", + api_key="brave-new-key", + ) + + payload = _build_web_search_payload(current, update) + + assert payload["default_provider"] == "tavily" + assert payload["providers"]["brave"]["api_key"] == "brave-new-key" + + @pytest.mark.asyncio async def test_tavily_adapter_normalizes_results(monkeypatch): config = WebSearchConfig( diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b952ef7b..a6648a45 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,23 @@ This project follows the repository versioning rule: - `improvement` -> `+0.0.1`（bugfix + 小功能混合） - `bugfix` -> `+0.0.1` +## [0.62.0] — 2026-05-21 + +Released: 2026-05-21 + +### Highlights +- Admin Next 转正为正式后台入口，旧 AntD 控制台完整迁移到 `/legacy/admin/*` 作为回退和对照，并保留 `/admin-next/*` 兼容重定向。 +- 新后台完成采集、AI Provider、Earth 内容、日志、BGP/告警、设置和认证链路的全量收口，新增 lazy tab loading、层级配置、移动端详情和 Markdown 文档渲染体验。 +- 抽出 Tactile UI 按钮、开关、tooltip、滚动条和表格滚动组件，为未来独立 npm 组件库打基础，同时补齐中英文开发文档。 + +### Added / Fixed / Improved +- 新增采集管理分层工作台，覆盖采集器配置、采集调度、采集历史/快照、凭证教程生成/重置、映射 propose/preview/create/activate 和真实连接/运行状态。 +- 改进 AI Provider 与工具调用配置，修复 key fallback、脱敏显示、轻量连通性测试、默认 provider、OpenCode Go 路由和 Playground Markdown 输出。 +- 更新 Earth 内容管理和 Earth 前台体验，支持品牌预览/上传/重置、TV 默认源与新增草稿、图层/新闻/3D 模型配置入口以及 live TV 预览。 +- 强化后端数据源、系统日志、WebSocket、AI client、位置 LLM fallback、Earth layer cache 和 datasource connectivity，并补充相关回归测试与运维脚本能力。 + +--- + ## [0.61.0] — 2026-05-18 Released: 2026-05-18 diff --git a/docs/plans/README.md b/docs/plans/README.md index f2d9afcc..f851a758 100644 --- a/docs/plans/README.md +++ b/docs/plans/README.md @@ -35,7 +35,11 @@ - [Docs Gatekeeper 鉴权系统计划](/home/ray/dev/linkong/planet/docs/plans/docs-gatekeeper-auth-plan.md) - [Location Resolver 共享管线计划](/home/ray/dev/linkong/planet/docs/plans/location-resolver-shared-pipeline-plan.md) - [frontend-ai-playground-development-plan.md](/home/ray/dev/linkong/planet/docs/plans/frontend-ai-playground-development-plan.md) +- [AI Provider OpenClaw-Style Routing Plan](/home/ray/dev/linkong/planet/docs/plans/ai-provider-openclaw-style-routing-plan.md) +- [统一集成配置 Schema 系统计划](/home/ray/dev/linkong/planet/docs/plans/integration-config-schema-system-plan.md) - [Lightweight Agent Orchestrator 与 WebSearch 证据层计划](/home/ray/dev/linkong/planet/docs/plans/agents-light-orchestrator-websearch-plan.md) +- [Admin Next Parity Checklist](/home/ray/dev/linkong/planet/docs/plans/admin-next-parity-checklist.md) +- [Admin Next Parity Audit Closeout](/home/ray/dev/linkong/planet/docs/plans/admin-next-parity-audit-closeout-plan.md) - [ue5-mvp-fused-plan.md](/home/ray/dev/linkong/planet/docs/plans/ue5-mvp-fused-plan.md) 不适合放入这里的内容： diff --git a/docs/plans/admin-next-dual-track-full-migration-plan.md b/docs/plans/admin-next-dual-track-full-migration-plan.md new file mode 100644 index 00000000..4873dd87 --- /dev/null +++ b/docs/plans/admin-next-dual-track-full-migration-plan.md @@ -0,0 +1,128 @@ +# Admin Next 双轨全量迁移修正计划 + +## Summary + +`/admin-next/*` 继续作为新版影子路由开发，旧 AntD 控制台必须完整保留作为生产回退和行为对照。只有新版完成全功能 parity 并通过验收后，才能逐个切换旧路由；删除旧页面和移除 AntD 必须作为最后独立步骤，并等待明确确认。 + +## Hard Rules + +- 未完成 parity checklist 前，不删除旧页面、不移除 AntD、不切旧入口。 +- “旧版能删除”只表示新版能力达到替代标准，不表示立即删除旧版。 +- 新版不能只做看板；旧版里可新增、编辑、删除、测试、触发、批量、轮询、WebSocket、权限控制、错误提示和确认弹窗的能力，都必须迁移。 +- `/admin-next/*` 可以重做交互和视觉，但业务语义、API payload、权限和危险操作保护必须对齐旧版。 + +## Design Direction + +- 风格采用“朴素工具化 + 轻微立体触感”：灰白/暗色低对比背景、清晰 1px 边框、轻微凸起/按下态、克制圆角、图标少量彩色。 +- 禁止回到 soft-glass、hero、大发光、大渐变、大色块和装饰性卡片。 +- 常规动作默认 icon-only + tooltip：刷新、重启、退出、复制、查看、编辑、删除、关闭、设置。 +- 强意图动作保留实心文字按钮：保存、创建、确认、执行、测试连接。 +- 页面统一 `16px` 外边距、`16px` 主区块间距、`12px` 面板 header/body 间距；表格标题和表体不能贴边。 +- 所有页面遵守一屏工作台：`PageHeader` 之外只允许一个主内容区域吃满剩余高度；详情区、表格区和 Playground 消息区内部滚动，不能被父级裁掉。 +- 页内 tabs 必须按旧版用户心智组织，而不是按后端接口名拆散；接口状态只能作为分区内部信息。 + +## Implementation Plan + +1. 恢复旧版安全网：恢复旧 AntD 页面、旧 `AppLayout`、旧 helper、AntD 依赖和旧路由；保留 `/admin-next/*` 影子路由。 +2. 建立新版基础层：整理 admin-next token、按钮、tooltip、theme switch、sidebar account、`EntityTable`、`FormDialog`、`ConfirmDialog`、`DetailPanel`、移动端表格/卡片切换。 +3. 拆掉通用看板页：每个模块建立真实业务页面、API adapter、form/mutation、轮询或 WebSocket 逻辑，不再用万能 Resource table 聚合展示。 +4. 按模块迁移： + - Dashboard：统计、WebSocket、健康检查、重启任务、任务日志、恢复探测。 + - DataSources：内置源/自定义源/实时源、详情、stats、task-status、启停、触发、批量触发、清理数据。 + - DataList：列表、summary、sources/types/countries、搜索、筛选、分页、详情、导出、分布。 + - Collection Management：configs CRUD、builtin connect/test、target schemas、mappings、run/stop、stream status。 + - Settings：system、notifications、security、SMTP、TV、integrations、collectors、测试连接、凭证指南。 + - AI：provider、web search、OCR、prompt registry、playground 完整会话操作。 + - Earth Content：brand、upload、delete/reset、boundary config/build/status、TV/content。 + - Logs：sources、level/date/search、刷新、详情、复制、tail/snapshot。 + - BGP：overview、collectors、incidents、anomalies、events、AI briefs。 + - Alerts：系统告警、BGP 告警、态势告警，包含 ack/resolve/stats/brief。 + - Users：list/create/edit/delete、role、Gatekeeper groups、权限显示。 + - Auth：登录、注册、验证邮箱、重发、找回、重置、logout。 +5. 每完成一个模块，对照旧页面跑 checklist；只有全部模块通过后，才提出旧路由切换和旧版删除。 + +## Test Plan + +- 恢复阶段：`cd frontend && /home/ray/.bun/bin/bun install && /home/ray/.bun/bin/bun run build`。 +- 旧路由检查：`/admin`、`/datasources`、`/settings`、`/ai`、`/logs` 必须继续打开旧版。 +- 新路由检查：`/admin-next/*` 必须继续打开新版。 +- 每模块 parity：读写删、测试连接、触发、批量、轮询、WebSocket、权限、错误提示、确认弹窗、移动端和滚动。 +- 视觉检查：light/dark/system、低高度窗口、125%/150% 缩放、按钮/状态/badge 等高、无双滚动条。 + +## Parity Checklist + +### Dashboard +- [x] `/dashboard/stats` 数据展示与旧版一致。 +- [x] WebSocket 连接状态、健康状态、自动刷新语义一致。 +- [x] 重启任务创建、轮询、日志查看、失败提示和恢复探测一致。 + +### DataSources +- [x] 内置源、自定义源、实时源分区清晰，不混表。 +- [x] 详情、stats、task-status、enable/disable、trigger、trigger-batch、trigger-all、delete-data 与旧版一致。 +- [x] 实时源 start/stop/restart、连接状态、任务状态与旧版一致。 + +### DataList +- [x] 列表、summary、sources/types/countries、搜索、筛选、分页参数与旧版一致。 +- [x] 详情 metadata、分布摘要、JSON/CSV 导出与旧版一致。 +- [x] 表格固定列、横向滚动、移动端卡片模式可用。 + +### Collection Management +- [x] configs/all、configs CRUD、builtin connect/test、custom sample 与旧版一致。 +- [x] target schemas、mappings preview/list/update/activate 与旧版一致；propose/create 已提供表单化入口。 +- [x] run-mapped、stop-mapped、stream-status 与旧版一致。 + +### Settings +- [x] system、notifications、security、SMTP、TV、integrations、collectors 表单字段完整。 +- [x] SMTP test、AI/Web/OCR secrets、connect tests、preset refresh 与旧版一致。 +- [x] credential guides、cache 清理、危险操作确认一致。 + +### AI +- [x] Provider、Web Search、OCR、Prompt Registry 配置保存/重置完整。 +- [x] Playground thread/session/messages/status/stop/resend/edit 完整。 +- [x] provider refresh、connect test、secrets 读取、错误提示和 loading 状态完整。 +- [x] 页内 tabs 按旧版心智恢复为模型供应商、工具调用、提示词、Playground。 + +### Earth Content +- [x] brand get/save/delete/reset/upload 完整。 +- [x] boundary status/config/build/build-status 完整。 +- [x] TV/content 配置与旧版 Settings 入口一致。 + +### Logs +- [x] sources、level/date/search、limit、刷新参数与旧版一致。 +- [x] 日志正文、详情展开、复制、空态和错误态完整。 + +### BGP +- [x] overview、collectors、incidents、anomalies、events 的 summary/list/detail 完整。 +- [x] AI brief list/detail/generate 完整。 +- [x] collector collect-location、刷新、结果详情和错误提示完整。 + +### Alerts +- [x] 系统告警 list/detail/ack/resolve/stats/AI brief 完整。 +- [x] BGP incidents/anomalies/brief 完整。 +- [x] 态势告警 stats/list/brief 完整。 + +### Users +- [x] list/create/edit/delete、role、Gatekeeper groups 完整。 +- [x] 权限显示、普通用户限制和 super admin 行为一致。 + +### Auth +- [x] login/register/verify/resend cooldown/forgot/reset/logout/me 完整。 +- [x] 未验证邮箱跳转、错误提示、成功跳转和 token store 行为一致。 + +### Hierarchy Repair +- [x] AI 不再按接口/状态拍平成一张表；模型供应商恢复为 provider -> model/API/key 层级。 +- [x] 工具调用恢复为 Web Search/OCR -> provider/API/高级参数层级。 +- [x] 提示词恢复为 group -> prompt 入口层级，保存/重置只作用于当前入口。 +- [x] Settings/Earth 配置型页面使用父级列表 + 右侧正式表单，长内容在一屏内内部滚动。 + +### Interaction Polish +- [x] 不会产生歧义的刷新、复制、详情、批量、状态、清理等动作优先改为 icon-only + tooltip/title。 +- [x] 强意图动作保留实心文字按钮，例如保存、创建、触发全部、生成简报、确认执行。 +- [x] 按钮字号、高度、图标容器和 hover 触感统一，接近侧栏深色模式滑块的轻微立体风格。 +- [x] 表格与详情之间增加可拖动竖向 resize handle，低高度和窄屏下不制造额外滚动条。 + +### Page Information Architecture +- [x] 页内 tab 默认使用中文；BGP、AI、OCR、Web Search、Playground、Schema 等专有名词按可识别性保留。 +- [x] 信息观测类页面使用列表 + 详情，例如 BGP、Alerts、Logs、DataSources。 +- [x] 配置管理类页面使用分层结构，例如 AI、Settings、Earth Content、Collection Management。 +- [x] 采集管理已从纯表格看板改为分层管理：采集器、映射模板、目标 Schema、运行状态先选父级，再编辑或执行动作。 diff --git a/docs/plans/admin-next-parity-audit-closeout-plan.md b/docs/plans/admin-next-parity-audit-closeout-plan.md new file mode 100644 index 00000000..6dd3f3ad --- /dev/null +++ b/docs/plans/admin-next-parity-audit-closeout-plan.md @@ -0,0 +1,135 @@ +# Admin Next Parity Audit Closeout + +Last updated: 2026-05-21 + +## Status + +Admin Next has been promoted to the official admin route family. The old AntD admin, old layout helpers, `antd`, and `@ant-design/icons` remain available under `/legacy/admin/*` as the rollback and comparison surface. + +`/admin-next/*` is now compatibility-only: old test links redirect to the official route. Do not add new capabilities there as a separate entry point. + +This closeout document is the final work log for the second parity audit. It records what has been fixed, what still requires manual verification, and what must not be treated as complete. + +## Fixed In This Audit + +- Restored the dual-track rule: old AntD pages and routes are not deleted and are not replaced before parity sign-off. +- Reworked management pages away from flat status tables toward business hierarchy: + - AI: `模型供应商 / 工具调用 / 提示词 / Playground`. + - Settings: system display, notification policy, security policy, SMTP only. + - Earth Content: brand, boundary precision, TV content. + - Collection: collector, collection schedule, history/snapshot. +- Repaired AI provider credential semantics: + - Provider status now comes from that provider's stored key or a runtime key that is explicitly scoped to that provider. + - Generic `.env` fallback no longer marks every provider as configured. + - Runtime fallback secrets stay masked by default; authorized administrators can reveal them through the console and each reveal must be audited without storing plaintext. + - Secret previews preserve the prefix before the first `-` when the backend preview exposes it, for example `sk-**********`. + - Connect-test buttons are icon-only plug actions and do not save configuration. +- Repaired key UI gaps: + - LLM API Key, proxy token, Web Search API Key, and OCR API Key use masked input with inline eye toggle. + - Save buttons use the disk icon. + - Default action is disabled when the item is already default. +- Repaired SMTP test flow: + - SMTP test has a recipient input. + - Request payload is `{ to, settings }`. +- Repaired Earth Content actions: + - Brand save, upload, delete/reset actions are available in the brand section. + - Brand preview renders with the same Earth left-top brand structure and starfield-style background. + - Boundary build is scoped to the boundary precision section, not the global page toolbar. + - TV default configuration is folded into item-level configuration; TV items support add/default/reset/delete semantics. + - TV default source now follows the same default-state semantics as AI Provider: if the response does not expose an explicit default, `cgtn-en` is treated as the runtime default; setting a default promotes that item visually and disables the redundant default action. +- Repaired Collection actions: + - Credential guide read/generate/reset entry points are available from collector-related panels. + - Create collector config uses field-first form controls with advanced JSON as a secondary path. + - Mapping propose/preview/create/activate flow is represented in the mapping workflow. +- Repaired shared UI rules: + - Management list names can wrap and remain readable instead of being squeezed by status tags. + - Status tags are fixed width and color-coded: default blue, configured green, unconfigured gray, error red. + - Configuration lists with a default item sort the default item to the top after save/default changes instead of leaving it in the previous alphabetical or API order. + - List footer actions are part of the scroll content; users see them after scrolling to the bottom. + - Footer icon buttons use the current tactile surface direction: external shadow first, without a separate inset/base-plate treatment. + - Detail panels use a dark neutral resize handle. + - Tooltip, dialog, modal backdrop, textarea scrollbar, custom scrollbar, mobile detail, and one-screen layout issues from the audit have been addressed in the shared admin-next layer. + - Multi-tab Admin Next pages now use lazy active-tab loading with local cache. Initial page load no longer requests every section endpoint; manual refresh and mutating actions refresh only the current section. + - Markdown-producing details, AI brief content, credential tutorials, and Playground assistant output use the shared Markdown renderer where the content is meant to be read as a document. Raw metadata remains available only where the page is explicitly showing original payloads. +- Added developer workflow scripts: + - `build:watch` + - `preview:auto` + - `bun run build` remains a production artifact build and does not reload an already-open dev page by itself. + +## Manual Verification Status + +The code has been updated. The list below records which areas are already manually checked and which still need route-level regression before replacement: + +- AI: + - Latest manual pass marked provider status, secret reveal, and Playground behavior as checked. +- DataSources: + - Single trigger performs old task-status precheck, handles running-conflict confirmation, supports force recollect, and refreshes task status afterward. + - Batch trigger uses old `/datasources/trigger-batch` semantics for selected IDs, filters, and force. + - Builtin, custom, and realtime source actions match old enable/disable/start/stop/restart/clear-data behavior. + - 2026-05-20 implementation pass: Admin Next builtin datasource rows now expose task/collection status and metrics, and single trigger now performs task-status precheck with running-conflict force confirmation plus post-trigger task refresh. + - 2026-05-20 manual pass: builtin trigger, task-status precheck, force recollect, batch trigger, enable/disable, and clear-data were verified in Admin Next. +- Collection: + - Collector config create/update/delete, builtin connect, custom test/sample/run/stop/status, schedule save, history/snapshot, mapping propose/preview/create/activate, and credential guide read/generate/reset all work with real backend payloads. + - 2026-05-20 implementation pass: `采集历史 / 快照` now reads real datasource snapshots from `/datasources/snapshots` instead of config rows. + - 2026-05-20 implementation pass: default credential guide actions no longer call unsupported providers blindly; unsupported collectors show a clear empty state and supported providers use `barentswatch` / `aisstream`. + - Collection action buttons must follow the AI page's single-responsibility rule: navigation/link actions do not save, save actions do not connect/test, and connect/test actions do not persist drafts. + - 2026-05-20 manual pass: collector create/edit `auth_config`, mapping propose/preview/create/activate, custom test/sample/run/stop/status, credential guide read/generate/reset, and history/snapshot behavior were verified in Admin Next. + - 2026-05-20 follow-up: dead JSON modal create/mapping code was removed so Collection creation now goes through the list-bottom draft workflow only. +- Earth Content: + - Brand preview visually matches the Earth runtime left-top brand at the same structural level. + - Brand upload writes back `logo_src` or `title_src` correctly. + - 2026-05-20 implementation pass: Brand upload validates and displays accepted suffixes (`png`, `jpg`, `jpeg`, `webp`, `svg`) and supports drag-and-drop. + - Boundary status/config/build and TV add/save/default/delete/reset match old behavior. +- Logs: + - Sources, snapshot/tail, level/date/search, copy, empty/error states, and internal scrolling all work. + - 2026-05-20 implementation pass: Logs now include database-backed `system-db` and `audit-db` sources as a fallback when file/docker/buffer logs are empty or missing, and the UI auto-selects an available source. Manual regression must still verify snapshot/tail-like reading, filters, copy, and scrolling. +- BGP / Alerts: + - BGP overview, collectors, collect-location detail, incidents, anomalies, events, latest/detail/generate AI brief. + - System/BGP/situational alert detail, ack, resolve with note, refresh after mutation, and AI brief details. +- Users/Auth: + - Users CRUD, role, Gatekeeper groups. + - 2026-05-20 implementation pass: User edit select dropdowns now use an opaque high-z-index menu surface, and Gatekeeper checkbox groups use an explicit surface background. Manual visual regression is still required. + - Login, register, verify email, resend cooldown, forgot/reset password. +- Mobile: + - 375px, 430px, and 768px widths default to list/group first. + - Detail opens only after tap/click and has a visible back action. + - Long forms, JSON, logs, and Playground messages scroll internally without body-level double scrollbars. + +## Explicit Non-Production Items + +The old AntD page itself treated these Earth content tabs as placeholder-level capabilities. Admin Next must show a visible “backend capability pending” empty state and must not mix in unrelated data: + +- `basemap` +- `layer_resources` +- `models_3d` +- `news_anchor_strategy` + +If backend endpoints are later added, these items must be promoted into `docs/plans/admin-next-parity-checklist.md` with concrete API and UI acceptance criteria. + +## Final Gate + +After the route promotion, the final gate is no longer “switch old routes.” The remaining gate is “keep legacy available until the promoted routes are stable enough to remove AntD.” + +1. Run `cd frontend && /home/ray/.bun/bin/bun run build`. +2. Run the static checks: + - `rg "map: \\(\\) => \\[\\]|暂不支持保存|placeholder" frontend/src/admin-next` + - `rg "ShadowPage|FeatureConsole|GlassPanel|InspectorDrawer" frontend/src/admin-next` +3. Manually verify every official route listed in `docs/plans/admin-next-parity-checklist.md`. +4. Confirm `/legacy/admin/*` still opens old AntD pages during the validation window. +5. Delete old AntD pages and remove AntD dependencies only as a separate final cleanup task after explicit confirmation. + +## Current Build Verification + +Last successful local build during this closeout, after the 2026-05-20 follow-up fixes: + +```bash +cd frontend && /home/ray/.bun/bin/bun run build +``` + +Result: passed. + +Backend syntax check also passed: + +```bash +/home/ray/.local/bin/uv run python -m py_compile backend/app/api/v1/datasources.py backend/app/api/v1/datasource_config.py backend/app/api/v1/system_control.py +``` diff --git a/docs/plans/admin-next-parity-checklist.md b/docs/plans/admin-next-parity-checklist.md new file mode 100644 index 00000000..ad829d4c --- /dev/null +++ b/docs/plans/admin-next-parity-checklist.md @@ -0,0 +1,43 @@ +# Admin Next Parity Checklist + +Last updated: 2026-05-21 + +This checklist is the hard gate for deleting the old AntD admin. Admin Next now owns the official admin routes; old AntD pages stay available under `/legacy/admin/*` until every production capability below is verified. + +## Route Gate + +- `/admin` Dashboard: stats, health probe, WebSocket status, restart task, restart logs, recovery probe. +- `/datasources`: builtin sources, custom sources, realtime sources, detail/stat/task-status, trigger, batch trigger, enable/disable, clear data, realtime start/stop/restart, custom source test/sample/run/stop/status link. +- `/data`: list, summary, source/type/country filters, search, pagination, detail metadata, export, distribution chart. +- `/collection-management`: collector config, mapping templates, target schemas, collection schedule, history/snapshot, create config, propose/preview/create/activate mapping, builtin connect, run/stop/stream-status, credential guide read/generate/reset. +- `/settings`: system display, notifications, security, SMTP get/save/test with recipient. +- `/ai`: model providers, tool calling, prompts, key reveal, provider refresh, connect tests, OCR/WebSearch secrets, Playground thread/chat/edit/resend/stop. +- `/earth-content`: brand preview as Earth renders it, upload/save/delete/reset, boundary status/build/config, TV source add/save/default/delete/reset. +- `/logs`: sources, filters, snapshot, empty/error states, copy. +- `/bgp`: overview, collectors, collect-location, incidents, anomalies, events, AI brief list/latest/detail/generate. +- `/alerts/system`, `/alerts/bgp`, `/alerts/situational`: list/stat/detail, ack, resolve with note, AI brief generation. +- `/users`: list/create/edit/delete, role, Gatekeeper permission groups. +- Auth pages: login, register, verify email, resend cooldown, forgot/reset password. + +Compatibility routes under `/admin-next/*` should redirect to these official paths and are not a separate validation target. + +## Verified Manual Passes + +- 2026-05-20 `/admin-next/collection-management`: user verified collector create/edit payloads, mapping propose/preview/create/activate, custom collector test/sample/run/stop/status, credential guide read/generate/reset, and collection history/snapshot behavior after the draft-form and Time Capsule updates. +- 2026-05-20 `/admin-next/datasources`: user verified builtin source trigger, task-status precheck, force recollect, batch trigger, enable/disable, and clear-data behavior. + +## Known Non-Production Tabs + +The old AntD page also marked these Earth content tabs as placeholder-level capabilities. Admin Next must show them as “backend capability pending” and must not mix in unrelated data: + +- `basemap` +- `layer_resources` +- `models_3d` +- `news_anchor_strategy` + +## Replacement Rules + +- Keep `/legacy/admin/*` available until the matching route gate above is manually verified after promotion. +- Do not delete old AntD pages, old layout helpers, `antd`, or `@ant-design/icons` until final explicit confirmation. +- Do not use fake rows, unrelated endpoint data, or empty adapters for a real old capability. +- Any backend-missing capability must be recorded here with the missing endpoint and visible UI empty state. diff --git a/docs/plans/admin-next-soft-glass-goal-driven-plan.md b/docs/plans/admin-next-soft-glass-goal-driven-plan.md new file mode 100644 index 00000000..2f31d3c2 --- /dev/null +++ b/docs/plans/admin-next-soft-glass-goal-driven-plan.md @@ -0,0 +1,217 @@ +# Admin Next Soft Glass Goal-Driven Plan + +## Goal + +Rebuild `/admin-next/*` into a modern soft-glass / light-neumorphic console while keeping the legacy Ant Design admin routes available for comparison. The new console must use visual references only as design anchors, not as imported templates, and must be implemented as Planet-owned reusable components and page patterns. + +The redesign must cover desktop and mobile. Data display, icon semantics, table readability, and scroll behavior are first-class acceptance criteria. + +## Criteria For Success + +- This plan exists at `docs/plans/admin-next-soft-glass-goal-driven-plan.md`. +- `/admin-next/*` has real pages for every route; route usage of `ShadowPage` is removed. +- Admin Next supports `system`, `light`, and `dark` theme modes using the same persistence and system-theme idea as Docs. +- The visual language reads as soft-glass / light-neumorphic instead of an AntD reskin: translucent panels, fine borders, subtle glow, cool backgrounds, restrained accent colors, crisp icons, and tactile controls. +- Mobile is explicitly designed: mobile navigation, filter sheets/chips, card-list data views, full-screen detail sheets, and bottom action bars instead of squeezed desktop tables. +- Tables do not show double scrollbars. Long fields are inspectable, copyable, or expandable; important columns stay readable. +- Existing custom scrollbar components remain the default scroll experience. +- `cd frontend && bun run build` passes. +- Static checks can prove no route-level placeholder remains. + +## Reference Strategy + +Use references to calibrate feel, not to take over code: + +- User-provided soft-glass / light-neumorphic dashboard images define the desired mood. +- Shadcn / Tailwind / glassmorphism admin templates may be inspected for shadow, spacing, panel, dark-mode, and app-shell ideas. +- Do not import a complete template, router, auth layer, state layer, or business page. +- Planet-specific interactions such as datasource tasks, BGP briefs, AI settings, log streams, Earth resources, and custom scrollbars are implemented in this repo. + +## Source-Of-Truth Mining + +Design content and page boundaries from existing repo history and docs: + +- `docs/CHANGELOG.md` +- `docs/plans/*` +- `docs/technical/*frontend*` +- `TODO.md` + +Important current boundaries: + +- `DataSources` is a datasource directory and runtime-control page, not a configuration editor. +- `Collection Management` owns collector endpoint, credential, custom source, mapping, and runtime configuration. +- `Earth Content` owns TV livestreams, brand assets, boundary precision, basemap, and Earth display resources. +- `AI` owns provider settings, tools, prompts, and Playground. +- `BGP` is an observability and evidence workspace, not just tables. +- `Alerts` is a duty/analysis workspace for system, BGP, and situational risk. +- `Logs` should become a usable log workbench with filtering and structured detail. +- `Settings` should keep platform settings only. + +## Reusable Architecture + +Recommended structure: + +- `admin-next/design`: theme tokens, theme mode, status colors, icon semantics, chart themes, scrollbar themes. +- `admin-next/components`: primitive UI such as button, input, select, tabs, dialog, drawer, badge, status pill, icon badge. +- `admin-next/patterns`: page patterns such as `PageFrame`, `GlassPanel`, `MetricCard`, `CommandBar`, `FilterRail`, `InspectorDrawer`, `DataWorkspace`, `SettingsWorkspace`, `ScrollableRegion`, `TableRegion`, and `MobileSheet`. +- `admin-next/features`: business modules such as datasources, data-list, bgp, alerts, ai, logs, settings, earth-content, collection-management, users. +- `admin-next/services`: feature API gateways and hooks. Pages should not scatter URL construction. +- `admin-next/routes`: route manifest used by navigation, command search, breadcrumbs, and legacy links. + +## Interaction Rules + +- Row click opens an inspector; frequent row actions live in the inspector header. +- Bulk selection opens a floating command bar. +- Common filters stay visible; advanced filters open a sheet/drawer. +- Detail, JSON, log context, and AI brief use inspector or reading panels instead of stacked modal dialogs. +- Settings are split into `FormSection` blocks with local save, dirty state, connection test, and reset. +- Dangerous actions use a confirm sheet. +- Loading, empty, error, and retry states are shared through reusable patterns. + +## Mobile Rules + +- Do not squeeze desktop layout into mobile. +- Use mobile navigation and sheets instead of a permanent desktop sidebar. +- Filters collapse into a filter sheet; active filters remain visible as chips. +- Data workspaces default to card-list mode on mobile and allow explicit table mode for dense comparison. +- Details open in full-screen mobile sheets. +- Batch actions use a sticky bottom command bar. +- Long logs and JSON use full-screen reading/editing surfaces with custom scrollbars. + +## Table And Data Display Rules + +- Tables must prioritize complete data display: + - key columns get stable widths and priority; + - long fields have peek/expand or inspector detail; + - IDs, URLs, metadata, and errors are copyable; + - complex fields are rendered fully in inspector. +- Desktop tables use one `TableRegion` scroll container. +- Mobile defaults to card-list display and supports dense table mode. +- Native scrollbar visuals are hidden where custom scrollbars apply. + +## Icon Rules + +- Use `lucide-react`. +- Define icon semantics by category: + - navigation icon; + - entity icon; + - status icon; + - action icon; + - severity icon. +- Icons must not be the only semantic carrier unless the icon is universally obvious; use labels or tooltips. +- Status icons pair with `StatusPill`. +- Colors come from tokens, not ad hoc hard-coded values. + +## Page Design + +### Dashboard + +- System health, realtime connection, datasource activity, task trend, alert summary, recent events, and quick commands. +- Restart flow uses a confirm sheet and log console. +- Mobile uses horizontal metric cards and collapsible event/command panels. + +### DataSources + +- Datasource directory and runtime control. +- Desktop: overview rail, datasource table/list, inspector. +- Mobile: datasource cards with status, latest task, trigger/toggle actions. +- Batch trigger uses command bar. +- Endpoint/config/task details are shown fully in inspector. + +### DataList + +- Collection result browsing, search, source/type filters, paging, distribution chart, and detail inspector. +- Fix double scrollbars. +- Mobile defaults to cards and allows table mode. + +### BGP + +- Views: collectors, incidents, anomalies, events, AI brief. +- Emphasize severity, region, ASN/prefix, collector coverage, and evidence. +- AI brief shows facts, judgment, and evidence gaps. + +### Alerts + +- Real pages for system, BGP, and situational alerts. +- Shared alert workspace with stats, severity/status filters, list, and inspector. +- AI brief appears in inspector/mobile sheet. + +### AI + +- Provider, tools, prompts, and Playground. +- Provider and tool settings use local save and connection tests. +- Prompt registry is grouped by task. +- Mobile Playground uses step tabs rather than cramped columns. + +### Logs + +- Log workbench with source, level, date, search, refresh, copy, and structured detail. +- Desktop uses a terminal-like log stream. +- Mobile rows expand or open a full-screen reading sheet. + +### Users + +- Search, role filter, create/edit/delete, Gatekeeper groups. +- Edit/create in drawer/sheet. +- Gatekeeper groups use checkbox chips. + +### Settings + +- Platform settings only: system, notification, security, SMTP. +- Each section saves independently. + +### Earth Content + +- TV livestreams, brand assets, boundary precision, basemap, and layer resources. +- Resource previews, upload state, and boundary build status are emphasized. + +### Collection Management + +- Collector config, custom sources, mapping, runtime control. +- Custom source creation uses a step sheet. +- Advanced JSON is isolated in a collapsible editor. + +## Milestones + +1. Save this plan and establish goal-driven criteria. +2. Build design/theme/pattern foundations. +3. Remove route-level placeholders and create real pages for all `/admin-next/*` routes. +4. Redesign layout, theme, mobile shell, scroll behavior, and table behavior. +5. Fill core workspaces with real API calls and reusable patterns. +6. Verify build, no `ShadowPage` route usage, and core route availability. + +## Verification + +Run: + +```bash +cd frontend && bun run build +rg "ShadowPage" frontend/src/admin-next +rg "axios\\." frontend/src/admin-next/pages +rg "overflow: auto|overflow-y: auto" frontend/src/admin-next +``` + +Manual route checks: + +- `/admin-next` +- `/admin-next/datasources` +- `/admin-next/data` +- `/admin-next/bgp` +- `/admin-next/alerts/system` +- `/admin-next/alerts/bgp` +- `/admin-next/alerts/situational` +- `/admin-next/ai` +- `/admin-next/logs` +- `/admin-next/users` +- `/admin-next/settings` +- `/admin-next/earth-content` +- `/admin-next/collection-management` + +Manual viewport checks: + +- desktop; +- mobile width; +- low height; +- 125% / 150% browser zoom; +- light / dark / system theme modes. + diff --git a/docs/plans/ai-provider-openclaw-style-routing-plan.md b/docs/plans/ai-provider-openclaw-style-routing-plan.md new file mode 100644 index 00000000..646893e3 --- /dev/null +++ b/docs/plans/ai-provider-openclaw-style-routing-plan.md @@ -0,0 +1,182 @@ +# AI Provider OpenClaw-Style Routing Plan + +Last updated: 2026-05-20 + +## Summary + +Planet 的 AI Provider 路由要从“运行时识别特殊 provider / 特殊模型”收敛到 OpenClaw 风格的配置驱动模型：模型引用、协议、鉴权、轻量探测、真实调用和模型级例外都由 provider catalog / preset / runtime metadata 描述，运行时只解释这些元数据，不再散落 `if provider == ... and model == ...` 这类硬编码。 + +这份计划覆盖 Admin Next 的 Provider 配置体验、backend settings API、`aiprovider` 适配服务和未来模型目录同步方式。目标是让 OpenCode Go、MiniMax、DeepSeek、OpenAI-compatible、Anthropic-compatible、Ollama、OpenRouter / One API 类代理都能用同一套规则扩展。 + +## Background + +当前实现已经完成了两步临时修正： + +- OpenCode Go 模型目录不再使用普通 Zen free 列表，而是使用 `https://opencode.ai/zen/go/v1/models`。 +- `minimax-m2.7` / `minimax-m2.5` 的协议例外已从 `aiprovider` 运行逻辑移到 `model_provider_apis` 元数据中。 + +但整体还没有完全达到 OpenClaw 式结构。OpenClaw 的关键思想是： + +- 模型引用使用 `provider/model`，由 provider 前缀确定 runtime provider。 +- provider 插件或 catalog 拥有 `normalizeModelId`、`normalizeTransport`、`normalizeConfig`、`prepareRuntimeAuth`、`createStreamFn` 等 provider 行为。 +- 主推理循环不认识具体模型名，只使用解析后的 provider config、transport 和 request adapter。 +- 上游网关能自己路由时，尽量透传 provider routing metadata，不在本地复制上游逻辑。 + +Planet 不需要完整复制 OpenClaw 插件系统，但需要学习它的边界划分。 + +## Design Principles + +- Provider catalog 是路由事实来源，runtime 不是。 +- 模型级协议例外必须是 metadata，例如 `model_provider_apis`，不能是 Python set / if 分支。 +- 轻量连通性测试只验证网络、鉴权和模型目录，不发真实 prompt。 +- 真实模型调用只发生在 Playground、AI brief、分析任务等明确需要生成的路径。 +- 保存配置不自动设为默认，不自动触发连接测试；保存、设默认、测试三种按钮职责分离。 +- 目录刷新使用增量合并语义：发现新模型，标记旧模型 stale，不直接删除用户选择或自定义模型。 +- 如果 provider 不提供可靠 `/models`，可以用内置 preset 确认已知模型，但 UI 必须说清楚这是 preset confirmation，不是假装 provider 返回了目录。 + +## Target Data Model + +Provider preset / runtime config 应逐步收敛为类似结构： + +```json +{ + "provider": "opencode-go", + "label": "OpenCode Go", + "default_transport": "openai-completions", + "base_url": "https://opencode.ai/zen/go/v1", + "auth": { + "type": "bearer", + "api_key_env": "OPENCODE_GO_API_KEY" + }, + "models": [ + { + "id": "glm-5.1", + "label": "GLM 5.1", + "transport": "openai-completions", + "context_window": null, + "capabilities": ["text"] + }, + { + "id": "minimax-m2.7", + "label": "MiniMax M2.7", + "transport": "anthropic-messages", + "capabilities": ["text", "reasoning"] + } + ], + "discovery": { + "type": "openai-models", + "url": "https://opencode.ai/zen/go/v1/models", + "auth": "provider-api-key" + } +} +``` + +Runtime 选择规则： + +1. 解析 provider。 +2. 解析 model。 +3. 从 `models[].transport` 找模型级 transport。 +4. 若没有模型级 transport，使用 provider `default_transport`。 +5. 将解析结果传给 `aiprovider`。 +6. `aiprovider` 只按 `transport` 组装请求，不认识 provider 专属模型名。 + +## Implementation Plan + +### Phase 1: Stabilize Current Metadata Path + +- Keep `model_provider_apis` as the immediate compatibility bridge. +- Ensure `_provider_defaults()` includes provider metadata such as `model_provider_apis`. +- Ensure `_runtime_config_from_ai_payload()` sends the resolved metadata through `AIProviderClient`. +- Ensure `AIProviderClient` forwards metadata to `aiprovider` with a structured header. +- Ensure `aiprovider.ProviderService` reads model metadata and resolves `provider_api = model_provider_apis[model] ?? provider_api`. +- Add tests proving `aiprovider` does not contain provider/model-specific literals for routing decisions. + +### Phase 2: Replace `model_provider_apis` With Structured Model Catalog + +- Extend `backend/app/services/llm_provider_catalog.py` preset shape with `models_metadata`. +- Preserve old `models` as a compatibility list for the UI. +- Add helpers: + - `get_provider_model_metadata(provider, model)` + - `resolve_provider_transport(provider_config, model)` + - `merge_discovered_models(existing, discovered)` +- Return both `models` and `models_metadata` from refresh endpoints. +- Admin Next should render model labels, capabilities and transport hints from metadata. + +### Phase 3: Provider Discovery And Incremental Sync + +- Add provider discovery descriptors: + - OpenAI-compatible `/models` + - Anthropic-compatible no-models / preset-confirmed path + - Ollama `/api/tags` + - OpenCode Go `/zen/go/v1/models` + - OpenRouter / One API passthrough model discovery +- Add incremental merge behavior: + - New discovered model: add. + - Existing discovered model: update `last_seen_at`, metadata. + - Missing discovered model: mark `stale`, do not delete. + - User custom model: keep unless explicitly removed. +- Surface discovery source in Admin Next: `实时发现 / 内置预设 / 用户自定义 / 已过期`. + +### Phase 4: Transport Adapters + +- Replace provider-specific request decisions with adapter descriptors: + - `openai-completions` + - `anthropic-messages` + - `ollama-generate` + - future `openai-responses` + - future `gemini-generate-content` +- Each adapter owns: + - path + - auth header format + - request body transform + - response text extraction + - reasoning/thinking block extraction + - models endpoint strategy +- `ProviderService.analyze()` should select adapter by resolved transport and call the adapter. + +### Phase 5: Admin Next UX + +- Model provider page should show: + - provider status tag + - default model tag + - source tag: env / runtime / preset / discovered + - model list with transport/capability hint + - separate buttons for save, set default, refresh model catalog, lightweight test +- The connect plug button remains lightweight. +- Full generation test lives only in Playground or a clearly named “试运行” action. +- If lightweight test falls back to preset confirmation, toast must say so explicitly. + +## TODO + +- [ ] Add `models_metadata` to provider presets and refresh responses. +- [ ] Add runtime resolver helper for provider/model transport selection. +- [ ] Remove any remaining provider/model-specific literals from `aiprovider` runtime routing. +- [ ] Add tests that `opencode-go/minimax-m2.7` resolves through metadata, not through runtime hardcode. +- [ ] Add tests for lightweight connectivity: + - [ ] 401 / 403 fail as auth error. + - [ ] 404 with known preset model passes as preset-confirmed. + - [ ] `/models` missing alias passes only when preset contains the alias. + - [ ] unknown model fails. +- [ ] Add discovery descriptors for OpenCode Go, OpenAI-compatible, Anthropic-compatible, Ollama, OpenRouter / One API. +- [ ] Add incremental model catalog merge semantics with stale marking. +- [ ] Update Admin Next model list to show model source, transport and capability. +- [ ] Keep save / set default / lightweight test / full test as separate actions. +- [ ] Document the final provider catalog schema in technical docs after implementation. + +## Current Acceptance Criteria + +- No runtime routing branch may depend on concrete model names like `minimax-m2.7`. +- OpenCode Go model refresh must not use the ordinary Zen free-model endpoint. +- Lightweight connect must not call `analyze()` or consume generation quota. +- Saving a provider must not automatically set it as default. +- Provider UI must distinguish configured key, fallback key, preset model and live-discovered model. + +## Related Files + +- `backend/app/services/llm_provider_catalog.py` +- `backend/app/api/v1/settings.py` +- `backend/app/services/ai_client.py` +- `aiprovider/main.py` +- `aiprovider/provider_service.py` +- `frontend/src/admin-next/pages/PlainResourcePages.tsx` +- `docs/plans/admin-next-parity-audit-closeout-plan.md` diff --git a/docs/plans/integration-config-schema-system-plan.md b/docs/plans/integration-config-schema-system-plan.md new file mode 100644 index 00000000..92f53def --- /dev/null +++ b/docs/plans/integration-config-schema-system-plan.md @@ -0,0 +1,349 @@ +# 统一集成配置 Schema 系统计划 + +Last updated: 2026-05-20 + +## Summary + +Planet 的采集器、AI Provider 和工具调用配置需要从“页面各自硬编码字段”收敛到同一套低代码 schema 系统。系统负责两件事： + +- 用后台可编辑 schema 生成配置表单。 +- 按字段 target 把表单值组装成后端运行时需要的请求、凭证和 JSON 配置。 + +这套 schema 不替代 `target_schema_registry`。`target_schema_registry` 继续负责采集结果映射和校验；本计划中的 `integration_config_schemas` 负责“怎么配置一个集成”。 + +## Current Problems + +- Admin Next 的采集器配置曾把不同凭证形态压成通用 `api_key`，导致 `barentswatch_vessels` 这种 OAuth client credentials 丢失 `client_id` 字段。 +- AI Provider、Web Search、OCR 和 DataSource 配置各自维护表单字段、secret 处理和 payload 组装逻辑，重复且容易漂移。 +- 新增字段时需要改前端字段列表和保存逻辑，无法做到后台配置化扩展。 + +## Target Model + +新增统一 registry：`integration_config_schemas`，存储在 `SystemSetting.payload`。 + +Registry 包含： + +- `fragments`：可复用字段片段，例如 endpoint、API Key、OAuth Client、HTTP 请求、WebSocket 订阅、AI Provider 基础字段、工具超时字段。 +- `auth_schemas`：可复用认证编排，例如 API Key、Bearer Token、Basic、OAuth2 Client Credentials、OAuth2 Authorization Code、Session Cookie Login。 +- `schemas`：具体配置对象使用的 schema，例如 `datasource:barentswatch_vessels`、`ai_provider:minimax`、`tool:web_search:tavily`。 +- `defaults`：每类集成的默认 schema，例如 `datasource`、`ai_provider`、`tool`。 + +Schema 必须声明 `kind`： + +- `datasource` +- `ai_provider` +- `tool` + +字段定义统一使用： + +```json +{ + "key": "client_secret", + "label": "Client Secret", + "type": "secret", + "secret": true, + "target": "auth_config.client_secret", + "required": true, + "wide": true +} +``` + +字段 target 支持写入： + +- DataSource：`root.*`、`auth_config.*`、`headers.*`、`config.*` +- AI Provider：`ai_provider.*`、`ai_provider.providers.{provider}.*` +- Tool：`web_search.*`、`web_search.providers.{provider}.*`、`ocr.*` + +## Auth Schema Model + +认证必须成为 schema 系统的一等能力，不能再把所有凭证强行压成 `api_key`。每个配置 schema 可以引用一个 `auth_schema`，也可以内联声明认证编排。 + +Auth schema 描述： + +- 凭证字段：哪些字段是 secret、是否必填、写入哪个 target。 +- 凭证来源：DB、env fallback、运行时草稿、用户授权回调。 +- 预认证请求：例如登录接口、token endpoint、OAuth callback。 +- 凭证注入方式：header、query、form、JSON body、cookie jar、WebSocket subscription payload。 +- reveal 策略：管理员可 reveal 并写 audit log；无 DB/env 值时显示空。 +- 测试策略：连接测试必须使用当前表单草稿优先，再 fallback 到已保存/env。 + +v1 需要支持的认证类型： + +- `none`：无认证。 +- `api_key`：API Key 写入 header/query/form/body。 +- `bearer_token`：Bearer token header。 +- `basic`：username/password，支持直接 Basic header 或 provider 特定登录。 +- `oauth2_client_credentials`：client_id/client_secret 换 access_token。 +- `oauth2_authorization_code`：第三方登录授权，包含 authorize URL、callback、token exchange、refresh。 +- `session_cookie_login`：用户名密码登录后保存 cookie jar，再访问数据接口。 +- `custom_auth_preflight`：无法归类时，用声明式 preflight 请求生成后续请求上下文。 + +Auth schema 示例： + +```json +{ + "key": "spacetrack_session", + "type": "session_cookie_login", + "fields": [ + { + "key": "username", + "label": "Username", + "target": "auth_config.username", + "required": true + }, + { + "key": "password", + "label": "Password", + "type": "secret", + "secret": true, + "target": "auth_config.password", + "required": true + } + ], + "preflight": { + "method": "POST", + "url": "https://www.space-track.org/ajaxauth/login", + "body_type": "form", + "body": { + "identity": "{{auth_config.username}}", + "password": "{{auth_config.password}}" + }, + "success": { + "type": "cookie" + } + }, + "inject": { + "type": "cookie_jar" + } +} +``` + +OAuth Authorization Code 示例： + +```json +{ + "key": "github_oauth", + "type": "oauth2_authorization_code", + "fields": [ + { "key": "client_id", "target": "auth_config.client_id", "required": true }, + { "key": "client_secret", "type": "secret", "secret": true, "target": "auth_config.client_secret", "required": true }, + { "key": "scopes", "type": "tags", "target": "auth_config.scopes" } + ], + "authorization": { + "url": "https://github.com/login/oauth/authorize", + "client_id": "{{auth_config.client_id}}", + "scopes": "{{auth_config.scopes}}", + "redirect_uri": "{{system.callback_base_url}}/api/v1/integrations/oauth/github/callback" + }, + "token": { + "method": "POST", + "url": "https://github.com/login/oauth/access_token", + "body_type": "form", + "body": { + "client_id": "{{auth_config.client_id}}", + "client_secret": "{{auth_config.client_secret}}", + "code": "{{oauth.code}}", + "redirect_uri": "{{oauth.redirect_uri}}" + } + }, + "inject": { + "type": "bearer_header", + "token_path": "access_token" + } +} +``` + +## Request And Runtime Assembly + +这套系统本质是面向 Planet 集成的低代码 Postman，但目标不是临时发请求，而是沉淀成可保存、可测试、可调度、可审计的运行配置。 + +Schema 支持描述： + +- HTTP method：`GET`、`POST` +- endpoint +- headers +- query params +- JSON body / form body +- auth schema / auth config / preflight auth flow +- WebSocket endpoint 和 subscription payload +- AI Provider 的 `provider_api`、`base_url`、`model`、`api_key`、`service_token` +- Tool 的 provider、base_url、api_key、timeout 和工具专属参数 + +请求执行顺序： + +1. 从 schema 字段 target 组装 root/auth_config/headers/config。 +2. 如果存在 auth schema，先解析凭证来源和草稿覆盖。 +3. 需要 preflight 时执行认证请求，例如 token exchange 或 login。 +4. 把认证结果注入正式请求，例如 bearer header、cookie jar、query token。 +5. 执行连接测试、采样、采集、AI Provider connect 或 tool connect。 + +## Default Schemas + +### DataSource + +默认采集器： + +- endpoint +- method +- headers JSON +- query/body/config JSON +- API Key auth +- advanced JSON + +`barentswatch_vessels`： + +- endpoint +- Client ID -> `auth_config.client_id` +- Client Secret -> `auth_config.client_secret` +- fixed `auth_type = oauth_client` + +`aisstream_vessels`： + +- WebSocket endpoint +- API Key -> `auth_config.api_key` +- subscription / bounding boxes config + +`spacetrack_tle`： + +- API Base URL / endpoint +- Username -> `auth_config.username` +- Password -> `auth_config.password` +- auth schema = `session_cookie_login` +- login endpoint = `https://www.space-track.org/ajaxauth/login` +- login body fields：`identity` / `password` +- run/test request uses returned session cookie + +### AI Provider + +默认字段： + +- provider +- provider_api +- base_url +- model +- api_key +- max_tokens +- anthropic_version +- service_url +- service_token +- timeout_seconds +- retry_attempts +- model_provider_apis + +Provider presets supply initial defaults, but the editable schema controls which fields appear and where values are saved. + +AI Provider auth variants： + +- OpenAI-compatible providers：`api_key` or `bearer_token`。 +- Local/sidecar services：`none`、`service_token` or custom header。 +- OAuth-backed providers：`oauth2_authorization_code`，适用于需要用户授权登录的 provider。 +- CLI/session-backed tools such as Codex：优先作为 `tool` 或本机 runner 集成；如果作为 provider，必须显式声明会话来源、权限边界、不可多用户复用的限制。 + +### Tool + +`web_search`： + +- enabled +- provider / default_provider +- base_url +- api_key +- max_results +- timeout_seconds +- endpoint_path +- search_depth +- engine +- include_answer / include_raw_content / include_text +- search_path / scrape_path / scrape_formats + +`ocr`： + +- enabled +- provider +- base_url +- api_key +- model +- languages +- timeout_seconds +- max_file_size_mb +- output_format + +Tool auth variants： + +- GitHub PAT：`bearer_token`。 +- GitHub OAuth App：`oauth2_authorization_code`，适合用户授权登录和代表用户访问。 +- GitHub App：`app_installation`，需要 app id/private key/installation id，并通过 schema 声明 installation token exchange。 +- Browser/session tools：必须显式标记为 `session_local_only`，不能作为后台多用户稳定凭证。 + +## API Plan + +- `GET /api/v1/integration-config-schemas` + - Return the full registry. +- `PUT /api/v1/integration-config-schemas` + - Save the registry. Admin only. +- `POST /api/v1/integration-config-schemas/validate` + - Validate full registry or one schema. +- `POST /api/v1/integration-config-schemas/auth/test` + - Test auth schema with draft credentials without saving. +- `GET /api/v1/integration-config-schemas/auth/secrets` + - Reveal stored/env-backed secret fields for admins; write audit log. +- `POST /api/v1/integrations/oauth/{provider}/start` + - Start OAuth Authorization Code flow. +- `GET /api/v1/integrations/oauth/{provider}/callback` + - Complete OAuth callback and store token material according to schema. +- `GET /api/v1/datasources/configs/all` + - Add `form_schema` to each row. +- `GET /api/v1/settings/integrations` + - Add `form_schema` for AI Provider and tools. + +Existing save APIs remain compatible: + +- DataSource saves to `DataSourceConfig`. +- AI Provider, Web Search and OCR save to `external_integrations`. + +Secret fields never return plaintext through list/config endpoints. They return configured state and masked preview only. + +Secret reveal endpoints return plaintext only on explicit administrator action and must log target, actor, source, result and timestamp. List/config endpoints must never leak secret plaintext. + +## Frontend Plan + +Admin Next extracts a reusable `SchemaForm`: + +- Render fields from `form_schema.fields`. +- Build payload by writing values to each field `target`. +- Preserve masked secret semantics: unchanged masked values do not overwrite stored secrets. +- Reveal secrets through a schema-aware reveal action; if neither DB nor env has a value, show an empty editable input. +- Support text, secret, number, boolean, select, textarea, JSON and tags controls. +- Support auth controls for API key, username/password, OAuth connect/disconnect, session cookie login and custom preflight status. +- Validate schema before saving registry changes. + +Pages migrated in v1: + +- Collection Management / collector configs. +- AI / Provider configuration. +- AI / Tools configuration for Web Search and OCR. + +Each detail page gets a schema editing action for admins. The editor saves registry JSON after validation. + +## Test Plan + +- Default registry initializes with datasource, AI Provider and tool schemas. +- Validation rejects duplicate keys, illegal targets, illegal field types and secret plaintext defaults. +- `barentswatch_vessels` renders and saves `client_id` / `client_secret`. +- `spacetrack_tle` renders username/password, tests with draft credentials, and does not require API Key. +- `aisstream_vessels` renders API Key and WebSocket subscription fields. +- Session cookie login auth executes preflight before sample/run and uses the resulting cookie jar. +- OAuth Authorization Code schema can start callback flow, store token metadata and inject bearer token. +- Secret reveal returns DB value, env fallback or empty value according to source, and writes audit log. +- Connection tests always prefer current draft credentials over saved/env credentials. +- AI Provider renders and saves `provider_api`, `base_url`, `model`, `api_key` and `service_token`. +- AI Provider and tool schemas can reuse the same auth schema primitives as DataSource. +- Web Search and OCR render and save provider-specific tool fields. +- Adding a schema field in the registry makes it appear in Admin Next without frontend code changes. +- Existing connection tests, datasource sampling, datasource run, AI Provider connect/reveal/refresh, and Web Search connect keep working. + +## Assumptions + +- `target_schema_registry` remains separate because it describes collected result shape, not configuration forms. +- The first implementation stores schema registry in `SystemSetting`; no new database table is required. +- Old AntD Settings pages stay compatible but are not migrated in v1. +- Tool scope in v1 is Web Search and OCR. diff --git a/docs/technical/en/README.md b/docs/technical/en/README.md index 8cf105a8..b0db171e 100644 --- a/docs/technical/en/README.md +++ b/docs/technical/en/README.md @@ -30,8 +30,10 @@ What belongs here: - [Collector Settings and Connectivity Validation](/home/ray/dev/linkong/planet/docs/technical/en/datasource-collector-settings-connectivity.md): Data source catalog, collector settings, connectivity validation, and BarentsWatch credentials - [Shared Location Resolution Pipeline Development Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-development.md): Backend location resolver / pipeline interfaces, registries, and extension points - [Docs Gatekeeper Development Guide](/home/ray/dev/linkong/planet/docs/technical/en/docs-gatekeeper-development.md): Backend Docs catalog, Markdown content loading, and Gatekeeper permission groups +- [Naming Glossary](/home/ray/dev/linkong/planet/docs/technical/en/naming-glossary.md): English/Chinese term mapping for the console, Earth, backend, and docs - [Earth Interactable Usage](/home/ray/dev/linkong/planet/docs/technical/en/earth-interactable-usage.md): API, lifecycle, and integration examples for Earth surface icon Interactable - [Earth Toolbar and Overlay Coordination](/home/ray/dev/linkong/planet/docs/technical/en/earth-toolbar-overlay-coordination.md): Closing matrix and integration rules for toolbar buttons, search, settings, news, and layer overlays +- [Tactile UI Components](/home/ray/dev/linkong/planet/docs/technical/en/tactile-ui-components.md): Portable button, switch, tooltip, and scrollbar APIs, theme tokens, and migration rules What does not belong here: diff --git a/docs/technical/en/agents-aiprovider.md b/docs/technical/en/agents-aiprovider.md index 08de2d37..bef76ab4 100644 --- a/docs/technical/en/agents-aiprovider.md +++ b/docs/technical/en/agents-aiprovider.md @@ -101,6 +101,19 @@ The AI settings page uses: These endpoints require an authenticated user. The `secrets` endpoint is only used when the settings page reveals a key or token; hiding the field restores the masked preview. +Admin Next keeps the AI page aligned with the legacy information architecture: + +- `Model Providers` + - Manages provider, wire adapter, default model, LLM API key, proxy URL, proxy token, model refresh, set-as-default, and lightweight connectivity testing. +- `Tool Calling` + - Manages tools such as WebSearch and OCR. Each tool first selects a provider, then edits that provider's API, key, and advanced parameters. +- `Prompts` + - Edits system/user prompts by prompt group and task key. Save and reset only affect the current task. +- `Playground` + - Runs real conversations with the active provider and prompt configuration. AI responses are rendered as Markdown. + +Save, set-as-default, and connectivity testing are separate responsibilities: save only persists the form, set-as-default only changes the active provider/tool, and connectivity testing only validates the current draft. It must not implicitly save or switch defaults. + The `ai-prompts` endpoints back the Prompts tab in AI settings. Shipped defaults come from versioned backend resources, while business code references stable task keys. The API stores only operator overrides. Resetting a prompt removes the override and falls back to the current shipped default. ### Prompt Boundary @@ -259,6 +272,34 @@ Each provider has its own key slot. Resolution order is: `.env` is only a fallback. After the settings page saves successfully, or after the connection test succeeds, PostgreSQL becomes the global default source. +Admin Next must compute key status per provider or tool: + +- If the database has a key for the current provider/tool, show `configured`. +- If the database has no key but the fallback provider, model, or tool matches the current item, show the fallback masked preview. +- If neither database nor matching fallback exists, show `not configured`; a generic `.env` key for another provider must not make this item appear configured. +- Masking keeps the prefix before the first `-`, for example `sk-********`. Plaintext reveal is only available inside the authorized configuration page. + +Tool keys follow the same rule. WebSearch and OCR must match the current tool and provider before they can use fallback credentials. + +### Lightweight Connectivity Testing + +The Admin Next plug button performs a lightweight connectivity check and does not save configuration. Common API-platform practice is two-tiered: + +- Check a provider catalog or low-cost endpoint to validate base URL, authentication, and model reachability. +- Send full model requests only when the user explicitly runs Playground or a business task. + +Connectivity results should be explicit: + +- `ok`: authentication, route, and model catalog are usable. +- `warning`: service is reachable, but the current model is missing from the catalog or capability metadata is incomplete. +- `error`: authentication, network, protocol, or model lookup failed. + +Toast titles must match the result; failures must not be titled as a successful connection. + +### OpenCode Go Routing Model + +Subscription channels such as OpenCode Go should not be handled by hard-coded frontend model sets. Prefer provider catalog or backend capability discovery that records per-model capabilities such as `chat_completions`, `anthropic_messages`, `models_endpoint`, and whether a subscription key is required. The frontend should display capabilities; the backend should map provider, base URL, model, and adapter into the real request. + #### Settings Page Behavior - The Provider select controls the global default provider. diff --git a/docs/technical/en/backend-collectors.md b/docs/technical/en/backend-collectors.md index 30b55c7e..1ad84403 100644 --- a/docs/technical/en/backend-collectors.md +++ b/docs/technical/en/backend-collectors.md @@ -93,6 +93,16 @@ Earth boundaries are no longer data collectors. They are Earth static rendering TOP500 and Epoch AI compute sources do not always provide usable coordinates. The unified Earth compute-center endpoint uses only valid source-provided coordinates or `compute_center_locations` dimension-table coordinates during the main map startup path; records without coordinates are returned as `unresolved` instead of being rendered from a local registry, country centroid, or guessed city. When users manually collect candidates, the backend queries ROR and Nominatim/OpenStreetMap from source fields; accepted candidates are saved into `compute_center_locations` and rendered from that table on the next layer refresh. +Admin Next collection management follows the business hierarchy instead of flattening every endpoint into one table: + +- `Collectors`: endpoint, authentication, headers, base parameters, enabled state, and credential guides. +- `Collection Schedule`: scheduler state and task controls. +- `Collection History / Snapshots`: history grouped by collector, with a detail-side snapshot selector for versions. + +Snapshot lists should not show every snapshot of the same collector as separate top-level records. The top-level list selects a collector; the detail area switches between time versions. + +Credential guides are maintained by `backend/app/services/credential_guides.py`. The console uses read / generate / reset actions to load or create Markdown instructions. The frontend should render the guide Markdown for operators, not expose generation prompts or raw metadata. + ## IV. Data Format (stored in CollectedData table) ```python diff --git a/docs/technical/en/datasource-collector-settings-connectivity.md b/docs/technical/en/datasource-collector-settings-connectivity.md index cbfb18cd..8f30d1a4 100644 --- a/docs/technical/en/datasource-collector-settings-connectivity.md +++ b/docs/technical/en/datasource-collector-settings-connectivity.md @@ -32,8 +32,8 @@ If endpoint, headers, base configuration, or credential fingerprint changes afte Files: -- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx) -- [index.css](/home/ray/dev/linkong/planet/frontend/src/index.css) +- [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx) +- [AdminNextRoutes.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/AdminNextRoutes.tsx) Current behavior: @@ -53,30 +53,35 @@ Current behavior: `data-source-bulk-toolbar__running-pill` is the styling entry point for the "Collecting" pill. It is aligned with other status tags, while hover treatment, arrow affordance, and blue outline indicate interactivity. -### Collectors +### Collection Management File: -- [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx) +- [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx) Current behavior: -- The `collector_credentials` tab is displayed as "Collectors" under `/collection-management`. -- A select lists built-in collectors and supports maintaining custom supplemental sources that merge into built-in data. -- The only button beside the select is a plug icon for health checks. -- Status tags below the select show: - - `Credentials required` / `No credentials required` - - Module - - `Enabled` / `Disabled` - - `Unchecked` / `Available` / `Unavailable` - - Whether the endpoint is overridden -- Collectors that require credentials place the credential card above base configuration. -- Collectors without credentials only show base configuration. +- `/collection-management` follows the legacy hierarchy: `Collectors`, `Collection Schedule`, and `Collection History / Snapshots`. +- `Collectors` is the configuration page. The left list shows collector configs; the right form edits endpoint, authentication, headers, collection parameters, and enabled state. +- New collectors and target schemas use draft detail pages instead of transparent JSON modals; save persists the draft, while cancel destroys it. +- The connection button only tests connectivity and does not save. The save button only persists the form. +- Credential guides open in a draggable Markdown modal. When no guide exists, the modal still opens and offers a generate action. Generation shows a centered waiting state; reset confirmation must render above the guide modal. +- `Collection History / Snapshots` groups by collector. The list must not repeat every snapshot for the same datasource; detail view switches versions through a Time Capsule / Time Machine style selector. - The AISStream collector uses WebSocket semantics: connecting, streaming, reconnecting, or stopped. It does not use a fixed completion percentage. - Custom source editing lives in collector settings. The data source catalog keeps overview, run controls, and read-only drawers. The connection button uses an inline Tabler-style plug icon with `plug-connected` semantics, avoiding the older refresh icon for a connection action. +### Mapping Templates and Target Schemas + +Mapping remains part of collection management, but not the main collector tab: + +- `Mapping Templates` owns sample payload, AI propose, preview, create/update, and activate. +- `Target Schemas` maintains writable target structures. +- `run-mapped`, `stop-mapped`, and `stream-status` run mapped custom collectors. + +The new UI should keep these paths form-first. Only advanced fields should collapse into JSON. Do not flatten templates, schemas, runtime status, and collector configuration into one table. + ## Backend APIs ### Data Source Configuration List @@ -140,6 +145,26 @@ Successful responses include: - `credential_provider` - `credential_source` +### Credential Guides + +```http +GET /api/v1/datasources/credential-guides/{provider} +POST /api/v1/datasources/credential-guides/{provider}/generate +POST /api/v1/datasources/credential-guides/{provider}/reset +``` + +Purpose: + +- Read the Markdown credential guide for a provider. +- Generate a guide from collector metadata when none exists. +- Reset back to the backend default guide. + +Frontend rules: + +- Render Markdown, not backend prompts or metadata. +- Generate/reset actions belong inside the guide modal, not in the collector configuration toolbar. +- Missing guides still open a modal so the user can generate one from there. + ### BarentsWatch AIS Connectivity Validation ```http diff --git a/docs/technical/en/earth-frontend-context.md b/docs/technical/en/earth-frontend-context.md index 6af91d48..64a4d39c 100644 --- a/docs/technical/en/earth-frontend-context.md +++ b/docs/technical/en/earth-frontend-context.md @@ -151,6 +151,15 @@ Each module is responsible for its own: `brand.js` manages Earth HUD brand resources. Static assets provide the default brand; runtime overrides come from `/api/v1/earth/brand`, and uploaded images are served from `/earth-brand-assets/...`. The frontend must treat logo/title images and text fallback separately: if an image fails, show the text title; if text fields are empty, rely on backend defaults so the HUD brand area never renders blank. The console Earth Content page owns saving and resetting brand configuration; the Earth frontend only consumes it. +The Admin Next Earth Content page must preserve runtime semantics: + +- `Brand`: brand preview should use the same dark starfield background, size, spacing, logo/title rendering, and text fallback as the Earth HUD top-left brand block, not a generic form preview. +- `Boundary Precision`: build boundary, refresh status, and restore defaults belong inside this section, not in the global page toolbar. +- `TV`: the list distinguishes built-in, collected, and custom sources. Card state represents enabled, disabled, draft, or error. Built-in sources cannot be deleted; collected and custom sources can. A new live source only enters draft state after the plus button is clicked; save persists it into the list, while cancel destroys the draft. +- `Basemap`, `Layer Resources`, `3D Models`, and `News Anchor Strategy`: if backend capability is not available yet, the console should show an explicit pending state instead of mixing those items into TV or brand configuration. + +TV preview should reuse the Earth runtime live-card structure and state labels as closely as possible so built-in markers, live loading state, stream source, region, and language match what users see on Earth. + The compute-center layer row has a notification badge for GeoJSON `unresolved` records. The badge means "no trustworthy coordinates, cannot render on the globe"; it is different from the `?` marker drawn on already positioned but unconfirmed compute centers. Clicking the badge opens a fixed info card beside the layer panel. Row-level `采集` fetches candidates only. Header-level `一键采用` processes the queue top-to-bottom, saves the highest-confidence valid candidate, removes successful rows, renumbers the list, and dispatches `earth:compute-center-unresolved-count-change` so the badge updates immediately. When the batch ends, `earth:compute-center-location-saved` refreshes the real layer. Location candidate state in the details card is cached in [info-card.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/info-card.js) by `entityType:entityId`. If the user closes the details card or unresolved queue and reopens the same compute center / BGP collector, previously collected candidates and status text are restored. Header-level `一键采用` prefers cached candidates, avoiding repeated online geocoding or LLM factcheck calls. After a location is saved, that entity's candidate list is cleared to a "refreshing layer" status so stale candidates do not keep misleading the user. diff --git a/docs/technical/en/frontend-admin-frontend-context.md b/docs/technical/en/frontend-admin-frontend-context.md index 50a65113..7c0f47ea 100644 --- a/docs/technical/en/frontend-admin-frontend-context.md +++ b/docs/technical/en/frontend-admin-frontend-context.md @@ -22,7 +22,7 @@ Main entry point: - [App.tsx](/home/ray/dev/linkong/planet/frontend/src/App.tsx) -Current admin-related routes: +Admin Next now owns the official admin routes: - `/admin` - `/users` @@ -37,15 +37,45 @@ Current admin-related routes: - `/collection-management` - `/settings` +These routes render [AdminNextRoutes.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/AdminNextRoutes.tsx). Page metadata and menu entries come from [manifest.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/routes/manifest.tsx). `/admin-next/*` remains only as a compatibility entry and redirects to the official route; it is no longer a parallel primary entry. + +The old AntD console remains available under `/legacy/admin/*` for comparison and rollback: + +- `/legacy/admin` +- `/legacy/admin/datasources` +- `/legacy/admin/data` +- `/legacy/admin/collection-management` +- `/legacy/admin/earth-content` +- `/legacy/admin/ai` +- `/legacy/admin/logs` +- `/legacy/admin/settings` +- `/legacy/admin/users` +- `/legacy/admin/bgp` +- `/legacy/admin/alerts/*` + +Legacy pages, `AppLayout`, `antd`, and `@ant-design/icons` stay in place during the legacy validation window. Do not remove them before Admin Next parity is accepted. + `/earth` is a standalone display page and is not part of the console shell. ## Current Page Shell -The console shared shell is at: +The official admin shell is at: + +- [AdminNextLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/components/layout/AdminNextLayout.tsx) + +Responsibilities: + +- Left navigation, grouped collapse, and mobile drawer +- Current account, version, logout, and theme switching +- Top search, breadcrumbs, and page shortcuts +- Single-screen content-area height closure +- Coordination for Admin Next internal scrolling, tables, detail panels, and mobile detail views + +The old AntD legacy shell remains at: - [AppLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/components/AppLayout/AppLayout.tsx) -Responsibilities: +Legacy responsibilities: - Left-side navigation - Collapse and expand @@ -66,7 +96,19 @@ Current structure: ``` -Future console pages should adapt to this shell rather than redefining full-page height semantics. +Future official admin pages should adapt to `AdminNextLayout` and Admin Next page patterns rather than adding new capability to the old `AppLayout`. Only `/legacy/admin/*` maintenance should change the old shell. + +## Admin Next Section Loading + +Multi-tab pages are currently coordinated by [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx), which hosts the current management and information workbench patterns. Section loading follows these rules: + +- Initial page load requests only the active tab; it does not prefetch every tab endpoint. +- Switching tabs lazily loads that tab. Loaded tabs stay cached in local `states`, so returning to a tab reuses the previous data. +- Explicit actions such as refresh, save, test, upload, and credential-guide generation refresh only the current section instead of fanning out to unrelated sections. +- Datasource directory filter changes clear the built-in-source section cache and reload the current section with the new filters. +- Top summary metrics count loaded sections only, so unopened tabs are not reported as failed interfaces. + +This keeps Earth, AI, collection management, and other multi-section pages from flooding backend APIs on cold start while preserving a fast cached tab-switching experience. Full health checks should use backend health endpoints or explicit refresh flows rather than relying on page initialization to touch every business endpoint. ## Current Shared Components @@ -74,7 +116,8 @@ Future console pages should adapt to this shell rather than redefining full-page File: -- [Scrollbar.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/Scrollbar.tsx) +- [Scrollbar.tsx](/home/ray/dev/linkong/planet/frontend/src/components/tactile-ui/Scrollbar.tsx) +- [Tactile UI Components](/home/ray/dev/linkong/planet/docs/technical/en/tactile-ui-components.md) Purpose: @@ -91,7 +134,7 @@ Current constraint: File: -- [ScrollbarOverlay.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/ScrollbarOverlay.tsx) +- [ScrollbarOverlay.tsx](/home/ray/dev/linkong/planet/frontend/src/components/tactile-ui/ScrollbarOverlay.tsx) Purpose: @@ -100,25 +143,42 @@ Purpose: Current usage: -- Data sources -- Collected data -- User management -- Settings page -- Alerts page -- BGP page +- Admin Next data sources, collected data, collection management, logs, alerts, and BGP pages +- Old AntD legacy pages continue using shared scrolling behavior through compatibility wrappers ### 3. `TableScrollRegion` File: -- [TableScrollRegion.tsx](/home/ray/dev/linkong/planet/frontend/src/components/Scrollbar/TableScrollRegion.tsx) +- [TableScrollRegion.tsx](/home/ray/dev/linkong/planet/frontend/src/components/tactile-ui/TableScrollRegion.tsx) Purpose: - Provides a unified wrapper for table scroll areas - New table pages should reuse this rather than repeating the "table area + overlay scrollbar" boilerplate -### 4. `SegmentedControl` +### 4. `TactileButton` / `TactileSwitch` / `ControlGroup` + +Files: + +- [Button.tsx](/home/ray/dev/linkong/planet/frontend/src/components/tactile-ui/Button.tsx) +- [Switch.tsx](/home/ray/dev/linkong/planet/frontend/src/components/tactile-ui/Switch.tsx) +- [ControlGroup.tsx](/home/ray/dev/linkong/planet/frontend/src/components/tactile-ui/ControlGroup.tsx) + +Purpose: + +- Admin Next global tool buttons and detail-panel toolbars +- Icon-only ordinary actions with tooltips +- Strong-intent actions such as save, create, confirm, delete, and stop +- Compact switches aligned with the Docs theme slider + +Current constraints: + +- Neutral buttons default to a white tactile surface with external shadow +- Unambiguous actions prefer icon + tooltip; strong-intent actions like save may keep text +- Feature pages should customize through props and CSS variables, not by rewriting core button CSS + +### 5. `SegmentedControl` Files: @@ -146,7 +206,7 @@ Current constraints: - Prefer CSS variable overrides for colors instead of hard-coding theme colors in feature components - Best for a small set of mutually exclusive choices; do not use it as a long list, navigation menu, or select replacement -### 5. `MarkdownRenderer` +### 6. `MarkdownRenderer` File: @@ -165,7 +225,7 @@ Current constraints: - Internal document links should be converted to `/docs/:slug` through `transformLink` - Heading anchors are injected through `getHeadingId`, keeping route state outside the renderer -### 6. `ConnectionTestInput` +### 7. `ConnectionTestInput` File: @@ -183,7 +243,7 @@ Current constraints: - Disabled integrations must grey out both the input and its connection-test action - The component only combines the input and action; callers still own form state, loading, disabled state, and the request itself -### 7. `TableActions` +### 8. `TableActions` File: diff --git a/docs/technical/en/naming-glossary.md b/docs/technical/en/naming-glossary.md new file mode 100644 index 00000000..6a3c370b --- /dev/null +++ b/docs/technical/en/naming-glossary.md @@ -0,0 +1,99 @@ +# Naming Glossary + +This document standardizes terms used across the Planet console, Earth, backend APIs, and documentation. When adding Chinese UI labels, chart labels, or translated documentation, prefer the Chinese display names listed here to avoid unnecessary mixed Chinese/English copy. + +## Usage Rules + +- Chinese UI should use the Chinese display name. +- API fields, database fields, code identifiers, and external brand names keep their original English or snake_case form. +- On first mention in long-form docs, use “Chinese Display Name (English Term)” when clarification helps; subsequent mentions can use the Chinese display name. +- Industry abbreviations such as BGP, AI, API, URL, ID, TLE, ASN, GPU, and AIS may remain English. +- External service brands keep their official English names, such as PeeringDB, Space-Track, BarentsWatch, AISStream, and GitHub. + +## Core Product Terms + +| English / Key | Chinese Display Name | Usage | +| --- | --- | --- | +| Planet | Planet | Product name | +| Admin Next | 新控制台 | New admin console | +| Earth | Earth | Visualization product name | +| datasource | 数据源 | APIs, lists, filters | +| collector | 采集器 | Collection jobs and credential configuration | +| collected data | 采集数据 | Data list and statistics | +| snapshot | 快照 | Collection history | +| mapping | 映射 | Field mapping | +| target schema | 目标 Schema | Mapping target structure | +| credential | 凭证 | Secrets, account passwords, OAuth material | +| secret reveal | 密钥 reveal | Explicit administrator secret reveal action | +| AI Provider | AI Provider | Service name | +| tool | 工具 | Web Search, OCR, and similar integrations | +| Playground | Playground | Interactive debugging entry | + +## Data Types + +| data_type | Chinese Display Name | Meaning | +| --- | --- | --- | +| `supercomputer` | 超算 | TOP500 supercomputer record | +| `gpu_cluster` | GPU 集群 | Epoch AI GPU cluster | +| `model` | 模型 | Hugging Face model | +| `dataset` | 数据集 | Hugging Face dataset | +| `space` | 空间 | Hugging Face Space | +| `ixp` | 互联网交换点 | PeeringDB IXP | +| `network` | 网络 | PeeringDB network | +| `facility` | 设施 | PeeringDB facility | +| `submarine_cable` | 海底光缆 | Submarine cable route | +| `landing_point` | 登陆点 | Cable landing point | +| `cable_system` | 海缆系统 | TeleGeography cable system | +| `cable_landing_relation` | 海缆登陆关系 | Cable-to-landing-point relation | +| `satellite_tle` | 卫星轨道根数 | CelesTrak / Space-Track TLE | +| `bgp_rib` | BGP 路由表 | BGP RIB snapshot | +| `bgp_update` | BGP 更新 | BGP update event | +| `prefix_geography` | 前缀地理位置 | IP prefix geography | +| `vessel_ais` | AIS 船舶 | AIS vessel observation | +| `vessel` | 船舶 | Aggregated vessel object | +| `news_live_stream` | 新闻直播源 | News live stream source | +| `news_item` | 新闻条目 | Archived news item | +| `device_stats` | 设备统计 | Cloudflare Radar device statistics | +| `traffic_stats` | 流量统计 | Cloudflare Radar traffic statistics | +| `as_stats` | 自治系统统计 | Cloudflare Radar AS statistics | +| `compute_center` | 算力中心 | Aggregated Earth compute-center object | +| `generic` | 通用数据 | Generic collector output | +| `generic_records` | 通用记录 | Generic mapping output | + +## Datasources And External Services + +| English / Key | Chinese Display Name | Notes | +| --- | --- | --- | +| TOP500 | TOP500 超算榜单 | Keep TOP500 as the external list name | +| Epoch AI | Epoch AI GPU 集群 | Keep Epoch AI as the service name | +| Hugging Face | Hugging Face | Official brand name | +| PeeringDB IXP | PeeringDB 交换中心 | IXP can also be explained as 互联网交换点 | +| PeeringDB Networks | PeeringDB 网络 | Network entity | +| PeeringDB Facilities | PeeringDB 设施 | Facility entity | +| Submarine Cables | 海底光缆 | Prefer Chinese in charts and lists | +| Cable Landing Points | 海缆登陆点 | Use 登陆点 consistently | +| CelesTrak TLE | CelesTrak 轨道根数 | TLE may remain abbreviated | +| Space-Track TLE | Space-Track 轨道根数 | Keep Space-Track as brand name | +| RIPE RIS Live BGP | RIPE RIS Live BGP | Keep service name and BGP abbreviation | +| IPtoASN Prefix Geography | IPtoASN 前缀地理位置 | Keep IPtoASN as dataset name | +| OpenGeoFeed Prefix Geography | OpenGeoFeed 前缀地理位置 | Keep OpenGeoFeed as brand name | +| NRO Delegated Prefix Geography | NRO Delegated 前缀地理位置 | Keep NRO Delegated as dataset name | +| BarentsWatch AIS Vessels | BarentsWatch AIS 船舶 | Keep BarentsWatch / AIS | +| AISStream Vessels | AISStream 船舶 | Keep AISStream as brand name | + +## UI Actions And Statuses + +| English / Key | Chinese Display Name | Usage | +| --- | --- | --- | +| success | 成功 | Status badge | +| failed | 失败 | Status badge | +| running | 运行中 | Task status | +| completed | 已完成 | Task status | +| cancelled | 已取消 | Task status | +| enabled | 已启用 | Toggle state | +| disabled | 已停用 | Toggle state | +| configured | 已配置 | Credential / integration state | +| unconfigured | 未配置 | Credential / integration state | +| refresh | 刷新 | Button | +| copy | 复制 | Button | +| reveal | reveal | Keep English for audited secret reveal semantics | diff --git a/docs/technical/en/ops-runbook.md b/docs/technical/en/ops-runbook.md index 1751acdf..5306ab0e 100644 --- a/docs/technical/en/ops-runbook.md +++ b/docs/technical/en/ops-runbook.md @@ -87,8 +87,8 @@ Recent logs: Follow: ```bash -./planet.sh log -f # frontend: /tmp/planet_frontend.log -./planet.sh log -b # backend: /tmp/planet_backend.log +./planet.sh log -f # frontend: ~/.local/state/planet/frontend.log +./planet.sh log -b # backend: ~/.local/state/planet/backend.log ./planet.sh log -a # AI Provider: planet_aiprovider container logs ``` @@ -234,6 +234,20 @@ Validate the frontend build: source ~/.zshrc && bun run build ``` +Use `bun run dev` during development; Vite HMR refreshes the browser after source saves. `bun run build` only writes the `dist` artifact and does not refresh an already-open dev page. + +To inspect the production bundle with automatic reload after successful builds: + +```bash +bun run preview:auto +``` + +To watch sources and rebuild continuously without starting the preview server: + +```bash +bun run build:watch +``` + Backend dependencies are managed with uv: ```bash diff --git a/docs/technical/en/tactile-ui-components.md b/docs/technical/en/tactile-ui-components.md new file mode 100644 index 00000000..0a7b15d6 --- /dev/null +++ b/docs/technical/en/tactile-ui-components.md @@ -0,0 +1,205 @@ +# Tactile UI Components + +Tactile UI is Planet's portable React control layer. It was extracted from the Admin Next button, switch, scrollbar, and tooltip work, but the components themselves do not depend on Admin Next, AntD, Radix, Tailwind, or `an-*` classes. The immediate goal is stable in-repo usage; the structure is intentionally close to something that can later be published as an npm package. + +## Design Goals + +- **Light tactile feel**: controls default to a white or themed surface, thin borders, and external shadow, matching the subtle depth of the Docs theme slider rather than large colored blocks or glow. +- **Portable styling**: classes use the `tui-*` prefix and styles live in `frontend/src/components/tactile-ui/styles.css`. +- **Low dependency surface**: components assume React and React DOM. Preset icons currently use `lucide-react`, and callers may also pass custom React nodes. +- **Theme friendly**: CSS variables expose the default styling surface. Planet pages adapt the library through theme variables and props. +- **Clear semantics**: unambiguous actions should prefer icon-only buttons with tooltips; strong-intent actions such as save, confirm, create, and run can keep text. + +## Import + +In this repository: + +```tsx +import { TactileButton, TactileSwitch, ControlGroup } from '@/components/tactile-ui' +import '@/components/tactile-ui/styles.css' +``` + +After a future package extraction: + +```tsx +import { TactileButton, TactileSwitch } from '@planet/tactile-ui' +import '@planet/tactile-ui/styles.css' +``` + +## Theme Tokens + +Core tokens are exposed as `--tui-*` CSS variables. Product themes should override variables instead of rewriting internal component classes. + +```css +:root { + --tui-surface: #ffffff; + --tui-surface-raised-hover: #f8fbff; + --tui-border-soft: #d6dfeb; + --tui-border-hover: #b8c6d9; + --tui-text: #0f172a; + --tui-primary: #2563eb; + --tui-danger: #dc2626; +} + +[data-theme='dark'] { + --tui-surface: #111827; + --tui-text: #e5edf8; +} +``` + +Most controls also accept a `tactile` prop for local width, height, radius, background, border, and shadow overrides. Use local overrides for small special cases; use CSS variables for product-wide styling. + +## `TactileButton` + +The button component covers regular buttons, icon buttons, strong-intent buttons, and link-like buttons. + +Common props: + +| Prop | Description | +| --- | --- | +| `variant` | `neutral`, `primary`, `danger`, `subtle`, or `ghost` | +| `size` | `sm`, `md`, `lg`, or `icon` | +| `shape` | `square` or `pill` | +| `icon` | Preset icon name or a custom React node | +| `iconOnly` | Fixed-size icon button; provide `tooltip` or `aria-label` | +| `tooltip` | Rendered through a portal and offset away from the cursor | +| `loading` | Disables the button and exposes `aria-disabled` | +| `tactile` | Overrides size, radius, shadow, background, and dark-mode variables | + +```tsx + +Save + +``` + +`variant="neutral"` defaults to a white tactile button. Colored buttons should still keep the same height and external shadow instead of relying on page-specific CSS overrides. + +## Icon Presets + +Preset icons are maintained in `tactileIconPresets`. Feature pages should call icons by semantic name so actions remain consistent across the console. + +Common semantics: + +| Name | Use | +| --- | --- | +| `refresh` | Refresh data | +| `save` | Save | +| `delete` | Delete | +| `trigger` / `collect` | Trigger collection | +| `start` / `play` | Start | +| `stop` | Stop | +| `connect` / `test` | Connectivity check | +| `guide` | Credential guide | +| `generate` | AI generation | +| `reset` | Restore defaults | +| `detail` | View details | +| `copy` | Copy | +| `upload` | Upload | + +Custom icons are also supported: + +```tsx +