release: bump version to 0.44.0
This commit is contained in:
linkong
209
backend/app/services/barentswatch.py
Normal file
209
backend/app/services/barentswatch.py
Normal file
@@ -0,0 +1,209 @@
|
||||
"""BarentsWatch AIS credential resolution and connectivity checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shlex
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
|
||||
|
||||
BARENTSWATCH_LATEST_URL = "https://live.ais.barentswatch.no/v1/latest/combined"
|
||||
BARENTSWATCH_TOKEN_URL = "https://id.barentswatch.no/connect/token"
|
||||
BARENTSWATCH_DATASOURCE_NAME = "barentswatch_vessels"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BarentsWatchConfig:
|
||||
endpoint: str
|
||||
client_id: str
|
||||
client_secret: str
|
||||
credential_source: str
|
||||
endpoint_source: str
|
||||
|
||||
|
||||
def _read_zshrc_env(path: Path | None = None) -> dict[str, str]:
|
||||
zshrc_path = path or Path.home() / ".zshrc"
|
||||
if not zshrc_path.exists():
|
||||
return {}
|
||||
|
||||
values: dict[str, str] = {}
|
||||
for raw_line in zshrc_path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if line.startswith("export "):
|
||||
line = line[len("export ") :].strip()
|
||||
if "=" not in line:
|
||||
continue
|
||||
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
if not key or not key.replace("_", "").isalnum() or not key[0].isalpha():
|
||||
continue
|
||||
|
||||
try:
|
||||
parsed = shlex.split(value, comments=True, posix=True)
|
||||
except ValueError:
|
||||
parsed = [value.strip().strip("'\"")]
|
||||
if parsed:
|
||||
values[key] = parsed[0]
|
||||
return values
|
||||
|
||||
|
||||
def _first_env_value(zshrc_env: dict[str, str], *keys: str) -> tuple[str, str]:
|
||||
for key in keys:
|
||||
value = os.getenv(key)
|
||||
if value:
|
||||
return value, "environment"
|
||||
for key in keys:
|
||||
value = zshrc_env.get(key)
|
||||
if value:
|
||||
return value, "~/.zshrc"
|
||||
return "", ""
|
||||
|
||||
|
||||
async def get_barentswatch_datasource_record(db: AsyncSession) -> DataSourceConfig | None:
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == BARENTSWATCH_DATASOURCE_NAME)
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def resolve_barentswatch_config(db: AsyncSession | None = None) -> BarentsWatchConfig:
|
||||
record = await get_barentswatch_datasource_record(db) if db else None
|
||||
auth_config = dict(record.auth_config or {}) if record else {}
|
||||
config = dict(record.config or {}) if record else {}
|
||||
zshrc_env = _read_zshrc_env()
|
||||
|
||||
env_client_id, env_source = _first_env_value(
|
||||
zshrc_env,
|
||||
"BARENTSWATCH_CLIENT_ID",
|
||||
"BARRENTSWATCH_CLIENT_ID",
|
||||
)
|
||||
env_client_secret, secret_env_source = _first_env_value(
|
||||
zshrc_env,
|
||||
"BARENTSWATCH_CLIENT_SECRET",
|
||||
"BARRENTSWATCH_CLIENT_SECRET",
|
||||
)
|
||||
client_id = auth_config.get("client_id") or config.get("client_id") or env_client_id
|
||||
client_secret = (
|
||||
auth_config.get("client_secret") or config.get("client_secret") or env_client_secret
|
||||
)
|
||||
|
||||
credential_source = ""
|
||||
if auth_config.get("client_id") or auth_config.get("client_secret"):
|
||||
credential_source = "datasource_config"
|
||||
elif config.get("client_id") or config.get("client_secret"):
|
||||
credential_source = "datasource_runtime_config"
|
||||
elif env_source or secret_env_source:
|
||||
credential_source = env_source or secret_env_source
|
||||
|
||||
yaml_endpoint = get_data_sources_config().get_yaml_url(BARENTSWATCH_DATASOURCE_NAME)
|
||||
endpoint = record.endpoint if record and record.endpoint else yaml_endpoint
|
||||
return BarentsWatchConfig(
|
||||
endpoint=endpoint or BARENTSWATCH_LATEST_URL,
|
||||
client_id=str(client_id or ""),
|
||||
client_secret=str(client_secret or ""),
|
||||
credential_source=credential_source or "missing",
|
||||
endpoint_source="datasource_config" if record and record.endpoint else "default",
|
||||
)
|
||||
|
||||
|
||||
async def fetch_barentswatch_access_token(
|
||||
client: httpx.AsyncClient,
|
||||
config: BarentsWatchConfig,
|
||||
) -> str | None:
|
||||
if not config.client_id or not config.client_secret:
|
||||
return None
|
||||
|
||||
response = await client.post(
|
||||
BARENTSWATCH_TOKEN_URL,
|
||||
data={
|
||||
"client_id": config.client_id,
|
||||
"client_secret": config.client_secret,
|
||||
"scope": "ais",
|
||||
"grant_type": "client_credentials",
|
||||
},
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
token = payload.get("access_token")
|
||||
return str(token) if token else None
|
||||
|
||||
|
||||
async def check_barentswatch_connectivity(db: AsyncSession) -> dict[str, Any]:
|
||||
config = await resolve_barentswatch_config(db)
|
||||
return await check_barentswatch_config(config)
|
||||
|
||||
|
||||
async def check_barentswatch_config(config: BarentsWatchConfig) -> dict[str, Any]:
|
||||
if not config.client_id or not config.client_secret:
|
||||
return {
|
||||
"success": False,
|
||||
"stage": "credentials",
|
||||
"message": "未找到 BarentsWatch client id/client secret，请先配置采集器凭证。",
|
||||
"endpoint": config.endpoint,
|
||||
"credential_source": config.credential_source,
|
||||
"settings_tab": "collector_credentials",
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=20.0) as client:
|
||||
token = await fetch_barentswatch_access_token(client, config)
|
||||
if not token:
|
||||
return {
|
||||
"success": False,
|
||||
"stage": "token",
|
||||
"message": "BarentsWatch token 响应中没有 access_token，请检查凭证。",
|
||||
"endpoint": config.endpoint,
|
||||
"credential_source": config.credential_source,
|
||||
"settings_tab": "collector_credentials",
|
||||
}
|
||||
|
||||
async with client.stream(
|
||||
"GET",
|
||||
config.endpoint,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"stage": "endpoint",
|
||||
"message": "BarentsWatch AIS token 和数据接口均可连通。",
|
||||
"endpoint": config.endpoint,
|
||||
"credential_source": config.credential_source,
|
||||
"endpoint_source": config.endpoint_source,
|
||||
}
|
||||
except httpx.HTTPStatusError as exc:
|
||||
status_code = exc.response.status_code
|
||||
stage = "token" if str(exc.request.url) == BARENTSWATCH_TOKEN_URL else "endpoint"
|
||||
return {
|
||||
"success": False,
|
||||
"stage": stage,
|
||||
"message": f"BarentsWatch {stage} 请求返回 HTTP {status_code}，请检查凭证或接口地址。",
|
||||
"endpoint": config.endpoint,
|
||||
"credential_source": config.credential_source,
|
||||
"settings_tab": "collector_credentials",
|
||||
}
|
||||
except httpx.HTTPError as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"stage": "network",
|
||||
"message": f"BarentsWatch 链路检查失败：{exc.__class__.__name__}",
|
||||
"endpoint": config.endpoint,
|
||||
"credential_source": config.credential_source,
|
||||
"settings_tab": "collector_credentials",
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
"""BarentsWatch AIS collector for vessel tracking."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -9,13 +8,14 @@ from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.vessel import VesselPosition, VesselStatic
|
||||
from app.services.barentswatch import (
|
||||
BARENTSWATCH_LATEST_URL,
|
||||
fetch_barentswatch_access_token,
|
||||
resolve_barentswatch_config,
|
||||
)
|
||||
from app.services.collectors.base import BaseCollector
|
||||
|
||||
|
||||
BARENTSWATCH_LATEST_URL = "https://live.ais.barentswatch.no/v1/latest/combined"
|
||||
BARENTSWATCH_TOKEN_URL = "https://id.barentswatch.no/connect/token"
|
||||
|
||||
|
||||
VESSEL_TYPE_NAMES = {
|
||||
30: "Fishing",
|
||||
35: "Military",
|
||||
@@ -38,62 +38,9 @@ class VesselAISCollector(BaseCollector):
|
||||
def base_url(self) -> str:
|
||||
return self._resolved_url or BARENTSWATCH_LATEST_URL
|
||||
|
||||
async def _load_datasource_config(self) -> dict[str, Any]:
|
||||
if not self._db_session:
|
||||
return {}
|
||||
try:
|
||||
from sqlalchemy import select
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
|
||||
result = await self._db_session.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == self.name)
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
datasource_config = result.scalar_one_or_none()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
if not datasource_config:
|
||||
return {}
|
||||
return {
|
||||
"auth_config": datasource_config.auth_config or {},
|
||||
"config": datasource_config.config or {},
|
||||
}
|
||||
|
||||
async def _get_access_token(self, client: httpx.AsyncClient) -> str | None:
|
||||
datasource_config = await self._load_datasource_config()
|
||||
auth_config = datasource_config.get("auth_config") or {}
|
||||
config = datasource_config.get("config") or {}
|
||||
client_id = (
|
||||
auth_config.get("client_id")
|
||||
or config.get("client_id")
|
||||
or os.getenv("BARENTSWATCH_CLIENT_ID")
|
||||
or os.getenv("BARRENTSWATCH_CLIENT_ID")
|
||||
)
|
||||
client_secret = (
|
||||
auth_config.get("client_secret")
|
||||
or config.get("client_secret")
|
||||
or os.getenv("BARENTSWATCH_CLIENT_SECRET")
|
||||
or os.getenv("BARRENTSWATCH_CLIENT_SECRET")
|
||||
)
|
||||
if not client_id or not client_secret:
|
||||
return None
|
||||
|
||||
response = await client.post(
|
||||
BARENTSWATCH_TOKEN_URL,
|
||||
data={
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"scope": "ais",
|
||||
"grant_type": "client_credentials",
|
||||
},
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
token = payload.get("access_token")
|
||||
return str(token) if token else None
|
||||
config = await resolve_barentswatch_config(self._db_session)
|
||||
return await fetch_barentswatch_access_token(client, config)
|
||||
|
||||
async def fetch(self) -> list[dict[str, Any]]:
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
|
||||
165
backend/app/services/credential_guides.py
Normal file
165
backend/app/services/credential_guides.py
Normal file
@@ -0,0 +1,165 @@
|
||||
"""Credential setup guides for collector integrations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient
|
||||
|
||||
|
||||
CREDENTIAL_GUIDES_CATEGORY = "collector_credential_guides"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CredentialGuideDefault:
|
||||
provider: str
|
||||
title: str
|
||||
prompt: str
|
||||
markdown: str
|
||||
|
||||
|
||||
BARENTSWATCH_DEFAULT_GUIDE = CredentialGuideDefault(
|
||||
provider="barentswatch",
|
||||
title="BarentsWatch AIS 凭证获取教程",
|
||||
prompt=(
|
||||
"请生成一份中文教程，指导开发者获取 BarentsWatch Live AIS API 的 "
|
||||
"OAuth client credentials。教程要面向已经有本地开发环境的人，包含注册/登录、"
|
||||
"创建 client、申请或确认 ais scope、复制 client id 和 client secret、"
|
||||
"在系统设置中填写并验证连接、常见失败排查。不要编造具体页面按钮文案，"
|
||||
"必须参考官方 tutorial：https://developer.barentswatch.no/docs/tutorial 。"
|
||||
"必须强调 Live AIS 要选择 AIS-client / AIS - API，而不是普通 API-client。"
|
||||
"如果步骤可能变化，要提醒以 BarentsWatch developer portal 当前页面为准。"
|
||||
),
|
||||
markdown="""## BarentsWatch AIS 凭证获取
|
||||
|
||||
官方教程：https://developer.barentswatch.no/docs/tutorial
|
||||
|
||||
1. 先打开上面的 BarentsWatch 官方 tutorial，按官方流程登录或注册开发者账号。
|
||||
2. 在 Developer access 页面选择 `AIS - API`，不要选择普通的 `BarentsWatch - API`。
|
||||
3. 在 `AIS - API` 下创建用于 Planet 的 AIS client。
|
||||
4. 创建时记下你设置的 password / client secret。
|
||||
5. 回到 My Page 复制完整 `Client ID`。它通常长得像 `your.email@example.com:client-name`。
|
||||
6. 回到 Planet 的 `设置 -> 采集器设置 -> BarentsWatch AIS`，填入 `Client ID` 和 `Client Secret`。
|
||||
7. 点击 `连接` 验证 token 和 AIS endpoint 是否可访问。
|
||||
8. 连接成功后保存凭证。
|
||||
|
||||
### 请求规则
|
||||
|
||||
- Token 地址：`https://id.barentswatch.no/connect/token`
|
||||
- 请求方式：`POST`
|
||||
- Content-Type：`application/x-www-form-urlencoded`
|
||||
- Body 必须包含：`grant_type=client_credentials`、`client_id`、`client_secret`、`scope=ais`
|
||||
- `client_id`、`client_secret`、`scope`、`grant_type` 都要放在 body，不要放在 header。
|
||||
- AIS 数据请求使用 header：`Authorization: Bearer <access_token>`
|
||||
|
||||
### 常见排查
|
||||
|
||||
- `未找到凭证`：确认 `Client ID` 和 `Client Secret` 已填写，或已经写入 `~/.zshrc`。
|
||||
- `HTTP 401/403`：通常是选成了普通 `BarentsWatch - API` client、client secret 错误，或 token 请求没有使用 `scope=ais`。
|
||||
- `network` 错误：检查本机是否能访问 `id.barentswatch.no` 和 `live.ais.barentswatch.no`。
|
||||
- Endpoint 建议保持默认：`https://live.ais.barentswatch.no/v1/latest/combined`。
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_CREDENTIAL_GUIDES = {
|
||||
BARENTSWATCH_DEFAULT_GUIDE.provider: BARENTSWATCH_DEFAULT_GUIDE,
|
||||
}
|
||||
|
||||
|
||||
async def _get_guide_store(db) -> tuple[SystemSetting | None, dict[str, Any]]:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == CREDENTIAL_GUIDES_CATEGORY)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
payload = dict(record.payload or {}) if record and isinstance(record.payload, dict) else {}
|
||||
return record, payload
|
||||
|
||||
|
||||
async def get_credential_guide(db, provider: str) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
|
||||
_record, store = await _get_guide_store(db)
|
||||
custom = store.get(provider) if isinstance(store.get(provider), dict) else None
|
||||
return {
|
||||
"provider": provider,
|
||||
"title": custom.get("title") if custom else default.title,
|
||||
"markdown": custom.get("markdown") if custom else default.markdown,
|
||||
"prompt": default.prompt,
|
||||
"source": "ai" if custom else "default",
|
||||
}
|
||||
|
||||
|
||||
async def save_credential_guide(db, provider: str, title: str, markdown: str) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
|
||||
record, store = await _get_guide_store(db)
|
||||
store[provider] = {
|
||||
"title": title or default.title,
|
||||
"markdown": markdown,
|
||||
}
|
||||
if record is None:
|
||||
db.add(SystemSetting(category=CREDENTIAL_GUIDES_CATEGORY, payload=store))
|
||||
else:
|
||||
record.payload = store
|
||||
await db.commit()
|
||||
return await get_credential_guide(db, provider)
|
||||
|
||||
|
||||
async def reset_credential_guide(db, provider: str) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
|
||||
record, store = await _get_guide_store(db)
|
||||
if provider in store:
|
||||
store.pop(provider, None)
|
||||
if record is not None:
|
||||
record.payload = store
|
||||
await db.commit()
|
||||
return await get_credential_guide(db, provider)
|
||||
|
||||
|
||||
async def generate_credential_guide(
|
||||
db,
|
||||
provider: str,
|
||||
ai_client: AIProviderClient,
|
||||
) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
|
||||
response = await ai_client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title=f"Generate credential guide for {provider}",
|
||||
objective=default.prompt,
|
||||
context={
|
||||
"provider": provider,
|
||||
"current_default_guide": default.markdown,
|
||||
"product_context": "Planet collector credential settings",
|
||||
},
|
||||
observations=[
|
||||
"Use concise Chinese markdown.",
|
||||
"Prefer stable concepts over brittle UI labels.",
|
||||
"Include verification and troubleshooting steps.",
|
||||
],
|
||||
constraints=[
|
||||
"Do not ask the user for secrets.",
|
||||
"Do not include fabricated screenshots.",
|
||||
"Return markdown only.",
|
||||
],
|
||||
)
|
||||
)
|
||||
markdown = response.content.strip()
|
||||
if not markdown:
|
||||
markdown = default.markdown
|
||||
return await save_credential_guide(db, provider, default.title, markdown)
|
||||
384
backend/app/services/datasource_connectivity.py
Normal file
384
backend/app/services/datasource_connectivity.py
Normal file
@@ -0,0 +1,384 @@
|
||||
"""Connectivity validation helpers for built-in datasource overrides."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.services.barentswatch import (
|
||||
_read_zshrc_env,
|
||||
fetch_barentswatch_access_token,
|
||||
resolve_barentswatch_config,
|
||||
)
|
||||
|
||||
|
||||
CONNECTIVITY_VALIDATION_KEY = "connectivity_validation"
|
||||
CONNECTIVITY_STORE_CATEGORY = "datasource_connectivity_validations"
|
||||
|
||||
|
||||
def _sha256_json(payload: Any) -> str:
|
||||
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _resolve_spacetrack_credentials() -> tuple[str, str, str]:
|
||||
zshrc_env = _read_zshrc_env()
|
||||
username = os.getenv("SPACETRACK_USERNAME") or zshrc_env.get("SPACETRACK_USERNAME") or ""
|
||||
password = os.getenv("SPACETRACK_PASSWORD") or zshrc_env.get("SPACETRACK_PASSWORD") or ""
|
||||
source = "environment" if os.getenv("SPACETRACK_USERNAME") or os.getenv("SPACETRACK_PASSWORD") else ""
|
||||
if not source and (username or password):
|
||||
source = "~/.zshrc"
|
||||
return username, password, source or "missing"
|
||||
|
||||
|
||||
def strip_connectivity_validation(config: dict | None) -> dict:
|
||||
cleaned = dict(config or {})
|
||||
cleaned.pop(CONNECTIVITY_VALIDATION_KEY, None)
|
||||
return cleaned
|
||||
|
||||
|
||||
def merge_connectivity_validation(existing_config: dict | None, next_config: dict | None) -> dict:
|
||||
merged = strip_connectivity_validation(next_config)
|
||||
validation = (existing_config or {}).get(CONNECTIVITY_VALIDATION_KEY)
|
||||
if validation:
|
||||
merged[CONNECTIVITY_VALIDATION_KEY] = validation
|
||||
return merged
|
||||
|
||||
|
||||
def get_connectivity_validation(config: DataSourceConfig | None) -> dict | None:
|
||||
validation = (config.config or {}).get(CONNECTIVITY_VALIDATION_KEY) if config else None
|
||||
return validation if isinstance(validation, dict) else None
|
||||
|
||||
|
||||
async def build_builtin_connectivity_checksum(
|
||||
source: str,
|
||||
endpoint: str,
|
||||
auth_type: str,
|
||||
headers: dict | None,
|
||||
config: dict | None,
|
||||
db=None,
|
||||
credential_override: dict[str, str] | None = None,
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
defaults = DEFAULT_DATASOURCES.get(source, {})
|
||||
credential_provider = defaults.get("credential_provider")
|
||||
credential_fingerprint = ""
|
||||
credential_source = "none"
|
||||
has_credentials = not defaults.get("requires_credentials", False)
|
||||
|
||||
if credential_provider == "barentswatch":
|
||||
if credential_override:
|
||||
client_id = credential_override.get("client_id", "")
|
||||
client_secret = credential_override.get("client_secret", "")
|
||||
credential_source = "draft"
|
||||
else:
|
||||
barentswatch_config = await resolve_barentswatch_config(db)
|
||||
client_id = barentswatch_config.client_id
|
||||
client_secret = barentswatch_config.client_secret
|
||||
credential_source = barentswatch_config.credential_source
|
||||
has_credentials = bool(client_id and client_secret)
|
||||
credential_fingerprint = _sha256_json(
|
||||
{
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
}
|
||||
)
|
||||
elif credential_provider == "spacetrack":
|
||||
username, password, credential_source = _resolve_spacetrack_credentials()
|
||||
has_credentials = bool(username and password)
|
||||
credential_fingerprint = _sha256_json(
|
||||
{
|
||||
"username": username,
|
||||
"password": password,
|
||||
}
|
||||
)
|
||||
elif defaults.get("requires_credentials"):
|
||||
credential_source = str(credential_provider or "unsupported")
|
||||
|
||||
checksum_payload = {
|
||||
"source": source,
|
||||
"endpoint": endpoint,
|
||||
"auth_type": "none",
|
||||
"headers": headers or {},
|
||||
"config": strip_connectivity_validation(config),
|
||||
"credential_provider": credential_provider or "none",
|
||||
"credential_fingerprint": credential_fingerprint,
|
||||
}
|
||||
return _sha256_json(checksum_payload), {
|
||||
"requires_credentials": bool(defaults.get("requires_credentials", False)),
|
||||
"credential_provider": credential_provider,
|
||||
"credential_source": credential_source,
|
||||
"has_credentials": has_credentials,
|
||||
}
|
||||
|
||||
|
||||
async def test_builtin_connectivity(
|
||||
source: str,
|
||||
endpoint: str,
|
||||
auth_type: str,
|
||||
headers: dict | None,
|
||||
config: dict | None,
|
||||
db=None,
|
||||
) -> dict[str, Any]:
|
||||
defaults = DEFAULT_DATASOURCES.get(source)
|
||||
if not defaults:
|
||||
return {
|
||||
"success": False,
|
||||
"message": "未知内置采集器，无法执行连接校验。",
|
||||
}
|
||||
|
||||
checksum, credential_context = await build_builtin_connectivity_checksum(
|
||||
source,
|
||||
endpoint,
|
||||
auth_type,
|
||||
headers,
|
||||
config,
|
||||
db,
|
||||
)
|
||||
if credential_context["requires_credentials"] and not credential_context["has_credentials"]:
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "credentials",
|
||||
"message": "该采集器需要凭证，请先到采集器凭证设置中配置。",
|
||||
"settings_tab": "collector_credentials",
|
||||
**credential_context,
|
||||
}
|
||||
supported_credential_providers = {"barentswatch", "spacetrack"}
|
||||
if (
|
||||
credential_context["requires_credentials"]
|
||||
and credential_context["credential_provider"] not in supported_credential_providers
|
||||
):
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "credentials",
|
||||
"message": "该采集器的凭证链路尚未接入，暂时无法完成连接校验。",
|
||||
"settings_tab": "collector_credentials",
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
request_headers = {str(key): str(value) for key, value in (headers or {}).items()}
|
||||
request_config = strip_connectivity_validation(config)
|
||||
timeout = float(request_config.get("timeout") or 30)
|
||||
request_endpoint = endpoint
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
if credential_context["credential_provider"] == "barentswatch":
|
||||
barentswatch_config = await resolve_barentswatch_config(db)
|
||||
token = await fetch_barentswatch_access_token(client, barentswatch_config)
|
||||
if not token:
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "token",
|
||||
"message": "凭证可读取，但 token 响应中没有 access_token。",
|
||||
"settings_tab": "collector_credentials",
|
||||
**credential_context,
|
||||
}
|
||||
request_headers["Authorization"] = f"Bearer {token}"
|
||||
elif credential_context["credential_provider"] == "spacetrack":
|
||||
username, password, _source = _resolve_spacetrack_credentials()
|
||||
login_url = "https://www.space-track.org/ajaxauth/login"
|
||||
login_response = await client.post(
|
||||
login_url,
|
||||
data={
|
||||
"identity": username,
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
login_response.raise_for_status()
|
||||
|
||||
started = datetime.now(UTC)
|
||||
async with client.stream("GET", request_endpoint, headers=request_headers) as response:
|
||||
response.raise_for_status()
|
||||
status_code = response.status_code
|
||||
elapsed_ms = (datetime.now(UTC) - started).total_seconds() * 1000
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"checksum": checksum,
|
||||
"stage": "endpoint",
|
||||
"message": "连接验证成功。",
|
||||
"status_code": status_code,
|
||||
"response_time_ms": elapsed_ms,
|
||||
**credential_context,
|
||||
}
|
||||
except httpx.HTTPStatusError as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "endpoint",
|
||||
"message": f"连接验证失败：HTTP {exc.response.status_code}",
|
||||
"error": f"HTTP Error: {exc.response.status_code}",
|
||||
**credential_context,
|
||||
}
|
||||
except httpx.HTTPError as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "network",
|
||||
"message": f"连接验证失败：{exc.__class__.__name__}",
|
||||
"error": str(exc),
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
|
||||
def make_success_validation(checksum: str, result: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"checksum": checksum,
|
||||
"status": "success",
|
||||
"validated_at": datetime.now(UTC).isoformat(),
|
||||
"status_code": result.get("status_code"),
|
||||
"credential_source": result.get("credential_source"),
|
||||
}
|
||||
|
||||
|
||||
def is_builtin_validation_current(config: DataSourceConfig | None, checksum: str) -> bool:
|
||||
validation = get_connectivity_validation(config)
|
||||
return bool(
|
||||
validation
|
||||
and validation.get("status") == "success"
|
||||
and validation.get("checksum") == checksum
|
||||
)
|
||||
|
||||
|
||||
async def get_connectivity_store(db) -> dict[str, Any]:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == CONNECTIVITY_STORE_CATEGORY)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
return dict(record.payload or {}) if record and isinstance(record.payload, dict) else {}
|
||||
|
||||
|
||||
async def save_connectivity_success(
|
||||
db,
|
||||
source: str,
|
||||
checksum: str,
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
connected_by: str,
|
||||
) -> dict[str, Any]:
|
||||
store = await get_connectivity_store(db)
|
||||
validation = {
|
||||
**make_success_validation(checksum, result),
|
||||
"connected_by": connected_by,
|
||||
}
|
||||
store[source] = validation
|
||||
|
||||
existing = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == CONNECTIVITY_STORE_CATEGORY)
|
||||
)
|
||||
record = existing.scalar_one_or_none()
|
||||
if record is None:
|
||||
db.add(SystemSetting(category=CONNECTIVITY_STORE_CATEGORY, payload=store))
|
||||
else:
|
||||
record.payload = store
|
||||
return validation
|
||||
|
||||
|
||||
async def load_builtin_override_config(db, source: str) -> DataSourceConfig | None:
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == source)
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_builtin_effective_candidate(db, source: str) -> dict[str, Any]:
|
||||
override = await load_builtin_override_config(db, source)
|
||||
default_endpoint = get_data_sources_config().get_yaml_url(source)
|
||||
return {
|
||||
"name": source,
|
||||
"endpoint": (override.endpoint if override and override.endpoint else default_endpoint) or "",
|
||||
"auth_type": override.auth_type if override else "none",
|
||||
"headers": override.headers if override else {},
|
||||
"config": strip_connectivity_validation(override.config if override else {}),
|
||||
}
|
||||
|
||||
|
||||
async def has_collected_data(db, source: str) -> bool:
|
||||
result = await db.execute(select(func.count(CollectedData.id)).where(CollectedData.source == source))
|
||||
if (result.scalar() or 0) > 0:
|
||||
return True
|
||||
|
||||
datasource_result = await db.execute(select(DataSource).where(DataSource.source == source))
|
||||
datasource = datasource_result.scalar_one_or_none()
|
||||
return bool(datasource and datasource.last_status == "success")
|
||||
|
||||
|
||||
async def get_builtin_connection_status(
|
||||
db,
|
||||
source: str,
|
||||
endpoint: str,
|
||||
auth_type: str,
|
||||
headers: dict | None,
|
||||
config: dict | None,
|
||||
) -> dict[str, Any]:
|
||||
checksum, credential_context = await build_builtin_connectivity_checksum(
|
||||
source,
|
||||
endpoint,
|
||||
auth_type,
|
||||
headers,
|
||||
config,
|
||||
db,
|
||||
)
|
||||
store = await get_connectivity_store(db)
|
||||
validation = store.get(source)
|
||||
if isinstance(validation, dict) and validation.get("status") == "success":
|
||||
if validation.get("checksum") == checksum:
|
||||
return {
|
||||
"connected": True,
|
||||
"checksum": checksum,
|
||||
"connected_by": validation.get("connected_by") or "connection_button",
|
||||
"message": "当前配置已完成连接验证。",
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
effective = await get_builtin_effective_candidate(db, source)
|
||||
effective_checksum, _ = await build_builtin_connectivity_checksum(
|
||||
source,
|
||||
effective["endpoint"],
|
||||
effective["auth_type"],
|
||||
effective["headers"],
|
||||
effective["config"],
|
||||
db,
|
||||
)
|
||||
if checksum == effective_checksum and await has_collected_data(db, source):
|
||||
return {
|
||||
"connected": True,
|
||||
"checksum": checksum,
|
||||
"connected_by": "collection",
|
||||
"message": "当前配置已有成功采集数据，视为已连接。",
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
if isinstance(validation, dict) and validation.get("status") == "success":
|
||||
return {
|
||||
"connected": False,
|
||||
"checksum": checksum,
|
||||
"connected_by": None,
|
||||
"message": "接口地址或凭证指纹已变化，请重新点击连接验证。",
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
return {
|
||||
"connected": False,
|
||||
"checksum": checksum,
|
||||
"connected_by": None,
|
||||
"message": "当前配置尚未连接，请点击连接验证。",
|
||||
**credential_context,
|
||||
}
|
||||
@@ -14,6 +14,11 @@ from app.core.time import to_iso8601_utc
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.task import CollectionTask
|
||||
from app.services.collectors.registry import collector_registry
|
||||
from app.services.datasource_connectivity import (
|
||||
build_builtin_connectivity_checksum,
|
||||
get_builtin_effective_candidate,
|
||||
save_connectivity_success,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -179,6 +184,23 @@ async def run_collector_task(collector_name: str):
|
||||
task_result = await collector.run(db)
|
||||
datasource.last_run_at = datetime.now(UTC)
|
||||
datasource.last_status = task_result.get("status")
|
||||
if datasource.last_status == "success":
|
||||
effective_candidate = await get_builtin_effective_candidate(db, datasource.source)
|
||||
checksum, _credential_context = await build_builtin_connectivity_checksum(
|
||||
datasource.source,
|
||||
effective_candidate["endpoint"],
|
||||
effective_candidate["auth_type"],
|
||||
effective_candidate["headers"],
|
||||
effective_candidate["config"],
|
||||
db,
|
||||
)
|
||||
await save_connectivity_success(
|
||||
db,
|
||||
datasource.source,
|
||||
checksum,
|
||||
{"status_code": None},
|
||||
connected_by="collection",
|
||||
)
|
||||
await _update_next_run_at(datasource, db)
|
||||
logger.info_event(
|
||||
"Collector completed",
|
||||
|
||||
@@ -19,6 +19,10 @@ ALLOWED_ACTIONS: dict[str, dict[str, Any]] = {
|
||||
"command": ["./planet.sh", "restart", "-b"],
|
||||
"recovery_mode": "backend",
|
||||
},
|
||||
"restart-frontend": {
|
||||
"command": ["./planet.sh", "restart", "-f"],
|
||||
"recovery_mode": "frontend",
|
||||
},
|
||||
"restart-ai-provider": {
|
||||
"command": ["./planet.sh", "restart", "-a"],
|
||||
"recovery_mode": "ai-provider",
|
||||
|
||||
Reference in New Issue
Block a user