release: bump version to 0.44.0
This commit is contained in:
linkong
@@ -12,6 +12,7 @@ from pydantic import BaseModel, Field
|
||||
import httpx
|
||||
|
||||
from app.core.target_schema_registry import get_target_schema, list_target_schemas
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.db.session import get_db
|
||||
from app.models.user import User
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
@@ -29,6 +30,12 @@ from app.services.datasource_mapping import (
|
||||
redact_for_llm,
|
||||
stable_payload_hash,
|
||||
)
|
||||
from app.services.datasource_connectivity import (
|
||||
get_builtin_connection_status,
|
||||
save_connectivity_success,
|
||||
strip_connectivity_validation,
|
||||
test_builtin_connectivity,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -73,6 +80,32 @@ class DataSourceConfigResponse(BaseModel):
|
||||
from_attributes = True
|
||||
|
||||
|
||||
def _is_builtin_config_name(name: str | None) -> bool:
|
||||
return bool(name and name in DEFAULT_DATASOURCES)
|
||||
|
||||
|
||||
async def _ensure_builtin_connection_verified(
|
||||
db: AsyncSession,
|
||||
config_data: DataSourceConfigCreate,
|
||||
) -> None:
|
||||
if not _is_builtin_config_name(config_data.name):
|
||||
return
|
||||
|
||||
status_result = await get_builtin_connection_status(
|
||||
db,
|
||||
config_data.name,
|
||||
config_data.endpoint,
|
||||
config_data.auth_type,
|
||||
config_data.headers,
|
||||
config_data.config,
|
||||
)
|
||||
if not status_result.get("connected"):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=status_result.get("message") or "请先完成连接验证，再保存内置采集器配置。",
|
||||
)
|
||||
|
||||
|
||||
class CustomSampleRequest(BaseModel):
|
||||
datasource_config_id: Optional[int] = None
|
||||
config: Optional[DataSourceConfigCreate] = None
|
||||
@@ -312,6 +345,47 @@ async def list_configs(
|
||||
}
|
||||
|
||||
|
||||
@router.get("/configs/all")
|
||||
async def list_all_datasources(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List all data sources: YAML defaults + DB overrides"""
|
||||
from app.core.data_sources import COLLECTOR_URL_KEYS, get_data_sources_config
|
||||
|
||||
config = get_data_sources_config()
|
||||
|
||||
db_query = await db.execute(select(DataSourceConfig))
|
||||
db_configs = {c.name: c for c in db_query.scalars().all()}
|
||||
|
||||
result = []
|
||||
for name, yaml_key in COLLECTOR_URL_KEYS.items():
|
||||
yaml_url = config.get_yaml_url(name)
|
||||
db_config = db_configs.get(name)
|
||||
|
||||
result.append(
|
||||
{
|
||||
"name": name,
|
||||
"default_url": yaml_url,
|
||||
"endpoint": db_config.endpoint if db_config else yaml_url,
|
||||
"is_overridden": db_config is not None and db_config.endpoint != yaml_url
|
||||
if yaml_url
|
||||
else db_config is not None,
|
||||
"is_active": db_config.is_active if db_config else True,
|
||||
"source_type": db_config.source_type if db_config else "http",
|
||||
"auth_type": db_config.auth_type if db_config else "none",
|
||||
"headers": db_config.headers if db_config else {},
|
||||
"config": strip_connectivity_validation(db_config.config if db_config else {}),
|
||||
"config_id": db_config.id if db_config else None,
|
||||
"description": db_config.description
|
||||
if db_config
|
||||
else f"Data source from YAML: {yaml_key}",
|
||||
}
|
||||
)
|
||||
|
||||
return {"total": len(result), "data": result}
|
||||
|
||||
|
||||
@router.get("/configs/{config_id}")
|
||||
async def get_config(
|
||||
config_id: int,
|
||||
@@ -356,7 +430,7 @@ async def create_config(
|
||||
auth_type=config_data.auth_type,
|
||||
auth_config=config_data.auth_config,
|
||||
headers=config_data.headers,
|
||||
config=config_data.config,
|
||||
config=strip_connectivity_validation(config_data.config),
|
||||
)
|
||||
|
||||
db.add(config)
|
||||
@@ -388,6 +462,8 @@ async def update_config(
|
||||
|
||||
update_data = config_data.model_dump(exclude_unset=True)
|
||||
for field, value in update_data.items():
|
||||
if field == "config":
|
||||
value = strip_connectivity_validation(value)
|
||||
setattr(config, field, value)
|
||||
|
||||
await db.commit()
|
||||
@@ -490,41 +566,61 @@ async def test_new_config(
|
||||
}
|
||||
|
||||
|
||||
@router.get("/configs/all")
|
||||
async def list_all_datasources(
|
||||
@router.post("/configs/builtin/connection-status")
|
||||
async def get_builtin_config_connection_status(
|
||||
config_data: DataSourceConfigCreate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List all data sources: YAML defaults + DB overrides"""
|
||||
from app.core.data_sources import COLLECTOR_URL_KEYS, get_data_sources_config
|
||||
if not _is_builtin_config_name(config_data.name):
|
||||
raise HTTPException(status_code=400, detail="Only built-in datasource configs are supported.")
|
||||
|
||||
config = get_data_sources_config()
|
||||
return await get_builtin_connection_status(
|
||||
db,
|
||||
config_data.name,
|
||||
config_data.endpoint,
|
||||
config_data.auth_type,
|
||||
config_data.headers,
|
||||
config_data.config,
|
||||
)
|
||||
|
||||
db_query = await db.execute(select(DataSourceConfig))
|
||||
db_configs = {c.name: c for c in db_query.scalars().all()}
|
||||
|
||||
result = []
|
||||
for name, yaml_key in COLLECTOR_URL_KEYS.items():
|
||||
yaml_url = config.get_yaml_url(name)
|
||||
db_config = db_configs.get(name)
|
||||
@router.post("/configs/builtin/connect")
|
||||
async def connect_builtin_config(
|
||||
config_data: DataSourceConfigCreate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
if not _is_builtin_config_name(config_data.name):
|
||||
raise HTTPException(status_code=400, detail="Only built-in datasource configs are supported.")
|
||||
|
||||
result.append(
|
||||
{
|
||||
"name": name,
|
||||
"default_url": yaml_url,
|
||||
"endpoint": db_config.endpoint if db_config else yaml_url,
|
||||
"is_overridden": db_config is not None and db_config.endpoint != yaml_url
|
||||
if yaml_url
|
||||
else db_config is not None,
|
||||
"is_active": db_config.is_active if db_config else True,
|
||||
"source_type": db_config.source_type if db_config else "http",
|
||||
"description": db_config.description
|
||||
if db_config
|
||||
else f"Data source from YAML: {yaml_key}",
|
||||
}
|
||||
result = await test_builtin_connectivity(
|
||||
config_data.name,
|
||||
config_data.endpoint,
|
||||
config_data.auth_type,
|
||||
config_data.headers,
|
||||
config_data.config,
|
||||
db,
|
||||
)
|
||||
if result.get("success") and result.get("checksum"):
|
||||
validation = await save_connectivity_success(
|
||||
db,
|
||||
config_data.name,
|
||||
result["checksum"],
|
||||
result,
|
||||
connected_by="connection_button",
|
||||
)
|
||||
await db.commit()
|
||||
return {
|
||||
**result,
|
||||
"connected": True,
|
||||
"validation": validation,
|
||||
}
|
||||
|
||||
return {"total": len(result), "data": result}
|
||||
return {
|
||||
**result,
|
||||
"connected": False,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/custom/sample")
|
||||
|
||||
@@ -17,6 +17,23 @@ from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.models.user import User
|
||||
from app.services.barentswatch import (
|
||||
BarentsWatchConfig,
|
||||
check_barentswatch_config,
|
||||
check_barentswatch_connectivity,
|
||||
get_barentswatch_datasource_record,
|
||||
resolve_barentswatch_config,
|
||||
)
|
||||
from app.services.credential_guides import (
|
||||
generate_credential_guide,
|
||||
get_credential_guide,
|
||||
reset_credential_guide,
|
||||
)
|
||||
from app.services.datasource_connectivity import (
|
||||
build_builtin_connectivity_checksum,
|
||||
save_connectivity_success,
|
||||
)
|
||||
from app.services.ai_client import AIProviderClient, get_ai_provider_client
|
||||
from app.services.llm_provider_catalog import (
|
||||
get_fallback_llm_provider_preset,
|
||||
list_fallback_llm_provider_presets,
|
||||
@@ -245,12 +262,7 @@ async def get_runtime_ai_provider_config(db: AsyncSession) -> dict:
|
||||
|
||||
|
||||
async def get_barentswatch_config_record(db: AsyncSession) -> Optional[DataSourceConfig]:
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == "barentswatch_vessels")
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
return await get_barentswatch_datasource_record(db)
|
||||
|
||||
|
||||
async def serialize_external_integrations(db: AsyncSession) -> dict:
|
||||
@@ -258,9 +270,9 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
|
||||
runtime_setting = await get_setting_record(db, "external_integrations")
|
||||
display_llm_config = ai_config["llm_config"] or DEFAULT_SETTINGS["external_integrations"]["ai_provider"]
|
||||
barentswatch_record = await get_barentswatch_config_record(db)
|
||||
yaml_config = get_data_sources_config()
|
||||
barentswatch_auth = barentswatch_record.auth_config if barentswatch_record else {}
|
||||
barentswatch_auth = barentswatch_auth or {}
|
||||
resolved_barentswatch = await resolve_barentswatch_config(db)
|
||||
return {
|
||||
"ai_provider": {
|
||||
"service_url": ai_config["service_url"],
|
||||
@@ -277,14 +289,12 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
|
||||
"source": "runtime" if runtime_setting else "env",
|
||||
},
|
||||
"barentswatch": {
|
||||
"endpoint": (
|
||||
barentswatch_record.endpoint
|
||||
if barentswatch_record and barentswatch_record.endpoint
|
||||
else yaml_config.get_yaml_url("barentswatch_vessels")
|
||||
"endpoint": resolved_barentswatch.endpoint,
|
||||
"client_id": barentswatch_auth.get("client_id") or resolved_barentswatch.client_id,
|
||||
"client_secret": _mask_secret(
|
||||
barentswatch_auth.get("client_secret") or resolved_barentswatch.client_secret
|
||||
),
|
||||
"client_id": barentswatch_auth.get("client_id") or "",
|
||||
"client_secret": _mask_secret(barentswatch_auth.get("client_secret")),
|
||||
"source": "datasource_config" if barentswatch_record else "default",
|
||||
"source": resolved_barentswatch.credential_source,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -328,7 +338,7 @@ async def save_external_integrations_payload(
|
||||
description="BarentsWatch Live AIS credentials",
|
||||
source_type="api",
|
||||
endpoint=update.barentswatch.endpoint.strip() or default_endpoint,
|
||||
auth_type="oauth_client_credentials",
|
||||
auth_type="oauth_client",
|
||||
auth_config={},
|
||||
headers={},
|
||||
config={},
|
||||
@@ -343,7 +353,7 @@ async def save_external_integrations_payload(
|
||||
current_auth["client_secret"] = update.barentswatch.client_secret
|
||||
current_auth["client_id"] = update.barentswatch.client_id.strip()
|
||||
barentswatch_record.endpoint = update.barentswatch.endpoint.strip() or default_endpoint
|
||||
barentswatch_record.auth_type = "oauth_client_credentials"
|
||||
barentswatch_record.auth_type = "oauth_client"
|
||||
barentswatch_record.auth_config = current_auth
|
||||
await db.commit()
|
||||
|
||||
@@ -461,6 +471,95 @@ async def get_external_integrations(
|
||||
return {"integrations": await serialize_external_integrations(db)}
|
||||
|
||||
|
||||
@router.get("/integrations/barentswatch/connectivity")
|
||||
async def get_barentswatch_connectivity(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await check_barentswatch_connectivity(db)
|
||||
|
||||
|
||||
@router.post("/integrations/barentswatch/connect")
|
||||
async def connect_barentswatch_integration(
|
||||
payload: BarentsWatchIntegrationUpdate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
current = await resolve_barentswatch_config(db)
|
||||
config = BarentsWatchConfig(
|
||||
endpoint=payload.endpoint.strip() or current.endpoint,
|
||||
client_id=payload.client_id.strip() or current.client_id,
|
||||
client_secret=(
|
||||
""
|
||||
if payload.clear_client_secret
|
||||
else payload.client_secret or current.client_secret
|
||||
),
|
||||
credential_source="draft",
|
||||
endpoint_source="draft",
|
||||
)
|
||||
result = await check_barentswatch_config(config)
|
||||
if result.get("success"):
|
||||
checksum, _context = await build_builtin_connectivity_checksum(
|
||||
"barentswatch_vessels",
|
||||
config.endpoint,
|
||||
"none",
|
||||
{},
|
||||
{},
|
||||
db,
|
||||
credential_override={
|
||||
"client_id": config.client_id,
|
||||
"client_secret": config.client_secret,
|
||||
},
|
||||
)
|
||||
validation = await save_connectivity_success(
|
||||
db,
|
||||
"barentswatch_vessels",
|
||||
checksum,
|
||||
result,
|
||||
connected_by="connection_button",
|
||||
)
|
||||
await db.commit()
|
||||
return {**result, "connected": True, "validation": validation}
|
||||
return {**result, "connected": False}
|
||||
|
||||
|
||||
@router.get("/credential-guides/{provider}")
|
||||
async def read_credential_guide(
|
||||
provider: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
try:
|
||||
return {"guide": await get_credential_guide(db, provider)}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/credential-guides/{provider}/generate")
|
||||
async def generate_provider_credential_guide(
|
||||
provider: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
ai_client: AIProviderClient = Depends(get_ai_provider_client),
|
||||
):
|
||||
try:
|
||||
return {"guide": await generate_credential_guide(db, provider, ai_client)}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/credential-guides/{provider}/reset")
|
||||
async def reset_provider_credential_guide(
|
||||
provider: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
try:
|
||||
return {"guide": await reset_credential_guide(db, provider)}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.get("/integrations/ai-provider/presets")
|
||||
async def get_ai_provider_presets(
|
||||
current_user: User = Depends(get_current_user),
|
||||
|
||||
209
backend/app/services/barentswatch.py
Normal file
209
backend/app/services/barentswatch.py
Normal file
@@ -0,0 +1,209 @@
|
||||
"""BarentsWatch AIS credential resolution and connectivity checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shlex
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
|
||||
|
||||
BARENTSWATCH_LATEST_URL = "https://live.ais.barentswatch.no/v1/latest/combined"
|
||||
BARENTSWATCH_TOKEN_URL = "https://id.barentswatch.no/connect/token"
|
||||
BARENTSWATCH_DATASOURCE_NAME = "barentswatch_vessels"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BarentsWatchConfig:
|
||||
endpoint: str
|
||||
client_id: str
|
||||
client_secret: str
|
||||
credential_source: str
|
||||
endpoint_source: str
|
||||
|
||||
|
||||
def _read_zshrc_env(path: Path | None = None) -> dict[str, str]:
|
||||
zshrc_path = path or Path.home() / ".zshrc"
|
||||
if not zshrc_path.exists():
|
||||
return {}
|
||||
|
||||
values: dict[str, str] = {}
|
||||
for raw_line in zshrc_path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if line.startswith("export "):
|
||||
line = line[len("export ") :].strip()
|
||||
if "=" not in line:
|
||||
continue
|
||||
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
if not key or not key.replace("_", "").isalnum() or not key[0].isalpha():
|
||||
continue
|
||||
|
||||
try:
|
||||
parsed = shlex.split(value, comments=True, posix=True)
|
||||
except ValueError:
|
||||
parsed = [value.strip().strip("'\"")]
|
||||
if parsed:
|
||||
values[key] = parsed[0]
|
||||
return values
|
||||
|
||||
|
||||
def _first_env_value(zshrc_env: dict[str, str], *keys: str) -> tuple[str, str]:
|
||||
for key in keys:
|
||||
value = os.getenv(key)
|
||||
if value:
|
||||
return value, "environment"
|
||||
for key in keys:
|
||||
value = zshrc_env.get(key)
|
||||
if value:
|
||||
return value, "~/.zshrc"
|
||||
return "", ""
|
||||
|
||||
|
||||
async def get_barentswatch_datasource_record(db: AsyncSession) -> DataSourceConfig | None:
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == BARENTSWATCH_DATASOURCE_NAME)
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def resolve_barentswatch_config(db: AsyncSession | None = None) -> BarentsWatchConfig:
|
||||
record = await get_barentswatch_datasource_record(db) if db else None
|
||||
auth_config = dict(record.auth_config or {}) if record else {}
|
||||
config = dict(record.config or {}) if record else {}
|
||||
zshrc_env = _read_zshrc_env()
|
||||
|
||||
env_client_id, env_source = _first_env_value(
|
||||
zshrc_env,
|
||||
"BARENTSWATCH_CLIENT_ID",
|
||||
"BARRENTSWATCH_CLIENT_ID",
|
||||
)
|
||||
env_client_secret, secret_env_source = _first_env_value(
|
||||
zshrc_env,
|
||||
"BARENTSWATCH_CLIENT_SECRET",
|
||||
"BARRENTSWATCH_CLIENT_SECRET",
|
||||
)
|
||||
client_id = auth_config.get("client_id") or config.get("client_id") or env_client_id
|
||||
client_secret = (
|
||||
auth_config.get("client_secret") or config.get("client_secret") or env_client_secret
|
||||
)
|
||||
|
||||
credential_source = ""
|
||||
if auth_config.get("client_id") or auth_config.get("client_secret"):
|
||||
credential_source = "datasource_config"
|
||||
elif config.get("client_id") or config.get("client_secret"):
|
||||
credential_source = "datasource_runtime_config"
|
||||
elif env_source or secret_env_source:
|
||||
credential_source = env_source or secret_env_source
|
||||
|
||||
yaml_endpoint = get_data_sources_config().get_yaml_url(BARENTSWATCH_DATASOURCE_NAME)
|
||||
endpoint = record.endpoint if record and record.endpoint else yaml_endpoint
|
||||
return BarentsWatchConfig(
|
||||
endpoint=endpoint or BARENTSWATCH_LATEST_URL,
|
||||
client_id=str(client_id or ""),
|
||||
client_secret=str(client_secret or ""),
|
||||
credential_source=credential_source or "missing",
|
||||
endpoint_source="datasource_config" if record and record.endpoint else "default",
|
||||
)
|
||||
|
||||
|
||||
async def fetch_barentswatch_access_token(
|
||||
client: httpx.AsyncClient,
|
||||
config: BarentsWatchConfig,
|
||||
) -> str | None:
|
||||
if not config.client_id or not config.client_secret:
|
||||
return None
|
||||
|
||||
response = await client.post(
|
||||
BARENTSWATCH_TOKEN_URL,
|
||||
data={
|
||||
"client_id": config.client_id,
|
||||
"client_secret": config.client_secret,
|
||||
"scope": "ais",
|
||||
"grant_type": "client_credentials",
|
||||
},
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
token = payload.get("access_token")
|
||||
return str(token) if token else None
|
||||
|
||||
|
||||
async def check_barentswatch_connectivity(db: AsyncSession) -> dict[str, Any]:
|
||||
config = await resolve_barentswatch_config(db)
|
||||
return await check_barentswatch_config(config)
|
||||
|
||||
|
||||
async def check_barentswatch_config(config: BarentsWatchConfig) -> dict[str, Any]:
|
||||
if not config.client_id or not config.client_secret:
|
||||
return {
|
||||
"success": False,
|
||||
"stage": "credentials",
|
||||
"message": "未找到 BarentsWatch client id/client secret，请先配置采集器凭证。",
|
||||
"endpoint": config.endpoint,
|
||||
"credential_source": config.credential_source,
|
||||
"settings_tab": "collector_credentials",
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=20.0) as client:
|
||||
token = await fetch_barentswatch_access_token(client, config)
|
||||
if not token:
|
||||
return {
|
||||
"success": False,
|
||||
"stage": "token",
|
||||
"message": "BarentsWatch token 响应中没有 access_token，请检查凭证。",
|
||||
"endpoint": config.endpoint,
|
||||
"credential_source": config.credential_source,
|
||||
"settings_tab": "collector_credentials",
|
||||
}
|
||||
|
||||
async with client.stream(
|
||||
"GET",
|
||||
config.endpoint,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"stage": "endpoint",
|
||||
"message": "BarentsWatch AIS token 和数据接口均可连通。",
|
||||
"endpoint": config.endpoint,
|
||||
"credential_source": config.credential_source,
|
||||
"endpoint_source": config.endpoint_source,
|
||||
}
|
||||
except httpx.HTTPStatusError as exc:
|
||||
status_code = exc.response.status_code
|
||||
stage = "token" if str(exc.request.url) == BARENTSWATCH_TOKEN_URL else "endpoint"
|
||||
return {
|
||||
"success": False,
|
||||
"stage": stage,
|
||||
"message": f"BarentsWatch {stage} 请求返回 HTTP {status_code}，请检查凭证或接口地址。",
|
||||
"endpoint": config.endpoint,
|
||||
"credential_source": config.credential_source,
|
||||
"settings_tab": "collector_credentials",
|
||||
}
|
||||
except httpx.HTTPError as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"stage": "network",
|
||||
"message": f"BarentsWatch 链路检查失败：{exc.__class__.__name__}",
|
||||
"endpoint": config.endpoint,
|
||||
"credential_source": config.credential_source,
|
||||
"settings_tab": "collector_credentials",
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
"""BarentsWatch AIS collector for vessel tracking."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -9,13 +8,14 @@ from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.vessel import VesselPosition, VesselStatic
|
||||
from app.services.barentswatch import (
|
||||
BARENTSWATCH_LATEST_URL,
|
||||
fetch_barentswatch_access_token,
|
||||
resolve_barentswatch_config,
|
||||
)
|
||||
from app.services.collectors.base import BaseCollector
|
||||
|
||||
|
||||
BARENTSWATCH_LATEST_URL = "https://live.ais.barentswatch.no/v1/latest/combined"
|
||||
BARENTSWATCH_TOKEN_URL = "https://id.barentswatch.no/connect/token"
|
||||
|
||||
|
||||
VESSEL_TYPE_NAMES = {
|
||||
30: "Fishing",
|
||||
35: "Military",
|
||||
@@ -38,62 +38,9 @@ class VesselAISCollector(BaseCollector):
|
||||
def base_url(self) -> str:
|
||||
return self._resolved_url or BARENTSWATCH_LATEST_URL
|
||||
|
||||
async def _load_datasource_config(self) -> dict[str, Any]:
|
||||
if not self._db_session:
|
||||
return {}
|
||||
try:
|
||||
from sqlalchemy import select
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
|
||||
result = await self._db_session.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == self.name)
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
datasource_config = result.scalar_one_or_none()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
if not datasource_config:
|
||||
return {}
|
||||
return {
|
||||
"auth_config": datasource_config.auth_config or {},
|
||||
"config": datasource_config.config or {},
|
||||
}
|
||||
|
||||
async def _get_access_token(self, client: httpx.AsyncClient) -> str | None:
|
||||
datasource_config = await self._load_datasource_config()
|
||||
auth_config = datasource_config.get("auth_config") or {}
|
||||
config = datasource_config.get("config") or {}
|
||||
client_id = (
|
||||
auth_config.get("client_id")
|
||||
or config.get("client_id")
|
||||
or os.getenv("BARENTSWATCH_CLIENT_ID")
|
||||
or os.getenv("BARRENTSWATCH_CLIENT_ID")
|
||||
)
|
||||
client_secret = (
|
||||
auth_config.get("client_secret")
|
||||
or config.get("client_secret")
|
||||
or os.getenv("BARENTSWATCH_CLIENT_SECRET")
|
||||
or os.getenv("BARRENTSWATCH_CLIENT_SECRET")
|
||||
)
|
||||
if not client_id or not client_secret:
|
||||
return None
|
||||
|
||||
response = await client.post(
|
||||
BARENTSWATCH_TOKEN_URL,
|
||||
data={
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"scope": "ais",
|
||||
"grant_type": "client_credentials",
|
||||
},
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
token = payload.get("access_token")
|
||||
return str(token) if token else None
|
||||
config = await resolve_barentswatch_config(self._db_session)
|
||||
return await fetch_barentswatch_access_token(client, config)
|
||||
|
||||
async def fetch(self) -> list[dict[str, Any]]:
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
|
||||
165
backend/app/services/credential_guides.py
Normal file
165
backend/app/services/credential_guides.py
Normal file
@@ -0,0 +1,165 @@
|
||||
"""Credential setup guides for collector integrations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient
|
||||
|
||||
|
||||
CREDENTIAL_GUIDES_CATEGORY = "collector_credential_guides"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CredentialGuideDefault:
|
||||
provider: str
|
||||
title: str
|
||||
prompt: str
|
||||
markdown: str
|
||||
|
||||
|
||||
BARENTSWATCH_DEFAULT_GUIDE = CredentialGuideDefault(
|
||||
provider="barentswatch",
|
||||
title="BarentsWatch AIS 凭证获取教程",
|
||||
prompt=(
|
||||
"请生成一份中文教程，指导开发者获取 BarentsWatch Live AIS API 的 "
|
||||
"OAuth client credentials。教程要面向已经有本地开发环境的人，包含注册/登录、"
|
||||
"创建 client、申请或确认 ais scope、复制 client id 和 client secret、"
|
||||
"在系统设置中填写并验证连接、常见失败排查。不要编造具体页面按钮文案，"
|
||||
"必须参考官方 tutorial：https://developer.barentswatch.no/docs/tutorial 。"
|
||||
"必须强调 Live AIS 要选择 AIS-client / AIS - API，而不是普通 API-client。"
|
||||
"如果步骤可能变化，要提醒以 BarentsWatch developer portal 当前页面为准。"
|
||||
),
|
||||
markdown="""## BarentsWatch AIS 凭证获取
|
||||
|
||||
官方教程：https://developer.barentswatch.no/docs/tutorial
|
||||
|
||||
1. 先打开上面的 BarentsWatch 官方 tutorial，按官方流程登录或注册开发者账号。
|
||||
2. 在 Developer access 页面选择 `AIS - API`，不要选择普通的 `BarentsWatch - API`。
|
||||
3. 在 `AIS - API` 下创建用于 Planet 的 AIS client。
|
||||
4. 创建时记下你设置的 password / client secret。
|
||||
5. 回到 My Page 复制完整 `Client ID`。它通常长得像 `your.email@example.com:client-name`。
|
||||
6. 回到 Planet 的 `设置 -> 采集器设置 -> BarentsWatch AIS`，填入 `Client ID` 和 `Client Secret`。
|
||||
7. 点击 `连接` 验证 token 和 AIS endpoint 是否可访问。
|
||||
8. 连接成功后保存凭证。
|
||||
|
||||
### 请求规则
|
||||
|
||||
- Token 地址：`https://id.barentswatch.no/connect/token`
|
||||
- 请求方式：`POST`
|
||||
- Content-Type：`application/x-www-form-urlencoded`
|
||||
- Body 必须包含：`grant_type=client_credentials`、`client_id`、`client_secret`、`scope=ais`
|
||||
- `client_id`、`client_secret`、`scope`、`grant_type` 都要放在 body，不要放在 header。
|
||||
- AIS 数据请求使用 header：`Authorization: Bearer <access_token>`
|
||||
|
||||
### 常见排查
|
||||
|
||||
- `未找到凭证`：确认 `Client ID` 和 `Client Secret` 已填写，或已经写入 `~/.zshrc`。
|
||||
- `HTTP 401/403`：通常是选成了普通 `BarentsWatch - API` client、client secret 错误，或 token 请求没有使用 `scope=ais`。
|
||||
- `network` 错误：检查本机是否能访问 `id.barentswatch.no` 和 `live.ais.barentswatch.no`。
|
||||
- Endpoint 建议保持默认：`https://live.ais.barentswatch.no/v1/latest/combined`。
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_CREDENTIAL_GUIDES = {
|
||||
BARENTSWATCH_DEFAULT_GUIDE.provider: BARENTSWATCH_DEFAULT_GUIDE,
|
||||
}
|
||||
|
||||
|
||||
async def _get_guide_store(db) -> tuple[SystemSetting | None, dict[str, Any]]:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == CREDENTIAL_GUIDES_CATEGORY)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
payload = dict(record.payload or {}) if record and isinstance(record.payload, dict) else {}
|
||||
return record, payload
|
||||
|
||||
|
||||
async def get_credential_guide(db, provider: str) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
|
||||
_record, store = await _get_guide_store(db)
|
||||
custom = store.get(provider) if isinstance(store.get(provider), dict) else None
|
||||
return {
|
||||
"provider": provider,
|
||||
"title": custom.get("title") if custom else default.title,
|
||||
"markdown": custom.get("markdown") if custom else default.markdown,
|
||||
"prompt": default.prompt,
|
||||
"source": "ai" if custom else "default",
|
||||
}
|
||||
|
||||
|
||||
async def save_credential_guide(db, provider: str, title: str, markdown: str) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
|
||||
record, store = await _get_guide_store(db)
|
||||
store[provider] = {
|
||||
"title": title or default.title,
|
||||
"markdown": markdown,
|
||||
}
|
||||
if record is None:
|
||||
db.add(SystemSetting(category=CREDENTIAL_GUIDES_CATEGORY, payload=store))
|
||||
else:
|
||||
record.payload = store
|
||||
await db.commit()
|
||||
return await get_credential_guide(db, provider)
|
||||
|
||||
|
||||
async def reset_credential_guide(db, provider: str) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
|
||||
record, store = await _get_guide_store(db)
|
||||
if provider in store:
|
||||
store.pop(provider, None)
|
||||
if record is not None:
|
||||
record.payload = store
|
||||
await db.commit()
|
||||
return await get_credential_guide(db, provider)
|
||||
|
||||
|
||||
async def generate_credential_guide(
|
||||
db,
|
||||
provider: str,
|
||||
ai_client: AIProviderClient,
|
||||
) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
|
||||
response = await ai_client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title=f"Generate credential guide for {provider}",
|
||||
objective=default.prompt,
|
||||
context={
|
||||
"provider": provider,
|
||||
"current_default_guide": default.markdown,
|
||||
"product_context": "Planet collector credential settings",
|
||||
},
|
||||
observations=[
|
||||
"Use concise Chinese markdown.",
|
||||
"Prefer stable concepts over brittle UI labels.",
|
||||
"Include verification and troubleshooting steps.",
|
||||
],
|
||||
constraints=[
|
||||
"Do not ask the user for secrets.",
|
||||
"Do not include fabricated screenshots.",
|
||||
"Return markdown only.",
|
||||
],
|
||||
)
|
||||
)
|
||||
markdown = response.content.strip()
|
||||
if not markdown:
|
||||
markdown = default.markdown
|
||||
return await save_credential_guide(db, provider, default.title, markdown)
|
||||
384
backend/app/services/datasource_connectivity.py
Normal file
384
backend/app/services/datasource_connectivity.py
Normal file
@@ -0,0 +1,384 @@
|
||||
"""Connectivity validation helpers for built-in datasource overrides."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.services.barentswatch import (
|
||||
_read_zshrc_env,
|
||||
fetch_barentswatch_access_token,
|
||||
resolve_barentswatch_config,
|
||||
)
|
||||
|
||||
|
||||
CONNECTIVITY_VALIDATION_KEY = "connectivity_validation"
|
||||
CONNECTIVITY_STORE_CATEGORY = "datasource_connectivity_validations"
|
||||
|
||||
|
||||
def _sha256_json(payload: Any) -> str:
|
||||
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _resolve_spacetrack_credentials() -> tuple[str, str, str]:
|
||||
zshrc_env = _read_zshrc_env()
|
||||
username = os.getenv("SPACETRACK_USERNAME") or zshrc_env.get("SPACETRACK_USERNAME") or ""
|
||||
password = os.getenv("SPACETRACK_PASSWORD") or zshrc_env.get("SPACETRACK_PASSWORD") or ""
|
||||
source = "environment" if os.getenv("SPACETRACK_USERNAME") or os.getenv("SPACETRACK_PASSWORD") else ""
|
||||
if not source and (username or password):
|
||||
source = "~/.zshrc"
|
||||
return username, password, source or "missing"
|
||||
|
||||
|
||||
def strip_connectivity_validation(config: dict | None) -> dict:
|
||||
cleaned = dict(config or {})
|
||||
cleaned.pop(CONNECTIVITY_VALIDATION_KEY, None)
|
||||
return cleaned
|
||||
|
||||
|
||||
def merge_connectivity_validation(existing_config: dict | None, next_config: dict | None) -> dict:
|
||||
merged = strip_connectivity_validation(next_config)
|
||||
validation = (existing_config or {}).get(CONNECTIVITY_VALIDATION_KEY)
|
||||
if validation:
|
||||
merged[CONNECTIVITY_VALIDATION_KEY] = validation
|
||||
return merged
|
||||
|
||||
|
||||
def get_connectivity_validation(config: DataSourceConfig | None) -> dict | None:
|
||||
validation = (config.config or {}).get(CONNECTIVITY_VALIDATION_KEY) if config else None
|
||||
return validation if isinstance(validation, dict) else None
|
||||
|
||||
|
||||
async def build_builtin_connectivity_checksum(
|
||||
source: str,
|
||||
endpoint: str,
|
||||
auth_type: str,
|
||||
headers: dict | None,
|
||||
config: dict | None,
|
||||
db=None,
|
||||
credential_override: dict[str, str] | None = None,
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
defaults = DEFAULT_DATASOURCES.get(source, {})
|
||||
credential_provider = defaults.get("credential_provider")
|
||||
credential_fingerprint = ""
|
||||
credential_source = "none"
|
||||
has_credentials = not defaults.get("requires_credentials", False)
|
||||
|
||||
if credential_provider == "barentswatch":
|
||||
if credential_override:
|
||||
client_id = credential_override.get("client_id", "")
|
||||
client_secret = credential_override.get("client_secret", "")
|
||||
credential_source = "draft"
|
||||
else:
|
||||
barentswatch_config = await resolve_barentswatch_config(db)
|
||||
client_id = barentswatch_config.client_id
|
||||
client_secret = barentswatch_config.client_secret
|
||||
credential_source = barentswatch_config.credential_source
|
||||
has_credentials = bool(client_id and client_secret)
|
||||
credential_fingerprint = _sha256_json(
|
||||
{
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
}
|
||||
)
|
||||
elif credential_provider == "spacetrack":
|
||||
username, password, credential_source = _resolve_spacetrack_credentials()
|
||||
has_credentials = bool(username and password)
|
||||
credential_fingerprint = _sha256_json(
|
||||
{
|
||||
"username": username,
|
||||
"password": password,
|
||||
}
|
||||
)
|
||||
elif defaults.get("requires_credentials"):
|
||||
credential_source = str(credential_provider or "unsupported")
|
||||
|
||||
checksum_payload = {
|
||||
"source": source,
|
||||
"endpoint": endpoint,
|
||||
"auth_type": "none",
|
||||
"headers": headers or {},
|
||||
"config": strip_connectivity_validation(config),
|
||||
"credential_provider": credential_provider or "none",
|
||||
"credential_fingerprint": credential_fingerprint,
|
||||
}
|
||||
return _sha256_json(checksum_payload), {
|
||||
"requires_credentials": bool(defaults.get("requires_credentials", False)),
|
||||
"credential_provider": credential_provider,
|
||||
"credential_source": credential_source,
|
||||
"has_credentials": has_credentials,
|
||||
}
|
||||
|
||||
|
||||
async def test_builtin_connectivity(
|
||||
source: str,
|
||||
endpoint: str,
|
||||
auth_type: str,
|
||||
headers: dict | None,
|
||||
config: dict | None,
|
||||
db=None,
|
||||
) -> dict[str, Any]:
|
||||
defaults = DEFAULT_DATASOURCES.get(source)
|
||||
if not defaults:
|
||||
return {
|
||||
"success": False,
|
||||
"message": "未知内置采集器，无法执行连接校验。",
|
||||
}
|
||||
|
||||
checksum, credential_context = await build_builtin_connectivity_checksum(
|
||||
source,
|
||||
endpoint,
|
||||
auth_type,
|
||||
headers,
|
||||
config,
|
||||
db,
|
||||
)
|
||||
if credential_context["requires_credentials"] and not credential_context["has_credentials"]:
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "credentials",
|
||||
"message": "该采集器需要凭证，请先到采集器凭证设置中配置。",
|
||||
"settings_tab": "collector_credentials",
|
||||
**credential_context,
|
||||
}
|
||||
supported_credential_providers = {"barentswatch", "spacetrack"}
|
||||
if (
|
||||
credential_context["requires_credentials"]
|
||||
and credential_context["credential_provider"] not in supported_credential_providers
|
||||
):
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "credentials",
|
||||
"message": "该采集器的凭证链路尚未接入，暂时无法完成连接校验。",
|
||||
"settings_tab": "collector_credentials",
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
request_headers = {str(key): str(value) for key, value in (headers or {}).items()}
|
||||
request_config = strip_connectivity_validation(config)
|
||||
timeout = float(request_config.get("timeout") or 30)
|
||||
request_endpoint = endpoint
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
if credential_context["credential_provider"] == "barentswatch":
|
||||
barentswatch_config = await resolve_barentswatch_config(db)
|
||||
token = await fetch_barentswatch_access_token(client, barentswatch_config)
|
||||
if not token:
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "token",
|
||||
"message": "凭证可读取，但 token 响应中没有 access_token。",
|
||||
"settings_tab": "collector_credentials",
|
||||
**credential_context,
|
||||
}
|
||||
request_headers["Authorization"] = f"Bearer {token}"
|
||||
elif credential_context["credential_provider"] == "spacetrack":
|
||||
username, password, _source = _resolve_spacetrack_credentials()
|
||||
login_url = "https://www.space-track.org/ajaxauth/login"
|
||||
login_response = await client.post(
|
||||
login_url,
|
||||
data={
|
||||
"identity": username,
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
login_response.raise_for_status()
|
||||
|
||||
started = datetime.now(UTC)
|
||||
async with client.stream("GET", request_endpoint, headers=request_headers) as response:
|
||||
response.raise_for_status()
|
||||
status_code = response.status_code
|
||||
elapsed_ms = (datetime.now(UTC) - started).total_seconds() * 1000
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"checksum": checksum,
|
||||
"stage": "endpoint",
|
||||
"message": "连接验证成功。",
|
||||
"status_code": status_code,
|
||||
"response_time_ms": elapsed_ms,
|
||||
**credential_context,
|
||||
}
|
||||
except httpx.HTTPStatusError as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "endpoint",
|
||||
"message": f"连接验证失败：HTTP {exc.response.status_code}",
|
||||
"error": f"HTTP Error: {exc.response.status_code}",
|
||||
**credential_context,
|
||||
}
|
||||
except httpx.HTTPError as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "network",
|
||||
"message": f"连接验证失败：{exc.__class__.__name__}",
|
||||
"error": str(exc),
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
|
||||
def make_success_validation(checksum: str, result: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"checksum": checksum,
|
||||
"status": "success",
|
||||
"validated_at": datetime.now(UTC).isoformat(),
|
||||
"status_code": result.get("status_code"),
|
||||
"credential_source": result.get("credential_source"),
|
||||
}
|
||||
|
||||
|
||||
def is_builtin_validation_current(config: DataSourceConfig | None, checksum: str) -> bool:
|
||||
validation = get_connectivity_validation(config)
|
||||
return bool(
|
||||
validation
|
||||
and validation.get("status") == "success"
|
||||
and validation.get("checksum") == checksum
|
||||
)
|
||||
|
||||
|
||||
async def get_connectivity_store(db) -> dict[str, Any]:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == CONNECTIVITY_STORE_CATEGORY)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
return dict(record.payload or {}) if record and isinstance(record.payload, dict) else {}
|
||||
|
||||
|
||||
async def save_connectivity_success(
|
||||
db,
|
||||
source: str,
|
||||
checksum: str,
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
connected_by: str,
|
||||
) -> dict[str, Any]:
|
||||
store = await get_connectivity_store(db)
|
||||
validation = {
|
||||
**make_success_validation(checksum, result),
|
||||
"connected_by": connected_by,
|
||||
}
|
||||
store[source] = validation
|
||||
|
||||
existing = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == CONNECTIVITY_STORE_CATEGORY)
|
||||
)
|
||||
record = existing.scalar_one_or_none()
|
||||
if record is None:
|
||||
db.add(SystemSetting(category=CONNECTIVITY_STORE_CATEGORY, payload=store))
|
||||
else:
|
||||
record.payload = store
|
||||
return validation
|
||||
|
||||
|
||||
async def load_builtin_override_config(db, source: str) -> DataSourceConfig | None:
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == source)
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_builtin_effective_candidate(db, source: str) -> dict[str, Any]:
|
||||
override = await load_builtin_override_config(db, source)
|
||||
default_endpoint = get_data_sources_config().get_yaml_url(source)
|
||||
return {
|
||||
"name": source,
|
||||
"endpoint": (override.endpoint if override and override.endpoint else default_endpoint) or "",
|
||||
"auth_type": override.auth_type if override else "none",
|
||||
"headers": override.headers if override else {},
|
||||
"config": strip_connectivity_validation(override.config if override else {}),
|
||||
}
|
||||
|
||||
|
||||
async def has_collected_data(db, source: str) -> bool:
|
||||
result = await db.execute(select(func.count(CollectedData.id)).where(CollectedData.source == source))
|
||||
if (result.scalar() or 0) > 0:
|
||||
return True
|
||||
|
||||
datasource_result = await db.execute(select(DataSource).where(DataSource.source == source))
|
||||
datasource = datasource_result.scalar_one_or_none()
|
||||
return bool(datasource and datasource.last_status == "success")
|
||||
|
||||
|
||||
async def get_builtin_connection_status(
|
||||
db,
|
||||
source: str,
|
||||
endpoint: str,
|
||||
auth_type: str,
|
||||
headers: dict | None,
|
||||
config: dict | None,
|
||||
) -> dict[str, Any]:
|
||||
checksum, credential_context = await build_builtin_connectivity_checksum(
|
||||
source,
|
||||
endpoint,
|
||||
auth_type,
|
||||
headers,
|
||||
config,
|
||||
db,
|
||||
)
|
||||
store = await get_connectivity_store(db)
|
||||
validation = store.get(source)
|
||||
if isinstance(validation, dict) and validation.get("status") == "success":
|
||||
if validation.get("checksum") == checksum:
|
||||
return {
|
||||
"connected": True,
|
||||
"checksum": checksum,
|
||||
"connected_by": validation.get("connected_by") or "connection_button",
|
||||
"message": "当前配置已完成连接验证。",
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
effective = await get_builtin_effective_candidate(db, source)
|
||||
effective_checksum, _ = await build_builtin_connectivity_checksum(
|
||||
source,
|
||||
effective["endpoint"],
|
||||
effective["auth_type"],
|
||||
effective["headers"],
|
||||
effective["config"],
|
||||
db,
|
||||
)
|
||||
if checksum == effective_checksum and await has_collected_data(db, source):
|
||||
return {
|
||||
"connected": True,
|
||||
"checksum": checksum,
|
||||
"connected_by": "collection",
|
||||
"message": "当前配置已有成功采集数据，视为已连接。",
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
if isinstance(validation, dict) and validation.get("status") == "success":
|
||||
return {
|
||||
"connected": False,
|
||||
"checksum": checksum,
|
||||
"connected_by": None,
|
||||
"message": "接口地址或凭证指纹已变化，请重新点击连接验证。",
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
return {
|
||||
"connected": False,
|
||||
"checksum": checksum,
|
||||
"connected_by": None,
|
||||
"message": "当前配置尚未连接，请点击连接验证。",
|
||||
**credential_context,
|
||||
}
|
||||
@@ -14,6 +14,11 @@ from app.core.time import to_iso8601_utc
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.task import CollectionTask
|
||||
from app.services.collectors.registry import collector_registry
|
||||
from app.services.datasource_connectivity import (
|
||||
build_builtin_connectivity_checksum,
|
||||
get_builtin_effective_candidate,
|
||||
save_connectivity_success,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -179,6 +184,23 @@ async def run_collector_task(collector_name: str):
|
||||
task_result = await collector.run(db)
|
||||
datasource.last_run_at = datetime.now(UTC)
|
||||
datasource.last_status = task_result.get("status")
|
||||
if datasource.last_status == "success":
|
||||
effective_candidate = await get_builtin_effective_candidate(db, datasource.source)
|
||||
checksum, _credential_context = await build_builtin_connectivity_checksum(
|
||||
datasource.source,
|
||||
effective_candidate["endpoint"],
|
||||
effective_candidate["auth_type"],
|
||||
effective_candidate["headers"],
|
||||
effective_candidate["config"],
|
||||
db,
|
||||
)
|
||||
await save_connectivity_success(
|
||||
db,
|
||||
datasource.source,
|
||||
checksum,
|
||||
{"status_code": None},
|
||||
connected_by="collection",
|
||||
)
|
||||
await _update_next_run_at(datasource, db)
|
||||
logger.info_event(
|
||||
"Collector completed",
|
||||
|
||||
@@ -19,6 +19,10 @@ ALLOWED_ACTIONS: dict[str, dict[str, Any]] = {
|
||||
"command": ["./planet.sh", "restart", "-b"],
|
||||
"recovery_mode": "backend",
|
||||
},
|
||||
"restart-frontend": {
|
||||
"command": ["./planet.sh", "restart", "-f"],
|
||||
"recovery_mode": "frontend",
|
||||
},
|
||||
"restart-ai-provider": {
|
||||
"command": ["./planet.sh", "restart", "-a"],
|
||||
"recovery_mode": "ai-provider",
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
@@ -59,6 +60,8 @@ def wait_for_recovery(action: str) -> tuple[bool, str]:
|
||||
recovery_mode = get_action_recovery_mode(action)
|
||||
if recovery_mode == "backend":
|
||||
return wait_for_http("http://localhost:8000/health"), "backend health recovery"
|
||||
if recovery_mode == "frontend":
|
||||
return wait_for_http("http://localhost:3000"), "frontend entrypoint recovery"
|
||||
if recovery_mode == "ai-provider":
|
||||
return wait_for_http("http://localhost:8010/health"), "ai provider health recovery"
|
||||
if recovery_mode == "database":
|
||||
@@ -108,8 +111,9 @@ def main() -> int:
|
||||
)
|
||||
append_task_log(args.task_id, "restart command started")
|
||||
|
||||
shell_command = shlex.join(command)
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
["zsh", "-ic", shell_command],
|
||||
cwd=str(ROOT_DIR),
|
||||
env=env,
|
||||
capture_output=True,
|
||||
|
||||
@@ -7,6 +7,7 @@ from app.api.v1.visualization import convert_vessels_to_geojson
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models.vessel import VesselPosition, VesselStatic
|
||||
from app.services import barentswatch
|
||||
from app.services.collectors.vessel_ais import VesselAISCollector
|
||||
|
||||
|
||||
@@ -34,6 +35,49 @@ def test_vessel_collector_transforms_barentswatch_like_records():
|
||||
assert records[0]["lat"] == pytest.approx(59.91)
|
||||
|
||||
|
||||
def test_barentswatch_reads_credentials_from_zshrc(tmp_path):
|
||||
zshrc = tmp_path / ".zshrc"
|
||||
zshrc.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"export BARENTSWATCH_CLIENT_ID='client-from-zshrc'",
|
||||
'export BARENTSWATCH_CLIENT_SECRET="secret-from-zshrc" # local dev credential',
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
values = barentswatch._read_zshrc_env(zshrc)
|
||||
|
||||
assert values["BARENTSWATCH_CLIENT_ID"] == "client-from-zshrc"
|
||||
assert values["BARENTSWATCH_CLIENT_SECRET"] == "secret-from-zshrc"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_barentswatch_resolves_config_from_zshrc(tmp_path, monkeypatch):
|
||||
zshrc = tmp_path / ".zshrc"
|
||||
zshrc.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"export BARENTSWATCH_CLIENT_ID=client-from-zshrc",
|
||||
"export BARENTSWATCH_CLIENT_SECRET=secret-from-zshrc",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.delenv("BARENTSWATCH_CLIENT_ID", raising=False)
|
||||
monkeypatch.delenv("BARENTSWATCH_CLIENT_SECRET", raising=False)
|
||||
monkeypatch.delenv("BARRENTSWATCH_CLIENT_ID", raising=False)
|
||||
monkeypatch.delenv("BARRENTSWATCH_CLIENT_SECRET", raising=False)
|
||||
monkeypatch.setattr(barentswatch.Path, "home", lambda: tmp_path)
|
||||
|
||||
config = await barentswatch.resolve_barentswatch_config(None)
|
||||
|
||||
assert config.client_id == "client-from-zshrc"
|
||||
assert config.client_secret == "secret-from-zshrc"
|
||||
assert config.credential_source == "~/.zshrc"
|
||||
|
||||
|
||||
def test_convert_vessels_to_geojson():
|
||||
position = VesselPosition(
|
||||
mmsi=257123000,
|
||||
|
||||
Reference in New Issue
Block a user