release: bump version to 0.43.0
This commit is contained in:
linkong
@@ -1,20 +1,34 @@
|
||||
"""DataSourceConfig API for user-defined data sources"""
|
||||
|
||||
from typing import Optional
|
||||
from typing import Any, Optional
|
||||
from datetime import datetime
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from pydantic import BaseModel, Field
|
||||
import httpx
|
||||
|
||||
from app.core.target_schema_registry import get_target_schema, list_target_schemas
|
||||
from app.db.session import get_db
|
||||
from app.models.user import User
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.datasource_mapping import DataSourceMappingTemplate
|
||||
from app.core.security import get_current_user
|
||||
from app.core.cache import cache
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient, get_ai_provider_client
|
||||
from app.services.datasource_mapping import (
|
||||
MappingError,
|
||||
build_heuristic_mapping,
|
||||
execute_mapping,
|
||||
persist_mapped_records,
|
||||
redact_for_llm,
|
||||
stable_payload_hash,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -59,6 +73,44 @@ class DataSourceConfigResponse(BaseModel):
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class CustomSampleRequest(BaseModel):
|
||||
datasource_config_id: Optional[int] = None
|
||||
config: Optional[DataSourceConfigCreate] = None
|
||||
limit_bytes: int = Field(default=200000, ge=1000, le=1000000)
|
||||
|
||||
|
||||
class MappingProposeRequest(BaseModel):
|
||||
sample_payload: Any
|
||||
target_schema: str
|
||||
use_ai: bool = True
|
||||
|
||||
|
||||
class MappingPreviewRequest(BaseModel):
|
||||
sample_payload: Any
|
||||
target_schema: str
|
||||
mapping_json: dict
|
||||
limit: int = Field(default=20, ge=1, le=100)
|
||||
|
||||
|
||||
class MappingTemplateCreate(BaseModel):
|
||||
datasource_config_id: int
|
||||
target_schema: str
|
||||
mapping_json: dict
|
||||
sample_payload: Any | None = None
|
||||
sample_payload_hash: Optional[str] = None
|
||||
validation_status: str = Field(default="draft", pattern="^(draft|valid|invalid)$")
|
||||
is_active: bool = False
|
||||
|
||||
|
||||
class MappingTemplateUpdate(BaseModel):
|
||||
target_schema: Optional[str] = None
|
||||
mapping_json: Optional[dict] = None
|
||||
sample_payload: Any | None = None
|
||||
sample_payload_hash: Optional[str] = None
|
||||
validation_status: Optional[str] = Field(default=None, pattern="^(draft|valid|invalid)$")
|
||||
is_active: Optional[bool] = None
|
||||
|
||||
|
||||
async def test_endpoint(
|
||||
endpoint: str,
|
||||
auth_type: str,
|
||||
@@ -96,6 +148,134 @@ async def test_endpoint(
|
||||
}
|
||||
|
||||
|
||||
def _build_request_headers(auth_type: str, auth_config: dict, headers: dict) -> dict[str, str]:
|
||||
request_headers = {str(key): str(value) for key, value in (headers or {}).items()}
|
||||
auth_type = str(auth_type or "none").lower()
|
||||
auth_config = auth_config or {}
|
||||
|
||||
if auth_type == "bearer" and auth_config.get("token"):
|
||||
request_headers["Authorization"] = f"Bearer {auth_config['token']}"
|
||||
elif auth_type == "api_key" and auth_config.get("api_key"):
|
||||
location = str(auth_config.get("in") or auth_config.get("location") or "header").lower()
|
||||
if location != "query":
|
||||
key_name = auth_config.get("key_name", "X-API-Key")
|
||||
request_headers[str(key_name)] = str(auth_config["api_key"])
|
||||
elif auth_type == "basic":
|
||||
username = auth_config.get("username", "")
|
||||
password = auth_config.get("password", "")
|
||||
credentials = f"{username}:{password}"
|
||||
encoded = base64.b64encode(credentials.encode()).decode()
|
||||
request_headers["Authorization"] = f"Basic {encoded}"
|
||||
return request_headers
|
||||
|
||||
|
||||
def _build_query_params(auth_type: str, auth_config: dict, config: dict) -> dict[str, Any]:
|
||||
params = {}
|
||||
candidate = (config or {}).get("params") or (config or {}).get("query_params")
|
||||
if isinstance(candidate, dict):
|
||||
params.update(candidate)
|
||||
|
||||
auth_type = str(auth_type or "none").lower()
|
||||
auth_config = auth_config or {}
|
||||
if auth_type == "api_key" and auth_config.get("api_key"):
|
||||
location = str(auth_config.get("in") or auth_config.get("location") or "header").lower()
|
||||
if location == "query":
|
||||
key_name = auth_config.get("key_name") or auth_config.get("param_name") or "api_key"
|
||||
params[str(key_name)] = auth_config["api_key"]
|
||||
return params
|
||||
|
||||
|
||||
async def fetch_custom_sample_from_config(config: DataSourceConfig, limit_bytes: int) -> Any:
|
||||
request_config = config.config or {}
|
||||
method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
|
||||
if method not in {"GET", "POST"}:
|
||||
raise HTTPException(status_code=400, detail="Only GET and POST sample requests are supported.")
|
||||
|
||||
headers = _build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {})
|
||||
params = _build_query_params(config.auth_type, config.auth_config or {}, request_config)
|
||||
timeout = float(request_config.get("timeout", 30))
|
||||
json_body = request_config.get("json_body")
|
||||
if json_body is None and str(request_config.get("body_type") or "").lower() in {"json", ""}:
|
||||
candidate = request_config.get("body")
|
||||
if isinstance(candidate, (dict, list)):
|
||||
json_body = candidate
|
||||
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
response = await client.request(
|
||||
method,
|
||||
config.endpoint,
|
||||
headers=headers,
|
||||
params=params or None,
|
||||
json=json_body,
|
||||
)
|
||||
response.raise_for_status()
|
||||
content = response.content[:limit_bytes]
|
||||
if "application/json" in response.headers.get("content-type", ""):
|
||||
return json.loads(content.decode(response.encoding or "utf-8"))
|
||||
return {"text": content.decode(response.encoding or "utf-8", errors="replace")}
|
||||
|
||||
|
||||
def _parse_mapping_from_ai_text(content: str) -> dict[str, Any] | None:
|
||||
if not content:
|
||||
return None
|
||||
|
||||
candidates = [content]
|
||||
fenced = re.findall(r"```(?:json)?\s*(\{.*?\})\s*```", content, flags=re.DOTALL)
|
||||
candidates = fenced + candidates
|
||||
for candidate in candidates:
|
||||
try:
|
||||
parsed = json.loads(candidate)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(parsed, dict) and isinstance(parsed.get("fields"), dict):
|
||||
return parsed
|
||||
return None
|
||||
|
||||
|
||||
async def _get_config_for_sample(
|
||||
payload: CustomSampleRequest,
|
||||
db: AsyncSession,
|
||||
) -> DataSourceConfig:
|
||||
if payload.datasource_config_id is not None:
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig).where(DataSourceConfig.id == payload.datasource_config_id)
|
||||
)
|
||||
config = result.scalar_one_or_none()
|
||||
if not config:
|
||||
raise HTTPException(status_code=404, detail="Configuration not found")
|
||||
return config
|
||||
|
||||
if payload.config is None:
|
||||
raise HTTPException(status_code=400, detail="datasource_config_id or config is required")
|
||||
|
||||
config_data = payload.config
|
||||
return DataSourceConfig(
|
||||
name=config_data.name,
|
||||
description=config_data.description,
|
||||
source_type=config_data.source_type,
|
||||
endpoint=config_data.endpoint,
|
||||
auth_type=config_data.auth_type,
|
||||
auth_config=config_data.auth_config,
|
||||
headers=config_data.headers,
|
||||
config=config_data.config,
|
||||
)
|
||||
|
||||
|
||||
def serialize_mapping_template(template: DataSourceMappingTemplate) -> dict[str, Any]:
|
||||
return {
|
||||
"id": template.id,
|
||||
"datasource_config_id": template.datasource_config_id,
|
||||
"target_schema": template.target_schema,
|
||||
"mapping_json": template.mapping_json,
|
||||
"sample_payload_hash": template.sample_payload_hash,
|
||||
"validation_status": template.validation_status,
|
||||
"version": template.version,
|
||||
"is_active": template.is_active,
|
||||
"created_at": to_iso8601_utc(template.created_at),
|
||||
"updated_at": to_iso8601_utc(template.updated_at),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/configs")
|
||||
async def list_configs(
|
||||
active_only: bool = False,
|
||||
@@ -345,3 +525,311 @@ async def list_all_datasources(
|
||||
)
|
||||
|
||||
return {"total": len(result), "data": result}
|
||||
|
||||
|
||||
@router.post("/custom/sample")
|
||||
async def fetch_custom_sample(
|
||||
payload: CustomSampleRequest,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Fetch a sample payload for a saved or draft custom data source."""
|
||||
config = await _get_config_for_sample(payload, db)
|
||||
try:
|
||||
sample = await fetch_custom_sample_from_config(config, payload.limit_bytes)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise HTTPException(
|
||||
status_code=exc.response.status_code,
|
||||
detail=f"Sample request failed: HTTP {exc.response.status_code}",
|
||||
) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise HTTPException(status_code=502, detail=f"Sample request failed: {exc}") from exc
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"sample_payload": sample,
|
||||
"sample_payload_hash": stable_payload_hash(sample),
|
||||
"redacted_preview": redact_for_llm(sample),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/target-schemas")
|
||||
async def get_datasource_target_schemas(
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List target schemas available for custom datasource mapping."""
|
||||
return {"data": list_target_schemas()}
|
||||
|
||||
|
||||
@router.post("/mappings/propose")
|
||||
async def propose_datasource_mapping(
|
||||
payload: MappingProposeRequest,
|
||||
current_user: User = Depends(get_current_user),
|
||||
ai_client: AIProviderClient = Depends(get_ai_provider_client),
|
||||
):
|
||||
"""Generate a mapping draft for a sample payload and target schema."""
|
||||
schema = get_target_schema(payload.target_schema)
|
||||
redacted_sample = redact_for_llm(payload.sample_payload)
|
||||
fallback_mapping = build_heuristic_mapping(redacted_sample, payload.target_schema)
|
||||
|
||||
ai_error: str | None = None
|
||||
mapping = fallback_mapping
|
||||
generated_by = "heuristic"
|
||||
if payload.use_ai:
|
||||
try:
|
||||
response = await ai_client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title=f"Generate datasource mapping for {schema.key}",
|
||||
objective=(
|
||||
"Return only JSON for a deterministic mapping DSL. "
|
||||
"The JSON must contain source.items_path and fields. "
|
||||
"Do not include prose or code."
|
||||
),
|
||||
context={
|
||||
"target_schema": schema.to_dict(),
|
||||
"sample_payload": redacted_sample,
|
||||
"mapping_dsl_example": fallback_mapping,
|
||||
},
|
||||
observations=[
|
||||
"Use JSONPath-like paths beginning with $.",
|
||||
"Never generate executable code.",
|
||||
"Use field types from the target schema.",
|
||||
],
|
||||
constraints=[
|
||||
"Return a single JSON object.",
|
||||
"Do not include credentials or secrets.",
|
||||
"Mark uncertain optional fields with default null.",
|
||||
],
|
||||
)
|
||||
)
|
||||
parsed = _parse_mapping_from_ai_text(response.content)
|
||||
if parsed:
|
||||
mapping = parsed
|
||||
generated_by = "ai_provider"
|
||||
else:
|
||||
ai_error = "AI provider did not return a valid mapping JSON object."
|
||||
except HTTPException as exc:
|
||||
ai_error = str(exc.detail)
|
||||
|
||||
mapping.setdefault("meta", {})
|
||||
if isinstance(mapping["meta"], dict):
|
||||
mapping["meta"].update(
|
||||
{
|
||||
"generated_by": generated_by,
|
||||
"requires_review": True,
|
||||
"ai_error": ai_error,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"target_schema": schema.to_dict(),
|
||||
"mapping_json": mapping,
|
||||
"sample_payload_hash": stable_payload_hash(payload.sample_payload),
|
||||
"redacted_sample_payload": redacted_sample,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/mappings/preview")
|
||||
async def preview_datasource_mapping(
|
||||
payload: MappingPreviewRequest,
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""Preview deterministic mapping output for a sample payload."""
|
||||
try:
|
||||
preview = execute_mapping(
|
||||
payload.sample_payload,
|
||||
payload.mapping_json,
|
||||
payload.target_schema,
|
||||
limit=payload.limit,
|
||||
)
|
||||
except (MappingError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
return {
|
||||
"success": preview["failed_count"] == 0,
|
||||
"preview": preview,
|
||||
"sample_payload_hash": stable_payload_hash(payload.sample_payload),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/mappings")
|
||||
async def list_datasource_mappings(
|
||||
datasource_config_id: Optional[int] = None,
|
||||
active_only: bool = False,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List saved mapping templates."""
|
||||
query = select(DataSourceMappingTemplate).order_by(
|
||||
DataSourceMappingTemplate.datasource_config_id,
|
||||
DataSourceMappingTemplate.version.desc(),
|
||||
)
|
||||
if datasource_config_id is not None:
|
||||
query = query.where(DataSourceMappingTemplate.datasource_config_id == datasource_config_id)
|
||||
if active_only:
|
||||
query = query.where(DataSourceMappingTemplate.is_active.is_(True))
|
||||
|
||||
result = await db.execute(query)
|
||||
mappings = result.scalars().all()
|
||||
return {"total": len(mappings), "data": [serialize_mapping_template(item) for item in mappings]}
|
||||
|
||||
|
||||
@router.post("/mappings")
|
||||
async def create_datasource_mapping(
|
||||
payload: MappingTemplateCreate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Save a mapping template for a datasource config."""
|
||||
get_target_schema(payload.target_schema)
|
||||
datasource = await db.get(DataSourceConfig, payload.datasource_config_id)
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Configuration not found")
|
||||
|
||||
if payload.sample_payload is not None:
|
||||
try:
|
||||
execute_mapping(payload.sample_payload, payload.mapping_json, payload.target_schema, limit=100)
|
||||
except (MappingError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=f"Mapping validation failed: {exc}") from exc
|
||||
|
||||
result = await db.execute(
|
||||
select(func.max(DataSourceMappingTemplate.version)).where(
|
||||
DataSourceMappingTemplate.datasource_config_id == payload.datasource_config_id,
|
||||
DataSourceMappingTemplate.target_schema == payload.target_schema,
|
||||
)
|
||||
)
|
||||
next_version = int(result.scalar() or 0) + 1
|
||||
|
||||
if payload.is_active:
|
||||
await db.execute(
|
||||
DataSourceMappingTemplate.__table__.update()
|
||||
.where(DataSourceMappingTemplate.datasource_config_id == payload.datasource_config_id)
|
||||
.values(is_active=False)
|
||||
)
|
||||
|
||||
template = DataSourceMappingTemplate(
|
||||
datasource_config_id=payload.datasource_config_id,
|
||||
target_schema=payload.target_schema,
|
||||
mapping_json=payload.mapping_json,
|
||||
sample_payload_hash=payload.sample_payload_hash
|
||||
or (stable_payload_hash(payload.sample_payload) if payload.sample_payload is not None else None),
|
||||
validation_status=payload.validation_status,
|
||||
version=next_version,
|
||||
is_active=payload.is_active,
|
||||
)
|
||||
db.add(template)
|
||||
await db.commit()
|
||||
await db.refresh(template)
|
||||
return {"message": "Mapping template saved successfully", "data": serialize_mapping_template(template)}
|
||||
|
||||
|
||||
@router.put("/mappings/{mapping_id}")
|
||||
async def update_datasource_mapping(
|
||||
mapping_id: int,
|
||||
payload: MappingTemplateUpdate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Update a mapping template in place."""
|
||||
template = await db.get(DataSourceMappingTemplate, mapping_id)
|
||||
if not template:
|
||||
raise HTTPException(status_code=404, detail="Mapping template not found")
|
||||
|
||||
target_schema = payload.target_schema or template.target_schema
|
||||
mapping_json = payload.mapping_json or template.mapping_json
|
||||
get_target_schema(target_schema)
|
||||
if payload.sample_payload is not None:
|
||||
try:
|
||||
execute_mapping(payload.sample_payload, mapping_json, target_schema, limit=100)
|
||||
except (MappingError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=f"Mapping validation failed: {exc}") from exc
|
||||
|
||||
if payload.is_active is True:
|
||||
await db.execute(
|
||||
DataSourceMappingTemplate.__table__.update()
|
||||
.where(DataSourceMappingTemplate.datasource_config_id == template.datasource_config_id)
|
||||
.where(DataSourceMappingTemplate.id != template.id)
|
||||
.values(is_active=False)
|
||||
)
|
||||
|
||||
template.target_schema = target_schema
|
||||
template.mapping_json = mapping_json
|
||||
if payload.sample_payload_hash is not None:
|
||||
template.sample_payload_hash = payload.sample_payload_hash
|
||||
elif payload.sample_payload is not None:
|
||||
template.sample_payload_hash = stable_payload_hash(payload.sample_payload)
|
||||
if payload.validation_status is not None:
|
||||
template.validation_status = payload.validation_status
|
||||
if payload.is_active is not None:
|
||||
template.is_active = payload.is_active
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(template)
|
||||
return {"message": "Mapping template updated successfully", "data": serialize_mapping_template(template)}
|
||||
|
||||
|
||||
@router.post("/{config_id}/run-mapped")
|
||||
async def run_mapped_datasource(
|
||||
config_id: int,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Run a saved custom datasource through its active deterministic mapping."""
|
||||
datasource = await db.get(DataSourceConfig, config_id)
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Configuration not found")
|
||||
|
||||
result = await db.execute(
|
||||
select(DataSourceMappingTemplate)
|
||||
.where(DataSourceMappingTemplate.datasource_config_id == config_id)
|
||||
.where(DataSourceMappingTemplate.is_active.is_(True))
|
||||
.order_by(DataSourceMappingTemplate.version.desc())
|
||||
.limit(1)
|
||||
)
|
||||
mapping = result.scalar_one_or_none()
|
||||
if not mapping:
|
||||
raise HTTPException(status_code=404, detail="No active mapping template found")
|
||||
|
||||
try:
|
||||
sample = await fetch_custom_sample_from_config(datasource, 5_000_000)
|
||||
mapped = execute_mapping(sample, mapping.mapping_json, mapping.target_schema)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise HTTPException(
|
||||
status_code=exc.response.status_code,
|
||||
detail=f"Datasource request failed: HTTP {exc.response.status_code}",
|
||||
) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise HTTPException(status_code=502, detail=f"Datasource request failed: {exc}") from exc
|
||||
except (MappingError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=f"Mapping failed: {exc}") from exc
|
||||
|
||||
if mapped["failed_count"] > 0:
|
||||
return {
|
||||
"status": "failed",
|
||||
"datasource_config_id": config_id,
|
||||
"mapping_id": mapping.id,
|
||||
"mapping_version": mapping.version,
|
||||
"target_schema": mapping.target_schema,
|
||||
"mapped_count": mapped["mapped_count"],
|
||||
"failed_count": mapped["failed_count"],
|
||||
"errors": mapped["errors"][:20],
|
||||
}
|
||||
|
||||
written_count = await persist_mapped_records(
|
||||
db,
|
||||
datasource_name=datasource.name,
|
||||
datasource_config_id=datasource.id,
|
||||
target_schema=mapping.target_schema,
|
||||
records=mapped["records"],
|
||||
mapping_version=mapping.version,
|
||||
)
|
||||
return {
|
||||
"status": "success",
|
||||
"datasource_config_id": config_id,
|
||||
"mapping_id": mapping.id,
|
||||
"mapping_version": mapping.version,
|
||||
"target_schema": mapping.target_schema,
|
||||
"fetched_count": mapped["total_items"],
|
||||
"mapped_count": mapped["mapped_count"],
|
||||
"written_count": written_count,
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.security import get_current_user
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.db.session import get_db
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.data_snapshot import DataSnapshot
|
||||
@@ -35,6 +36,17 @@ def format_frequency_label(minutes: int) -> str:
|
||||
return f"{minutes}m"
|
||||
|
||||
|
||||
def datasource_metadata(source: str) -> dict:
|
||||
info = DEFAULT_DATASOURCES.get(source, {})
|
||||
return {
|
||||
"display_name": info.get("display_name") or info.get("name") or source,
|
||||
"is_free": bool(info.get("is_free", True)),
|
||||
"requires_credentials": bool(info.get("requires_credentials", False)),
|
||||
"credential_provider": info.get("credential_provider"),
|
||||
"credential_status": info.get("credential_status", "none"),
|
||||
}
|
||||
|
||||
|
||||
def is_due_for_collection(datasource: DataSource, now: datetime) -> bool:
|
||||
if datasource.last_run_at is None:
|
||||
return True
|
||||
@@ -72,31 +84,6 @@ async def _load_latest_running_tasks(
|
||||
return {task.datasource_id: task for task in result.scalars().all()}
|
||||
|
||||
|
||||
async def _load_latest_completed_tasks(
|
||||
db: AsyncSession,
|
||||
datasource_ids: list[int],
|
||||
) -> dict[int, CollectionTask]:
|
||||
if not datasource_ids:
|
||||
return {}
|
||||
|
||||
ranked_tasks = (
|
||||
select(
|
||||
CollectionTask.id.label("task_id"),
|
||||
_task_rank_column(CollectionTask.completed_at),
|
||||
)
|
||||
.where(CollectionTask.datasource_id.in_(datasource_ids))
|
||||
.where(CollectionTask.completed_at.isnot(None))
|
||||
.where(CollectionTask.status.in_(("success", "failed", "cancelled")))
|
||||
.subquery()
|
||||
)
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id)
|
||||
.where(ranked_tasks.c.row_num == 1)
|
||||
)
|
||||
return {task.datasource_id: task for task in result.scalars().all()}
|
||||
|
||||
|
||||
async def _load_latest_task_ids(
|
||||
db: AsyncSession,
|
||||
datasource_ids: list[int],
|
||||
@@ -123,21 +110,6 @@ async def _load_latest_task_ids(
|
||||
return {datasource_id: task_id for datasource_id, task_id in result.all()}
|
||||
|
||||
|
||||
async def _load_datasource_data_counts(
|
||||
db: AsyncSession,
|
||||
sources: list[str],
|
||||
) -> dict[str, int]:
|
||||
if not sources:
|
||||
return {}
|
||||
|
||||
result = await db.execute(
|
||||
select(CollectedData.source, func.count(CollectedData.id))
|
||||
.where(CollectedData.source.in_(sources))
|
||||
.group_by(CollectedData.source)
|
||||
)
|
||||
return {source: count for source, count in result.all()}
|
||||
|
||||
|
||||
async def _load_datasource_endpoint_overrides(
|
||||
db: AsyncSession,
|
||||
sources: list[str],
|
||||
@@ -161,7 +133,7 @@ async def _load_datasource_endpoint_overrides(
|
||||
async def _load_datasource_list_context(
|
||||
db: AsyncSession,
|
||||
datasources: list[DataSource],
|
||||
) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, int], dict[str, str]]:
|
||||
) -> tuple[dict[int, CollectionTask], dict[str, str]]:
|
||||
datasource_ids = [datasource.id for datasource in datasources]
|
||||
sources = [datasource.source for datasource in datasources]
|
||||
|
||||
@@ -185,10 +157,8 @@ async def _load_datasource_list_context(
|
||||
if stale_datasource_ids:
|
||||
running_tasks = await _load_latest_running_tasks(db, datasource_ids)
|
||||
|
||||
completed_tasks = await _load_latest_completed_tasks(db, datasource_ids)
|
||||
data_counts = await _load_datasource_data_counts(db, sources)
|
||||
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
|
||||
return running_tasks, completed_tasks, data_counts, endpoint_overrides
|
||||
return running_tasks, endpoint_overrides
|
||||
|
||||
|
||||
async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
|
||||
@@ -401,27 +371,19 @@ async def list_datasources(
|
||||
|
||||
collector_list = []
|
||||
config = get_data_sources_config()
|
||||
running_tasks, completed_tasks, data_counts, endpoint_overrides = await _load_datasource_list_context(
|
||||
db,
|
||||
datasources,
|
||||
)
|
||||
running_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
|
||||
for datasource in datasources:
|
||||
running_task = running_tasks.get(datasource.id)
|
||||
last_task = completed_tasks.get(datasource.id)
|
||||
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(
|
||||
datasource.source,
|
||||
)
|
||||
data_count = data_counts.get(datasource.source, 0)
|
||||
|
||||
last_run_at = datasource.last_run_at or (last_task.completed_at if last_task else None)
|
||||
last_run = to_iso8601_utc(last_run_at)
|
||||
last_status = datasource.last_status or (last_task.status if last_task else None)
|
||||
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
|
||||
last_run_at = datasource.last_run_at
|
||||
last_status = datasource.last_status
|
||||
|
||||
collector_list.append(
|
||||
{
|
||||
"id": datasource.id,
|
||||
"source": datasource.source,
|
||||
"name": datasource.name,
|
||||
**datasource_metadata(datasource.source),
|
||||
"module": datasource.module,
|
||||
"priority": datasource.priority,
|
||||
"frequency": format_frequency_label(datasource.frequency_minutes),
|
||||
@@ -429,11 +391,9 @@ async def list_datasources(
|
||||
"is_active": datasource.is_active,
|
||||
"collector_class": datasource.collector_class,
|
||||
"endpoint": endpoint,
|
||||
"last_run": last_run,
|
||||
"last_run": to_iso8601_utc(last_run_at),
|
||||
"last_run_at": to_iso8601_utc(last_run_at),
|
||||
"last_status": last_status,
|
||||
"last_records_processed": last_task.records_processed if last_task else None,
|
||||
"data_count": data_count,
|
||||
"is_running": running_task is not None,
|
||||
"task_id": running_task.id if running_task else None,
|
||||
"progress": running_task.progress if running_task else None,
|
||||
@@ -576,6 +536,7 @@ async def get_datasource(
|
||||
return {
|
||||
"id": datasource.id,
|
||||
"name": datasource.name,
|
||||
**datasource_metadata(datasource.source),
|
||||
"module": datasource.module,
|
||||
"priority": datasource.priority,
|
||||
"frequency": format_frequency_label(datasource.frequency_minutes),
|
||||
|
||||
@@ -9,10 +9,19 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.security import get_current_user
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.db.session import get_db
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.models.user import User
|
||||
from app.services.llm_provider_catalog import (
|
||||
get_fallback_llm_provider_preset,
|
||||
list_fallback_llm_provider_presets,
|
||||
refresh_llm_provider_preset,
|
||||
)
|
||||
from app.services.scheduler import sync_datasource_job
|
||||
from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings
|
||||
|
||||
@@ -39,6 +48,21 @@ DEFAULT_SETTINGS = {
|
||||
"password_policy": "medium",
|
||||
},
|
||||
"tv": DEFAULT_TV_SETTINGS,
|
||||
"external_integrations": {
|
||||
"ai_provider": {
|
||||
"service_url": "",
|
||||
"service_token": "",
|
||||
"provider": "minimax",
|
||||
"provider_api": "anthropic-messages",
|
||||
"base_url": "https://api.minimaxi.com/anthropic",
|
||||
"model": "MiniMax-M2.7",
|
||||
"api_key": "",
|
||||
"max_tokens": 1200,
|
||||
"anthropic_version": "2023-06-01",
|
||||
"timeout_seconds": 60,
|
||||
"retry_attempts": 2,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -96,6 +120,34 @@ class TVSettingsUpdate(BaseModel):
|
||||
sources: list[TVStreamSourceUpdate] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AIProviderIntegrationUpdate(BaseModel):
|
||||
service_url: str = ""
|
||||
service_token: Optional[str] = None
|
||||
provider: str = Field(default="minimax", max_length=80)
|
||||
provider_api: str = Field(default="anthropic-messages", max_length=80)
|
||||
base_url: str = Field(default="", max_length=500)
|
||||
model: str = Field(default="", max_length=200)
|
||||
api_key: Optional[str] = None
|
||||
max_tokens: int = Field(default=1200, ge=1, le=200000)
|
||||
anthropic_version: str = Field(default="2023-06-01", max_length=40)
|
||||
timeout_seconds: int = Field(default=60, ge=5, le=600)
|
||||
retry_attempts: int = Field(default=2, ge=1, le=10)
|
||||
clear_service_token: bool = False
|
||||
clear_api_key: bool = False
|
||||
|
||||
|
||||
class BarentsWatchIntegrationUpdate(BaseModel):
|
||||
endpoint: str = ""
|
||||
client_id: str = ""
|
||||
client_secret: Optional[str] = None
|
||||
clear_client_secret: bool = False
|
||||
|
||||
|
||||
class ExternalIntegrationsUpdate(BaseModel):
|
||||
ai_provider: AIProviderIntegrationUpdate
|
||||
barentswatch: BarentsWatchIntegrationUpdate
|
||||
|
||||
|
||||
def merge_with_defaults(category: str, payload: Optional[dict]) -> dict:
|
||||
merged = deepcopy(DEFAULT_SETTINGS[category])
|
||||
if payload:
|
||||
@@ -146,6 +198,158 @@ async def save_setting_payload(db: AsyncSession, category: str, payload: dict) -
|
||||
return merge_with_defaults(category, record.payload)
|
||||
|
||||
|
||||
def _mask_secret(value: Optional[str]) -> dict:
|
||||
if not value:
|
||||
return {"configured": False, "preview": ""}
|
||||
text = str(value)
|
||||
if "-" in text:
|
||||
prefix = text.split("-", 1)[0] + "-"
|
||||
preview = prefix + ("*" * max(len(text) - len(prefix), 1))
|
||||
else:
|
||||
prefix_len = min(4, len(text))
|
||||
preview = text[:prefix_len] + ("*" * max(len(text) - prefix_len, 1))
|
||||
return {"configured": True, "preview": preview}
|
||||
|
||||
|
||||
async def get_runtime_ai_provider_config(db: AsyncSession) -> dict:
|
||||
runtime_record = await get_setting_record(db, "external_integrations")
|
||||
payload = merge_with_defaults(
|
||||
"external_integrations",
|
||||
runtime_record.payload if runtime_record else None,
|
||||
)
|
||||
ai_payload = payload.get("ai_provider") or {}
|
||||
has_runtime_llm_config = bool(
|
||||
runtime_record
|
||||
and isinstance(runtime_record.payload, dict)
|
||||
and isinstance(runtime_record.payload.get("ai_provider"), dict)
|
||||
)
|
||||
return {
|
||||
"service_url": ai_payload.get("service_url") or app_settings.AI_PROVIDER_SERVICE_URL,
|
||||
"service_token": ai_payload.get("service_token") or app_settings.AI_PROVIDER_SERVICE_TOKEN,
|
||||
"timeout_seconds": int(
|
||||
ai_payload.get("timeout_seconds") or app_settings.AI_PROVIDER_TIMEOUT_SECONDS
|
||||
),
|
||||
"retry_attempts": int(
|
||||
ai_payload.get("retry_attempts") or app_settings.AI_PROVIDER_RETRY_ATTEMPTS
|
||||
),
|
||||
"llm_config": {
|
||||
"provider": ai_payload.get("provider") or "minimax",
|
||||
"provider_api": ai_payload.get("provider_api") or "anthropic-messages",
|
||||
"base_url": ai_payload.get("base_url") or "https://api.minimaxi.com/anthropic",
|
||||
"model": ai_payload.get("model") or "MiniMax-M2.7",
|
||||
"api_key": ai_payload.get("api_key") or "",
|
||||
"max_tokens": int(ai_payload.get("max_tokens") or 1200),
|
||||
"anthropic_version": ai_payload.get("anthropic_version") or "2023-06-01",
|
||||
} if has_runtime_llm_config else {},
|
||||
}
|
||||
|
||||
|
||||
async def get_barentswatch_config_record(db: AsyncSession) -> Optional[DataSourceConfig]:
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == "barentswatch_vessels")
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def serialize_external_integrations(db: AsyncSession) -> dict:
|
||||
ai_config = await get_runtime_ai_provider_config(db)
|
||||
runtime_setting = await get_setting_record(db, "external_integrations")
|
||||
display_llm_config = ai_config["llm_config"] or DEFAULT_SETTINGS["external_integrations"]["ai_provider"]
|
||||
barentswatch_record = await get_barentswatch_config_record(db)
|
||||
yaml_config = get_data_sources_config()
|
||||
barentswatch_auth = barentswatch_record.auth_config if barentswatch_record else {}
|
||||
barentswatch_auth = barentswatch_auth or {}
|
||||
return {
|
||||
"ai_provider": {
|
||||
"service_url": ai_config["service_url"],
|
||||
"service_token": _mask_secret(ai_config["service_token"]),
|
||||
"provider": display_llm_config.get("provider") or "minimax",
|
||||
"provider_api": display_llm_config.get("provider_api") or "anthropic-messages",
|
||||
"base_url": display_llm_config.get("base_url") or "https://api.minimaxi.com/anthropic",
|
||||
"model": display_llm_config.get("model") or "MiniMax-M2.7",
|
||||
"api_key": _mask_secret(display_llm_config.get("api_key")),
|
||||
"max_tokens": int(display_llm_config.get("max_tokens") or 1200),
|
||||
"anthropic_version": display_llm_config.get("anthropic_version") or "2023-06-01",
|
||||
"timeout_seconds": ai_config["timeout_seconds"],
|
||||
"retry_attempts": ai_config["retry_attempts"],
|
||||
"source": "runtime" if runtime_setting else "env",
|
||||
},
|
||||
"barentswatch": {
|
||||
"endpoint": (
|
||||
barentswatch_record.endpoint
|
||||
if barentswatch_record and barentswatch_record.endpoint
|
||||
else yaml_config.get_yaml_url("barentswatch_vessels")
|
||||
),
|
||||
"client_id": barentswatch_auth.get("client_id") or "",
|
||||
"client_secret": _mask_secret(barentswatch_auth.get("client_secret")),
|
||||
"source": "datasource_config" if barentswatch_record else "default",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def save_external_integrations_payload(
|
||||
db: AsyncSession,
|
||||
update: ExternalIntegrationsUpdate,
|
||||
) -> dict:
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
current_ai = current_payload.get("ai_provider") or {}
|
||||
ai_payload = {
|
||||
"service_url": update.ai_provider.service_url.strip()
|
||||
or app_settings.AI_PROVIDER_SERVICE_URL,
|
||||
"service_token": current_ai.get("service_token") or "",
|
||||
"provider": update.ai_provider.provider.strip() or "minimax",
|
||||
"provider_api": update.ai_provider.provider_api.strip() or "anthropic-messages",
|
||||
"base_url": update.ai_provider.base_url.strip(),
|
||||
"model": update.ai_provider.model.strip(),
|
||||
"api_key": current_ai.get("api_key") or "",
|
||||
"max_tokens": update.ai_provider.max_tokens,
|
||||
"anthropic_version": update.ai_provider.anthropic_version.strip() or "2023-06-01",
|
||||
"timeout_seconds": update.ai_provider.timeout_seconds,
|
||||
"retry_attempts": update.ai_provider.retry_attempts,
|
||||
}
|
||||
if update.ai_provider.clear_service_token:
|
||||
ai_payload["service_token"] = ""
|
||||
elif update.ai_provider.service_token not in (None, ""):
|
||||
ai_payload["service_token"] = update.ai_provider.service_token
|
||||
if update.ai_provider.clear_api_key:
|
||||
ai_payload["api_key"] = ""
|
||||
elif update.ai_provider.api_key not in (None, ""):
|
||||
ai_payload["api_key"] = update.ai_provider.api_key
|
||||
|
||||
await save_setting_payload(db, "external_integrations", {"ai_provider": ai_payload})
|
||||
|
||||
default_endpoint = get_data_sources_config().get_yaml_url("barentswatch_vessels")
|
||||
barentswatch_record = await get_barentswatch_config_record(db)
|
||||
if barentswatch_record is None:
|
||||
barentswatch_record = DataSourceConfig(
|
||||
name="barentswatch_vessels",
|
||||
description="BarentsWatch Live AIS credentials",
|
||||
source_type="api",
|
||||
endpoint=update.barentswatch.endpoint.strip() or default_endpoint,
|
||||
auth_type="oauth_client_credentials",
|
||||
auth_config={},
|
||||
headers={},
|
||||
config={},
|
||||
is_active=True,
|
||||
)
|
||||
db.add(barentswatch_record)
|
||||
|
||||
current_auth = dict(barentswatch_record.auth_config or {})
|
||||
if update.barentswatch.clear_client_secret:
|
||||
current_auth.pop("client_secret", None)
|
||||
elif update.barentswatch.client_secret not in (None, ""):
|
||||
current_auth["client_secret"] = update.barentswatch.client_secret
|
||||
current_auth["client_id"] = update.barentswatch.client_id.strip()
|
||||
barentswatch_record.endpoint = update.barentswatch.endpoint.strip() or default_endpoint
|
||||
barentswatch_record.auth_type = "oauth_client_credentials"
|
||||
barentswatch_record.auth_config = current_auth
|
||||
await db.commit()
|
||||
|
||||
return await serialize_external_integrations(db)
|
||||
|
||||
|
||||
def format_frequency_label(minutes: int) -> str:
|
||||
if minutes % 1440 == 0:
|
||||
return f"{minutes // 1440}d"
|
||||
@@ -155,9 +359,11 @@ def format_frequency_label(minutes: int) -> str:
|
||||
|
||||
|
||||
def serialize_collector(datasource: DataSource) -> dict:
|
||||
defaults = DEFAULT_DATASOURCES.get(datasource.source, {})
|
||||
return {
|
||||
"id": datasource.id,
|
||||
"name": datasource.name,
|
||||
"display_name": defaults.get("display_name") or datasource.name,
|
||||
"source": datasource.source,
|
||||
"module": datasource.module,
|
||||
"priority": datasource.priority,
|
||||
@@ -167,6 +373,10 @@ def serialize_collector(datasource: DataSource) -> dict:
|
||||
"last_run_at": to_iso8601_utc(datasource.last_run_at),
|
||||
"last_status": datasource.last_status,
|
||||
"next_run_at": to_iso8601_utc(datasource.next_run_at),
|
||||
"is_free": bool(defaults.get("is_free", True)),
|
||||
"requires_credentials": bool(defaults.get("requires_credentials", False)),
|
||||
"credential_provider": defaults.get("credential_provider"),
|
||||
"credential_status": defaults.get("credential_status", "none"),
|
||||
}
|
||||
|
||||
|
||||
@@ -243,6 +453,46 @@ async def update_tv_settings(
|
||||
return {"status": "updated", "tv": normalize_tv_settings(saved)}
|
||||
|
||||
|
||||
@router.get("/integrations")
|
||||
async def get_external_integrations(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return {"integrations": await serialize_external_integrations(db)}
|
||||
|
||||
|
||||
@router.get("/integrations/ai-provider/presets")
|
||||
async def get_ai_provider_presets(
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
return {"data": list_fallback_llm_provider_presets()}
|
||||
|
||||
|
||||
@router.post("/integrations/ai-provider/presets/{provider}/refresh")
|
||||
async def refresh_ai_provider_preset(
|
||||
provider: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
return {"data": await refresh_llm_provider_preset(provider)}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
fallback = get_fallback_llm_provider_preset(provider)
|
||||
fallback["refresh_error"] = str(exc)
|
||||
return {"data": fallback}
|
||||
|
||||
|
||||
@router.put("/integrations")
|
||||
async def update_external_integrations(
|
||||
payload: ExternalIntegrationsUpdate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
saved = await save_external_integrations_payload(db, payload)
|
||||
return {"status": "updated", "integrations": saved}
|
||||
|
||||
|
||||
@router.get("/collectors")
|
||||
async def get_collector_settings(
|
||||
current_user: User = Depends(get_current_user),
|
||||
@@ -289,6 +539,7 @@ async def get_all_settings(
|
||||
"notifications": setting_payloads["notifications"],
|
||||
"security": setting_payloads["security"],
|
||||
"tv": await get_tv_settings_payload(db),
|
||||
"integrations": await serialize_external_integrations(db),
|
||||
"collectors": [serialize_collector(datasource) for datasource in datasources],
|
||||
"generated_at": to_iso8601_utc(datetime.now(UTC)),
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ Unified API for all visualization data sources.
|
||||
Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium.
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import math
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Depends, Query, Response
|
||||
@@ -20,6 +20,7 @@ from app.db.session import get_db
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.vessel import VesselPosition, VesselStatic
|
||||
from app.services.bgp_collectors import build_bgp_collector_coverage
|
||||
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
|
||||
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
|
||||
@@ -511,7 +512,7 @@ def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str
|
||||
if unit in {"pflop/s", "pflops", "pflop"}:
|
||||
normalized_tflops = capacity_value * 1000
|
||||
elif unit in {"gflop/s", "gflops", "gflop"}:
|
||||
normalized_tflops = capacity_value / 1000
|
||||
normalized_tflops = capacity_value
|
||||
else:
|
||||
normalized_tflops = capacity_value
|
||||
|
||||
@@ -609,6 +610,108 @@ def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str
|
||||
return {"type": "FeatureCollection", "features": features}
|
||||
|
||||
|
||||
VESSEL_TYPE_FILTERS = {
|
||||
"cargo": lambda props: str(props.get("vessel_type_name", "")).lower() == "cargo"
|
||||
or 70 <= int(props.get("vessel_type") or -1) <= 79,
|
||||
"tanker": lambda props: str(props.get("vessel_type_name", "")).lower() == "tanker"
|
||||
or 80 <= int(props.get("vessel_type") or -1) <= 89,
|
||||
"passenger": lambda props: str(props.get("vessel_type_name", "")).lower() == "passenger"
|
||||
or 60 <= int(props.get("vessel_type") or -1) <= 69,
|
||||
"fishing": lambda props: str(props.get("vessel_type_name", "")).lower() == "fishing"
|
||||
or int(props.get("vessel_type") or -1) == 30,
|
||||
"military": lambda props: str(props.get("vessel_type_name", "")).lower() == "military"
|
||||
or int(props.get("vessel_type") or -1) == 35,
|
||||
"other": lambda props: str(props.get("vessel_type_name", "")).lower()
|
||||
not in {"cargo", "tanker", "passenger", "fishing", "military"},
|
||||
}
|
||||
|
||||
|
||||
def convert_vessels_to_geojson(rows: List[Any]) -> Dict[str, Any]:
|
||||
features = []
|
||||
for position, static in rows:
|
||||
if position.lat is None or position.lon is None:
|
||||
continue
|
||||
props = {
|
||||
"mmsi": position.mmsi,
|
||||
"name": getattr(static, "name", None) or f"MMSI {position.mmsi}",
|
||||
"callsign": getattr(static, "callsign", None),
|
||||
"imo": getattr(static, "imo", None),
|
||||
"vessel_type": getattr(static, "vessel_type", None),
|
||||
"vessel_type_name": getattr(static, "vessel_type_name", None) or "Other",
|
||||
"flag": getattr(static, "flag", None),
|
||||
"length": getattr(static, "length", None),
|
||||
"width": getattr(static, "width", None),
|
||||
"draught": getattr(static, "draught", None),
|
||||
"sog": position.sog,
|
||||
"cog": position.cog,
|
||||
"heading": position.heading,
|
||||
"nav_status": position.nav_status,
|
||||
"received_at": to_iso8601_utc(position.received_at),
|
||||
"data_type": "vessel",
|
||||
}
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": position.mmsi,
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [position.lon, position.lat],
|
||||
},
|
||||
"properties": props,
|
||||
}
|
||||
)
|
||||
|
||||
return {"type": "FeatureCollection", "features": features}
|
||||
|
||||
|
||||
def _parse_bbox(value: Optional[str]) -> tuple[float, float, float, float] | None:
|
||||
if not value:
|
||||
return None
|
||||
parts = [part.strip() for part in value.split(",")]
|
||||
if len(parts) != 4:
|
||||
raise HTTPException(status_code=400, detail="bbox must be lon_min,lat_min,lon_max,lat_max")
|
||||
try:
|
||||
lon_min, lat_min, lon_max, lat_max = [float(part) for part in parts]
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail="bbox values must be numbers") from exc
|
||||
if lat_min > lat_max:
|
||||
lat_min, lat_max = lat_max, lat_min
|
||||
if lon_min > lon_max:
|
||||
lon_min, lon_max = lon_max, lon_min
|
||||
return lon_min, lat_min, lon_max, lat_max
|
||||
|
||||
|
||||
def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bool:
|
||||
if not requested_types:
|
||||
return True
|
||||
for requested_type in requested_types:
|
||||
predicate = VESSEL_TYPE_FILTERS.get(requested_type)
|
||||
if predicate and predicate(props):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]:
|
||||
by_type: dict[str, int] = {}
|
||||
underway = 0
|
||||
anchored_or_moored = 0
|
||||
for feature in features:
|
||||
props = feature.get("properties", {})
|
||||
vessel_type = str(props.get("vessel_type_name") or "Other")
|
||||
by_type[vessel_type] = by_type.get(vessel_type, 0) + 1
|
||||
nav_status = props.get("nav_status")
|
||||
if nav_status in (1, 5):
|
||||
anchored_or_moored += 1
|
||||
else:
|
||||
underway += 1
|
||||
return {
|
||||
"total": len(features),
|
||||
"by_type": by_type,
|
||||
"underway": underway,
|
||||
"anchored_or_moored": anchored_or_moored,
|
||||
}
|
||||
|
||||
|
||||
def convert_bgp_anomalies_to_geojson(
|
||||
records: List[BGPAnomaly],
|
||||
geography_hints: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
@@ -1298,6 +1401,137 @@ async def get_compute_centers_geojson(
|
||||
}
|
||||
|
||||
|
||||
@router.get("/geo/vessels")
|
||||
async def get_vessels_geojson(
|
||||
bbox: Optional[str] = Query(
|
||||
None,
|
||||
description="Viewport bbox as lon_min,lat_min,lon_max,lat_max",
|
||||
),
|
||||
type: Optional[str] = Query(
|
||||
None,
|
||||
description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other",
|
||||
),
|
||||
limit: int = Query(5000, ge=1, le=50000),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Return latest vessel positions as GeoJSON points."""
|
||||
latest_times = (
|
||||
select(
|
||||
VesselPosition.mmsi.label("mmsi"),
|
||||
func.max(VesselPosition.received_at).label("received_at"),
|
||||
)
|
||||
.group_by(VesselPosition.mmsi)
|
||||
.subquery()
|
||||
)
|
||||
stmt = (
|
||||
select(VesselPosition, VesselStatic)
|
||||
.join(
|
||||
latest_times,
|
||||
(VesselPosition.mmsi == latest_times.c.mmsi)
|
||||
& (VesselPosition.received_at == latest_times.c.received_at),
|
||||
)
|
||||
.outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
|
||||
.order_by(VesselPosition.received_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
|
||||
parsed_bbox = _parse_bbox(bbox)
|
||||
if parsed_bbox is not None:
|
||||
lon_min, lat_min, lon_max, lat_max = parsed_bbox
|
||||
stmt = stmt.where(
|
||||
VesselPosition.lon >= lon_min,
|
||||
VesselPosition.lon <= lon_max,
|
||||
VesselPosition.lat >= lat_min,
|
||||
VesselPosition.lat <= lat_max,
|
||||
)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
rows = list(result.all())
|
||||
geojson = convert_vessels_to_geojson(rows)
|
||||
requested_types = {
|
||||
item.strip().lower()
|
||||
for item in (type or "").split(",")
|
||||
if item.strip()
|
||||
}
|
||||
if requested_types:
|
||||
geojson["features"] = [
|
||||
feature
|
||||
for feature in geojson.get("features", [])
|
||||
if _matches_vessel_type(feature.get("properties", {}), requested_types)
|
||||
]
|
||||
|
||||
features = geojson.get("features", [])
|
||||
return {
|
||||
**geojson,
|
||||
"count": len(features),
|
||||
"stats": _build_vessel_stats(features),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/vessels/{mmsi}")
|
||||
async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)):
|
||||
latest_position_stmt = (
|
||||
select(VesselPosition)
|
||||
.where(VesselPosition.mmsi == mmsi)
|
||||
.order_by(VesselPosition.received_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
static = await db.get(VesselStatic, mmsi)
|
||||
result = await db.execute(latest_position_stmt)
|
||||
position = result.scalar_one_or_none()
|
||||
if position is None:
|
||||
raise HTTPException(status_code=404, detail="Vessel not found")
|
||||
geojson = convert_vessels_to_geojson([(position, static)])
|
||||
return {
|
||||
**(geojson["features"][0]["properties"]),
|
||||
"latitude": position.lat,
|
||||
"longitude": position.lon,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/vessels/{mmsi}/track")
|
||||
async def get_vessel_track(
|
||||
mmsi: int,
|
||||
hours: int = Query(6, ge=1, le=24),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
cutoff = datetime.now(UTC) - timedelta(hours=hours)
|
||||
result = await db.execute(
|
||||
select(VesselPosition)
|
||||
.where(VesselPosition.mmsi == mmsi)
|
||||
.where(VesselPosition.received_at >= cutoff)
|
||||
.order_by(VesselPosition.received_at.asc())
|
||||
)
|
||||
positions = list(result.scalars().all())
|
||||
if not positions:
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": [],
|
||||
"count": 0,
|
||||
}
|
||||
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "LineString",
|
||||
"coordinates": [[position.lon, position.lat] for position in positions],
|
||||
},
|
||||
"properties": {
|
||||
"mmsi": mmsi,
|
||||
"hours": hours,
|
||||
"point_count": len(positions),
|
||||
"start_at": to_iso8601_utc(positions[0].received_at),
|
||||
"end_at": to_iso8601_utc(positions[-1].received_at),
|
||||
},
|
||||
}
|
||||
],
|
||||
"count": 1,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/geo/bgp-anomalies")
|
||||
async def get_bgp_anomalies_geojson(
|
||||
severity: Optional[str] = Query(None),
|
||||
@@ -1394,6 +1628,10 @@ async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
|
||||
db,
|
||||
source_filter=("ris_live_bgp", "bgpstream_bgp"),
|
||||
)
|
||||
vessel_count_result = await db.execute(
|
||||
select(func.count(func.distinct(VesselPosition.mmsi))),
|
||||
)
|
||||
vessel_count = int(vessel_count_result.scalar() or 0)
|
||||
|
||||
return {
|
||||
"generated_at": to_iso8601_utc(datetime.now(UTC)),
|
||||
@@ -1402,6 +1640,7 @@ async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
|
||||
"landing_point_count": len(landing_points.get("features", [])),
|
||||
"satellite_count": len(satellites.get("features", [])),
|
||||
"compute_center_count": len(compute_features),
|
||||
"vessel_count": vessel_count,
|
||||
"supercomputer_count": sum(
|
||||
1 for feature in compute_features
|
||||
if feature.get("properties", {}).get("site_type") == "supercomputer"
|
||||
|
||||
@@ -31,6 +31,7 @@ COLLECTOR_URL_KEYS = {
|
||||
"opengeofeed_prefix_geo": "opengeofeed.public_csv_url",
|
||||
"nro_delegated_prefix_geo": "nro.delegated_stats_url",
|
||||
"news_live_streams": "news_live_streams.channels_url",
|
||||
"barentswatch_vessels": "barentswatch_vessels.url",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -94,3 +94,7 @@ news_live_streams:
|
||||
streams_url: "https://iptv-org.github.io/api/streams.json"
|
||||
# IPTV-org 台标 JSON
|
||||
logos_url: "https://iptv-org.github.io/api/logos.json"
|
||||
|
||||
barentswatch_vessels:
|
||||
# BarentsWatch Live AIS latest combined endpoint. Requires an AIS bearer token.
|
||||
url: "https://live.ais.barentswatch.no/v1/latest/combined"
|
||||
|
||||
@@ -4,163 +4,246 @@ DEFAULT_DATASOURCES = {
|
||||
"top500": {
|
||||
"id": 1,
|
||||
"name": "TOP500 Supercomputers",
|
||||
"display_name": "TOP500 超算榜单",
|
||||
"module": "L1",
|
||||
"priority": "P0",
|
||||
"frequency_minutes": 240,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"epoch_ai_gpu": {
|
||||
"id": 2,
|
||||
"name": "Epoch AI GPU Clusters",
|
||||
"display_name": "Epoch AI GPU 集群",
|
||||
"module": "L1",
|
||||
"priority": "P0",
|
||||
"frequency_minutes": 360,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"huggingface_models": {
|
||||
"id": 3,
|
||||
"name": "HuggingFace Models",
|
||||
"display_name": "Hugging Face 模型",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 720,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"huggingface_datasets": {
|
||||
"id": 4,
|
||||
"name": "HuggingFace Datasets",
|
||||
"display_name": "Hugging Face 数据集",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 720,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"huggingface_spaces": {
|
||||
"id": 5,
|
||||
"name": "HuggingFace Spaces",
|
||||
"display_name": "Hugging Face Spaces",
|
||||
"module": "L2",
|
||||
"priority": "P2",
|
||||
"frequency_minutes": 1440,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"peeringdb_ixp": {
|
||||
"id": 6,
|
||||
"name": "PeeringDB IXP",
|
||||
"display_name": "PeeringDB 交换中心",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 1440,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"peeringdb_network": {
|
||||
"id": 7,
|
||||
"name": "PeeringDB Networks",
|
||||
"display_name": "PeeringDB 网络",
|
||||
"module": "L2",
|
||||
"priority": "P2",
|
||||
"frequency_minutes": 2880,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"peeringdb_facility": {
|
||||
"id": 8,
|
||||
"name": "PeeringDB Facilities",
|
||||
"display_name": "PeeringDB 设施",
|
||||
"module": "L2",
|
||||
"priority": "P2",
|
||||
"frequency_minutes": 2880,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"telegeography_cables": {
|
||||
"id": 9,
|
||||
"name": "Submarine Cables",
|
||||
"display_name": "海底光缆",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"telegeography_landing": {
|
||||
"id": 10,
|
||||
"name": "Cable Landing Points",
|
||||
"display_name": "光缆登陆点",
|
||||
"module": "L2",
|
||||
"priority": "P2",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"telegeography_systems": {
|
||||
"id": 11,
|
||||
"name": "Cable Systems",
|
||||
"display_name": "光缆系统",
|
||||
"module": "L2",
|
||||
"priority": "P2",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"arcgis_cables": {
|
||||
"id": 15,
|
||||
"name": "ArcGIS Submarine Cables",
|
||||
"display_name": "ArcGIS 海底光缆",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"arcgis_landing_points": {
|
||||
"id": 16,
|
||||
"name": "ArcGIS Landing Points",
|
||||
"display_name": "ArcGIS 登陆点",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"arcgis_cable_landing_relation": {
|
||||
"id": 17,
|
||||
"name": "ArcGIS Cable-Landing Relations",
|
||||
"display_name": "ArcGIS 光缆登陆关系",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"fao_landing_points": {
|
||||
"id": 18,
|
||||
"name": "FAO Landing Points",
|
||||
"display_name": "FAO 登陆点",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"spacetrack_tle": {
|
||||
"id": 19,
|
||||
"name": "Space-Track TLE",
|
||||
"display_name": "Space-Track 轨道根数",
|
||||
"module": "L3",
|
||||
"priority": "P2",
|
||||
"frequency_minutes": 1440,
|
||||
"is_free": True,
|
||||
"requires_credentials": True,
|
||||
"credential_provider": "spacetrack",
|
||||
"credential_status": "planned",
|
||||
},
|
||||
"celestrak_tle": {
|
||||
"id": 20,
|
||||
"name": "CelesTrak TLE",
|
||||
"display_name": "CelesTrak 轨道根数",
|
||||
"module": "L3",
|
||||
"priority": "P2",
|
||||
"frequency_minutes": 1440,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"ris_live_bgp": {
|
||||
"id": 21,
|
||||
"name": "RIPE RIS Live BGP",
|
||||
"display_name": "RIPE RIS 实时 BGP",
|
||||
"module": "L3",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 15,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"bgpstream_bgp": {
|
||||
"id": 22,
|
||||
"name": "CAIDA BGPStream Backfill",
|
||||
"display_name": "CAIDA BGPStream 回填",
|
||||
"module": "L3",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 360,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"iptoasn_prefix_geo": {
|
||||
"id": 23,
|
||||
"name": "IPtoASN Prefix Geography",
|
||||
"display_name": "IPtoASN 前缀地理",
|
||||
"module": "L3",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 1440,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"opengeofeed_prefix_geo": {
|
||||
"id": 24,
|
||||
"name": "OpenGeoFeed Prefix Geography",
|
||||
"display_name": "OpenGeoFeed 前缀地理",
|
||||
"module": "L3",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 1440,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"nro_delegated_prefix_geo": {
|
||||
"id": 25,
|
||||
"name": "NRO Delegated Prefix Geography",
|
||||
"display_name": "NRO 分配前缀地理",
|
||||
"module": "L3",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 1440,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"news_live_streams": {
|
||||
"id": 26,
|
||||
"name": "News Live Streams",
|
||||
"display_name": "新闻直播源",
|
||||
"module": "L4",
|
||||
"priority": "P2",
|
||||
"frequency_minutes": 720,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"barentswatch_vessels": {
|
||||
"id": 27,
|
||||
"name": "BarentsWatch AIS Vessels",
|
||||
"display_name": "BarentsWatch AIS 船舶",
|
||||
"module": "L4",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 1,
|
||||
"is_free": True,
|
||||
"requires_credentials": True,
|
||||
"credential_provider": "barentswatch",
|
||||
"credential_status": "supported",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
151
backend/app/core/target_schema_registry.py
Normal file
151
backend/app/core/target_schema_registry.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""Registry of target schemas supported by mapped custom data sources."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, ValidationError, field_validator
|
||||
|
||||
|
||||
class VesselAISRecord(BaseModel):
|
||||
mmsi: int = Field(ge=100000000, le=999999999)
|
||||
lat: float = Field(ge=-90, le=90)
|
||||
lon: float = Field(ge=-180, le=180)
|
||||
sog: float | None = None
|
||||
cog: float | None = Field(default=None, ge=0, le=360)
|
||||
heading: int | None = Field(default=None, ge=0, le=511)
|
||||
name: str | None = None
|
||||
vessel_type: str | int | None = None
|
||||
received_at: datetime | None = None
|
||||
|
||||
|
||||
class GeoPointRecord(BaseModel):
|
||||
lat: float = Field(ge=-90, le=90)
|
||||
lon: float = Field(ge=-180, le=180)
|
||||
name: str | None = None
|
||||
type: str | None = None
|
||||
source_id: str | None = None
|
||||
observed_at: datetime | None = None
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class GenericRecord(BaseModel):
|
||||
data: dict[str, Any] = Field(default_factory=dict)
|
||||
source_id: str | None = None
|
||||
observed_at: datetime | None = None
|
||||
|
||||
@field_validator("data")
|
||||
@classmethod
|
||||
def require_payload(cls, value: dict[str, Any]) -> dict[str, Any]:
|
||||
if not value:
|
||||
raise ValueError("generic_records requires a non-empty data object")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TargetField:
|
||||
name: str
|
||||
type: str
|
||||
required: bool = False
|
||||
description: str = ""
|
||||
example: Any = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"type": self.type,
|
||||
"required": self.required,
|
||||
"description": self.description,
|
||||
"example": self.example,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TargetSchema:
|
||||
key: str
|
||||
label: str
|
||||
description: str
|
||||
fields: tuple[TargetField, ...]
|
||||
model: type[BaseModel]
|
||||
destination: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"key": self.key,
|
||||
"label": self.label,
|
||||
"description": self.description,
|
||||
"destination": self.destination,
|
||||
"fields": [field.to_dict() for field in self.fields],
|
||||
}
|
||||
|
||||
def validate_record(self, record: dict[str, Any]) -> tuple[dict[str, Any] | None, list[str]]:
|
||||
try:
|
||||
return self.model.model_validate(record).model_dump(mode="json"), []
|
||||
except ValidationError as exc:
|
||||
return None, [
|
||||
".".join(str(part) for part in error["loc"]) + f": {error['msg']}"
|
||||
for error in exc.errors()
|
||||
]
|
||||
|
||||
|
||||
TARGET_SCHEMAS: dict[str, TargetSchema] = {
|
||||
"vessel_ais": TargetSchema(
|
||||
key="vessel_ais",
|
||||
label="船舶 AIS",
|
||||
description="船只位置、航速、航向、MMSI 等 AIS 数据。",
|
||||
destination="vessel_position",
|
||||
model=VesselAISRecord,
|
||||
fields=(
|
||||
TargetField("mmsi", "integer", True, "MMSI 九位船舶标识", 257123000),
|
||||
TargetField("lat", "float", True, "纬度", 59.91),
|
||||
TargetField("lon", "float", True, "经度", 10.75),
|
||||
TargetField("sog", "float", False, "对地航速，单位节", 12.4),
|
||||
TargetField("cog", "float", False, "对地航向，0-360 度", 184.5),
|
||||
TargetField("heading", "integer", False, "船首向，0-511", 186),
|
||||
TargetField("name", "string", False, "船名", "OSLO EXPRESS"),
|
||||
TargetField("vessel_type", "string", False, "船型", "cargo"),
|
||||
TargetField("received_at", "datetime", False, "数据接收时间", "2026-04-28T00:00:00Z"),
|
||||
),
|
||||
),
|
||||
"geo_points": TargetSchema(
|
||||
key="geo_points",
|
||||
label="通用地理点",
|
||||
description="带经纬度的通用实体或事件点位。",
|
||||
destination="generic_geo_points",
|
||||
model=GeoPointRecord,
|
||||
fields=(
|
||||
TargetField("lat", "float", True, "纬度", 1.3),
|
||||
TargetField("lon", "float", True, "经度", 103.8),
|
||||
TargetField("name", "string", False, "点位名称", "Singapore"),
|
||||
TargetField("type", "string", False, "点位类型", "datacenter"),
|
||||
TargetField("source_id", "string", False, "来源侧 ID", "sg-1"),
|
||||
TargetField("observed_at", "datetime", False, "观测时间", "2026-04-28T00:00:00Z"),
|
||||
TargetField("metadata", "object", False, "扩展字段", {"provider": "example"}),
|
||||
),
|
||||
),
|
||||
"generic_records": TargetSchema(
|
||||
key="generic_records",
|
||||
label="通用结构化记录",
|
||||
description="未知结构数据沉淀，不直接进入 Earth 图层。",
|
||||
destination="collected_data",
|
||||
model=GenericRecord,
|
||||
fields=(
|
||||
TargetField("data", "object", True, "结构化记录主体", {"raw": "value"}),
|
||||
TargetField("source_id", "string", False, "来源侧 ID", "record-1"),
|
||||
TargetField("observed_at", "datetime", False, "观测时间", "2026-04-28T00:00:00Z"),
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def list_target_schemas() -> list[dict[str, Any]]:
|
||||
return [schema.to_dict() for schema in TARGET_SCHEMAS.values()]
|
||||
|
||||
|
||||
def get_target_schema(key: str) -> TargetSchema:
|
||||
try:
|
||||
return TARGET_SCHEMAS[key]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"Unsupported target schema: {key}") from exc
|
||||
@@ -110,6 +110,8 @@ async def init_db():
|
||||
import app.models.playground_session # noqa: F401
|
||||
import app.models.playground_message # noqa: F401
|
||||
import app.models.system_log # noqa: F401
|
||||
import app.models.vessel # noqa: F401
|
||||
import app.models.datasource_mapping # noqa: F401
|
||||
|
||||
logger.warning_event(
|
||||
"Database pool settings active",
|
||||
|
||||
@@ -12,6 +12,8 @@ from app.models.system_setting import SystemSetting
|
||||
from app.models.playground_session import PlaygroundSession
|
||||
from app.models.playground_message import PlaygroundMessage
|
||||
from app.models.system_log import SystemLog, AuditLog
|
||||
from app.models.vessel import VesselPosition, VesselStatic
|
||||
from app.models.datasource_mapping import DataSourceMappingTemplate
|
||||
|
||||
__all__ = [
|
||||
"User",
|
||||
@@ -29,4 +31,7 @@ __all__ = [
|
||||
"BGPObservation",
|
||||
"SystemLog",
|
||||
"AuditLog",
|
||||
"VesselPosition",
|
||||
"VesselStatic",
|
||||
"DataSourceMappingTemplate",
|
||||
]
|
||||
|
||||
32
backend/app/models/datasource_mapping.py
Normal file
32
backend/app/models/datasource_mapping.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""Mapping templates for user-defined data source payloads."""
|
||||
|
||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, JSON, String
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.db.session import Base
|
||||
|
||||
|
||||
class DataSourceMappingTemplate(Base):
|
||||
__tablename__ = "datasource_mapping_templates"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
datasource_config_id = Column(
|
||||
Integer,
|
||||
ForeignKey("datasource_configs.id"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
target_schema = Column(String(80), nullable=False, index=True)
|
||||
mapping_json = Column(JSON, nullable=False, default={})
|
||||
sample_payload_hash = Column(String(64), nullable=True)
|
||||
validation_status = Column(String(30), nullable=False, default="draft")
|
||||
version = Column(Integer, nullable=False, default=1)
|
||||
is_active = Column(Boolean, nullable=False, default=False, index=True)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"<DataSourceMappingTemplate {self.id}: "
|
||||
f"{self.datasource_config_id}/{self.target_schema}/v{self.version}>"
|
||||
)
|
||||
75
backend/app/models/vessel.py
Normal file
75
backend/app/models/vessel.py
Normal file
@@ -0,0 +1,75 @@
|
||||
"""Vessel AIS models for live maritime tracking."""
|
||||
|
||||
from sqlalchemy import BigInteger, Column, DateTime, Float, Index, Integer, SmallInteger, String
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.db.session import Base
|
||||
|
||||
|
||||
class VesselStatic(Base):
|
||||
"""Slow-changing vessel identity and dimensions."""
|
||||
|
||||
__tablename__ = "vessel_static"
|
||||
|
||||
mmsi = Column(BigInteger, primary_key=True)
|
||||
name = Column(String(128), nullable=True)
|
||||
callsign = Column(String(16), nullable=True)
|
||||
vessel_type = Column(SmallInteger, nullable=True, index=True)
|
||||
vessel_type_name = Column(String(64), nullable=True, index=True)
|
||||
flag = Column(String(4), nullable=True, index=True)
|
||||
length = Column(Float, nullable=True)
|
||||
width = Column(Float, nullable=True)
|
||||
draught = Column(Float, nullable=True)
|
||||
imo = Column(BigInteger, nullable=True)
|
||||
updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"mmsi": self.mmsi,
|
||||
"name": self.name,
|
||||
"callsign": self.callsign,
|
||||
"vessel_type": self.vessel_type,
|
||||
"vessel_type_name": self.vessel_type_name,
|
||||
"flag": self.flag,
|
||||
"length": self.length,
|
||||
"width": self.width,
|
||||
"draught": self.draught,
|
||||
"imo": self.imo,
|
||||
"updated_at": to_iso8601_utc(self.updated_at),
|
||||
}
|
||||
|
||||
|
||||
class VesselPosition(Base):
|
||||
"""Append-only AIS positions retained for short history windows."""
|
||||
|
||||
__tablename__ = "vessel_position"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
mmsi = Column(BigInteger, nullable=False, index=True)
|
||||
lat = Column(Float, nullable=False)
|
||||
lon = Column(Float, nullable=False)
|
||||
sog = Column(Float, nullable=True)
|
||||
cog = Column(Float, nullable=True)
|
||||
heading = Column(SmallInteger, nullable=True)
|
||||
nav_status = Column(SmallInteger, nullable=True, index=True)
|
||||
received_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_vessel_pos_mmsi_time", "mmsi", "received_at"),
|
||||
Index("idx_vessel_pos_time", "received_at"),
|
||||
Index("idx_vessel_pos_lat_lon", "lat", "lon"),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"mmsi": self.mmsi,
|
||||
"lat": self.lat,
|
||||
"lon": self.lon,
|
||||
"sog": self.sog,
|
||||
"cog": self.cog,
|
||||
"heading": self.heading,
|
||||
"nav_status": self.nav_status,
|
||||
"received_at": to_iso8601_utc(self.received_at),
|
||||
}
|
||||
@@ -3,9 +3,11 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException, status
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.db.session import get_db
|
||||
from app.schemas.ai import (
|
||||
AIProviderStatusResponse,
|
||||
SituationalAnalysisRequest,
|
||||
@@ -14,11 +16,27 @@ from app.schemas.ai import (
|
||||
|
||||
|
||||
class AIProviderClient:
|
||||
def __init__(self) -> None:
|
||||
self.service_url = settings.AI_PROVIDER_SERVICE_URL.rstrip("/")
|
||||
self.service_token = settings.AI_PROVIDER_SERVICE_TOKEN
|
||||
self.timeout = settings.AI_PROVIDER_TIMEOUT_SECONDS
|
||||
self.retry_attempts = max(settings.AI_PROVIDER_RETRY_ATTEMPTS, 1)
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
service_url: str | None = None,
|
||||
service_token: str | None = None,
|
||||
timeout: int | None = None,
|
||||
retry_attempts: int | None = None,
|
||||
llm_config: dict | None = None,
|
||||
) -> None:
|
||||
self.service_url = (
|
||||
service_url if service_url is not None else settings.AI_PROVIDER_SERVICE_URL
|
||||
).rstrip("/")
|
||||
self.service_token = (
|
||||
service_token if service_token is not None else settings.AI_PROVIDER_SERVICE_TOKEN
|
||||
)
|
||||
self.timeout = timeout if timeout is not None else settings.AI_PROVIDER_TIMEOUT_SECONDS
|
||||
self.retry_attempts = max(
|
||||
retry_attempts if retry_attempts is not None else settings.AI_PROVIDER_RETRY_ATTEMPTS,
|
||||
1,
|
||||
)
|
||||
self.llm_config = llm_config or {}
|
||||
|
||||
def _headers(self, request_id: str | None = None) -> dict[str, str]:
|
||||
headers = {"Content-Type": "application/json"}
|
||||
@@ -26,6 +44,19 @@ class AIProviderClient:
|
||||
headers["X-Provider-Token"] = self.service_token
|
||||
if request_id:
|
||||
headers["X-Request-ID"] = request_id
|
||||
llm_header_map = {
|
||||
"provider": "X-AI-Provider",
|
||||
"provider_api": "X-AI-Provider-API",
|
||||
"base_url": "X-AI-Base-URL",
|
||||
"api_key": "X-AI-API-Key",
|
||||
"model": "X-AI-Model",
|
||||
"max_tokens": "X-AI-Max-Tokens",
|
||||
"anthropic_version": "X-AI-Anthropic-Version",
|
||||
}
|
||||
for key, header_name in llm_header_map.items():
|
||||
value = self.llm_config.get(key)
|
||||
if value not in (None, ""):
|
||||
headers[header_name] = str(value)
|
||||
return headers
|
||||
|
||||
async def get_status(self, request_id: str | None = None) -> AIProviderStatusResponse:
|
||||
@@ -105,5 +136,14 @@ class AIProviderClient:
|
||||
)
|
||||
|
||||
|
||||
def get_ai_provider_client() -> AIProviderClient:
|
||||
return AIProviderClient()
|
||||
async def get_ai_provider_client(db: AsyncSession = Depends(get_db)) -> AIProviderClient:
|
||||
from app.api.v1.settings import get_runtime_ai_provider_config
|
||||
|
||||
runtime_config = await get_runtime_ai_provider_config(db)
|
||||
return AIProviderClient(
|
||||
service_url=runtime_config["service_url"],
|
||||
service_token=runtime_config["service_token"],
|
||||
timeout=runtime_config["timeout_seconds"],
|
||||
retry_attempts=runtime_config["retry_attempts"],
|
||||
llm_config=runtime_config.get("llm_config") or {},
|
||||
)
|
||||
|
||||
@@ -36,6 +36,7 @@ from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
|
||||
from app.services.collectors.opengeofeed import OpenGeoFeedPrefixGeoCollector
|
||||
from app.services.collectors.nro_delegated import NRODelegatedPrefixGeoCollector
|
||||
from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
|
||||
from app.services.collectors.vessel_ais import VesselAISCollector
|
||||
|
||||
collector_registry.register(TOP500Collector())
|
||||
collector_registry.register(EpochAIGPUCollector())
|
||||
@@ -63,3 +64,4 @@ collector_registry.register(IPtoASNPrefixGeoCollector())
|
||||
collector_registry.register(OpenGeoFeedPrefixGeoCollector())
|
||||
collector_registry.register(NRODelegatedPrefixGeoCollector())
|
||||
collector_registry.register(NewsLiveStreamsCollector())
|
||||
collector_registry.register(VesselAISCollector())
|
||||
|
||||
326
backend/app/services/collectors/vessel_ais.py
Normal file
326
backend/app/services/collectors/vessel_ais.py
Normal file
@@ -0,0 +1,326 @@
|
||||
"""BarentsWatch AIS collector for vessel tracking."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.vessel import VesselPosition, VesselStatic
|
||||
from app.services.collectors.base import BaseCollector
|
||||
|
||||
|
||||
BARENTSWATCH_LATEST_URL = "https://live.ais.barentswatch.no/v1/latest/combined"
|
||||
BARENTSWATCH_TOKEN_URL = "https://id.barentswatch.no/connect/token"
|
||||
|
||||
|
||||
VESSEL_TYPE_NAMES = {
|
||||
30: "Fishing",
|
||||
35: "Military",
|
||||
60: "Passenger",
|
||||
70: "Cargo",
|
||||
80: "Tanker",
|
||||
}
|
||||
|
||||
|
||||
class VesselAISCollector(BaseCollector):
|
||||
"""Collect latest AIS positions and append them to vessel tables."""
|
||||
|
||||
name = "barentswatch_vessels"
|
||||
priority = "P1"
|
||||
module = "L4"
|
||||
frequency_hours = 1
|
||||
data_type = "vessel_ais"
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return self._resolved_url or BARENTSWATCH_LATEST_URL
|
||||
|
||||
async def _load_datasource_config(self) -> dict[str, Any]:
|
||||
if not self._db_session:
|
||||
return {}
|
||||
try:
|
||||
from sqlalchemy import select
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
|
||||
result = await self._db_session.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == self.name)
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
datasource_config = result.scalar_one_or_none()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
if not datasource_config:
|
||||
return {}
|
||||
return {
|
||||
"auth_config": datasource_config.auth_config or {},
|
||||
"config": datasource_config.config or {},
|
||||
}
|
||||
|
||||
async def _get_access_token(self, client: httpx.AsyncClient) -> str | None:
|
||||
datasource_config = await self._load_datasource_config()
|
||||
auth_config = datasource_config.get("auth_config") or {}
|
||||
config = datasource_config.get("config") or {}
|
||||
client_id = (
|
||||
auth_config.get("client_id")
|
||||
or config.get("client_id")
|
||||
or os.getenv("BARENTSWATCH_CLIENT_ID")
|
||||
or os.getenv("BARRENTSWATCH_CLIENT_ID")
|
||||
)
|
||||
client_secret = (
|
||||
auth_config.get("client_secret")
|
||||
or config.get("client_secret")
|
||||
or os.getenv("BARENTSWATCH_CLIENT_SECRET")
|
||||
or os.getenv("BARRENTSWATCH_CLIENT_SECRET")
|
||||
)
|
||||
if not client_id or not client_secret:
|
||||
return None
|
||||
|
||||
response = await client.post(
|
||||
BARENTSWATCH_TOKEN_URL,
|
||||
data={
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"scope": "ais",
|
||||
"grant_type": "client_credentials",
|
||||
},
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
token = payload.get("access_token")
|
||||
return str(token) if token else None
|
||||
|
||||
async def fetch(self) -> list[dict[str, Any]]:
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
headers: dict[str, str] = {}
|
||||
token = await self._get_access_token(client)
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
response = await client.get(self.base_url, headers=headers)
|
||||
if response.status_code == 401 and not token:
|
||||
return self._get_sample_data()
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
|
||||
if isinstance(payload, list):
|
||||
return [item for item in payload if isinstance(item, dict)]
|
||||
if isinstance(payload, dict):
|
||||
for key in ("features", "data", "items", "vessels"):
|
||||
value = payload.get(key)
|
||||
if isinstance(value, list):
|
||||
if key == "features":
|
||||
return [
|
||||
{
|
||||
**(item.get("properties") or {}),
|
||||
"geometry": item.get("geometry"),
|
||||
}
|
||||
for item in value
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
return self._get_sample_data()
|
||||
|
||||
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
transformed = []
|
||||
for item in raw_data:
|
||||
record = self._normalize_record(item)
|
||||
if record:
|
||||
transformed.append(record)
|
||||
return transformed
|
||||
|
||||
async def _save_data(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
data: list[dict[str, Any]],
|
||||
task_id: int | None = None,
|
||||
snapshot_id: int | None = None,
|
||||
) -> int:
|
||||
now = datetime.now(UTC)
|
||||
records_added = 0
|
||||
|
||||
for index, item in enumerate(data):
|
||||
static = await db.get(VesselStatic, item["mmsi"])
|
||||
if static is None:
|
||||
static = VesselStatic(mmsi=item["mmsi"])
|
||||
db.add(static)
|
||||
|
||||
for field in (
|
||||
"name",
|
||||
"callsign",
|
||||
"vessel_type",
|
||||
"vessel_type_name",
|
||||
"flag",
|
||||
"length",
|
||||
"width",
|
||||
"draught",
|
||||
"imo",
|
||||
):
|
||||
value = item.get(field)
|
||||
if value not in (None, ""):
|
||||
setattr(static, field, value)
|
||||
static.updated_at = now
|
||||
|
||||
db.add(
|
||||
VesselPosition(
|
||||
mmsi=item["mmsi"],
|
||||
lat=item["lat"],
|
||||
lon=item["lon"],
|
||||
sog=item.get("sog"),
|
||||
cog=item.get("cog"),
|
||||
heading=item.get("heading"),
|
||||
nav_status=item.get("nav_status"),
|
||||
received_at=item.get("received_at") or now,
|
||||
)
|
||||
)
|
||||
records_added += 1
|
||||
|
||||
if (index + 1) % 1000 == 0:
|
||||
await self.update_progress(index + 1, commit=True)
|
||||
|
||||
await db.execute(
|
||||
delete(VesselPosition).where(VesselPosition.received_at < now - timedelta(hours=24))
|
||||
)
|
||||
await db.commit()
|
||||
await self.update_progress(records_added, force=True)
|
||||
return records_added
|
||||
|
||||
def _normalize_record(self, item: dict[str, Any]) -> dict[str, Any] | None:
|
||||
mmsi = _as_int(_pick(item, "mmsi", "MMSI", "Mmsi"))
|
||||
lat = _as_float(_pick(item, "lat", "latitude", "Latitude"))
|
||||
lon = _as_float(_pick(item, "lon", "lng", "longitude", "Longitude"))
|
||||
|
||||
geometry = item.get("geometry")
|
||||
coordinates = geometry.get("coordinates") if isinstance(geometry, dict) else None
|
||||
if (lat is None or lon is None) and isinstance(coordinates, list) and len(coordinates) >= 2:
|
||||
lon = _as_float(coordinates[0])
|
||||
lat = _as_float(coordinates[1])
|
||||
|
||||
if mmsi is None or lat is None or lon is None:
|
||||
return None
|
||||
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
|
||||
return None
|
||||
|
||||
vessel_type = _as_int(_pick(item, "vessel_type", "shipType", "ship_type", "ShipType"))
|
||||
vessel_type_name = (
|
||||
_pick(item, "vessel_type_name", "shipTypeName", "ship_type_name", "VesselTypeName")
|
||||
or _vessel_type_name(vessel_type)
|
||||
)
|
||||
received_at = _parse_datetime(_pick(item, "received_at", "timestamp", "time", "msgtime"))
|
||||
|
||||
return {
|
||||
"mmsi": mmsi,
|
||||
"name": _pick(item, "name", "shipName", "ship_name", "Name"),
|
||||
"callsign": _pick(item, "callsign", "callSign", "CallSign"),
|
||||
"vessel_type": vessel_type,
|
||||
"vessel_type_name": vessel_type_name,
|
||||
"flag": _pick(item, "flag", "country", "Flag"),
|
||||
"length": _as_float(_pick(item, "length", "shipLength", "Length")),
|
||||
"width": _as_float(_pick(item, "width", "shipWidth", "Width")),
|
||||
"draught": _as_float(_pick(item, "draught", "draft", "Draught")),
|
||||
"imo": _as_int(_pick(item, "imo", "IMO", "imoNumber")),
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"sog": _as_float(_pick(item, "sog", "speedOverGround", "SOG")),
|
||||
"cog": _as_float(_pick(item, "cog", "courseOverGround", "COG")),
|
||||
"heading": _as_int(_pick(item, "heading", "trueHeading", "Heading")),
|
||||
"nav_status": _as_int(_pick(item, "nav_status", "navStatus", "NavigationalStatus")),
|
||||
"received_at": received_at,
|
||||
}
|
||||
|
||||
def _get_sample_data(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"mmsi": 257123000,
|
||||
"name": "OSLO TRADER",
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
"sog": 12.4,
|
||||
"cog": 214,
|
||||
"heading": 215,
|
||||
"nav_status": 0,
|
||||
"vessel_type": 70,
|
||||
"vessel_type_name": "Cargo",
|
||||
"flag": "NO",
|
||||
"length": 185,
|
||||
},
|
||||
{
|
||||
"mmsi": 257456000,
|
||||
"name": "NORDIC FJORD",
|
||||
"lat": 60.39,
|
||||
"lon": 5.32,
|
||||
"sog": 0.2,
|
||||
"cog": 82,
|
||||
"heading": 80,
|
||||
"nav_status": 1,
|
||||
"vessel_type": 60,
|
||||
"vessel_type_name": "Passenger",
|
||||
"flag": "NO",
|
||||
"length": 126,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _pick(item: dict[str, Any], *keys: str) -> Any:
|
||||
for key in keys:
|
||||
if key in item and item[key] not in (None, ""):
|
||||
return item[key]
|
||||
return None
|
||||
|
||||
|
||||
def _as_float(value: Any) -> float | None:
|
||||
try:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _as_int(value: Any) -> int | None:
|
||||
try:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if not value:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
timestamp = float(value)
|
||||
if timestamp > 10_000_000_000:
|
||||
timestamp /= 1000
|
||||
return datetime.fromtimestamp(timestamp, UTC)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _vessel_type_name(vessel_type: int | None) -> str:
|
||||
if vessel_type is None:
|
||||
return "Other"
|
||||
if 70 <= vessel_type <= 79:
|
||||
return "Cargo"
|
||||
if 80 <= vessel_type <= 89:
|
||||
return "Tanker"
|
||||
if 60 <= vessel_type <= 69:
|
||||
return "Passenger"
|
||||
if vessel_type == 30:
|
||||
return "Fishing"
|
||||
if vessel_type == 35:
|
||||
return "Military"
|
||||
return VESSEL_TYPE_NAMES.get(vessel_type, "Other")
|
||||
358
backend/app/services/datasource_mapping.py
Normal file
358
backend/app/services/datasource_mapping.py
Normal file
@@ -0,0 +1,358 @@
|
||||
"""Deterministic mapping support for custom data sources."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.target_schema_registry import TargetSchema, get_target_schema
|
||||
|
||||
SECRET_KEY_PATTERN = re.compile(
|
||||
r"(token|secret|password|passwd|authorization|api[_-]?key|client[_-]?secret)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
class MappingError(ValueError):
|
||||
"""Raised when a mapping definition cannot be executed."""
|
||||
|
||||
|
||||
def stable_payload_hash(payload: Any) -> str:
|
||||
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode()
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def redact_for_llm(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
redacted = {}
|
||||
for key, item in value.items():
|
||||
if SECRET_KEY_PATTERN.search(str(key)):
|
||||
redacted[key] = "[REDACTED]"
|
||||
else:
|
||||
redacted[key] = redact_for_llm(item)
|
||||
return redacted
|
||||
if isinstance(value, list):
|
||||
return [redact_for_llm(item) for item in value[:20]]
|
||||
return value
|
||||
|
||||
|
||||
def extract_path(payload: Any, path: str | None) -> Any:
|
||||
if not path or path == "$":
|
||||
return payload
|
||||
|
||||
normalized = path.strip()
|
||||
if normalized.startswith("$."):
|
||||
normalized = normalized[2:]
|
||||
elif normalized.startswith("$"):
|
||||
normalized = normalized[1:]
|
||||
normalized = normalized.strip(".")
|
||||
if not normalized:
|
||||
return payload
|
||||
|
||||
current = payload
|
||||
for raw_segment in normalized.split("."):
|
||||
segment = raw_segment.strip()
|
||||
if not segment:
|
||||
continue
|
||||
|
||||
list_all = segment.endswith("[*]")
|
||||
if list_all:
|
||||
segment = segment[:-3]
|
||||
|
||||
index = None
|
||||
match = re.fullmatch(r"(.+)\[(\d+)\]", segment)
|
||||
if match:
|
||||
segment = match.group(1)
|
||||
index = int(match.group(2))
|
||||
|
||||
if segment:
|
||||
if isinstance(current, dict):
|
||||
current = current.get(segment)
|
||||
else:
|
||||
return None
|
||||
|
||||
if list_all:
|
||||
return current if isinstance(current, list) else []
|
||||
|
||||
if index is not None:
|
||||
if not isinstance(current, list) or index >= len(current):
|
||||
return None
|
||||
current = current[index]
|
||||
|
||||
return current
|
||||
|
||||
|
||||
def _convert_value(value: Any, target_type: str | None) -> Any:
|
||||
if value is None or target_type in (None, "", "any"):
|
||||
return value
|
||||
|
||||
if target_type == "string":
|
||||
return str(value)
|
||||
if target_type == "integer":
|
||||
return int(value)
|
||||
if target_type == "float":
|
||||
return float(value)
|
||||
if target_type == "boolean":
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
return bool(value)
|
||||
if target_type == "datetime":
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
return datetime.fromtimestamp(value)
|
||||
if isinstance(value, str):
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return value
|
||||
if target_type == "object":
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
raise ValueError("expected object")
|
||||
if target_type == "array":
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
raise ValueError("expected array")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def _apply_enum(value: Any, enum_map: Any) -> Any:
|
||||
if not isinstance(enum_map, dict):
|
||||
return value
|
||||
key = str(value)
|
||||
return enum_map.get(key, enum_map.get(value, value))
|
||||
|
||||
|
||||
def _map_one(item: Any, field_mapping: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
|
||||
output: dict[str, Any] = {}
|
||||
errors: list[str] = []
|
||||
|
||||
for field_name, rule in field_mapping.items():
|
||||
if isinstance(rule, str):
|
||||
rule = {"path": rule}
|
||||
if not isinstance(rule, dict):
|
||||
errors.append(f"{field_name}: mapping rule must be an object or path string")
|
||||
continue
|
||||
|
||||
value = extract_path(item, rule.get("path"))
|
||||
if value is None and "default" in rule:
|
||||
value = rule.get("default")
|
||||
value = _apply_enum(value, rule.get("enum"))
|
||||
|
||||
try:
|
||||
value = _convert_value(value, rule.get("type"))
|
||||
except (TypeError, ValueError) as exc:
|
||||
errors.append(f"{field_name}: failed to convert value {value!r}: {exc}")
|
||||
continue
|
||||
|
||||
if value is not None or rule.get("include_null", False):
|
||||
output[field_name] = value
|
||||
|
||||
return output, errors
|
||||
|
||||
|
||||
def execute_mapping(
|
||||
payload: Any,
|
||||
mapping_json: dict[str, Any],
|
||||
target_schema: str | TargetSchema,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
schema = get_target_schema(target_schema) if isinstance(target_schema, str) else target_schema
|
||||
source = mapping_json.get("source") or {}
|
||||
fields = mapping_json.get("fields")
|
||||
if not isinstance(fields, dict) or not fields:
|
||||
raise MappingError("mapping_json.fields must be a non-empty object")
|
||||
|
||||
items_path = source.get("items_path") or mapping_json.get("items_path") or "$"
|
||||
items = extract_path(payload, items_path)
|
||||
if isinstance(items, dict):
|
||||
items = [items]
|
||||
elif not isinstance(items, list):
|
||||
items = []
|
||||
|
||||
if limit is not None:
|
||||
items = items[:limit]
|
||||
|
||||
mapped_records: list[dict[str, Any]] = []
|
||||
errors: list[dict[str, Any]] = []
|
||||
for index, item in enumerate(items):
|
||||
mapped, mapping_errors = _map_one(item, fields)
|
||||
validated, validation_errors = schema.validate_record(mapped)
|
||||
all_errors = mapping_errors + validation_errors
|
||||
if all_errors:
|
||||
errors.append({"index": index, "errors": all_errors, "record": mapped})
|
||||
continue
|
||||
if validated is not None:
|
||||
mapped_records.append(validated)
|
||||
|
||||
return {
|
||||
"target_schema": schema.key,
|
||||
"total_items": len(items),
|
||||
"mapped_count": len(mapped_records),
|
||||
"failed_count": len(errors),
|
||||
"records": mapped_records,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def build_heuristic_mapping(sample_payload: Any, target_schema_key: str) -> dict[str, Any]:
|
||||
schema = get_target_schema(target_schema_key)
|
||||
items_path = "$"
|
||||
sample_item = sample_payload
|
||||
if isinstance(sample_payload, dict):
|
||||
for key in ("data", "items", "results", "features", "vessels"):
|
||||
candidate = sample_payload.get(key)
|
||||
if isinstance(candidate, list) and candidate:
|
||||
items_path = f"$.{key}[*]"
|
||||
sample_item = candidate[0]
|
||||
break
|
||||
elif isinstance(sample_payload, list) and sample_payload:
|
||||
items_path = "$"
|
||||
sample_item = sample_payload[0]
|
||||
|
||||
available = _flatten_keys(sample_item if isinstance(sample_item, dict) else {})
|
||||
fields: dict[str, Any] = {}
|
||||
for field in schema.fields:
|
||||
candidate = _best_field_match(field.name, available)
|
||||
if candidate:
|
||||
fields[field.name] = {"path": f"$.{candidate}", "type": field.type}
|
||||
elif field.name == "data" and target_schema_key == "generic_records":
|
||||
fields[field.name] = {"path": "$", "type": "object"}
|
||||
elif not field.required:
|
||||
fields[field.name] = {"path": f"$.{field.name}", "type": field.type, "default": None}
|
||||
|
||||
return {
|
||||
"source": {"items_path": items_path},
|
||||
"fields": fields,
|
||||
"meta": {
|
||||
"generated_by": "heuristic",
|
||||
"requires_review": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _flatten_keys(payload: dict[str, Any], prefix: str = "") -> list[str]:
|
||||
keys: list[str] = []
|
||||
for key, value in payload.items():
|
||||
dotted = f"{prefix}.{key}" if prefix else str(key)
|
||||
keys.append(dotted)
|
||||
if isinstance(value, dict):
|
||||
keys.extend(_flatten_keys(value, dotted))
|
||||
return keys
|
||||
|
||||
|
||||
def _best_field_match(field_name: str, candidates: list[str]) -> str | None:
|
||||
aliases = {
|
||||
"lat": ("lat", "latitude", "y"),
|
||||
"lon": ("lon", "lng", "longitude", "x"),
|
||||
"mmsi": ("mmsi",),
|
||||
"sog": ("sog", "speed", "speedOverGround"),
|
||||
"cog": ("cog", "course", "courseOverGround"),
|
||||
"received_at": ("received_at", "timestamp", "time", "updated_at"),
|
||||
"observed_at": ("observed_at", "timestamp", "time", "updated_at"),
|
||||
"source_id": ("id", "source_id", "uuid"),
|
||||
}.get(field_name, (field_name,))
|
||||
|
||||
lowered = {candidate.lower(): candidate for candidate in candidates}
|
||||
for alias in aliases:
|
||||
if alias.lower() in lowered:
|
||||
return lowered[alias.lower()]
|
||||
for candidate in candidates:
|
||||
tail = candidate.split(".")[-1].lower()
|
||||
if tail in {alias.lower() for alias in aliases}:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return None
|
||||
|
||||
|
||||
async def persist_mapped_records(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
datasource_name: str,
|
||||
datasource_config_id: int,
|
||||
target_schema: str,
|
||||
records: list[dict[str, Any]],
|
||||
mapping_version: int,
|
||||
) -> int:
|
||||
"""Persist validated mapped records to the destination for a target schema."""
|
||||
if target_schema == "vessel_ais":
|
||||
from app.models.vessel import VesselPosition
|
||||
|
||||
for record in records:
|
||||
db.add(
|
||||
VesselPosition(
|
||||
mmsi=record["mmsi"],
|
||||
lat=record["lat"],
|
||||
lon=record["lon"],
|
||||
sog=record.get("sog"),
|
||||
cog=record.get("cog"),
|
||||
heading=record.get("heading"),
|
||||
received_at=_parse_datetime(record.get("received_at")) or datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
return len(records)
|
||||
|
||||
from app.models.collected_data import CollectedData
|
||||
|
||||
collected_at = datetime.now(UTC)
|
||||
for index, record in enumerate(records):
|
||||
if target_schema == "geo_points":
|
||||
source_id = record.get("source_id") or f"{datasource_config_id}:{index}"
|
||||
name = record.get("name")
|
||||
metadata = {
|
||||
"latitude": record.get("lat"),
|
||||
"longitude": record.get("lon"),
|
||||
"type": record.get("type"),
|
||||
"mapping_version": mapping_version,
|
||||
"target_schema": target_schema,
|
||||
**(record.get("metadata") or {}),
|
||||
}
|
||||
reference_date = _parse_datetime(record.get("observed_at"))
|
||||
else:
|
||||
source_id = record.get("source_id") or f"{datasource_config_id}:{index}"
|
||||
name = None
|
||||
metadata = {
|
||||
"data": record.get("data") or {},
|
||||
"mapping_version": mapping_version,
|
||||
"target_schema": target_schema,
|
||||
}
|
||||
reference_date = _parse_datetime(record.get("observed_at"))
|
||||
|
||||
db.add(
|
||||
CollectedData(
|
||||
source=datasource_name,
|
||||
source_id=str(source_id),
|
||||
entity_key=f"{datasource_name}:{source_id}",
|
||||
data_type=target_schema,
|
||||
name=name,
|
||||
title=name,
|
||||
extra_data=metadata,
|
||||
collected_at=collected_at,
|
||||
reference_date=reference_date,
|
||||
is_valid=1,
|
||||
is_current=True,
|
||||
change_type="created",
|
||||
change_summary={},
|
||||
)
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
return len(records)
|
||||
150
backend/app/services/llm_provider_catalog.py
Normal file
150
backend/app/services/llm_provider_catalog.py
Normal file
@@ -0,0 +1,150 @@
|
||||
"""LLM provider presets used by Settings and the runtime AI provider bridge."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
MODELS_DEV_URL = "https://models.dev/api.json"
|
||||
|
||||
|
||||
FALLBACK_LLM_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
|
||||
"minimax": {
|
||||
"provider": "minimax",
|
||||
"label": "MiniMax",
|
||||
"provider_api": "anthropic-messages",
|
||||
"base_url": "https://api.minimaxi.com/anthropic",
|
||||
"model": "MiniMax-M2.7",
|
||||
"models": ["MiniMax-M2.7", "MiniMax-M2.7-highspeed", "MiniMax-M2.5", "MiniMax-M2"],
|
||||
"api_key_env": "MINIMAX_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"openai": {
|
||||
"provider": "openai",
|
||||
"label": "OpenAI",
|
||||
"provider_api": "openai-completions",
|
||||
"base_url": "https://api.openai.com/v1",
|
||||
"model": "gpt-5.1",
|
||||
"models": ["gpt-5.1", "gpt-5.1-codex", "gpt-4.1", "gpt-4o"],
|
||||
"api_key_env": "OPENAI_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"anthropic": {
|
||||
"provider": "anthropic",
|
||||
"label": "Anthropic",
|
||||
"provider_api": "anthropic-messages",
|
||||
"base_url": "https://api.anthropic.com/v1",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"models": ["claude-sonnet-4-6", "claude-opus-4-5", "claude-3-5-haiku-20241022"],
|
||||
"api_key_env": "ANTHROPIC_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"deepseek": {
|
||||
"provider": "deepseek",
|
||||
"label": "DeepSeek",
|
||||
"provider_api": "openai-completions",
|
||||
"base_url": "https://api.deepseek.com/v1",
|
||||
"model": "deepseek-chat",
|
||||
"models": ["deepseek-chat", "deepseek-reasoner"],
|
||||
"api_key_env": "DEEPSEEK_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"alibaba": {
|
||||
"provider": "alibaba",
|
||||
"label": "Alibaba Qwen / DashScope",
|
||||
"provider_api": "openai-completions",
|
||||
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
"model": "qwen3-max",
|
||||
"models": ["qwen3-max", "qwen3.5-plus", "qwen-max", "qwen-plus"],
|
||||
"api_key_env": "DASHSCOPE_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"moonshotai": {
|
||||
"provider": "moonshotai",
|
||||
"label": "Moonshot AI / Kimi",
|
||||
"provider_api": "openai-completions",
|
||||
"base_url": "https://api.moonshot.ai/v1",
|
||||
"model": "kimi-k2.5",
|
||||
"models": ["kimi-k2.5", "kimi-k2-thinking", "kimi-k2-turbo-preview"],
|
||||
"api_key_env": "MOONSHOT_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"openrouter": {
|
||||
"provider": "openrouter",
|
||||
"label": "OpenRouter",
|
||||
"provider_api": "openai-completions",
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
"model": "openai/gpt-5.1",
|
||||
"models": ["openai/gpt-5.1", "anthropic/claude-sonnet-4.5", "qwen/qwen3-max"],
|
||||
"api_key_env": "OPENROUTER_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"ollama": {
|
||||
"provider": "ollama",
|
||||
"label": "Ollama Local",
|
||||
"provider_api": "ollama-generate",
|
||||
"base_url": "http://127.0.0.1:11434",
|
||||
"model": "qwen2.5:7b",
|
||||
"models": ["qwen2.5:7b", "llama3.1:8b", "mistral:7b"],
|
||||
"api_key_env": "",
|
||||
"source": "fallback",
|
||||
},
|
||||
}
|
||||
|
||||
MODELS_DEV_PROVIDER_KEYS = {
|
||||
"minimax": "minimax",
|
||||
"openai": "openai",
|
||||
"anthropic": "anthropic",
|
||||
"deepseek": "deepseek",
|
||||
"alibaba": "alibaba",
|
||||
"moonshotai": "moonshotai",
|
||||
"openrouter": "openrouter",
|
||||
}
|
||||
|
||||
|
||||
def list_fallback_llm_provider_presets() -> list[dict[str, Any]]:
|
||||
return [dict(value) for value in FALLBACK_LLM_PROVIDER_PRESETS.values()]
|
||||
|
||||
|
||||
def get_fallback_llm_provider_preset(provider: str) -> dict[str, Any]:
|
||||
key = provider.strip().lower()
|
||||
if key not in FALLBACK_LLM_PROVIDER_PRESETS:
|
||||
raise ValueError(f"Unsupported LLM provider preset: {provider}")
|
||||
return dict(FALLBACK_LLM_PROVIDER_PRESETS[key])
|
||||
|
||||
|
||||
async def refresh_llm_provider_preset(provider: str) -> dict[str, Any]:
|
||||
fallback = get_fallback_llm_provider_preset(provider)
|
||||
models_dev_key = MODELS_DEV_PROVIDER_KEYS.get(fallback["provider"])
|
||||
if not models_dev_key:
|
||||
return fallback
|
||||
|
||||
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
|
||||
response = await client.get(
|
||||
MODELS_DEV_URL,
|
||||
headers={"User-Agent": "Planet/1.0"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
catalog = response.json()
|
||||
|
||||
upstream = catalog.get(models_dev_key)
|
||||
if not isinstance(upstream, dict):
|
||||
return fallback
|
||||
|
||||
upstream_models = upstream.get("models") if isinstance(upstream.get("models"), dict) else {}
|
||||
model_ids = list(upstream_models.keys())[:80]
|
||||
base_url = upstream.get("api") or fallback["base_url"]
|
||||
if fallback["provider"] == "deepseek" and base_url == "https://api.deepseek.com":
|
||||
base_url = "https://api.deepseek.com/v1"
|
||||
|
||||
refreshed = {
|
||||
**fallback,
|
||||
"label": upstream.get("name") or fallback["label"],
|
||||
"base_url": base_url,
|
||||
"model": model_ids[0] if model_ids else fallback["model"],
|
||||
"models": model_ids or fallback["models"],
|
||||
"api_key_env": (upstream.get("env") or [fallback["api_key_env"]])[0],
|
||||
"source": MODELS_DEV_URL,
|
||||
}
|
||||
return refreshed
|
||||
Reference in New Issue
Block a user