release: bump version to 0.43.0

backend/app/api/v1/datasource_config.py

@@ -1,20 +1,34 @@
 """DataSourceConfig API for user-defined data sources"""
 
-from typing import Optional
+from typing import Any, Optional
 from datetime import datetime
 import base64
+import json
+import re
 from fastapi import APIRouter, Depends, HTTPException, status
 from sqlalchemy import select, func
 from sqlalchemy.ext.asyncio import AsyncSession
 from pydantic import BaseModel, Field
 import httpx
 
+from app.core.target_schema_registry import get_target_schema, list_target_schemas
 from app.db.session import get_db
 from app.models.user import User
 from app.models.datasource_config import DataSourceConfig
+from app.models.datasource_mapping import DataSourceMappingTemplate
 from app.core.security import get_current_user
 from app.core.cache import cache
 from app.core.time import to_iso8601_utc
+from app.schemas.ai import SituationalAnalysisRequest
+from app.services.ai_client import AIProviderClient, get_ai_provider_client
+from app.services.datasource_mapping import (
+    MappingError,
+    build_heuristic_mapping,
+    execute_mapping,
+    persist_mapped_records,
+    redact_for_llm,
+    stable_payload_hash,
+)
 
 router = APIRouter()
 
@@ -59,6 +73,44 @@ class DataSourceConfigResponse(BaseModel):
         from_attributes = True
 
 
+class CustomSampleRequest(BaseModel):
+    datasource_config_id: Optional[int] = None
+    config: Optional[DataSourceConfigCreate] = None
+    limit_bytes: int = Field(default=200000, ge=1000, le=1000000)
+
+
+class MappingProposeRequest(BaseModel):
+    sample_payload: Any
+    target_schema: str
+    use_ai: bool = True
+
+
+class MappingPreviewRequest(BaseModel):
+    sample_payload: Any
+    target_schema: str
+    mapping_json: dict
+    limit: int = Field(default=20, ge=1, le=100)
+
+
+class MappingTemplateCreate(BaseModel):
+    datasource_config_id: int
+    target_schema: str
+    mapping_json: dict
+    sample_payload: Any | None = None
+    sample_payload_hash: Optional[str] = None
+    validation_status: str = Field(default="draft", pattern="^(draft|valid|invalid)$")
+    is_active: bool = False
+
+
+class MappingTemplateUpdate(BaseModel):
+    target_schema: Optional[str] = None
+    mapping_json: Optional[dict] = None
+    sample_payload: Any | None = None
+    sample_payload_hash: Optional[str] = None
+    validation_status: Optional[str] = Field(default=None, pattern="^(draft|valid|invalid)$")
+    is_active: Optional[bool] = None
+
+
 async def test_endpoint(
     endpoint: str,
     auth_type: str,
@@ -96,6 +148,134 @@ async def test_endpoint(
         }
 
 
+def _build_request_headers(auth_type: str, auth_config: dict, headers: dict) -> dict[str, str]:
+    request_headers = {str(key): str(value) for key, value in (headers or {}).items()}
+    auth_type = str(auth_type or "none").lower()
+    auth_config = auth_config or {}
+
+    if auth_type == "bearer" and auth_config.get("token"):
+        request_headers["Authorization"] = f"Bearer {auth_config['token']}"
+    elif auth_type == "api_key" and auth_config.get("api_key"):
+        location = str(auth_config.get("in") or auth_config.get("location") or "header").lower()
+        if location != "query":
+            key_name = auth_config.get("key_name", "X-API-Key")
+            request_headers[str(key_name)] = str(auth_config["api_key"])
+    elif auth_type == "basic":
+        username = auth_config.get("username", "")
+        password = auth_config.get("password", "")
+        credentials = f"{username}:{password}"
+        encoded = base64.b64encode(credentials.encode()).decode()
+        request_headers["Authorization"] = f"Basic {encoded}"
+    return request_headers
+
+
+def _build_query_params(auth_type: str, auth_config: dict, config: dict) -> dict[str, Any]:
+    params = {}
+    candidate = (config or {}).get("params") or (config or {}).get("query_params")
+    if isinstance(candidate, dict):
+        params.update(candidate)
+
+    auth_type = str(auth_type or "none").lower()
+    auth_config = auth_config or {}
+    if auth_type == "api_key" and auth_config.get("api_key"):
+        location = str(auth_config.get("in") or auth_config.get("location") or "header").lower()
+        if location == "query":
+            key_name = auth_config.get("key_name") or auth_config.get("param_name") or "api_key"
+            params[str(key_name)] = auth_config["api_key"]
+    return params
+
+
+async def fetch_custom_sample_from_config(config: DataSourceConfig, limit_bytes: int) -> Any:
+    request_config = config.config or {}
+    method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
+    if method not in {"GET", "POST"}:
+        raise HTTPException(status_code=400, detail="Only GET and POST sample requests are supported.")
+
+    headers = _build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {})
+    params = _build_query_params(config.auth_type, config.auth_config or {}, request_config)
+    timeout = float(request_config.get("timeout", 30))
+    json_body = request_config.get("json_body")
+    if json_body is None and str(request_config.get("body_type") or "").lower() in {"json", ""}:
+        candidate = request_config.get("body")
+        if isinstance(candidate, (dict, list)):
+            json_body = candidate
+
+    async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+        response = await client.request(
+            method,
+            config.endpoint,
+            headers=headers,
+            params=params or None,
+            json=json_body,
+        )
+        response.raise_for_status()
+        content = response.content[:limit_bytes]
+        if "application/json" in response.headers.get("content-type", ""):
+            return json.loads(content.decode(response.encoding or "utf-8"))
+        return {"text": content.decode(response.encoding or "utf-8", errors="replace")}
+
+
+def _parse_mapping_from_ai_text(content: str) -> dict[str, Any] | None:
+    if not content:
+        return None
+
+    candidates = [content]
+    fenced = re.findall(r"```(?:json)?\s*(\{.*?\})\s*```", content, flags=re.DOTALL)
+    candidates = fenced + candidates
+    for candidate in candidates:
+        try:
+            parsed = json.loads(candidate)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, dict) and isinstance(parsed.get("fields"), dict):
+            return parsed
+    return None
+
+
+async def _get_config_for_sample(
+    payload: CustomSampleRequest,
+    db: AsyncSession,
+) -> DataSourceConfig:
+    if payload.datasource_config_id is not None:
+        result = await db.execute(
+            select(DataSourceConfig).where(DataSourceConfig.id == payload.datasource_config_id)
+        )
+        config = result.scalar_one_or_none()
+        if not config:
+            raise HTTPException(status_code=404, detail="Configuration not found")
+        return config
+
+    if payload.config is None:
+        raise HTTPException(status_code=400, detail="datasource_config_id or config is required")
+
+    config_data = payload.config
+    return DataSourceConfig(
+        name=config_data.name,
+        description=config_data.description,
+        source_type=config_data.source_type,
+        endpoint=config_data.endpoint,
+        auth_type=config_data.auth_type,
+        auth_config=config_data.auth_config,
+        headers=config_data.headers,
+        config=config_data.config,
+    )
+
+
+def serialize_mapping_template(template: DataSourceMappingTemplate) -> dict[str, Any]:
+    return {
+        "id": template.id,
+        "datasource_config_id": template.datasource_config_id,
+        "target_schema": template.target_schema,
+        "mapping_json": template.mapping_json,
+        "sample_payload_hash": template.sample_payload_hash,
+        "validation_status": template.validation_status,
+        "version": template.version,
+        "is_active": template.is_active,
+        "created_at": to_iso8601_utc(template.created_at),
+        "updated_at": to_iso8601_utc(template.updated_at),
+    }
+
+
 @router.get("/configs")
 async def list_configs(
     active_only: bool = False,
@@ -345,3 +525,311 @@ async def list_all_datasources(
         )
 
     return {"total": len(result), "data": result}
+
+
+@router.post("/custom/sample")
+async def fetch_custom_sample(
+    payload: CustomSampleRequest,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    """Fetch a sample payload for a saved or draft custom data source."""
+    config = await _get_config_for_sample(payload, db)
+    try:
+        sample = await fetch_custom_sample_from_config(config, payload.limit_bytes)
+    except httpx.HTTPStatusError as exc:
+        raise HTTPException(
+            status_code=exc.response.status_code,
+            detail=f"Sample request failed: HTTP {exc.response.status_code}",
+        ) from exc
+    except httpx.HTTPError as exc:
+        raise HTTPException(status_code=502, detail=f"Sample request failed: {exc}") from exc
+
+    return {
+        "success": True,
+        "sample_payload": sample,
+        "sample_payload_hash": stable_payload_hash(sample),
+        "redacted_preview": redact_for_llm(sample),
+    }
+
+
+@router.get("/target-schemas")
+async def get_datasource_target_schemas(
+    current_user: User = Depends(get_current_user),
+):
+    """List target schemas available for custom datasource mapping."""
+    return {"data": list_target_schemas()}
+
+
+@router.post("/mappings/propose")
+async def propose_datasource_mapping(
+    payload: MappingProposeRequest,
+    current_user: User = Depends(get_current_user),
+    ai_client: AIProviderClient = Depends(get_ai_provider_client),
+):
+    """Generate a mapping draft for a sample payload and target schema."""
+    schema = get_target_schema(payload.target_schema)
+    redacted_sample = redact_for_llm(payload.sample_payload)
+    fallback_mapping = build_heuristic_mapping(redacted_sample, payload.target_schema)
+
+    ai_error: str | None = None
+    mapping = fallback_mapping
+    generated_by = "heuristic"
+    if payload.use_ai:
+        try:
+            response = await ai_client.analyze(
+                SituationalAnalysisRequest(
+                    title=f"Generate datasource mapping for {schema.key}",
+                    objective=(
+                        "Return only JSON for a deterministic mapping DSL. "
+                        "The JSON must contain source.items_path and fields. "
+                        "Do not include prose or code."
+                    ),
+                    context={
+                        "target_schema": schema.to_dict(),
+                        "sample_payload": redacted_sample,
+                        "mapping_dsl_example": fallback_mapping,
+                    },
+                    observations=[
+                        "Use JSONPath-like paths beginning with $.",
+                        "Never generate executable code.",
+                        "Use field types from the target schema.",
+                    ],
+                    constraints=[
+                        "Return a single JSON object.",
+                        "Do not include credentials or secrets.",
+                        "Mark uncertain optional fields with default null.",
+                    ],
+                )
+            )
+            parsed = _parse_mapping_from_ai_text(response.content)
+            if parsed:
+                mapping = parsed
+                generated_by = "ai_provider"
+            else:
+                ai_error = "AI provider did not return a valid mapping JSON object."
+        except HTTPException as exc:
+            ai_error = str(exc.detail)
+
+    mapping.setdefault("meta", {})
+    if isinstance(mapping["meta"], dict):
+        mapping["meta"].update(
+            {
+                "generated_by": generated_by,
+                "requires_review": True,
+                "ai_error": ai_error,
+            }
+        )
+
+    return {
+        "target_schema": schema.to_dict(),
+        "mapping_json": mapping,
+        "sample_payload_hash": stable_payload_hash(payload.sample_payload),
+        "redacted_sample_payload": redacted_sample,
+    }
+
+
+@router.post("/mappings/preview")
+async def preview_datasource_mapping(
+    payload: MappingPreviewRequest,
+    current_user: User = Depends(get_current_user),
+):
+    """Preview deterministic mapping output for a sample payload."""
+    try:
+        preview = execute_mapping(
+            payload.sample_payload,
+            payload.mapping_json,
+            payload.target_schema,
+            limit=payload.limit,
+        )
+    except (MappingError, ValueError) as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    return {
+        "success": preview["failed_count"] == 0,
+        "preview": preview,
+        "sample_payload_hash": stable_payload_hash(payload.sample_payload),
+    }
+
+
+@router.get("/mappings")
+async def list_datasource_mappings(
+    datasource_config_id: Optional[int] = None,
+    active_only: bool = False,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    """List saved mapping templates."""
+    query = select(DataSourceMappingTemplate).order_by(
+        DataSourceMappingTemplate.datasource_config_id,
+        DataSourceMappingTemplate.version.desc(),
+    )
+    if datasource_config_id is not None:
+        query = query.where(DataSourceMappingTemplate.datasource_config_id == datasource_config_id)
+    if active_only:
+        query = query.where(DataSourceMappingTemplate.is_active.is_(True))
+
+    result = await db.execute(query)
+    mappings = result.scalars().all()
+    return {"total": len(mappings), "data": [serialize_mapping_template(item) for item in mappings]}
+
+
+@router.post("/mappings")
+async def create_datasource_mapping(
+    payload: MappingTemplateCreate,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    """Save a mapping template for a datasource config."""
+    get_target_schema(payload.target_schema)
+    datasource = await db.get(DataSourceConfig, payload.datasource_config_id)
+    if not datasource:
+        raise HTTPException(status_code=404, detail="Configuration not found")
+
+    if payload.sample_payload is not None:
+        try:
+            execute_mapping(payload.sample_payload, payload.mapping_json, payload.target_schema, limit=100)
+        except (MappingError, ValueError) as exc:
+            raise HTTPException(status_code=400, detail=f"Mapping validation failed: {exc}") from exc
+
+    result = await db.execute(
+        select(func.max(DataSourceMappingTemplate.version)).where(
+            DataSourceMappingTemplate.datasource_config_id == payload.datasource_config_id,
+            DataSourceMappingTemplate.target_schema == payload.target_schema,
+        )
+    )
+    next_version = int(result.scalar() or 0) + 1
+
+    if payload.is_active:
+        await db.execute(
+            DataSourceMappingTemplate.__table__.update()
+            .where(DataSourceMappingTemplate.datasource_config_id == payload.datasource_config_id)
+            .values(is_active=False)
+        )
+
+    template = DataSourceMappingTemplate(
+        datasource_config_id=payload.datasource_config_id,
+        target_schema=payload.target_schema,
+        mapping_json=payload.mapping_json,
+        sample_payload_hash=payload.sample_payload_hash
+        or (stable_payload_hash(payload.sample_payload) if payload.sample_payload is not None else None),
+        validation_status=payload.validation_status,
+        version=next_version,
+        is_active=payload.is_active,
+    )
+    db.add(template)
+    await db.commit()
+    await db.refresh(template)
+    return {"message": "Mapping template saved successfully", "data": serialize_mapping_template(template)}
+
+
+@router.put("/mappings/{mapping_id}")
+async def update_datasource_mapping(
+    mapping_id: int,
+    payload: MappingTemplateUpdate,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    """Update a mapping template in place."""
+    template = await db.get(DataSourceMappingTemplate, mapping_id)
+    if not template:
+        raise HTTPException(status_code=404, detail="Mapping template not found")
+
+    target_schema = payload.target_schema or template.target_schema
+    mapping_json = payload.mapping_json or template.mapping_json
+    get_target_schema(target_schema)
+    if payload.sample_payload is not None:
+        try:
+            execute_mapping(payload.sample_payload, mapping_json, target_schema, limit=100)
+        except (MappingError, ValueError) as exc:
+            raise HTTPException(status_code=400, detail=f"Mapping validation failed: {exc}") from exc
+
+    if payload.is_active is True:
+        await db.execute(
+            DataSourceMappingTemplate.__table__.update()
+            .where(DataSourceMappingTemplate.datasource_config_id == template.datasource_config_id)
+            .where(DataSourceMappingTemplate.id != template.id)
+            .values(is_active=False)
+        )
+
+    template.target_schema = target_schema
+    template.mapping_json = mapping_json
+    if payload.sample_payload_hash is not None:
+        template.sample_payload_hash = payload.sample_payload_hash
+    elif payload.sample_payload is not None:
+        template.sample_payload_hash = stable_payload_hash(payload.sample_payload)
+    if payload.validation_status is not None:
+        template.validation_status = payload.validation_status
+    if payload.is_active is not None:
+        template.is_active = payload.is_active
+
+    await db.commit()
+    await db.refresh(template)
+    return {"message": "Mapping template updated successfully", "data": serialize_mapping_template(template)}
+
+
+@router.post("/{config_id}/run-mapped")
+async def run_mapped_datasource(
+    config_id: int,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    """Run a saved custom datasource through its active deterministic mapping."""
+    datasource = await db.get(DataSourceConfig, config_id)
+    if not datasource:
+        raise HTTPException(status_code=404, detail="Configuration not found")
+
+    result = await db.execute(
+        select(DataSourceMappingTemplate)
+        .where(DataSourceMappingTemplate.datasource_config_id == config_id)
+        .where(DataSourceMappingTemplate.is_active.is_(True))
+        .order_by(DataSourceMappingTemplate.version.desc())
+        .limit(1)
+    )
+    mapping = result.scalar_one_or_none()
+    if not mapping:
+        raise HTTPException(status_code=404, detail="No active mapping template found")
+
+    try:
+        sample = await fetch_custom_sample_from_config(datasource, 5_000_000)
+        mapped = execute_mapping(sample, mapping.mapping_json, mapping.target_schema)
+    except httpx.HTTPStatusError as exc:
+        raise HTTPException(
+            status_code=exc.response.status_code,
+            detail=f"Datasource request failed: HTTP {exc.response.status_code}",
+        ) from exc
+    except httpx.HTTPError as exc:
+        raise HTTPException(status_code=502, detail=f"Datasource request failed: {exc}") from exc
+    except (MappingError, ValueError) as exc:
+        raise HTTPException(status_code=400, detail=f"Mapping failed: {exc}") from exc
+
+    if mapped["failed_count"] > 0:
+        return {
+            "status": "failed",
+            "datasource_config_id": config_id,
+            "mapping_id": mapping.id,
+            "mapping_version": mapping.version,
+            "target_schema": mapping.target_schema,
+            "mapped_count": mapped["mapped_count"],
+            "failed_count": mapped["failed_count"],
+            "errors": mapped["errors"][:20],
+        }
+
+    written_count = await persist_mapped_records(
+        db,
+        datasource_name=datasource.name,
+        datasource_config_id=datasource.id,
+        target_schema=mapping.target_schema,
+        records=mapped["records"],
+        mapping_version=mapping.version,
+    )
+    return {
+        "status": "success",
+        "datasource_config_id": config_id,
+        "mapping_id": mapping.id,
+        "mapping_version": mapping.version,
+        "target_schema": mapping.target_schema,
+        "fetched_count": mapped["total_items"],
+        "mapped_count": mapped["mapped_count"],
+        "written_count": written_count,
+    }

backend/app/api/v1/datasources.py

@@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.core.time import to_iso8601_utc
 from app.core.security import get_current_user
 from app.core.data_sources import get_data_sources_config
+from app.core.datasource_defaults import DEFAULT_DATASOURCES
 from app.db.session import get_db
 from app.models.collected_data import CollectedData
 from app.models.data_snapshot import DataSnapshot
@@ -35,6 +36,17 @@ def format_frequency_label(minutes: int) -> str:
     return f"{minutes}m"
 
 
+def datasource_metadata(source: str) -> dict:
+    info = DEFAULT_DATASOURCES.get(source, {})
+    return {
+        "display_name": info.get("display_name") or info.get("name") or source,
+        "is_free": bool(info.get("is_free", True)),
+        "requires_credentials": bool(info.get("requires_credentials", False)),
+        "credential_provider": info.get("credential_provider"),
+        "credential_status": info.get("credential_status", "none"),
+    }
+
+
 def is_due_for_collection(datasource: DataSource, now: datetime) -> bool:
     if datasource.last_run_at is None:
         return True
@@ -72,31 +84,6 @@ async def _load_latest_running_tasks(
     return {task.datasource_id: task for task in result.scalars().all()}
 
 
-async def _load_latest_completed_tasks(
-    db: AsyncSession,
-    datasource_ids: list[int],
-) -> dict[int, CollectionTask]:
-    if not datasource_ids:
-        return {}
-
-    ranked_tasks = (
-        select(
-            CollectionTask.id.label("task_id"),
-            _task_rank_column(CollectionTask.completed_at),
-        )
-        .where(CollectionTask.datasource_id.in_(datasource_ids))
-        .where(CollectionTask.completed_at.isnot(None))
-        .where(CollectionTask.status.in_(("success", "failed", "cancelled")))
-        .subquery()
-    )
-    result = await db.execute(
-        select(CollectionTask)
-        .join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id)
-        .where(ranked_tasks.c.row_num == 1)
-    )
-    return {task.datasource_id: task for task in result.scalars().all()}
-
-
 async def _load_latest_task_ids(
     db: AsyncSession,
     datasource_ids: list[int],
@@ -123,21 +110,6 @@ async def _load_latest_task_ids(
     return {datasource_id: task_id for datasource_id, task_id in result.all()}
 
 
-async def _load_datasource_data_counts(
-    db: AsyncSession,
-    sources: list[str],
-) -> dict[str, int]:
-    if not sources:
-        return {}
-
-    result = await db.execute(
-        select(CollectedData.source, func.count(CollectedData.id))
-        .where(CollectedData.source.in_(sources))
-        .group_by(CollectedData.source)
-    )
-    return {source: count for source, count in result.all()}
-
-
 async def _load_datasource_endpoint_overrides(
     db: AsyncSession,
     sources: list[str],
@@ -161,7 +133,7 @@ async def _load_datasource_endpoint_overrides(
 async def _load_datasource_list_context(
     db: AsyncSession,
     datasources: list[DataSource],
-) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, int], dict[str, str]]:
+) -> tuple[dict[int, CollectionTask], dict[str, str]]:
     datasource_ids = [datasource.id for datasource in datasources]
     sources = [datasource.source for datasource in datasources]
 
@@ -185,10 +157,8 @@ async def _load_datasource_list_context(
     if stale_datasource_ids:
         running_tasks = await _load_latest_running_tasks(db, datasource_ids)
 
-    completed_tasks = await _load_latest_completed_tasks(db, datasource_ids)
-    data_counts = await _load_datasource_data_counts(db, sources)
     endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
-    return running_tasks, completed_tasks, data_counts, endpoint_overrides
+    return running_tasks, endpoint_overrides
 
 
 async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
@@ -401,27 +371,19 @@ async def list_datasources(
 
     collector_list = []
     config = get_data_sources_config()
-    running_tasks, completed_tasks, data_counts, endpoint_overrides = await _load_datasource_list_context(
-        db,
-        datasources,
-    )
+    running_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
     for datasource in datasources:
         running_task = running_tasks.get(datasource.id)
-        last_task = completed_tasks.get(datasource.id)
-        endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(
-            datasource.source,
-        )
-        data_count = data_counts.get(datasource.source, 0)
-
-        last_run_at = datasource.last_run_at or (last_task.completed_at if last_task else None)
-        last_run = to_iso8601_utc(last_run_at)
-        last_status = datasource.last_status or (last_task.status if last_task else None)
+        endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
+        last_run_at = datasource.last_run_at
+        last_status = datasource.last_status
 
         collector_list.append(
             {
                 "id": datasource.id,
                 "source": datasource.source,
                 "name": datasource.name,
+                **datasource_metadata(datasource.source),
                 "module": datasource.module,
                 "priority": datasource.priority,
                 "frequency": format_frequency_label(datasource.frequency_minutes),
@@ -429,11 +391,9 @@ async def list_datasources(
                 "is_active": datasource.is_active,
                 "collector_class": datasource.collector_class,
                 "endpoint": endpoint,
-                "last_run": last_run,
+                "last_run": to_iso8601_utc(last_run_at),
                 "last_run_at": to_iso8601_utc(last_run_at),
                 "last_status": last_status,
-                "last_records_processed": last_task.records_processed if last_task else None,
-                "data_count": data_count,
                 "is_running": running_task is not None,
                 "task_id": running_task.id if running_task else None,
                 "progress": running_task.progress if running_task else None,
@@ -576,6 +536,7 @@ async def get_datasource(
     return {
         "id": datasource.id,
         "name": datasource.name,
+        **datasource_metadata(datasource.source),
         "module": datasource.module,
         "priority": datasource.priority,
         "frequency": format_frequency_label(datasource.frequency_minutes),

backend/app/api/v1/settings.py

@@ -9,10 +9,19 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.security import get_current_user
 from app.core.time import to_iso8601_utc
+from app.core.config import settings as app_settings
+from app.core.data_sources import get_data_sources_config
+from app.core.datasource_defaults import DEFAULT_DATASOURCES
 from app.db.session import get_db
 from app.models.datasource import DataSource
+from app.models.datasource_config import DataSourceConfig
 from app.models.system_setting import SystemSetting
 from app.models.user import User
+from app.services.llm_provider_catalog import (
+    get_fallback_llm_provider_preset,
+    list_fallback_llm_provider_presets,
+    refresh_llm_provider_preset,
+)
 from app.services.scheduler import sync_datasource_job
 from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings
 
@@ -39,6 +48,21 @@ DEFAULT_SETTINGS = {
         "password_policy": "medium",
     },
     "tv": DEFAULT_TV_SETTINGS,
+    "external_integrations": {
+        "ai_provider": {
+            "service_url": "",
+            "service_token": "",
+            "provider": "minimax",
+            "provider_api": "anthropic-messages",
+            "base_url": "https://api.minimaxi.com/anthropic",
+            "model": "MiniMax-M2.7",
+            "api_key": "",
+            "max_tokens": 1200,
+            "anthropic_version": "2023-06-01",
+            "timeout_seconds": 60,
+            "retry_attempts": 2,
+        }
+    },
 }
 
 
@@ -96,6 +120,34 @@ class TVSettingsUpdate(BaseModel):
     sources: list[TVStreamSourceUpdate] = Field(default_factory=list)
 
 
+class AIProviderIntegrationUpdate(BaseModel):
+    service_url: str = ""
+    service_token: Optional[str] = None
+    provider: str = Field(default="minimax", max_length=80)
+    provider_api: str = Field(default="anthropic-messages", max_length=80)
+    base_url: str = Field(default="", max_length=500)
+    model: str = Field(default="", max_length=200)
+    api_key: Optional[str] = None
+    max_tokens: int = Field(default=1200, ge=1, le=200000)
+    anthropic_version: str = Field(default="2023-06-01", max_length=40)
+    timeout_seconds: int = Field(default=60, ge=5, le=600)
+    retry_attempts: int = Field(default=2, ge=1, le=10)
+    clear_service_token: bool = False
+    clear_api_key: bool = False
+
+
+class BarentsWatchIntegrationUpdate(BaseModel):
+    endpoint: str = ""
+    client_id: str = ""
+    client_secret: Optional[str] = None
+    clear_client_secret: bool = False
+
+
+class ExternalIntegrationsUpdate(BaseModel):
+    ai_provider: AIProviderIntegrationUpdate
+    barentswatch: BarentsWatchIntegrationUpdate
+
+
 def merge_with_defaults(category: str, payload: Optional[dict]) -> dict:
     merged = deepcopy(DEFAULT_SETTINGS[category])
     if payload:
@@ -146,6 +198,158 @@ async def save_setting_payload(db: AsyncSession, category: str, payload: dict) -
     return merge_with_defaults(category, record.payload)
 
 
+def _mask_secret(value: Optional[str]) -> dict:
+    if not value:
+        return {"configured": False, "preview": ""}
+    text = str(value)
+    if "-" in text:
+        prefix = text.split("-", 1)[0] + "-"
+        preview = prefix + ("*" * max(len(text) - len(prefix), 1))
+    else:
+        prefix_len = min(4, len(text))
+        preview = text[:prefix_len] + ("*" * max(len(text) - prefix_len, 1))
+    return {"configured": True, "preview": preview}
+
+
+async def get_runtime_ai_provider_config(db: AsyncSession) -> dict:
+    runtime_record = await get_setting_record(db, "external_integrations")
+    payload = merge_with_defaults(
+        "external_integrations",
+        runtime_record.payload if runtime_record else None,
+    )
+    ai_payload = payload.get("ai_provider") or {}
+    has_runtime_llm_config = bool(
+        runtime_record
+        and isinstance(runtime_record.payload, dict)
+        and isinstance(runtime_record.payload.get("ai_provider"), dict)
+    )
+    return {
+        "service_url": ai_payload.get("service_url") or app_settings.AI_PROVIDER_SERVICE_URL,
+        "service_token": ai_payload.get("service_token") or app_settings.AI_PROVIDER_SERVICE_TOKEN,
+        "timeout_seconds": int(
+            ai_payload.get("timeout_seconds") or app_settings.AI_PROVIDER_TIMEOUT_SECONDS
+        ),
+        "retry_attempts": int(
+            ai_payload.get("retry_attempts") or app_settings.AI_PROVIDER_RETRY_ATTEMPTS
+        ),
+        "llm_config": {
+            "provider": ai_payload.get("provider") or "minimax",
+            "provider_api": ai_payload.get("provider_api") or "anthropic-messages",
+            "base_url": ai_payload.get("base_url") or "https://api.minimaxi.com/anthropic",
+            "model": ai_payload.get("model") or "MiniMax-M2.7",
+            "api_key": ai_payload.get("api_key") or "",
+            "max_tokens": int(ai_payload.get("max_tokens") or 1200),
+            "anthropic_version": ai_payload.get("anthropic_version") or "2023-06-01",
+        } if has_runtime_llm_config else {},
+    }
+
+
+async def get_barentswatch_config_record(db: AsyncSession) -> Optional[DataSourceConfig]:
+    result = await db.execute(
+        select(DataSourceConfig)
+        .where(DataSourceConfig.name == "barentswatch_vessels")
+        .where(DataSourceConfig.is_active.is_(True))
+    )
+    return result.scalar_one_or_none()
+
+
+async def serialize_external_integrations(db: AsyncSession) -> dict:
+    ai_config = await get_runtime_ai_provider_config(db)
+    runtime_setting = await get_setting_record(db, "external_integrations")
+    display_llm_config = ai_config["llm_config"] or DEFAULT_SETTINGS["external_integrations"]["ai_provider"]
+    barentswatch_record = await get_barentswatch_config_record(db)
+    yaml_config = get_data_sources_config()
+    barentswatch_auth = barentswatch_record.auth_config if barentswatch_record else {}
+    barentswatch_auth = barentswatch_auth or {}
+    return {
+        "ai_provider": {
+            "service_url": ai_config["service_url"],
+            "service_token": _mask_secret(ai_config["service_token"]),
+            "provider": display_llm_config.get("provider") or "minimax",
+            "provider_api": display_llm_config.get("provider_api") or "anthropic-messages",
+            "base_url": display_llm_config.get("base_url") or "https://api.minimaxi.com/anthropic",
+            "model": display_llm_config.get("model") or "MiniMax-M2.7",
+            "api_key": _mask_secret(display_llm_config.get("api_key")),
+            "max_tokens": int(display_llm_config.get("max_tokens") or 1200),
+            "anthropic_version": display_llm_config.get("anthropic_version") or "2023-06-01",
+            "timeout_seconds": ai_config["timeout_seconds"],
+            "retry_attempts": ai_config["retry_attempts"],
+            "source": "runtime" if runtime_setting else "env",
+        },
+        "barentswatch": {
+            "endpoint": (
+                barentswatch_record.endpoint
+                if barentswatch_record and barentswatch_record.endpoint
+                else yaml_config.get_yaml_url("barentswatch_vessels")
+            ),
+            "client_id": barentswatch_auth.get("client_id") or "",
+            "client_secret": _mask_secret(barentswatch_auth.get("client_secret")),
+            "source": "datasource_config" if barentswatch_record else "default",
+        },
+    }
+
+
+async def save_external_integrations_payload(
+    db: AsyncSession,
+    update: ExternalIntegrationsUpdate,
+) -> dict:
+    current_payload = await get_setting_payload(db, "external_integrations")
+    current_ai = current_payload.get("ai_provider") or {}
+    ai_payload = {
+        "service_url": update.ai_provider.service_url.strip()
+        or app_settings.AI_PROVIDER_SERVICE_URL,
+        "service_token": current_ai.get("service_token") or "",
+        "provider": update.ai_provider.provider.strip() or "minimax",
+        "provider_api": update.ai_provider.provider_api.strip() or "anthropic-messages",
+        "base_url": update.ai_provider.base_url.strip(),
+        "model": update.ai_provider.model.strip(),
+        "api_key": current_ai.get("api_key") or "",
+        "max_tokens": update.ai_provider.max_tokens,
+        "anthropic_version": update.ai_provider.anthropic_version.strip() or "2023-06-01",
+        "timeout_seconds": update.ai_provider.timeout_seconds,
+        "retry_attempts": update.ai_provider.retry_attempts,
+    }
+    if update.ai_provider.clear_service_token:
+        ai_payload["service_token"] = ""
+    elif update.ai_provider.service_token not in (None, ""):
+        ai_payload["service_token"] = update.ai_provider.service_token
+    if update.ai_provider.clear_api_key:
+        ai_payload["api_key"] = ""
+    elif update.ai_provider.api_key not in (None, ""):
+        ai_payload["api_key"] = update.ai_provider.api_key
+
+    await save_setting_payload(db, "external_integrations", {"ai_provider": ai_payload})
+
+    default_endpoint = get_data_sources_config().get_yaml_url("barentswatch_vessels")
+    barentswatch_record = await get_barentswatch_config_record(db)
+    if barentswatch_record is None:
+        barentswatch_record = DataSourceConfig(
+            name="barentswatch_vessels",
+            description="BarentsWatch Live AIS credentials",
+            source_type="api",
+            endpoint=update.barentswatch.endpoint.strip() or default_endpoint,
+            auth_type="oauth_client_credentials",
+            auth_config={},
+            headers={},
+            config={},
+            is_active=True,
+        )
+        db.add(barentswatch_record)
+
+    current_auth = dict(barentswatch_record.auth_config or {})
+    if update.barentswatch.clear_client_secret:
+        current_auth.pop("client_secret", None)
+    elif update.barentswatch.client_secret not in (None, ""):
+        current_auth["client_secret"] = update.barentswatch.client_secret
+    current_auth["client_id"] = update.barentswatch.client_id.strip()
+    barentswatch_record.endpoint = update.barentswatch.endpoint.strip() or default_endpoint
+    barentswatch_record.auth_type = "oauth_client_credentials"
+    barentswatch_record.auth_config = current_auth
+    await db.commit()
+
+    return await serialize_external_integrations(db)
+
+
 def format_frequency_label(minutes: int) -> str:
     if minutes % 1440 == 0:
         return f"{minutes // 1440}d"
@@ -155,9 +359,11 @@ def format_frequency_label(minutes: int) -> str:
 
 
 def serialize_collector(datasource: DataSource) -> dict:
+    defaults = DEFAULT_DATASOURCES.get(datasource.source, {})
     return {
         "id": datasource.id,
         "name": datasource.name,
+        "display_name": defaults.get("display_name") or datasource.name,
         "source": datasource.source,
         "module": datasource.module,
         "priority": datasource.priority,
@@ -167,6 +373,10 @@ def serialize_collector(datasource: DataSource) -> dict:
         "last_run_at": to_iso8601_utc(datasource.last_run_at),
         "last_status": datasource.last_status,
         "next_run_at": to_iso8601_utc(datasource.next_run_at),
+        "is_free": bool(defaults.get("is_free", True)),
+        "requires_credentials": bool(defaults.get("requires_credentials", False)),
+        "credential_provider": defaults.get("credential_provider"),
+        "credential_status": defaults.get("credential_status", "none"),
     }
 
 
@@ -243,6 +453,46 @@ async def update_tv_settings(
     return {"status": "updated", "tv": normalize_tv_settings(saved)}
 
 
+@router.get("/integrations")
+async def get_external_integrations(
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    return {"integrations": await serialize_external_integrations(db)}
+
+
+@router.get("/integrations/ai-provider/presets")
+async def get_ai_provider_presets(
+    current_user: User = Depends(get_current_user),
+):
+    return {"data": list_fallback_llm_provider_presets()}
+
+
+@router.post("/integrations/ai-provider/presets/{provider}/refresh")
+async def refresh_ai_provider_preset(
+    provider: str,
+    current_user: User = Depends(get_current_user),
+):
+    try:
+        return {"data": await refresh_llm_provider_preset(provider)}
+    except ValueError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+    except Exception as exc:
+        fallback = get_fallback_llm_provider_preset(provider)
+        fallback["refresh_error"] = str(exc)
+        return {"data": fallback}
+
+
+@router.put("/integrations")
+async def update_external_integrations(
+    payload: ExternalIntegrationsUpdate,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    saved = await save_external_integrations_payload(db, payload)
+    return {"status": "updated", "integrations": saved}
+
+
 @router.get("/collectors")
 async def get_collector_settings(
     current_user: User = Depends(get_current_user),
@@ -289,6 +539,7 @@ async def get_all_settings(
         "notifications": setting_payloads["notifications"],
         "security": setting_payloads["security"],
         "tv": await get_tv_settings_payload(db),
+        "integrations": await serialize_external_integrations(db),
         "collectors": [serialize_collector(datasource) for datasource in datasources],
         "generated_at": to_iso8601_utc(datetime.now(UTC)),
     }

backend/app/api/v1/visualization.py

@@ -4,7 +4,7 @@ Unified API for all visualization data sources.
 Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium.
 """
 
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 import math
 import httpx
 from fastapi import APIRouter, HTTPException, Depends, Query, Response
@@ -20,6 +20,7 @@ from app.db.session import get_db
 from app.models.bgp_anomaly import BGPAnomaly
 from app.models.bgp_incident import BGPIncident
 from app.models.collected_data import CollectedData
+from app.models.vessel import VesselPosition, VesselStatic
 from app.services.bgp_collectors import build_bgp_collector_coverage
 from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
 from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
@@ -511,7 +512,7 @@ def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str
     if unit in {"pflop/s", "pflops", "pflop"}:
         normalized_tflops = capacity_value * 1000
     elif unit in {"gflop/s", "gflops", "gflop"}:
-        normalized_tflops = capacity_value / 1000
+        normalized_tflops = capacity_value
     else:
         normalized_tflops = capacity_value
 
@@ -609,6 +610,108 @@ def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str
     return {"type": "FeatureCollection", "features": features}
 
 
+VESSEL_TYPE_FILTERS = {
+    "cargo": lambda props: str(props.get("vessel_type_name", "")).lower() == "cargo"
+    or 70 <= int(props.get("vessel_type") or -1) <= 79,
+    "tanker": lambda props: str(props.get("vessel_type_name", "")).lower() == "tanker"
+    or 80 <= int(props.get("vessel_type") or -1) <= 89,
+    "passenger": lambda props: str(props.get("vessel_type_name", "")).lower() == "passenger"
+    or 60 <= int(props.get("vessel_type") or -1) <= 69,
+    "fishing": lambda props: str(props.get("vessel_type_name", "")).lower() == "fishing"
+    or int(props.get("vessel_type") or -1) == 30,
+    "military": lambda props: str(props.get("vessel_type_name", "")).lower() == "military"
+    or int(props.get("vessel_type") or -1) == 35,
+    "other": lambda props: str(props.get("vessel_type_name", "")).lower()
+    not in {"cargo", "tanker", "passenger", "fishing", "military"},
+}
+
+
+def convert_vessels_to_geojson(rows: List[Any]) -> Dict[str, Any]:
+    features = []
+    for position, static in rows:
+        if position.lat is None or position.lon is None:
+            continue
+        props = {
+            "mmsi": position.mmsi,
+            "name": getattr(static, "name", None) or f"MMSI {position.mmsi}",
+            "callsign": getattr(static, "callsign", None),
+            "imo": getattr(static, "imo", None),
+            "vessel_type": getattr(static, "vessel_type", None),
+            "vessel_type_name": getattr(static, "vessel_type_name", None) or "Other",
+            "flag": getattr(static, "flag", None),
+            "length": getattr(static, "length", None),
+            "width": getattr(static, "width", None),
+            "draught": getattr(static, "draught", None),
+            "sog": position.sog,
+            "cog": position.cog,
+            "heading": position.heading,
+            "nav_status": position.nav_status,
+            "received_at": to_iso8601_utc(position.received_at),
+            "data_type": "vessel",
+        }
+        features.append(
+            {
+                "type": "Feature",
+                "id": position.mmsi,
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [position.lon, position.lat],
+                },
+                "properties": props,
+            }
+        )
+
+    return {"type": "FeatureCollection", "features": features}
+
+
+def _parse_bbox(value: Optional[str]) -> tuple[float, float, float, float] | None:
+    if not value:
+        return None
+    parts = [part.strip() for part in value.split(",")]
+    if len(parts) != 4:
+        raise HTTPException(status_code=400, detail="bbox must be lon_min,lat_min,lon_max,lat_max")
+    try:
+        lon_min, lat_min, lon_max, lat_max = [float(part) for part in parts]
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail="bbox values must be numbers") from exc
+    if lat_min > lat_max:
+        lat_min, lat_max = lat_max, lat_min
+    if lon_min > lon_max:
+        lon_min, lon_max = lon_max, lon_min
+    return lon_min, lat_min, lon_max, lat_max
+
+
+def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bool:
+    if not requested_types:
+        return True
+    for requested_type in requested_types:
+        predicate = VESSEL_TYPE_FILTERS.get(requested_type)
+        if predicate and predicate(props):
+            return True
+    return False
+
+
+def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]:
+    by_type: dict[str, int] = {}
+    underway = 0
+    anchored_or_moored = 0
+    for feature in features:
+        props = feature.get("properties", {})
+        vessel_type = str(props.get("vessel_type_name") or "Other")
+        by_type[vessel_type] = by_type.get(vessel_type, 0) + 1
+        nav_status = props.get("nav_status")
+        if nav_status in (1, 5):
+            anchored_or_moored += 1
+        else:
+            underway += 1
+    return {
+        "total": len(features),
+        "by_type": by_type,
+        "underway": underway,
+        "anchored_or_moored": anchored_or_moored,
+    }
+
+
 def convert_bgp_anomalies_to_geojson(
     records: List[BGPAnomaly],
     geography_hints: Optional[Dict[str, Dict[str, Any]]] = None,
@@ -1298,6 +1401,137 @@ async def get_compute_centers_geojson(
     }
 
 
+@router.get("/geo/vessels")
+async def get_vessels_geojson(
+    bbox: Optional[str] = Query(
+        None,
+        description="Viewport bbox as lon_min,lat_min,lon_max,lat_max",
+    ),
+    type: Optional[str] = Query(
+        None,
+        description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other",
+    ),
+    limit: int = Query(5000, ge=1, le=50000),
+    db: AsyncSession = Depends(get_db),
+):
+    """Return latest vessel positions as GeoJSON points."""
+    latest_times = (
+        select(
+            VesselPosition.mmsi.label("mmsi"),
+            func.max(VesselPosition.received_at).label("received_at"),
+        )
+        .group_by(VesselPosition.mmsi)
+        .subquery()
+    )
+    stmt = (
+        select(VesselPosition, VesselStatic)
+        .join(
+            latest_times,
+            (VesselPosition.mmsi == latest_times.c.mmsi)
+            & (VesselPosition.received_at == latest_times.c.received_at),
+        )
+        .outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
+        .order_by(VesselPosition.received_at.desc())
+        .limit(limit)
+    )
+
+    parsed_bbox = _parse_bbox(bbox)
+    if parsed_bbox is not None:
+        lon_min, lat_min, lon_max, lat_max = parsed_bbox
+        stmt = stmt.where(
+            VesselPosition.lon >= lon_min,
+            VesselPosition.lon <= lon_max,
+            VesselPosition.lat >= lat_min,
+            VesselPosition.lat <= lat_max,
+        )
+
+    result = await db.execute(stmt)
+    rows = list(result.all())
+    geojson = convert_vessels_to_geojson(rows)
+    requested_types = {
+        item.strip().lower()
+        for item in (type or "").split(",")
+        if item.strip()
+    }
+    if requested_types:
+        geojson["features"] = [
+            feature
+            for feature in geojson.get("features", [])
+            if _matches_vessel_type(feature.get("properties", {}), requested_types)
+        ]
+
+    features = geojson.get("features", [])
+    return {
+        **geojson,
+        "count": len(features),
+        "stats": _build_vessel_stats(features),
+    }
+
+
+@router.get("/vessels/{mmsi}")
+async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)):
+    latest_position_stmt = (
+        select(VesselPosition)
+        .where(VesselPosition.mmsi == mmsi)
+        .order_by(VesselPosition.received_at.desc())
+        .limit(1)
+    )
+    static = await db.get(VesselStatic, mmsi)
+    result = await db.execute(latest_position_stmt)
+    position = result.scalar_one_or_none()
+    if position is None:
+        raise HTTPException(status_code=404, detail="Vessel not found")
+    geojson = convert_vessels_to_geojson([(position, static)])
+    return {
+        **(geojson["features"][0]["properties"]),
+        "latitude": position.lat,
+        "longitude": position.lon,
+    }
+
+
+@router.get("/vessels/{mmsi}/track")
+async def get_vessel_track(
+    mmsi: int,
+    hours: int = Query(6, ge=1, le=24),
+    db: AsyncSession = Depends(get_db),
+):
+    cutoff = datetime.now(UTC) - timedelta(hours=hours)
+    result = await db.execute(
+        select(VesselPosition)
+        .where(VesselPosition.mmsi == mmsi)
+        .where(VesselPosition.received_at >= cutoff)
+        .order_by(VesselPosition.received_at.asc())
+    )
+    positions = list(result.scalars().all())
+    if not positions:
+        return {
+            "type": "FeatureCollection",
+            "features": [],
+            "count": 0,
+        }
+
+    return {
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "geometry": {
+                    "type": "LineString",
+                    "coordinates": [[position.lon, position.lat] for position in positions],
+                },
+                "properties": {
+                    "mmsi": mmsi,
+                    "hours": hours,
+                    "point_count": len(positions),
+                    "start_at": to_iso8601_utc(positions[0].received_at),
+                    "end_at": to_iso8601_utc(positions[-1].received_at),
+                },
+            }
+        ],
+        "count": 1,
+    }
+
+
 @router.get("/geo/bgp-anomalies")
 async def get_bgp_anomalies_geojson(
     severity: Optional[str] = Query(None),
@@ -1394,6 +1628,10 @@ async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
         db,
         source_filter=("ris_live_bgp", "bgpstream_bgp"),
     )
+    vessel_count_result = await db.execute(
+        select(func.count(func.distinct(VesselPosition.mmsi))),
+    )
+    vessel_count = int(vessel_count_result.scalar() or 0)
 
     return {
         "generated_at": to_iso8601_utc(datetime.now(UTC)),
@@ -1402,6 +1640,7 @@ async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
             "landing_point_count": len(landing_points.get("features", [])),
             "satellite_count": len(satellites.get("features", [])),
             "compute_center_count": len(compute_features),
+            "vessel_count": vessel_count,
             "supercomputer_count": sum(
                 1 for feature in compute_features
                 if feature.get("properties", {}).get("site_type") == "supercomputer"