release: bump version to 0.53.0

backend/app/api/main.py

@@ -18,6 +18,7 @@ from app.api.v1 import (
     vessels,
     bgp,
     news,
+    realtime_sources,
     system_control,
     tv,
 )
@@ -50,3 +51,4 @@ api_router.include_router(vessels.router, prefix="/vessels", tags=["vessels"])
 api_router.include_router(bgp.router, prefix="/bgp", tags=["bgp"])
 api_router.include_router(tv.router, prefix="/tv", tags=["tv"])
 api_router.include_router(news.router, prefix="/news", tags=["news"])
+api_router.include_router(realtime_sources.router, prefix="/realtime-sources", tags=["realtime-sources"])

backend/app/api/v1/datasources.py

@@ -18,6 +18,8 @@ from app.models.datasource import DataSource
 from app.models.datasource_config import DataSourceConfig
 from app.models.task import CollectionTask
 from app.models.user import User
+from app.models.vessel import AISRawObservation
+from app.services.vessel_ais_aggregation import VESSEL_AIS_SCHEMA
 from app.services.scheduler import (
     cancel_running_collector_now,
     get_latest_task_id_for_datasource,
@@ -161,7 +163,26 @@ async def _load_collected_record_counts(
         .where(CollectedData.is_current.is_(True))
         .group_by(CollectedData.source)
     )
-    return {source: int(count or 0) for source, count in result.all()}
+    counts = {source: int(count or 0) for source, count in result.all()}
+
+    vessel_sources = [
+        source
+        for source in sources
+        if datasource_metadata(source)["credential_provider"] in {"aisstream", "barentswatch"}
+        or "vessel" in source
+        or "ais" in source
+    ]
+    if vessel_sources:
+        raw_result = await db.execute(
+            select(AISRawObservation.source, func.count(AISRawObservation.id))
+            .where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
+            .where(AISRawObservation.source.in_(vessel_sources))
+            .group_by(AISRawObservation.source)
+        )
+        for source, count in raw_result.all():
+            counts[source] = max(counts.get(source, 0), int(count or 0))
+
+    return counts
 
 
 async def _load_datasource_endpoint_overrides(

backend/app/api/v1/layers.py

@@ -117,7 +117,7 @@ async def get_vessel_layer_snapshot(
         bbox=parsed_bbox,
         zoom=zoom,
         limit=limit,
-        vessel_type=vessel_type,
+        type_filter=vessel_type,
         since_minutes=since_minutes,
     )
 

backend/app/api/v1/realtime_sources.py

@@ -0,0 +1,280 @@
+"""Realtime datasource operations and runtime statistics."""
+
+from datetime import UTC, datetime, timedelta
+import os
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy import distinct, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.data_sources import get_data_sources_config
+from app.core.security import get_current_user
+from app.core.time import to_iso8601_utc
+from app.db.session import get_db
+from app.models.datasource import DataSource
+from app.models.datasource_config import DataSourceConfig
+from app.models.user import User
+from app.models.vessel import AISRawObservation, AISSourceHealth
+from app.services.custom_datasource_runtime import (
+    get_custom_stream_status,
+    start_custom_stream,
+    stop_custom_stream,
+)
+from app.services.scheduler import (
+    cancel_running_collector_now,
+    is_collector_running,
+    run_collector_now,
+)
+from app.services.vessel_ais_aggregation import VESSEL_AIS_SCHEMA, update_ais_source_health
+
+router = APIRouter()
+
+BUILTIN_REALTIME_SOURCES = {"aisstream_vessels"}
+REALTIME_SOURCE_TYPES = {"websocket", "ws"}
+
+
+def _is_realtime_config(config: DataSourceConfig) -> bool:
+    return str(config.source_type or "").lower() in REALTIME_SOURCE_TYPES
+
+
+def _safe_config_dict(value: Any) -> dict[str, Any]:
+    return value if isinstance(value, dict) else {}
+
+
+def _credential_configured(config: DataSourceConfig | None) -> bool:
+    if config is not None:
+        auth_config = _safe_config_dict(config.auth_config)
+        config_payload = _safe_config_dict(config.config)
+        if auth_config.get("api_key") or config_payload.get("api_key"):
+            return True
+    return bool(os.getenv("AISSTREAM_API_KEY"))
+
+
+async def _load_realtime_stats(db: AsyncSession, source: str) -> dict[str, Any]:
+    now = datetime.now(UTC)
+    observed_24h = now - timedelta(hours=24)
+    observed_1h = now - timedelta(hours=1)
+    payload_mmsi = AISRawObservation.entity_key
+
+    result = await db.execute(
+        select(
+            func.count(AISRawObservation.id).label("total_observations"),
+            func.count(AISRawObservation.id)
+            .filter(AISRawObservation.observed_at >= observed_24h)
+            .label("observations_24h"),
+            func.count(AISRawObservation.id)
+            .filter(AISRawObservation.observed_at >= observed_1h)
+            .label("observations_1h"),
+            func.count(distinct(payload_mmsi)).label("unique_mmsi_total"),
+            func.count(distinct(payload_mmsi))
+            .filter(AISRawObservation.observed_at >= observed_24h)
+            .label("unique_mmsi_24h"),
+            func.max(AISRawObservation.observed_at).label("latest_observed_at"),
+            func.max(AISRawObservation.collected_at).label("latest_collected_at"),
+        )
+        .where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
+        .where(AISRawObservation.source == source)
+    )
+    row = result.mappings().one()
+    return {
+        "total_observations": int(row["total_observations"] or 0),
+        "observations_24h": int(row["observations_24h"] or 0),
+        "observations_1h": int(row["observations_1h"] or 0),
+        "unique_mmsi_total": int(row["unique_mmsi_total"] or 0),
+        "unique_mmsi_24h": int(row["unique_mmsi_24h"] or 0),
+        "latest_observed_at": to_iso8601_utc(row["latest_observed_at"]),
+        "latest_collected_at": to_iso8601_utc(row["latest_collected_at"]),
+    }
+
+
+def _runtime_status_for_builtin(source: str) -> dict[str, Any]:
+    running = is_collector_running(source)
+    return {
+        "running": running,
+        "done": False,
+        "runtime": "collector",
+    }
+
+
+def _runtime_status_for_custom(config_id: int) -> dict[str, Any]:
+    status = get_custom_stream_status(config_id)
+    return {
+        "running": bool(status.get("running")),
+        "done": bool(status.get("done")),
+        "runtime": "custom_stream",
+    }
+
+
+async def _serialize_builtin_aisstream(
+    db: AsyncSession,
+    datasource: DataSource,
+    config: DataSourceConfig | None,
+) -> dict[str, Any]:
+    health = await db.get(AISSourceHealth, datasource.source)
+    config_payload = _safe_config_dict(config.config if config else {})
+    endpoint = (
+        (config.endpoint if config else None)
+        or get_data_sources_config().get_yaml_url(datasource.source)
+    )
+    return {
+        "source": datasource.source,
+        "name": datasource.name,
+        "display_name": "AISStream 实时船舶",
+        "kind": "builtin",
+        "source_type": "websocket",
+        "endpoint": endpoint,
+        "is_active": bool(datasource.is_active),
+        "credential_configured": _credential_configured(config),
+        "message_types": config_payload.get("message_types") or ["PositionReport", "ShipStaticData"],
+        "bounding_boxes": config_payload.get("bounding_boxes") or [[[-90, -180], [90, 180]]],
+        "config": config_payload,
+        "runtime": _runtime_status_for_builtin(datasource.source),
+        "health": health.to_dict() if health else None,
+        "stats": await _load_realtime_stats(db, datasource.source),
+    }
+
+
+async def _serialize_custom_stream(
+    db: AsyncSession,
+    config: DataSourceConfig,
+) -> dict[str, Any]:
+    health = await db.get(AISSourceHealth, config.name)
+    config_payload = _safe_config_dict(config.config)
+    return {
+        "source": config.name,
+        "name": config.name,
+        "display_name": config.description or config.name,
+        "kind": "custom",
+        "config_id": config.id,
+        "source_type": config.source_type,
+        "endpoint": config.endpoint,
+        "is_active": bool(config.is_active),
+        "credential_configured": config.auth_type == "none" or bool(_safe_config_dict(config.auth_config)),
+        "message_types": config_payload.get("message_types") or [],
+        "bounding_boxes": config_payload.get("bounding_boxes") or [],
+        "config": config_payload,
+        "runtime": _runtime_status_for_custom(config.id),
+        "health": health.to_dict() if health else None,
+        "stats": await _load_realtime_stats(db, config.name),
+    }
+
+
+async def _load_builtin_aisstream(db: AsyncSession) -> tuple[DataSource | None, DataSourceConfig | None]:
+    result = await db.execute(select(DataSource).where(DataSource.source == "aisstream_vessels"))
+    datasource = result.scalar_one_or_none()
+    config_result = await db.execute(
+        select(DataSourceConfig)
+        .where(DataSourceConfig.name == "aisstream_vessels")
+        .where(DataSourceConfig.is_active.is_(True))
+        .order_by(DataSourceConfig.id.desc())
+        .limit(1)
+    )
+    return datasource, config_result.scalar_one_or_none()
+
+
+async def _load_custom_realtime_config(db: AsyncSession, source: str) -> DataSourceConfig | None:
+    result = await db.execute(
+        select(DataSourceConfig)
+        .where(DataSourceConfig.name == source)
+        .order_by(DataSourceConfig.id.desc())
+        .limit(1)
+    )
+    config = result.scalar_one_or_none()
+    return config if config is not None and _is_realtime_config(config) else None
+
+
+@router.get("")
+async def list_realtime_sources(
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    sources: list[dict[str, Any]] = []
+    datasource, builtin_config = await _load_builtin_aisstream(db)
+    if datasource is not None:
+        sources.append(await _serialize_builtin_aisstream(db, datasource, builtin_config))
+
+    custom_result = await db.execute(
+        select(DataSourceConfig)
+        .where(func.lower(DataSourceConfig.source_type).in_(REALTIME_SOURCE_TYPES))
+        .order_by(DataSourceConfig.name)
+    )
+    for config in custom_result.scalars().all():
+        if config.name in BUILTIN_REALTIME_SOURCES:
+            continue
+        sources.append(await _serialize_custom_stream(db, config))
+
+    return {"total": len(sources), "data": sources}
+
+
+async def _ensure_builtin_startable(db: AsyncSession) -> DataSourceConfig | None:
+    datasource, config = await _load_builtin_aisstream(db)
+    if datasource is None:
+        raise HTTPException(status_code=404, detail="Realtime source not found")
+    if not datasource.is_active:
+        raise HTTPException(status_code=400, detail="Realtime source is disabled")
+    if not _credential_configured(config):
+        raise HTTPException(status_code=400, detail="AISStream API key is not configured")
+    return config
+
+
+@router.post("/{source}/start")
+async def start_realtime_source(
+    source: str,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    if source == "aisstream_vessels":
+        await _ensure_builtin_startable(db)
+        if is_collector_running(source):
+            return {"status": "already_running", "source": source, "runtime": _runtime_status_for_builtin(source)}
+        if not run_collector_now(source):
+            raise HTTPException(status_code=409, detail="Realtime source could not be started")
+        return {"status": "started", "source": source, "runtime": _runtime_status_for_builtin(source)}
+
+    config = await _load_custom_realtime_config(db, source)
+    if config is None:
+        raise HTTPException(status_code=404, detail="Realtime source not found")
+    if not config.is_active:
+        raise HTTPException(status_code=400, detail="Realtime source is disabled")
+    started = start_custom_stream(config.id)
+    return {
+        "status": "started" if started else "already_running",
+        "source": source,
+        "runtime": _runtime_status_for_custom(config.id),
+    }
+
+
+@router.post("/{source}/stop")
+async def stop_realtime_source(
+    source: str,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    if source == "aisstream_vessels":
+        stopped = await cancel_running_collector_now(source)
+        await update_ais_source_health(db, source=source, connection_state="disconnected", last_error=None)
+        await db.commit()
+        return {"status": "stopped" if stopped else "not_running", "source": source, "runtime": _runtime_status_for_builtin(source)}
+
+    config = await _load_custom_realtime_config(db, source)
+    if config is None:
+        raise HTTPException(status_code=404, detail="Realtime source not found")
+    stopped = await stop_custom_stream(config.id)
+    await update_ais_source_health(db, source=source, connection_state="disconnected", last_error=None)
+    await db.commit()
+    return {
+        "status": "stopped" if stopped else "not_running",
+        "source": source,
+        "runtime": _runtime_status_for_custom(config.id),
+    }
+
+
+@router.post("/{source}/restart")
+async def restart_realtime_source(
+    source: str,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    await stop_realtime_source(source, current_user=current_user, db=db)
+    return await start_realtime_source(source, current_user=current_user, db=db)

backend/app/api/v1/visualization.py

@@ -68,6 +68,7 @@ TERRAIN_TILE_BATCH_MAX_ITEMS = 128
 TERRAIN_TILE_BATCH_CONCURRENCY = 16
 _terrain_tile_cache: OrderedDict[tuple[int, int, int], tuple[bytes, str, dict[str, str]]] = OrderedDict()
 VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE)
+VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED = True
 
 
 class TerrariumTileRequest(BaseModel):
@@ -941,6 +942,39 @@ def _merge_vessel_features(
     }
 
 
+async def _load_legacy_vessel_snapshot_features(
+    db: AsyncSession,
+    *,
+    bbox: tuple[float, float, float, float] | None,
+    limit: int,
+) -> list[dict[str, Any]]:
+    latest_positions = select(
+        VesselPosition.mmsi.label("mmsi"),
+        func.max(VesselPosition.received_at).label("received_at"),
+    )
+    if bbox is not None:
+        lon_min, lat_min, lon_max, lat_max = bbox
+        latest_positions = latest_positions.where(VesselPosition.lon >= lon_min)
+        latest_positions = latest_positions.where(VesselPosition.lon <= lon_max)
+        latest_positions = latest_positions.where(VesselPosition.lat >= lat_min)
+        latest_positions = latest_positions.where(VesselPosition.lat <= lat_max)
+
+    latest_positions = latest_positions.group_by(VesselPosition.mmsi).subquery()
+    result = await db.execute(
+        select(VesselPosition, VesselStatic)
+        .join(
+            latest_positions,
+            (VesselPosition.mmsi == latest_positions.c.mmsi)
+            & (VesselPosition.received_at == latest_positions.c.received_at),
+        )
+        .outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
+        .order_by(VesselPosition.received_at.desc())
+        .limit(limit)
+    )
+    legacy_geojson = convert_vessels_to_geojson(list(result.all()))
+    return legacy_geojson.get("features", [])[:limit]
+
+
 def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]:
     by_type: dict[str, int] = {}
     underway = 0
@@ -2073,33 +2107,6 @@ async def _load_compute_center_record(db: AsyncSession, source_id: str) -> Colle
     return result.scalars().first()
 
 
-@router.get("/geo/vessels")
-async def get_vessels_geojson(
-    bbox: Optional[str] = Query(
-        None,
-        description="Viewport bbox as lon_min,lat_min,lon_max,lat_max",
-    ),
-    type: Optional[str] = Query(
-        None,
-        description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other",
-    ),
-    limit: Optional[int] = Query(
-        None,
-        ge=0,
-        description="Maximum vessel features to return. Omit or pass 0 for no limit.",
-    ),
-    db: AsyncSession = Depends(get_db),
-):
-    """Legacy vessel endpoint removed in favor of /api/v1/vessels/snapshot."""
-    raise HTTPException(
-        status_code=410,
-        detail=(
-            "Legacy vessel GeoJSON endpoint has been removed. "
-            "Use /api/v1/vessels/snapshot with bbox, zoom, and limit."
-        ),
-    )
-
-
 async def _load_raw_vessel_snapshot_features(
     db: AsyncSession,
     *,
@@ -2121,18 +2128,38 @@ async def _load_raw_vessel_snapshot_features(
             observed_since=observed_since,
         )
     raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
-    features = raw_geojson.get("features", [])
+    raw_features = raw_geojson.get("features", [])
+    features = raw_features
+    legacy_features: list[dict[str, Any]] = []
+    legacy_fallback_used = False
+    if not raw_features and VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED:
+        legacy_features = await _load_legacy_vessel_snapshot_features(
+            db,
+            bbox=bbox,
+            limit=limit,
+        )
+        features, _merge_diagnostics = _merge_vessel_features(raw_features, legacy_features)
+        legacy_fallback_used = bool(legacy_features)
+
     return features, {
-        "raw_feature_count": len(features),
+        "raw_feature_count": len(raw_features),
         "raw_unique_mmsi": len(
             {
                 key
-                for key in (_feature_mmsi_key(feature) for feature in features)
+                for key in (_feature_mmsi_key(feature) for feature in raw_features)
                 if key is not None
             }
         ),
-        "legacy_feature_count": 0,
-        "legacy_backfilled_mmsi": 0,
+        "legacy_feature_count": len(legacy_features),
+        "legacy_backfilled_mmsi": len(
+            {
+                key
+                for key in (_feature_mmsi_key(feature) for feature in legacy_features)
+                if key is not None
+            }
+        ),
+        "legacy_fallback_enabled": VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED,
+        "legacy_fallback_used": legacy_fallback_used,
         "final_unique_mmsi": len(
             {
                 key
@@ -2466,7 +2493,12 @@ async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
         select(func.count(func.distinct(VesselPosition.mmsi)))
     )
     legacy_unique_mmsi = int(legacy_unique_result.scalar() or 0)
-    vessel_count = max(raw_unique_mmsi, legacy_unique_mmsi)
+    legacy_fallback_active = (
+        VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED
+        and raw_unique_mmsi == 0
+        and legacy_unique_mmsi > 0
+    )
+    vessel_count = legacy_unique_mmsi if legacy_fallback_active else raw_unique_mmsi
     aisstream_health = await db.get(AISSourceHealth, "aisstream_vessels")
 
     return {
@@ -2477,6 +2509,8 @@ async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
             "satellite_count": satellite_count,
             "compute_center_count": compute_center_count,
             "vessel_count": vessel_count,
+            "vessel_count_source": "legacy_fallback" if legacy_fallback_active else "raw_recent",
+            "vessel_legacy_fallback_enabled": VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED,
             "vessel_raw_unique_mmsi": raw_unique_mmsi,
             "vessel_raw_unique_window_hours": raw_unique_window_hours,
             "vessel_legacy_unique_mmsi": legacy_unique_mmsi,

backend/app/services/scheduler.py

@@ -374,6 +374,11 @@ def run_collector_now(collector_name: str) -> bool:
         return False
 
 
+def is_collector_running(collector_name: str) -> bool:
+    task = get_running_collector_task(collector_name)
+    return bool(task is not None and not task.done())
+
+
 async def cancel_running_collector_now(collector_name: str) -> bool:
     task = get_running_collector_task(collector_name)
     if task is None or task.done():