281 lines
11 KiB
Python
281 lines
11 KiB
Python
"""Realtime datasource operations and runtime statistics."""
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
import os
|
|
from typing import Any
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException
|
|
from sqlalchemy import distinct, func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.core.data_sources import get_data_sources_config
|
|
from app.core.security import get_current_user
|
|
from app.core.time import to_iso8601_utc
|
|
from app.db.session import get_db
|
|
from app.models.datasource import DataSource
|
|
from app.models.datasource_config import DataSourceConfig
|
|
from app.models.user import User
|
|
from app.models.vessel import AISRawObservation, AISSourceHealth
|
|
from app.services.custom_datasource_runtime import (
|
|
get_custom_stream_status,
|
|
start_custom_stream,
|
|
stop_custom_stream,
|
|
)
|
|
from app.services.scheduler import (
|
|
cancel_running_collector_now,
|
|
is_collector_running,
|
|
run_collector_now,
|
|
)
|
|
from app.services.vessel_ais_aggregation import VESSEL_AIS_SCHEMA, update_ais_source_health
|
|
|
|
router = APIRouter()
|
|
|
|
BUILTIN_REALTIME_SOURCES = {"aisstream_vessels"}
|
|
REALTIME_SOURCE_TYPES = {"websocket", "ws"}
|
|
|
|
|
|
def _is_realtime_config(config: DataSourceConfig) -> bool:
|
|
return str(config.source_type or "").lower() in REALTIME_SOURCE_TYPES
|
|
|
|
|
|
def _safe_config_dict(value: Any) -> dict[str, Any]:
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def _credential_configured(config: DataSourceConfig | None) -> bool:
|
|
if config is not None:
|
|
auth_config = _safe_config_dict(config.auth_config)
|
|
config_payload = _safe_config_dict(config.config)
|
|
if auth_config.get("api_key") or config_payload.get("api_key"):
|
|
return True
|
|
return bool(os.getenv("AISSTREAM_API_KEY"))
|
|
|
|
|
|
async def _load_realtime_stats(db: AsyncSession, source: str) -> dict[str, Any]:
|
|
now = datetime.now(UTC)
|
|
observed_24h = now - timedelta(hours=24)
|
|
observed_1h = now - timedelta(hours=1)
|
|
payload_mmsi = AISRawObservation.entity_key
|
|
|
|
result = await db.execute(
|
|
select(
|
|
func.count(AISRawObservation.id).label("total_observations"),
|
|
func.count(AISRawObservation.id)
|
|
.filter(AISRawObservation.observed_at >= observed_24h)
|
|
.label("observations_24h"),
|
|
func.count(AISRawObservation.id)
|
|
.filter(AISRawObservation.observed_at >= observed_1h)
|
|
.label("observations_1h"),
|
|
func.count(distinct(payload_mmsi)).label("unique_mmsi_total"),
|
|
func.count(distinct(payload_mmsi))
|
|
.filter(AISRawObservation.observed_at >= observed_24h)
|
|
.label("unique_mmsi_24h"),
|
|
func.max(AISRawObservation.observed_at).label("latest_observed_at"),
|
|
func.max(AISRawObservation.collected_at).label("latest_collected_at"),
|
|
)
|
|
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
|
|
.where(AISRawObservation.source == source)
|
|
)
|
|
row = result.mappings().one()
|
|
return {
|
|
"total_observations": int(row["total_observations"] or 0),
|
|
"observations_24h": int(row["observations_24h"] or 0),
|
|
"observations_1h": int(row["observations_1h"] or 0),
|
|
"unique_mmsi_total": int(row["unique_mmsi_total"] or 0),
|
|
"unique_mmsi_24h": int(row["unique_mmsi_24h"] or 0),
|
|
"latest_observed_at": to_iso8601_utc(row["latest_observed_at"]),
|
|
"latest_collected_at": to_iso8601_utc(row["latest_collected_at"]),
|
|
}
|
|
|
|
|
|
def _runtime_status_for_builtin(source: str) -> dict[str, Any]:
|
|
running = is_collector_running(source)
|
|
return {
|
|
"running": running,
|
|
"done": False,
|
|
"runtime": "collector",
|
|
}
|
|
|
|
|
|
def _runtime_status_for_custom(config_id: int) -> dict[str, Any]:
|
|
status = get_custom_stream_status(config_id)
|
|
return {
|
|
"running": bool(status.get("running")),
|
|
"done": bool(status.get("done")),
|
|
"runtime": "custom_stream",
|
|
}
|
|
|
|
|
|
async def _serialize_builtin_aisstream(
|
|
db: AsyncSession,
|
|
datasource: DataSource,
|
|
config: DataSourceConfig | None,
|
|
) -> dict[str, Any]:
|
|
health = await db.get(AISSourceHealth, datasource.source)
|
|
config_payload = _safe_config_dict(config.config if config else {})
|
|
endpoint = (
|
|
(config.endpoint if config else None)
|
|
or get_data_sources_config().get_yaml_url(datasource.source)
|
|
)
|
|
return {
|
|
"source": datasource.source,
|
|
"name": datasource.name,
|
|
"display_name": "AISStream 实时船舶",
|
|
"kind": "builtin",
|
|
"source_type": "websocket",
|
|
"endpoint": endpoint,
|
|
"is_active": bool(datasource.is_active),
|
|
"credential_configured": _credential_configured(config),
|
|
"message_types": config_payload.get("message_types") or ["PositionReport", "ShipStaticData"],
|
|
"bounding_boxes": config_payload.get("bounding_boxes") or [[[-90, -180], [90, 180]]],
|
|
"config": config_payload,
|
|
"runtime": _runtime_status_for_builtin(datasource.source),
|
|
"health": health.to_dict() if health else None,
|
|
"stats": await _load_realtime_stats(db, datasource.source),
|
|
}
|
|
|
|
|
|
async def _serialize_custom_stream(
|
|
db: AsyncSession,
|
|
config: DataSourceConfig,
|
|
) -> dict[str, Any]:
|
|
health = await db.get(AISSourceHealth, config.name)
|
|
config_payload = _safe_config_dict(config.config)
|
|
return {
|
|
"source": config.name,
|
|
"name": config.name,
|
|
"display_name": config.description or config.name,
|
|
"kind": "custom",
|
|
"config_id": config.id,
|
|
"source_type": config.source_type,
|
|
"endpoint": config.endpoint,
|
|
"is_active": bool(config.is_active),
|
|
"credential_configured": config.auth_type == "none" or bool(_safe_config_dict(config.auth_config)),
|
|
"message_types": config_payload.get("message_types") or [],
|
|
"bounding_boxes": config_payload.get("bounding_boxes") or [],
|
|
"config": config_payload,
|
|
"runtime": _runtime_status_for_custom(config.id),
|
|
"health": health.to_dict() if health else None,
|
|
"stats": await _load_realtime_stats(db, config.name),
|
|
}
|
|
|
|
|
|
async def _load_builtin_aisstream(db: AsyncSession) -> tuple[DataSource | None, DataSourceConfig | None]:
|
|
result = await db.execute(select(DataSource).where(DataSource.source == "aisstream_vessels"))
|
|
datasource = result.scalar_one_or_none()
|
|
config_result = await db.execute(
|
|
select(DataSourceConfig)
|
|
.where(DataSourceConfig.name == "aisstream_vessels")
|
|
.where(DataSourceConfig.is_active.is_(True))
|
|
.order_by(DataSourceConfig.id.desc())
|
|
.limit(1)
|
|
)
|
|
return datasource, config_result.scalar_one_or_none()
|
|
|
|
|
|
async def _load_custom_realtime_config(db: AsyncSession, source: str) -> DataSourceConfig | None:
|
|
result = await db.execute(
|
|
select(DataSourceConfig)
|
|
.where(DataSourceConfig.name == source)
|
|
.order_by(DataSourceConfig.id.desc())
|
|
.limit(1)
|
|
)
|
|
config = result.scalar_one_or_none()
|
|
return config if config is not None and _is_realtime_config(config) else None
|
|
|
|
|
|
@router.get("")
|
|
async def list_realtime_sources(
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
sources: list[dict[str, Any]] = []
|
|
datasource, builtin_config = await _load_builtin_aisstream(db)
|
|
if datasource is not None:
|
|
sources.append(await _serialize_builtin_aisstream(db, datasource, builtin_config))
|
|
|
|
custom_result = await db.execute(
|
|
select(DataSourceConfig)
|
|
.where(func.lower(DataSourceConfig.source_type).in_(REALTIME_SOURCE_TYPES))
|
|
.order_by(DataSourceConfig.name)
|
|
)
|
|
for config in custom_result.scalars().all():
|
|
if config.name in BUILTIN_REALTIME_SOURCES:
|
|
continue
|
|
sources.append(await _serialize_custom_stream(db, config))
|
|
|
|
return {"total": len(sources), "data": sources}
|
|
|
|
|
|
async def _ensure_builtin_startable(db: AsyncSession) -> DataSourceConfig | None:
|
|
datasource, config = await _load_builtin_aisstream(db)
|
|
if datasource is None:
|
|
raise HTTPException(status_code=404, detail="Realtime source not found")
|
|
if not datasource.is_active:
|
|
raise HTTPException(status_code=400, detail="Realtime source is disabled")
|
|
if not _credential_configured(config):
|
|
raise HTTPException(status_code=400, detail="AISStream API key is not configured")
|
|
return config
|
|
|
|
|
|
@router.post("/{source}/start")
|
|
async def start_realtime_source(
|
|
source: str,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
if source == "aisstream_vessels":
|
|
await _ensure_builtin_startable(db)
|
|
if is_collector_running(source):
|
|
return {"status": "already_running", "source": source, "runtime": _runtime_status_for_builtin(source)}
|
|
if not run_collector_now(source):
|
|
raise HTTPException(status_code=409, detail="Realtime source could not be started")
|
|
return {"status": "started", "source": source, "runtime": _runtime_status_for_builtin(source)}
|
|
|
|
config = await _load_custom_realtime_config(db, source)
|
|
if config is None:
|
|
raise HTTPException(status_code=404, detail="Realtime source not found")
|
|
if not config.is_active:
|
|
raise HTTPException(status_code=400, detail="Realtime source is disabled")
|
|
started = start_custom_stream(config.id)
|
|
return {
|
|
"status": "started" if started else "already_running",
|
|
"source": source,
|
|
"runtime": _runtime_status_for_custom(config.id),
|
|
}
|
|
|
|
|
|
@router.post("/{source}/stop")
|
|
async def stop_realtime_source(
|
|
source: str,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
if source == "aisstream_vessels":
|
|
stopped = await cancel_running_collector_now(source)
|
|
await update_ais_source_health(db, source=source, connection_state="disconnected", last_error=None)
|
|
await db.commit()
|
|
return {"status": "stopped" if stopped else "not_running", "source": source, "runtime": _runtime_status_for_builtin(source)}
|
|
|
|
config = await _load_custom_realtime_config(db, source)
|
|
if config is None:
|
|
raise HTTPException(status_code=404, detail="Realtime source not found")
|
|
stopped = await stop_custom_stream(config.id)
|
|
await update_ais_source_health(db, source=source, connection_state="disconnected", last_error=None)
|
|
await db.commit()
|
|
return {
|
|
"status": "stopped" if stopped else "not_running",
|
|
"source": source,
|
|
"runtime": _runtime_status_for_custom(config.id),
|
|
}
|
|
|
|
|
|
@router.post("/{source}/restart")
|
|
async def restart_realtime_source(
|
|
source: str,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
await stop_realtime_source(source, current_user=current_user, db=db)
|
|
return await start_realtime_source(source, current_user=current_user, db=db)
|