release: bump version to 0.53.0
This commit is contained in:
rayd1o
280
backend/app/api/v1/realtime_sources.py
Normal file
280
backend/app/api/v1/realtime_sources.py
Normal file
@@ -0,0 +1,280 @@
|
||||
"""Realtime datasource operations and runtime statistics."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy import distinct, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.core.security import get_current_user
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.db.session import get_db
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.user import User
|
||||
from app.models.vessel import AISRawObservation, AISSourceHealth
|
||||
from app.services.custom_datasource_runtime import (
|
||||
get_custom_stream_status,
|
||||
start_custom_stream,
|
||||
stop_custom_stream,
|
||||
)
|
||||
from app.services.scheduler import (
|
||||
cancel_running_collector_now,
|
||||
is_collector_running,
|
||||
run_collector_now,
|
||||
)
|
||||
from app.services.vessel_ais_aggregation import VESSEL_AIS_SCHEMA, update_ais_source_health
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
BUILTIN_REALTIME_SOURCES = {"aisstream_vessels"}
|
||||
REALTIME_SOURCE_TYPES = {"websocket", "ws"}
|
||||
|
||||
|
||||
def _is_realtime_config(config: DataSourceConfig) -> bool:
|
||||
return str(config.source_type or "").lower() in REALTIME_SOURCE_TYPES
|
||||
|
||||
|
||||
def _safe_config_dict(value: Any) -> dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _credential_configured(config: DataSourceConfig | None) -> bool:
|
||||
if config is not None:
|
||||
auth_config = _safe_config_dict(config.auth_config)
|
||||
config_payload = _safe_config_dict(config.config)
|
||||
if auth_config.get("api_key") or config_payload.get("api_key"):
|
||||
return True
|
||||
return bool(os.getenv("AISSTREAM_API_KEY"))
|
||||
|
||||
|
||||
async def _load_realtime_stats(db: AsyncSession, source: str) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
observed_24h = now - timedelta(hours=24)
|
||||
observed_1h = now - timedelta(hours=1)
|
||||
payload_mmsi = AISRawObservation.entity_key
|
||||
|
||||
result = await db.execute(
|
||||
select(
|
||||
func.count(AISRawObservation.id).label("total_observations"),
|
||||
func.count(AISRawObservation.id)
|
||||
.filter(AISRawObservation.observed_at >= observed_24h)
|
||||
.label("observations_24h"),
|
||||
func.count(AISRawObservation.id)
|
||||
.filter(AISRawObservation.observed_at >= observed_1h)
|
||||
.label("observations_1h"),
|
||||
func.count(distinct(payload_mmsi)).label("unique_mmsi_total"),
|
||||
func.count(distinct(payload_mmsi))
|
||||
.filter(AISRawObservation.observed_at >= observed_24h)
|
||||
.label("unique_mmsi_24h"),
|
||||
func.max(AISRawObservation.observed_at).label("latest_observed_at"),
|
||||
func.max(AISRawObservation.collected_at).label("latest_collected_at"),
|
||||
)
|
||||
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
|
||||
.where(AISRawObservation.source == source)
|
||||
)
|
||||
row = result.mappings().one()
|
||||
return {
|
||||
"total_observations": int(row["total_observations"] or 0),
|
||||
"observations_24h": int(row["observations_24h"] or 0),
|
||||
"observations_1h": int(row["observations_1h"] or 0),
|
||||
"unique_mmsi_total": int(row["unique_mmsi_total"] or 0),
|
||||
"unique_mmsi_24h": int(row["unique_mmsi_24h"] or 0),
|
||||
"latest_observed_at": to_iso8601_utc(row["latest_observed_at"]),
|
||||
"latest_collected_at": to_iso8601_utc(row["latest_collected_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _runtime_status_for_builtin(source: str) -> dict[str, Any]:
|
||||
running = is_collector_running(source)
|
||||
return {
|
||||
"running": running,
|
||||
"done": False,
|
||||
"runtime": "collector",
|
||||
}
|
||||
|
||||
|
||||
def _runtime_status_for_custom(config_id: int) -> dict[str, Any]:
|
||||
status = get_custom_stream_status(config_id)
|
||||
return {
|
||||
"running": bool(status.get("running")),
|
||||
"done": bool(status.get("done")),
|
||||
"runtime": "custom_stream",
|
||||
}
|
||||
|
||||
|
||||
async def _serialize_builtin_aisstream(
|
||||
db: AsyncSession,
|
||||
datasource: DataSource,
|
||||
config: DataSourceConfig | None,
|
||||
) -> dict[str, Any]:
|
||||
health = await db.get(AISSourceHealth, datasource.source)
|
||||
config_payload = _safe_config_dict(config.config if config else {})
|
||||
endpoint = (
|
||||
(config.endpoint if config else None)
|
||||
or get_data_sources_config().get_yaml_url(datasource.source)
|
||||
)
|
||||
return {
|
||||
"source": datasource.source,
|
||||
"name": datasource.name,
|
||||
"display_name": "AISStream 实时船舶",
|
||||
"kind": "builtin",
|
||||
"source_type": "websocket",
|
||||
"endpoint": endpoint,
|
||||
"is_active": bool(datasource.is_active),
|
||||
"credential_configured": _credential_configured(config),
|
||||
"message_types": config_payload.get("message_types") or ["PositionReport", "ShipStaticData"],
|
||||
"bounding_boxes": config_payload.get("bounding_boxes") or [[[-90, -180], [90, 180]]],
|
||||
"config": config_payload,
|
||||
"runtime": _runtime_status_for_builtin(datasource.source),
|
||||
"health": health.to_dict() if health else None,
|
||||
"stats": await _load_realtime_stats(db, datasource.source),
|
||||
}
|
||||
|
||||
|
||||
async def _serialize_custom_stream(
|
||||
db: AsyncSession,
|
||||
config: DataSourceConfig,
|
||||
) -> dict[str, Any]:
|
||||
health = await db.get(AISSourceHealth, config.name)
|
||||
config_payload = _safe_config_dict(config.config)
|
||||
return {
|
||||
"source": config.name,
|
||||
"name": config.name,
|
||||
"display_name": config.description or config.name,
|
||||
"kind": "custom",
|
||||
"config_id": config.id,
|
||||
"source_type": config.source_type,
|
||||
"endpoint": config.endpoint,
|
||||
"is_active": bool(config.is_active),
|
||||
"credential_configured": config.auth_type == "none" or bool(_safe_config_dict(config.auth_config)),
|
||||
"message_types": config_payload.get("message_types") or [],
|
||||
"bounding_boxes": config_payload.get("bounding_boxes") or [],
|
||||
"config": config_payload,
|
||||
"runtime": _runtime_status_for_custom(config.id),
|
||||
"health": health.to_dict() if health else None,
|
||||
"stats": await _load_realtime_stats(db, config.name),
|
||||
}
|
||||
|
||||
|
||||
async def _load_builtin_aisstream(db: AsyncSession) -> tuple[DataSource | None, DataSourceConfig | None]:
|
||||
result = await db.execute(select(DataSource).where(DataSource.source == "aisstream_vessels"))
|
||||
datasource = result.scalar_one_or_none()
|
||||
config_result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == "aisstream_vessels")
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
.order_by(DataSourceConfig.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return datasource, config_result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _load_custom_realtime_config(db: AsyncSession, source: str) -> DataSourceConfig | None:
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == source)
|
||||
.order_by(DataSourceConfig.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
config = result.scalar_one_or_none()
|
||||
return config if config is not None and _is_realtime_config(config) else None
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_realtime_sources(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
sources: list[dict[str, Any]] = []
|
||||
datasource, builtin_config = await _load_builtin_aisstream(db)
|
||||
if datasource is not None:
|
||||
sources.append(await _serialize_builtin_aisstream(db, datasource, builtin_config))
|
||||
|
||||
custom_result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(func.lower(DataSourceConfig.source_type).in_(REALTIME_SOURCE_TYPES))
|
||||
.order_by(DataSourceConfig.name)
|
||||
)
|
||||
for config in custom_result.scalars().all():
|
||||
if config.name in BUILTIN_REALTIME_SOURCES:
|
||||
continue
|
||||
sources.append(await _serialize_custom_stream(db, config))
|
||||
|
||||
return {"total": len(sources), "data": sources}
|
||||
|
||||
|
||||
async def _ensure_builtin_startable(db: AsyncSession) -> DataSourceConfig | None:
|
||||
datasource, config = await _load_builtin_aisstream(db)
|
||||
if datasource is None:
|
||||
raise HTTPException(status_code=404, detail="Realtime source not found")
|
||||
if not datasource.is_active:
|
||||
raise HTTPException(status_code=400, detail="Realtime source is disabled")
|
||||
if not _credential_configured(config):
|
||||
raise HTTPException(status_code=400, detail="AISStream API key is not configured")
|
||||
return config
|
||||
|
||||
|
||||
@router.post("/{source}/start")
|
||||
async def start_realtime_source(
|
||||
source: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
if source == "aisstream_vessels":
|
||||
await _ensure_builtin_startable(db)
|
||||
if is_collector_running(source):
|
||||
return {"status": "already_running", "source": source, "runtime": _runtime_status_for_builtin(source)}
|
||||
if not run_collector_now(source):
|
||||
raise HTTPException(status_code=409, detail="Realtime source could not be started")
|
||||
return {"status": "started", "source": source, "runtime": _runtime_status_for_builtin(source)}
|
||||
|
||||
config = await _load_custom_realtime_config(db, source)
|
||||
if config is None:
|
||||
raise HTTPException(status_code=404, detail="Realtime source not found")
|
||||
if not config.is_active:
|
||||
raise HTTPException(status_code=400, detail="Realtime source is disabled")
|
||||
started = start_custom_stream(config.id)
|
||||
return {
|
||||
"status": "started" if started else "already_running",
|
||||
"source": source,
|
||||
"runtime": _runtime_status_for_custom(config.id),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source}/stop")
|
||||
async def stop_realtime_source(
|
||||
source: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
if source == "aisstream_vessels":
|
||||
stopped = await cancel_running_collector_now(source)
|
||||
await update_ais_source_health(db, source=source, connection_state="disconnected", last_error=None)
|
||||
await db.commit()
|
||||
return {"status": "stopped" if stopped else "not_running", "source": source, "runtime": _runtime_status_for_builtin(source)}
|
||||
|
||||
config = await _load_custom_realtime_config(db, source)
|
||||
if config is None:
|
||||
raise HTTPException(status_code=404, detail="Realtime source not found")
|
||||
stopped = await stop_custom_stream(config.id)
|
||||
await update_ais_source_health(db, source=source, connection_state="disconnected", last_error=None)
|
||||
await db.commit()
|
||||
return {
|
||||
"status": "stopped" if stopped else "not_running",
|
||||
"source": source,
|
||||
"runtime": _runtime_status_for_custom(config.id),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source}/restart")
|
||||
async def restart_realtime_source(
|
||||
source: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await stop_realtime_source(source, current_user=current_user, db=db)
|
||||
return await start_realtime_source(source, current_user=current_user, db=db)
|
||||
Reference in New Issue
Block a user