linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.48.0

Browse Source
This commit is contained in:
linkong
2026-05-07 18:06:06 +08:00
parent 421234301a
commit bb9183b8a4
51 changed files with 4609 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
backend/app/api/v1/datasource_config.py
View File

@@ -5,8 +5,8 @@ from datetime import datetime
import base64
import json
import re
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select, func
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import delete, select, func
from sqlalchemy.ext.asyncio import AsyncSession
from pydantic import BaseModel, Field
import httpx
@@ -17,6 +17,8 @@ from app.db.session import get_db
from app.models.user import User
from app.models.datasource_config import DataSourceConfig
from app.models.datasource_mapping import DataSourceMappingTemplate
from app.models.collected_data import CollectedData
from app.models.vessel import AISRawObservation, AISSourceHealth
from app.core.security import get_current_user
from app.core.cache import cache
from app.core.time import to_iso8601_utc
@@ -26,10 +28,19 @@ from app.services.datasource_mapping import (
MappingError,
build_heuristic_mapping,
execute_mapping,
persist_mapped_records,
redact_for_llm,
stable_payload_hash,
)
from app.services.custom_datasource_runtime import (
CustomDatasourceRuntimeError,
fetch_rest_payload,
get_custom_stream_status,
run_mapped_rest_config,
run_mapped_websocket_config,
start_custom_stream,
stop_custom_stream,
test_websocket_config,
)
from app.services.datasource_connectivity import (
get_builtin_connection_status,
save_connectivity_success,
@@ -43,7 +54,7 @@ router = APIRouter()
class DataSourceConfigCreate(BaseModel):
name: str = Field(..., min_length=1, max_length=100)
description: Optional[str] = None
source_type: str = Field(..., description="http, api, database")
source_type: str = Field(..., description="rest, websocket, http, api, database")
endpoint: str = Field(..., max_length=500)
auth_type: str = Field(default="none", description="none, bearer, api_key, basic")
auth_config: dict = Field(default={})
@@ -219,6 +230,8 @@ def _build_query_params(auth_type: str, auth_config: dict, config: dict) -> dict
async def fetch_custom_sample_from_config(config: DataSourceConfig, limit_bytes: int) -> Any:
if str(config.source_type or "").lower() in {"websocket", "ws"}:
raise HTTPException(status_code=400, detail="WebSocket sources must use connection test or run-mapped stream.")
request_config = config.config or {}
method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
if method not in {"GET", "POST"}:
@@ -488,6 +501,8 @@ async def update_config(
@router.delete("/configs/{config_id}")
async def delete_config(
config_id: int,
delete_mappings: bool = Query(False),
delete_source_data: bool = Query(False),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
@@ -498,12 +513,59 @@ async def delete_config(
if not config:
raise HTTPException(status_code=404, detail="Configuration not found")
deleted_mappings = 0
deleted_records = {
"collected_data": 0,
"ais_raw_observations": 0,
"ais_source_health": 0,
}
if delete_source_data:
collected_result = await db.execute(
delete(CollectedData).where(CollectedData.source == config.name)
)
raw_result = await db.execute(
delete(AISRawObservation).where(AISRawObservation.source == config.name)
)
health_result = await db.execute(
delete(AISSourceHealth).where(AISSourceHealth.source == config.name)
)
deleted_records = {
"collected_data": collected_result.rowcount or 0,
"ais_raw_observations": raw_result.rowcount or 0,
"ais_source_health": health_result.rowcount or 0,
}
if delete_mappings or delete_source_data:
mapping_result = await db.execute(
delete(DataSourceMappingTemplate).where(
DataSourceMappingTemplate.datasource_config_id == config_id
)
)
deleted_mappings = mapping_result.rowcount or 0
await db.delete(config)
await db.commit()
cache.delete_pattern("datasource_configs:*")
return {"message": "Configuration deleted successfully"}
if delete_source_data and (config.config or {}).get("target_schema") == "vessel_ais":
from app.core.websocket.broadcaster import broadcaster
await broadcaster.broadcast_custom(
"vessels",
{
"action": "reload",
"source": config.name,
"reason": "custom_source_deleted",
},
)
return {
"message": "Configuration deleted successfully",
"deleted_mappings": deleted_mappings,
"deleted_records": deleted_records,
}
@router.post("/configs/{config_id}/test")
@@ -520,6 +582,8 @@ async def test_config(
raise HTTPException(status_code=404, detail="Configuration not found")
try:
if str(config.source_type or "").lower() in {"websocket", "ws"}:
return await test_websocket_config(config)
result = await test_endpoint(
endpoint=config.endpoint,
auth_type=config.auth_type,
@@ -550,6 +614,18 @@ async def test_new_config(
):
"""Test a new data source configuration without saving"""
try:
if str(config_data.source_type or "").lower() in {"websocket", "ws"}:
config = DataSourceConfig(
name=config_data.name,
description=config_data.description,
source_type=config_data.source_type,
endpoint=config_data.endpoint,
auth_type=config_data.auth_type,
auth_config=config_data.auth_config,
headers=config_data.headers,
config=config_data.config,
)
return await test_websocket_config(config)
result = await test_endpoint(
endpoint=config_data.endpoint,
auth_type=config_data.auth_type,
@@ -875,6 +951,8 @@ async def update_datasource_mapping(
@router.post("/{config_id}/run-mapped")
async def run_mapped_datasource(
config_id: int,
background: bool = Query(False, description="For WebSocket sources, start a background stream task."),
debug_max_messages: int | None = Query(None, ge=1),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
@@ -883,20 +961,24 @@ async def run_mapped_datasource(
if not datasource:
raise HTTPException(status_code=404, detail="Configuration not found")
result = await db.execute(
select(DataSourceMappingTemplate)
.where(DataSourceMappingTemplate.datasource_config_id == config_id)
.where(DataSourceMappingTemplate.is_active.is_(True))
.order_by(DataSourceMappingTemplate.version.desc())
.limit(1)
)
mapping = result.scalar_one_or_none()
if not mapping:
raise HTTPException(status_code=404, detail="No active mapping template found")
try:
sample = await fetch_custom_sample_from_config(datasource, 5_000_000)
mapped = execute_mapping(sample, mapping.mapping_json, mapping.target_schema)
if str(datasource.source_type or "").lower() in {"websocket", "ws"}:
if background and debug_max_messages is None:
started = start_custom_stream(config_id)
if not started:
raise HTTPException(status_code=409, detail="Custom WebSocket source is already running")
return {
"status": "started",
"datasource_config_id": config_id,
"stream": get_custom_stream_status(config_id),
}
return await run_mapped_websocket_config(
db,
datasource,
debug_max_messages=debug_max_messages,
)
return await run_mapped_rest_config(db, datasource)
except httpx.HTTPStatusError as exc:
raise HTTPException(
status_code=exc.response.status_code,
@@ -904,36 +986,26 @@ async def run_mapped_datasource(
) from exc
except httpx.HTTPError as exc:
raise HTTPException(status_code=502, detail=f"Datasource request failed: {exc}") from exc
except (MappingError, ValueError) as exc:
except (CustomDatasourceRuntimeError, MappingError, ValueError) as exc:
raise HTTPException(status_code=400, detail=f"Mapping failed: {exc}") from exc
if mapped["failed_count"] > 0:
return {
"status": "failed",
"datasource_config_id": config_id,
"mapping_id": mapping.id,
"mapping_version": mapping.version,
"target_schema": mapping.target_schema,
"mapped_count": mapped["mapped_count"],
"failed_count": mapped["failed_count"],
"errors": mapped["errors"][:20],
}
written_count = await persist_mapped_records(
db,
datasource_name=datasource.name,
datasource_config_id=datasource.id,
target_schema=mapping.target_schema,
records=mapped["records"],
mapping_version=mapping.version,
)
@router.post("/{config_id}/stop-mapped")
async def stop_mapped_datasource(
config_id: int,
current_user: User = Depends(get_current_user),
):
stopped = await stop_custom_stream(config_id)
return {
"status": "success",
"status": "stopped" if stopped else "not_running",
"datasource_config_id": config_id,
"mapping_id": mapping.id,
"mapping_version": mapping.version,
"target_schema": mapping.target_schema,
"fetched_count": mapped["total_items"],
"mapped_count": mapped["mapped_count"],
"written_count": written_count,
"stream": get_custom_stream_status(config_id),
}
@router.get("/{config_id}/stream-status")
async def get_mapped_stream_status(
config_id: int,
current_user: User = Depends(get_current_user),
):
return get_custom_stream_status(config_id)

132
backend/app/api/v1/vessel_aggregation.py Normal file
View File

@@ -0,0 +1,132 @@
"""v4 strategy + v5 conflict-promotion + enrichment APIs for vessel_ais."""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.user import User
from app.models.vessel import AISConflictRecord
from app.services.vessel_aggregation_strategy import (
StrategyValidationError,
load_strategy,
reset_strategy,
save_strategy,
)
from app.services.vessel_enrichment import (
get_vessel_enrichment_bundle,
upsert_vessel_media_enrichment,
upsert_vessel_profile_enrichment,
)
router = APIRouter()
@router.get("/strategy")
async def get_aggregation_strategy(db: AsyncSession = Depends(get_db)):
return await load_strategy(db)
@router.put("/strategy")
async def put_aggregation_strategy(
payload: dict[str, Any],
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
return await save_strategy(db, payload)
except StrategyValidationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.delete("/strategy")
async def reset_aggregation_strategy(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await reset_strategy(db)
@router.post("/conflicts/{mmsi}/{field}/promote-to-rule")
async def promote_conflict_to_rule(
mmsi: int,
field: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Lift the current conflict resolution into a persistent strategy rule."""
result = await db.execute(
select(AISConflictRecord)
.where(AISConflictRecord.target_schema == "vessel_ais")
.where(AISConflictRecord.entity_key == str(mmsi))
.where(AISConflictRecord.field == field)
.order_by(AISConflictRecord.updated_at.desc(), AISConflictRecord.id.desc())
.limit(1)
)
record = result.scalar_one_or_none()
if record is None or not record.selected_source:
raise HTTPException(status_code=404, detail="Conflict record with selected_source not found")
strategy = await load_strategy(db)
vessel_ais = dict(strategy.get("vessel_ais") or {})
field_rules = dict(vessel_ais.get("field_rules") or {})
field_rules[field] = {"mode": "source_priority", "source_priority": [record.selected_source]}
vessel_ais["field_rules"] = field_rules
incoming = {"version": int(strategy.get("version") or 0), "vessel_ais": vessel_ais}
try:
return await save_strategy(db, incoming)
except StrategyValidationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.delete("/conflicts/{mmsi}/{field}/promote-to-rule")
async def revert_conflict_rule(
mmsi: int,
field: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
strategy = await load_strategy(db)
vessel_ais = dict(strategy.get("vessel_ais") or {})
field_rules = dict(vessel_ais.get("field_rules") or {})
if field in field_rules:
del field_rules[field]
vessel_ais["field_rules"] = field_rules
incoming = {"version": int(strategy.get("version") or 0), "vessel_ais": vessel_ais}
try:
return await save_strategy(db, incoming)
except StrategyValidationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.get("/enrichment/{mmsi}")
async def get_vessel_enrichment(mmsi: int, db: AsyncSession = Depends(get_db)):
return await get_vessel_enrichment_bundle(db, mmsi)
@router.put("/enrichment/{mmsi}/profile")
async def put_vessel_profile_enrichment(
mmsi: int,
payload: dict[str, Any],
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await upsert_vessel_profile_enrichment(db, mmsi=mmsi, payload=payload)
@router.put("/enrichment/{mmsi}/media")
async def put_vessel_media_enrichment(
mmsi: int,
payload: dict[str, Any],
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return await upsert_vessel_media_enrichment(db, mmsi=mmsi, payload=payload)

489
backend/app/api/v1/visualization.py
View File

@@ -6,6 +6,7 @@ Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium.
from datetime import UTC, datetime, timedelta
import math
import re
import httpx
from fastapi import APIRouter, HTTPException, Depends, Query, Response
from sqlalchemy.ext.asyncio import AsyncSession
@@ -19,14 +20,16 @@ from app.core.time import to_iso8601_utc
from app.db.session import get_db
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.models.collected_data import CollectedData
from app.models.vessel import VesselPosition, VesselStatic
from app.models.vessel import AISSourceHealth, VesselPosition, VesselStatic
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
from app.services.persistent_logs import record_system_log
from app.services.vessel_ais_aggregation import (
build_field_conflict_candidates,
count_unique_raw_vessel_mmsi,
get_aggregated_vessel,
get_aggregated_vessel_track,
get_aggregated_vessels,
@@ -40,6 +43,7 @@ logger = get_logger(__name__, service="api")
TERRAIN_TILE_URL_TEMPLATE = (
"https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png"
)
VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE)
# ============== Converter Functions ==============
@@ -281,6 +285,120 @@ async def _load_current_collected_data(
return list(result.scalars().all())
async def _latest_task_id_for_source(
db: AsyncSession,
source: str,
*,
exclude_unknown_name: bool = False,
) -> int | None:
stmt = (
select(
CollectedData.task_id,
func.max(CollectedData.collected_at).label("latest_collected_at"),
func.max(CollectedData.id).label("latest_id"),
)
.where(CollectedData.source == source)
.where(CollectedData.task_id.isnot(None))
.group_by(CollectedData.task_id)
.order_by(func.max(CollectedData.collected_at).desc(), func.max(CollectedData.id).desc())
.limit(1)
)
if exclude_unknown_name:
stmt = stmt.where(CollectedData.name != "Unknown")
result = await db.execute(stmt)
row = result.first()
return int(row.task_id) if row and row.task_id is not None else None
async def _load_current_or_latest_task_data(
db: AsyncSession,
source: str,
*,
exclude_unknown_name: bool = False,
limit: Optional[int] = None,
) -> List[CollectedData]:
records = await _load_current_collected_data(
db,
source,
exclude_unknown_name=exclude_unknown_name,
limit=limit,
)
if records:
return records
latest_task_id = await _latest_task_id_for_source(
db,
source,
exclude_unknown_name=exclude_unknown_name,
)
if latest_task_id is None:
return []
stmt = (
select(CollectedData)
.where(CollectedData.source == source)
.where(CollectedData.task_id == latest_task_id)
.order_by(CollectedData.id.desc())
)
if exclude_unknown_name:
stmt = stmt.where(CollectedData.name != "Unknown")
if limit is not None:
stmt = stmt.limit(limit)
result = await db.execute(stmt)
return list(result.scalars().all())
async def _count_current_or_latest_task_data(
db: AsyncSession,
source: str,
*,
exclude_unknown_name: bool = False,
) -> int:
current_stmt = (
select(func.count(CollectedData.id))
.where(CollectedData.source == source)
.where(CollectedData.is_current.is_(True))
)
if exclude_unknown_name:
current_stmt = current_stmt.where(CollectedData.name != "Unknown")
current_result = await db.execute(current_stmt)
current_scalar = current_result.scalar()
if current_scalar is None and hasattr(current_result, "scalars"):
current_rows = current_result.scalars().all()
current_count = sum(
1
for row in current_rows
if getattr(row, "source", None) == source
and (not exclude_unknown_name or getattr(row, "name", None) != "Unknown")
)
else:
current_count = int(current_scalar or 0)
if current_count > 0:
return current_count
latest_task_id = await _latest_task_id_for_source(
db,
source,
exclude_unknown_name=exclude_unknown_name,
)
if latest_task_id is None:
return 0
latest_stmt = (
select(func.count(CollectedData.id))
.where(CollectedData.source == source)
.where(CollectedData.task_id == latest_task_id)
)
if exclude_unknown_name:
latest_stmt = latest_stmt.where(CollectedData.name != "Unknown")
latest_result = await db.execute(latest_stmt)
return int(latest_result.scalar() or 0)
async def _load_current_collected_data_by_sources(
db: AsyncSession,
sources: List[str],
@@ -636,14 +754,21 @@ VESSEL_TYPE_FILTERS = {
def convert_vessels_to_geojson(rows: List[Any]) -> Dict[str, Any]:
features = []
seen_mmsi: set[int] = set()
for position, static in rows:
if position.lat is None or position.lon is None:
continue
if position.mmsi in seen_mmsi:
continue
seen_mmsi.add(position.mmsi)
props = {
"mmsi": position.mmsi,
"mmsi_display": str(position.mmsi),
"name": getattr(static, "name", None) or f"MMSI {position.mmsi}",
"name_is_fallback": _is_vessel_name_fallback(getattr(static, "name", None), position.mmsi),
"callsign": getattr(static, "callsign", None),
"imo": getattr(static, "imo", None),
"imo_display": str(getattr(static, "imo")) if getattr(static, "imo", None) else None,
"vessel_type": getattr(static, "vessel_type", None),
"vessel_type_name": getattr(static, "vessel_type_name", None) or "Other",
"flag": getattr(static, "flag", None),
@@ -685,9 +810,12 @@ def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict
}
props = {
"mmsi": vessel["mmsi"],
"mmsi_display": str(vessel["mmsi"]),
"name": vessel.get("name") or f"MMSI {vessel['mmsi']}",
"name_is_fallback": _is_vessel_name_fallback(vessel.get("name"), vessel["mmsi"]),
"callsign": vessel.get("callsign"),
"imo": vessel.get("imo"),
"imo_display": str(vessel.get("imo")) if vessel.get("imo") else None,
"vessel_type": vessel.get("vessel_type"),
"vessel_type_name": vessel.get("vessel_type_name") or "Other",
"flag": vessel.get("flag"),
@@ -704,6 +832,7 @@ def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict
"source_summary": source_summary,
"quality_flags": vessel.get("quality_flags") or [],
"conflict_count": vessel.get("conflict_count", 0),
"aggregation_strategy_version": vessel.get("aggregation_strategy_version", 0),
"data_type": "vessel",
}
features.append(
@@ -737,6 +866,24 @@ def _parse_bbox(value: Optional[str]) -> tuple[float, float, float, float] | Non
return lon_min, lat_min, lon_max, lat_max
def _is_vessel_name_fallback(name: Any, mmsi: Any) -> bool:
text = str(name or "").strip()
mmsi_text = str(mmsi or "").strip()
if not text:
return True
if mmsi_text and text == mmsi_text:
return True
return bool(VESSEL_NAME_FALLBACK_PATTERN.match(text))
def _requested_vessel_types(value: Optional[str]) -> set[str]:
return {
item.strip().lower()
for item in (value or "").split(",")
if item.strip()
}
def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bool:
if not requested_types:
return True
@@ -747,6 +894,88 @@ def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bo
return False
def _feature_mmsi_key(feature: dict[str, Any]) -> str | None:
props = feature.get("properties", {})
mmsi = props.get("mmsi") or feature.get("id")
if mmsi in (None, ""):
return None
return str(mmsi)
def _feature_in_bbox(feature: dict[str, Any], bbox: tuple[float, float, float, float] | None) -> bool:
if bbox is None:
return True
coordinates = feature.get("geometry", {}).get("coordinates") or []
if len(coordinates) < 2:
return False
try:
lon = float(coordinates[0])
lat = float(coordinates[1])
except (TypeError, ValueError):
return False
lon_min, lat_min, lon_max, lat_max = bbox
return lon_min <= lon <= lon_max and lat_min <= lat <= lat_max
def _filter_vessel_features(
features: list[dict[str, Any]],
*,
bbox: tuple[float, float, float, float] | None,
requested_types: set[str],
) -> list[dict[str, Any]]:
return [
feature
for feature in features
if _feature_in_bbox(feature, bbox)
and _matches_vessel_type(feature.get("properties", {}), requested_types)
]
def _merge_vessel_features(
raw_features: list[dict[str, Any]],
legacy_features: list[dict[str, Any]],
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
"""Prefer aggregated raw observations as the canonical source of truth.
Legacy `vessel_position` rows only fill MMSIs that the unified pipeline does
not yet know about, so a vessel never appears twice when both BarentsWatch
and AISStream observe it. Once the legacy table drains, this branch becomes
a no-op.
"""
merged: list[dict[str, Any]] = []
seen: set[str] = set()
raw_keys: set[str] = set()
legacy_keys: set[str] = set()
for feature in raw_features:
key = _feature_mmsi_key(feature)
if key is None or key in seen:
continue
seen.add(key)
raw_keys.add(key)
merged.append(feature)
legacy_added = 0
for feature in legacy_features:
key = _feature_mmsi_key(feature)
if key is None:
continue
legacy_keys.add(key)
if key in seen:
continue
seen.add(key)
legacy_added += 1
merged.append(feature)
return merged, {
"raw_unique_mmsi": len(raw_keys),
"legacy_unique_mmsi": len(legacy_keys),
"legacy_backfilled_mmsi": legacy_added,
"final_unique_mmsi": len(seen),
}
def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]:
by_type: dict[str, int] = {}
underway = 0
@@ -1347,7 +1576,7 @@ async def get_satellites_geojson(
db: AsyncSession = Depends(get_db),
):
"""获取卫星 TLE GeoJSON 数据"""
records = await _load_current_collected_data(
records = await _load_current_or_latest_task_data(
db,
"celestrak_tle",
exclude_unknown_name=True,
@@ -1476,27 +1705,30 @@ async def get_vessels_geojson(
):
"""Return latest vessel positions as GeoJSON points."""
parsed_bbox = _parse_bbox(bbox)
aggregated_vessels = await get_aggregated_vessels(db, bbox=parsed_bbox, limit=limit)
if aggregated_vessels:
geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
requested_types = {
item.strip().lower()
for item in (type or "").split(",")
if item.strip()
}
if requested_types:
geojson["features"] = [
feature
for feature in geojson.get("features", [])
if _matches_vessel_type(feature.get("properties", {}), requested_types)
]
requested_types = _requested_vessel_types(type)
merged_features, diagnostics = await _load_merged_vessel_features(db)
features = _filter_vessel_features(
merged_features,
bbox=parsed_bbox,
requested_types=requested_types,
)
if limit and limit > 0:
features = features[:limit]
return {
"type": "FeatureCollection",
"features": features,
"count": len(features),
"stats": _build_vessel_stats(features),
"diagnostics": {
**diagnostics,
"filtered_count": len(features),
},
}
features = geojson.get("features", [])
return {
**geojson,
"count": len(features),
"stats": _build_vessel_stats(features),
}
async def _load_merged_vessel_features(db: AsyncSession) -> tuple[list[dict[str, Any]], dict[str, Any]]:
aggregated_vessels = await get_aggregated_vessels(db)
raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
latest_times = (
select(
@@ -1516,50 +1748,122 @@ async def get_vessels_geojson(
.outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
.order_by(VesselPosition.received_at.desc())
)
if limit and limit > 0:
stmt = stmt.limit(limit)
if parsed_bbox is not None:
lon_min, lat_min, lon_max, lat_max = parsed_bbox
stmt = stmt.where(
VesselPosition.lon >= lon_min,
VesselPosition.lon <= lon_max,
VesselPosition.lat >= lat_min,
VesselPosition.lat <= lat_max,
)
result = await db.execute(stmt)
rows = list(result.all())
geojson = convert_vessels_to_geojson(rows)
requested_types = {
item.strip().lower()
for item in (type or "").split(",")
if item.strip()
legacy_geojson = convert_vessels_to_geojson(rows)
merged_features, diagnostics = _merge_vessel_features(
raw_geojson.get("features", []),
legacy_geojson.get("features", []),
)
return merged_features, {
**diagnostics,
"raw_feature_count": len(raw_geojson.get("features", [])),
"legacy_feature_count": len(legacy_geojson.get("features", [])),
}
if requested_types:
geojson["features"] = [
feature
for feature in geojson.get("features", [])
if _matches_vessel_type(feature.get("properties", {}), requested_types)
]
features = geojson.get("features", [])
@router.get("/vessels/custom-supplements")
async def get_vessel_custom_supplements(db: AsyncSession = Depends(get_db)):
"""Group custom vessel_ais sources by their declared merge target for diagnostics."""
from app.models.datasource_config import DataSourceConfig
result = await db.execute(
select(DataSourceConfig.name, DataSourceConfig.config, DataSourceConfig.is_active)
.where(DataSourceConfig.config["target_schema"].as_string() == "vessel_ais")
)
grouped: dict[str, dict[str, Any]] = {}
for name, config, is_active in result.all():
config = config or {}
merge_target = str(config.get("merge_target_source") or "barentswatch_vessels")
bucket = grouped.setdefault(merge_target, {"merge_target": merge_target, "sources": []})
bucket["sources"].append({"name": name, "is_active": bool(is_active)})
return {"groups": list(grouped.values())}
@router.get("/vessels/name-fallbacks")
async def get_vessel_name_fallbacks(
limit: int = Query(500, ge=0, description="Maximum fallback-name vessels to return. 0 means no limit."),
db: AsyncSession = Depends(get_db),
):
"""Return vessels whose display name still falls back to MMSI."""
aggregated_vessels = await get_aggregated_vessels(db)
raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
latest_times = (
select(
VesselPosition.mmsi.label("mmsi"),
func.max(VesselPosition.received_at).label("received_at"),
)
.group_by(VesselPosition.mmsi)
.subquery()
)
result = await db.execute(
select(VesselPosition, VesselStatic)
.join(
latest_times,
(VesselPosition.mmsi == latest_times.c.mmsi)
& (VesselPosition.received_at == latest_times.c.received_at),
)
.outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
.order_by(VesselPosition.received_at.desc())
)
legacy_geojson = convert_vessels_to_geojson(list(result.all()))
features, diagnostics = _merge_vessel_features(
raw_geojson.get("features", []),
legacy_geojson.get("features", []),
)
fallback_items = []
for feature in features:
props = feature.get("properties", {})
mmsi = props.get("mmsi")
name = props.get("name")
if not _is_vessel_name_fallback(name, mmsi):
continue
source_summary = props.get("source_summary") or {}
fallback_items.append(
{
"mmsi": str(mmsi),
"display_name": name or f"MMSI {mmsi}",
"reason": "missing_real_name",
"received_at": props.get("received_at"),
"sources": sorted(source_summary.keys()),
"source_summary": source_summary,
"message_types": sorted(
{
message_type
for summary in source_summary.values()
for message_type in (summary.get("message_types") or [])
}
),
"field_sources": props.get("field_sources") or {},
}
)
if limit and limit > 0:
fallback_items = fallback_items[:limit]
return {
**geojson,
"count": len(features),
"stats": _build_vessel_stats(features),
"count": len(fallback_items),
"items": fallback_items,
"diagnostics": diagnostics,
}
@router.get("/vessels/{mmsi}")
async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)):
from app.services.vessel_enrichment import get_vessel_enrichment_bundle
aggregated = await get_aggregated_vessel(db, mmsi)
enrichment = await get_vessel_enrichment_bundle(db, mmsi)
if aggregated is not None:
return {
**aggregated,
"received_at": to_iso8601_utc(aggregated.get("received_at")),
"latitude": aggregated["lat"],
"longitude": aggregated["lon"],
"enrichment": enrichment,
}
latest_position_stmt = (
@@ -1578,6 +1882,7 @@ async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)):
**(geojson["features"][0]["properties"]),
"latitude": position.lat,
"longitude": position.lon,
"enrichment": enrichment,
}
@@ -1737,31 +2042,16 @@ async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db)):
@router.get("/geo/summary")
async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
"""Return lightweight Earth HUD counts without loading layer GeoJSON payloads."""
records_by_source = await _load_current_collected_data_by_sources(
cable_count = await _count_current_or_latest_task_data(db, "arcgis_cables")
landing_point_count = await _count_current_or_latest_task_data(db, "arcgis_landing_points")
satellite_count = await _count_current_or_latest_task_data(
db,
[
"arcgis_cables",
"arcgis_landing_points",
"celestrak_tle",
"top500",
"epoch_ai_gpu",
],
"celestrak_tle",
exclude_unknown_name=True,
)
cables = convert_cable_to_geojson(records_by_source.get("arcgis_cables", []))
landing_points = convert_landing_point_to_geojson(
records_by_source.get("arcgis_landing_points", []),
)
satellites = convert_satellite_to_geojson(
_filter_known_records(records_by_source.get("celestrak_tle", [])),
)
compute_centers = convert_compute_centers_to_geojson(
_filter_known_records(
records_by_source.get("top500", [])
+ records_by_source.get("epoch_ai_gpu", []),
),
)
compute_features = compute_centers.get("features", [])
supercomputer_count = await _count_current_or_latest_task_data(db, "top500")
gpu_cluster_count = await _count_current_or_latest_task_data(db, "epoch_ai_gpu")
compute_center_count = supercomputer_count + gpu_cluster_count
active_incident_result = await db.execute(
select(func.count(BGPIncident.id)).where(BGPIncident.status == "active"),
@@ -1771,35 +2061,56 @@ async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
)
active_incident_count = int(active_incident_result.scalar() or 0)
active_anomaly_count = int(active_anomaly_result.scalar() or 0)
bgp_collectors = await build_bgp_collector_coverage(
bgp_collector_result = await db.execute(
select(func.count(func.distinct(BGPObservation.collector)))
.where(BGPObservation.collector.isnot(None))
.where(func.length(func.btrim(BGPObservation.collector)) > 0)
.where(BGPObservation.source.in_(("ris_live_bgp", "bgpstream_bgp")))
)
bgp_collector_scalar = bgp_collector_result.scalar()
if bgp_collector_scalar is None:
bgp_collectors = await build_bgp_collector_coverage(
db,
source_filter=("ris_live_bgp", "bgpstream_bgp"),
)
bgp_collector_count = len(
[item for item in bgp_collectors if item.get("collector")]
)
else:
bgp_collector_count = int(bgp_collector_scalar or 0)
raw_unique_window_hours = 24
raw_unique_mmsi = await count_unique_raw_vessel_mmsi(
db,
source_filter=("ris_live_bgp", "bgpstream_bgp"),
observed_since=datetime.now(UTC) - timedelta(hours=raw_unique_window_hours),
)
vessel_count_result = await db.execute(
select(func.count(func.distinct(VesselPosition.mmsi))),
legacy_unique_result = await db.execute(
select(func.count(func.distinct(VesselPosition.mmsi)))
)
vessel_count = int(vessel_count_result.scalar() or 0)
legacy_unique_mmsi = int(legacy_unique_result.scalar() or 0)
vessel_count = max(raw_unique_mmsi, legacy_unique_mmsi)
aisstream_health = await db.get(AISSourceHealth, "aisstream_vessels")
return {
"generated_at": to_iso8601_utc(datetime.now(UTC)),
"stats": {
"cable_count": len(cables.get("features", [])),
"landing_point_count": len(landing_points.get("features", [])),
"satellite_count": len(satellites.get("features", [])),
"compute_center_count": len(compute_features),
"cable_count": cable_count,
"landing_point_count": landing_point_count,
"satellite_count": satellite_count,
"compute_center_count": compute_center_count,
"vessel_count": vessel_count,
"supercomputer_count": sum(
1 for feature in compute_features
if feature.get("properties", {}).get("site_type") == "supercomputer"
),
"gpu_cluster_count": sum(
1 for feature in compute_features
if feature.get("properties", {}).get("site_type") == "gpu_cluster"
),
"vessel_raw_unique_mmsi": raw_unique_mmsi,
"vessel_raw_unique_window_hours": raw_unique_window_hours,
"vessel_legacy_unique_mmsi": legacy_unique_mmsi,
"aisstream_connection_state": aisstream_health.connection_state if aisstream_health else None,
"aisstream_last_seen_at": to_iso8601_utc(aisstream_health.last_seen_at) if aisstream_health else None,
"aisstream_message_rate": aisstream_health.message_rate if aisstream_health else None,
"aisstream_lag_seconds": aisstream_health.lag_seconds if aisstream_health else None,
"supercomputer_count": supercomputer_count,
"gpu_cluster_count": gpu_cluster_count,
"bgp_event_count": active_incident_count or active_anomaly_count,
"bgp_incident_count": active_incident_count,
"bgp_anomaly_count": active_anomaly_count,
"bgp_collector_count": len([item for item in bgp_collectors if item.get("collector")]),
"bgp_collector_count": bgp_collector_count,
},
}

41
backend/app/api/v1/websocket.py
View File

@@ -40,16 +40,16 @@ async def authenticate_token(token: str) -> Optional[dict]:
@router.websocket("/ws")
async def websocket_endpoint(
websocket: WebSocket,
token: str = Query(...),
token: str | None = Query(None),
):
"""WebSocket endpoint for real-time data"""
logger.info_event(
"WebSocket connection attempt",
event="auth.websocket.connection_attempt",
context={"token_preview": f"{token[:8]}..."},
context={"token_preview": f"{token[:8]}..." if token else "anonymous"},
)
payload = await authenticate_token(token)
if payload is None:
payload = await authenticate_token(token) if token else None
if token and payload is None:
logger.warning_event(
"WebSocket authentication failed, closing connection",
event="auth.websocket.connection_rejected",
@@ -57,7 +57,17 @@ async def websocket_endpoint(
await websocket.close(code=4001)
return
user_id = str(payload.get("sub"))
is_anonymous = payload is None
user_id = str(payload.get("sub")) if payload else f"anonymous:{id(websocket)}"
supported_channels = ["vessels"] if is_anonymous else [
"gpu_clusters",
"submarine_cables",
"ixp_nodes",
"alerts",
"dashboard",
"datasource_tasks",
"vessels",
]
await manager.connect(websocket, user_id)
try:
@@ -68,14 +78,7 @@ async def websocket_endpoint(
"connection_id": f"conn_{user_id}",
"server_version": settings.VERSION,
"heartbeat_interval": 30,
"supported_channels": [
"gpu_clusters",
"submarine_cables",
"ixp_nodes",
"alerts",
"dashboard",
"datasource_tasks",
],
"supported_channels": supported_channels,
},
}
)
@@ -93,12 +96,24 @@ async def websocket_endpoint(
)
elif data.get("type") == "subscribe":
channels = data.get("data", {}).get("channels", [])
if is_anonymous:
channels = [channel for channel in channels if channel in supported_channels]
manager.subscribe(websocket, channels)
await websocket.send_json(
{
"type": "subscription_confirmed",
"data": {"action": "subscribe", "channels": channels},
}
)
elif data.get("type") == "unsubscribe":
channels = data.get("data", {}).get("channels", [])
manager.unsubscribe(websocket, channels)
await websocket.send_json(
{
"type": "subscription_confirmed",
"data": {"action": "unsubscribe", "channels": channels},
}
)
elif data.get("type") == "control_frame":
await websocket.send_json(
{"type": "control_acknowledged", "data": {"received": True}}