release: bump version to 0.47.0
This commit is contained in:
@@ -5,7 +5,7 @@ from datetime import datetime
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from pydantic import BaseModel, Field
|
||||
@@ -318,7 +318,7 @@ async def list_configs(
|
||||
"""List all user-defined data source configurations"""
|
||||
query = select(DataSourceConfig)
|
||||
if active_only:
|
||||
query = query.where(DataSourceConfig.is_active == True)
|
||||
query = query.where(DataSourceConfig.is_active)
|
||||
query = query.order_by(DataSourceConfig.created_at.desc())
|
||||
|
||||
result = await db.execute(query)
|
||||
@@ -374,6 +374,11 @@ async def list_all_datasources(
|
||||
"is_active": db_config.is_active if db_config else True,
|
||||
"source_type": db_config.source_type if db_config else "http",
|
||||
"auth_type": db_config.auth_type if db_config else "none",
|
||||
"auth_configured": {
|
||||
"api_key": bool((db_config.auth_config or {}).get("api_key"))
|
||||
if db_config
|
||||
else False,
|
||||
},
|
||||
"headers": db_config.headers if db_config else {},
|
||||
"config": strip_connectivity_validation(db_config.config if db_config else {}),
|
||||
"config_id": db_config.id if db_config else None,
|
||||
@@ -464,6 +469,8 @@ async def update_config(
|
||||
for field, value in update_data.items():
|
||||
if field == "config":
|
||||
value = strip_connectivity_validation(value)
|
||||
if field == "auth_config" and value == {} and (config.auth_config or {}):
|
||||
continue
|
||||
setattr(config, field, value)
|
||||
|
||||
await db.commit()
|
||||
@@ -601,6 +608,7 @@ async def connect_builtin_config(
|
||||
config_data.headers,
|
||||
config_data.config,
|
||||
db,
|
||||
config_data.auth_config,
|
||||
)
|
||||
if result.get("success") and result.get("checksum"):
|
||||
validation = await save_connectivity_success(
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.models.user import User
|
||||
from app.models.vessel import AISSourceHealth
|
||||
from app.services.barentswatch import (
|
||||
BarentsWatchConfig,
|
||||
check_barentswatch_config,
|
||||
@@ -368,7 +369,12 @@ def format_frequency_label(minutes: int) -> str:
|
||||
return f"{minutes}m"
|
||||
|
||||
|
||||
def serialize_collector(datasource: DataSource) -> dict:
|
||||
async def get_ais_source_health_by_source(db: AsyncSession) -> dict[str, dict]:
|
||||
result = await db.execute(select(AISSourceHealth))
|
||||
return {item.source: item.to_dict() for item in result.scalars().all()}
|
||||
|
||||
|
||||
def serialize_collector(datasource: DataSource, ais_health_by_source: dict[str, dict] | None = None) -> dict:
|
||||
defaults = DEFAULT_DATASOURCES.get(datasource.source, {})
|
||||
return {
|
||||
"id": datasource.id,
|
||||
@@ -387,6 +393,7 @@ def serialize_collector(datasource: DataSource) -> dict:
|
||||
"requires_credentials": bool(defaults.get("requires_credentials", False)),
|
||||
"credential_provider": defaults.get("credential_provider"),
|
||||
"credential_status": defaults.get("credential_status", "none"),
|
||||
"ais_health": (ais_health_by_source or {}).get(datasource.source),
|
||||
}
|
||||
|
||||
|
||||
@@ -599,7 +606,8 @@ async def get_collector_settings(
|
||||
):
|
||||
result = await db.execute(select(DataSource).order_by(DataSource.module, DataSource.id))
|
||||
datasources = result.scalars().all()
|
||||
return {"collectors": [serialize_collector(datasource) for datasource in datasources]}
|
||||
ais_health_by_source = await get_ais_source_health_by_source(db)
|
||||
return {"collectors": [serialize_collector(datasource, ais_health_by_source) for datasource in datasources]}
|
||||
|
||||
|
||||
@router.put("/collectors/{datasource_id}")
|
||||
@@ -619,7 +627,8 @@ async def update_collector_settings(
|
||||
await db.commit()
|
||||
await db.refresh(datasource)
|
||||
await sync_datasource_job(datasource.id)
|
||||
return {"status": "updated", "collector": serialize_collector(datasource)}
|
||||
ais_health_by_source = await get_ais_source_health_by_source(db)
|
||||
return {"status": "updated", "collector": serialize_collector(datasource, ais_health_by_source)}
|
||||
|
||||
|
||||
@router.get("")
|
||||
@@ -633,12 +642,13 @@ async def get_all_settings(
|
||||
db,
|
||||
["system", "notifications", "security"],
|
||||
)
|
||||
ais_health_by_source = await get_ais_source_health_by_source(db)
|
||||
return {
|
||||
"system": setting_payloads["system"],
|
||||
"notifications": setting_payloads["notifications"],
|
||||
"security": setting_payloads["security"],
|
||||
"tv": await get_tv_settings_payload(db),
|
||||
"integrations": await serialize_external_integrations(db),
|
||||
"collectors": [serialize_collector(datasource) for datasource in datasources],
|
||||
"collectors": [serialize_collector(datasource, ais_health_by_source) for datasource in datasources],
|
||||
"generated_at": to_iso8601_utc(datetime.now(UTC)),
|
||||
}
|
||||
|
||||
@@ -25,6 +25,14 @@ from app.services.bgp_collectors import build_bgp_collector_coverage
|
||||
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
|
||||
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
|
||||
from app.services.persistent_logs import record_system_log
|
||||
from app.services.vessel_ais_aggregation import (
|
||||
build_field_conflict_candidates,
|
||||
get_aggregated_vessel,
|
||||
get_aggregated_vessel_track,
|
||||
get_aggregated_vessels,
|
||||
get_vessel_conflict_records,
|
||||
get_vessel_raw_observations,
|
||||
)
|
||||
from app.core.logging import get_logger
|
||||
|
||||
router = APIRouter()
|
||||
@@ -664,6 +672,54 @@ def convert_vessels_to_geojson(rows: List[Any]) -> Dict[str, Any]:
|
||||
return {"type": "FeatureCollection", "features": features}
|
||||
|
||||
|
||||
def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict[str, Any]:
|
||||
features = []
|
||||
for vessel in vessels:
|
||||
if vessel.get("lat") is None or vessel.get("lon") is None:
|
||||
continue
|
||||
source_summary = {}
|
||||
for source, summary in (vessel.get("source_summary") or {}).items():
|
||||
source_summary[source] = {
|
||||
**summary,
|
||||
"latest_observed_at": to_iso8601_utc(summary.get("latest_observed_at")),
|
||||
}
|
||||
props = {
|
||||
"mmsi": vessel["mmsi"],
|
||||
"name": vessel.get("name") or f"MMSI {vessel['mmsi']}",
|
||||
"callsign": vessel.get("callsign"),
|
||||
"imo": vessel.get("imo"),
|
||||
"vessel_type": vessel.get("vessel_type"),
|
||||
"vessel_type_name": vessel.get("vessel_type_name") or "Other",
|
||||
"flag": vessel.get("flag"),
|
||||
"length": vessel.get("length"),
|
||||
"width": vessel.get("width"),
|
||||
"draught": vessel.get("draught"),
|
||||
"sog": vessel.get("sog"),
|
||||
"cog": vessel.get("cog"),
|
||||
"heading": vessel.get("heading"),
|
||||
"nav_status": vessel.get("nav_status"),
|
||||
"received_at": to_iso8601_utc(vessel.get("received_at")),
|
||||
"field_sources": vessel.get("field_sources") or {},
|
||||
"selected_reasons": vessel.get("selected_reasons") or {},
|
||||
"source_summary": source_summary,
|
||||
"quality_flags": vessel.get("quality_flags") or [],
|
||||
"conflict_count": vessel.get("conflict_count", 0),
|
||||
"data_type": "vessel",
|
||||
}
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": vessel["mmsi"],
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [vessel["lon"], vessel["lat"]],
|
||||
},
|
||||
"properties": props,
|
||||
}
|
||||
)
|
||||
return {"type": "FeatureCollection", "features": features}
|
||||
|
||||
|
||||
def _parse_bbox(value: Optional[str]) -> tuple[float, float, float, float] | None:
|
||||
if not value:
|
||||
return None
|
||||
@@ -1411,10 +1467,37 @@ async def get_vessels_geojson(
|
||||
None,
|
||||
description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other",
|
||||
),
|
||||
limit: int = Query(5000, ge=1, le=50000),
|
||||
limit: Optional[int] = Query(
|
||||
None,
|
||||
ge=0,
|
||||
description="Maximum vessel features to return. Omit or pass 0 for no limit.",
|
||||
),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Return latest vessel positions as GeoJSON points."""
|
||||
parsed_bbox = _parse_bbox(bbox)
|
||||
aggregated_vessels = await get_aggregated_vessels(db, bbox=parsed_bbox, limit=limit)
|
||||
if aggregated_vessels:
|
||||
geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
|
||||
requested_types = {
|
||||
item.strip().lower()
|
||||
for item in (type or "").split(",")
|
||||
if item.strip()
|
||||
}
|
||||
if requested_types:
|
||||
geojson["features"] = [
|
||||
feature
|
||||
for feature in geojson.get("features", [])
|
||||
if _matches_vessel_type(feature.get("properties", {}), requested_types)
|
||||
]
|
||||
|
||||
features = geojson.get("features", [])
|
||||
return {
|
||||
**geojson,
|
||||
"count": len(features),
|
||||
"stats": _build_vessel_stats(features),
|
||||
}
|
||||
|
||||
latest_times = (
|
||||
select(
|
||||
VesselPosition.mmsi.label("mmsi"),
|
||||
@@ -1432,10 +1515,10 @@ async def get_vessels_geojson(
|
||||
)
|
||||
.outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
|
||||
.order_by(VesselPosition.received_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if limit and limit > 0:
|
||||
stmt = stmt.limit(limit)
|
||||
|
||||
parsed_bbox = _parse_bbox(bbox)
|
||||
if parsed_bbox is not None:
|
||||
lon_min, lat_min, lon_max, lat_max = parsed_bbox
|
||||
stmt = stmt.where(
|
||||
@@ -1470,6 +1553,15 @@ async def get_vessels_geojson(
|
||||
|
||||
@router.get("/vessels/{mmsi}")
|
||||
async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)):
|
||||
aggregated = await get_aggregated_vessel(db, mmsi)
|
||||
if aggregated is not None:
|
||||
return {
|
||||
**aggregated,
|
||||
"received_at": to_iso8601_utc(aggregated.get("received_at")),
|
||||
"latitude": aggregated["lat"],
|
||||
"longitude": aggregated["lon"],
|
||||
}
|
||||
|
||||
latest_position_stmt = (
|
||||
select(VesselPosition)
|
||||
.where(VesselPosition.mmsi == mmsi)
|
||||
@@ -1496,6 +1588,30 @@ async def get_vessel_track(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
cutoff = datetime.now(UTC) - timedelta(hours=hours)
|
||||
aggregated_points = await get_aggregated_vessel_track(db, mmsi, cutoff=cutoff)
|
||||
if aggregated_points:
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "LineString",
|
||||
"coordinates": [[point["lon"], point["lat"]] for point in aggregated_points],
|
||||
},
|
||||
"properties": {
|
||||
"mmsi": mmsi,
|
||||
"hours": hours,
|
||||
"point_count": len(aggregated_points),
|
||||
"start_at": to_iso8601_utc(aggregated_points[0]["observed_at"]),
|
||||
"end_at": to_iso8601_utc(aggregated_points[-1]["observed_at"]),
|
||||
"point_sources": [point["source"] for point in aggregated_points],
|
||||
},
|
||||
}
|
||||
],
|
||||
"count": 1,
|
||||
}
|
||||
|
||||
result = await db.execute(
|
||||
select(VesselPosition)
|
||||
.where(VesselPosition.mmsi == mmsi)
|
||||
@@ -1532,6 +1648,37 @@ async def get_vessel_track(
|
||||
}
|
||||
|
||||
|
||||
@router.get("/vessels/{mmsi}/observations")
|
||||
async def get_vessel_observations(
|
||||
mmsi: int,
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Return raw AIS observations for debugging source-level collector facts."""
|
||||
|
||||
observations = await get_vessel_raw_observations(db, mmsi, limit=limit)
|
||||
return {
|
||||
"mmsi": mmsi,
|
||||
"count": len(observations),
|
||||
"observations": [item.to_dict() for item in observations],
|
||||
"conflict_candidates": build_field_conflict_candidates(observations),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/vessels/{mmsi}/conflicts")
|
||||
async def get_vessel_conflicts(mmsi: int, db: AsyncSession = Depends(get_db)):
|
||||
"""Return recorded AIS conflicts plus current raw-observation candidates."""
|
||||
|
||||
records = await get_vessel_conflict_records(db, mmsi)
|
||||
observations = await get_vessel_raw_observations(db, mmsi, limit=500)
|
||||
return {
|
||||
"mmsi": mmsi,
|
||||
"count": len(records),
|
||||
"conflicts": [item.to_dict() for item in records],
|
||||
"candidates": build_field_conflict_candidates(observations),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/geo/bgp-anomalies")
|
||||
async def get_bgp_anomalies_geojson(
|
||||
severity: Optional[str] = Query(None),
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import os
|
||||
import yaml
|
||||
from functools import lru_cache
|
||||
from typing import Optional
|
||||
|
||||
|
||||
COLLECTOR_URL_KEYS = {
|
||||
@@ -32,6 +31,7 @@ COLLECTOR_URL_KEYS = {
|
||||
"nro_delegated_prefix_geo": "nro.delegated_stats_url",
|
||||
"news_live_streams": "news_live_streams.channels_url",
|
||||
"barentswatch_vessels": "barentswatch_vessels.url",
|
||||
"aisstream_vessels": "aisstream_vessels.url",
|
||||
}
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ class DataSourcesConfig:
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
|
||||
query = select(DataSourceConfig).where(
|
||||
DataSourceConfig.name == collector_name, DataSourceConfig.is_active == True
|
||||
DataSourceConfig.name == collector_name, DataSourceConfig.is_active
|
||||
)
|
||||
result = await db.execute(query)
|
||||
db_config = result.scalar_one_or_none()
|
||||
|
||||
@@ -98,3 +98,7 @@ news_live_streams:
|
||||
barentswatch_vessels:
|
||||
# BarentsWatch Live AIS latest combined endpoint. Requires an AIS bearer token.
|
||||
url: "https://live.ais.barentswatch.no/v1/latest/combined"
|
||||
|
||||
aisstream_vessels:
|
||||
# AISStream realtime WebSocket endpoint. Requires an AISStream API key.
|
||||
url: "wss://stream.aisstream.io/v0/stream"
|
||||
|
||||
@@ -245,6 +245,18 @@ DEFAULT_DATASOURCES = {
|
||||
"credential_provider": "barentswatch",
|
||||
"credential_status": "supported",
|
||||
},
|
||||
"aisstream_vessels": {
|
||||
"id": 28,
|
||||
"name": "AISStream Vessels",
|
||||
"display_name": "AISStream 实时船舶",
|
||||
"module": "L4",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 1,
|
||||
"is_free": True,
|
||||
"requires_credentials": True,
|
||||
"credential_provider": "aisstream",
|
||||
"credential_status": "supported",
|
||||
},
|
||||
}
|
||||
|
||||
ID_TO_COLLECTOR = {info["id"]: name for name, info in DEFAULT_DATASOURCES.items()}
|
||||
|
||||
@@ -12,7 +12,7 @@ from app.models.system_setting import SystemSetting
|
||||
from app.models.playground_session import PlaygroundSession
|
||||
from app.models.playground_message import PlaygroundMessage
|
||||
from app.models.system_log import SystemLog, AuditLog
|
||||
from app.models.vessel import VesselPosition, VesselStatic
|
||||
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth, VesselPosition, VesselStatic
|
||||
from app.models.datasource_mapping import DataSourceMappingTemplate
|
||||
|
||||
__all__ = [
|
||||
@@ -31,7 +31,12 @@ __all__ = [
|
||||
"BGPObservation",
|
||||
"SystemLog",
|
||||
"AuditLog",
|
||||
"PlaygroundSession",
|
||||
"PlaygroundMessage",
|
||||
"VesselPosition",
|
||||
"VesselStatic",
|
||||
"AISRawObservation",
|
||||
"AISConflictRecord",
|
||||
"AISSourceHealth",
|
||||
"DataSourceMappingTemplate",
|
||||
]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Vessel AIS models for live maritime tracking."""
|
||||
|
||||
from sqlalchemy import BigInteger, Column, DateTime, Float, Index, Integer, SmallInteger, String
|
||||
from sqlalchemy import BigInteger, Column, DateTime, Float, Index, Integer, JSON, SmallInteger, String
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.core.time import to_iso8601_utc
|
||||
@@ -73,3 +73,113 @@ class VesselPosition(Base):
|
||||
"nav_status": self.nav_status,
|
||||
"received_at": to_iso8601_utc(self.received_at),
|
||||
}
|
||||
|
||||
|
||||
class AISRawObservation(Base):
|
||||
"""Source-level AIS fact before aggregation and conflict resolution."""
|
||||
|
||||
__tablename__ = "ais_raw_observations"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
target_schema = Column(String(64), nullable=False, default="vessel_ais", index=True)
|
||||
source = Column(String(100), nullable=False, index=True)
|
||||
entity_key = Column(String(64), nullable=False, index=True)
|
||||
delivery_mode = Column(String(32), nullable=False, index=True)
|
||||
transport = Column(String(32), nullable=False, index=True)
|
||||
message_type = Column(String(64), nullable=True, index=True)
|
||||
source_message_id = Column(String(128), nullable=True, index=True)
|
||||
observation_hash = Column(String(64), nullable=False, unique=True, index=True)
|
||||
observed_at = Column(DateTime(timezone=True), nullable=False, index=True)
|
||||
collected_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
|
||||
normalized_payload = Column(JSON, default=dict)
|
||||
raw_payload = Column(JSON, default=dict)
|
||||
quality_flags = Column(JSON, default=list)
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_ais_raw_entity_observed", "target_schema", "entity_key", "observed_at"),
|
||||
Index("idx_ais_raw_source_entity", "source", "entity_key"),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"target_schema": self.target_schema,
|
||||
"source": self.source,
|
||||
"entity_key": self.entity_key,
|
||||
"delivery_mode": self.delivery_mode,
|
||||
"transport": self.transport,
|
||||
"message_type": self.message_type,
|
||||
"source_message_id": self.source_message_id,
|
||||
"observation_hash": self.observation_hash,
|
||||
"observed_at": to_iso8601_utc(self.observed_at),
|
||||
"collected_at": to_iso8601_utc(self.collected_at),
|
||||
"normalized_payload": self.normalized_payload or {},
|
||||
"raw_payload": self.raw_payload or {},
|
||||
"quality_flags": self.quality_flags or [],
|
||||
}
|
||||
|
||||
|
||||
class AISConflictRecord(Base):
|
||||
"""Recorded field-level disagreement between AIS sources."""
|
||||
|
||||
__tablename__ = "ais_conflict_records"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
target_schema = Column(String(64), nullable=False, default="vessel_ais", index=True)
|
||||
entity_key = Column(String(64), nullable=False, index=True)
|
||||
field = Column(String(64), nullable=False, index=True)
|
||||
candidates = Column(JSON, default=dict)
|
||||
selected_source = Column(String(100), nullable=True, index=True)
|
||||
selected_value = Column(JSON, nullable=True)
|
||||
selected_reason = Column(String(64), nullable=True, index=True)
|
||||
resolved_by = Column(String(32), nullable=False, default="system", index=True)
|
||||
status = Column(String(32), nullable=False, default="open", index=True)
|
||||
created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
|
||||
updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_ais_conflict_entity_field", "target_schema", "entity_key", "field"),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"target_schema": self.target_schema,
|
||||
"entity_key": self.entity_key,
|
||||
"field": self.field,
|
||||
"candidates": self.candidates or {},
|
||||
"selected_source": self.selected_source,
|
||||
"selected_value": self.selected_value,
|
||||
"selected_reason": self.selected_reason,
|
||||
"resolved_by": self.resolved_by,
|
||||
"status": self.status,
|
||||
"created_at": to_iso8601_utc(self.created_at),
|
||||
"updated_at": to_iso8601_utc(self.updated_at),
|
||||
}
|
||||
|
||||
|
||||
class AISSourceHealth(Base):
|
||||
"""Runtime health signal for an AIS collector source."""
|
||||
|
||||
__tablename__ = "ais_source_health"
|
||||
|
||||
source = Column(String(100), primary_key=True)
|
||||
connection_state = Column(String(32), nullable=False, default="disconnected", index=True)
|
||||
last_seen_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
last_success_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
last_error = Column(String(500), nullable=True)
|
||||
message_rate = Column(Float, nullable=True)
|
||||
lag_seconds = Column(Float, nullable=True)
|
||||
updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"source": self.source,
|
||||
"connection_state": self.connection_state,
|
||||
"last_seen_at": to_iso8601_utc(self.last_seen_at),
|
||||
"last_success_at": to_iso8601_utc(self.last_success_at),
|
||||
"last_error": self.last_error,
|
||||
"message_rate": self.message_rate,
|
||||
"lag_seconds": self.lag_seconds,
|
||||
"updated_at": to_iso8601_utc(self.updated_at),
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
|
||||
from app.services.collectors.opengeofeed import OpenGeoFeedPrefixGeoCollector
|
||||
from app.services.collectors.nro_delegated import NRODelegatedPrefixGeoCollector
|
||||
from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
|
||||
from app.services.collectors.aisstream import AISStreamCollector
|
||||
from app.services.collectors.vessel_ais import VesselAISCollector
|
||||
|
||||
collector_registry.register(TOP500Collector())
|
||||
@@ -65,3 +66,40 @@ collector_registry.register(OpenGeoFeedPrefixGeoCollector())
|
||||
collector_registry.register(NRODelegatedPrefixGeoCollector())
|
||||
collector_registry.register(NewsLiveStreamsCollector())
|
||||
collector_registry.register(VesselAISCollector())
|
||||
collector_registry.register(AISStreamCollector())
|
||||
|
||||
__all__ = [
|
||||
"BaseCollector",
|
||||
"HTTPCollector",
|
||||
"IntervalCollector",
|
||||
"collector_registry",
|
||||
"CollectorRegistry",
|
||||
"TOP500Collector",
|
||||
"EpochAIGPUCollector",
|
||||
"HuggingFaceModelCollector",
|
||||
"HuggingFaceDatasetCollector",
|
||||
"HuggingFaceSpacesCollector",
|
||||
"PeeringDBIXPCollector",
|
||||
"PeeringDBNetworkCollector",
|
||||
"PeeringDBFacilityCollector",
|
||||
"TeleGeographyCableCollector",
|
||||
"TeleGeographyLandingPointCollector",
|
||||
"TeleGeographyCableSystemCollector",
|
||||
"CloudflareRadarDeviceCollector",
|
||||
"CloudflareRadarTrafficCollector",
|
||||
"CloudflareRadarTopASCollector",
|
||||
"ArcGISCableCollector",
|
||||
"FAOLandingPointCollector",
|
||||
"ArcGISLandingPointCollector",
|
||||
"ArcGISCableLandingRelationCollector",
|
||||
"SpaceTrackTLECollector",
|
||||
"CelesTrakTLECollector",
|
||||
"RISLiveCollector",
|
||||
"BGPStreamBackfillCollector",
|
||||
"IPtoASNPrefixGeoCollector",
|
||||
"OpenGeoFeedPrefixGeoCollector",
|
||||
"NRODelegatedPrefixGeoCollector",
|
||||
"NewsLiveStreamsCollector",
|
||||
"VesselAISCollector",
|
||||
"AISStreamCollector",
|
||||
]
|
||||
|
||||
276
backend/app/services/collectors/aisstream.py
Normal file
276
backend/app/services/collectors/aisstream.py
Normal file
@@ -0,0 +1,276 @@
|
||||
"""AISStream WebSocket collector for realtime vessel AIS observations."""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.services.collectors.base import BaseCollector
|
||||
from app.services.vessel_ais_aggregation import (
|
||||
AISSTREAM_DELIVERY_MODE,
|
||||
AISSTREAM_TRANSPORT,
|
||||
record_vessel_ais_observation,
|
||||
update_ais_source_health,
|
||||
)
|
||||
from app.services.vessel_types import normalize_vessel_type_name
|
||||
|
||||
DEFAULT_AISSTREAM_URL = "wss://stream.aisstream.io/v0/stream"
|
||||
DEFAULT_BOUNDING_BOXES = [[[-90, -180], [90, 180]]]
|
||||
DEFAULT_MESSAGE_TYPES = ["PositionReport", "ShipStaticData"]
|
||||
|
||||
|
||||
class AISStreamCollector(BaseCollector):
|
||||
"""Collect AISStream WebSocket messages into the raw AIS observation layer."""
|
||||
|
||||
name = "aisstream_vessels"
|
||||
priority = "P1"
|
||||
module = "L4"
|
||||
frequency_hours = 1
|
||||
data_type = "vessel_ais"
|
||||
fail_on_empty = False
|
||||
|
||||
async def _load_datasource_config(self) -> DataSourceConfig | None:
|
||||
if self._db_session is None:
|
||||
return None
|
||||
result = await self._db_session.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == self.name)
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def _get_effective_config(self) -> dict[str, Any]:
|
||||
datasource_config = await self._load_datasource_config()
|
||||
config = dict(datasource_config.config or {}) if datasource_config else {}
|
||||
auth_config = dict(datasource_config.auth_config or {}) if datasource_config else {}
|
||||
endpoint = (
|
||||
(datasource_config.endpoint if datasource_config else None)
|
||||
or self._resolved_url
|
||||
or get_data_sources_config().get_yaml_url(self.name)
|
||||
or DEFAULT_AISSTREAM_URL
|
||||
)
|
||||
api_key = (
|
||||
auth_config.get("api_key")
|
||||
or config.get("api_key")
|
||||
or os.getenv("AISSTREAM_API_KEY")
|
||||
)
|
||||
return {
|
||||
"endpoint": endpoint,
|
||||
"api_key": api_key,
|
||||
"bounding_boxes": config.get("bounding_boxes") or DEFAULT_BOUNDING_BOXES,
|
||||
"message_types": config.get("message_types") or DEFAULT_MESSAGE_TYPES,
|
||||
"max_messages": int(config.get("max_messages") or 500),
|
||||
"receive_timeout_seconds": float(config.get("receive_timeout_seconds") or 30),
|
||||
}
|
||||
|
||||
async def fetch(self) -> list[dict[str, Any]]:
|
||||
config = await self._get_effective_config()
|
||||
if not config["api_key"]:
|
||||
raise RuntimeError("AISStream API key is not configured")
|
||||
|
||||
try:
|
||||
import websockets
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("Python package 'websockets' is required for AISStream") from exc
|
||||
|
||||
subscription = {
|
||||
"APIKey": config["api_key"],
|
||||
"BoundingBoxes": config["bounding_boxes"],
|
||||
"FilterMessageTypes": config["message_types"],
|
||||
}
|
||||
|
||||
messages: list[dict[str, Any]] = []
|
||||
try:
|
||||
async with websockets.connect(config["endpoint"]) as websocket:
|
||||
await websocket.send(json.dumps(subscription))
|
||||
while len(messages) < config["max_messages"]:
|
||||
try:
|
||||
raw_message = await asyncio.wait_for(
|
||||
websocket.recv(),
|
||||
timeout=config["receive_timeout_seconds"],
|
||||
)
|
||||
except TimeoutError:
|
||||
break
|
||||
payload = json.loads(raw_message)
|
||||
if isinstance(payload, dict):
|
||||
messages.append(payload)
|
||||
except Exception as exc:
|
||||
if self._db_session is not None:
|
||||
await update_ais_source_health(
|
||||
self._db_session,
|
||||
source=self.name,
|
||||
connection_state="disconnected",
|
||||
last_error=f"{exc.__class__.__name__}: {exc}",
|
||||
)
|
||||
await self._db_session.commit()
|
||||
raise
|
||||
|
||||
return messages
|
||||
|
||||
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
records = []
|
||||
for item in raw_data:
|
||||
record = self._normalize_message(item)
|
||||
if record:
|
||||
records.append(record)
|
||||
return records
|
||||
|
||||
async def _save_data(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
data: list[dict[str, Any]],
|
||||
task_id: int | None = None,
|
||||
snapshot_id: int | None = None,
|
||||
) -> int:
|
||||
now = datetime.now(UTC)
|
||||
records_added = 0
|
||||
latest_observed_at = now
|
||||
for index, item in enumerate(data):
|
||||
observed_at = item.get("received_at") or now
|
||||
observation = await record_vessel_ais_observation(
|
||||
db,
|
||||
source=self.name,
|
||||
normalized_payload=item,
|
||||
raw_payload=item.get("_raw_payload") or item,
|
||||
delivery_mode=AISSTREAM_DELIVERY_MODE,
|
||||
transport=AISSTREAM_TRANSPORT,
|
||||
message_type=item.get("_message_type") or "PositionReport",
|
||||
source_message_id=item.get("_source_message_id"),
|
||||
observed_at=observed_at,
|
||||
collected_at=now,
|
||||
)
|
||||
if observation is not None:
|
||||
records_added += 1
|
||||
if isinstance(observed_at, datetime) and observed_at > latest_observed_at:
|
||||
latest_observed_at = observed_at
|
||||
if (index + 1) % 1000 == 0:
|
||||
await self.update_progress(index + 1, commit=True)
|
||||
|
||||
await update_ais_source_health(
|
||||
db,
|
||||
source=self.name,
|
||||
connection_state="connected",
|
||||
observed_count=len(data),
|
||||
last_seen_at=latest_observed_at,
|
||||
last_success_at=now if data else None,
|
||||
lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
|
||||
)
|
||||
await db.commit()
|
||||
await self.update_progress(records_added, force=True)
|
||||
return records_added
|
||||
|
||||
def _normalize_message(self, item: dict[str, Any]) -> dict[str, Any] | None:
|
||||
message_type = str(item.get("MessageType") or item.get("message_type") or "")
|
||||
metadata = item.get("MetaData") if isinstance(item.get("MetaData"), dict) else {}
|
||||
message = item.get("Message") if isinstance(item.get("Message"), dict) else {}
|
||||
body = message.get(message_type) if isinstance(message.get(message_type), dict) else message
|
||||
if not isinstance(body, dict):
|
||||
body = {}
|
||||
|
||||
mmsi = _as_int(_pick(metadata, "MMSI", "mmsi") or _pick(body, "MMSI", "mmsi"))
|
||||
if mmsi is None:
|
||||
return None
|
||||
|
||||
received_at = _parse_datetime(
|
||||
_pick(metadata, "time_utc", "Time_UTC", "timestamp")
|
||||
or _pick(body, "Timestamp", "timestamp", "time")
|
||||
)
|
||||
ship_name = _clean_text(
|
||||
_pick(body, "Name", "ShipName", "name")
|
||||
or _pick(metadata, "ShipName", "ship_name", "name")
|
||||
)
|
||||
record: dict[str, Any] = {
|
||||
"mmsi": mmsi,
|
||||
"received_at": received_at,
|
||||
"_message_type": message_type or None,
|
||||
"_source_message_id": item.get("MessageID") or item.get("message_id"),
|
||||
"_raw_payload": item,
|
||||
}
|
||||
|
||||
lat = _as_float(_pick(body, "Latitude", "lat", "latitude"))
|
||||
lon = _as_float(_pick(body, "Longitude", "lon", "lng", "longitude"))
|
||||
if lat is not None and lon is not None:
|
||||
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
|
||||
return None
|
||||
record.update(
|
||||
{
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"sog": _as_float(_pick(body, "Sog", "SOG", "speedOverGround")),
|
||||
"cog": _as_float(_pick(body, "Cog", "COG", "courseOverGround")),
|
||||
"heading": _as_int(_pick(body, "TrueHeading", "Heading", "heading")),
|
||||
"nav_status": _as_int(_pick(body, "NavigationalStatus", "nav_status")),
|
||||
}
|
||||
)
|
||||
|
||||
vessel_type = _as_int(_pick(body, "Type", "ShipType", "vessel_type"))
|
||||
record.update(
|
||||
{
|
||||
"name": ship_name,
|
||||
"callsign": _pick(body, "CallSign", "callsign"),
|
||||
"imo": _as_int(_pick(body, "ImoNumber", "IMO", "imo")),
|
||||
"vessel_type": vessel_type,
|
||||
"vessel_type_name": _pick(body, "TypeName", "ShipTypeName", "vessel_type_name")
|
||||
or normalize_vessel_type_name(vessel_type),
|
||||
"length": _as_float(_pick(body, "DimensionToBow", "Length", "length")),
|
||||
"width": _as_float(_pick(body, "DimensionToPort", "Width", "width")),
|
||||
}
|
||||
)
|
||||
return record
|
||||
|
||||
|
||||
def _pick(item: dict[str, Any], *keys: str) -> Any:
|
||||
for key in keys:
|
||||
if key in item and item[key] not in (None, ""):
|
||||
return item[key]
|
||||
return None
|
||||
|
||||
|
||||
def _clean_text(value: Any) -> str | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _as_float(value: Any) -> float | None:
|
||||
try:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _as_int(value: Any) -> int | None:
|
||||
try:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if not value:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
timestamp = float(value)
|
||||
if timestamp > 10_000_000_000:
|
||||
timestamp /= 1000
|
||||
return datetime.fromtimestamp(timestamp, UTC)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
@@ -4,7 +4,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.vessel import VesselPosition, VesselStatic
|
||||
@@ -14,15 +14,13 @@ from app.services.barentswatch import (
|
||||
resolve_barentswatch_config,
|
||||
)
|
||||
from app.services.collectors.base import BaseCollector
|
||||
|
||||
|
||||
VESSEL_TYPE_NAMES = {
|
||||
30: "Fishing",
|
||||
35: "Military",
|
||||
60: "Passenger",
|
||||
70: "Cargo",
|
||||
80: "Tanker",
|
||||
}
|
||||
from app.services.vessel_ais_aggregation import (
|
||||
BARENTSWATCH_DELIVERY_MODE,
|
||||
BARENTSWATCH_TRANSPORT,
|
||||
record_vessel_ais_observation,
|
||||
update_ais_source_health,
|
||||
)
|
||||
from app.services.vessel_types import normalize_vessel_type_name
|
||||
|
||||
|
||||
class VesselAISCollector(BaseCollector):
|
||||
@@ -92,6 +90,18 @@ class VesselAISCollector(BaseCollector):
|
||||
records_added = 0
|
||||
|
||||
for index, item in enumerate(data):
|
||||
observed_at = item.get("received_at") or now
|
||||
await record_vessel_ais_observation(
|
||||
db,
|
||||
source=self.name,
|
||||
normalized_payload=item,
|
||||
raw_payload=item,
|
||||
delivery_mode=BARENTSWATCH_DELIVERY_MODE,
|
||||
transport=BARENTSWATCH_TRANSPORT,
|
||||
observed_at=observed_at,
|
||||
collected_at=now,
|
||||
)
|
||||
|
||||
static = await db.get(VesselStatic, item["mmsi"])
|
||||
if static is None:
|
||||
static = VesselStatic(mmsi=item["mmsi"])
|
||||
@@ -122,7 +132,7 @@ class VesselAISCollector(BaseCollector):
|
||||
cog=item.get("cog"),
|
||||
heading=item.get("heading"),
|
||||
nav_status=item.get("nav_status"),
|
||||
received_at=item.get("received_at") or now,
|
||||
received_at=observed_at,
|
||||
)
|
||||
)
|
||||
records_added += 1
|
||||
@@ -130,6 +140,19 @@ class VesselAISCollector(BaseCollector):
|
||||
if (index + 1) % 1000 == 0:
|
||||
await self.update_progress(index + 1, commit=True)
|
||||
|
||||
latest_observed_at = max(
|
||||
(item.get("received_at") for item in data if item.get("received_at")),
|
||||
default=now,
|
||||
)
|
||||
await update_ais_source_health(
|
||||
db,
|
||||
source=self.name,
|
||||
connection_state="connected",
|
||||
observed_count=len(data),
|
||||
last_seen_at=latest_observed_at,
|
||||
last_success_at=now if data else None,
|
||||
lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
|
||||
)
|
||||
await db.execute(
|
||||
delete(VesselPosition).where(VesselPosition.received_at < now - timedelta(hours=24))
|
||||
)
|
||||
@@ -156,7 +179,7 @@ class VesselAISCollector(BaseCollector):
|
||||
vessel_type = _as_int(_pick(item, "vessel_type", "shipType", "ship_type", "ShipType"))
|
||||
vessel_type_name = (
|
||||
_pick(item, "vessel_type_name", "shipTypeName", "ship_type_name", "VesselTypeName")
|
||||
or _vessel_type_name(vessel_type)
|
||||
or normalize_vessel_type_name(vessel_type)
|
||||
)
|
||||
received_at = _parse_datetime(_pick(item, "received_at", "timestamp", "time", "msgtime"))
|
||||
|
||||
@@ -255,19 +278,3 @@ def _parse_datetime(value: Any) -> datetime | None:
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _vessel_type_name(vessel_type: int | None) -> str:
|
||||
if vessel_type is None:
|
||||
return "Other"
|
||||
if 70 <= vessel_type <= 79:
|
||||
return "Cargo"
|
||||
if 80 <= vessel_type <= 89:
|
||||
return "Tanker"
|
||||
if 60 <= vessel_type <= 69:
|
||||
return "Passenger"
|
||||
if vessel_type == 30:
|
||||
return "Fishing"
|
||||
if vessel_type == 35:
|
||||
return "Military"
|
||||
return VESSEL_TYPE_NAMES.get(vessel_type, "Other")
|
||||
|
||||
@@ -66,9 +66,68 @@ BARENTSWATCH_DEFAULT_GUIDE = CredentialGuideDefault(
|
||||
""",
|
||||
)
|
||||
|
||||
AISSTREAM_DEFAULT_GUIDE = CredentialGuideDefault(
|
||||
provider="aisstream",
|
||||
title="AISStream API Key 获取教程",
|
||||
prompt=(
|
||||
"请生成一份中文教程,指导开发者获取 AISStream 的 API Key 并配置到 Planet。"
|
||||
"教程要面向已经有本地开发环境的人,包含注册/登录 AISStream、获取 API Key、"
|
||||
"理解免费额度和订阅范围、在 Planet 设置中心填写 API Key、配置 bounding boxes "
|
||||
"和 message types、验证连接、常见失败排查。必须提醒用户以 AISStream 当前官网和"
|
||||
"服务条款为准,不要编造具体页面按钮文案。"
|
||||
),
|
||||
markdown="""## AISStream API Key 获取
|
||||
|
||||
官方入口:https://aisstream.io/
|
||||
|
||||
1. 打开 AISStream 官网,按当前页面指引注册或登录账号。
|
||||
2. 在账号/API 管理页面创建或复制你的 API Key。
|
||||
3. 先确认当前账号额度、使用条款和可订阅区域。实时 AIS 流量可能很大,不建议一开始订阅全球范围。
|
||||
4. 回到 Planet 的 `设置 -> 采集器设置 -> AISStream 实时船舶`。
|
||||
5. 在 `AISStream 凭证` 中填入 API Key。
|
||||
6. Endpoint 通常保持默认:`wss://stream.aisstream.io/v0/stream`。
|
||||
7. 按需配置 `Bounding Boxes JSON` 和 `消息类型`。
|
||||
8. 点击连接测试,确认系统能读取凭证且 WebSocket endpoint 格式有效。
|
||||
9. 保存采集器设置后再运行 `aisstream_vessels` collector。
|
||||
|
||||
### 推荐配置
|
||||
|
||||
默认消息类型:
|
||||
|
||||
```json
|
||||
["PositionReport", "ShipStaticData"]
|
||||
```
|
||||
|
||||
默认 Bounding Boxes 示例:
|
||||
|
||||
```json
|
||||
[[[-90, -180], [90, 180]]]
|
||||
```
|
||||
|
||||
这个示例表示全球范围。实际使用时建议先改成较小区域,降低消息量和处理压力。
|
||||
|
||||
### 请求规则
|
||||
|
||||
- Endpoint:`wss://stream.aisstream.io/v0/stream`
|
||||
- 传输方式:WebSocket
|
||||
- API Key 放在订阅 payload 中,不放在 HTTP header。
|
||||
- Planet 会把 AISStream 标记为 `delivery_mode = realtime_stream`、`transport = websocket`。
|
||||
- AISStream collector 只写入 AIS raw observations,不直接覆盖最终船只展示表。
|
||||
|
||||
### 常见排查
|
||||
|
||||
- `未找到凭证`:确认 API Key 已保存到采集器设置,或设置了 `AISSTREAM_API_KEY` 环境变量 / `~/.zshrc`。
|
||||
- `endpoint 必须是 ws:// 或 wss://`:AISStream 是 WebSocket 流接口,不要填普通 `https://` API 地址。
|
||||
- 采集量过大:缩小 `Bounding Boxes JSON`,减少 `message_types`,或降低单次最大消息数。
|
||||
- 没有船只数据:确认订阅区域内确实有 AIS 活动,并检查 API Key 当前额度和权限。
|
||||
- 连接中断:实时流可能受网络和上游限流影响,collector 会记录源健康状态供聚合服务回退。
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_CREDENTIAL_GUIDES = {
|
||||
BARENTSWATCH_DEFAULT_GUIDE.provider: BARENTSWATCH_DEFAULT_GUIDE,
|
||||
AISSTREAM_DEFAULT_GUIDE.provider: AISSTREAM_DEFAULT_GUIDE,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from app.services.barentswatch import (
|
||||
|
||||
CONNECTIVITY_VALIDATION_KEY = "connectivity_validation"
|
||||
CONNECTIVITY_STORE_CATEGORY = "datasource_connectivity_validations"
|
||||
SUPPORTED_CREDENTIAL_PROVIDERS = {"barentswatch", "spacetrack", "aisstream"}
|
||||
|
||||
|
||||
def _sha256_json(payload: Any) -> str:
|
||||
@@ -43,6 +44,36 @@ def _resolve_spacetrack_credentials() -> tuple[str, str, str]:
|
||||
return username, password, source or "missing"
|
||||
|
||||
|
||||
async def _resolve_aisstream_api_key(
|
||||
db=None,
|
||||
credential_override: dict[str, str] | None = None,
|
||||
) -> tuple[str, str]:
|
||||
if credential_override and credential_override.get("api_key"):
|
||||
return str(credential_override["api_key"]), "draft"
|
||||
|
||||
env_key = os.getenv("AISSTREAM_API_KEY")
|
||||
zshrc_key = _read_zshrc_env().get("AISSTREAM_API_KEY")
|
||||
if db is not None:
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == "aisstream_vessels")
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
if record:
|
||||
auth_config = record.auth_config or {}
|
||||
runtime_config = record.config or {}
|
||||
api_key = auth_config.get("api_key") or runtime_config.get("api_key")
|
||||
if api_key:
|
||||
return str(api_key), "datasource_config"
|
||||
|
||||
if env_key:
|
||||
return env_key, "environment"
|
||||
if zshrc_key:
|
||||
return zshrc_key, "~/.zshrc"
|
||||
return "", "missing"
|
||||
|
||||
|
||||
def strip_connectivity_validation(config: dict | None) -> dict:
|
||||
cleaned = dict(config or {})
|
||||
cleaned.pop(CONNECTIVITY_VALIDATION_KEY, None)
|
||||
@@ -103,6 +134,10 @@ async def build_builtin_connectivity_checksum(
|
||||
"password": password,
|
||||
}
|
||||
)
|
||||
elif credential_provider == "aisstream":
|
||||
api_key, credential_source = await _resolve_aisstream_api_key(db, credential_override)
|
||||
has_credentials = bool(api_key)
|
||||
credential_fingerprint = _sha256_json({"api_key": api_key})
|
||||
elif defaults.get("requires_credentials"):
|
||||
credential_source = str(credential_provider or "unsupported")
|
||||
|
||||
@@ -130,6 +165,7 @@ async def test_builtin_connectivity(
|
||||
headers: dict | None,
|
||||
config: dict | None,
|
||||
db=None,
|
||||
credential_override: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
defaults = DEFAULT_DATASOURCES.get(source)
|
||||
if not defaults:
|
||||
@@ -145,6 +181,7 @@ async def test_builtin_connectivity(
|
||||
headers,
|
||||
config,
|
||||
db,
|
||||
credential_override,
|
||||
)
|
||||
if credential_context["requires_credentials"] and not credential_context["has_credentials"]:
|
||||
return {
|
||||
@@ -155,10 +192,9 @@ async def test_builtin_connectivity(
|
||||
"settings_tab": "collector_credentials",
|
||||
**credential_context,
|
||||
}
|
||||
supported_credential_providers = {"barentswatch", "spacetrack"}
|
||||
if (
|
||||
credential_context["requires_credentials"]
|
||||
and credential_context["credential_provider"] not in supported_credential_providers
|
||||
and credential_context["credential_provider"] not in SUPPORTED_CREDENTIAL_PROVIDERS
|
||||
):
|
||||
return {
|
||||
"success": False,
|
||||
@@ -174,6 +210,23 @@ async def test_builtin_connectivity(
|
||||
timeout = float(request_config.get("timeout") or 30)
|
||||
request_endpoint = endpoint
|
||||
|
||||
if credential_context["credential_provider"] == "aisstream":
|
||||
if not str(request_endpoint).startswith(("ws://", "wss://")):
|
||||
return {
|
||||
"success": False,
|
||||
"checksum": checksum,
|
||||
"stage": "endpoint",
|
||||
"message": "AISStream endpoint 必须是 ws:// 或 wss:// WebSocket 地址。",
|
||||
**credential_context,
|
||||
}
|
||||
return {
|
||||
"success": True,
|
||||
"checksum": checksum,
|
||||
"stage": "credentials",
|
||||
"message": "AISStream 凭证已配置,WebSocket endpoint 格式有效。",
|
||||
**credential_context,
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
if credential_context["credential_provider"] == "barentswatch":
|
||||
|
||||
574
backend/app/services/vessel_ais_aggregation.py
Normal file
574
backend/app/services/vessel_ais_aggregation.py
Normal file
@@ -0,0 +1,574 @@
|
||||
"""AIS raw observation and aggregation support for vessel collectors."""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from typing import Any, Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth
|
||||
from app.services.vessel_types import normalize_vessel_type_name
|
||||
|
||||
VESSEL_AIS_SCHEMA = "vessel_ais"
|
||||
BARENTSWATCH_DELIVERY_MODE = "polling"
|
||||
BARENTSWATCH_TRANSPORT = "http"
|
||||
AISSTREAM_DELIVERY_MODE = "realtime_stream"
|
||||
AISSTREAM_TRANSPORT = "websocket"
|
||||
DELIVERY_MODE_PRIORITY = {
|
||||
"realtime_stream": 40,
|
||||
"batch_stream": 30,
|
||||
"polling": 20,
|
||||
"snapshot": 10,
|
||||
}
|
||||
DYNAMIC_FIELDS = ("lat", "lon", "sog", "cog", "heading", "nav_status")
|
||||
CONFLICT_FIELDS = (
|
||||
"name",
|
||||
"callsign",
|
||||
"imo",
|
||||
"flag",
|
||||
"vessel_type",
|
||||
"vessel_type_name",
|
||||
"length",
|
||||
"width",
|
||||
"draught",
|
||||
)
|
||||
|
||||
|
||||
def _json_default(value: Any) -> Any:
|
||||
if isinstance(value, datetime):
|
||||
return value.astimezone(UTC).isoformat()
|
||||
return str(value)
|
||||
|
||||
|
||||
def _stable_payload(value: Any) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=_json_default)
|
||||
|
||||
|
||||
def _jsonable(value: Any) -> Any:
|
||||
if isinstance(value, datetime):
|
||||
return value.astimezone(UTC).isoformat()
|
||||
if isinstance(value, dict):
|
||||
return {str(key): _jsonable(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_jsonable(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _coerce_datetime(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if isinstance(value, (int, float)):
|
||||
timestamp = float(value)
|
||||
if timestamp > 10_000_000_000:
|
||||
timestamp /= 1000
|
||||
return datetime.fromtimestamp(timestamp, UTC)
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def build_observation_hash(
|
||||
*,
|
||||
source: str,
|
||||
entity_key: str,
|
||||
message_type: str | None,
|
||||
observed_at: datetime,
|
||||
normalized_payload: dict[str, Any],
|
||||
source_message_id: str | None = None,
|
||||
) -> str:
|
||||
"""Build a deterministic idempotency key for one source-level AIS observation."""
|
||||
|
||||
if source_message_id:
|
||||
basis = {
|
||||
"source": source,
|
||||
"entity_key": entity_key,
|
||||
"source_message_id": source_message_id,
|
||||
}
|
||||
else:
|
||||
basis = {
|
||||
"source": source,
|
||||
"entity_key": entity_key,
|
||||
"message_type": message_type,
|
||||
"observed_at": observed_at.astimezone(UTC).isoformat(),
|
||||
"payload": normalized_payload,
|
||||
}
|
||||
return sha256(_stable_payload(basis).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_field_conflict_candidates(
|
||||
observations: Iterable[AISRawObservation],
|
||||
fields: Iterable[str] = CONFLICT_FIELDS,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return current field disagreements from raw observations without mutating state."""
|
||||
|
||||
candidates_by_field: dict[str, dict[str, Any]] = {}
|
||||
for observation in observations:
|
||||
payload = observation.normalized_payload or {}
|
||||
for field in fields:
|
||||
value = payload.get(field)
|
||||
if value in (None, ""):
|
||||
continue
|
||||
field_candidates = candidates_by_field.setdefault(field, {})
|
||||
field_candidates[observation.source] = value
|
||||
|
||||
conflicts = []
|
||||
for field, candidates in sorted(candidates_by_field.items()):
|
||||
unique_values = {_stable_payload(value) for value in candidates.values()}
|
||||
if len(unique_values) <= 1:
|
||||
continue
|
||||
conflicts.append(
|
||||
{
|
||||
"field": field,
|
||||
"candidates": candidates,
|
||||
"status": "candidate",
|
||||
}
|
||||
)
|
||||
return conflicts
|
||||
|
||||
|
||||
def _payload_value(payload: dict[str, Any], field: str) -> Any:
|
||||
value = payload.get(field)
|
||||
return None if value in (None, "") else value
|
||||
|
||||
|
||||
def _clean_text(value: Any) -> str | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _raw_metadata_value(observation: AISRawObservation, field: str) -> Any:
|
||||
raw_payload = observation.raw_payload or {}
|
||||
metadata = raw_payload.get("MetaData") if isinstance(raw_payload, dict) else None
|
||||
if not isinstance(metadata, dict):
|
||||
return None
|
||||
if field == "name":
|
||||
return _clean_text(metadata.get("ShipName") or metadata.get("ship_name") or metadata.get("name"))
|
||||
return None
|
||||
|
||||
|
||||
def _delivery_priority(observation: AISRawObservation) -> int:
|
||||
return DELIVERY_MODE_PRIORITY.get(str(observation.delivery_mode or ""), 0)
|
||||
|
||||
|
||||
def _has_valid_position(payload: dict[str, Any]) -> bool:
|
||||
try:
|
||||
lat = float(payload.get("lat"))
|
||||
lon = float(payload.get("lon"))
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return -90 <= lat <= 90 and -180 <= lon <= 180
|
||||
|
||||
|
||||
def _is_future_observation(observation: AISRawObservation, now: datetime) -> bool:
|
||||
return observation.observed_at > now
|
||||
|
||||
|
||||
def _select_position_observation(
|
||||
observations: list[AISRawObservation],
|
||||
*,
|
||||
now: datetime,
|
||||
) -> tuple[AISRawObservation | None, list[str]]:
|
||||
rejected_flags: list[str] = []
|
||||
candidates = []
|
||||
for observation in observations:
|
||||
payload = observation.normalized_payload or {}
|
||||
if not _has_valid_position(payload):
|
||||
rejected_flags.append("invalid_position")
|
||||
continue
|
||||
if _is_future_observation(observation, now):
|
||||
rejected_flags.append("future_timestamp")
|
||||
continue
|
||||
candidates.append(observation)
|
||||
|
||||
if not candidates:
|
||||
return None, sorted(set(rejected_flags))
|
||||
|
||||
candidates.sort(
|
||||
key=lambda item: (
|
||||
item.observed_at,
|
||||
_delivery_priority(item),
|
||||
item.collected_at,
|
||||
item.id or 0,
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return candidates[0], sorted(set(rejected_flags))
|
||||
|
||||
|
||||
def _select_static_field(
|
||||
observations: list[AISRawObservation],
|
||||
field: str,
|
||||
) -> tuple[Any, str | None, str | None]:
|
||||
candidates = []
|
||||
for observation in observations:
|
||||
value = _payload_value(observation.normalized_payload or {}, field)
|
||||
if value is None:
|
||||
value = _raw_metadata_value(observation, field)
|
||||
if value is None:
|
||||
continue
|
||||
candidates.append((observation, value))
|
||||
|
||||
if not candidates:
|
||||
return None, None, None
|
||||
|
||||
candidates.sort(
|
||||
key=lambda item: (
|
||||
_delivery_priority(item[0]),
|
||||
item[0].observed_at,
|
||||
item[0].collected_at,
|
||||
item[0].id or 0,
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
selected_observation, selected_value = candidates[0]
|
||||
unique_values = {_stable_payload(value) for _, value in candidates}
|
||||
reason = "delivery_mode_priority" if len(unique_values) > 1 else "non_empty_priority"
|
||||
return selected_value, selected_observation.source, reason
|
||||
|
||||
|
||||
def _build_source_summary(observations: list[AISRawObservation]) -> dict[str, dict[str, Any]]:
|
||||
summary: dict[str, dict[str, Any]] = {}
|
||||
for observation in observations:
|
||||
source_summary = summary.setdefault(
|
||||
observation.source,
|
||||
{
|
||||
"observation_count": 0,
|
||||
"latest_observed_at": None,
|
||||
"delivery_mode": observation.delivery_mode,
|
||||
"transport": observation.transport,
|
||||
"message_types": [],
|
||||
},
|
||||
)
|
||||
source_summary["observation_count"] += 1
|
||||
latest_observed_at = source_summary["latest_observed_at"]
|
||||
if latest_observed_at is None or observation.observed_at > latest_observed_at:
|
||||
source_summary["latest_observed_at"] = observation.observed_at
|
||||
if observation.message_type and observation.message_type not in source_summary["message_types"]:
|
||||
source_summary["message_types"].append(observation.message_type)
|
||||
return summary
|
||||
|
||||
|
||||
def _build_aggregated_vessel(
|
||||
entity_key: str,
|
||||
observations: list[AISRawObservation],
|
||||
*,
|
||||
now: datetime,
|
||||
) -> dict[str, Any] | None:
|
||||
position_observation, rejected_flags = _select_position_observation(observations, now=now)
|
||||
if position_observation is None:
|
||||
return None
|
||||
|
||||
payload = position_observation.normalized_payload or {}
|
||||
mmsi = int(entity_key)
|
||||
result: dict[str, Any] = {
|
||||
"mmsi": mmsi,
|
||||
"lat": float(payload["lat"]),
|
||||
"lon": float(payload["lon"]),
|
||||
"received_at": position_observation.observed_at,
|
||||
"field_sources": {},
|
||||
"selected_reasons": {},
|
||||
"source_summary": _build_source_summary(observations),
|
||||
"quality_flags": sorted(
|
||||
set((position_observation.quality_flags or []) + rejected_flags)
|
||||
),
|
||||
}
|
||||
|
||||
for field in DYNAMIC_FIELDS:
|
||||
value = _payload_value(payload, field)
|
||||
if field in ("lat", "lon") or value is not None:
|
||||
result[field] = value
|
||||
result["field_sources"][field] = position_observation.source
|
||||
result["selected_reasons"][field] = "newest_observation"
|
||||
|
||||
for field in CONFLICT_FIELDS:
|
||||
selected_value, selected_source, reason = _select_static_field(observations, field)
|
||||
if selected_value is None:
|
||||
continue
|
||||
result[field] = selected_value
|
||||
result["field_sources"][field] = selected_source
|
||||
result["selected_reasons"][field] = reason
|
||||
|
||||
result["name"] = result.get("name") or f"MMSI {mmsi}"
|
||||
result["vessel_type_name"] = result.get("vessel_type_name") or normalize_vessel_type_name(
|
||||
result.get("vessel_type")
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def _upsert_conflict_records(
|
||||
db: AsyncSession,
|
||||
entity_key: str,
|
||||
observations: list[AISRawObservation],
|
||||
aggregated: dict[str, Any],
|
||||
) -> int:
|
||||
conflicts = build_field_conflict_candidates(observations)
|
||||
now = datetime.now(UTC)
|
||||
for conflict in conflicts:
|
||||
field = conflict["field"]
|
||||
result = await db.execute(
|
||||
select(AISConflictRecord)
|
||||
.where(AISConflictRecord.target_schema == VESSEL_AIS_SCHEMA)
|
||||
.where(AISConflictRecord.entity_key == entity_key)
|
||||
.where(AISConflictRecord.field == field)
|
||||
.limit(1)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
if record is None:
|
||||
record = AISConflictRecord(
|
||||
target_schema=VESSEL_AIS_SCHEMA,
|
||||
entity_key=entity_key,
|
||||
field=field,
|
||||
)
|
||||
db.add(record)
|
||||
record.candidates = conflict["candidates"]
|
||||
record.selected_source = (aggregated.get("field_sources") or {}).get(field)
|
||||
record.selected_value = aggregated.get(field)
|
||||
record.selected_reason = (aggregated.get("selected_reasons") or {}).get(field)
|
||||
record.resolved_by = "system"
|
||||
record.status = "open"
|
||||
record.updated_at = now
|
||||
return len(conflicts)
|
||||
|
||||
|
||||
def _group_observations(observations: Iterable[AISRawObservation]) -> dict[str, list[AISRawObservation]]:
|
||||
grouped: dict[str, list[AISRawObservation]] = {}
|
||||
for observation in observations:
|
||||
grouped.setdefault(str(observation.entity_key), []).append(observation)
|
||||
return grouped
|
||||
|
||||
|
||||
async def record_vessel_ais_observation(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source: str,
|
||||
normalized_payload: dict[str, Any],
|
||||
raw_payload: dict[str, Any] | None = None,
|
||||
delivery_mode: str,
|
||||
transport: str,
|
||||
message_type: str | None = "PositionReport",
|
||||
source_message_id: str | None = None,
|
||||
observed_at: datetime | None = None,
|
||||
collected_at: datetime | None = None,
|
||||
quality_flags: list[str] | None = None,
|
||||
) -> AISRawObservation | None:
|
||||
"""Insert one raw observation if the source-level fact has not already been stored."""
|
||||
|
||||
entity_key = str(normalized_payload["mmsi"])
|
||||
collected_at = collected_at or datetime.now(UTC)
|
||||
observed_at = (
|
||||
_coerce_datetime(observed_at)
|
||||
or _coerce_datetime(normalized_payload.get("received_at"))
|
||||
or collected_at
|
||||
)
|
||||
normalized_json = _jsonable(normalized_payload)
|
||||
raw_json = _jsonable(raw_payload or {})
|
||||
|
||||
observation_hash = build_observation_hash(
|
||||
source=source,
|
||||
entity_key=entity_key,
|
||||
message_type=message_type,
|
||||
observed_at=observed_at,
|
||||
normalized_payload=normalized_json,
|
||||
source_message_id=source_message_id,
|
||||
)
|
||||
existing_result = await db.execute(
|
||||
select(AISRawObservation.id).where(AISRawObservation.observation_hash == observation_hash)
|
||||
)
|
||||
if existing_result.scalar_one_or_none() is not None:
|
||||
return None
|
||||
|
||||
observation = AISRawObservation(
|
||||
target_schema=VESSEL_AIS_SCHEMA,
|
||||
source=source,
|
||||
entity_key=entity_key,
|
||||
delivery_mode=delivery_mode,
|
||||
transport=transport,
|
||||
message_type=message_type,
|
||||
source_message_id=source_message_id,
|
||||
observation_hash=observation_hash,
|
||||
observed_at=observed_at,
|
||||
collected_at=collected_at,
|
||||
normalized_payload=normalized_json,
|
||||
raw_payload=raw_json,
|
||||
quality_flags=quality_flags or [],
|
||||
)
|
||||
db.add(observation)
|
||||
return observation
|
||||
|
||||
|
||||
async def aggregate_vessel_observations(
|
||||
db: AsyncSession,
|
||||
observations: Iterable[AISRawObservation],
|
||||
*,
|
||||
write_conflicts: bool = True,
|
||||
) -> list[dict[str, Any]]:
|
||||
now = datetime.now(UTC)
|
||||
vessels = []
|
||||
for entity_key, entity_observations in _group_observations(observations).items():
|
||||
aggregated = _build_aggregated_vessel(entity_key, entity_observations, now=now)
|
||||
if aggregated is None:
|
||||
continue
|
||||
if write_conflicts:
|
||||
aggregated["conflict_count"] = await _upsert_conflict_records(
|
||||
db,
|
||||
entity_key,
|
||||
entity_observations,
|
||||
aggregated,
|
||||
)
|
||||
else:
|
||||
aggregated["conflict_count"] = len(build_field_conflict_candidates(entity_observations))
|
||||
vessels.append(aggregated)
|
||||
|
||||
vessels.sort(key=lambda item: item.get("received_at") or datetime.min.replace(tzinfo=UTC), reverse=True)
|
||||
return vessels
|
||||
|
||||
|
||||
async def get_aggregated_vessels(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
bbox: tuple[float, float, float, float] | None = None,
|
||||
limit: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(AISRawObservation)
|
||||
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
|
||||
.order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc())
|
||||
)
|
||||
if limit and limit > 0:
|
||||
stmt = stmt.limit(max(limit * 20, limit))
|
||||
|
||||
result = await db.execute(stmt)
|
||||
if not hasattr(result, "scalars"):
|
||||
return []
|
||||
vessels = await aggregate_vessel_observations(db, result.scalars().all())
|
||||
|
||||
if bbox is not None:
|
||||
lon_min, lat_min, lon_max, lat_max = bbox
|
||||
vessels = [
|
||||
vessel
|
||||
for vessel in vessels
|
||||
if lon_min <= float(vessel["lon"]) <= lon_max
|
||||
and lat_min <= float(vessel["lat"]) <= lat_max
|
||||
]
|
||||
|
||||
if limit and limit > 0:
|
||||
return vessels[:limit]
|
||||
return vessels
|
||||
|
||||
|
||||
async def get_aggregated_vessel(db: AsyncSession, mmsi: int) -> dict[str, Any] | None:
|
||||
observations = await get_vessel_raw_observations(db, mmsi, limit=1000)
|
||||
vessels = await aggregate_vessel_observations(db, observations)
|
||||
return vessels[0] if vessels else None
|
||||
|
||||
|
||||
async def get_aggregated_vessel_track(
|
||||
db: AsyncSession,
|
||||
mmsi: int,
|
||||
*,
|
||||
cutoff: datetime,
|
||||
) -> list[dict[str, Any]]:
|
||||
result = await db.execute(
|
||||
select(AISRawObservation)
|
||||
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
|
||||
.where(AISRawObservation.entity_key == str(mmsi))
|
||||
.where(AISRawObservation.observed_at >= cutoff)
|
||||
.order_by(AISRawObservation.observed_at.asc(), AISRawObservation.id.asc())
|
||||
)
|
||||
if not hasattr(result, "scalars"):
|
||||
return []
|
||||
|
||||
points: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, float, float, str]] = set()
|
||||
for observation in result.scalars().all():
|
||||
payload = observation.normalized_payload or {}
|
||||
if not _has_valid_position(payload):
|
||||
continue
|
||||
lat = float(payload["lat"])
|
||||
lon = float(payload["lon"])
|
||||
key = (
|
||||
observation.observed_at.isoformat(),
|
||||
round(lat, 5),
|
||||
round(lon, 5),
|
||||
observation.source,
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
points.append(
|
||||
{
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"observed_at": observation.observed_at,
|
||||
"source": observation.source,
|
||||
"selected_reason": "track_timeline",
|
||||
"quality_flags": observation.quality_flags or [],
|
||||
}
|
||||
)
|
||||
return points
|
||||
|
||||
|
||||
async def update_ais_source_health(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source: str,
|
||||
connection_state: str,
|
||||
observed_count: int = 0,
|
||||
last_seen_at: datetime | None = None,
|
||||
last_success_at: datetime | None = None,
|
||||
last_error: str | None = None,
|
||||
lag_seconds: float | None = None,
|
||||
) -> AISSourceHealth:
|
||||
"""Upsert the health row for an AIS source."""
|
||||
|
||||
now = datetime.now(UTC)
|
||||
health = await db.get(AISSourceHealth, source)
|
||||
if health is None:
|
||||
health = AISSourceHealth(source=source)
|
||||
db.add(health)
|
||||
|
||||
health.connection_state = connection_state
|
||||
health.last_seen_at = last_seen_at or health.last_seen_at
|
||||
health.last_success_at = last_success_at or health.last_success_at
|
||||
health.last_error = last_error
|
||||
health.message_rate = float(observed_count)
|
||||
health.lag_seconds = lag_seconds
|
||||
health.updated_at = now
|
||||
return health
|
||||
|
||||
|
||||
async def get_vessel_raw_observations(
|
||||
db: AsyncSession,
|
||||
mmsi: int,
|
||||
*,
|
||||
limit: int = 100,
|
||||
) -> list[AISRawObservation]:
|
||||
result = await db.execute(
|
||||
select(AISRawObservation)
|
||||
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
|
||||
.where(AISRawObservation.entity_key == str(mmsi))
|
||||
.order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def get_vessel_conflict_records(
|
||||
db: AsyncSession,
|
||||
mmsi: int,
|
||||
) -> list[AISConflictRecord]:
|
||||
result = await db.execute(
|
||||
select(AISConflictRecord)
|
||||
.where(AISConflictRecord.target_schema == VESSEL_AIS_SCHEMA)
|
||||
.where(AISConflictRecord.entity_key == str(mmsi))
|
||||
.order_by(AISConflictRecord.updated_at.desc(), AISConflictRecord.id.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
31
backend/app/services/vessel_types.py
Normal file
31
backend/app/services/vessel_types.py
Normal file
@@ -0,0 +1,31 @@
|
||||
"""Shared AIS vessel type helpers."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
VESSEL_TYPE_NAMES = {
|
||||
30: "Fishing",
|
||||
35: "Military",
|
||||
60: "Passenger",
|
||||
70: "Cargo",
|
||||
80: "Tanker",
|
||||
}
|
||||
|
||||
|
||||
def normalize_vessel_type_name(vessel_type: Any) -> str:
|
||||
"""Map AIS numeric vessel type codes to display buckets."""
|
||||
|
||||
try:
|
||||
type_code = int(float(vessel_type))
|
||||
except (TypeError, ValueError):
|
||||
return "Other"
|
||||
if 70 <= type_code <= 79:
|
||||
return "Cargo"
|
||||
if 80 <= type_code <= 89:
|
||||
return "Tanker"
|
||||
if 60 <= type_code <= 69:
|
||||
return "Passenger"
|
||||
if type_code == 30:
|
||||
return "Fishing"
|
||||
if type_code == 35:
|
||||
return "Military"
|
||||
return VESSEL_TYPE_NAMES.get(type_code, "Other")
|
||||
@@ -1,11 +1,13 @@
|
||||
"""Unit tests for data collectors"""
|
||||
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.services.credential_guides import DEFAULT_CREDENTIAL_GUIDES
|
||||
from app.services.collectors.top500 import TOP500Collector
|
||||
from app.services.collectors.base import BaseCollector, HTTPCollector
|
||||
from app.services.collectors.registry import collector_registry
|
||||
from app.services.datasource_connectivity import SUPPORTED_CREDENTIAL_PROVIDERS
|
||||
from app.models.task import CollectionTask
|
||||
|
||||
|
||||
@@ -145,3 +147,30 @@ class TestHTTPCollector:
|
||||
assert hasattr(collector, "parse_response")
|
||||
assert callable(collector.fetch)
|
||||
assert callable(collector.parse_response)
|
||||
|
||||
|
||||
def test_aisstream_collector_is_registered():
|
||||
collector = collector_registry.get("aisstream_vessels")
|
||||
|
||||
assert collector is not None
|
||||
assert collector.data_type == "vessel_ais"
|
||||
|
||||
|
||||
def test_supported_credential_collectors_have_guides_and_connectivity_provider():
|
||||
missing: list[str] = []
|
||||
for source, info in DEFAULT_DATASOURCES.items():
|
||||
if not info.get("requires_credentials"):
|
||||
continue
|
||||
if info.get("credential_status") != "supported":
|
||||
continue
|
||||
|
||||
provider = info.get("credential_provider")
|
||||
if not provider:
|
||||
missing.append(f"{source}: missing credential_provider")
|
||||
continue
|
||||
if provider not in DEFAULT_CREDENTIAL_GUIDES:
|
||||
missing.append(f"{source}: missing credential guide for {provider}")
|
||||
if provider not in SUPPORTED_CREDENTIAL_PROVIDERS:
|
||||
missing.append(f"{source}: missing connectivity provider for {provider}")
|
||||
|
||||
assert missing == []
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
@@ -6,9 +7,16 @@ from httpx import ASGITransport, AsyncClient
|
||||
from app.api.v1.visualization import convert_vessels_to_geojson
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models.vessel import VesselPosition, VesselStatic
|
||||
from app.models.vessel import AISRawObservation, VesselPosition, VesselStatic
|
||||
from app.services import barentswatch
|
||||
from app.services.collectors.aisstream import AISStreamCollector
|
||||
from app.services.collectors.vessel_ais import VesselAISCollector
|
||||
from app.services.vessel_ais_aggregation import (
|
||||
aggregate_vessel_observations,
|
||||
build_field_conflict_candidates,
|
||||
build_observation_hash,
|
||||
record_vessel_ais_observation,
|
||||
)
|
||||
|
||||
|
||||
def test_vessel_collector_transforms_barentswatch_like_records():
|
||||
@@ -35,6 +43,328 @@ def test_vessel_collector_transforms_barentswatch_like_records():
|
||||
assert records[0]["lat"] == pytest.approx(59.91)
|
||||
|
||||
|
||||
def test_vessel_observation_hash_is_stable_for_same_payload():
|
||||
observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc)
|
||||
payload = {
|
||||
"mmsi": 257123000,
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
"received_at": observed_at,
|
||||
}
|
||||
|
||||
first = build_observation_hash(
|
||||
source="barentswatch_vessels",
|
||||
entity_key="257123000",
|
||||
message_type="PositionReport",
|
||||
observed_at=observed_at,
|
||||
normalized_payload=payload,
|
||||
)
|
||||
second = build_observation_hash(
|
||||
source="barentswatch_vessels",
|
||||
entity_key="257123000",
|
||||
message_type="PositionReport",
|
||||
observed_at=observed_at,
|
||||
normalized_payload=dict(reversed(payload.items())),
|
||||
)
|
||||
|
||||
assert first == second
|
||||
assert len(first) == 64
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_vessel_ais_observation_skips_existing_hash():
|
||||
observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
class _Result:
|
||||
def scalar_one_or_none(self):
|
||||
return 123
|
||||
|
||||
class _Session:
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
|
||||
async def execute(self, _stmt):
|
||||
return _Result()
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
db = _Session()
|
||||
observation = await record_vessel_ais_observation(
|
||||
db,
|
||||
source="barentswatch_vessels",
|
||||
normalized_payload={
|
||||
"mmsi": 257123000,
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
"received_at": observed_at,
|
||||
},
|
||||
delivery_mode="polling",
|
||||
transport="http",
|
||||
observed_at=observed_at.isoformat(),
|
||||
)
|
||||
|
||||
assert observation is None
|
||||
assert db.added == []
|
||||
|
||||
|
||||
def test_build_field_conflict_candidates_from_raw_observations():
|
||||
observations = [
|
||||
AISRawObservation(
|
||||
source="barentswatch_vessels",
|
||||
normalized_payload={"name": "OSLO TRADER", "flag": "NO"},
|
||||
),
|
||||
AISRawObservation(
|
||||
source="aisstream_vessels",
|
||||
normalized_payload={"name": "OSLO TRADER II", "flag": "NO"},
|
||||
),
|
||||
]
|
||||
|
||||
conflicts = build_field_conflict_candidates(observations)
|
||||
|
||||
assert conflicts == [
|
||||
{
|
||||
"field": "name",
|
||||
"candidates": {
|
||||
"aisstream_vessels": "OSLO TRADER II",
|
||||
"barentswatch_vessels": "OSLO TRADER",
|
||||
},
|
||||
"status": "candidate",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aggregate_vessel_observations_prefers_realtime_and_records_conflict():
|
||||
observed_at = datetime.now(timezone.utc) - timedelta(minutes=5)
|
||||
|
||||
class _Result:
|
||||
def scalar_one_or_none(self):
|
||||
return None
|
||||
|
||||
class _Session:
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
|
||||
async def execute(self, _stmt):
|
||||
return _Result()
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
db = _Session()
|
||||
observations = [
|
||||
AISRawObservation(
|
||||
id=1,
|
||||
source="barentswatch_vessels",
|
||||
entity_key="257123000",
|
||||
delivery_mode="polling",
|
||||
transport="http",
|
||||
observed_at=observed_at,
|
||||
collected_at=observed_at,
|
||||
normalized_payload={
|
||||
"mmsi": 257123000,
|
||||
"name": "OSLO TRADER",
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
},
|
||||
),
|
||||
AISRawObservation(
|
||||
id=2,
|
||||
source="aisstream_vessels",
|
||||
entity_key="257123000",
|
||||
delivery_mode="realtime_stream",
|
||||
transport="websocket",
|
||||
observed_at=observed_at + timedelta(seconds=10),
|
||||
collected_at=observed_at + timedelta(seconds=10),
|
||||
normalized_payload={
|
||||
"mmsi": 257123000,
|
||||
"vessel_type": 79,
|
||||
"lat": 59.92,
|
||||
"lon": 10.74,
|
||||
},
|
||||
raw_payload={"MetaData": {"ShipName": "OSLO TRADER II "}},
|
||||
),
|
||||
]
|
||||
|
||||
vessels = await aggregate_vessel_observations(db, observations)
|
||||
|
||||
assert vessels[0]["lat"] == pytest.approx(59.92)
|
||||
assert vessels[0]["field_sources"]["lat"] == "aisstream_vessels"
|
||||
assert vessels[0]["name"] == "OSLO TRADER II"
|
||||
assert vessels[0]["vessel_type_name"] == "Cargo"
|
||||
assert vessels[0]["source_summary"]["aisstream_vessels"]["observation_count"] == 1
|
||||
assert vessels[0]["source_summary"]["barentswatch_vessels"]["delivery_mode"] == "polling"
|
||||
assert vessels[0]["conflict_count"] == 0
|
||||
assert db.added == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vessel_collector_writes_raw_observations_without_changing_position_save(monkeypatch):
|
||||
collector = VesselAISCollector()
|
||||
collector.update_progress = AsyncMock()
|
||||
record_observation = AsyncMock()
|
||||
update_health = AsyncMock()
|
||||
monkeypatch.setattr(
|
||||
"app.services.collectors.vessel_ais.record_vessel_ais_observation",
|
||||
record_observation,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.collectors.vessel_ais.update_ais_source_health",
|
||||
update_health,
|
||||
)
|
||||
|
||||
class _Session:
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
self.committed = False
|
||||
|
||||
async def get(self, *_args):
|
||||
return None
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
async def execute(self, _stmt):
|
||||
return None
|
||||
|
||||
async def commit(self):
|
||||
self.committed = True
|
||||
|
||||
db = _Session()
|
||||
observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
saved = await collector._save_data(
|
||||
db,
|
||||
[
|
||||
{
|
||||
"mmsi": 257123000,
|
||||
"name": "OSLO TRADER",
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
"received_at": observed_at,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert saved == 1
|
||||
assert db.committed is True
|
||||
assert any(isinstance(item, VesselStatic) for item in db.added)
|
||||
assert any(isinstance(item, VesselPosition) for item in db.added)
|
||||
record_observation.assert_awaited_once()
|
||||
assert record_observation.await_args.kwargs["source"] == "barentswatch_vessels"
|
||||
assert record_observation.await_args.kwargs["normalized_payload"]["mmsi"] == 257123000
|
||||
update_health.assert_awaited_once()
|
||||
|
||||
|
||||
def test_aisstream_collector_normalizes_position_report():
|
||||
collector = AISStreamCollector()
|
||||
|
||||
records = collector.transform(
|
||||
[
|
||||
{
|
||||
"MessageType": "PositionReport",
|
||||
"MetaData": {
|
||||
"MMSI": 257123000,
|
||||
"ShipName": "OSLO TRADER ",
|
||||
"time_utc": "2026-04-30T12:00:00Z",
|
||||
},
|
||||
"Message": {
|
||||
"PositionReport": {
|
||||
"Latitude": 59.91,
|
||||
"Longitude": 10.73,
|
||||
"Sog": 12.4,
|
||||
"Cog": 214,
|
||||
"TrueHeading": 215,
|
||||
"NavigationalStatus": 0,
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["mmsi"] == 257123000
|
||||
assert records[0]["lat"] == pytest.approx(59.91)
|
||||
assert records[0]["name"] == "OSLO TRADER"
|
||||
assert records[0]["_message_type"] == "PositionReport"
|
||||
|
||||
|
||||
def test_aisstream_collector_maps_ship_static_type_name():
|
||||
collector = AISStreamCollector()
|
||||
|
||||
records = collector.transform(
|
||||
[
|
||||
{
|
||||
"MessageType": "ShipStaticData",
|
||||
"MetaData": {
|
||||
"MMSI": 257123000,
|
||||
"time_utc": "2026-04-30T12:00:00Z",
|
||||
},
|
||||
"Message": {
|
||||
"ShipStaticData": {
|
||||
"Name": "OSLO TRADER",
|
||||
"Type": 79,
|
||||
"CallSign": "LAAB",
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["vessel_type"] == 79
|
||||
assert records[0]["vessel_type_name"] == "Cargo"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aisstream_collector_writes_only_raw_observations(monkeypatch):
|
||||
collector = AISStreamCollector()
|
||||
collector.update_progress = AsyncMock()
|
||||
record_observation = AsyncMock(return_value=object())
|
||||
update_health = AsyncMock()
|
||||
monkeypatch.setattr(
|
||||
"app.services.collectors.aisstream.record_vessel_ais_observation",
|
||||
record_observation,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.collectors.aisstream.update_ais_source_health",
|
||||
update_health,
|
||||
)
|
||||
|
||||
class _Session:
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
self.committed = False
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
async def commit(self):
|
||||
self.committed = True
|
||||
|
||||
db = _Session()
|
||||
saved = await collector._save_data(
|
||||
db,
|
||||
[
|
||||
{
|
||||
"mmsi": 257123000,
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
"received_at": datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc),
|
||||
"_message_type": "PositionReport",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert saved == 1
|
||||
assert db.added == []
|
||||
assert db.committed is True
|
||||
record_observation.assert_awaited_once()
|
||||
assert record_observation.await_args.kwargs["source"] == "aisstream_vessels"
|
||||
update_health.assert_awaited_once()
|
||||
|
||||
|
||||
def test_barentswatch_reads_credentials_from_zshrc(tmp_path):
|
||||
zshrc = tmp_path / ".zshrc"
|
||||
zshrc.write_text(
|
||||
@@ -137,7 +467,7 @@ async def test_vessels_geojson_endpoint_filters_type_and_bbox():
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get(
|
||||
"/api/v1/visualization/geo/vessels",
|
||||
params={"bbox": "0,50,20,70", "type": "cargo"},
|
||||
params={"bbox": "0,50,20,70", "type": "cargo", "limit": 0},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
Reference in New Issue
Block a user