release: bump version to 0.47.0

This commit is contained in:
linkong
2026-04-30 16:56:37 +08:00
parent f22079d33a
commit 421234301a
42 changed files with 2501 additions and 454 deletions

View File

@@ -5,7 +5,7 @@ from datetime import datetime
import base64
import json
import re
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from pydantic import BaseModel, Field
@@ -318,7 +318,7 @@ async def list_configs(
"""List all user-defined data source configurations"""
query = select(DataSourceConfig)
if active_only:
query = query.where(DataSourceConfig.is_active == True)
query = query.where(DataSourceConfig.is_active)
query = query.order_by(DataSourceConfig.created_at.desc())
result = await db.execute(query)
@@ -374,6 +374,11 @@ async def list_all_datasources(
"is_active": db_config.is_active if db_config else True,
"source_type": db_config.source_type if db_config else "http",
"auth_type": db_config.auth_type if db_config else "none",
"auth_configured": {
"api_key": bool((db_config.auth_config or {}).get("api_key"))
if db_config
else False,
},
"headers": db_config.headers if db_config else {},
"config": strip_connectivity_validation(db_config.config if db_config else {}),
"config_id": db_config.id if db_config else None,
@@ -464,6 +469,8 @@ async def update_config(
for field, value in update_data.items():
if field == "config":
value = strip_connectivity_validation(value)
if field == "auth_config" and value == {} and (config.auth_config or {}):
continue
setattr(config, field, value)
await db.commit()
@@ -601,6 +608,7 @@ async def connect_builtin_config(
config_data.headers,
config_data.config,
db,
config_data.auth_config,
)
if result.get("success") and result.get("checksum"):
validation = await save_connectivity_success(

View File

@@ -17,6 +17,7 @@ from app.models.datasource import DataSource
from app.models.datasource_config import DataSourceConfig
from app.models.system_setting import SystemSetting
from app.models.user import User
from app.models.vessel import AISSourceHealth
from app.services.barentswatch import (
BarentsWatchConfig,
check_barentswatch_config,
@@ -368,7 +369,12 @@ def format_frequency_label(minutes: int) -> str:
return f"{minutes}m"
def serialize_collector(datasource: DataSource) -> dict:
async def get_ais_source_health_by_source(db: AsyncSession) -> dict[str, dict]:
result = await db.execute(select(AISSourceHealth))
return {item.source: item.to_dict() for item in result.scalars().all()}
def serialize_collector(datasource: DataSource, ais_health_by_source: dict[str, dict] | None = None) -> dict:
defaults = DEFAULT_DATASOURCES.get(datasource.source, {})
return {
"id": datasource.id,
@@ -387,6 +393,7 @@ def serialize_collector(datasource: DataSource) -> dict:
"requires_credentials": bool(defaults.get("requires_credentials", False)),
"credential_provider": defaults.get("credential_provider"),
"credential_status": defaults.get("credential_status", "none"),
"ais_health": (ais_health_by_source or {}).get(datasource.source),
}
@@ -599,7 +606,8 @@ async def get_collector_settings(
):
result = await db.execute(select(DataSource).order_by(DataSource.module, DataSource.id))
datasources = result.scalars().all()
return {"collectors": [serialize_collector(datasource) for datasource in datasources]}
ais_health_by_source = await get_ais_source_health_by_source(db)
return {"collectors": [serialize_collector(datasource, ais_health_by_source) for datasource in datasources]}
@router.put("/collectors/{datasource_id}")
@@ -619,7 +627,8 @@ async def update_collector_settings(
await db.commit()
await db.refresh(datasource)
await sync_datasource_job(datasource.id)
return {"status": "updated", "collector": serialize_collector(datasource)}
ais_health_by_source = await get_ais_source_health_by_source(db)
return {"status": "updated", "collector": serialize_collector(datasource, ais_health_by_source)}
@router.get("")
@@ -633,12 +642,13 @@ async def get_all_settings(
db,
["system", "notifications", "security"],
)
ais_health_by_source = await get_ais_source_health_by_source(db)
return {
"system": setting_payloads["system"],
"notifications": setting_payloads["notifications"],
"security": setting_payloads["security"],
"tv": await get_tv_settings_payload(db),
"integrations": await serialize_external_integrations(db),
"collectors": [serialize_collector(datasource) for datasource in datasources],
"collectors": [serialize_collector(datasource, ais_health_by_source) for datasource in datasources],
"generated_at": to_iso8601_utc(datetime.now(UTC)),
}

View File

@@ -25,6 +25,14 @@ from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
from app.services.persistent_logs import record_system_log
from app.services.vessel_ais_aggregation import (
build_field_conflict_candidates,
get_aggregated_vessel,
get_aggregated_vessel_track,
get_aggregated_vessels,
get_vessel_conflict_records,
get_vessel_raw_observations,
)
from app.core.logging import get_logger
router = APIRouter()
@@ -664,6 +672,54 @@ def convert_vessels_to_geojson(rows: List[Any]) -> Dict[str, Any]:
return {"type": "FeatureCollection", "features": features}
def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict[str, Any]:
features = []
for vessel in vessels:
if vessel.get("lat") is None or vessel.get("lon") is None:
continue
source_summary = {}
for source, summary in (vessel.get("source_summary") or {}).items():
source_summary[source] = {
**summary,
"latest_observed_at": to_iso8601_utc(summary.get("latest_observed_at")),
}
props = {
"mmsi": vessel["mmsi"],
"name": vessel.get("name") or f"MMSI {vessel['mmsi']}",
"callsign": vessel.get("callsign"),
"imo": vessel.get("imo"),
"vessel_type": vessel.get("vessel_type"),
"vessel_type_name": vessel.get("vessel_type_name") or "Other",
"flag": vessel.get("flag"),
"length": vessel.get("length"),
"width": vessel.get("width"),
"draught": vessel.get("draught"),
"sog": vessel.get("sog"),
"cog": vessel.get("cog"),
"heading": vessel.get("heading"),
"nav_status": vessel.get("nav_status"),
"received_at": to_iso8601_utc(vessel.get("received_at")),
"field_sources": vessel.get("field_sources") or {},
"selected_reasons": vessel.get("selected_reasons") or {},
"source_summary": source_summary,
"quality_flags": vessel.get("quality_flags") or [],
"conflict_count": vessel.get("conflict_count", 0),
"data_type": "vessel",
}
features.append(
{
"type": "Feature",
"id": vessel["mmsi"],
"geometry": {
"type": "Point",
"coordinates": [vessel["lon"], vessel["lat"]],
},
"properties": props,
}
)
return {"type": "FeatureCollection", "features": features}
def _parse_bbox(value: Optional[str]) -> tuple[float, float, float, float] | None:
if not value:
return None
@@ -1411,10 +1467,37 @@ async def get_vessels_geojson(
None,
description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other",
),
limit: int = Query(5000, ge=1, le=50000),
limit: Optional[int] = Query(
None,
ge=0,
description="Maximum vessel features to return. Omit or pass 0 for no limit.",
),
db: AsyncSession = Depends(get_db),
):
"""Return latest vessel positions as GeoJSON points."""
parsed_bbox = _parse_bbox(bbox)
aggregated_vessels = await get_aggregated_vessels(db, bbox=parsed_bbox, limit=limit)
if aggregated_vessels:
geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
requested_types = {
item.strip().lower()
for item in (type or "").split(",")
if item.strip()
}
if requested_types:
geojson["features"] = [
feature
for feature in geojson.get("features", [])
if _matches_vessel_type(feature.get("properties", {}), requested_types)
]
features = geojson.get("features", [])
return {
**geojson,
"count": len(features),
"stats": _build_vessel_stats(features),
}
latest_times = (
select(
VesselPosition.mmsi.label("mmsi"),
@@ -1432,10 +1515,10 @@ async def get_vessels_geojson(
)
.outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
.order_by(VesselPosition.received_at.desc())
.limit(limit)
)
if limit and limit > 0:
stmt = stmt.limit(limit)
parsed_bbox = _parse_bbox(bbox)
if parsed_bbox is not None:
lon_min, lat_min, lon_max, lat_max = parsed_bbox
stmt = stmt.where(
@@ -1470,6 +1553,15 @@ async def get_vessels_geojson(
@router.get("/vessels/{mmsi}")
async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)):
aggregated = await get_aggregated_vessel(db, mmsi)
if aggregated is not None:
return {
**aggregated,
"received_at": to_iso8601_utc(aggregated.get("received_at")),
"latitude": aggregated["lat"],
"longitude": aggregated["lon"],
}
latest_position_stmt = (
select(VesselPosition)
.where(VesselPosition.mmsi == mmsi)
@@ -1496,6 +1588,30 @@ async def get_vessel_track(
db: AsyncSession = Depends(get_db),
):
cutoff = datetime.now(UTC) - timedelta(hours=hours)
aggregated_points = await get_aggregated_vessel_track(db, mmsi, cutoff=cutoff)
if aggregated_points:
return {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {
"type": "LineString",
"coordinates": [[point["lon"], point["lat"]] for point in aggregated_points],
},
"properties": {
"mmsi": mmsi,
"hours": hours,
"point_count": len(aggregated_points),
"start_at": to_iso8601_utc(aggregated_points[0]["observed_at"]),
"end_at": to_iso8601_utc(aggregated_points[-1]["observed_at"]),
"point_sources": [point["source"] for point in aggregated_points],
},
}
],
"count": 1,
}
result = await db.execute(
select(VesselPosition)
.where(VesselPosition.mmsi == mmsi)
@@ -1532,6 +1648,37 @@ async def get_vessel_track(
}
@router.get("/vessels/{mmsi}/observations")
async def get_vessel_observations(
mmsi: int,
limit: int = Query(100, ge=1, le=500),
db: AsyncSession = Depends(get_db),
):
"""Return raw AIS observations for debugging source-level collector facts."""
observations = await get_vessel_raw_observations(db, mmsi, limit=limit)
return {
"mmsi": mmsi,
"count": len(observations),
"observations": [item.to_dict() for item in observations],
"conflict_candidates": build_field_conflict_candidates(observations),
}
@router.get("/vessels/{mmsi}/conflicts")
async def get_vessel_conflicts(mmsi: int, db: AsyncSession = Depends(get_db)):
"""Return recorded AIS conflicts plus current raw-observation candidates."""
records = await get_vessel_conflict_records(db, mmsi)
observations = await get_vessel_raw_observations(db, mmsi, limit=500)
return {
"mmsi": mmsi,
"count": len(records),
"conflicts": [item.to_dict() for item in records],
"candidates": build_field_conflict_candidates(observations),
}
@router.get("/geo/bgp-anomalies")
async def get_bgp_anomalies_geojson(
severity: Optional[str] = Query(None),

View File

@@ -1,7 +1,6 @@
import os
import yaml
from functools import lru_cache
from typing import Optional
COLLECTOR_URL_KEYS = {
@@ -32,6 +31,7 @@ COLLECTOR_URL_KEYS = {
"nro_delegated_prefix_geo": "nro.delegated_stats_url",
"news_live_streams": "news_live_streams.channels_url",
"barentswatch_vessels": "barentswatch_vessels.url",
"aisstream_vessels": "aisstream_vessels.url",
}
@@ -74,7 +74,7 @@ class DataSourcesConfig:
from app.models.datasource_config import DataSourceConfig
query = select(DataSourceConfig).where(
DataSourceConfig.name == collector_name, DataSourceConfig.is_active == True
DataSourceConfig.name == collector_name, DataSourceConfig.is_active
)
result = await db.execute(query)
db_config = result.scalar_one_or_none()

View File

@@ -98,3 +98,7 @@ news_live_streams:
barentswatch_vessels:
# BarentsWatch Live AIS latest combined endpoint. Requires an AIS bearer token.
url: "https://live.ais.barentswatch.no/v1/latest/combined"
aisstream_vessels:
# AISStream realtime WebSocket endpoint. Requires an AISStream API key.
url: "wss://stream.aisstream.io/v0/stream"

View File

@@ -245,6 +245,18 @@ DEFAULT_DATASOURCES = {
"credential_provider": "barentswatch",
"credential_status": "supported",
},
"aisstream_vessels": {
"id": 28,
"name": "AISStream Vessels",
"display_name": "AISStream 实时船舶",
"module": "L4",
"priority": "P1",
"frequency_minutes": 1,
"is_free": True,
"requires_credentials": True,
"credential_provider": "aisstream",
"credential_status": "supported",
},
}
ID_TO_COLLECTOR = {info["id"]: name for name, info in DEFAULT_DATASOURCES.items()}

View File

@@ -12,7 +12,7 @@ from app.models.system_setting import SystemSetting
from app.models.playground_session import PlaygroundSession
from app.models.playground_message import PlaygroundMessage
from app.models.system_log import SystemLog, AuditLog
from app.models.vessel import VesselPosition, VesselStatic
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth, VesselPosition, VesselStatic
from app.models.datasource_mapping import DataSourceMappingTemplate
__all__ = [
@@ -31,7 +31,12 @@ __all__ = [
"BGPObservation",
"SystemLog",
"AuditLog",
"PlaygroundSession",
"PlaygroundMessage",
"VesselPosition",
"VesselStatic",
"AISRawObservation",
"AISConflictRecord",
"AISSourceHealth",
"DataSourceMappingTemplate",
]

View File

@@ -1,6 +1,6 @@
"""Vessel AIS models for live maritime tracking."""
from sqlalchemy import BigInteger, Column, DateTime, Float, Index, Integer, SmallInteger, String
from sqlalchemy import BigInteger, Column, DateTime, Float, Index, Integer, JSON, SmallInteger, String
from sqlalchemy.sql import func
from app.core.time import to_iso8601_utc
@@ -73,3 +73,113 @@ class VesselPosition(Base):
"nav_status": self.nav_status,
"received_at": to_iso8601_utc(self.received_at),
}
class AISRawObservation(Base):
"""Source-level AIS fact before aggregation and conflict resolution."""
__tablename__ = "ais_raw_observations"
id = Column(Integer, primary_key=True, autoincrement=True)
target_schema = Column(String(64), nullable=False, default="vessel_ais", index=True)
source = Column(String(100), nullable=False, index=True)
entity_key = Column(String(64), nullable=False, index=True)
delivery_mode = Column(String(32), nullable=False, index=True)
transport = Column(String(32), nullable=False, index=True)
message_type = Column(String(64), nullable=True, index=True)
source_message_id = Column(String(128), nullable=True, index=True)
observation_hash = Column(String(64), nullable=False, unique=True, index=True)
observed_at = Column(DateTime(timezone=True), nullable=False, index=True)
collected_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
normalized_payload = Column(JSON, default=dict)
raw_payload = Column(JSON, default=dict)
quality_flags = Column(JSON, default=list)
__table_args__ = (
Index("idx_ais_raw_entity_observed", "target_schema", "entity_key", "observed_at"),
Index("idx_ais_raw_source_entity", "source", "entity_key"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"target_schema": self.target_schema,
"source": self.source,
"entity_key": self.entity_key,
"delivery_mode": self.delivery_mode,
"transport": self.transport,
"message_type": self.message_type,
"source_message_id": self.source_message_id,
"observation_hash": self.observation_hash,
"observed_at": to_iso8601_utc(self.observed_at),
"collected_at": to_iso8601_utc(self.collected_at),
"normalized_payload": self.normalized_payload or {},
"raw_payload": self.raw_payload or {},
"quality_flags": self.quality_flags or [],
}
class AISConflictRecord(Base):
"""Recorded field-level disagreement between AIS sources."""
__tablename__ = "ais_conflict_records"
id = Column(Integer, primary_key=True, autoincrement=True)
target_schema = Column(String(64), nullable=False, default="vessel_ais", index=True)
entity_key = Column(String(64), nullable=False, index=True)
field = Column(String(64), nullable=False, index=True)
candidates = Column(JSON, default=dict)
selected_source = Column(String(100), nullable=True, index=True)
selected_value = Column(JSON, nullable=True)
selected_reason = Column(String(64), nullable=True, index=True)
resolved_by = Column(String(32), nullable=False, default="system", index=True)
status = Column(String(32), nullable=False, default="open", index=True)
created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
__table_args__ = (
Index("idx_ais_conflict_entity_field", "target_schema", "entity_key", "field"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"target_schema": self.target_schema,
"entity_key": self.entity_key,
"field": self.field,
"candidates": self.candidates or {},
"selected_source": self.selected_source,
"selected_value": self.selected_value,
"selected_reason": self.selected_reason,
"resolved_by": self.resolved_by,
"status": self.status,
"created_at": to_iso8601_utc(self.created_at),
"updated_at": to_iso8601_utc(self.updated_at),
}
class AISSourceHealth(Base):
"""Runtime health signal for an AIS collector source."""
__tablename__ = "ais_source_health"
source = Column(String(100), primary_key=True)
connection_state = Column(String(32), nullable=False, default="disconnected", index=True)
last_seen_at = Column(DateTime(timezone=True), nullable=True, index=True)
last_success_at = Column(DateTime(timezone=True), nullable=True, index=True)
last_error = Column(String(500), nullable=True)
message_rate = Column(Float, nullable=True)
lag_seconds = Column(Float, nullable=True)
updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), index=True)
def to_dict(self) -> dict:
return {
"source": self.source,
"connection_state": self.connection_state,
"last_seen_at": to_iso8601_utc(self.last_seen_at),
"last_success_at": to_iso8601_utc(self.last_success_at),
"last_error": self.last_error,
"message_rate": self.message_rate,
"lag_seconds": self.lag_seconds,
"updated_at": to_iso8601_utc(self.updated_at),
}

View File

@@ -36,6 +36,7 @@ from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
from app.services.collectors.opengeofeed import OpenGeoFeedPrefixGeoCollector
from app.services.collectors.nro_delegated import NRODelegatedPrefixGeoCollector
from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
from app.services.collectors.aisstream import AISStreamCollector
from app.services.collectors.vessel_ais import VesselAISCollector
collector_registry.register(TOP500Collector())
@@ -65,3 +66,40 @@ collector_registry.register(OpenGeoFeedPrefixGeoCollector())
collector_registry.register(NRODelegatedPrefixGeoCollector())
collector_registry.register(NewsLiveStreamsCollector())
collector_registry.register(VesselAISCollector())
collector_registry.register(AISStreamCollector())
__all__ = [
"BaseCollector",
"HTTPCollector",
"IntervalCollector",
"collector_registry",
"CollectorRegistry",
"TOP500Collector",
"EpochAIGPUCollector",
"HuggingFaceModelCollector",
"HuggingFaceDatasetCollector",
"HuggingFaceSpacesCollector",
"PeeringDBIXPCollector",
"PeeringDBNetworkCollector",
"PeeringDBFacilityCollector",
"TeleGeographyCableCollector",
"TeleGeographyLandingPointCollector",
"TeleGeographyCableSystemCollector",
"CloudflareRadarDeviceCollector",
"CloudflareRadarTrafficCollector",
"CloudflareRadarTopASCollector",
"ArcGISCableCollector",
"FAOLandingPointCollector",
"ArcGISLandingPointCollector",
"ArcGISCableLandingRelationCollector",
"SpaceTrackTLECollector",
"CelesTrakTLECollector",
"RISLiveCollector",
"BGPStreamBackfillCollector",
"IPtoASNPrefixGeoCollector",
"OpenGeoFeedPrefixGeoCollector",
"NRODelegatedPrefixGeoCollector",
"NewsLiveStreamsCollector",
"VesselAISCollector",
"AISStreamCollector",
]

View File

@@ -0,0 +1,276 @@
"""AISStream WebSocket collector for realtime vessel AIS observations."""
from datetime import UTC, datetime
import asyncio
import json
import os
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.data_sources import get_data_sources_config
from app.models.datasource_config import DataSourceConfig
from app.services.collectors.base import BaseCollector
from app.services.vessel_ais_aggregation import (
AISSTREAM_DELIVERY_MODE,
AISSTREAM_TRANSPORT,
record_vessel_ais_observation,
update_ais_source_health,
)
from app.services.vessel_types import normalize_vessel_type_name
DEFAULT_AISSTREAM_URL = "wss://stream.aisstream.io/v0/stream"
DEFAULT_BOUNDING_BOXES = [[[-90, -180], [90, 180]]]
DEFAULT_MESSAGE_TYPES = ["PositionReport", "ShipStaticData"]
class AISStreamCollector(BaseCollector):
"""Collect AISStream WebSocket messages into the raw AIS observation layer."""
name = "aisstream_vessels"
priority = "P1"
module = "L4"
frequency_hours = 1
data_type = "vessel_ais"
fail_on_empty = False
async def _load_datasource_config(self) -> DataSourceConfig | None:
if self._db_session is None:
return None
result = await self._db_session.execute(
select(DataSourceConfig)
.where(DataSourceConfig.name == self.name)
.where(DataSourceConfig.is_active.is_(True))
)
return result.scalar_one_or_none()
async def _get_effective_config(self) -> dict[str, Any]:
datasource_config = await self._load_datasource_config()
config = dict(datasource_config.config or {}) if datasource_config else {}
auth_config = dict(datasource_config.auth_config or {}) if datasource_config else {}
endpoint = (
(datasource_config.endpoint if datasource_config else None)
or self._resolved_url
or get_data_sources_config().get_yaml_url(self.name)
or DEFAULT_AISSTREAM_URL
)
api_key = (
auth_config.get("api_key")
or config.get("api_key")
or os.getenv("AISSTREAM_API_KEY")
)
return {
"endpoint": endpoint,
"api_key": api_key,
"bounding_boxes": config.get("bounding_boxes") or DEFAULT_BOUNDING_BOXES,
"message_types": config.get("message_types") or DEFAULT_MESSAGE_TYPES,
"max_messages": int(config.get("max_messages") or 500),
"receive_timeout_seconds": float(config.get("receive_timeout_seconds") or 30),
}
async def fetch(self) -> list[dict[str, Any]]:
config = await self._get_effective_config()
if not config["api_key"]:
raise RuntimeError("AISStream API key is not configured")
try:
import websockets
except ImportError as exc:
raise RuntimeError("Python package 'websockets' is required for AISStream") from exc
subscription = {
"APIKey": config["api_key"],
"BoundingBoxes": config["bounding_boxes"],
"FilterMessageTypes": config["message_types"],
}
messages: list[dict[str, Any]] = []
try:
async with websockets.connect(config["endpoint"]) as websocket:
await websocket.send(json.dumps(subscription))
while len(messages) < config["max_messages"]:
try:
raw_message = await asyncio.wait_for(
websocket.recv(),
timeout=config["receive_timeout_seconds"],
)
except TimeoutError:
break
payload = json.loads(raw_message)
if isinstance(payload, dict):
messages.append(payload)
except Exception as exc:
if self._db_session is not None:
await update_ais_source_health(
self._db_session,
source=self.name,
connection_state="disconnected",
last_error=f"{exc.__class__.__name__}: {exc}",
)
await self._db_session.commit()
raise
return messages
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
records = []
for item in raw_data:
record = self._normalize_message(item)
if record:
records.append(record)
return records
async def _save_data(
self,
db: AsyncSession,
data: list[dict[str, Any]],
task_id: int | None = None,
snapshot_id: int | None = None,
) -> int:
now = datetime.now(UTC)
records_added = 0
latest_observed_at = now
for index, item in enumerate(data):
observed_at = item.get("received_at") or now
observation = await record_vessel_ais_observation(
db,
source=self.name,
normalized_payload=item,
raw_payload=item.get("_raw_payload") or item,
delivery_mode=AISSTREAM_DELIVERY_MODE,
transport=AISSTREAM_TRANSPORT,
message_type=item.get("_message_type") or "PositionReport",
source_message_id=item.get("_source_message_id"),
observed_at=observed_at,
collected_at=now,
)
if observation is not None:
records_added += 1
if isinstance(observed_at, datetime) and observed_at > latest_observed_at:
latest_observed_at = observed_at
if (index + 1) % 1000 == 0:
await self.update_progress(index + 1, commit=True)
await update_ais_source_health(
db,
source=self.name,
connection_state="connected",
observed_count=len(data),
last_seen_at=latest_observed_at,
last_success_at=now if data else None,
lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
)
await db.commit()
await self.update_progress(records_added, force=True)
return records_added
def _normalize_message(self, item: dict[str, Any]) -> dict[str, Any] | None:
message_type = str(item.get("MessageType") or item.get("message_type") or "")
metadata = item.get("MetaData") if isinstance(item.get("MetaData"), dict) else {}
message = item.get("Message") if isinstance(item.get("Message"), dict) else {}
body = message.get(message_type) if isinstance(message.get(message_type), dict) else message
if not isinstance(body, dict):
body = {}
mmsi = _as_int(_pick(metadata, "MMSI", "mmsi") or _pick(body, "MMSI", "mmsi"))
if mmsi is None:
return None
received_at = _parse_datetime(
_pick(metadata, "time_utc", "Time_UTC", "timestamp")
or _pick(body, "Timestamp", "timestamp", "time")
)
ship_name = _clean_text(
_pick(body, "Name", "ShipName", "name")
or _pick(metadata, "ShipName", "ship_name", "name")
)
record: dict[str, Any] = {
"mmsi": mmsi,
"received_at": received_at,
"_message_type": message_type or None,
"_source_message_id": item.get("MessageID") or item.get("message_id"),
"_raw_payload": item,
}
lat = _as_float(_pick(body, "Latitude", "lat", "latitude"))
lon = _as_float(_pick(body, "Longitude", "lon", "lng", "longitude"))
if lat is not None and lon is not None:
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
return None
record.update(
{
"lat": lat,
"lon": lon,
"sog": _as_float(_pick(body, "Sog", "SOG", "speedOverGround")),
"cog": _as_float(_pick(body, "Cog", "COG", "courseOverGround")),
"heading": _as_int(_pick(body, "TrueHeading", "Heading", "heading")),
"nav_status": _as_int(_pick(body, "NavigationalStatus", "nav_status")),
}
)
vessel_type = _as_int(_pick(body, "Type", "ShipType", "vessel_type"))
record.update(
{
"name": ship_name,
"callsign": _pick(body, "CallSign", "callsign"),
"imo": _as_int(_pick(body, "ImoNumber", "IMO", "imo")),
"vessel_type": vessel_type,
"vessel_type_name": _pick(body, "TypeName", "ShipTypeName", "vessel_type_name")
or normalize_vessel_type_name(vessel_type),
"length": _as_float(_pick(body, "DimensionToBow", "Length", "length")),
"width": _as_float(_pick(body, "DimensionToPort", "Width", "width")),
}
)
return record
def _pick(item: dict[str, Any], *keys: str) -> Any:
for key in keys:
if key in item and item[key] not in (None, ""):
return item[key]
return None
def _clean_text(value: Any) -> str | None:
if value in (None, ""):
return None
text = str(value).strip()
return text or None
def _as_float(value: Any) -> float | None:
try:
if value in (None, ""):
return None
return float(value)
except (TypeError, ValueError):
return None
def _as_int(value: Any) -> int | None:
try:
if value in (None, ""):
return None
return int(float(value))
except (TypeError, ValueError):
return None
def _parse_datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if not value:
return None
if isinstance(value, (int, float)):
timestamp = float(value)
if timestamp > 10_000_000_000:
timestamp /= 1000
return datetime.fromtimestamp(timestamp, UTC)
if isinstance(value, str):
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
except ValueError:
return None
return None

View File

@@ -4,7 +4,7 @@ from datetime import UTC, datetime, timedelta
from typing import Any
import httpx
from sqlalchemy import delete, select
from sqlalchemy import delete
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.vessel import VesselPosition, VesselStatic
@@ -14,15 +14,13 @@ from app.services.barentswatch import (
resolve_barentswatch_config,
)
from app.services.collectors.base import BaseCollector
VESSEL_TYPE_NAMES = {
30: "Fishing",
35: "Military",
60: "Passenger",
70: "Cargo",
80: "Tanker",
}
from app.services.vessel_ais_aggregation import (
BARENTSWATCH_DELIVERY_MODE,
BARENTSWATCH_TRANSPORT,
record_vessel_ais_observation,
update_ais_source_health,
)
from app.services.vessel_types import normalize_vessel_type_name
class VesselAISCollector(BaseCollector):
@@ -92,6 +90,18 @@ class VesselAISCollector(BaseCollector):
records_added = 0
for index, item in enumerate(data):
observed_at = item.get("received_at") or now
await record_vessel_ais_observation(
db,
source=self.name,
normalized_payload=item,
raw_payload=item,
delivery_mode=BARENTSWATCH_DELIVERY_MODE,
transport=BARENTSWATCH_TRANSPORT,
observed_at=observed_at,
collected_at=now,
)
static = await db.get(VesselStatic, item["mmsi"])
if static is None:
static = VesselStatic(mmsi=item["mmsi"])
@@ -122,7 +132,7 @@ class VesselAISCollector(BaseCollector):
cog=item.get("cog"),
heading=item.get("heading"),
nav_status=item.get("nav_status"),
received_at=item.get("received_at") or now,
received_at=observed_at,
)
)
records_added += 1
@@ -130,6 +140,19 @@ class VesselAISCollector(BaseCollector):
if (index + 1) % 1000 == 0:
await self.update_progress(index + 1, commit=True)
latest_observed_at = max(
(item.get("received_at") for item in data if item.get("received_at")),
default=now,
)
await update_ais_source_health(
db,
source=self.name,
connection_state="connected",
observed_count=len(data),
last_seen_at=latest_observed_at,
last_success_at=now if data else None,
lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
)
await db.execute(
delete(VesselPosition).where(VesselPosition.received_at < now - timedelta(hours=24))
)
@@ -156,7 +179,7 @@ class VesselAISCollector(BaseCollector):
vessel_type = _as_int(_pick(item, "vessel_type", "shipType", "ship_type", "ShipType"))
vessel_type_name = (
_pick(item, "vessel_type_name", "shipTypeName", "ship_type_name", "VesselTypeName")
or _vessel_type_name(vessel_type)
or normalize_vessel_type_name(vessel_type)
)
received_at = _parse_datetime(_pick(item, "received_at", "timestamp", "time", "msgtime"))
@@ -255,19 +278,3 @@ def _parse_datetime(value: Any) -> datetime | None:
except ValueError:
return None
return None
def _vessel_type_name(vessel_type: int | None) -> str:
if vessel_type is None:
return "Other"
if 70 <= vessel_type <= 79:
return "Cargo"
if 80 <= vessel_type <= 89:
return "Tanker"
if 60 <= vessel_type <= 69:
return "Passenger"
if vessel_type == 30:
return "Fishing"
if vessel_type == 35:
return "Military"
return VESSEL_TYPE_NAMES.get(vessel_type, "Other")

View File

@@ -66,9 +66,68 @@ BARENTSWATCH_DEFAULT_GUIDE = CredentialGuideDefault(
""",
)
AISSTREAM_DEFAULT_GUIDE = CredentialGuideDefault(
provider="aisstream",
title="AISStream API Key 获取教程",
prompt=(
"请生成一份中文教程,指导开发者获取 AISStream 的 API Key 并配置到 Planet。"
"教程要面向已经有本地开发环境的人,包含注册/登录 AISStream、获取 API Key、"
"理解免费额度和订阅范围、在 Planet 设置中心填写 API Key、配置 bounding boxes "
"和 message types、验证连接、常见失败排查。必须提醒用户以 AISStream 当前官网和"
"服务条款为准,不要编造具体页面按钮文案。"
),
markdown="""## AISStream API Key 获取
官方入口https://aisstream.io/
1. 打开 AISStream 官网,按当前页面指引注册或登录账号。
2. 在账号/API 管理页面创建或复制你的 API Key。
3. 先确认当前账号额度、使用条款和可订阅区域。实时 AIS 流量可能很大,不建议一开始订阅全球范围。
4. 回到 Planet 的 `设置 -> 采集器设置 -> AISStream 实时船舶`。
5. 在 `AISStream 凭证` 中填入 API Key。
6. Endpoint 通常保持默认:`wss://stream.aisstream.io/v0/stream`。
7. 按需配置 `Bounding Boxes JSON` 和 `消息类型`。
8. 点击连接测试,确认系统能读取凭证且 WebSocket endpoint 格式有效。
9. 保存采集器设置后再运行 `aisstream_vessels` collector。
### 推荐配置
默认消息类型:
```json
["PositionReport", "ShipStaticData"]
```
默认 Bounding Boxes 示例:
```json
[[[-90, -180], [90, 180]]]
```
这个示例表示全球范围。实际使用时建议先改成较小区域,降低消息量和处理压力。
### 请求规则
- Endpoint`wss://stream.aisstream.io/v0/stream`
- 传输方式WebSocket
- API Key 放在订阅 payload 中,不放在 HTTP header。
- Planet 会把 AISStream 标记为 `delivery_mode = realtime_stream`、`transport = websocket`。
- AISStream collector 只写入 AIS raw observations不直接覆盖最终船只展示表。
### 常见排查
- `未找到凭证`:确认 API Key 已保存到采集器设置,或设置了 `AISSTREAM_API_KEY` 环境变量 / `~/.zshrc`。
- `endpoint 必须是 ws:// 或 wss://`AISStream 是 WebSocket 流接口,不要填普通 `https://` API 地址。
- 采集量过大:缩小 `Bounding Boxes JSON`,减少 `message_types`,或降低单次最大消息数。
- 没有船只数据:确认订阅区域内确实有 AIS 活动,并检查 API Key 当前额度和权限。
- 连接中断实时流可能受网络和上游限流影响collector 会记录源健康状态供聚合服务回退。
""",
)
DEFAULT_CREDENTIAL_GUIDES = {
BARENTSWATCH_DEFAULT_GUIDE.provider: BARENTSWATCH_DEFAULT_GUIDE,
AISSTREAM_DEFAULT_GUIDE.provider: AISSTREAM_DEFAULT_GUIDE,
}

View File

@@ -26,6 +26,7 @@ from app.services.barentswatch import (
CONNECTIVITY_VALIDATION_KEY = "connectivity_validation"
CONNECTIVITY_STORE_CATEGORY = "datasource_connectivity_validations"
SUPPORTED_CREDENTIAL_PROVIDERS = {"barentswatch", "spacetrack", "aisstream"}
def _sha256_json(payload: Any) -> str:
@@ -43,6 +44,36 @@ def _resolve_spacetrack_credentials() -> tuple[str, str, str]:
return username, password, source or "missing"
async def _resolve_aisstream_api_key(
db=None,
credential_override: dict[str, str] | None = None,
) -> tuple[str, str]:
if credential_override and credential_override.get("api_key"):
return str(credential_override["api_key"]), "draft"
env_key = os.getenv("AISSTREAM_API_KEY")
zshrc_key = _read_zshrc_env().get("AISSTREAM_API_KEY")
if db is not None:
result = await db.execute(
select(DataSourceConfig)
.where(DataSourceConfig.name == "aisstream_vessels")
.where(DataSourceConfig.is_active.is_(True))
)
record = result.scalar_one_or_none()
if record:
auth_config = record.auth_config or {}
runtime_config = record.config or {}
api_key = auth_config.get("api_key") or runtime_config.get("api_key")
if api_key:
return str(api_key), "datasource_config"
if env_key:
return env_key, "environment"
if zshrc_key:
return zshrc_key, "~/.zshrc"
return "", "missing"
def strip_connectivity_validation(config: dict | None) -> dict:
cleaned = dict(config or {})
cleaned.pop(CONNECTIVITY_VALIDATION_KEY, None)
@@ -103,6 +134,10 @@ async def build_builtin_connectivity_checksum(
"password": password,
}
)
elif credential_provider == "aisstream":
api_key, credential_source = await _resolve_aisstream_api_key(db, credential_override)
has_credentials = bool(api_key)
credential_fingerprint = _sha256_json({"api_key": api_key})
elif defaults.get("requires_credentials"):
credential_source = str(credential_provider or "unsupported")
@@ -130,6 +165,7 @@ async def test_builtin_connectivity(
headers: dict | None,
config: dict | None,
db=None,
credential_override: dict[str, str] | None = None,
) -> dict[str, Any]:
defaults = DEFAULT_DATASOURCES.get(source)
if not defaults:
@@ -145,6 +181,7 @@ async def test_builtin_connectivity(
headers,
config,
db,
credential_override,
)
if credential_context["requires_credentials"] and not credential_context["has_credentials"]:
return {
@@ -155,10 +192,9 @@ async def test_builtin_connectivity(
"settings_tab": "collector_credentials",
**credential_context,
}
supported_credential_providers = {"barentswatch", "spacetrack"}
if (
credential_context["requires_credentials"]
and credential_context["credential_provider"] not in supported_credential_providers
and credential_context["credential_provider"] not in SUPPORTED_CREDENTIAL_PROVIDERS
):
return {
"success": False,
@@ -174,6 +210,23 @@ async def test_builtin_connectivity(
timeout = float(request_config.get("timeout") or 30)
request_endpoint = endpoint
if credential_context["credential_provider"] == "aisstream":
if not str(request_endpoint).startswith(("ws://", "wss://")):
return {
"success": False,
"checksum": checksum,
"stage": "endpoint",
"message": "AISStream endpoint 必须是 ws:// 或 wss:// WebSocket 地址。",
**credential_context,
}
return {
"success": True,
"checksum": checksum,
"stage": "credentials",
"message": "AISStream 凭证已配置WebSocket endpoint 格式有效。",
**credential_context,
}
try:
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
if credential_context["credential_provider"] == "barentswatch":

View File

@@ -0,0 +1,574 @@
"""AIS raw observation and aggregation support for vessel collectors."""
from datetime import UTC, datetime
from hashlib import sha256
import json
from typing import Any, Iterable
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth
from app.services.vessel_types import normalize_vessel_type_name
VESSEL_AIS_SCHEMA = "vessel_ais"
BARENTSWATCH_DELIVERY_MODE = "polling"
BARENTSWATCH_TRANSPORT = "http"
AISSTREAM_DELIVERY_MODE = "realtime_stream"
AISSTREAM_TRANSPORT = "websocket"
DELIVERY_MODE_PRIORITY = {
"realtime_stream": 40,
"batch_stream": 30,
"polling": 20,
"snapshot": 10,
}
DYNAMIC_FIELDS = ("lat", "lon", "sog", "cog", "heading", "nav_status")
CONFLICT_FIELDS = (
"name",
"callsign",
"imo",
"flag",
"vessel_type",
"vessel_type_name",
"length",
"width",
"draught",
)
def _json_default(value: Any) -> Any:
if isinstance(value, datetime):
return value.astimezone(UTC).isoformat()
return str(value)
def _stable_payload(value: Any) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=_json_default)
def _jsonable(value: Any) -> Any:
if isinstance(value, datetime):
return value.astimezone(UTC).isoformat()
if isinstance(value, dict):
return {str(key): _jsonable(item) for key, item in value.items()}
if isinstance(value, list):
return [_jsonable(item) for item in value]
return value
def _coerce_datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, (int, float)):
timestamp = float(value)
if timestamp > 10_000_000_000:
timestamp /= 1000
return datetime.fromtimestamp(timestamp, UTC)
if isinstance(value, str) and value:
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
except ValueError:
return None
return None
def build_observation_hash(
*,
source: str,
entity_key: str,
message_type: str | None,
observed_at: datetime,
normalized_payload: dict[str, Any],
source_message_id: str | None = None,
) -> str:
"""Build a deterministic idempotency key for one source-level AIS observation."""
if source_message_id:
basis = {
"source": source,
"entity_key": entity_key,
"source_message_id": source_message_id,
}
else:
basis = {
"source": source,
"entity_key": entity_key,
"message_type": message_type,
"observed_at": observed_at.astimezone(UTC).isoformat(),
"payload": normalized_payload,
}
return sha256(_stable_payload(basis).encode("utf-8")).hexdigest()
def build_field_conflict_candidates(
observations: Iterable[AISRawObservation],
fields: Iterable[str] = CONFLICT_FIELDS,
) -> list[dict[str, Any]]:
"""Return current field disagreements from raw observations without mutating state."""
candidates_by_field: dict[str, dict[str, Any]] = {}
for observation in observations:
payload = observation.normalized_payload or {}
for field in fields:
value = payload.get(field)
if value in (None, ""):
continue
field_candidates = candidates_by_field.setdefault(field, {})
field_candidates[observation.source] = value
conflicts = []
for field, candidates in sorted(candidates_by_field.items()):
unique_values = {_stable_payload(value) for value in candidates.values()}
if len(unique_values) <= 1:
continue
conflicts.append(
{
"field": field,
"candidates": candidates,
"status": "candidate",
}
)
return conflicts
def _payload_value(payload: dict[str, Any], field: str) -> Any:
value = payload.get(field)
return None if value in (None, "") else value
def _clean_text(value: Any) -> str | None:
if value in (None, ""):
return None
text = str(value).strip()
return text or None
def _raw_metadata_value(observation: AISRawObservation, field: str) -> Any:
raw_payload = observation.raw_payload or {}
metadata = raw_payload.get("MetaData") if isinstance(raw_payload, dict) else None
if not isinstance(metadata, dict):
return None
if field == "name":
return _clean_text(metadata.get("ShipName") or metadata.get("ship_name") or metadata.get("name"))
return None
def _delivery_priority(observation: AISRawObservation) -> int:
return DELIVERY_MODE_PRIORITY.get(str(observation.delivery_mode or ""), 0)
def _has_valid_position(payload: dict[str, Any]) -> bool:
try:
lat = float(payload.get("lat"))
lon = float(payload.get("lon"))
except (TypeError, ValueError):
return False
return -90 <= lat <= 90 and -180 <= lon <= 180
def _is_future_observation(observation: AISRawObservation, now: datetime) -> bool:
return observation.observed_at > now
def _select_position_observation(
observations: list[AISRawObservation],
*,
now: datetime,
) -> tuple[AISRawObservation | None, list[str]]:
rejected_flags: list[str] = []
candidates = []
for observation in observations:
payload = observation.normalized_payload or {}
if not _has_valid_position(payload):
rejected_flags.append("invalid_position")
continue
if _is_future_observation(observation, now):
rejected_flags.append("future_timestamp")
continue
candidates.append(observation)
if not candidates:
return None, sorted(set(rejected_flags))
candidates.sort(
key=lambda item: (
item.observed_at,
_delivery_priority(item),
item.collected_at,
item.id or 0,
),
reverse=True,
)
return candidates[0], sorted(set(rejected_flags))
def _select_static_field(
observations: list[AISRawObservation],
field: str,
) -> tuple[Any, str | None, str | None]:
candidates = []
for observation in observations:
value = _payload_value(observation.normalized_payload or {}, field)
if value is None:
value = _raw_metadata_value(observation, field)
if value is None:
continue
candidates.append((observation, value))
if not candidates:
return None, None, None
candidates.sort(
key=lambda item: (
_delivery_priority(item[0]),
item[0].observed_at,
item[0].collected_at,
item[0].id or 0,
),
reverse=True,
)
selected_observation, selected_value = candidates[0]
unique_values = {_stable_payload(value) for _, value in candidates}
reason = "delivery_mode_priority" if len(unique_values) > 1 else "non_empty_priority"
return selected_value, selected_observation.source, reason
def _build_source_summary(observations: list[AISRawObservation]) -> dict[str, dict[str, Any]]:
summary: dict[str, dict[str, Any]] = {}
for observation in observations:
source_summary = summary.setdefault(
observation.source,
{
"observation_count": 0,
"latest_observed_at": None,
"delivery_mode": observation.delivery_mode,
"transport": observation.transport,
"message_types": [],
},
)
source_summary["observation_count"] += 1
latest_observed_at = source_summary["latest_observed_at"]
if latest_observed_at is None or observation.observed_at > latest_observed_at:
source_summary["latest_observed_at"] = observation.observed_at
if observation.message_type and observation.message_type not in source_summary["message_types"]:
source_summary["message_types"].append(observation.message_type)
return summary
def _build_aggregated_vessel(
entity_key: str,
observations: list[AISRawObservation],
*,
now: datetime,
) -> dict[str, Any] | None:
position_observation, rejected_flags = _select_position_observation(observations, now=now)
if position_observation is None:
return None
payload = position_observation.normalized_payload or {}
mmsi = int(entity_key)
result: dict[str, Any] = {
"mmsi": mmsi,
"lat": float(payload["lat"]),
"lon": float(payload["lon"]),
"received_at": position_observation.observed_at,
"field_sources": {},
"selected_reasons": {},
"source_summary": _build_source_summary(observations),
"quality_flags": sorted(
set((position_observation.quality_flags or []) + rejected_flags)
),
}
for field in DYNAMIC_FIELDS:
value = _payload_value(payload, field)
if field in ("lat", "lon") or value is not None:
result[field] = value
result["field_sources"][field] = position_observation.source
result["selected_reasons"][field] = "newest_observation"
for field in CONFLICT_FIELDS:
selected_value, selected_source, reason = _select_static_field(observations, field)
if selected_value is None:
continue
result[field] = selected_value
result["field_sources"][field] = selected_source
result["selected_reasons"][field] = reason
result["name"] = result.get("name") or f"MMSI {mmsi}"
result["vessel_type_name"] = result.get("vessel_type_name") or normalize_vessel_type_name(
result.get("vessel_type")
)
return result
async def _upsert_conflict_records(
db: AsyncSession,
entity_key: str,
observations: list[AISRawObservation],
aggregated: dict[str, Any],
) -> int:
conflicts = build_field_conflict_candidates(observations)
now = datetime.now(UTC)
for conflict in conflicts:
field = conflict["field"]
result = await db.execute(
select(AISConflictRecord)
.where(AISConflictRecord.target_schema == VESSEL_AIS_SCHEMA)
.where(AISConflictRecord.entity_key == entity_key)
.where(AISConflictRecord.field == field)
.limit(1)
)
record = result.scalar_one_or_none()
if record is None:
record = AISConflictRecord(
target_schema=VESSEL_AIS_SCHEMA,
entity_key=entity_key,
field=field,
)
db.add(record)
record.candidates = conflict["candidates"]
record.selected_source = (aggregated.get("field_sources") or {}).get(field)
record.selected_value = aggregated.get(field)
record.selected_reason = (aggregated.get("selected_reasons") or {}).get(field)
record.resolved_by = "system"
record.status = "open"
record.updated_at = now
return len(conflicts)
def _group_observations(observations: Iterable[AISRawObservation]) -> dict[str, list[AISRawObservation]]:
grouped: dict[str, list[AISRawObservation]] = {}
for observation in observations:
grouped.setdefault(str(observation.entity_key), []).append(observation)
return grouped
async def record_vessel_ais_observation(
db: AsyncSession,
*,
source: str,
normalized_payload: dict[str, Any],
raw_payload: dict[str, Any] | None = None,
delivery_mode: str,
transport: str,
message_type: str | None = "PositionReport",
source_message_id: str | None = None,
observed_at: datetime | None = None,
collected_at: datetime | None = None,
quality_flags: list[str] | None = None,
) -> AISRawObservation | None:
"""Insert one raw observation if the source-level fact has not already been stored."""
entity_key = str(normalized_payload["mmsi"])
collected_at = collected_at or datetime.now(UTC)
observed_at = (
_coerce_datetime(observed_at)
or _coerce_datetime(normalized_payload.get("received_at"))
or collected_at
)
normalized_json = _jsonable(normalized_payload)
raw_json = _jsonable(raw_payload or {})
observation_hash = build_observation_hash(
source=source,
entity_key=entity_key,
message_type=message_type,
observed_at=observed_at,
normalized_payload=normalized_json,
source_message_id=source_message_id,
)
existing_result = await db.execute(
select(AISRawObservation.id).where(AISRawObservation.observation_hash == observation_hash)
)
if existing_result.scalar_one_or_none() is not None:
return None
observation = AISRawObservation(
target_schema=VESSEL_AIS_SCHEMA,
source=source,
entity_key=entity_key,
delivery_mode=delivery_mode,
transport=transport,
message_type=message_type,
source_message_id=source_message_id,
observation_hash=observation_hash,
observed_at=observed_at,
collected_at=collected_at,
normalized_payload=normalized_json,
raw_payload=raw_json,
quality_flags=quality_flags or [],
)
db.add(observation)
return observation
async def aggregate_vessel_observations(
db: AsyncSession,
observations: Iterable[AISRawObservation],
*,
write_conflicts: bool = True,
) -> list[dict[str, Any]]:
now = datetime.now(UTC)
vessels = []
for entity_key, entity_observations in _group_observations(observations).items():
aggregated = _build_aggregated_vessel(entity_key, entity_observations, now=now)
if aggregated is None:
continue
if write_conflicts:
aggregated["conflict_count"] = await _upsert_conflict_records(
db,
entity_key,
entity_observations,
aggregated,
)
else:
aggregated["conflict_count"] = len(build_field_conflict_candidates(entity_observations))
vessels.append(aggregated)
vessels.sort(key=lambda item: item.get("received_at") or datetime.min.replace(tzinfo=UTC), reverse=True)
return vessels
async def get_aggregated_vessels(
db: AsyncSession,
*,
bbox: tuple[float, float, float, float] | None = None,
limit: int | None = None,
) -> list[dict[str, Any]]:
stmt = (
select(AISRawObservation)
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
.order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc())
)
if limit and limit > 0:
stmt = stmt.limit(max(limit * 20, limit))
result = await db.execute(stmt)
if not hasattr(result, "scalars"):
return []
vessels = await aggregate_vessel_observations(db, result.scalars().all())
if bbox is not None:
lon_min, lat_min, lon_max, lat_max = bbox
vessels = [
vessel
for vessel in vessels
if lon_min <= float(vessel["lon"]) <= lon_max
and lat_min <= float(vessel["lat"]) <= lat_max
]
if limit and limit > 0:
return vessels[:limit]
return vessels
async def get_aggregated_vessel(db: AsyncSession, mmsi: int) -> dict[str, Any] | None:
observations = await get_vessel_raw_observations(db, mmsi, limit=1000)
vessels = await aggregate_vessel_observations(db, observations)
return vessels[0] if vessels else None
async def get_aggregated_vessel_track(
db: AsyncSession,
mmsi: int,
*,
cutoff: datetime,
) -> list[dict[str, Any]]:
result = await db.execute(
select(AISRawObservation)
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
.where(AISRawObservation.entity_key == str(mmsi))
.where(AISRawObservation.observed_at >= cutoff)
.order_by(AISRawObservation.observed_at.asc(), AISRawObservation.id.asc())
)
if not hasattr(result, "scalars"):
return []
points: list[dict[str, Any]] = []
seen: set[tuple[str, float, float, str]] = set()
for observation in result.scalars().all():
payload = observation.normalized_payload or {}
if not _has_valid_position(payload):
continue
lat = float(payload["lat"])
lon = float(payload["lon"])
key = (
observation.observed_at.isoformat(),
round(lat, 5),
round(lon, 5),
observation.source,
)
if key in seen:
continue
seen.add(key)
points.append(
{
"lat": lat,
"lon": lon,
"observed_at": observation.observed_at,
"source": observation.source,
"selected_reason": "track_timeline",
"quality_flags": observation.quality_flags or [],
}
)
return points
async def update_ais_source_health(
db: AsyncSession,
*,
source: str,
connection_state: str,
observed_count: int = 0,
last_seen_at: datetime | None = None,
last_success_at: datetime | None = None,
last_error: str | None = None,
lag_seconds: float | None = None,
) -> AISSourceHealth:
"""Upsert the health row for an AIS source."""
now = datetime.now(UTC)
health = await db.get(AISSourceHealth, source)
if health is None:
health = AISSourceHealth(source=source)
db.add(health)
health.connection_state = connection_state
health.last_seen_at = last_seen_at or health.last_seen_at
health.last_success_at = last_success_at or health.last_success_at
health.last_error = last_error
health.message_rate = float(observed_count)
health.lag_seconds = lag_seconds
health.updated_at = now
return health
async def get_vessel_raw_observations(
db: AsyncSession,
mmsi: int,
*,
limit: int = 100,
) -> list[AISRawObservation]:
result = await db.execute(
select(AISRawObservation)
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
.where(AISRawObservation.entity_key == str(mmsi))
.order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc())
.limit(limit)
)
return list(result.scalars().all())
async def get_vessel_conflict_records(
db: AsyncSession,
mmsi: int,
) -> list[AISConflictRecord]:
result = await db.execute(
select(AISConflictRecord)
.where(AISConflictRecord.target_schema == VESSEL_AIS_SCHEMA)
.where(AISConflictRecord.entity_key == str(mmsi))
.order_by(AISConflictRecord.updated_at.desc(), AISConflictRecord.id.desc())
)
return list(result.scalars().all())

View File

@@ -0,0 +1,31 @@
"""Shared AIS vessel type helpers."""
from typing import Any
VESSEL_TYPE_NAMES = {
30: "Fishing",
35: "Military",
60: "Passenger",
70: "Cargo",
80: "Tanker",
}
def normalize_vessel_type_name(vessel_type: Any) -> str:
"""Map AIS numeric vessel type codes to display buckets."""
try:
type_code = int(float(vessel_type))
except (TypeError, ValueError):
return "Other"
if 70 <= type_code <= 79:
return "Cargo"
if 80 <= type_code <= 89:
return "Tanker"
if 60 <= type_code <= 69:
return "Passenger"
if type_code == 30:
return "Fishing"
if type_code == 35:
return "Military"
return VESSEL_TYPE_NAMES.get(type_code, "Other")

View File

@@ -1,11 +1,13 @@
"""Unit tests for data collectors"""
import pytest
from datetime import datetime
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, patch
from app.core.datasource_defaults import DEFAULT_DATASOURCES
from app.services.credential_guides import DEFAULT_CREDENTIAL_GUIDES
from app.services.collectors.top500 import TOP500Collector
from app.services.collectors.base import BaseCollector, HTTPCollector
from app.services.collectors.registry import collector_registry
from app.services.datasource_connectivity import SUPPORTED_CREDENTIAL_PROVIDERS
from app.models.task import CollectionTask
@@ -145,3 +147,30 @@ class TestHTTPCollector:
assert hasattr(collector, "parse_response")
assert callable(collector.fetch)
assert callable(collector.parse_response)
def test_aisstream_collector_is_registered():
collector = collector_registry.get("aisstream_vessels")
assert collector is not None
assert collector.data_type == "vessel_ais"
def test_supported_credential_collectors_have_guides_and_connectivity_provider():
missing: list[str] = []
for source, info in DEFAULT_DATASOURCES.items():
if not info.get("requires_credentials"):
continue
if info.get("credential_status") != "supported":
continue
provider = info.get("credential_provider")
if not provider:
missing.append(f"{source}: missing credential_provider")
continue
if provider not in DEFAULT_CREDENTIAL_GUIDES:
missing.append(f"{source}: missing credential guide for {provider}")
if provider not in SUPPORTED_CREDENTIAL_PROVIDERS:
missing.append(f"{source}: missing connectivity provider for {provider}")
assert missing == []

View File

@@ -1,4 +1,5 @@
from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock
import pytest
from httpx import ASGITransport, AsyncClient
@@ -6,9 +7,16 @@ from httpx import ASGITransport, AsyncClient
from app.api.v1.visualization import convert_vessels_to_geojson
from app.db.session import get_db
from app.main import app
from app.models.vessel import VesselPosition, VesselStatic
from app.models.vessel import AISRawObservation, VesselPosition, VesselStatic
from app.services import barentswatch
from app.services.collectors.aisstream import AISStreamCollector
from app.services.collectors.vessel_ais import VesselAISCollector
from app.services.vessel_ais_aggregation import (
aggregate_vessel_observations,
build_field_conflict_candidates,
build_observation_hash,
record_vessel_ais_observation,
)
def test_vessel_collector_transforms_barentswatch_like_records():
@@ -35,6 +43,328 @@ def test_vessel_collector_transforms_barentswatch_like_records():
assert records[0]["lat"] == pytest.approx(59.91)
def test_vessel_observation_hash_is_stable_for_same_payload():
observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc)
payload = {
"mmsi": 257123000,
"lat": 59.91,
"lon": 10.73,
"received_at": observed_at,
}
first = build_observation_hash(
source="barentswatch_vessels",
entity_key="257123000",
message_type="PositionReport",
observed_at=observed_at,
normalized_payload=payload,
)
second = build_observation_hash(
source="barentswatch_vessels",
entity_key="257123000",
message_type="PositionReport",
observed_at=observed_at,
normalized_payload=dict(reversed(payload.items())),
)
assert first == second
assert len(first) == 64
@pytest.mark.asyncio
async def test_record_vessel_ais_observation_skips_existing_hash():
observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc)
class _Result:
def scalar_one_or_none(self):
return 123
class _Session:
def __init__(self):
self.added = []
async def execute(self, _stmt):
return _Result()
def add(self, item):
self.added.append(item)
db = _Session()
observation = await record_vessel_ais_observation(
db,
source="barentswatch_vessels",
normalized_payload={
"mmsi": 257123000,
"lat": 59.91,
"lon": 10.73,
"received_at": observed_at,
},
delivery_mode="polling",
transport="http",
observed_at=observed_at.isoformat(),
)
assert observation is None
assert db.added == []
def test_build_field_conflict_candidates_from_raw_observations():
observations = [
AISRawObservation(
source="barentswatch_vessels",
normalized_payload={"name": "OSLO TRADER", "flag": "NO"},
),
AISRawObservation(
source="aisstream_vessels",
normalized_payload={"name": "OSLO TRADER II", "flag": "NO"},
),
]
conflicts = build_field_conflict_candidates(observations)
assert conflicts == [
{
"field": "name",
"candidates": {
"aisstream_vessels": "OSLO TRADER II",
"barentswatch_vessels": "OSLO TRADER",
},
"status": "candidate",
}
]
@pytest.mark.asyncio
async def test_aggregate_vessel_observations_prefers_realtime_and_records_conflict():
observed_at = datetime.now(timezone.utc) - timedelta(minutes=5)
class _Result:
def scalar_one_or_none(self):
return None
class _Session:
def __init__(self):
self.added = []
async def execute(self, _stmt):
return _Result()
def add(self, item):
self.added.append(item)
db = _Session()
observations = [
AISRawObservation(
id=1,
source="barentswatch_vessels",
entity_key="257123000",
delivery_mode="polling",
transport="http",
observed_at=observed_at,
collected_at=observed_at,
normalized_payload={
"mmsi": 257123000,
"name": "OSLO TRADER",
"lat": 59.91,
"lon": 10.73,
},
),
AISRawObservation(
id=2,
source="aisstream_vessels",
entity_key="257123000",
delivery_mode="realtime_stream",
transport="websocket",
observed_at=observed_at + timedelta(seconds=10),
collected_at=observed_at + timedelta(seconds=10),
normalized_payload={
"mmsi": 257123000,
"vessel_type": 79,
"lat": 59.92,
"lon": 10.74,
},
raw_payload={"MetaData": {"ShipName": "OSLO TRADER II "}},
),
]
vessels = await aggregate_vessel_observations(db, observations)
assert vessels[0]["lat"] == pytest.approx(59.92)
assert vessels[0]["field_sources"]["lat"] == "aisstream_vessels"
assert vessels[0]["name"] == "OSLO TRADER II"
assert vessels[0]["vessel_type_name"] == "Cargo"
assert vessels[0]["source_summary"]["aisstream_vessels"]["observation_count"] == 1
assert vessels[0]["source_summary"]["barentswatch_vessels"]["delivery_mode"] == "polling"
assert vessels[0]["conflict_count"] == 0
assert db.added == []
@pytest.mark.asyncio
async def test_vessel_collector_writes_raw_observations_without_changing_position_save(monkeypatch):
collector = VesselAISCollector()
collector.update_progress = AsyncMock()
record_observation = AsyncMock()
update_health = AsyncMock()
monkeypatch.setattr(
"app.services.collectors.vessel_ais.record_vessel_ais_observation",
record_observation,
)
monkeypatch.setattr(
"app.services.collectors.vessel_ais.update_ais_source_health",
update_health,
)
class _Session:
def __init__(self):
self.added = []
self.committed = False
async def get(self, *_args):
return None
def add(self, item):
self.added.append(item)
async def execute(self, _stmt):
return None
async def commit(self):
self.committed = True
db = _Session()
observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc)
saved = await collector._save_data(
db,
[
{
"mmsi": 257123000,
"name": "OSLO TRADER",
"lat": 59.91,
"lon": 10.73,
"received_at": observed_at,
}
],
)
assert saved == 1
assert db.committed is True
assert any(isinstance(item, VesselStatic) for item in db.added)
assert any(isinstance(item, VesselPosition) for item in db.added)
record_observation.assert_awaited_once()
assert record_observation.await_args.kwargs["source"] == "barentswatch_vessels"
assert record_observation.await_args.kwargs["normalized_payload"]["mmsi"] == 257123000
update_health.assert_awaited_once()
def test_aisstream_collector_normalizes_position_report():
collector = AISStreamCollector()
records = collector.transform(
[
{
"MessageType": "PositionReport",
"MetaData": {
"MMSI": 257123000,
"ShipName": "OSLO TRADER ",
"time_utc": "2026-04-30T12:00:00Z",
},
"Message": {
"PositionReport": {
"Latitude": 59.91,
"Longitude": 10.73,
"Sog": 12.4,
"Cog": 214,
"TrueHeading": 215,
"NavigationalStatus": 0,
}
},
}
]
)
assert len(records) == 1
assert records[0]["mmsi"] == 257123000
assert records[0]["lat"] == pytest.approx(59.91)
assert records[0]["name"] == "OSLO TRADER"
assert records[0]["_message_type"] == "PositionReport"
def test_aisstream_collector_maps_ship_static_type_name():
collector = AISStreamCollector()
records = collector.transform(
[
{
"MessageType": "ShipStaticData",
"MetaData": {
"MMSI": 257123000,
"time_utc": "2026-04-30T12:00:00Z",
},
"Message": {
"ShipStaticData": {
"Name": "OSLO TRADER",
"Type": 79,
"CallSign": "LAAB",
}
},
}
]
)
assert len(records) == 1
assert records[0]["vessel_type"] == 79
assert records[0]["vessel_type_name"] == "Cargo"
@pytest.mark.asyncio
async def test_aisstream_collector_writes_only_raw_observations(monkeypatch):
collector = AISStreamCollector()
collector.update_progress = AsyncMock()
record_observation = AsyncMock(return_value=object())
update_health = AsyncMock()
monkeypatch.setattr(
"app.services.collectors.aisstream.record_vessel_ais_observation",
record_observation,
)
monkeypatch.setattr(
"app.services.collectors.aisstream.update_ais_source_health",
update_health,
)
class _Session:
def __init__(self):
self.added = []
self.committed = False
def add(self, item):
self.added.append(item)
async def commit(self):
self.committed = True
db = _Session()
saved = await collector._save_data(
db,
[
{
"mmsi": 257123000,
"lat": 59.91,
"lon": 10.73,
"received_at": datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc),
"_message_type": "PositionReport",
}
],
)
assert saved == 1
assert db.added == []
assert db.committed is True
record_observation.assert_awaited_once()
assert record_observation.await_args.kwargs["source"] == "aisstream_vessels"
update_health.assert_awaited_once()
def test_barentswatch_reads_credentials_from_zshrc(tmp_path):
zshrc = tmp_path / ".zshrc"
zshrc.write_text(
@@ -137,7 +467,7 @@ async def test_vessels_geojson_endpoint_filters_type_and_bbox():
async with AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get(
"/api/v1/visualization/geo/vessels",
params={"bbox": "0,50,20,70", "type": "cargo"},
params={"bbox": "0,50,20,70", "type": "cargo", "limit": 0},
)
assert response.status_code == 200