296 lines
11 KiB
Python
296 lines
11 KiB
Python
"""BarentsWatch AIS collector for vessel tracking."""
|
|
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.core.enums import SnapshotStatus
|
|
from app.core.time import to_iso8601_utc
|
|
from app.core.websocket.broadcaster import broadcaster
|
|
from app.services.barentswatch import (
|
|
BARENTSWATCH_LATEST_URL,
|
|
fetch_barentswatch_access_token,
|
|
resolve_barentswatch_config,
|
|
)
|
|
from app.services.collectors.base import BaseCollector
|
|
from app.services.vessel_ais_aggregation import (
|
|
BARENTSWATCH_DELIVERY_MODE,
|
|
BARENTSWATCH_TRANSPORT,
|
|
record_vessel_ais_observation,
|
|
update_ais_source_health,
|
|
)
|
|
from app.services.vessel_types import normalize_vessel_type_name
|
|
|
|
|
|
class VesselAISCollector(BaseCollector):
|
|
"""Collect latest AIS positions and append them to vessel tables."""
|
|
|
|
name = "barentswatch_vessels"
|
|
priority = "P1"
|
|
module = "L4"
|
|
frequency_hours = 1
|
|
data_type = "vessel_ais"
|
|
|
|
@property
|
|
def base_url(self) -> str:
|
|
return self._resolved_url or BARENTSWATCH_LATEST_URL
|
|
|
|
async def _get_access_token(self, client: httpx.AsyncClient) -> str | None:
|
|
config = await resolve_barentswatch_config(self._db_session)
|
|
return await fetch_barentswatch_access_token(client, config)
|
|
|
|
async def fetch(self) -> list[dict[str, Any]]:
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
|
headers: dict[str, str] = {}
|
|
token = await self._get_access_token(client)
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
|
|
response = await client.get(self.base_url, headers=headers)
|
|
if response.status_code == 401 and not token:
|
|
return self._get_sample_data()
|
|
response.raise_for_status()
|
|
payload = response.json()
|
|
|
|
if isinstance(payload, list):
|
|
return [item for item in payload if isinstance(item, dict)]
|
|
if isinstance(payload, dict):
|
|
for key in ("features", "data", "items", "vessels"):
|
|
value = payload.get(key)
|
|
if isinstance(value, list):
|
|
if key == "features":
|
|
return [
|
|
{
|
|
**(item.get("properties") or {}),
|
|
"geometry": item.get("geometry"),
|
|
}
|
|
for item in value
|
|
if isinstance(item, dict)
|
|
]
|
|
return [item for item in value if isinstance(item, dict)]
|
|
return self._get_sample_data()
|
|
|
|
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
transformed = []
|
|
for item in raw_data:
|
|
record = self._normalize_record(item)
|
|
if record:
|
|
transformed.append(record)
|
|
return transformed
|
|
|
|
async def _save_data(
|
|
self,
|
|
db: AsyncSession,
|
|
data: list[dict[str, Any]],
|
|
task_id: int | None = None,
|
|
snapshot_id: int | None = None,
|
|
) -> int:
|
|
now = datetime.now(UTC)
|
|
records_added = 0
|
|
|
|
for index, item in enumerate(data):
|
|
observed_at = item.get("received_at") or now
|
|
await record_vessel_ais_observation(
|
|
db,
|
|
source=self.name,
|
|
normalized_payload=item,
|
|
raw_payload=item,
|
|
delivery_mode=BARENTSWATCH_DELIVERY_MODE,
|
|
transport=BARENTSWATCH_TRANSPORT,
|
|
observed_at=observed_at,
|
|
collected_at=now,
|
|
)
|
|
records_added += 1
|
|
|
|
if (index + 1) % 1000 == 0:
|
|
await self.update_progress(index + 1, commit=True)
|
|
|
|
latest_observed_at = max(
|
|
(item.get("received_at") for item in data if item.get("received_at")),
|
|
default=now,
|
|
)
|
|
await update_ais_source_health(
|
|
db,
|
|
source=self.name,
|
|
connection_state="connected",
|
|
observed_count=len(data),
|
|
last_seen_at=latest_observed_at,
|
|
last_success_at=now if data else None,
|
|
lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
|
|
)
|
|
if snapshot_id is not None:
|
|
from app.models.data_snapshot import DataSnapshot
|
|
|
|
snapshot = await db.get(DataSnapshot, snapshot_id)
|
|
if snapshot:
|
|
snapshot.record_count = records_added
|
|
snapshot.status = SnapshotStatus.SUCCESS.value
|
|
snapshot.completed_at = now
|
|
snapshot.summary = {
|
|
"created": records_added,
|
|
"updated": 0,
|
|
"unchanged": 0,
|
|
"deleted": 0,
|
|
"storage": "ais_raw_observations",
|
|
}
|
|
await db.commit()
|
|
await self._broadcast_vessel_snapshot(data)
|
|
await self.update_progress(records_added, force=True)
|
|
return records_added
|
|
|
|
async def _broadcast_vessel_snapshot(self, data: list[dict[str, Any]]) -> None:
|
|
"""Push REST collector updates through the same realtime vessel channel."""
|
|
if not data:
|
|
return
|
|
|
|
batch_size = 500
|
|
for offset in range(0, len(data), batch_size):
|
|
batch = data[offset : offset + batch_size]
|
|
await broadcaster.broadcast_custom(
|
|
"vessels",
|
|
{
|
|
"action": "upsert",
|
|
"source": self.name,
|
|
"created": True,
|
|
"vessels": [
|
|
{
|
|
"mmsi": item.get("mmsi"),
|
|
"mmsi_display": str(item.get("mmsi")) if item.get("mmsi") is not None else None,
|
|
"name": item.get("name"),
|
|
"callsign": item.get("callsign"),
|
|
"lat": item.get("lat"),
|
|
"lon": item.get("lon"),
|
|
"sog": item.get("sog"),
|
|
"cog": item.get("cog"),
|
|
"heading": item.get("heading"),
|
|
"nav_status": item.get("nav_status"),
|
|
"vessel_type": item.get("vessel_type"),
|
|
"vessel_type_name": item.get("vessel_type_name"),
|
|
"received_at": to_iso8601_utc(item.get("received_at")),
|
|
}
|
|
for item in batch
|
|
],
|
|
},
|
|
)
|
|
|
|
def _normalize_record(self, item: dict[str, Any]) -> dict[str, Any] | None:
|
|
mmsi = _as_int(_pick(item, "mmsi", "MMSI", "Mmsi"))
|
|
lat = _as_float(_pick(item, "lat", "latitude", "Latitude"))
|
|
lon = _as_float(_pick(item, "lon", "lng", "longitude", "Longitude"))
|
|
|
|
geometry = item.get("geometry")
|
|
coordinates = geometry.get("coordinates") if isinstance(geometry, dict) else None
|
|
if (lat is None or lon is None) and isinstance(coordinates, list) and len(coordinates) >= 2:
|
|
lon = _as_float(coordinates[0])
|
|
lat = _as_float(coordinates[1])
|
|
|
|
if mmsi is None or lat is None or lon is None:
|
|
return None
|
|
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
|
|
return None
|
|
|
|
vessel_type = _as_int(_pick(item, "vessel_type", "shipType", "ship_type", "ShipType"))
|
|
vessel_type_name = (
|
|
_pick(item, "vessel_type_name", "shipTypeName", "ship_type_name", "VesselTypeName")
|
|
or normalize_vessel_type_name(vessel_type)
|
|
)
|
|
received_at = _parse_datetime(_pick(item, "received_at", "timestamp", "time", "msgtime"))
|
|
|
|
return {
|
|
"mmsi": mmsi,
|
|
"name": _pick(item, "name", "shipName", "ship_name", "Name"),
|
|
"callsign": _pick(item, "callsign", "callSign", "CallSign"),
|
|
"vessel_type": vessel_type,
|
|
"vessel_type_name": vessel_type_name,
|
|
"flag": _pick(item, "flag", "country", "Flag"),
|
|
"length": _as_float(_pick(item, "length", "shipLength", "Length")),
|
|
"width": _as_float(_pick(item, "width", "shipWidth", "Width")),
|
|
"draught": _as_float(_pick(item, "draught", "draft", "Draught")),
|
|
"imo": _as_int(_pick(item, "imo", "IMO", "imoNumber")),
|
|
"lat": lat,
|
|
"lon": lon,
|
|
"sog": _as_float(_pick(item, "sog", "speedOverGround", "SOG")),
|
|
"cog": _as_float(_pick(item, "cog", "courseOverGround", "COG")),
|
|
"heading": _as_int(_pick(item, "heading", "trueHeading", "Heading")),
|
|
"nav_status": _as_int(_pick(item, "nav_status", "navStatus", "NavigationalStatus")),
|
|
"received_at": received_at,
|
|
}
|
|
|
|
def _get_sample_data(self) -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"mmsi": 257123000,
|
|
"name": "OSLO TRADER",
|
|
"lat": 59.91,
|
|
"lon": 10.73,
|
|
"sog": 12.4,
|
|
"cog": 214,
|
|
"heading": 215,
|
|
"nav_status": 0,
|
|
"vessel_type": 70,
|
|
"vessel_type_name": "Cargo",
|
|
"flag": "NO",
|
|
"length": 185,
|
|
},
|
|
{
|
|
"mmsi": 257456000,
|
|
"name": "NORDIC FJORD",
|
|
"lat": 60.39,
|
|
"lon": 5.32,
|
|
"sog": 0.2,
|
|
"cog": 82,
|
|
"heading": 80,
|
|
"nav_status": 1,
|
|
"vessel_type": 60,
|
|
"vessel_type_name": "Passenger",
|
|
"flag": "NO",
|
|
"length": 126,
|
|
},
|
|
]
|
|
|
|
|
|
def _pick(item: dict[str, Any], *keys: str) -> Any:
|
|
for key in keys:
|
|
if key in item and item[key] not in (None, ""):
|
|
return item[key]
|
|
return None
|
|
|
|
|
|
def _as_float(value: Any) -> float | None:
|
|
try:
|
|
if value in (None, ""):
|
|
return None
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _as_int(value: Any) -> int | None:
|
|
try:
|
|
if value in (None, ""):
|
|
return None
|
|
return int(float(value))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _parse_datetime(value: Any) -> datetime | None:
|
|
if isinstance(value, datetime):
|
|
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
|
if not value:
|
|
return None
|
|
if isinstance(value, (int, float)):
|
|
timestamp = float(value)
|
|
if timestamp > 10_000_000_000:
|
|
timestamp /= 1000
|
|
return datetime.fromtimestamp(timestamp, UTC)
|
|
if isinstance(value, str):
|
|
try:
|
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
|
except ValueError:
|
|
return None
|
|
return None
|