linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.47.0

Browse Source
This commit is contained in:
linkong
2026-04-30 16:56:37 +08:00
parent f22079d33a
commit 421234301a
42 changed files with 2501 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
backend/app/services/collectors/__init__.py
View File

@@ -36,6 +36,7 @@ from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
from app.services.collectors.opengeofeed import OpenGeoFeedPrefixGeoCollector
from app.services.collectors.nro_delegated import NRODelegatedPrefixGeoCollector
from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
from app.services.collectors.aisstream import AISStreamCollector
from app.services.collectors.vessel_ais import VesselAISCollector
collector_registry.register(TOP500Collector())
@@ -65,3 +66,40 @@ collector_registry.register(OpenGeoFeedPrefixGeoCollector())
collector_registry.register(NRODelegatedPrefixGeoCollector())
collector_registry.register(NewsLiveStreamsCollector())
collector_registry.register(VesselAISCollector())
collector_registry.register(AISStreamCollector())
__all__ = [
"BaseCollector",
"HTTPCollector",
"IntervalCollector",
"collector_registry",
"CollectorRegistry",
"TOP500Collector",
"EpochAIGPUCollector",
"HuggingFaceModelCollector",
"HuggingFaceDatasetCollector",
"HuggingFaceSpacesCollector",
"PeeringDBIXPCollector",
"PeeringDBNetworkCollector",
"PeeringDBFacilityCollector",
"TeleGeographyCableCollector",
"TeleGeographyLandingPointCollector",
"TeleGeographyCableSystemCollector",
"CloudflareRadarDeviceCollector",
"CloudflareRadarTrafficCollector",
"CloudflareRadarTopASCollector",
"ArcGISCableCollector",
"FAOLandingPointCollector",
"ArcGISLandingPointCollector",
"ArcGISCableLandingRelationCollector",
"SpaceTrackTLECollector",
"CelesTrakTLECollector",
"RISLiveCollector",
"BGPStreamBackfillCollector",
"IPtoASNPrefixGeoCollector",
"OpenGeoFeedPrefixGeoCollector",
"NRODelegatedPrefixGeoCollector",
"NewsLiveStreamsCollector",
"VesselAISCollector",
"AISStreamCollector",
]

276
backend/app/services/collectors/aisstream.py Normal file
View File

@@ -0,0 +1,276 @@
"""AISStream WebSocket collector for realtime vessel AIS observations."""
from datetime import UTC, datetime
import asyncio
import json
import os
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.data_sources import get_data_sources_config
from app.models.datasource_config import DataSourceConfig
from app.services.collectors.base import BaseCollector
from app.services.vessel_ais_aggregation import (
AISSTREAM_DELIVERY_MODE,
AISSTREAM_TRANSPORT,
record_vessel_ais_observation,
update_ais_source_health,
)
from app.services.vessel_types import normalize_vessel_type_name
DEFAULT_AISSTREAM_URL = "wss://stream.aisstream.io/v0/stream"
DEFAULT_BOUNDING_BOXES = [[[-90, -180], [90, 180]]]
DEFAULT_MESSAGE_TYPES = ["PositionReport", "ShipStaticData"]
class AISStreamCollector(BaseCollector):
"""Collect AISStream WebSocket messages into the raw AIS observation layer."""
name = "aisstream_vessels"
priority = "P1"
module = "L4"
frequency_hours = 1
data_type = "vessel_ais"
fail_on_empty = False
async def _load_datasource_config(self) -> DataSourceConfig | None:
if self._db_session is None:
return None
result = await self._db_session.execute(
select(DataSourceConfig)
.where(DataSourceConfig.name == self.name)
.where(DataSourceConfig.is_active.is_(True))
)
return result.scalar_one_or_none()
async def _get_effective_config(self) -> dict[str, Any]:
datasource_config = await self._load_datasource_config()
config = dict(datasource_config.config or {}) if datasource_config else {}
auth_config = dict(datasource_config.auth_config or {}) if datasource_config else {}
endpoint = (
(datasource_config.endpoint if datasource_config else None)
or self._resolved_url
or get_data_sources_config().get_yaml_url(self.name)
or DEFAULT_AISSTREAM_URL
)
api_key = (
auth_config.get("api_key")
or config.get("api_key")
or os.getenv("AISSTREAM_API_KEY")
)
return {
"endpoint": endpoint,
"api_key": api_key,
"bounding_boxes": config.get("bounding_boxes") or DEFAULT_BOUNDING_BOXES,
"message_types": config.get("message_types") or DEFAULT_MESSAGE_TYPES,
"max_messages": int(config.get("max_messages") or 500),
"receive_timeout_seconds": float(config.get("receive_timeout_seconds") or 30),
}
async def fetch(self) -> list[dict[str, Any]]:
config = await self._get_effective_config()
if not config["api_key"]:
raise RuntimeError("AISStream API key is not configured")
try:
import websockets
except ImportError as exc:
raise RuntimeError("Python package 'websockets' is required for AISStream") from exc
subscription = {
"APIKey": config["api_key"],
"BoundingBoxes": config["bounding_boxes"],
"FilterMessageTypes": config["message_types"],
}
messages: list[dict[str, Any]] = []
try:
async with websockets.connect(config["endpoint"]) as websocket:
await websocket.send(json.dumps(subscription))
while len(messages) < config["max_messages"]:
try:
raw_message = await asyncio.wait_for(
websocket.recv(),
timeout=config["receive_timeout_seconds"],
)
except TimeoutError:
break
payload = json.loads(raw_message)
if isinstance(payload, dict):
messages.append(payload)
except Exception as exc:
if self._db_session is not None:
await update_ais_source_health(
self._db_session,
source=self.name,
connection_state="disconnected",
last_error=f"{exc.__class__.__name__}: {exc}",
)
await self._db_session.commit()
raise
return messages
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
records = []
for item in raw_data:
record = self._normalize_message(item)
if record:
records.append(record)
return records
async def _save_data(
self,
db: AsyncSession,
data: list[dict[str, Any]],
task_id: int | None = None,
snapshot_id: int | None = None,
) -> int:
now = datetime.now(UTC)
records_added = 0
latest_observed_at = now
for index, item in enumerate(data):
observed_at = item.get("received_at") or now
observation = await record_vessel_ais_observation(
db,
source=self.name,
normalized_payload=item,
raw_payload=item.get("_raw_payload") or item,
delivery_mode=AISSTREAM_DELIVERY_MODE,
transport=AISSTREAM_TRANSPORT,
message_type=item.get("_message_type") or "PositionReport",
source_message_id=item.get("_source_message_id"),
observed_at=observed_at,
collected_at=now,
)
if observation is not None:
records_added += 1
if isinstance(observed_at, datetime) and observed_at > latest_observed_at:
latest_observed_at = observed_at
if (index + 1) % 1000 == 0:
await self.update_progress(index + 1, commit=True)
await update_ais_source_health(
db,
source=self.name,
connection_state="connected",
observed_count=len(data),
last_seen_at=latest_observed_at,
last_success_at=now if data else None,
lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
)
await db.commit()
await self.update_progress(records_added, force=True)
return records_added
def _normalize_message(self, item: dict[str, Any]) -> dict[str, Any] | None:
message_type = str(item.get("MessageType") or item.get("message_type") or "")
metadata = item.get("MetaData") if isinstance(item.get("MetaData"), dict) else {}
message = item.get("Message") if isinstance(item.get("Message"), dict) else {}
body = message.get(message_type) if isinstance(message.get(message_type), dict) else message
if not isinstance(body, dict):
body = {}
mmsi = _as_int(_pick(metadata, "MMSI", "mmsi") or _pick(body, "MMSI", "mmsi"))
if mmsi is None:
return None
received_at = _parse_datetime(
_pick(metadata, "time_utc", "Time_UTC", "timestamp")
or _pick(body, "Timestamp", "timestamp", "time")
)
ship_name = _clean_text(
_pick(body, "Name", "ShipName", "name")
or _pick(metadata, "ShipName", "ship_name", "name")
)
record: dict[str, Any] = {
"mmsi": mmsi,
"received_at": received_at,
"_message_type": message_type or None,
"_source_message_id": item.get("MessageID") or item.get("message_id"),
"_raw_payload": item,
}
lat = _as_float(_pick(body, "Latitude", "lat", "latitude"))
lon = _as_float(_pick(body, "Longitude", "lon", "lng", "longitude"))
if lat is not None and lon is not None:
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
return None
record.update(
{
"lat": lat,
"lon": lon,
"sog": _as_float(_pick(body, "Sog", "SOG", "speedOverGround")),
"cog": _as_float(_pick(body, "Cog", "COG", "courseOverGround")),
"heading": _as_int(_pick(body, "TrueHeading", "Heading", "heading")),
"nav_status": _as_int(_pick(body, "NavigationalStatus", "nav_status")),
}
)
vessel_type = _as_int(_pick(body, "Type", "ShipType", "vessel_type"))
record.update(
{
"name": ship_name,
"callsign": _pick(body, "CallSign", "callsign"),
"imo": _as_int(_pick(body, "ImoNumber", "IMO", "imo")),
"vessel_type": vessel_type,
"vessel_type_name": _pick(body, "TypeName", "ShipTypeName", "vessel_type_name")
or normalize_vessel_type_name(vessel_type),
"length": _as_float(_pick(body, "DimensionToBow", "Length", "length")),
"width": _as_float(_pick(body, "DimensionToPort", "Width", "width")),
}
)
return record
def _pick(item: dict[str, Any], *keys: str) -> Any:
for key in keys:
if key in item and item[key] not in (None, ""):
return item[key]
return None
def _clean_text(value: Any) -> str | None:
if value in (None, ""):
return None
text = str(value).strip()
return text or None
def _as_float(value: Any) -> float | None:
try:
if value in (None, ""):
return None
return float(value)
except (TypeError, ValueError):
return None
def _as_int(value: Any) -> int | None:
try:
if value in (None, ""):
return None
return int(float(value))
except (TypeError, ValueError):
return None
def _parse_datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if not value:
return None
if isinstance(value, (int, float)):
timestamp = float(value)
if timestamp > 10_000_000_000:
timestamp /= 1000
return datetime.fromtimestamp(timestamp, UTC)
if isinstance(value, str):
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
except ValueError:
return None
return None

63
backend/app/services/collectors/vessel_ais.py
View File

@@ -4,7 +4,7 @@ from datetime import UTC, datetime, timedelta
from typing import Any
import httpx
from sqlalchemy import delete, select
from sqlalchemy import delete
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.vessel import VesselPosition, VesselStatic
@@ -14,15 +14,13 @@ from app.services.barentswatch import (
resolve_barentswatch_config,
)
from app.services.collectors.base import BaseCollector
VESSEL_TYPE_NAMES = {
30: "Fishing",
35: "Military",
60: "Passenger",
70: "Cargo",
80: "Tanker",
}
from app.services.vessel_ais_aggregation import (
BARENTSWATCH_DELIVERY_MODE,
BARENTSWATCH_TRANSPORT,
record_vessel_ais_observation,
update_ais_source_health,
)
from app.services.vessel_types import normalize_vessel_type_name
class VesselAISCollector(BaseCollector):
@@ -92,6 +90,18 @@ class VesselAISCollector(BaseCollector):
records_added = 0
for index, item in enumerate(data):
observed_at = item.get("received_at") or now
await record_vessel_ais_observation(
db,
source=self.name,
normalized_payload=item,
raw_payload=item,
delivery_mode=BARENTSWATCH_DELIVERY_MODE,
transport=BARENTSWATCH_TRANSPORT,
observed_at=observed_at,
collected_at=now,
)
static = await db.get(VesselStatic, item["mmsi"])
if static is None:
static = VesselStatic(mmsi=item["mmsi"])
@@ -122,7 +132,7 @@ class VesselAISCollector(BaseCollector):
cog=item.get("cog"),
heading=item.get("heading"),
nav_status=item.get("nav_status"),
received_at=item.get("received_at") or now,
received_at=observed_at,
)
)
records_added += 1
@@ -130,6 +140,19 @@ class VesselAISCollector(BaseCollector):
if (index + 1) % 1000 == 0:
await self.update_progress(index + 1, commit=True)
latest_observed_at = max(
(item.get("received_at") for item in data if item.get("received_at")),
default=now,
)
await update_ais_source_health(
db,
source=self.name,
connection_state="connected",
observed_count=len(data),
last_seen_at=latest_observed_at,
last_success_at=now if data else None,
lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
)
await db.execute(
delete(VesselPosition).where(VesselPosition.received_at < now - timedelta(hours=24))
)
@@ -156,7 +179,7 @@ class VesselAISCollector(BaseCollector):
vessel_type = _as_int(_pick(item, "vessel_type", "shipType", "ship_type", "ShipType"))
vessel_type_name = (
_pick(item, "vessel_type_name", "shipTypeName", "ship_type_name", "VesselTypeName")
or _vessel_type_name(vessel_type)
or normalize_vessel_type_name(vessel_type)
)
received_at = _parse_datetime(_pick(item, "received_at", "timestamp", "time", "msgtime"))
@@ -255,19 +278,3 @@ def _parse_datetime(value: Any) -> datetime | None:
except ValueError:
return None
return None
def _vessel_type_name(vessel_type: int | None) -> str:
if vessel_type is None:
return "Other"
if 70 <= vessel_type <= 79:
return "Cargo"
if 80 <= vessel_type <= 89:
return "Tanker"
if 60 <= vessel_type <= 69:
return "Passenger"
if vessel_type == 30:
return "Fishing"
if vessel_type == 35:
return "Military"
return VESSEL_TYPE_NAMES.get(vessel_type, "Other")

59
backend/app/services/credential_guides.py
View File

@@ -66,9 +66,68 @@ BARENTSWATCH_DEFAULT_GUIDE = CredentialGuideDefault(
""",
)
AISSTREAM_DEFAULT_GUIDE = CredentialGuideDefault(
provider="aisstream",
title="AISStream API Key 获取教程",
prompt=(
"请生成一份中文教程，指导开发者获取 AISStream 的 API Key 并配置到 Planet。"
"教程要面向已经有本地开发环境的人，包含注册/登录 AISStream、获取 API Key、"
"理解免费额度和订阅范围、在 Planet 设置中心填写 API Key、配置 bounding boxes "
"和 message types、验证连接、常见失败排查。必须提醒用户以 AISStream 当前官网和"
"服务条款为准，不要编造具体页面按钮文案。"
),
markdown="""## AISStream API Key 获取
官方入口：https://aisstream.io/
1. 打开 AISStream 官网，按当前页面指引注册或登录账号。
2. 在账号/API 管理页面创建或复制你的 API Key。
3. 先确认当前账号额度、使用条款和可订阅区域。实时 AIS 流量可能很大，不建议一开始订阅全球范围。
4. 回到 Planet 的 `设置 -> 采集器设置 -> AISStream 实时船舶`。
5. 在 `AISStream 凭证` 中填入 API Key。
6. Endpoint 通常保持默认：`wss://stream.aisstream.io/v0/stream`。
7. 按需配置 `Bounding Boxes JSON` 和 `消息类型`。
8. 点击连接测试，确认系统能读取凭证且 WebSocket endpoint 格式有效。
9. 保存采集器设置后再运行 `aisstream_vessels` collector。
### 推荐配置
默认消息类型：
```json
["PositionReport", "ShipStaticData"]
```
默认 Bounding Boxes 示例：
```json
[[[-90, -180], [90, 180]]]
```
这个示例表示全球范围。实际使用时建议先改成较小区域，降低消息量和处理压力。
### 请求规则
- Endpoint：`wss://stream.aisstream.io/v0/stream`
- 传输方式：WebSocket
- API Key 放在订阅 payload 中，不放在 HTTP header。
- Planet 会把 AISStream 标记为 `delivery_mode = realtime_stream`、`transport = websocket`。
- AISStream collector 只写入 AIS raw observations，不直接覆盖最终船只展示表。
### 常见排查
- `未找到凭证`：确认 API Key 已保存到采集器设置，或设置了 `AISSTREAM_API_KEY` 环境变量 / `~/.zshrc`。
- `endpoint 必须是 ws:// 或 wss://`：AISStream 是 WebSocket 流接口，不要填普通 `https://` API 地址。
- 采集量过大：缩小 `Bounding Boxes JSON`，减少 `message_types`，或降低单次最大消息数。
- 没有船只数据：确认订阅区域内确实有 AIS 活动，并检查 API Key 当前额度和权限。
- 连接中断：实时流可能受网络和上游限流影响，collector 会记录源健康状态供聚合服务回退。
""",
)
DEFAULT_CREDENTIAL_GUIDES = {
BARENTSWATCH_DEFAULT_GUIDE.provider: BARENTSWATCH_DEFAULT_GUIDE,
AISSTREAM_DEFAULT_GUIDE.provider: AISSTREAM_DEFAULT_GUIDE,
}

57
backend/app/services/datasource_connectivity.py
View File

@@ -26,6 +26,7 @@ from app.services.barentswatch import (
CONNECTIVITY_VALIDATION_KEY = "connectivity_validation"
CONNECTIVITY_STORE_CATEGORY = "datasource_connectivity_validations"
SUPPORTED_CREDENTIAL_PROVIDERS = {"barentswatch", "spacetrack", "aisstream"}
def _sha256_json(payload: Any) -> str:
@@ -43,6 +44,36 @@ def _resolve_spacetrack_credentials() -> tuple[str, str, str]:
return username, password, source or "missing"
async def _resolve_aisstream_api_key(
db=None,
credential_override: dict[str, str] | None = None,
) -> tuple[str, str]:
if credential_override and credential_override.get("api_key"):
return str(credential_override["api_key"]), "draft"
env_key = os.getenv("AISSTREAM_API_KEY")
zshrc_key = _read_zshrc_env().get("AISSTREAM_API_KEY")
if db is not None:
result = await db.execute(
select(DataSourceConfig)
.where(DataSourceConfig.name == "aisstream_vessels")
.where(DataSourceConfig.is_active.is_(True))
)
record = result.scalar_one_or_none()
if record:
auth_config = record.auth_config or {}
runtime_config = record.config or {}
api_key = auth_config.get("api_key") or runtime_config.get("api_key")
if api_key:
return str(api_key), "datasource_config"
if env_key:
return env_key, "environment"
if zshrc_key:
return zshrc_key, "~/.zshrc"
return "", "missing"
def strip_connectivity_validation(config: dict | None) -> dict:
cleaned = dict(config or {})
cleaned.pop(CONNECTIVITY_VALIDATION_KEY, None)
@@ -103,6 +134,10 @@ async def build_builtin_connectivity_checksum(
"password": password,
}
)
elif credential_provider == "aisstream":
api_key, credential_source = await _resolve_aisstream_api_key(db, credential_override)
has_credentials = bool(api_key)
credential_fingerprint = _sha256_json({"api_key": api_key})
elif defaults.get("requires_credentials"):
credential_source = str(credential_provider or "unsupported")
@@ -130,6 +165,7 @@ async def test_builtin_connectivity(
headers: dict | None,
config: dict | None,
db=None,
credential_override: dict[str, str] | None = None,
) -> dict[str, Any]:
defaults = DEFAULT_DATASOURCES.get(source)
if not defaults:
@@ -145,6 +181,7 @@ async def test_builtin_connectivity(
headers,
config,
db,
credential_override,
)
if credential_context["requires_credentials"] and not credential_context["has_credentials"]:
return {
@@ -155,10 +192,9 @@ async def test_builtin_connectivity(
"settings_tab": "collector_credentials",
**credential_context,
}
supported_credential_providers = {"barentswatch", "spacetrack"}
if (
credential_context["requires_credentials"]
and credential_context["credential_provider"] not in supported_credential_providers
and credential_context["credential_provider"] not in SUPPORTED_CREDENTIAL_PROVIDERS
):
return {
"success": False,
@@ -174,6 +210,23 @@ async def test_builtin_connectivity(
timeout = float(request_config.get("timeout") or 30)
request_endpoint = endpoint
if credential_context["credential_provider"] == "aisstream":
if not str(request_endpoint).startswith(("ws://", "wss://")):
return {
"success": False,
"checksum": checksum,
"stage": "endpoint",
"message": "AISStream endpoint 必须是 ws:// 或 wss:// WebSocket 地址。",
**credential_context,
}
return {
"success": True,
"checksum": checksum,
"stage": "credentials",
"message": "AISStream 凭证已配置，WebSocket endpoint 格式有效。",
**credential_context,
}
try:
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
if credential_context["credential_provider"] == "barentswatch":

574
backend/app/services/vessel_ais_aggregation.py Normal file
View File

@@ -0,0 +1,574 @@
"""AIS raw observation and aggregation support for vessel collectors."""
from datetime import UTC, datetime
from hashlib import sha256
import json
from typing import Any, Iterable
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth
from app.services.vessel_types import normalize_vessel_type_name
VESSEL_AIS_SCHEMA = "vessel_ais"
BARENTSWATCH_DELIVERY_MODE = "polling"
BARENTSWATCH_TRANSPORT = "http"
AISSTREAM_DELIVERY_MODE = "realtime_stream"
AISSTREAM_TRANSPORT = "websocket"
DELIVERY_MODE_PRIORITY = {
"realtime_stream": 40,
"batch_stream": 30,
"polling": 20,
"snapshot": 10,
}
DYNAMIC_FIELDS = ("lat", "lon", "sog", "cog", "heading", "nav_status")
CONFLICT_FIELDS = (
"name",
"callsign",
"imo",
"flag",
"vessel_type",
"vessel_type_name",
"length",
"width",
"draught",
)
def _json_default(value: Any) -> Any:
if isinstance(value, datetime):
return value.astimezone(UTC).isoformat()
return str(value)
def _stable_payload(value: Any) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=_json_default)
def _jsonable(value: Any) -> Any:
if isinstance(value, datetime):
return value.astimezone(UTC).isoformat()
if isinstance(value, dict):
return {str(key): _jsonable(item) for key, item in value.items()}
if isinstance(value, list):
return [_jsonable(item) for item in value]
return value
def _coerce_datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, (int, float)):
timestamp = float(value)
if timestamp > 10_000_000_000:
timestamp /= 1000
return datetime.fromtimestamp(timestamp, UTC)
if isinstance(value, str) and value:
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
except ValueError:
return None
return None
def build_observation_hash(
*,
source: str,
entity_key: str,
message_type: str | None,
observed_at: datetime,
normalized_payload: dict[str, Any],
source_message_id: str | None = None,
) -> str:
"""Build a deterministic idempotency key for one source-level AIS observation."""
if source_message_id:
basis = {
"source": source,
"entity_key": entity_key,
"source_message_id": source_message_id,
}
else:
basis = {
"source": source,
"entity_key": entity_key,
"message_type": message_type,
"observed_at": observed_at.astimezone(UTC).isoformat(),
"payload": normalized_payload,
}
return sha256(_stable_payload(basis).encode("utf-8")).hexdigest()
def build_field_conflict_candidates(
observations: Iterable[AISRawObservation],
fields: Iterable[str] = CONFLICT_FIELDS,
) -> list[dict[str, Any]]:
"""Return current field disagreements from raw observations without mutating state."""
candidates_by_field: dict[str, dict[str, Any]] = {}
for observation in observations:
payload = observation.normalized_payload or {}
for field in fields:
value = payload.get(field)
if value in (None, ""):
continue
field_candidates = candidates_by_field.setdefault(field, {})
field_candidates[observation.source] = value
conflicts = []
for field, candidates in sorted(candidates_by_field.items()):
unique_values = {_stable_payload(value) for value in candidates.values()}
if len(unique_values) <= 1:
continue
conflicts.append(
{
"field": field,
"candidates": candidates,
"status": "candidate",
}
)
return conflicts
def _payload_value(payload: dict[str, Any], field: str) -> Any:
value = payload.get(field)
return None if value in (None, "") else value
def _clean_text(value: Any) -> str | None:
if value in (None, ""):
return None
text = str(value).strip()
return text or None
def _raw_metadata_value(observation: AISRawObservation, field: str) -> Any:
raw_payload = observation.raw_payload or {}
metadata = raw_payload.get("MetaData") if isinstance(raw_payload, dict) else None
if not isinstance(metadata, dict):
return None
if field == "name":
return _clean_text(metadata.get("ShipName") or metadata.get("ship_name") or metadata.get("name"))
return None
def _delivery_priority(observation: AISRawObservation) -> int:
return DELIVERY_MODE_PRIORITY.get(str(observation.delivery_mode or ""), 0)
def _has_valid_position(payload: dict[str, Any]) -> bool:
try:
lat = float(payload.get("lat"))
lon = float(payload.get("lon"))
except (TypeError, ValueError):
return False
return -90 <= lat <= 90 and -180 <= lon <= 180
def _is_future_observation(observation: AISRawObservation, now: datetime) -> bool:
return observation.observed_at > now
def _select_position_observation(
observations: list[AISRawObservation],
*,
now: datetime,
) -> tuple[AISRawObservation | None, list[str]]:
rejected_flags: list[str] = []
candidates = []
for observation in observations:
payload = observation.normalized_payload or {}
if not _has_valid_position(payload):
rejected_flags.append("invalid_position")
continue
if _is_future_observation(observation, now):
rejected_flags.append("future_timestamp")
continue
candidates.append(observation)
if not candidates:
return None, sorted(set(rejected_flags))
candidates.sort(
key=lambda item: (
item.observed_at,
_delivery_priority(item),
item.collected_at,
item.id or 0,
),
reverse=True,
)
return candidates[0], sorted(set(rejected_flags))
def _select_static_field(
observations: list[AISRawObservation],
field: str,
) -> tuple[Any, str | None, str | None]:
candidates = []
for observation in observations:
value = _payload_value(observation.normalized_payload or {}, field)
if value is None:
value = _raw_metadata_value(observation, field)
if value is None:
continue
candidates.append((observation, value))
if not candidates:
return None, None, None
candidates.sort(
key=lambda item: (
_delivery_priority(item[0]),
item[0].observed_at,
item[0].collected_at,
item[0].id or 0,
),
reverse=True,
)
selected_observation, selected_value = candidates[0]
unique_values = {_stable_payload(value) for _, value in candidates}
reason = "delivery_mode_priority" if len(unique_values) > 1 else "non_empty_priority"
return selected_value, selected_observation.source, reason
def _build_source_summary(observations: list[AISRawObservation]) -> dict[str, dict[str, Any]]:
summary: dict[str, dict[str, Any]] = {}
for observation in observations:
source_summary = summary.setdefault(
observation.source,
{
"observation_count": 0,
"latest_observed_at": None,
"delivery_mode": observation.delivery_mode,
"transport": observation.transport,
"message_types": [],
},
)
source_summary["observation_count"] += 1
latest_observed_at = source_summary["latest_observed_at"]
if latest_observed_at is None or observation.observed_at > latest_observed_at:
source_summary["latest_observed_at"] = observation.observed_at
if observation.message_type and observation.message_type not in source_summary["message_types"]:
source_summary["message_types"].append(observation.message_type)
return summary
def _build_aggregated_vessel(
entity_key: str,
observations: list[AISRawObservation],
*,
now: datetime,
) -> dict[str, Any] | None:
position_observation, rejected_flags = _select_position_observation(observations, now=now)
if position_observation is None:
return None
payload = position_observation.normalized_payload or {}
mmsi = int(entity_key)
result: dict[str, Any] = {
"mmsi": mmsi,
"lat": float(payload["lat"]),
"lon": float(payload["lon"]),
"received_at": position_observation.observed_at,
"field_sources": {},
"selected_reasons": {},
"source_summary": _build_source_summary(observations),
"quality_flags": sorted(
set((position_observation.quality_flags or []) + rejected_flags)
),
}
for field in DYNAMIC_FIELDS:
value = _payload_value(payload, field)
if field in ("lat", "lon") or value is not None:
result[field] = value
result["field_sources"][field] = position_observation.source
result["selected_reasons"][field] = "newest_observation"
for field in CONFLICT_FIELDS:
selected_value, selected_source, reason = _select_static_field(observations, field)
if selected_value is None:
continue
result[field] = selected_value
result["field_sources"][field] = selected_source
result["selected_reasons"][field] = reason
result["name"] = result.get("name") or f"MMSI {mmsi}"
result["vessel_type_name"] = result.get("vessel_type_name") or normalize_vessel_type_name(
result.get("vessel_type")
)
return result
async def _upsert_conflict_records(
db: AsyncSession,
entity_key: str,
observations: list[AISRawObservation],
aggregated: dict[str, Any],
) -> int:
conflicts = build_field_conflict_candidates(observations)
now = datetime.now(UTC)
for conflict in conflicts:
field = conflict["field"]
result = await db.execute(
select(AISConflictRecord)
.where(AISConflictRecord.target_schema == VESSEL_AIS_SCHEMA)
.where(AISConflictRecord.entity_key == entity_key)
.where(AISConflictRecord.field == field)
.limit(1)
)
record = result.scalar_one_or_none()
if record is None:
record = AISConflictRecord(
target_schema=VESSEL_AIS_SCHEMA,
entity_key=entity_key,
field=field,
)
db.add(record)
record.candidates = conflict["candidates"]
record.selected_source = (aggregated.get("field_sources") or {}).get(field)
record.selected_value = aggregated.get(field)
record.selected_reason = (aggregated.get("selected_reasons") or {}).get(field)
record.resolved_by = "system"
record.status = "open"
record.updated_at = now
return len(conflicts)
def _group_observations(observations: Iterable[AISRawObservation]) -> dict[str, list[AISRawObservation]]:
grouped: dict[str, list[AISRawObservation]] = {}
for observation in observations:
grouped.setdefault(str(observation.entity_key), []).append(observation)
return grouped
async def record_vessel_ais_observation(
db: AsyncSession,
*,
source: str,
normalized_payload: dict[str, Any],
raw_payload: dict[str, Any] | None = None,
delivery_mode: str,
transport: str,
message_type: str | None = "PositionReport",
source_message_id: str | None = None,
observed_at: datetime | None = None,
collected_at: datetime | None = None,
quality_flags: list[str] | None = None,
) -> AISRawObservation | None:
"""Insert one raw observation if the source-level fact has not already been stored."""
entity_key = str(normalized_payload["mmsi"])
collected_at = collected_at or datetime.now(UTC)
observed_at = (
_coerce_datetime(observed_at)
or _coerce_datetime(normalized_payload.get("received_at"))
or collected_at
)
normalized_json = _jsonable(normalized_payload)
raw_json = _jsonable(raw_payload or {})
observation_hash = build_observation_hash(
source=source,
entity_key=entity_key,
message_type=message_type,
observed_at=observed_at,
normalized_payload=normalized_json,
source_message_id=source_message_id,
)
existing_result = await db.execute(
select(AISRawObservation.id).where(AISRawObservation.observation_hash == observation_hash)
)
if existing_result.scalar_one_or_none() is not None:
return None
observation = AISRawObservation(
target_schema=VESSEL_AIS_SCHEMA,
source=source,
entity_key=entity_key,
delivery_mode=delivery_mode,
transport=transport,
message_type=message_type,
source_message_id=source_message_id,
observation_hash=observation_hash,
observed_at=observed_at,
collected_at=collected_at,
normalized_payload=normalized_json,
raw_payload=raw_json,
quality_flags=quality_flags or [],
)
db.add(observation)
return observation
async def aggregate_vessel_observations(
db: AsyncSession,
observations: Iterable[AISRawObservation],
*,
write_conflicts: bool = True,
) -> list[dict[str, Any]]:
now = datetime.now(UTC)
vessels = []
for entity_key, entity_observations in _group_observations(observations).items():
aggregated = _build_aggregated_vessel(entity_key, entity_observations, now=now)
if aggregated is None:
continue
if write_conflicts:
aggregated["conflict_count"] = await _upsert_conflict_records(
db,
entity_key,
entity_observations,
aggregated,
)
else:
aggregated["conflict_count"] = len(build_field_conflict_candidates(entity_observations))
vessels.append(aggregated)
vessels.sort(key=lambda item: item.get("received_at") or datetime.min.replace(tzinfo=UTC), reverse=True)
return vessels
async def get_aggregated_vessels(
db: AsyncSession,
*,
bbox: tuple[float, float, float, float] | None = None,
limit: int | None = None,
) -> list[dict[str, Any]]:
stmt = (
select(AISRawObservation)
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
.order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc())
)
if limit and limit > 0:
stmt = stmt.limit(max(limit * 20, limit))
result = await db.execute(stmt)
if not hasattr(result, "scalars"):
return []
vessels = await aggregate_vessel_observations(db, result.scalars().all())
if bbox is not None:
lon_min, lat_min, lon_max, lat_max = bbox
vessels = [
vessel
for vessel in vessels
if lon_min <= float(vessel["lon"]) <= lon_max
and lat_min <= float(vessel["lat"]) <= lat_max
]
if limit and limit > 0:
return vessels[:limit]
return vessels
async def get_aggregated_vessel(db: AsyncSession, mmsi: int) -> dict[str, Any] | None:
observations = await get_vessel_raw_observations(db, mmsi, limit=1000)
vessels = await aggregate_vessel_observations(db, observations)
return vessels[0] if vessels else None
async def get_aggregated_vessel_track(
db: AsyncSession,
mmsi: int,
*,
cutoff: datetime,
) -> list[dict[str, Any]]:
result = await db.execute(
select(AISRawObservation)
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
.where(AISRawObservation.entity_key == str(mmsi))
.where(AISRawObservation.observed_at >= cutoff)
.order_by(AISRawObservation.observed_at.asc(), AISRawObservation.id.asc())
)
if not hasattr(result, "scalars"):
return []
points: list[dict[str, Any]] = []
seen: set[tuple[str, float, float, str]] = set()
for observation in result.scalars().all():
payload = observation.normalized_payload or {}
if not _has_valid_position(payload):
continue
lat = float(payload["lat"])
lon = float(payload["lon"])
key = (
observation.observed_at.isoformat(),
round(lat, 5),
round(lon, 5),
observation.source,
)
if key in seen:
continue
seen.add(key)
points.append(
{
"lat": lat,
"lon": lon,
"observed_at": observation.observed_at,
"source": observation.source,
"selected_reason": "track_timeline",
"quality_flags": observation.quality_flags or [],
}
)
return points
async def update_ais_source_health(
db: AsyncSession,
*,
source: str,
connection_state: str,
observed_count: int = 0,
last_seen_at: datetime | None = None,
last_success_at: datetime | None = None,
last_error: str | None = None,
lag_seconds: float | None = None,
) -> AISSourceHealth:
"""Upsert the health row for an AIS source."""
now = datetime.now(UTC)
health = await db.get(AISSourceHealth, source)
if health is None:
health = AISSourceHealth(source=source)
db.add(health)
health.connection_state = connection_state
health.last_seen_at = last_seen_at or health.last_seen_at
health.last_success_at = last_success_at or health.last_success_at
health.last_error = last_error
health.message_rate = float(observed_count)
health.lag_seconds = lag_seconds
health.updated_at = now
return health
async def get_vessel_raw_observations(
db: AsyncSession,
mmsi: int,
*,
limit: int = 100,
) -> list[AISRawObservation]:
result = await db.execute(
select(AISRawObservation)
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
.where(AISRawObservation.entity_key == str(mmsi))
.order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc())
.limit(limit)
)
return list(result.scalars().all())
async def get_vessel_conflict_records(
db: AsyncSession,
mmsi: int,
) -> list[AISConflictRecord]:
result = await db.execute(
select(AISConflictRecord)
.where(AISConflictRecord.target_schema == VESSEL_AIS_SCHEMA)
.where(AISConflictRecord.entity_key == str(mmsi))
.order_by(AISConflictRecord.updated_at.desc(), AISConflictRecord.id.desc())
)
return list(result.scalars().all())

31
backend/app/services/vessel_types.py Normal file
View File

@@ -0,0 +1,31 @@
"""Shared AIS vessel type helpers."""
from typing import Any
VESSEL_TYPE_NAMES = {
30: "Fishing",
35: "Military",
60: "Passenger",
70: "Cargo",
80: "Tanker",
}
def normalize_vessel_type_name(vessel_type: Any) -> str:
"""Map AIS numeric vessel type codes to display buckets."""
try:
type_code = int(float(vessel_type))
except (TypeError, ValueError):
return "Other"
if 70 <= type_code <= 79:
return "Cargo"
if 80 <= type_code <= 89:
return "Tanker"
if 60 <= type_code <= 69:
return "Passenger"
if type_code == 30:
return "Fishing"
if type_code == 35:
return "Military"
return VESSEL_TYPE_NAMES.get(type_code, "Other")