linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

release: bump version to 0.48.0

Browse Source
This commit is contained in:
linkong
2026-05-07 18:06:06 +08:00
parent 421234301a
commit bb9183b8a4
51 changed files with 4609 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
backend/app/services/vessel_ais_aggregation.py
View File

@@ -1,6 +1,6 @@
"""AIS raw observation and aggregation support for vessel collectors."""
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from hashlib import sha256
import json
from typing import Any, Iterable
@@ -9,9 +9,14 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth
from app.services.vessel_aggregation_strategy import (
DEFAULT_STRATEGY,
load_strategy,
)
from app.services.vessel_types import normalize_vessel_type_name
VESSEL_AIS_SCHEMA = "vessel_ais"
DEFAULT_AGGREGATION_WINDOW_HOURS = 24
BARENTSWATCH_DELIVERY_MODE = "polling"
BARENTSWATCH_TRANSPORT = "http"
AISSTREAM_DELIVERY_MODE = "realtime_stream"
@@ -171,13 +176,43 @@ def _is_future_observation(observation: AISRawObservation, now: datetime) -> boo
return observation.observed_at > now
def _strategy_source_rank(
source: str,
strategy: dict[str, Any],
) -> int:
priority = (strategy.get("vessel_ais") or {}).get("source_priority") or []
if source in priority:
return len(priority) - priority.index(source)
return 0
def _is_stream_stale(
observation: AISRawObservation,
*,
now: datetime,
strategy: dict[str, Any],
) -> bool:
delivery_mode = str(observation.delivery_mode or "")
freshness = (strategy.get("vessel_ais") or {}).get("freshness") or {}
if delivery_mode == "realtime_stream":
window = int(freshness.get("realtime_stream_seconds", 0) or 0)
else:
window = int(freshness.get("polling_seconds", 0) or 0)
if window <= 0:
return False
return (now - observation.observed_at).total_seconds() > window
def _select_position_observation(
observations: list[AISRawObservation],
*,
now: datetime,
strategy: dict[str, Any] | None = None,
) -> tuple[AISRawObservation | None, list[str]]:
strategy = strategy or DEFAULT_STRATEGY
rejected_flags: list[str] = []
candidates = []
fresh_candidates: list[AISRawObservation] = []
stale_candidates: list[AISRawObservation] = []
for observation in observations:
payload = observation.normalized_payload or {}
if not _has_valid_position(payload):
@@ -186,8 +221,13 @@ def _select_position_observation(
if _is_future_observation(observation, now):
rejected_flags.append("future_timestamp")
continue
candidates.append(observation)
if _is_stream_stale(observation, now=now, strategy=strategy):
stale_candidates.append(observation)
rejected_flags.append("freshness_fallback")
continue
fresh_candidates.append(observation)
candidates = fresh_candidates or stale_candidates
if not candidates:
return None, sorted(set(rejected_flags))
@@ -195,6 +235,7 @@ def _select_position_observation(
key=lambda item: (
item.observed_at,
_delivery_priority(item),
_strategy_source_rank(item.source, strategy),
item.collected_at,
item.id or 0,
),
@@ -206,7 +247,9 @@ def _select_position_observation(
def _select_static_field(
observations: list[AISRawObservation],
field: str,
strategy: dict[str, Any] | None = None,
) -> tuple[Any, str | None, str | None]:
strategy = strategy or DEFAULT_STRATEGY
candidates = []
for observation in observations:
value = _payload_value(observation.normalized_payload or {}, field)
@@ -219,6 +262,39 @@ def _select_static_field(
if not candidates:
return None, None, None
field_rules = (strategy.get("vessel_ais") or {}).get("field_rules") or {}
rule = field_rules.get(field) or {"mode": "source_priority"}
mode = rule.get("mode")
if mode == "locked":
locked_source = rule.get("locked_source")
for observation, value in candidates:
if observation.source == locked_source:
return value, observation.source, "locked"
if mode in ("source_priority", "locked"):
priority = rule.get("source_priority") or (strategy.get("vessel_ais") or {}).get("source_priority") or []
ranked = sorted(
candidates,
key=lambda item: (
priority.index(item[0].source) if item[0].source in priority else len(priority) + 1,
-_delivery_priority(item[0]),
-(item[0].observed_at.timestamp() if item[0].observed_at else 0),
),
)
observation, value = ranked[0]
return value, observation.source, "source_priority"
if mode == "newest":
ranked = sorted(
candidates,
key=lambda item: (item[0].observed_at, _delivery_priority(item[0]), item[0].id or 0),
reverse=True,
)
observation, value = ranked[0]
return value, observation.source, "newest_observation"
# default / non_empty: prefer delivery mode priority, then newest
candidates.sort(
key=lambda item: (
_delivery_priority(item[0]),
@@ -261,8 +337,12 @@ def _build_aggregated_vessel(
observations: list[AISRawObservation],
*,
now: datetime,
strategy: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
position_observation, rejected_flags = _select_position_observation(observations, now=now)
strategy = strategy or DEFAULT_STRATEGY
position_observation, rejected_flags = _select_position_observation(
observations, now=now, strategy=strategy
)
if position_observation is None:
return None
@@ -279,6 +359,7 @@ def _build_aggregated_vessel(
"quality_flags": sorted(
set((position_observation.quality_flags or []) + rejected_flags)
),
"aggregation_strategy_version": int(strategy.get("version") or 0),
}
for field in DYNAMIC_FIELDS:
@@ -289,7 +370,9 @@ def _build_aggregated_vessel(
result["selected_reasons"][field] = "newest_observation"
for field in CONFLICT_FIELDS:
selected_value, selected_source, reason = _select_static_field(observations, field)
selected_value, selected_source, reason = _select_static_field(
observations, field, strategy=strategy
)
if selected_value is None:
continue
result[field] = selected_value
@@ -408,12 +491,16 @@ async def aggregate_vessel_observations(
db: AsyncSession,
observations: Iterable[AISRawObservation],
*,
write_conflicts: bool = True,
write_conflicts: bool = False,
strategy: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
strategy = strategy if strategy is not None else await _safe_load_strategy(db)
now = datetime.now(UTC)
vessels = []
for entity_key, entity_observations in _group_observations(observations).items():
aggregated = _build_aggregated_vessel(entity_key, entity_observations, now=now)
aggregated = _build_aggregated_vessel(
entity_key, entity_observations, now=now, strategy=strategy
)
if aggregated is None:
continue
if write_conflicts:
@@ -431,15 +518,28 @@ async def aggregate_vessel_observations(
return vessels
async def _safe_load_strategy(db: AsyncSession) -> dict[str, Any]:
"""Tolerate fake test sessions where load_strategy may misbehave."""
try:
return await load_strategy(db)
except Exception:
return DEFAULT_STRATEGY
async def get_aggregated_vessels(
db: AsyncSession,
*,
bbox: tuple[float, float, float, float] | None = None,
limit: int | None = None,
observed_since: datetime | None = None,
) -> list[dict[str, Any]]:
observed_since = observed_since or (
datetime.now(UTC) - timedelta(hours=DEFAULT_AGGREGATION_WINDOW_HOURS)
)
stmt = (
select(AISRawObservation)
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
.where(AISRawObservation.observed_at >= observed_since)
.order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc())
)
if limit and limit > 0:
@@ -545,6 +645,30 @@ async def update_ais_source_health(
return health
async def count_unique_raw_vessel_mmsi(
db: AsyncSession,
*,
observed_since: datetime | None = None,
) -> int:
"""Count unique raw vessel MMSI values for HUD counts; never aggregates."""
from sqlalchemy import func as sa_func
unique_mmsi_stmt = (
select(AISRawObservation.entity_key)
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
.distinct()
)
if observed_since is not None:
unique_mmsi_stmt = unique_mmsi_stmt.where(
AISRawObservation.observed_at >= observed_since,
)
result = await db.execute(
select(sa_func.count()).select_from(unique_mmsi_stmt.subquery()),
)
return int(result.scalar() or 0)
async def get_vessel_raw_observations(
db: AsyncSession,
mmsi: int,