release: bump version to 0.47.0
This commit is contained in:
linkong
@@ -1,11 +1,13 @@
|
||||
"""Unit tests for data collectors"""
|
||||
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.services.credential_guides import DEFAULT_CREDENTIAL_GUIDES
|
||||
from app.services.collectors.top500 import TOP500Collector
|
||||
from app.services.collectors.base import BaseCollector, HTTPCollector
|
||||
from app.services.collectors.registry import collector_registry
|
||||
from app.services.datasource_connectivity import SUPPORTED_CREDENTIAL_PROVIDERS
|
||||
from app.models.task import CollectionTask
|
||||
|
||||
|
||||
@@ -145,3 +147,30 @@ class TestHTTPCollector:
|
||||
assert hasattr(collector, "parse_response")
|
||||
assert callable(collector.fetch)
|
||||
assert callable(collector.parse_response)
|
||||
|
||||
|
||||
def test_aisstream_collector_is_registered():
|
||||
collector = collector_registry.get("aisstream_vessels")
|
||||
|
||||
assert collector is not None
|
||||
assert collector.data_type == "vessel_ais"
|
||||
|
||||
|
||||
def test_supported_credential_collectors_have_guides_and_connectivity_provider():
|
||||
missing: list[str] = []
|
||||
for source, info in DEFAULT_DATASOURCES.items():
|
||||
if not info.get("requires_credentials"):
|
||||
continue
|
||||
if info.get("credential_status") != "supported":
|
||||
continue
|
||||
|
||||
provider = info.get("credential_provider")
|
||||
if not provider:
|
||||
missing.append(f"{source}: missing credential_provider")
|
||||
continue
|
||||
if provider not in DEFAULT_CREDENTIAL_GUIDES:
|
||||
missing.append(f"{source}: missing credential guide for {provider}")
|
||||
if provider not in SUPPORTED_CREDENTIAL_PROVIDERS:
|
||||
missing.append(f"{source}: missing connectivity provider for {provider}")
|
||||
|
||||
assert missing == []
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
@@ -6,9 +7,16 @@ from httpx import ASGITransport, AsyncClient
|
||||
from app.api.v1.visualization import convert_vessels_to_geojson
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models.vessel import VesselPosition, VesselStatic
|
||||
from app.models.vessel import AISRawObservation, VesselPosition, VesselStatic
|
||||
from app.services import barentswatch
|
||||
from app.services.collectors.aisstream import AISStreamCollector
|
||||
from app.services.collectors.vessel_ais import VesselAISCollector
|
||||
from app.services.vessel_ais_aggregation import (
|
||||
aggregate_vessel_observations,
|
||||
build_field_conflict_candidates,
|
||||
build_observation_hash,
|
||||
record_vessel_ais_observation,
|
||||
)
|
||||
|
||||
|
||||
def test_vessel_collector_transforms_barentswatch_like_records():
|
||||
@@ -35,6 +43,328 @@ def test_vessel_collector_transforms_barentswatch_like_records():
|
||||
assert records[0]["lat"] == pytest.approx(59.91)
|
||||
|
||||
|
||||
def test_vessel_observation_hash_is_stable_for_same_payload():
|
||||
observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc)
|
||||
payload = {
|
||||
"mmsi": 257123000,
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
"received_at": observed_at,
|
||||
}
|
||||
|
||||
first = build_observation_hash(
|
||||
source="barentswatch_vessels",
|
||||
entity_key="257123000",
|
||||
message_type="PositionReport",
|
||||
observed_at=observed_at,
|
||||
normalized_payload=payload,
|
||||
)
|
||||
second = build_observation_hash(
|
||||
source="barentswatch_vessels",
|
||||
entity_key="257123000",
|
||||
message_type="PositionReport",
|
||||
observed_at=observed_at,
|
||||
normalized_payload=dict(reversed(payload.items())),
|
||||
)
|
||||
|
||||
assert first == second
|
||||
assert len(first) == 64
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_vessel_ais_observation_skips_existing_hash():
|
||||
observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
class _Result:
|
||||
def scalar_one_or_none(self):
|
||||
return 123
|
||||
|
||||
class _Session:
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
|
||||
async def execute(self, _stmt):
|
||||
return _Result()
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
db = _Session()
|
||||
observation = await record_vessel_ais_observation(
|
||||
db,
|
||||
source="barentswatch_vessels",
|
||||
normalized_payload={
|
||||
"mmsi": 257123000,
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
"received_at": observed_at,
|
||||
},
|
||||
delivery_mode="polling",
|
||||
transport="http",
|
||||
observed_at=observed_at.isoformat(),
|
||||
)
|
||||
|
||||
assert observation is None
|
||||
assert db.added == []
|
||||
|
||||
|
||||
def test_build_field_conflict_candidates_from_raw_observations():
|
||||
observations = [
|
||||
AISRawObservation(
|
||||
source="barentswatch_vessels",
|
||||
normalized_payload={"name": "OSLO TRADER", "flag": "NO"},
|
||||
),
|
||||
AISRawObservation(
|
||||
source="aisstream_vessels",
|
||||
normalized_payload={"name": "OSLO TRADER II", "flag": "NO"},
|
||||
),
|
||||
]
|
||||
|
||||
conflicts = build_field_conflict_candidates(observations)
|
||||
|
||||
assert conflicts == [
|
||||
{
|
||||
"field": "name",
|
||||
"candidates": {
|
||||
"aisstream_vessels": "OSLO TRADER II",
|
||||
"barentswatch_vessels": "OSLO TRADER",
|
||||
},
|
||||
"status": "candidate",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aggregate_vessel_observations_prefers_realtime_and_records_conflict():
|
||||
observed_at = datetime.now(timezone.utc) - timedelta(minutes=5)
|
||||
|
||||
class _Result:
|
||||
def scalar_one_or_none(self):
|
||||
return None
|
||||
|
||||
class _Session:
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
|
||||
async def execute(self, _stmt):
|
||||
return _Result()
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
db = _Session()
|
||||
observations = [
|
||||
AISRawObservation(
|
||||
id=1,
|
||||
source="barentswatch_vessels",
|
||||
entity_key="257123000",
|
||||
delivery_mode="polling",
|
||||
transport="http",
|
||||
observed_at=observed_at,
|
||||
collected_at=observed_at,
|
||||
normalized_payload={
|
||||
"mmsi": 257123000,
|
||||
"name": "OSLO TRADER",
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
},
|
||||
),
|
||||
AISRawObservation(
|
||||
id=2,
|
||||
source="aisstream_vessels",
|
||||
entity_key="257123000",
|
||||
delivery_mode="realtime_stream",
|
||||
transport="websocket",
|
||||
observed_at=observed_at + timedelta(seconds=10),
|
||||
collected_at=observed_at + timedelta(seconds=10),
|
||||
normalized_payload={
|
||||
"mmsi": 257123000,
|
||||
"vessel_type": 79,
|
||||
"lat": 59.92,
|
||||
"lon": 10.74,
|
||||
},
|
||||
raw_payload={"MetaData": {"ShipName": "OSLO TRADER II "}},
|
||||
),
|
||||
]
|
||||
|
||||
vessels = await aggregate_vessel_observations(db, observations)
|
||||
|
||||
assert vessels[0]["lat"] == pytest.approx(59.92)
|
||||
assert vessels[0]["field_sources"]["lat"] == "aisstream_vessels"
|
||||
assert vessels[0]["name"] == "OSLO TRADER II"
|
||||
assert vessels[0]["vessel_type_name"] == "Cargo"
|
||||
assert vessels[0]["source_summary"]["aisstream_vessels"]["observation_count"] == 1
|
||||
assert vessels[0]["source_summary"]["barentswatch_vessels"]["delivery_mode"] == "polling"
|
||||
assert vessels[0]["conflict_count"] == 0
|
||||
assert db.added == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vessel_collector_writes_raw_observations_without_changing_position_save(monkeypatch):
|
||||
collector = VesselAISCollector()
|
||||
collector.update_progress = AsyncMock()
|
||||
record_observation = AsyncMock()
|
||||
update_health = AsyncMock()
|
||||
monkeypatch.setattr(
|
||||
"app.services.collectors.vessel_ais.record_vessel_ais_observation",
|
||||
record_observation,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.collectors.vessel_ais.update_ais_source_health",
|
||||
update_health,
|
||||
)
|
||||
|
||||
class _Session:
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
self.committed = False
|
||||
|
||||
async def get(self, *_args):
|
||||
return None
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
async def execute(self, _stmt):
|
||||
return None
|
||||
|
||||
async def commit(self):
|
||||
self.committed = True
|
||||
|
||||
db = _Session()
|
||||
observed_at = datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
saved = await collector._save_data(
|
||||
db,
|
||||
[
|
||||
{
|
||||
"mmsi": 257123000,
|
||||
"name": "OSLO TRADER",
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
"received_at": observed_at,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert saved == 1
|
||||
assert db.committed is True
|
||||
assert any(isinstance(item, VesselStatic) for item in db.added)
|
||||
assert any(isinstance(item, VesselPosition) for item in db.added)
|
||||
record_observation.assert_awaited_once()
|
||||
assert record_observation.await_args.kwargs["source"] == "barentswatch_vessels"
|
||||
assert record_observation.await_args.kwargs["normalized_payload"]["mmsi"] == 257123000
|
||||
update_health.assert_awaited_once()
|
||||
|
||||
|
||||
def test_aisstream_collector_normalizes_position_report():
|
||||
collector = AISStreamCollector()
|
||||
|
||||
records = collector.transform(
|
||||
[
|
||||
{
|
||||
"MessageType": "PositionReport",
|
||||
"MetaData": {
|
||||
"MMSI": 257123000,
|
||||
"ShipName": "OSLO TRADER ",
|
||||
"time_utc": "2026-04-30T12:00:00Z",
|
||||
},
|
||||
"Message": {
|
||||
"PositionReport": {
|
||||
"Latitude": 59.91,
|
||||
"Longitude": 10.73,
|
||||
"Sog": 12.4,
|
||||
"Cog": 214,
|
||||
"TrueHeading": 215,
|
||||
"NavigationalStatus": 0,
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["mmsi"] == 257123000
|
||||
assert records[0]["lat"] == pytest.approx(59.91)
|
||||
assert records[0]["name"] == "OSLO TRADER"
|
||||
assert records[0]["_message_type"] == "PositionReport"
|
||||
|
||||
|
||||
def test_aisstream_collector_maps_ship_static_type_name():
|
||||
collector = AISStreamCollector()
|
||||
|
||||
records = collector.transform(
|
||||
[
|
||||
{
|
||||
"MessageType": "ShipStaticData",
|
||||
"MetaData": {
|
||||
"MMSI": 257123000,
|
||||
"time_utc": "2026-04-30T12:00:00Z",
|
||||
},
|
||||
"Message": {
|
||||
"ShipStaticData": {
|
||||
"Name": "OSLO TRADER",
|
||||
"Type": 79,
|
||||
"CallSign": "LAAB",
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["vessel_type"] == 79
|
||||
assert records[0]["vessel_type_name"] == "Cargo"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aisstream_collector_writes_only_raw_observations(monkeypatch):
|
||||
collector = AISStreamCollector()
|
||||
collector.update_progress = AsyncMock()
|
||||
record_observation = AsyncMock(return_value=object())
|
||||
update_health = AsyncMock()
|
||||
monkeypatch.setattr(
|
||||
"app.services.collectors.aisstream.record_vessel_ais_observation",
|
||||
record_observation,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.collectors.aisstream.update_ais_source_health",
|
||||
update_health,
|
||||
)
|
||||
|
||||
class _Session:
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
self.committed = False
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
async def commit(self):
|
||||
self.committed = True
|
||||
|
||||
db = _Session()
|
||||
saved = await collector._save_data(
|
||||
db,
|
||||
[
|
||||
{
|
||||
"mmsi": 257123000,
|
||||
"lat": 59.91,
|
||||
"lon": 10.73,
|
||||
"received_at": datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc),
|
||||
"_message_type": "PositionReport",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert saved == 1
|
||||
assert db.added == []
|
||||
assert db.committed is True
|
||||
record_observation.assert_awaited_once()
|
||||
assert record_observation.await_args.kwargs["source"] == "aisstream_vessels"
|
||||
update_health.assert_awaited_once()
|
||||
|
||||
|
||||
def test_barentswatch_reads_credentials_from_zshrc(tmp_path):
|
||||
zshrc = tmp_path / ".zshrc"
|
||||
zshrc.write_text(
|
||||
@@ -137,7 +467,7 @@ async def test_vessels_geojson_endpoint_filters_type_and_bbox():
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get(
|
||||
"/api/v1/visualization/geo/vessels",
|
||||
params={"bbox": "0,50,20,70", "type": "cargo"},
|
||||
params={"bbox": "0,50,20,70", "type": "cargo", "limit": 0},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
Reference in New Issue
Block a user