153 lines
5.0 KiB
Python
153 lines
5.0 KiB
Python
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
from app.api.v1 import datasources as datasources_api
|
|
from app.models.datasource import DataSource
|
|
from app.services import earth_layer_cache as earth_cache
|
|
|
|
|
|
def make_datasource(
|
|
datasource_id: int,
|
|
source: str,
|
|
*,
|
|
name: str | None = None,
|
|
module: str = "L4",
|
|
is_active: bool = True,
|
|
last_status: str | None = None,
|
|
last_run_at: datetime | None = None,
|
|
frequency_minutes: int = 60,
|
|
) -> DataSource:
|
|
return DataSource(
|
|
id=datasource_id,
|
|
name=name or source,
|
|
source=source,
|
|
module=module,
|
|
priority="P1",
|
|
frequency_minutes=frequency_minutes,
|
|
collector_class=source,
|
|
is_active=is_active,
|
|
last_status=last_status,
|
|
last_run_at=last_run_at,
|
|
)
|
|
|
|
|
|
def test_datasource_product_key_groups_domain_specific_sources():
|
|
assert datasources_api.datasource_product_key(make_datasource(1, "aisstream_vessels")) == "vessels"
|
|
assert datasources_api.datasource_product_key(make_datasource(2, "telegeography_cables")) == "cables"
|
|
assert datasources_api.datasource_product_key(make_datasource(3, "celestrak_tle")) == "satellites"
|
|
assert datasources_api.datasource_product_key(make_datasource(4, "ris_live_bgp")) == "bgp"
|
|
|
|
|
|
def test_filter_datasources_by_product_status_and_collected_state():
|
|
vessels = make_datasource(1, "aisstream_vessels", last_status="success")
|
|
cables = make_datasource(2, "telegeography_cables", last_status="failed")
|
|
filtered = datasources_api._filter_datasources_in_memory(
|
|
[vessels, cables],
|
|
running_tasks={},
|
|
record_counts={"aisstream_vessels": 12, "telegeography_cables": 0},
|
|
product="vessels",
|
|
run_status="success",
|
|
collected=True,
|
|
)
|
|
|
|
assert filtered == [vessels]
|
|
|
|
|
|
def test_serialize_datasource_row_can_skip_endpoint_resolution():
|
|
datasource = make_datasource(7, "arcgis_cables", last_status="success", module="L2")
|
|
|
|
class ExplodingConfig:
|
|
def get_yaml_url(self, _source):
|
|
raise AssertionError("endpoint resolution should be skipped")
|
|
|
|
row = datasources_api.serialize_datasource_row(
|
|
datasource,
|
|
running_tasks={},
|
|
latest_tasks={},
|
|
record_counts={"arcgis_cables": 343},
|
|
endpoint_overrides={},
|
|
config=ExplodingConfig(),
|
|
include_endpoint=False,
|
|
)
|
|
|
|
assert row["id"] == 7
|
|
assert row["source"] == "arcgis_cables"
|
|
assert row["module"] == "L2"
|
|
assert row["last_status"] == "success"
|
|
assert row["collected_records"] == 343
|
|
assert row["has_collected_data"] is True
|
|
assert "endpoint" not in row
|
|
|
|
|
|
def test_serialize_datasource_row_includes_endpoint_when_requested():
|
|
datasource = make_datasource(8, "arcgis_landing_points")
|
|
|
|
class Config:
|
|
def get_yaml_url(self, source):
|
|
return f"https://example.test/{source}"
|
|
|
|
row = datasources_api.serialize_datasource_row(
|
|
datasource,
|
|
running_tasks={},
|
|
latest_tasks={},
|
|
record_counts={},
|
|
endpoint_overrides={},
|
|
config=Config(),
|
|
include_endpoint=True,
|
|
)
|
|
|
|
assert row["endpoint"] == "https://example.test/arcgis_landing_points"
|
|
|
|
|
|
def test_invalidate_earth_layer_cache_for_source_covers_datasource_aliases(monkeypatch):
|
|
patterns: list[str] = []
|
|
|
|
def fake_delete_pattern(pattern: str) -> int:
|
|
patterns.append(pattern)
|
|
return 1
|
|
|
|
monkeypatch.setattr(earth_cache.earth_layer_cache, "delete_pattern", fake_delete_pattern)
|
|
|
|
deleted = earth_cache.invalidate_earth_layer_cache_for_source("telegeography_cables")
|
|
|
|
assert deleted == 3
|
|
assert patterns == [
|
|
"earth:layer:v1:cables*",
|
|
"earth:layer:v1:landing-points*",
|
|
"earth:layer:v1:summary*",
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_trigger_datasource_batch_skips_disabled_and_frequency_window(monkeypatch):
|
|
now = datetime.now(timezone.utc)
|
|
disabled = make_datasource(1, "aisstream_vessels", is_active=False)
|
|
not_due = make_datasource(2, "telegeography_cables", last_run_at=now, frequency_minutes=120)
|
|
due = make_datasource(3, "ris_live_bgp", last_run_at=now - timedelta(hours=2))
|
|
triggered_sources: list[str] = []
|
|
|
|
async def fake_running_tasks(_db, _ids):
|
|
return {}
|
|
|
|
async def fake_latest_task_ids(_db, _ids):
|
|
return {}
|
|
|
|
monkeypatch.setattr(datasources_api, "_load_latest_running_tasks", fake_running_tasks)
|
|
monkeypatch.setattr(datasources_api, "_load_latest_task_ids", fake_latest_task_ids)
|
|
monkeypatch.setattr(
|
|
datasources_api,
|
|
"run_collector_now",
|
|
lambda source: triggered_sources.append(source) or True,
|
|
)
|
|
|
|
result = await datasources_api._trigger_datasource_batch(
|
|
object(),
|
|
[disabled, not_due, due],
|
|
force=False,
|
|
)
|
|
|
|
assert [item["source"] for item in result["triggered"]] == ["ris_live_bgp"]
|
|
assert {item["reason"] for item in result["skipped"]} == {"disabled", "within_frequency_window"}
|
|
assert triggered_sources == ["ris_live_bgp"]
|