import asyncio import inspect from datetime import datetime, timedelta, timezone from app.api.v1 import datasources as datasources_api from app.models.datasource import DataSource from app.models.task import CollectionTask from app.services import data_jobs from app.services import earth_layer_cache as earth_cache def make_datasource( datasource_id: int, source: str, *, name: str | None = None, module: str = "L4", is_active: bool = True, last_status: str | None = None, last_run_at: datetime | None = None, frequency_minutes: int = 60, ) -> DataSource: return DataSource( id=datasource_id, name=name or source, source=source, module=module, priority="P1", frequency_minutes=frequency_minutes, collector_class=source, is_active=is_active, last_status=last_status, last_run_at=last_run_at, ) def test_datasource_product_key_groups_domain_specific_sources(): assert datasources_api.datasource_product_key(make_datasource(1, "aisstream_vessels")) == "vessels" assert datasources_api.datasource_product_key(make_datasource(2, "telegeography_cables")) == "cables" assert datasources_api.datasource_product_key(make_datasource(3, "celestrak_tle")) == "satellites" assert datasources_api.datasource_product_key(make_datasource(4, "ris_live_bgp")) == "bgp" def test_filter_datasources_by_product_status_and_collected_state(): vessels = make_datasource(1, "aisstream_vessels", last_status="success") cables = make_datasource(2, "telegeography_cables", last_status="failed") filtered = datasources_api._filter_datasources_in_memory( [vessels, cables], running_tasks={}, record_counts={"aisstream_vessels": 12, "telegeography_cables": 0}, product="vessels", run_status="success", collected=True, ) assert filtered == [vessels] def test_serialize_datasource_row_can_skip_endpoint_resolution(): datasource = make_datasource(7, "arcgis_cables", last_status="success", module="L2") class ExplodingConfig: def get_yaml_url(self, _source): raise AssertionError("endpoint resolution should be skipped") row = datasources_api.serialize_datasource_row( datasource, running_tasks={}, latest_tasks={}, record_counts={"arcgis_cables": 343}, endpoint_overrides={}, config=ExplodingConfig(), include_endpoint=False, ) assert row["id"] == 7 assert row["source"] == "arcgis_cables" assert row["module"] == "L2" assert row["last_status"] == "success" assert row["collected_records"] == 343 assert row["has_collected_data"] is True assert "endpoint" not in row def test_serialize_datasource_row_includes_endpoint_when_requested(): datasource = make_datasource(8, "arcgis_landing_points") class Config: def get_yaml_url(self, source): return f"https://example.test/{source}" row = datasources_api.serialize_datasource_row( datasource, running_tasks={}, latest_tasks={}, record_counts={}, endpoint_overrides={}, config=Config(), include_endpoint=True, ) assert row["endpoint"] == "https://example.test/arcgis_landing_points" def test_serialize_datasource_row_separates_collector_running_from_delete_task(): datasource = make_datasource(9, "top500", last_status="success") task = CollectionTask( id=44, datasource_id=9, source="top500", task_type="clear_data", status="running", phase="clearing_data", ) row = datasources_api.serialize_datasource_row( datasource, running_tasks={9: task}, latest_tasks={9: task}, record_counts={"top500": 500}, endpoint_overrides={}, config=object(), include_endpoint=False, ) assert row["is_task_active"] is True assert row["is_running"] is False assert row["task_type"] == "clear_data" assert row["task_status"] == "running" def test_cancel_queued_delete_task_finishes_immediately(monkeypatch): task = CollectionTask( id=45, datasource_id=9, source="top500", task_type="clear_data", status="queued", phase="queued", ) class FakeDb: async def commit(self): return None async def refresh(self, _task): return None async def fake_broadcast(_task): return None monkeypatch.setattr(data_jobs, "_broadcast_task_update", fake_broadcast) async def run(): return await data_jobs.request_cancel_datasource_task(FakeDb(), task) result = asyncio.run(run()) assert result.status == "cancelled" assert result.phase == "cancelled" assert result.completed_at is not None def test_clear_data_job_relies_on_db_outbox_instead_of_extra_earth_refresh_task(): source = inspect.getsource(data_jobs._run_clear_data_job) assert "enqueue_earth_refresh_job" not in source def test_invalidate_earth_layer_cache_for_source_covers_datasource_aliases(monkeypatch): patterns: list[str] = [] def fake_delete_pattern(pattern: str) -> int: patterns.append(pattern) return 1 monkeypatch.setattr(earth_cache.earth_layer_cache, "delete_pattern", fake_delete_pattern) deleted = earth_cache.invalidate_earth_layer_cache_for_source("telegeography_cables") assert deleted == 3 assert patterns == [ "earth:layer:v1:cables*", "earth:layer:v1:landing-points*", "earth:layer:v1:summary*", ] def test_trigger_datasource_batch_skips_disabled_and_frequency_window(monkeypatch): async def run(): now = datetime.now(timezone.utc) disabled = make_datasource(1, "aisstream_vessels", is_active=False) not_due = make_datasource(2, "telegeography_cables", last_run_at=now, frequency_minutes=120) due = make_datasource(3, "ris_live_bgp", last_run_at=now - timedelta(hours=2)) queued_sources: list[str] = [] async def fake_running_tasks(_db, _ids): return {} async def fake_enqueue(_db, datasource, task_type, **_kwargs): queued_sources.append(datasource.source) return CollectionTask(id=100 + datasource.id, datasource_id=datasource.id, source=datasource.source, task_type=task_type, status="queued") monkeypatch.setattr(datasources_api, "_load_latest_running_tasks", fake_running_tasks) monkeypatch.setattr( datasources_api, "enqueue_datasource_job", fake_enqueue, ) result = await datasources_api._trigger_datasource_batch( object(), [disabled, not_due, due], force=False, ) assert [item["source"] for item in result["triggered"]] == ["ris_live_bgp"] assert {item["reason"] for item in result["skipped"]} == {"disabled", "within_frequency_window"} assert queued_sources == ["ris_live_bgp"] asyncio.run(run())