"""Unit tests for data collectors""" import json import pytest from unittest.mock import AsyncMock, patch from app.core.datasource_defaults import DEFAULT_DATASOURCES from app.services.collectors.celestrak import CelesTrakTLECollector from app.services.collectors.downloads import DownloadHTTPStatusError, ResumableFileDownloader from app.services.credential_guides import DEFAULT_CREDENTIAL_GUIDES from app.services.collectors.top500 import TOP500Collector from app.services.collectors.registry import collector_registry from app.services.datasource_connectivity import SUPPORTED_CREDENTIAL_PROVIDERS from app.models.task import CollectionTask class TestBaseCollector: """Tests for BaseCollector""" def test_base_collector_attributes(self): """Test base collector has correct default attributes via concrete class""" collector = TOP500Collector() assert collector.name == "top500" assert collector.priority == "P0" assert collector.module == "L1" assert collector.frequency_hours == 4 @pytest.mark.asyncio async def test_update_phase_progress_tracks_phase_fields(self, mock_db_session): """Test phase-level progress updates independently from record totals""" collector = TOP500Collector() task = CollectionTask(datasource_id=1, status="running", phase="fetching") collector._current_task = task collector._db_session = mock_db_session with patch.object(collector, "_publish_task_update", new=AsyncMock()) as publish: await collector.update_phase_progress( current=512, total=1024, unit="bytes", message="Downloading dataset", commit=True, ) assert task.phase_progress == 50.0 assert task.phase_current == 512 assert task.phase_total == 1024 assert task.phase_unit == "bytes" assert task.phase_message == "Downloading dataset" mock_db_session.commit.assert_awaited_once() publish.assert_awaited_once() class TestTOP500Collector: """Tests for TOP500Collector""" def test_parse_coordinate_valid_float(self): """Test parsing valid float coordinate""" collector = TOP500Collector() assert collector._parse_coordinate(45.5) == 45.5 def test_parse_coordinate_valid_string(self): """Test parsing valid string coordinate""" collector = TOP500Collector() assert collector._parse_coordinate("45.5") == 45.5 def test_parse_coordinate_invalid_string(self): """Test parsing invalid string coordinate""" collector = TOP500Collector() assert collector._parse_coordinate("invalid") == 0.0 def test_parse_coordinate_none(self): """Test parsing None coordinate""" collector = TOP500Collector() assert collector._parse_coordinate(None) == 0.0 def test_parse_response_empty(self): """Test parsing empty response""" collector = TOP500Collector() result = collector.parse_response("
") assert len(result) > 0 def test_parse_response_single_item(self): """Test parsing single item response""" collector = TOP500Collector() response = """
RankSystemCoresRmaxRpeakPower
1 Test Supercomputer, Test Corp\nTest Site\nUSA 100000 100 PFLOP/s 150 PFLOP/s 5000
""" result = collector.parse_response(response) assert len(result) == 1 assert result[0]["source_id"] == "top500_1" assert result[0]["name"] == "Test Supercomputer" assert result[0]["country"] == "USA" assert result[0]["metadata"]["rank"] == 1 assert "Test Corp" in result[0]["metadata"]["manufacturer"] def test_parse_response_skips_invalid_item(self): """Test parsing skips items with missing data""" collector = TOP500Collector() response = """
RankSystemCoresRmaxRpeakPower
1 Valid\nVendor\nSite\nUSA 1000 10 PFLOP/s 12 PFLOP/s 100
- Invalid 1000 10 PFLOP/s 12 PFLOP/s 100
""" result = collector.parse_response(response) assert len(result) == 1 assert result[0]["name"] == "Valid" class TestHTTPCollector: """Tests for HTTPCollector""" def test_http_collector_attributes(self): """Test HTTP collector has correct default attributes via concrete class""" collector = TOP500Collector() assert collector.name == "top500" assert collector.priority == "P0" assert hasattr(collector, "fetch") def test_collector_has_required_methods(self): """Test HTTP collector has required methods""" collector = TOP500Collector() assert hasattr(collector, "fetch") assert hasattr(collector, "parse_response") assert callable(collector.fetch) assert callable(collector.parse_response) class TestCelesTrakTLECollector: def test_transform_uses_norad_as_source_id_and_preserves_starlink_group(self): collector = CelesTrakTLECollector() result = collector.transform([ { "NORAD_CAT_ID": 44720, "OBJECT_NAME": "STARLINK-1000", "OBJECT_ID": "2019-029AZ", "EPOCH": "2026-03-13T00:00:00Z", "MEAN_MOTION": 15.79234567, "ECCENTRICITY": 0.0001234, "INCLINATION": 53.0, "RA_OF_ASC_NODE": 10.0, "ARG_OF_PERICENTER": 20.0, "MEAN_ANOMALY": 30.0, "_celestrak_query_group": "active", "_celestrak_source_url": "https://celestrak.example/gp.php?GROUP=active&FORMAT=json", } ]) assert result[0]["source_id"] == "44720" assert result[0]["metadata"]["constellation_group"] == "starlink" assert result[0]["metadata"]["celestrak_query_group"] == "active" assert result[0]["metadata"]["norad_cat_id"] == 44720 assert result[0]["metadata"]["tle_line1"] assert result[0]["metadata"]["tle_line2"] def test_load_active_payload_rejects_invalid_records(self, tmp_path): collector = CelesTrakTLECollector() payload_path = tmp_path / "active.json" payload_path.write_text(json.dumps([{"OBJECT_NAME": "missing norad"}]), encoding="utf-8") with pytest.raises(RuntimeError, match="invalid record"): collector._load_active_payload(payload_path) def test_load_active_payload_accepts_complete_array(self, tmp_path): collector = CelesTrakTLECollector() payload_path = tmp_path / "active.json" payload_path.write_text( json.dumps([{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}]), encoding="utf-8", ) records = collector._load_active_payload(payload_path) assert records == [{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}] @pytest.mark.asyncio async def test_fetch_retries_and_raises_instead_of_returning_partial_data(self, monkeypatch, tmp_path): collector = CelesTrakTLECollector() collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php" attempts = 0 async def fake_download_file(*args, **kwargs): nonlocal attempts attempts += 1 raise RuntimeError("network interrupted") async def fake_emit_business_log(*args, **kwargs): return None monkeypatch.setattr(collector._downloader, "download_file", fake_download_file) monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log) monkeypatch.setattr("app.services.collectors.celestrak.asyncio.sleep", AsyncMock()) with pytest.raises(RuntimeError, match="failed after retries"): await collector.fetch() assert attempts == 3 @pytest.mark.asyncio async def test_fetch_uses_cache_when_celestrak_reports_not_updated(self, monkeypatch, tmp_path): collector = CelesTrakTLECollector() collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php" collector._downloader = ResumableFileDownloader(cache_namespace="celestrak-test", cache_root=tmp_path) url = collector._active_url() cached_path = collector._downloader.cached_file_path(url, ".json") cached_path.parent.mkdir(parents=True, exist_ok=True) cached_path.write_text( json.dumps([{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}]), encoding="utf-8", ) async def fake_download_file(*args, **kwargs): raise DownloadHTTPStatusError( url=url, status_code=403, body="GP data has not updated since your last successful download of GROUP=active.", ) async def fake_emit_business_log(*args, **kwargs): return None monkeypatch.setattr(collector._downloader, "download_file", fake_download_file) monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log) records = await collector.fetch() assert records[0]["NORAD_CAT_ID"] == 25544 assert records[0]["_celestrak_query_group"] == "active" @pytest.mark.asyncio async def test_fetch_not_updated_without_cache_does_not_retry(self, monkeypatch, tmp_path): collector = CelesTrakTLECollector() collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php" collector._downloader = ResumableFileDownloader(cache_namespace="celestrak-test", cache_root=tmp_path) attempts = 0 async def fake_download_file(*args, **kwargs): nonlocal attempts attempts += 1 raise DownloadHTTPStatusError( url=collector._active_url(), status_code=403, body="GP data has not updated since your last successful download of GROUP=active.", ) async def fake_emit_business_log(*args, **kwargs): return None monkeypatch.setattr(collector._downloader, "download_file", fake_download_file) monkeypatch.setattr("app.services.collectors.celestrak.FALLBACK_GROUPS", ("starlink",)) monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log) monkeypatch.setattr("app.services.collectors.celestrak.asyncio.sleep", AsyncMock()) with pytest.raises(RuntimeError, match="fallback group mode failed"): await collector.fetch() assert attempts == 2 @pytest.mark.asyncio async def test_fetch_falls_back_to_all_groups_when_active_not_updated_without_cache(self, monkeypatch, tmp_path): collector = CelesTrakTLECollector() collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php" collector._downloader = ResumableFileDownloader(cache_namespace="celestrak-test", cache_root=tmp_path) payload_by_group = { "starlink": [{"NORAD_CAT_ID": 100, "OBJECT_NAME": "STARLINK-100"}], "gps-ops": [{"NORAD_CAT_ID": 200, "OBJECT_NAME": "GPS BIIR-2"}], } async def fake_download_file(_client, url, **_kwargs): if "GROUP=active" in url: raise DownloadHTTPStatusError( url=url, status_code=403, body="GP data has not updated since your last successful download of GROUP=active.", ) group = "starlink" if "GROUP=starlink" in url else "gps-ops" path = tmp_path / f"{group}.json" path.write_text(json.dumps(payload_by_group[group]), encoding="utf-8") return path async def fake_emit_business_log(*args, **kwargs): return None monkeypatch.setattr(collector._downloader, "download_file", fake_download_file) monkeypatch.setattr("app.services.collectors.celestrak.FALLBACK_GROUPS", tuple(payload_by_group)) monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log) records = await collector.fetch() assert [item["NORAD_CAT_ID"] for item in records] == [100, 200] assert records[0]["_celestrak_query_group"] == "starlink" assert records[1]["_celestrak_group"] == "gps-ops" def test_aisstream_collector_is_registered(): collector = collector_registry.get("aisstream_vessels") assert collector is not None assert collector.data_type == "vessel_ais" def test_supported_credential_collectors_have_guides_and_connectivity_provider(): missing: list[str] = [] for source, info in DEFAULT_DATASOURCES.items(): if not info.get("requires_credentials"): continue if info.get("credential_status") != "supported": continue provider = info.get("credential_provider") if not provider: missing.append(f"{source}: missing credential_provider") continue if provider not in DEFAULT_CREDENTIAL_GUIDES: missing.append(f"{source}: missing credential guide for {provider}") if provider not in SUPPORTED_CREDENTIAL_PROVIDERS: missing.append(f"{source}: missing connectivity provider for {provider}") assert missing == []