"""Unit tests for data collectors""" import pytest from unittest.mock import AsyncMock, patch from app.core.datasource_defaults import DEFAULT_DATASOURCES from app.services.credential_guides import DEFAULT_CREDENTIAL_GUIDES from app.services.collectors.top500 import TOP500Collector from app.services.collectors.registry import collector_registry from app.services.datasource_connectivity import SUPPORTED_CREDENTIAL_PROVIDERS from app.models.task import CollectionTask class TestBaseCollector: """Tests for BaseCollector""" def test_base_collector_attributes(self): """Test base collector has correct default attributes via concrete class""" collector = TOP500Collector() assert collector.name == "top500" assert collector.priority == "P0" assert collector.module == "L1" assert collector.frequency_hours == 4 @pytest.mark.asyncio async def test_update_phase_progress_tracks_phase_fields(self, mock_db_session): """Test phase-level progress updates independently from record totals""" collector = TOP500Collector() task = CollectionTask(datasource_id=1, status="running", phase="fetching") collector._current_task = task collector._db_session = mock_db_session with patch.object(collector, "_publish_task_update", new=AsyncMock()) as publish: await collector.update_phase_progress( current=512, total=1024, unit="bytes", message="Downloading dataset", commit=True, ) assert task.phase_progress == 50.0 assert task.phase_current == 512 assert task.phase_total == 1024 assert task.phase_unit == "bytes" assert task.phase_message == "Downloading dataset" mock_db_session.commit.assert_awaited_once() publish.assert_awaited_once() class TestTOP500Collector: """Tests for TOP500Collector""" def test_parse_coordinate_valid_float(self): """Test parsing valid float coordinate""" collector = TOP500Collector() assert collector._parse_coordinate(45.5) == 45.5 def test_parse_coordinate_valid_string(self): """Test parsing valid string coordinate""" collector = TOP500Collector() assert collector._parse_coordinate("45.5") == 45.5 def test_parse_coordinate_invalid_string(self): """Test parsing invalid string coordinate""" collector = TOP500Collector() assert collector._parse_coordinate("invalid") == 0.0 def test_parse_coordinate_none(self): """Test parsing None coordinate""" collector = TOP500Collector() assert collector._parse_coordinate(None) == 0.0 def test_parse_response_empty(self): """Test parsing empty response""" collector = TOP500Collector() result = collector.parse_response("
") assert len(result) > 0 def test_parse_response_single_item(self): """Test parsing single item response""" collector = TOP500Collector() response = """
RankSystemCoresRmaxRpeakPower
1 Test Supercomputer, Test Corp\nTest Site\nUSA 100000 100 PFLOP/s 150 PFLOP/s 5000
""" result = collector.parse_response(response) assert len(result) == 1 assert result[0]["source_id"] == "top500_1" assert result[0]["name"] == "Test Supercomputer" assert result[0]["country"] == "USA" assert result[0]["metadata"]["rank"] == 1 assert "Test Corp" in result[0]["metadata"]["manufacturer"] def test_parse_response_skips_invalid_item(self): """Test parsing skips items with missing data""" collector = TOP500Collector() response = """
RankSystemCoresRmaxRpeakPower
1 Valid\nVendor\nSite\nUSA 1000 10 PFLOP/s 12 PFLOP/s 100
- Invalid 1000 10 PFLOP/s 12 PFLOP/s 100
""" result = collector.parse_response(response) assert len(result) == 1 assert result[0]["name"] == "Valid" class TestHTTPCollector: """Tests for HTTPCollector""" def test_http_collector_attributes(self): """Test HTTP collector has correct default attributes via concrete class""" collector = TOP500Collector() assert collector.name == "top500" assert collector.priority == "P0" assert hasattr(collector, "fetch") def test_collector_has_required_methods(self): """Test HTTP collector has required methods""" collector = TOP500Collector() assert hasattr(collector, "fetch") assert hasattr(collector, "parse_response") assert callable(collector.fetch) assert callable(collector.parse_response) def test_aisstream_collector_is_registered(): collector = collector_registry.get("aisstream_vessels") assert collector is not None assert collector.data_type == "vessel_ais" def test_supported_credential_collectors_have_guides_and_connectivity_provider(): missing: list[str] = [] for source, info in DEFAULT_DATASOURCES.items(): if not info.get("requires_credentials"): continue if info.get("credential_status") != "supported": continue provider = info.get("credential_provider") if not provider: missing.append(f"{source}: missing credential_provider") continue if provider not in DEFAULT_CREDENTIAL_GUIDES: missing.append(f"{source}: missing credential guide for {provider}") if provider not in SUPPORTED_CREDENTIAL_PROVIDERS: missing.append(f"{source}: missing connectivity provider for {provider}") assert missing == []