177 lines
6.3 KiB
Python
177 lines
6.3 KiB
Python
"""Unit tests for data collectors"""
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
|
from app.services.credential_guides import DEFAULT_CREDENTIAL_GUIDES
|
|
from app.services.collectors.top500 import TOP500Collector
|
|
from app.services.collectors.registry import collector_registry
|
|
from app.services.datasource_connectivity import SUPPORTED_CREDENTIAL_PROVIDERS
|
|
from app.models.task import CollectionTask
|
|
|
|
|
|
class TestBaseCollector:
|
|
"""Tests for BaseCollector"""
|
|
|
|
def test_base_collector_attributes(self):
|
|
"""Test base collector has correct default attributes via concrete class"""
|
|
collector = TOP500Collector()
|
|
assert collector.name == "top500"
|
|
assert collector.priority == "P0"
|
|
assert collector.module == "L1"
|
|
assert collector.frequency_hours == 4
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_update_phase_progress_tracks_phase_fields(self, mock_db_session):
|
|
"""Test phase-level progress updates independently from record totals"""
|
|
collector = TOP500Collector()
|
|
task = CollectionTask(datasource_id=1, status="running", phase="fetching")
|
|
collector._current_task = task
|
|
collector._db_session = mock_db_session
|
|
|
|
with patch.object(collector, "_publish_task_update", new=AsyncMock()) as publish:
|
|
await collector.update_phase_progress(
|
|
current=512,
|
|
total=1024,
|
|
unit="bytes",
|
|
message="Downloading dataset",
|
|
commit=True,
|
|
)
|
|
|
|
assert task.phase_progress == 50.0
|
|
assert task.phase_current == 512
|
|
assert task.phase_total == 1024
|
|
assert task.phase_unit == "bytes"
|
|
assert task.phase_message == "Downloading dataset"
|
|
mock_db_session.commit.assert_awaited_once()
|
|
publish.assert_awaited_once()
|
|
|
|
|
|
class TestTOP500Collector:
|
|
"""Tests for TOP500Collector"""
|
|
|
|
def test_parse_coordinate_valid_float(self):
|
|
"""Test parsing valid float coordinate"""
|
|
collector = TOP500Collector()
|
|
assert collector._parse_coordinate(45.5) == 45.5
|
|
|
|
def test_parse_coordinate_valid_string(self):
|
|
"""Test parsing valid string coordinate"""
|
|
collector = TOP500Collector()
|
|
assert collector._parse_coordinate("45.5") == 45.5
|
|
|
|
def test_parse_coordinate_invalid_string(self):
|
|
"""Test parsing invalid string coordinate"""
|
|
collector = TOP500Collector()
|
|
assert collector._parse_coordinate("invalid") == 0.0
|
|
|
|
def test_parse_coordinate_none(self):
|
|
"""Test parsing None coordinate"""
|
|
collector = TOP500Collector()
|
|
assert collector._parse_coordinate(None) == 0.0
|
|
|
|
def test_parse_response_empty(self):
|
|
"""Test parsing empty response"""
|
|
collector = TOP500Collector()
|
|
result = collector.parse_response("<html><body><table></table></body></html>")
|
|
assert len(result) > 0
|
|
|
|
def test_parse_response_single_item(self):
|
|
"""Test parsing single item response"""
|
|
collector = TOP500Collector()
|
|
response = """
|
|
<table class="top500-table">
|
|
<tr><th>Rank</th><th>System</th><th>Cores</th><th>Rmax</th><th>Rpeak</th><th>Power</th></tr>
|
|
<tr>
|
|
<td>1</td>
|
|
<td><a href="/system/1/">Test Supercomputer</a>, Test Corp\nTest Site\nUSA</td>
|
|
<td>100000</td>
|
|
<td>100 PFLOP/s</td>
|
|
<td>150 PFLOP/s</td>
|
|
<td>5000</td>
|
|
</tr>
|
|
</table>
|
|
"""
|
|
result = collector.parse_response(response)
|
|
assert len(result) == 1
|
|
assert result[0]["source_id"] == "top500_1"
|
|
assert result[0]["name"] == "Test Supercomputer"
|
|
assert result[0]["country"] == "USA"
|
|
assert result[0]["metadata"]["rank"] == 1
|
|
assert "Test Corp" in result[0]["metadata"]["manufacturer"]
|
|
|
|
def test_parse_response_skips_invalid_item(self):
|
|
"""Test parsing skips items with missing data"""
|
|
collector = TOP500Collector()
|
|
response = """
|
|
<table class="top500-table">
|
|
<tr><th>Rank</th><th>System</th><th>Cores</th><th>Rmax</th><th>Rpeak</th><th>Power</th></tr>
|
|
<tr>
|
|
<td>1</td>
|
|
<td>Valid\nVendor\nSite\nUSA</td>
|
|
<td>1000</td>
|
|
<td>10 PFLOP/s</td>
|
|
<td>12 PFLOP/s</td>
|
|
<td>100</td>
|
|
</tr>
|
|
<tr>
|
|
<td>-</td>
|
|
<td>Invalid</td>
|
|
<td>1000</td>
|
|
<td>10 PFLOP/s</td>
|
|
<td>12 PFLOP/s</td>
|
|
<td>100</td>
|
|
</tr>
|
|
</table>
|
|
"""
|
|
result = collector.parse_response(response)
|
|
assert len(result) == 1
|
|
assert result[0]["name"] == "Valid"
|
|
|
|
|
|
class TestHTTPCollector:
|
|
"""Tests for HTTPCollector"""
|
|
|
|
def test_http_collector_attributes(self):
|
|
"""Test HTTP collector has correct default attributes via concrete class"""
|
|
collector = TOP500Collector()
|
|
assert collector.name == "top500"
|
|
assert collector.priority == "P0"
|
|
assert hasattr(collector, "fetch")
|
|
|
|
def test_collector_has_required_methods(self):
|
|
"""Test HTTP collector has required methods"""
|
|
collector = TOP500Collector()
|
|
assert hasattr(collector, "fetch")
|
|
assert hasattr(collector, "parse_response")
|
|
assert callable(collector.fetch)
|
|
assert callable(collector.parse_response)
|
|
|
|
|
|
def test_aisstream_collector_is_registered():
|
|
collector = collector_registry.get("aisstream_vessels")
|
|
|
|
assert collector is not None
|
|
assert collector.data_type == "vessel_ais"
|
|
|
|
|
|
def test_supported_credential_collectors_have_guides_and_connectivity_provider():
|
|
missing: list[str] = []
|
|
for source, info in DEFAULT_DATASOURCES.items():
|
|
if not info.get("requires_credentials"):
|
|
continue
|
|
if info.get("credential_status") != "supported":
|
|
continue
|
|
|
|
provider = info.get("credential_provider")
|
|
if not provider:
|
|
missing.append(f"{source}: missing credential_provider")
|
|
continue
|
|
if provider not in DEFAULT_CREDENTIAL_GUIDES:
|
|
missing.append(f"{source}: missing credential guide for {provider}")
|
|
if provider not in SUPPORTED_CREDENTIAL_PROVIDERS:
|
|
missing.append(f"{source}: missing connectivity provider for {provider}")
|
|
|
|
assert missing == []
|