347 lines
14 KiB
Python
347 lines
14 KiB
Python
"""Unit tests for data collectors"""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
|
from app.services.collectors.celestrak import CelesTrakTLECollector
|
|
from app.services.collectors.downloads import DownloadHTTPStatusError, ResumableFileDownloader
|
|
from app.services.credential_guides import DEFAULT_CREDENTIAL_GUIDES
|
|
from app.services.collectors.top500 import TOP500Collector
|
|
from app.services.collectors.registry import collector_registry
|
|
from app.services.datasource_connectivity import SUPPORTED_CREDENTIAL_PROVIDERS
|
|
from app.models.task import CollectionTask
|
|
|
|
|
|
class TestBaseCollector:
|
|
"""Tests for BaseCollector"""
|
|
|
|
def test_base_collector_attributes(self):
|
|
"""Test base collector has correct default attributes via concrete class"""
|
|
collector = TOP500Collector()
|
|
assert collector.name == "top500"
|
|
assert collector.priority == "P0"
|
|
assert collector.module == "L1"
|
|
assert collector.frequency_hours == 4
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_update_phase_progress_tracks_phase_fields(self, mock_db_session):
|
|
"""Test phase-level progress updates independently from record totals"""
|
|
collector = TOP500Collector()
|
|
task = CollectionTask(datasource_id=1, status="running", phase="fetching")
|
|
collector._current_task = task
|
|
collector._db_session = mock_db_session
|
|
|
|
with patch.object(collector, "_publish_task_update", new=AsyncMock()) as publish:
|
|
await collector.update_phase_progress(
|
|
current=512,
|
|
total=1024,
|
|
unit="bytes",
|
|
message="Downloading dataset",
|
|
commit=True,
|
|
)
|
|
|
|
assert task.phase_progress == 50.0
|
|
assert task.phase_current == 512
|
|
assert task.phase_total == 1024
|
|
assert task.phase_unit == "bytes"
|
|
assert task.phase_message == "Downloading dataset"
|
|
mock_db_session.commit.assert_awaited_once()
|
|
publish.assert_awaited_once()
|
|
|
|
|
|
class TestTOP500Collector:
|
|
"""Tests for TOP500Collector"""
|
|
|
|
def test_parse_coordinate_valid_float(self):
|
|
"""Test parsing valid float coordinate"""
|
|
collector = TOP500Collector()
|
|
assert collector._parse_coordinate(45.5) == 45.5
|
|
|
|
def test_parse_coordinate_valid_string(self):
|
|
"""Test parsing valid string coordinate"""
|
|
collector = TOP500Collector()
|
|
assert collector._parse_coordinate("45.5") == 45.5
|
|
|
|
def test_parse_coordinate_invalid_string(self):
|
|
"""Test parsing invalid string coordinate"""
|
|
collector = TOP500Collector()
|
|
assert collector._parse_coordinate("invalid") == 0.0
|
|
|
|
def test_parse_coordinate_none(self):
|
|
"""Test parsing None coordinate"""
|
|
collector = TOP500Collector()
|
|
assert collector._parse_coordinate(None) == 0.0
|
|
|
|
def test_parse_response_empty(self):
|
|
"""Test parsing empty response"""
|
|
collector = TOP500Collector()
|
|
result = collector.parse_response("<html><body><table></table></body></html>")
|
|
assert len(result) > 0
|
|
|
|
def test_parse_response_single_item(self):
|
|
"""Test parsing single item response"""
|
|
collector = TOP500Collector()
|
|
response = """
|
|
<table class="top500-table">
|
|
<tr><th>Rank</th><th>System</th><th>Cores</th><th>Rmax</th><th>Rpeak</th><th>Power</th></tr>
|
|
<tr>
|
|
<td>1</td>
|
|
<td><a href="/system/1/">Test Supercomputer</a>, Test Corp\nTest Site\nUSA</td>
|
|
<td>100000</td>
|
|
<td>100 PFLOP/s</td>
|
|
<td>150 PFLOP/s</td>
|
|
<td>5000</td>
|
|
</tr>
|
|
</table>
|
|
"""
|
|
result = collector.parse_response(response)
|
|
assert len(result) == 1
|
|
assert result[0]["source_id"] == "top500_1"
|
|
assert result[0]["name"] == "Test Supercomputer"
|
|
assert result[0]["country"] == "USA"
|
|
assert result[0]["metadata"]["rank"] == 1
|
|
assert "Test Corp" in result[0]["metadata"]["manufacturer"]
|
|
|
|
def test_parse_response_skips_invalid_item(self):
|
|
"""Test parsing skips items with missing data"""
|
|
collector = TOP500Collector()
|
|
response = """
|
|
<table class="top500-table">
|
|
<tr><th>Rank</th><th>System</th><th>Cores</th><th>Rmax</th><th>Rpeak</th><th>Power</th></tr>
|
|
<tr>
|
|
<td>1</td>
|
|
<td>Valid\nVendor\nSite\nUSA</td>
|
|
<td>1000</td>
|
|
<td>10 PFLOP/s</td>
|
|
<td>12 PFLOP/s</td>
|
|
<td>100</td>
|
|
</tr>
|
|
<tr>
|
|
<td>-</td>
|
|
<td>Invalid</td>
|
|
<td>1000</td>
|
|
<td>10 PFLOP/s</td>
|
|
<td>12 PFLOP/s</td>
|
|
<td>100</td>
|
|
</tr>
|
|
</table>
|
|
"""
|
|
result = collector.parse_response(response)
|
|
assert len(result) == 1
|
|
assert result[0]["name"] == "Valid"
|
|
|
|
|
|
class TestHTTPCollector:
|
|
"""Tests for HTTPCollector"""
|
|
|
|
def test_http_collector_attributes(self):
|
|
"""Test HTTP collector has correct default attributes via concrete class"""
|
|
collector = TOP500Collector()
|
|
assert collector.name == "top500"
|
|
assert collector.priority == "P0"
|
|
assert hasattr(collector, "fetch")
|
|
|
|
def test_collector_has_required_methods(self):
|
|
"""Test HTTP collector has required methods"""
|
|
collector = TOP500Collector()
|
|
assert hasattr(collector, "fetch")
|
|
assert hasattr(collector, "parse_response")
|
|
assert callable(collector.fetch)
|
|
assert callable(collector.parse_response)
|
|
|
|
|
|
class TestCelesTrakTLECollector:
|
|
def test_transform_uses_norad_as_source_id_and_preserves_starlink_group(self):
|
|
collector = CelesTrakTLECollector()
|
|
result = collector.transform([
|
|
{
|
|
"NORAD_CAT_ID": 44720,
|
|
"OBJECT_NAME": "STARLINK-1000",
|
|
"OBJECT_ID": "2019-029AZ",
|
|
"EPOCH": "2026-03-13T00:00:00Z",
|
|
"MEAN_MOTION": 15.79234567,
|
|
"ECCENTRICITY": 0.0001234,
|
|
"INCLINATION": 53.0,
|
|
"RA_OF_ASC_NODE": 10.0,
|
|
"ARG_OF_PERICENTER": 20.0,
|
|
"MEAN_ANOMALY": 30.0,
|
|
"_celestrak_query_group": "active",
|
|
"_celestrak_source_url": "https://celestrak.example/gp.php?GROUP=active&FORMAT=json",
|
|
}
|
|
])
|
|
|
|
assert result[0]["source_id"] == "44720"
|
|
assert result[0]["metadata"]["constellation_group"] == "starlink"
|
|
assert result[0]["metadata"]["celestrak_query_group"] == "active"
|
|
assert result[0]["metadata"]["norad_cat_id"] == 44720
|
|
assert result[0]["metadata"]["tle_line1"]
|
|
assert result[0]["metadata"]["tle_line2"]
|
|
|
|
def test_load_active_payload_rejects_invalid_records(self, tmp_path):
|
|
collector = CelesTrakTLECollector()
|
|
payload_path = tmp_path / "active.json"
|
|
payload_path.write_text(json.dumps([{"OBJECT_NAME": "missing norad"}]), encoding="utf-8")
|
|
|
|
with pytest.raises(RuntimeError, match="invalid record"):
|
|
collector._load_active_payload(payload_path)
|
|
|
|
def test_load_active_payload_accepts_complete_array(self, tmp_path):
|
|
collector = CelesTrakTLECollector()
|
|
payload_path = tmp_path / "active.json"
|
|
payload_path.write_text(
|
|
json.dumps([{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}]),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
records = collector._load_active_payload(payload_path)
|
|
|
|
assert records == [{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_retries_and_raises_instead_of_returning_partial_data(self, monkeypatch, tmp_path):
|
|
collector = CelesTrakTLECollector()
|
|
collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php"
|
|
attempts = 0
|
|
|
|
async def fake_download_file(*args, **kwargs):
|
|
nonlocal attempts
|
|
attempts += 1
|
|
raise RuntimeError("network interrupted")
|
|
|
|
async def fake_emit_business_log(*args, **kwargs):
|
|
return None
|
|
|
|
monkeypatch.setattr(collector._downloader, "download_file", fake_download_file)
|
|
monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log)
|
|
monkeypatch.setattr("app.services.collectors.celestrak.asyncio.sleep", AsyncMock())
|
|
|
|
with pytest.raises(RuntimeError, match="failed after retries"):
|
|
await collector.fetch()
|
|
|
|
assert attempts == 3
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_uses_cache_when_celestrak_reports_not_updated(self, monkeypatch, tmp_path):
|
|
collector = CelesTrakTLECollector()
|
|
collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php"
|
|
collector._downloader = ResumableFileDownloader(cache_namespace="celestrak-test", cache_root=tmp_path)
|
|
url = collector._active_url()
|
|
cached_path = collector._downloader.cached_file_path(url, ".json")
|
|
cached_path.parent.mkdir(parents=True, exist_ok=True)
|
|
cached_path.write_text(
|
|
json.dumps([{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}]),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
async def fake_download_file(*args, **kwargs):
|
|
raise DownloadHTTPStatusError(
|
|
url=url,
|
|
status_code=403,
|
|
body="GP data has not updated since your last successful download of GROUP=active.",
|
|
)
|
|
|
|
async def fake_emit_business_log(*args, **kwargs):
|
|
return None
|
|
|
|
monkeypatch.setattr(collector._downloader, "download_file", fake_download_file)
|
|
monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log)
|
|
|
|
records = await collector.fetch()
|
|
|
|
assert records[0]["NORAD_CAT_ID"] == 25544
|
|
assert records[0]["_celestrak_query_group"] == "active"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_not_updated_without_cache_does_not_retry(self, monkeypatch, tmp_path):
|
|
collector = CelesTrakTLECollector()
|
|
collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php"
|
|
collector._downloader = ResumableFileDownloader(cache_namespace="celestrak-test", cache_root=tmp_path)
|
|
attempts = 0
|
|
|
|
async def fake_download_file(*args, **kwargs):
|
|
nonlocal attempts
|
|
attempts += 1
|
|
raise DownloadHTTPStatusError(
|
|
url=collector._active_url(),
|
|
status_code=403,
|
|
body="GP data has not updated since your last successful download of GROUP=active.",
|
|
)
|
|
|
|
async def fake_emit_business_log(*args, **kwargs):
|
|
return None
|
|
|
|
monkeypatch.setattr(collector._downloader, "download_file", fake_download_file)
|
|
monkeypatch.setattr("app.services.collectors.celestrak.FALLBACK_GROUPS", ("starlink",))
|
|
monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log)
|
|
monkeypatch.setattr("app.services.collectors.celestrak.asyncio.sleep", AsyncMock())
|
|
|
|
with pytest.raises(RuntimeError, match="fallback group mode failed"):
|
|
await collector.fetch()
|
|
|
|
assert attempts == 2
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_falls_back_to_all_groups_when_active_not_updated_without_cache(self, monkeypatch, tmp_path):
|
|
collector = CelesTrakTLECollector()
|
|
collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php"
|
|
collector._downloader = ResumableFileDownloader(cache_namespace="celestrak-test", cache_root=tmp_path)
|
|
payload_by_group = {
|
|
"starlink": [{"NORAD_CAT_ID": 100, "OBJECT_NAME": "STARLINK-100"}],
|
|
"gps-ops": [{"NORAD_CAT_ID": 200, "OBJECT_NAME": "GPS BIIR-2"}],
|
|
}
|
|
|
|
async def fake_download_file(_client, url, **_kwargs):
|
|
if "GROUP=active" in url:
|
|
raise DownloadHTTPStatusError(
|
|
url=url,
|
|
status_code=403,
|
|
body="GP data has not updated since your last successful download of GROUP=active.",
|
|
)
|
|
group = "starlink" if "GROUP=starlink" in url else "gps-ops"
|
|
path = tmp_path / f"{group}.json"
|
|
path.write_text(json.dumps(payload_by_group[group]), encoding="utf-8")
|
|
return path
|
|
|
|
async def fake_emit_business_log(*args, **kwargs):
|
|
return None
|
|
|
|
monkeypatch.setattr(collector._downloader, "download_file", fake_download_file)
|
|
monkeypatch.setattr("app.services.collectors.celestrak.FALLBACK_GROUPS", tuple(payload_by_group))
|
|
monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log)
|
|
|
|
records = await collector.fetch()
|
|
|
|
assert [item["NORAD_CAT_ID"] for item in records] == [100, 200]
|
|
assert records[0]["_celestrak_query_group"] == "starlink"
|
|
assert records[1]["_celestrak_group"] == "gps-ops"
|
|
|
|
|
|
def test_aisstream_collector_is_registered():
|
|
collector = collector_registry.get("aisstream_vessels")
|
|
|
|
assert collector is not None
|
|
assert collector.data_type == "vessel_ais"
|
|
|
|
|
|
def test_supported_credential_collectors_have_guides_and_connectivity_provider():
|
|
missing: list[str] = []
|
|
for source, info in DEFAULT_DATASOURCES.items():
|
|
if not info.get("requires_credentials"):
|
|
continue
|
|
if info.get("credential_status") != "supported":
|
|
continue
|
|
|
|
provider = info.get("credential_provider")
|
|
if not provider:
|
|
missing.append(f"{source}: missing credential_provider")
|
|
continue
|
|
if provider not in DEFAULT_CREDENTIAL_GUIDES:
|
|
missing.append(f"{source}: missing credential guide for {provider}")
|
|
if provider not in SUPPORTED_CREDENTIAL_PROVIDERS:
|
|
missing.append(f"{source}: missing connectivity provider for {provider}")
|
|
|
|
assert missing == []
|