planet/backend/tests/test_datasources_batch.py

import asyncio
import inspect
from datetime import datetime, timedelta, timezone

from app.api.v1 import datasources as datasources_api
from app.models.datasource import DataSource
from app.models.task import CollectionTask
from app.services import data_jobs
from app.services import earth_layer_cache as earth_cache


def make_datasource(
    datasource_id: int,
    source: str,
    *,
    name: str | None = None,
    module: str = "L4",
    is_active: bool = True,
    last_status: str | None = None,
    last_run_at: datetime | None = None,
    frequency_minutes: int = 60,
) -> DataSource:
    return DataSource(
        id=datasource_id,
        name=name or source,
        source=source,
        module=module,
        priority="P1",
        frequency_minutes=frequency_minutes,
        collector_class=source,
        is_active=is_active,
        last_status=last_status,
        last_run_at=last_run_at,
    )


def test_datasource_product_key_groups_domain_specific_sources():
    assert datasources_api.datasource_product_key(make_datasource(1, "aisstream_vessels")) == "vessels"
    assert datasources_api.datasource_product_key(make_datasource(2, "telegeography_cables")) == "cables"
    assert datasources_api.datasource_product_key(make_datasource(3, "celestrak_tle")) == "satellites"
    assert datasources_api.datasource_product_key(make_datasource(4, "ris_live_bgp")) == "bgp"


def test_filter_datasources_by_product_status_and_collected_state():
    vessels = make_datasource(1, "aisstream_vessels", last_status="success")
    cables = make_datasource(2, "telegeography_cables", last_status="failed")
    filtered = datasources_api._filter_datasources_in_memory(
        [vessels, cables],
        running_tasks={},
        record_counts={"aisstream_vessels": 12, "telegeography_cables": 0},
        product="vessels",
        run_status="success",
        collected=True,
    )

    assert filtered == [vessels]


def test_serialize_datasource_row_can_skip_endpoint_resolution():
    datasource = make_datasource(7, "arcgis_cables", last_status="success", module="L2")

    class ExplodingConfig:
        def get_yaml_url(self, _source):
            raise AssertionError("endpoint resolution should be skipped")

    row = datasources_api.serialize_datasource_row(
        datasource,
        running_tasks={},
        latest_tasks={},
        record_counts={"arcgis_cables": 343},
        endpoint_overrides={},
        config=ExplodingConfig(),
        include_endpoint=False,
    )

    assert row["id"] == 7
    assert row["source"] == "arcgis_cables"
    assert row["module"] == "L2"
    assert row["last_status"] == "success"
    assert row["collected_records"] == 343
    assert row["has_collected_data"] is True
    assert "endpoint" not in row


def test_serialize_datasource_row_includes_endpoint_when_requested():
    datasource = make_datasource(8, "arcgis_landing_points")

    class Config:
        def get_yaml_url(self, source):
            return f"https://example.test/{source}"

    row = datasources_api.serialize_datasource_row(
        datasource,
        running_tasks={},
        latest_tasks={},
        record_counts={},
        endpoint_overrides={},
        config=Config(),
        include_endpoint=True,
    )

    assert row["endpoint"] == "https://example.test/arcgis_landing_points"


def test_serialize_datasource_row_separates_collector_running_from_delete_task():
    datasource = make_datasource(9, "top500", last_status="success")
    task = CollectionTask(
        id=44,
        datasource_id=9,
        source="top500",
        task_type="clear_data",
        status="running",
        phase="clearing_data",
    )

    row = datasources_api.serialize_datasource_row(
        datasource,
        running_tasks={9: task},
        latest_tasks={9: task},
        record_counts={"top500": 500},
        endpoint_overrides={},
        config=object(),
        include_endpoint=False,
    )

    assert row["is_task_active"] is True
    assert row["is_running"] is False
    assert row["task_type"] == "clear_data"
    assert row["task_status"] == "running"


def test_cancel_queued_delete_task_finishes_immediately(monkeypatch):
    task = CollectionTask(
        id=45,
        datasource_id=9,
        source="top500",
        task_type="clear_data",
        status="queued",
        phase="queued",
    )

    class FakeDb:
        async def commit(self):
            return None

        async def refresh(self, _task):
            return None

    async def fake_broadcast(_task):
        return None

    monkeypatch.setattr(data_jobs, "_broadcast_task_update", fake_broadcast)

    async def run():
        return await data_jobs.request_cancel_datasource_task(FakeDb(), task)

    result = asyncio.run(run())

    assert result.status == "cancelled"
    assert result.phase == "cancelled"
    assert result.completed_at is not None


def test_clear_data_job_relies_on_db_outbox_instead_of_extra_earth_refresh_task():
    source = inspect.getsource(data_jobs._run_clear_data_job)

    assert "enqueue_earth_refresh_job" not in source


def test_invalidate_earth_layer_cache_for_source_covers_datasource_aliases(monkeypatch):
    patterns: list[str] = []

    def fake_delete_pattern(pattern: str) -> int:
        patterns.append(pattern)
        return 1

    monkeypatch.setattr(earth_cache.earth_layer_cache, "delete_pattern", fake_delete_pattern)

    deleted = earth_cache.invalidate_earth_layer_cache_for_source("telegeography_cables")

    assert deleted == 3
    assert patterns == [
        "earth:layer:v1:cables*",
        "earth:layer:v1:landing-points*",
        "earth:layer:v1:summary*",
    ]


def test_trigger_datasource_batch_skips_disabled_and_frequency_window(monkeypatch):
    async def run():
        now = datetime.now(timezone.utc)
        disabled = make_datasource(1, "aisstream_vessels", is_active=False)
        not_due = make_datasource(2, "telegeography_cables", last_run_at=now, frequency_minutes=120)
        due = make_datasource(3, "ris_live_bgp", last_run_at=now - timedelta(hours=2))
        queued_sources: list[str] = []

        async def fake_running_tasks(_db, _ids):
            return {}

        async def fake_enqueue(_db, datasource, task_type, **_kwargs):
            queued_sources.append(datasource.source)
            return CollectionTask(id=100 + datasource.id, datasource_id=datasource.id, source=datasource.source, task_type=task_type, status="queued")

        monkeypatch.setattr(datasources_api, "_load_latest_running_tasks", fake_running_tasks)
        monkeypatch.setattr(
            datasources_api,
            "enqueue_datasource_job",
            fake_enqueue,
        )

        result = await datasources_api._trigger_datasource_batch(
            object(),
            [disabled, not_due, due],
            force=False,
        )

        assert [item["source"] for item in result["triggered"]] == ["ris_live_bgp"]
        assert {item["reason"] for item in result["skipped"]} == {"disabled", "within_frequency_window"}
        assert queued_sources == ["ris_live_bgp"]

    asyncio.run(run())