189 lines
6.7 KiB
Python
189 lines
6.7 KiB
Python
"""Earth layer adapter registry for datasource-backed refresh behavior."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EarthLayerAdapter:
|
|
sources: frozenset[str]
|
|
layers: tuple[str, ...]
|
|
cache_patterns: tuple[str, ...]
|
|
tables: frozenset[str] = field(default_factory=frozenset)
|
|
derived_models: tuple[str, ...] = field(default_factory=tuple)
|
|
refresh_strategy: str = "clear_then_reload"
|
|
|
|
|
|
EARTH_LAYER_ADAPTERS: tuple[EarthLayerAdapter, ...] = (
|
|
EarthLayerAdapter(
|
|
sources=frozenset({"barentswatch_vessels", "aisstream_vessels", "vessel_static", "vessel_position", "vessel_current_state", "ais_raw_observations", "ais_source_health"}),
|
|
tables=frozenset({"vessel_static", "vessel_position", "vessel_current_state", "ais_raw_observations", "ais_source_health"}),
|
|
layers=("vessels",),
|
|
cache_patterns=("vessels*", "summary*"),
|
|
derived_models=("ais_raw_observations", "ais_conflict_records", "ais_source_health"),
|
|
refresh_strategy="delta",
|
|
),
|
|
EarthLayerAdapter(
|
|
sources=frozenset(
|
|
{
|
|
"telegeography_cables",
|
|
"telegeography_landing",
|
|
"telegeography_landing_points",
|
|
"telegeography_systems",
|
|
"telegeography_cable_systems",
|
|
"arcgis_cables",
|
|
"arcgis_landing_points",
|
|
"arcgis_cable_landing_relation",
|
|
"arcgis_cable_landing_relations",
|
|
"fao_landing_points",
|
|
}
|
|
),
|
|
tables=frozenset({"collected_data"}),
|
|
layers=("cables",),
|
|
cache_patterns=("cables*", "landing-points*", "summary*"),
|
|
),
|
|
EarthLayerAdapter(
|
|
sources=frozenset({"celestrak_tle", "spacetrack_tle"}),
|
|
tables=frozenset({"collected_data"}),
|
|
layers=("satellites",),
|
|
cache_patterns=("satellites*", "summary*"),
|
|
),
|
|
EarthLayerAdapter(
|
|
sources=frozenset(
|
|
{
|
|
"top500",
|
|
"top500_supercomputers",
|
|
"epoch_ai_gpu",
|
|
"huggingface_models",
|
|
"huggingface_datasets",
|
|
"huggingface_spaces",
|
|
"compute_center_locations",
|
|
}
|
|
),
|
|
tables=frozenset({"compute_center_locations"}),
|
|
layers=("computeCenters",),
|
|
cache_patterns=("compute-centers*", "summary*"),
|
|
refresh_strategy="reload",
|
|
),
|
|
EarthLayerAdapter(
|
|
sources=frozenset(
|
|
{
|
|
"ris_live_bgp",
|
|
"bgpstream_bgp",
|
|
"iptoasn_prefix_geo",
|
|
"opengeofeed_prefix_geo",
|
|
"nro_delegated_prefix_geo",
|
|
"bgp_observations",
|
|
"bgp_anomalies",
|
|
"bgp_incidents",
|
|
"bgp_collector_locations",
|
|
}
|
|
),
|
|
tables=frozenset({"bgp_observations", "bgp_anomalies", "bgp_incidents", "bgp_collector_locations"}),
|
|
layers=("bgp",),
|
|
cache_patterns=("bgp*", "summary*"),
|
|
derived_models=("bgp_observations", "bgp_anomalies", "bgp_incidents"),
|
|
),
|
|
EarthLayerAdapter(
|
|
sources=frozenset({"news_live_streams"}),
|
|
tables=frozenset({"collected_data"}),
|
|
layers=("media",),
|
|
cache_patterns=("summary*",),
|
|
refresh_strategy="reload",
|
|
),
|
|
EarthLayerAdapter(
|
|
sources=frozenset({"media_news_archive", "earth_news_items"}),
|
|
tables=frozenset({"earth_news_items"}),
|
|
layers=("news",),
|
|
cache_patterns=("summary*",),
|
|
refresh_strategy="reload",
|
|
),
|
|
EarthLayerAdapter(
|
|
sources=frozenset({"earth_interactables"}),
|
|
tables=frozenset({"earth_interactables"}),
|
|
layers=("interactables",),
|
|
cache_patterns=("interactables*", "summary*"),
|
|
refresh_strategy="delta",
|
|
),
|
|
)
|
|
|
|
_ADAPTERS_BY_SOURCE = {
|
|
source: adapter
|
|
for adapter in EARTH_LAYER_ADAPTERS
|
|
for source in adapter.sources
|
|
}
|
|
_ADAPTERS_BY_TABLE = {
|
|
table: adapter
|
|
for adapter in EARTH_LAYER_ADAPTERS
|
|
for table in adapter.tables
|
|
}
|
|
|
|
|
|
def get_earth_layer_adapter_for_source(source: str | None) -> EarthLayerAdapter | None:
|
|
return _ADAPTERS_BY_SOURCE.get(str(source or "").strip())
|
|
|
|
|
|
def get_earth_layer_adapter_for_change(table: str | None, source: str | None) -> EarthLayerAdapter | None:
|
|
table_key = str(table or "").strip()
|
|
source_key = str(source or "").strip()
|
|
if table_key and table_key != "collected_data":
|
|
adapter = _ADAPTERS_BY_TABLE.get(table_key)
|
|
if adapter is not None:
|
|
return adapter
|
|
return get_earth_layer_adapter_for_source(source_key)
|
|
|
|
|
|
def get_earth_update_layers_for_source(source: str | None) -> list[str]:
|
|
adapter = get_earth_layer_adapter_for_source(source)
|
|
return list(adapter.layers) if adapter else []
|
|
|
|
|
|
def get_earth_update_layers_for_change(table: str | None, source: str | None) -> list[str]:
|
|
adapter = get_earth_layer_adapter_for_change(table, source)
|
|
return list(adapter.layers) if adapter else []
|
|
|
|
|
|
def get_earth_refresh_strategy_for_change(table: str | None, source: str | None) -> str | None:
|
|
adapter = get_earth_layer_adapter_for_change(table, source)
|
|
return adapter.refresh_strategy if adapter else None
|
|
|
|
|
|
def get_earth_cache_patterns_for_source(source: str | None) -> list[str]:
|
|
adapter = get_earth_layer_adapter_for_source(source)
|
|
return list(adapter.cache_patterns) if adapter else []
|
|
|
|
|
|
async def clear_derived_datasource_data(db: AsyncSession, source: str) -> dict[str, int]:
|
|
adapter = get_earth_layer_adapter_for_source(source)
|
|
if adapter is None or not adapter.derived_models:
|
|
return {}
|
|
|
|
from app.models.bgp_anomaly import BGPAnomaly
|
|
from app.models.bgp_incident import BGPIncident
|
|
from app.models.bgp_observation import BGPObservation
|
|
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth
|
|
|
|
model_by_key: dict[str, Any] = {
|
|
"bgp_observations": BGPObservation,
|
|
"bgp_anomalies": BGPAnomaly,
|
|
"bgp_incidents": BGPIncident,
|
|
"ais_raw_observations": AISRawObservation,
|
|
"ais_conflict_records": AISConflictRecord,
|
|
"ais_source_health": AISSourceHealth,
|
|
}
|
|
deleted_counts: dict[str, int] = {}
|
|
for key in adapter.derived_models:
|
|
model = model_by_key.get(key)
|
|
if model is None:
|
|
continue
|
|
if key == "ais_conflict_records":
|
|
result = await db.execute(model.__table__.delete().where(model.selected_source == source))
|
|
else:
|
|
result = await db.execute(model.__table__.delete().where(model.source == source))
|
|
deleted_counts[key] = int(result.rowcount or 0)
|
|
return deleted_counts
|