"""Visualization API - GeoJSON endpoints for 3D Earth display Unified API for all visualization data sources. Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium. """ import asyncio import base64 from collections import OrderedDict from datetime import UTC, datetime, timedelta import math import re import httpx from fastapi import APIRouter, HTTPException, Depends, Query, Response from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, func from typing import List, Dict, Any, Optional from app.core.collected_data_fields import get_record_field from app.core.satellite_tle import build_tle_lines_from_elements from app.core.time import to_iso8601_utc from app.db.session import get_db from app.models.bgp_anomaly import BGPAnomaly from app.models.bgp_incident import BGPIncident from app.models.bgp_observation import BGPObservation from app.models.collected_data import CollectedData from app.models.vessel import AISSourceHealth, VesselPosition, VesselStatic from app.services.bgp_collectors import build_bgp_collector_coverage from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance from app.services.compute_center_locations import ( RENDERABLE_PRECISIONS, ResolutionDiagnostic, build_compute_center_location_query, collect_location_candidates, refresh_compute_center_location_cache, resolve_compute_center_location_full, upsert_compute_center_location, ) from app.services.ai_client import get_ai_provider_client from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS from app.services.location.llm_fallback import collect_llm_location_fallback_candidate from app.services.persistent_logs import record_system_log from app.services.vessel_ais_aggregation import ( build_field_conflict_candidates, count_unique_raw_vessel_mmsi, get_aggregated_vessel, get_aggregated_vessel_track, get_aggregated_vessels, get_vessel_conflict_records, get_vessel_raw_observations, ) from app.core.logging import get_logger router = APIRouter() logger = get_logger(__name__, service="api") TERRAIN_TILE_URL_TEMPLATE = ( "https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png" ) TERRAIN_TILE_CACHE_MAX_ITEMS = 512 TERRAIN_TILE_BATCH_MAX_ITEMS = 128 TERRAIN_TILE_BATCH_CONCURRENCY = 16 _terrain_tile_cache: OrderedDict[tuple[int, int, int], tuple[bytes, str, dict[str, str]]] = OrderedDict() VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE) class TerrariumTileRequest(BaseModel): z: int = Field(ge=0, le=14) x: int = Field(ge=0) y: int = Field(ge=0) class TerrariumTileBatchRequest(BaseModel): tiles: List[TerrariumTileRequest] = Field(min_length=1, max_length=TERRAIN_TILE_BATCH_MAX_ITEMS) # ============== Converter Functions ============== def convert_cable_to_geojson(records: List[CollectedData]) -> Dict[str, Any]: """Convert cable records to GeoJSON FeatureCollection""" features = [] for record in records: metadata = record.extra_data or {} route_coords = metadata.get("route_coordinates", []) if not route_coords: continue all_lines = [] # Handle both old format (flat array) and new format (array of arrays) if route_coords and isinstance(route_coords[0], list): # New format: array of arrays (MultiLineString structure) if route_coords and isinstance(route_coords[0][0], list): # Array of arrays of arrays - multiple lines for line in route_coords: line_coords = [] for point in line: if len(point) >= 2: try: lon = float(point[0]) lat = float(point[1]) line_coords.append([lon, lat]) except (ValueError, TypeError): continue if len(line_coords) >= 2: all_lines.append(line_coords) else: # Old format: flat array of points - treat as single line line_coords = [] for point in route_coords: if len(point) >= 2: try: lon = float(point[0]) lat = float(point[1]) line_coords.append([lon, lat]) except (ValueError, TypeError): continue if len(line_coords) >= 2: all_lines.append(line_coords) if not all_lines: continue # Use MultiLineString format to preserve cable segments features.append( { "type": "Feature", "geometry": {"type": "MultiLineString", "coordinates": all_lines}, "properties": { "id": record.id, "cable_id": record.name, "source_id": record.source_id, "Name": record.name, "name": record.name, "owner": metadata.get("owners"), "owners": metadata.get("owners"), "rfs": metadata.get("rfs"), "RFS": metadata.get("rfs"), "status": metadata.get("status", "active"), "length": get_record_field(record, "value"), "length_km": get_record_field(record, "value"), "SHAPE__Length": get_record_field(record, "value"), "url": metadata.get("url"), "color": metadata.get("color"), "year": metadata.get("year"), }, } ) return {"type": "FeatureCollection", "features": features} def convert_landing_point_to_geojson(records: List[CollectedData], city_to_cable_ids_map: Dict[int, List[int]] = None, cable_id_to_name_map: Dict[int, str] = None) -> Dict[str, Any]: features = [] for record in records: try: latitude = get_record_field(record, "latitude") longitude = get_record_field(record, "longitude") lat = float(latitude) if latitude else None lon = float(longitude) if longitude else None except (ValueError, TypeError): continue if lat is None or lon is None: continue metadata = record.extra_data or {} city_id = metadata.get("city_id") props = { "id": record.id, "source_id": record.source_id, "name": record.name, "country": get_record_field(record, "country"), "city": get_record_field(record, "city"), "is_tbd": metadata.get("is_tbd", False), } cable_names = [] if city_to_cable_ids_map and city_id in city_to_cable_ids_map: for cable_id in city_to_cable_ids_map[city_id]: if cable_id_to_name_map and cable_id in cable_id_to_name_map: cable_names.append(cable_id_to_name_map[cable_id]) if cable_names: props["cable_names"] = cable_names features.append( { "type": "Feature", "geometry": {"type": "Point", "coordinates": [lon, lat]}, "properties": props, } ) return {"type": "FeatureCollection", "features": features} def convert_satellite_to_geojson(records: List[CollectedData]) -> Dict[str, Any]: """Convert satellite TLE records to GeoJSON""" features = [] for record in records: metadata = record.extra_data or {} norad_id = metadata.get("norad_cat_id") if not norad_id: continue tle_line1 = metadata.get("tle_line1") tle_line2 = metadata.get("tle_line2") if not tle_line1 or not tle_line2: tle_line1, tle_line2 = build_tle_lines_from_elements( norad_cat_id=norad_id, epoch=metadata.get("epoch"), inclination=metadata.get("inclination"), raan=metadata.get("raan"), eccentricity=metadata.get("eccentricity"), arg_of_perigee=metadata.get("arg_of_perigee"), mean_anomaly=metadata.get("mean_anomaly"), mean_motion=metadata.get("mean_motion"), ) constellation_group = _normalize_satellite_constellation_group( metadata.get("constellation_group"), record.name, ) footprint_policy = _get_satellite_footprint_policy(constellation_group) features.append( { "type": "Feature", "id": norad_id, "geometry": {"type": "Point", "coordinates": [0, 0, 0]}, "properties": { "id": record.id, "norad_cat_id": norad_id, "name": record.name, "constellation_group": constellation_group, "footprint_policy": footprint_policy, "international_designator": metadata.get("international_designator"), "epoch": metadata.get("epoch"), "inclination": metadata.get("inclination"), "raan": metadata.get("raan"), "eccentricity": metadata.get("eccentricity"), "arg_of_perigee": metadata.get("arg_of_perigee"), "mean_anomaly": metadata.get("mean_anomaly"), "mean_motion": metadata.get("mean_motion"), "bstar": metadata.get("bstar"), "classification_type": metadata.get("classification_type"), "tle_line1": tle_line1, "tle_line2": tle_line2, "data_type": "satellite_tle", }, } ) return {"type": "FeatureCollection", "features": features} def _normalize_satellite_constellation_group( raw_group: Any, name: Optional[str], ) -> Optional[str]: normalized_group = str(raw_group or "").strip().lower() if normalized_group: return normalized_group normalized_name = str(name or "").strip().upper() if normalized_name.startswith("STARLINK"): return "starlink" if normalized_name.startswith("IRIDIUM"): return "iridium-next" return None def _get_satellite_footprint_policy(constellation_group: Optional[str]) -> str: if constellation_group == "starlink": return "starlink_ground_footprint" if constellation_group == "iridium-next": return "iridium_coverage_ring" return "none" def _current_collected_data_stmt(source: str): return ( select(CollectedData) .where(CollectedData.source == source) .where(CollectedData.is_current.is_(True)) .order_by(CollectedData.id.desc()) ) async def _load_current_collected_data( db: AsyncSession, source: str, *, exclude_unknown_name: bool = False, limit: Optional[int] = None, ) -> List[CollectedData]: stmt = _current_collected_data_stmt(source) if exclude_unknown_name: stmt = stmt.where(CollectedData.name != "Unknown") if limit is not None: stmt = stmt.limit(limit) result = await db.execute(stmt) return list(result.scalars().all()) async def _latest_task_id_for_source( db: AsyncSession, source: str, *, exclude_unknown_name: bool = False, ) -> int | None: stmt = ( select( CollectedData.task_id, func.max(CollectedData.collected_at).label("latest_collected_at"), func.max(CollectedData.id).label("latest_id"), ) .where(CollectedData.source == source) .where(CollectedData.task_id.isnot(None)) .group_by(CollectedData.task_id) .order_by(func.max(CollectedData.collected_at).desc(), func.max(CollectedData.id).desc()) .limit(1) ) if exclude_unknown_name: stmt = stmt.where(CollectedData.name != "Unknown") result = await db.execute(stmt) row = result.first() return int(row.task_id) if row and row.task_id is not None else None async def _load_current_or_latest_task_data( db: AsyncSession, source: str, *, exclude_unknown_name: bool = False, limit: Optional[int] = None, ) -> List[CollectedData]: records = await _load_current_collected_data( db, source, exclude_unknown_name=exclude_unknown_name, limit=limit, ) if records: return records latest_task_id = await _latest_task_id_for_source( db, source, exclude_unknown_name=exclude_unknown_name, ) if latest_task_id is None: return [] stmt = ( select(CollectedData) .where(CollectedData.source == source) .where(CollectedData.task_id == latest_task_id) .order_by(CollectedData.id.desc()) ) if exclude_unknown_name: stmt = stmt.where(CollectedData.name != "Unknown") if limit is not None: stmt = stmt.limit(limit) result = await db.execute(stmt) return list(result.scalars().all()) async def _count_current_or_latest_task_data( db: AsyncSession, source: str, *, exclude_unknown_name: bool = False, ) -> int: current_stmt = ( select(func.count(CollectedData.id)) .where(CollectedData.source == source) .where(CollectedData.is_current.is_(True)) ) if exclude_unknown_name: current_stmt = current_stmt.where(CollectedData.name != "Unknown") current_result = await db.execute(current_stmt) current_scalar = current_result.scalar() if current_scalar is None and hasattr(current_result, "scalars"): current_rows = current_result.scalars().all() current_count = sum( 1 for row in current_rows if getattr(row, "source", None) == source and (not exclude_unknown_name or getattr(row, "name", None) != "Unknown") ) else: current_count = int(current_scalar or 0) if current_count > 0: return current_count latest_task_id = await _latest_task_id_for_source( db, source, exclude_unknown_name=exclude_unknown_name, ) if latest_task_id is None: return 0 latest_stmt = ( select(func.count(CollectedData.id)) .where(CollectedData.source == source) .where(CollectedData.task_id == latest_task_id) ) if exclude_unknown_name: latest_stmt = latest_stmt.where(CollectedData.name != "Unknown") latest_result = await db.execute(latest_stmt) return int(latest_result.scalar() or 0) async def _load_current_collected_data_by_sources( db: AsyncSession, sources: List[str], ) -> Dict[str, List[CollectedData]]: if not sources: return {} stmt = ( select(CollectedData) .where(CollectedData.source.in_(sources)) .where(CollectedData.is_current.is_(True)) .order_by(CollectedData.source.asc(), CollectedData.id.desc()) ) result = await db.execute(stmt) grouped_records: Dict[str, List[CollectedData]] = {source: [] for source in sources} for record in result.scalars().all(): grouped_records.setdefault(record.source, []).append(record) return grouped_records def _build_landing_point_cable_maps( relation_records: List[CollectedData], cable_records: List[CollectedData], ) -> tuple[Dict[int, List[int]], Dict[int, str]]: city_to_cable_ids_map: Dict[int, List[int]] = {} for relation_record in relation_records: if not relation_record.extra_data: continue city_id = relation_record.extra_data.get("city_id") cable_id = relation_record.extra_data.get("cable_id") if city_id is None or cable_id is None: continue city_to_cable_ids_map.setdefault(city_id, []) if cable_id not in city_to_cable_ids_map[city_id]: city_to_cable_ids_map[city_id].append(cable_id) cable_id_to_name_map: Dict[int, str] = {} for cable_record in cable_records: if not cable_record.extra_data: continue cable_id = cable_record.extra_data.get("cable_id") cable_name = cable_record.name if cable_id and cable_name: cable_id_to_name_map[cable_id] = cable_name return city_to_cable_ids_map, cable_id_to_name_map def _filter_known_records(records: List[CollectedData]) -> List[CollectedData]: return [record for record in records if record.name != "Unknown"] def convert_supercomputer_to_geojson(records: List[CollectedData]) -> Dict[str, Any]: """Convert TOP500 supercomputer records to GeoJSON""" features = [] for record in records: try: latitude = get_record_field(record, "latitude") longitude = get_record_field(record, "longitude") lat = float(latitude) if latitude and latitude != "0.0" else None lon = ( float(longitude) if longitude and longitude != "0.0" else None ) except (ValueError, TypeError): lat, lon = None, None metadata = record.extra_data or {} features.append( { "type": "Feature", "id": record.id, "geometry": {"type": "Point", "coordinates": [lon or 0, lat or 0]}, "properties": { "id": record.id, "name": record.name, "rank": metadata.get("rank"), "r_max": get_record_field(record, "rmax"), "r_peak": get_record_field(record, "rpeak"), "cores": get_record_field(record, "cores"), "power": get_record_field(record, "power"), "country": get_record_field(record, "country"), "city": get_record_field(record, "city"), "data_type": "supercomputer", }, } ) return {"type": "FeatureCollection", "features": features} def convert_gpu_cluster_to_geojson(records: List[CollectedData]) -> Dict[str, Any]: """Convert GPU cluster records to GeoJSON""" features = [] for record in records: try: latitude = get_record_field(record, "latitude") longitude = get_record_field(record, "longitude") lat = float(latitude) if latitude else None lon = float(longitude) if longitude else None except (ValueError, TypeError): lat, lon = None, None metadata = record.extra_data or {} features.append( { "type": "Feature", "id": record.id, "geometry": {"type": "Point", "coordinates": [lon or 0, lat or 0]}, "properties": { "id": record.id, "name": record.name, "country": get_record_field(record, "country"), "city": get_record_field(record, "city"), "metadata": metadata, "data_type": "gpu_cluster", }, } ) return {"type": "FeatureCollection", "features": features} def _parse_float(value: Any) -> Optional[float]: try: if value in (None, ""): return None return float(value) except (TypeError, ValueError): return None def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str) -> str: if capacity_value is None: return "unknown" unit = str(capacity_unit or "").strip().lower() if unit in {"pflop/s", "pflops", "pflop"}: normalized_tflops = capacity_value * 1000 elif unit in {"gflop/s", "gflops", "gflop"}: normalized_tflops = capacity_value else: normalized_tflops = capacity_value if normalized_tflops >= 1_000_000: return "exascale" if normalized_tflops >= 100_000: return "ultra" if normalized_tflops >= 10_000: return "large" if normalized_tflops > 0: return "regional" return "unknown" def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str, Any]: """Convert compute infrastructure records into a unified GeoJSON layer. Records that cannot be resolved to at least city-level precision are NOT silently dropped: they are returned in ``unresolved`` so the UI can offer the click-to-collect coordinate flow. The features list never contains ``[0, 0]`` placeholders or country/region/unknown precision points. """ features: List[Dict[str, Any]] = [] unresolved: List[Dict[str, Any]] = [] for record in records: metadata = record.extra_data or {} result = resolve_compute_center_location_full(record, metadata) site_type = ( "supercomputer" if record.source == "top500" or record.data_type == "supercomputer" else "gpu_cluster" ) if not result.is_resolved: diagnostic = result.diagnostic or ResolutionDiagnostic( failure_reason="Unknown resolver failure", attempted_queries=(), record_id=getattr(record, "id", None), source=getattr(record, "source", None), source_id=getattr(record, "source_id", None), name=getattr(record, "name", None), ) unresolved.append({ **diagnostic.to_dict(), "site_type": site_type, }) continue location = result.location if location is None or not location.is_renderable: # Defensive: should not happen because is_resolved guards this. continue location_props = location.to_geojson_properties() latitude = location.latitude longitude = location.longitude if site_type == "supercomputer": capacity_value = _parse_float(get_record_field(record, "rmax")) capacity_unit = "GFlops" else: capacity_value = _parse_float(get_record_field(record, "value")) capacity_unit = str(get_record_field(record, "unit") or "TFlop/s") vendor = ( metadata.get("manufacturer") or metadata.get("vendor") or metadata.get("gpu_type") ) operator = ( metadata.get("organization") or metadata.get("operator") or metadata.get("owner") ) rank = metadata.get("rank") if rank in (None, "") and site_type == "supercomputer": rank = get_record_field(record, "rank") updated_at = to_iso8601_utc(record.reference_date or record.collected_at) features.append( { "type": "Feature", "id": record.id, "geometry": { "type": "Point", "coordinates": [longitude, latitude], }, "properties": { "id": record.id, "source_id": record.source_id, "name": record.name, "site_type": site_type, "country": get_record_field(record, "country") or location.country, "city": get_record_field(record, "city") or location.city, "region": location.region, "latitude": latitude, "longitude": longitude, "operator": operator, "vendor": vendor, "capacity_value": capacity_value, "capacity_unit": capacity_unit, "capacity_band": _normalize_capacity_band(capacity_value, capacity_unit), "rank": rank, "gpu_count": metadata.get("gpu_count"), "gpu_type": metadata.get("gpu_type"), "cores": get_record_field(record, "cores"), "power": get_record_field(record, "power"), "source": record.source, "updated_at": updated_at, "status": "observed", **location_props, "data_type": "compute_center", "metadata": metadata, }, } ) return {"type": "FeatureCollection", "features": features, "unresolved": unresolved} VESSEL_TYPE_FILTERS = { "cargo": lambda props: str(props.get("vessel_type_name", "")).lower() == "cargo" or 70 <= int(props.get("vessel_type") or -1) <= 79, "tanker": lambda props: str(props.get("vessel_type_name", "")).lower() == "tanker" or 80 <= int(props.get("vessel_type") or -1) <= 89, "passenger": lambda props: str(props.get("vessel_type_name", "")).lower() == "passenger" or 60 <= int(props.get("vessel_type") or -1) <= 69, "fishing": lambda props: str(props.get("vessel_type_name", "")).lower() == "fishing" or int(props.get("vessel_type") or -1) == 30, "military": lambda props: str(props.get("vessel_type_name", "")).lower() == "military" or int(props.get("vessel_type") or -1) == 35, "other": lambda props: str(props.get("vessel_type_name", "")).lower() not in {"cargo", "tanker", "passenger", "fishing", "military"}, } def convert_vessels_to_geojson(rows: List[Any]) -> Dict[str, Any]: features = [] seen_mmsi: set[int] = set() for position, static in rows: if position.lat is None or position.lon is None: continue if position.mmsi in seen_mmsi: continue seen_mmsi.add(position.mmsi) props = { "mmsi": position.mmsi, "mmsi_display": str(position.mmsi), "name": getattr(static, "name", None) or f"MMSI {position.mmsi}", "name_is_fallback": _is_vessel_name_fallback(getattr(static, "name", None), position.mmsi), "callsign": getattr(static, "callsign", None), "imo": getattr(static, "imo", None), "imo_display": str(getattr(static, "imo")) if getattr(static, "imo", None) else None, "vessel_type": getattr(static, "vessel_type", None), "vessel_type_name": getattr(static, "vessel_type_name", None) or "Other", "flag": getattr(static, "flag", None), "length": getattr(static, "length", None), "width": getattr(static, "width", None), "draught": getattr(static, "draught", None), "sog": position.sog, "cog": position.cog, "heading": position.heading, "nav_status": position.nav_status, "received_at": to_iso8601_utc(position.received_at), "data_type": "vessel", } features.append( { "type": "Feature", "id": position.mmsi, "geometry": { "type": "Point", "coordinates": [position.lon, position.lat], }, "properties": props, } ) return {"type": "FeatureCollection", "features": features} def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict[str, Any]: features = [] for vessel in vessels: if vessel.get("lat") is None or vessel.get("lon") is None: continue source_summary = {} for source, summary in (vessel.get("source_summary") or {}).items(): source_summary[source] = { **summary, "latest_observed_at": to_iso8601_utc(summary.get("latest_observed_at")), } props = { "mmsi": vessel["mmsi"], "mmsi_display": str(vessel["mmsi"]), "name": vessel.get("name") or f"MMSI {vessel['mmsi']}", "name_is_fallback": _is_vessel_name_fallback(vessel.get("name"), vessel["mmsi"]), "callsign": vessel.get("callsign"), "imo": vessel.get("imo"), "imo_display": str(vessel.get("imo")) if vessel.get("imo") else None, "vessel_type": vessel.get("vessel_type"), "vessel_type_name": vessel.get("vessel_type_name") or "Other", "flag": vessel.get("flag"), "length": vessel.get("length"), "width": vessel.get("width"), "draught": vessel.get("draught"), "sog": vessel.get("sog"), "cog": vessel.get("cog"), "heading": vessel.get("heading"), "nav_status": vessel.get("nav_status"), "received_at": to_iso8601_utc(vessel.get("received_at")), "field_sources": vessel.get("field_sources") or {}, "selected_reasons": vessel.get("selected_reasons") or {}, "source_summary": source_summary, "quality_flags": vessel.get("quality_flags") or [], "conflict_count": vessel.get("conflict_count", 0), "aggregation_strategy_version": vessel.get("aggregation_strategy_version", 0), "data_type": "vessel", } features.append( { "type": "Feature", "id": vessel["mmsi"], "geometry": { "type": "Point", "coordinates": [vessel["lon"], vessel["lat"]], }, "properties": props, } ) return {"type": "FeatureCollection", "features": features} def _parse_bbox(value: Optional[str]) -> tuple[float, float, float, float] | None: if not value: return None parts = [part.strip() for part in value.split(",")] if len(parts) != 4: raise HTTPException(status_code=400, detail="bbox must be lon_min,lat_min,lon_max,lat_max") try: lon_min, lat_min, lon_max, lat_max = [float(part) for part in parts] except ValueError as exc: raise HTTPException(status_code=400, detail="bbox values must be numbers") from exc if lat_min > lat_max: lat_min, lat_max = lat_max, lat_min if lon_min > lon_max: lon_min, lon_max = lon_max, lon_min return lon_min, lat_min, lon_max, lat_max def _is_vessel_name_fallback(name: Any, mmsi: Any) -> bool: text = str(name or "").strip() mmsi_text = str(mmsi or "").strip() if not text: return True if mmsi_text and text == mmsi_text: return True return bool(VESSEL_NAME_FALLBACK_PATTERN.match(text)) def _requested_vessel_types(value: Optional[str]) -> set[str]: return { item.strip().lower() for item in (value or "").split(",") if item.strip() } def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bool: if not requested_types: return True for requested_type in requested_types: predicate = VESSEL_TYPE_FILTERS.get(requested_type) if predicate and predicate(props): return True return False def _feature_mmsi_key(feature: dict[str, Any]) -> str | None: props = feature.get("properties", {}) mmsi = props.get("mmsi") or feature.get("id") if mmsi in (None, ""): return None return str(mmsi) def _feature_in_bbox(feature: dict[str, Any], bbox: tuple[float, float, float, float] | None) -> bool: if bbox is None: return True coordinates = feature.get("geometry", {}).get("coordinates") or [] if len(coordinates) < 2: return False try: lon = float(coordinates[0]) lat = float(coordinates[1]) except (TypeError, ValueError): return False lon_min, lat_min, lon_max, lat_max = bbox return lon_min <= lon <= lon_max and lat_min <= lat <= lat_max def _filter_vessel_features( features: list[dict[str, Any]], *, bbox: tuple[float, float, float, float] | None, requested_types: set[str], ) -> list[dict[str, Any]]: return [ feature for feature in features if _feature_in_bbox(feature, bbox) and _matches_vessel_type(feature.get("properties", {}), requested_types) ] def _merge_vessel_features( raw_features: list[dict[str, Any]], legacy_features: list[dict[str, Any]], ) -> tuple[list[dict[str, Any]], dict[str, Any]]: """Prefer aggregated raw observations as the canonical source of truth. Legacy `vessel_position` rows only fill MMSIs that the unified pipeline does not yet know about, so a vessel never appears twice when both BarentsWatch and AISStream observe it. Once the legacy table drains, this branch becomes a no-op. """ merged: list[dict[str, Any]] = [] seen: set[str] = set() raw_keys: set[str] = set() legacy_keys: set[str] = set() for feature in raw_features: key = _feature_mmsi_key(feature) if key is None or key in seen: continue seen.add(key) raw_keys.add(key) merged.append(feature) legacy_added = 0 for feature in legacy_features: key = _feature_mmsi_key(feature) if key is None: continue legacy_keys.add(key) if key in seen: continue seen.add(key) legacy_added += 1 merged.append(feature) return merged, { "raw_unique_mmsi": len(raw_keys), "legacy_unique_mmsi": len(legacy_keys), "legacy_backfilled_mmsi": legacy_added, "final_unique_mmsi": len(seen), } def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]: by_type: dict[str, int] = {} underway = 0 anchored_or_moored = 0 for feature in features: props = feature.get("properties", {}) vessel_type = str(props.get("vessel_type_name") or "Other") by_type[vessel_type] = by_type.get(vessel_type, 0) + 1 nav_status = props.get("nav_status") if nav_status in (1, 5): anchored_or_moored += 1 else: underway += 1 return { "total": len(features), "by_type": by_type, "underway": underway, "anchored_or_moored": anchored_or_moored, } def convert_bgp_anomalies_to_geojson( records: List[BGPAnomaly], geography_hints: Optional[Dict[str, Dict[str, Any]]] = None, ) -> Dict[str, Any]: features = [] geography_hints = geography_hints or {} for record in records: evidence = record.evidence or {} hint = geography_hints.get(str(record.entity_key or record.id), {}) collectors = evidence.get("collectors") or record.peer_scope or [] if not collectors: nested = evidence.get("events") or [] collectors = [ str((item or {}).get("collector") or "").strip() for item in nested if (item or {}).get("collector") ] collectors = [collector for collector in collectors if collector] if not collectors: collectors = [] as_path = [] if isinstance(evidence.get("as_path"), list): as_path = evidence.get("as_path") or [] if not as_path: nested = evidence.get("events") or [] for item in nested: candidate_path = (item or {}).get("as_path") if isinstance(candidate_path, list) and candidate_path: as_path = candidate_path break impacted_regions = [] seen_regions = set() for collector_name in collectors: collector_location = RIPE_RIS_COLLECTOR_COORDS.get(str(collector_name)) if not collector_location: continue region_key = ( collector_location.get("country"), collector_location.get("city"), ) if region_key in seen_regions: continue seen_regions.add(region_key) impacted_regions.append( { "collector": collector_name, "country": collector_location.get("country"), "city": collector_location.get("city"), "latitude": collector_location.get("latitude"), "longitude": collector_location.get("longitude"), } ) geography_regions = _normalize_geo_regions(hint.get("regions") or []) geography_mode = hint.get("geography_mode") or "collector_centroid" collector = collectors[0] if collectors else None location = geography_regions[0] if geography_regions else None if location is None and collector: location = RIPE_RIS_COLLECTOR_COORDS.get(str(collector)) if location is None: nested = evidence.get("events") or [] for item in nested: collector_name = (item or {}).get("collector") if collector_name and collector_name in RIPE_RIS_COLLECTOR_COORDS: location = RIPE_RIS_COLLECTOR_COORDS[collector_name] collector = collector_name geography_mode = "collector_centroid" break if location is None: continue features.append( { "type": "Feature", "geometry": { "type": "Point", "coordinates": [location["longitude"], location["latitude"]], }, "properties": { "id": record.id, "collector": collector, "city": location.get("city"), "country": location.get("country"), "source": record.source, "anomaly_type": record.anomaly_type, "severity": record.severity, "status": record.status, "prefix": record.prefix, "origin_asn": record.origin_asn, "new_origin_asn": record.new_origin_asn, "collectors": collectors, "collector_count": len(collectors) or 1, "as_path": as_path, "impacted_regions": impacted_regions, "geography_mode": geography_mode, "confidence": record.confidence, "summary": record.summary, "created_at": to_iso8601_utc(record.created_at), }, } ) return {"type": "FeatureCollection", "features": features} async def build_anomaly_geography_hints( db: AsyncSession, records: List[BGPAnomaly], ) -> Dict[str, Dict[str, Any]]: hints: Dict[str, Dict[str, Any]] = {} for record in records: hint = _extract_evidence_geography_hint(record.evidence or {}) if hint: hints[str(record.entity_key or record.id)] = hint return hints def convert_bgp_collectors_to_geojson( coverage_by_collector: Dict[str, Dict[str, Any]] | None = None, ) -> Dict[str, Any]: features = [] coverage_by_collector = coverage_by_collector or {} for collector, location in sorted(RIPE_RIS_COLLECTOR_COORDS.items()): coverage = coverage_by_collector.get(collector, {}) features.append( { "type": "Feature", "geometry": { "type": "Point", "coordinates": [location["longitude"], location["latitude"]], }, "properties": { "collector": collector, "city": coverage.get("city") or location.get("city"), "country": coverage.get("country") or location.get("country"), "status": "online", "observation_count": coverage.get("observation_count", 0), "prefix_count": coverage.get("prefix_count", 0), "origin_asn_count": coverage.get("origin_asn_count", 0), "peer_asn_count": coverage.get("peer_asn_count", 0), "recent_15m_observation_count": coverage.get("recent_15m_observation_count", 0), "recent_24h_observation_count": coverage.get("recent_24h_observation_count", 0), "recent_7d_observation_count": coverage.get("recent_7d_observation_count", 0), "recent_15m_prefix_count": coverage.get("recent_15m_prefix_count", 0), "recent_24h_prefix_count": coverage.get("recent_24h_prefix_count", 0), "recent_7d_prefix_count": coverage.get("recent_7d_prefix_count", 0), "top_event_types": coverage.get("top_event_types", []), "latest_observed_at": coverage.get("latest_observed_at"), "latest_event_type": coverage.get("latest_event_type"), "baseline_scope": coverage.get( "baseline_scope", { "countries": [location.get("country")] if location.get("country") else [], "cities": [location.get("city")] if location.get("city") else [], }, ), }, } ) return {"type": "FeatureCollection", "features": features} def _incident_estimated_center(valid_regions: List[Dict[str, Any]]) -> Dict[str, float]: x = 0.0 y = 0.0 z = 0.0 for region in valid_regions: lat_rad = math.radians(float(region["latitude"])) lon_rad = math.radians(float(region["longitude"])) x += math.cos(lat_rad) * math.cos(lon_rad) y += math.cos(lat_rad) * math.sin(lon_rad) z += math.sin(lat_rad) total = float(len(valid_regions)) if total <= 0: return {"latitude": 0.0, "longitude": 0.0} x /= total y /= total z /= total hyp = math.sqrt((x * x) + (y * y)) if hyp == 0: return {"latitude": 0.0, "longitude": 0.0} return { "latitude": math.degrees(math.atan2(z, hyp)), "longitude": math.degrees(math.atan2(y, x)), } def _incident_estimated_radius_km(center: Dict[str, float], valid_regions: List[Dict[str, Any]]) -> float: center_coords = (float(center["longitude"]), float(center["latitude"])) distances = [ haversine_distance( center_coords, (float(region["longitude"]), float(region["latitude"])), ) for region in valid_regions ] return round(max(distances) if distances else 0.0, 1) def _normalize_geo_regions(regions: List[Dict[str, Any]]) -> List[Dict[str, Any]]: normalized: list[dict[str, Any]] = [] seen: set[tuple[Any, ...]] = set() for region in regions: if not isinstance(region, dict): continue latitude = region.get("latitude") longitude = region.get("longitude") if not isinstance(latitude, (int, float)) or not isinstance(longitude, (int, float)): continue item = { "collector": region.get("collector"), "country": region.get("country"), "city": region.get("city"), "latitude": float(latitude), "longitude": float(longitude), } key = ( item["collector"], item["country"], item["city"], item["latitude"], item["longitude"], ) if key in seen: continue seen.add(key) normalized.append(item) return normalized def _extract_evidence_geography_hint(evidence: Dict[str, Any]) -> Dict[str, Any] | None: prefix_geo_regions = [] prefix_regions = [] asn_regions = [] evidence_prefix_geography = evidence.get("prefix_geography") or {} prefix_geo_regions.extend( _normalize_geo_regions(evidence_prefix_geography.get("regions") or []) ) prefix_scope = evidence.get("prefix_scope") or {} prefix_regions.extend(_normalize_geo_regions(prefix_scope.get("regions") or [])) for profile_key in ("origin_asn_profile", "new_origin_asn_profile"): profile = evidence.get(profile_key) or {} latitude = profile.get("latitude") longitude = profile.get("longitude") if isinstance(latitude, (int, float)) and isinstance(longitude, (int, float)): asn_regions.append( { "country": profile.get("country"), "city": profile.get("city"), "latitude": float(latitude), "longitude": float(longitude), } ) prefix_geo_regions = _normalize_geo_regions(prefix_geo_regions) prefix_regions = _normalize_geo_regions(prefix_regions) asn_regions = _normalize_geo_regions(asn_regions) if prefix_geo_regions: return {"regions": prefix_geo_regions, "geography_mode": "prefix_geography"} if prefix_regions: return {"regions": prefix_regions, "geography_mode": "prefix_scope"} if asn_regions: return {"regions": asn_regions, "geography_mode": "asn_region"} return None async def build_incident_geography_hints( db: AsyncSession, records: List[BGPIncident], ) -> Dict[str, Dict[str, Any]]: evidence_refs = sorted( { str(ref) for record in records for ref in (record.evidence_refs or []) if ref } ) anomalies = [] if evidence_refs: result = await db.execute( select(BGPAnomaly).where(BGPAnomaly.entity_key.in_(evidence_refs)) ) anomalies = result.scalars().all() anomaly_by_key = { str(anomaly.entity_key): anomaly for anomaly in anomalies if anomaly.entity_key } hints: Dict[str, Dict[str, Any]] = {} for record in records: merged_hint: Dict[str, Any] | None = None priority = {"prefix_geography": 3, "prefix_scope": 2, "asn_region": 1} for ref in record.evidence_refs or []: anomaly = anomaly_by_key.get(str(ref)) if anomaly is None: continue hint = _extract_evidence_geography_hint(anomaly.evidence or {}) if hint is None: continue if merged_hint is None: merged_hint = { "regions": list(hint["regions"]), "geography_mode": hint["geography_mode"], } continue if priority[hint["geography_mode"]] > priority[merged_hint["geography_mode"]]: merged_hint = { "regions": list(hint["regions"]), "geography_mode": hint["geography_mode"], } elif priority[hint["geography_mode"]] == priority[merged_hint["geography_mode"]]: merged_hint["regions"].extend(hint["regions"]) if merged_hint: merged_hint["regions"] = _normalize_geo_regions(merged_hint["regions"]) hints[record.incident_key] = merged_hint return hints def convert_bgp_incidents_to_geojson( records: List[BGPIncident], geography_hints: Optional[Dict[str, Dict[str, Any]]] = None, ) -> Dict[str, Any]: features = [] for record in records: hint = (geography_hints or {}).get(record.incident_key, {}) regions = hint.get("regions") or (record.affected_regions or []) if not regions: continue valid_regions = _normalize_geo_regions(regions) if not valid_regions: continue estimated_center = _incident_estimated_center(valid_regions) estimated_radius_km = _incident_estimated_radius_km(estimated_center, valid_regions) features.append( { "type": "Feature", "geometry": { "type": "Point", "coordinates": [ estimated_center["longitude"], estimated_center["latitude"], ], }, "properties": { "id": record.id, "incident_key": record.incident_key, "incident_type": record.incident_type, "title": record.title, "summary": record.summary, "severity": record.severity, "status": record.status, "confidence": record.confidence, "affected_prefixes": record.affected_prefixes or [], "affected_asns": record.affected_asns or [], "affected_collectors": record.affected_collectors or [], "affected_regions": valid_regions, "estimated_center": estimated_center, "estimated_radius_km": estimated_radius_km, "geography_mode": hint.get("geography_mode") or "collector_centroid", "related_cables": record.related_cables or [], "related_ixps": record.related_ixps or [], "created_at": to_iso8601_utc(record.created_at), "started_at": to_iso8601_utc(record.started_at), }, } ) return {"type": "FeatureCollection", "features": features} # ============== API Endpoints ============== @router.get("/geo/cables") async def get_cables_geojson(db: AsyncSession = Depends(get_db)): """获取海底电缆 GeoJSON 数据 (LineString)""" try: records = await _load_current_collected_data(db, "arcgis_cables") if not records: raise HTTPException( status_code=404, detail="No cable data found. Please run the arcgis_cables collector first.", ) return convert_cable_to_geojson(records) except HTTPException: raise except Exception as e: logger.exception_event( "Failed to build cables GeoJSON response", event="visualization.cables.load_failed", context={"error": str(e)}, ) await record_system_log( source="backend", service="api", module=__name__, event="visualization.cables.load_failed", level="error", message="Failed to build cables GeoJSON response", category="visualization", context={"error": str(e)}, ) raise HTTPException(status_code=500, detail=f"Internal error: {str(e)}") @router.get("/geo/landing-points") async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)): try: records_by_source = await _load_current_collected_data_by_sources( db, [ "arcgis_landing_points", "arcgis_cable_landing_relation", "arcgis_cables", ], ) records = records_by_source.get("arcgis_landing_points", []) relation_records = records_by_source.get( "arcgis_cable_landing_relation", [], ) cable_records = records_by_source.get("arcgis_cables", []) city_to_cable_ids_map, cable_id_to_name_map = _build_landing_point_cable_maps( relation_records, cable_records, ) if not records: raise HTTPException( status_code=404, detail="No landing point data found. Please run the arcgis_landing_points collector first.", ) return convert_landing_point_to_geojson(records, city_to_cable_ids_map, cable_id_to_name_map) except HTTPException: raise except Exception as e: logger.exception_event( "Failed to build landing points GeoJSON response", event="visualization.landing_points.load_failed", context={"error": str(e)}, ) await record_system_log( source="backend", service="api", module=__name__, event="visualization.landing_points.load_failed", level="error", message="Failed to build landing points GeoJSON response", category="visualization", context={"error": str(e)}, ) raise HTTPException(status_code=500, detail=f"Internal error: {str(e)}") @router.get("/terrain/terrarium/{z}/{x}/{y}.png") async def get_terrarium_tile(z: int, x: int, y: int): """Proxy Terrarium elevation tiles through the backend to avoid browser CORS issues.""" if not _is_valid_terrain_tile(z, x, y): raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates") try: async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client: content, content_type, headers = await _fetch_terrain_tile(client, z, x, y) except httpx.HTTPStatusError as exc: raise HTTPException( status_code=exc.response.status_code, detail=f"Terrain tile upstream error: {exc.response.status_code}", ) from exc except httpx.HTTPError as exc: raise HTTPException( status_code=502, detail=f"Terrain tile fetch failed: {exc}", ) from exc return Response( content=content, media_type=content_type, headers=headers, ) def _is_valid_terrain_tile(z: int, x: int, y: int) -> bool: if z < 0 or x < 0 or y < 0: return False max_tile = 2 ** z return x < max_tile and y < max_tile def _get_cached_terrain_tile(z: int, x: int, y: int) -> tuple[bytes, str, dict[str, str]] | None: key = (z, x, y) cached = _terrain_tile_cache.get(key) if cached is None: return None _terrain_tile_cache.move_to_end(key) content, content_type, headers = cached return content, content_type, dict(headers) def _cache_terrain_tile( z: int, x: int, y: int, content: bytes, content_type: str, headers: dict[str, str], ) -> None: key = (z, x, y) _terrain_tile_cache[key] = (content, content_type, dict(headers)) _terrain_tile_cache.move_to_end(key) while len(_terrain_tile_cache) > TERRAIN_TILE_CACHE_MAX_ITEMS: _terrain_tile_cache.popitem(last=False) async def _fetch_terrain_tile( client: httpx.AsyncClient, z: int, x: int, y: int, ) -> tuple[bytes, str, dict[str, str]]: cached = _get_cached_terrain_tile(z, x, y) if cached is not None: return cached url = TERRAIN_TILE_URL_TEMPLATE.format(z=z, x=x, y=y) upstream = await client.get(url) upstream.raise_for_status() cache_control = upstream.headers.get("cache-control") or "public, max-age=86400" headers = { "Cache-Control": cache_control, } etag = upstream.headers.get("etag") last_modified = upstream.headers.get("last-modified") if etag: headers["ETag"] = etag if last_modified: headers["Last-Modified"] = last_modified content_type = upstream.headers.get("content-type", "image/png") content = upstream.content _cache_terrain_tile(z, x, y, content, content_type, headers) return content, content_type, dict(headers) @router.post("/terrain/terrarium/batch") async def get_terrarium_tile_batch(payload: TerrariumTileBatchRequest): """Fetch Terrarium elevation tiles in batches so the browser avoids many tiny requests.""" unique_tiles: list[TerrariumTileRequest] = [] seen: set[tuple[int, int, int]] = set() for tile in payload.tiles: key = (tile.z, tile.x, tile.y) if key in seen: continue seen.add(key) if not _is_valid_terrain_tile(tile.z, tile.x, tile.y): raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates") unique_tiles.append(tile) semaphore = asyncio.Semaphore(TERRAIN_TILE_BATCH_CONCURRENCY) results: list[dict[str, Any]] = [] errors: list[dict[str, Any]] = [] async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client: async def fetch_one(tile: TerrariumTileRequest) -> None: async with semaphore: try: content, content_type, _headers = await _fetch_terrain_tile( client, tile.z, tile.x, tile.y, ) results.append( { "z": tile.z, "x": tile.x, "y": tile.y, "content_type": content_type, "data": base64.b64encode(content).decode("ascii"), }, ) except httpx.HTTPStatusError as exc: errors.append( { "z": tile.z, "x": tile.x, "y": tile.y, "status_code": exc.response.status_code, "message": f"upstream error: {exc.response.status_code}", }, ) except httpx.HTTPError as exc: errors.append( { "z": tile.z, "x": tile.x, "y": tile.y, "status_code": 502, "message": str(exc), }, ) await asyncio.gather(*(fetch_one(tile) for tile in unique_tiles)) return { "tiles": results, "errors": errors, } @router.get("/geo/all") async def get_all_geojson(db: AsyncSession = Depends(get_db)): records_by_source = await _load_current_collected_data_by_sources( db, [ "arcgis_cables", "arcgis_landing_points", "arcgis_cable_landing_relation", ], ) cables_records = records_by_source.get("arcgis_cables", []) points_records = records_by_source.get("arcgis_landing_points", []) relation_records = records_by_source.get("arcgis_cable_landing_relation", []) city_to_cable_ids_map, cable_id_to_name_map = _build_landing_point_cable_maps( relation_records, cables_records, ) cables = ( convert_cable_to_geojson(cables_records) if cables_records else {"type": "FeatureCollection", "features": []} ) points = ( convert_landing_point_to_geojson(points_records, city_to_cable_ids_map, cable_id_to_name_map) if points_records else {"type": "FeatureCollection", "features": []} ) return { "cables": cables, "landing_points": points, "stats": { "cable_count": len(cables.get("features", [])) if cables else 0, "landing_point_count": len(points.get("features", [])) if points else 0, }, } @router.get("/geo/satellites") async def get_satellites_geojson( limit: Optional[int] = Query( None, ge=1, description="Maximum number of satellites to return. Omit for no limit.", ), db: AsyncSession = Depends(get_db), ): """获取卫星 TLE GeoJSON 数据""" records = await _load_current_or_latest_task_data( db, "celestrak_tle", exclude_unknown_name=True, limit=limit, ) if not records: return {"type": "FeatureCollection", "features": [], "count": 0} geojson = convert_satellite_to_geojson(list(records)) return { **geojson, "count": len(geojson.get("features", [])), } @router.get("/geo/supercomputers") async def get_supercomputers_geojson( limit: int = 500, db: AsyncSession = Depends(get_db), ): """获取 TOP500 超算中心 GeoJSON 数据""" records = await _load_current_collected_data( db, "top500", exclude_unknown_name=True, limit=limit, ) if not records: return {"type": "FeatureCollection", "features": [], "count": 0} geojson = convert_supercomputer_to_geojson(list(records)) return { **geojson, "count": len(geojson.get("features", [])), } @router.get("/geo/gpu-clusters") async def get_gpu_clusters_geojson( limit: int = 100, db: AsyncSession = Depends(get_db), ): """获取 GPU 集群 GeoJSON 数据""" records = await _load_current_collected_data( db, "epoch_ai_gpu", exclude_unknown_name=True, limit=limit, ) if not records: return {"type": "FeatureCollection", "features": [], "count": 0} geojson = convert_gpu_cluster_to_geojson(list(records)) return { **geojson, "count": len(geojson.get("features", [])), } @router.get("/geo/compute-centers") async def get_compute_centers_geojson( limit: int = Query(200, ge=1, le=1000), db: AsyncSession = Depends(get_db), ): """获取统一算力中心 GeoJSON 数据""" records_by_source = await _load_current_collected_data_by_sources( db, ["top500", "epoch_ai_gpu"], ) records = _filter_known_records( records_by_source.get("top500", []) + records_by_source.get("epoch_ai_gpu", []), ) if limit is not None: records = records[:limit] if not records: return { "type": "FeatureCollection", "features": [], "unresolved": [], "count": 0, "stats": { "total": 0, "supercomputers": 0, "gpu_clusters": 0, "unresolved": 0, }, } await refresh_compute_center_location_cache(db) geojson = convert_compute_centers_to_geojson(records) features = geojson.get("features", []) unresolved = geojson.get("unresolved", []) # Belt-and-suspenders: ensure no Feature ever sneaks through without # city-or-better precision and finite, non-zero coordinates. sanitized_features: List[Dict[str, Any]] = [] for feature in features: coords = feature.get("geometry", {}).get("coordinates") or [] precision = feature.get("properties", {}).get("location_precision") if precision not in RENDERABLE_PRECISIONS: unresolved.append({ "failure_reason": f"Rejected non-renderable precision '{precision}'", "record_id": feature.get("id"), "source_id": feature.get("properties", {}).get("source_id"), "name": feature.get("properties", {}).get("name"), }) continue if ( len(coords) != 2 or coords[0] in (None, 0, 0.0) or coords[1] in (None, 0, 0.0) ): unresolved.append({ "failure_reason": "Rejected feature with [0,0] or invalid coordinates", "record_id": feature.get("id"), "source_id": feature.get("properties", {}).get("source_id"), "name": feature.get("properties", {}).get("name"), }) continue sanitized_features.append(feature) return { "type": "FeatureCollection", "features": sanitized_features, "unresolved": unresolved, "count": len(sanitized_features), "stats": { "total": len(sanitized_features), "supercomputers": sum( 1 for feature in sanitized_features if feature.get("properties", {}).get("site_type") == "supercomputer" ), "gpu_clusters": sum( 1 for feature in sanitized_features if feature.get("properties", {}).get("site_type") == "gpu_cluster" ), "unresolved": len(unresolved), }, } class CollectComputeCenterLocationRequest(BaseModel): name: Optional[str] = None source: Optional[str] = None operator: Optional[str] = None site: Optional[str] = None organization: Optional[str] = None city: Optional[str] = None country: Optional[str] = None record_id: Optional[int] = Field(default=None, alias="id") model_config = {"populate_by_name": True} class SaveComputeCenterLocationRequest(BaseModel): source: Optional[str] = None name: Optional[str] = None operator: Optional[str] = None site: Optional[str] = None city: Optional[str] = None country: Optional[str] = None latitude: float longitude: float precision: str = "city" confidence: Optional[float] = None location_source: Optional[str] = None source_url: Optional[str] = None source_note: Optional[str] = None raw_payload: Dict[str, Any] = Field(default_factory=dict) needs_confirmation: bool = False verification_status: Optional[str] = None model_config = {"populate_by_name": True} @router.post("/compute-centers/{source_id}/collect-location") async def collect_compute_center_location( source_id: str, payload: CollectComputeCenterLocationRequest, db: AsyncSession = Depends(get_db), ): """Run the full multi-query location collection pipeline for a record. The endpoint accepts the source_id of a compute center plus contextual fields (name/operator/site/city/country/...) and returns ranked candidate locations from source coordinates, open organization lookups, and online geocoding combinations. The caller never has to type coordinates by hand: if any candidate is accepted it can be applied directly. If no candidate can reach city-level precision the response includes an explicit ``failure_reason`` and the list of attempted queries. """ if not source_id or not source_id.strip(): raise HTTPException(status_code=400, detail="source_id is required") record = await _load_compute_center_record(db, source_id) name = payload.name or (record.name if record else None) metadata = (record.extra_data or {}) if record else {} operator = payload.operator or metadata.get("operator") or metadata.get("organization") or metadata.get("owner") site = payload.site or metadata.get("site") organization = payload.organization or metadata.get("organization") city = payload.city or get_record_field(record, "city") if record else payload.city country = payload.country or (get_record_field(record, "country") if record else None) source = payload.source or (record.source if record else None) record_id = payload.record_id or (record.id if record else None) candidates, attempted_queries = collect_location_candidates( name=name, source=source, source_id=source_id, operator=operator, site=site, organization=organization, city=city, country=country, record_id=record_id, ) llm_failure_reason = None if not candidates: query = build_compute_center_location_query( name=name, source=source, source_id=source_id, operator=operator, site=site, organization=organization, city=city, country=country, ) try: provider_client = await get_ai_provider_client(db) llm_result = await collect_llm_location_fallback_candidate( provider_client=provider_client, query=query, entity_type="compute_center", attempted_queries=attempted_queries, ) except Exception as exc: llm_result = None llm_failure_reason = f"LLM location factcheck unavailable: {exc}" attempted_queries = [ *attempted_queries, f"llm_factcheck:compute_center:{name or source_id or 'unknown'}", ] if llm_result is not None: attempted_queries = [*attempted_queries, *llm_result.attempted_queries] candidates = llm_result.candidates llm_failure_reason = llm_result.failure_reason if not candidates: return { "source_id": source_id, "record_id": record_id, "name": name, "success": False, "failure_reason": ( "No source coordinates, organization lookup, or online geocoding" " result reached city-level precision." ), "candidates": [], "attempted_queries": list(attempted_queries), "llm_failure_reason": llm_failure_reason, "context": { "name": name, "operator": operator, "site": site, "city": city, "country": country, }, } return { "source_id": source_id, "record_id": record_id, "name": name, "success": True, "candidates": [candidate.to_dict() for candidate in candidates], "best_candidate": candidates[0].to_dict(), "attempted_queries": list(attempted_queries), "context": { "name": name, "operator": operator, "site": site, "city": city, "country": country, }, } @router.post("/compute-centers/{source_id}/location") async def save_compute_center_location( source_id: str, payload: SaveComputeCenterLocationRequest, db: AsyncSession = Depends(get_db), ): """Persist the user-selected compute-center location candidate.""" if not source_id or not source_id.strip(): raise HTTPException(status_code=400, detail="source_id is required") if payload.latitude in (0.0, None) or payload.longitude in (0.0, None): raise HTTPException(status_code=400, detail="latitude/longitude are required") if payload.precision not in RENDERABLE_PRECISIONS: raise HTTPException(status_code=400, detail="precision must be precise, site, or city") record = await _load_compute_center_record(db, source_id) metadata = (record.extra_data or {}) if record else {} record_source = payload.source or (record.source if record else None) if not record_source: raise HTTPException(status_code=400, detail="source is required for unknown compute center") operator = ( payload.operator or metadata.get("operator") or metadata.get("organization") or metadata.get("owner") or metadata.get("manufacturer") ) site = payload.site or metadata.get("site") or metadata.get("organization") saved = await upsert_compute_center_location( db, source=record_source, source_id=source_id, name=payload.name or (record.name if record else None), operator=operator, site=site, city=payload.city or (get_record_field(record, "city") if record else None), country=payload.country or (get_record_field(record, "country") if record else None), latitude=payload.latitude, longitude=payload.longitude, precision=payload.precision, confidence=payload.confidence, location_source=payload.location_source or "manual_selection", source_url=payload.source_url, source_note=payload.source_note, raw_payload=payload.raw_payload, needs_confirmation=payload.needs_confirmation, verification_status=payload.verification_status or ("unverified" if payload.needs_confirmation else "verified"), ) return { "success": True, "source": saved.source, "source_id": saved.source_id, "location": saved.to_location_dict(), } async def _load_compute_center_record(db: AsyncSession, source_id: str) -> CollectedData | None: stmt = ( select(CollectedData) .where(CollectedData.source_id == source_id) .where(CollectedData.source.in_(["top500", "epoch_ai_gpu"])) .order_by(CollectedData.is_current.desc(), CollectedData.id.desc()) .limit(1) ) result = await db.execute(stmt) return result.scalars().first() @router.get("/geo/vessels") async def get_vessels_geojson( bbox: Optional[str] = Query( None, description="Viewport bbox as lon_min,lat_min,lon_max,lat_max", ), type: Optional[str] = Query( None, description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other", ), limit: Optional[int] = Query( None, ge=0, description="Maximum vessel features to return. Omit or pass 0 for no limit.", ), db: AsyncSession = Depends(get_db), ): """Return latest vessel positions as GeoJSON points.""" parsed_bbox = _parse_bbox(bbox) requested_types = _requested_vessel_types(type) merged_features, diagnostics = await _load_merged_vessel_features(db) features = _filter_vessel_features( merged_features, bbox=parsed_bbox, requested_types=requested_types, ) if limit and limit > 0: features = features[:limit] return { "type": "FeatureCollection", "features": features, "count": len(features), "stats": _build_vessel_stats(features), "diagnostics": { **diagnostics, "filtered_count": len(features), }, } async def _load_merged_vessel_features(db: AsyncSession) -> tuple[list[dict[str, Any]], dict[str, Any]]: aggregated_vessels = await get_aggregated_vessels(db) raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels) latest_times = ( select( VesselPosition.mmsi.label("mmsi"), func.max(VesselPosition.received_at).label("received_at"), ) .group_by(VesselPosition.mmsi) .subquery() ) stmt = ( select(VesselPosition, VesselStatic) .join( latest_times, (VesselPosition.mmsi == latest_times.c.mmsi) & (VesselPosition.received_at == latest_times.c.received_at), ) .outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi) .order_by(VesselPosition.received_at.desc()) ) result = await db.execute(stmt) rows = list(result.all()) legacy_geojson = convert_vessels_to_geojson(rows) merged_features, diagnostics = _merge_vessel_features( raw_geojson.get("features", []), legacy_geojson.get("features", []), ) return merged_features, { **diagnostics, "raw_feature_count": len(raw_geojson.get("features", [])), "legacy_feature_count": len(legacy_geojson.get("features", [])), } @router.get("/vessels/custom-supplements") async def get_vessel_custom_supplements(db: AsyncSession = Depends(get_db)): """Group custom vessel_ais sources by their declared merge target for diagnostics.""" from app.models.datasource_config import DataSourceConfig result = await db.execute( select(DataSourceConfig.name, DataSourceConfig.config, DataSourceConfig.is_active) .where(DataSourceConfig.config["target_schema"].as_string() == "vessel_ais") ) grouped: dict[str, dict[str, Any]] = {} for name, config, is_active in result.all(): config = config or {} merge_target = str(config.get("merge_target_source") or "barentswatch_vessels") bucket = grouped.setdefault(merge_target, {"merge_target": merge_target, "sources": []}) bucket["sources"].append({"name": name, "is_active": bool(is_active)}) return {"groups": list(grouped.values())} @router.get("/vessels/name-fallbacks") async def get_vessel_name_fallbacks( limit: int = Query(500, ge=0, description="Maximum fallback-name vessels to return. 0 means no limit."), db: AsyncSession = Depends(get_db), ): """Return vessels whose display name still falls back to MMSI.""" aggregated_vessels = await get_aggregated_vessels(db) raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels) latest_times = ( select( VesselPosition.mmsi.label("mmsi"), func.max(VesselPosition.received_at).label("received_at"), ) .group_by(VesselPosition.mmsi) .subquery() ) result = await db.execute( select(VesselPosition, VesselStatic) .join( latest_times, (VesselPosition.mmsi == latest_times.c.mmsi) & (VesselPosition.received_at == latest_times.c.received_at), ) .outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi) .order_by(VesselPosition.received_at.desc()) ) legacy_geojson = convert_vessels_to_geojson(list(result.all())) features, diagnostics = _merge_vessel_features( raw_geojson.get("features", []), legacy_geojson.get("features", []), ) fallback_items = [] for feature in features: props = feature.get("properties", {}) mmsi = props.get("mmsi") name = props.get("name") if not _is_vessel_name_fallback(name, mmsi): continue source_summary = props.get("source_summary") or {} fallback_items.append( { "mmsi": str(mmsi), "display_name": name or f"MMSI {mmsi}", "reason": "missing_real_name", "received_at": props.get("received_at"), "sources": sorted(source_summary.keys()), "source_summary": source_summary, "message_types": sorted( { message_type for summary in source_summary.values() for message_type in (summary.get("message_types") or []) } ), "field_sources": props.get("field_sources") or {}, } ) if limit and limit > 0: fallback_items = fallback_items[:limit] return { "count": len(fallback_items), "items": fallback_items, "diagnostics": diagnostics, } @router.get("/vessels/{mmsi}") async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)): from app.services.vessel_enrichment import get_vessel_enrichment_bundle aggregated = await get_aggregated_vessel(db, mmsi) enrichment = await get_vessel_enrichment_bundle(db, mmsi) if aggregated is not None: return { **aggregated, "received_at": to_iso8601_utc(aggregated.get("received_at")), "latitude": aggregated["lat"], "longitude": aggregated["lon"], "enrichment": enrichment, } latest_position_stmt = ( select(VesselPosition) .where(VesselPosition.mmsi == mmsi) .order_by(VesselPosition.received_at.desc()) .limit(1) ) static = await db.get(VesselStatic, mmsi) result = await db.execute(latest_position_stmt) position = result.scalar_one_or_none() if position is None: raise HTTPException(status_code=404, detail="Vessel not found") geojson = convert_vessels_to_geojson([(position, static)]) return { **(geojson["features"][0]["properties"]), "latitude": position.lat, "longitude": position.lon, "enrichment": enrichment, } @router.get("/vessels/{mmsi}/track") async def get_vessel_track( mmsi: int, hours: int = Query(6, ge=1, le=24), db: AsyncSession = Depends(get_db), ): cutoff = datetime.now(UTC) - timedelta(hours=hours) aggregated_points = await get_aggregated_vessel_track(db, mmsi, cutoff=cutoff) if aggregated_points: return { "type": "FeatureCollection", "features": [ { "type": "Feature", "geometry": { "type": "LineString", "coordinates": [[point["lon"], point["lat"]] for point in aggregated_points], }, "properties": { "mmsi": mmsi, "hours": hours, "point_count": len(aggregated_points), "start_at": to_iso8601_utc(aggregated_points[0]["observed_at"]), "end_at": to_iso8601_utc(aggregated_points[-1]["observed_at"]), "point_sources": [point["source"] for point in aggregated_points], }, } ], "count": 1, } result = await db.execute( select(VesselPosition) .where(VesselPosition.mmsi == mmsi) .where(VesselPosition.received_at >= cutoff) .order_by(VesselPosition.received_at.asc()) ) positions = list(result.scalars().all()) if not positions: return { "type": "FeatureCollection", "features": [], "count": 0, } return { "type": "FeatureCollection", "features": [ { "type": "Feature", "geometry": { "type": "LineString", "coordinates": [[position.lon, position.lat] for position in positions], }, "properties": { "mmsi": mmsi, "hours": hours, "point_count": len(positions), "start_at": to_iso8601_utc(positions[0].received_at), "end_at": to_iso8601_utc(positions[-1].received_at), }, } ], "count": 1, } @router.get("/vessels/{mmsi}/observations") async def get_vessel_observations( mmsi: int, limit: int = Query(100, ge=1, le=500), db: AsyncSession = Depends(get_db), ): """Return raw AIS observations for debugging source-level collector facts.""" observations = await get_vessel_raw_observations(db, mmsi, limit=limit) return { "mmsi": mmsi, "count": len(observations), "observations": [item.to_dict() for item in observations], "conflict_candidates": build_field_conflict_candidates(observations), } @router.get("/vessels/{mmsi}/conflicts") async def get_vessel_conflicts(mmsi: int, db: AsyncSession = Depends(get_db)): """Return recorded AIS conflicts plus current raw-observation candidates.""" records = await get_vessel_conflict_records(db, mmsi) observations = await get_vessel_raw_observations(db, mmsi, limit=500) return { "mmsi": mmsi, "count": len(records), "conflicts": [item.to_dict() for item in records], "candidates": build_field_conflict_candidates(observations), } @router.get("/geo/bgp-anomalies") async def get_bgp_anomalies_geojson( severity: Optional[str] = Query(None), status: Optional[str] = Query("active"), limit: int = Query(200, ge=1, le=1000), db: AsyncSession = Depends(get_db), ): stmt = select(BGPAnomaly).order_by(BGPAnomaly.created_at.desc()).limit(limit) if severity: stmt = stmt.where(BGPAnomaly.severity == severity) if status: stmt = stmt.where(BGPAnomaly.status == status) result = await db.execute(stmt) records = list(result.scalars().all()) geography_hints = await build_anomaly_geography_hints(db, records) geojson = convert_bgp_anomalies_to_geojson(records, geography_hints) return {**geojson, "count": len(geojson.get("features", []))} @router.get("/geo/bgp-incidents") async def get_bgp_incidents_geojson( severity: Optional[str] = Query(None), status: Optional[str] = Query("active"), limit: int = Query(100, ge=1, le=500), db: AsyncSession = Depends(get_db), ): stmt = select(BGPIncident).order_by(BGPIncident.created_at.desc()).limit(limit) if severity: stmt = stmt.where(BGPIncident.severity == severity) if status: stmt = stmt.where(BGPIncident.status == status) result = await db.execute(stmt) records = list(result.scalars().all()) geography_hints = await build_incident_geography_hints(db, records) geojson = convert_bgp_incidents_to_geojson(records, geography_hints) return {**geojson, "count": len(geojson.get("features", []))} @router.get("/geo/bgp-collectors") async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db)): coverage = await build_bgp_collector_coverage( db, source_filter=("ris_live_bgp", "bgpstream_bgp"), ) coverage_by_collector = { item["collector"]: item for item in coverage if item.get("collector") } geojson = convert_bgp_collectors_to_geojson(coverage_by_collector) return {**geojson, "count": len(geojson.get("features", []))} @router.get("/geo/summary") async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)): """Return lightweight Earth HUD counts without loading layer GeoJSON payloads.""" cable_count = await _count_current_or_latest_task_data(db, "arcgis_cables") landing_point_count = await _count_current_or_latest_task_data(db, "arcgis_landing_points") satellite_count = await _count_current_or_latest_task_data( db, "celestrak_tle", exclude_unknown_name=True, ) supercomputer_count = await _count_current_or_latest_task_data(db, "top500") gpu_cluster_count = await _count_current_or_latest_task_data(db, "epoch_ai_gpu") compute_center_count = supercomputer_count + gpu_cluster_count active_incident_result = await db.execute( select(func.count(BGPIncident.id)).where(BGPIncident.status == "active"), ) active_anomaly_result = await db.execute( select(func.count(BGPAnomaly.id)).where(BGPAnomaly.status == "active"), ) active_incident_count = int(active_incident_result.scalar() or 0) active_anomaly_count = int(active_anomaly_result.scalar() or 0) bgp_collector_result = await db.execute( select(func.count(func.distinct(BGPObservation.collector))) .where(BGPObservation.collector.isnot(None)) .where(func.length(func.btrim(BGPObservation.collector)) > 0) .where(BGPObservation.source.in_(("ris_live_bgp", "bgpstream_bgp"))) ) bgp_collector_scalar = bgp_collector_result.scalar() if bgp_collector_scalar is None: bgp_collectors = await build_bgp_collector_coverage( db, source_filter=("ris_live_bgp", "bgpstream_bgp"), ) bgp_collector_count = len( [item for item in bgp_collectors if item.get("collector")] ) else: bgp_collector_count = int(bgp_collector_scalar or 0) raw_unique_window_hours = 24 raw_unique_mmsi = await count_unique_raw_vessel_mmsi( db, observed_since=datetime.now(UTC) - timedelta(hours=raw_unique_window_hours), ) legacy_unique_result = await db.execute( select(func.count(func.distinct(VesselPosition.mmsi))) ) legacy_unique_mmsi = int(legacy_unique_result.scalar() or 0) vessel_count = max(raw_unique_mmsi, legacy_unique_mmsi) aisstream_health = await db.get(AISSourceHealth, "aisstream_vessels") return { "generated_at": to_iso8601_utc(datetime.now(UTC)), "stats": { "cable_count": cable_count, "landing_point_count": landing_point_count, "satellite_count": satellite_count, "compute_center_count": compute_center_count, "vessel_count": vessel_count, "vessel_raw_unique_mmsi": raw_unique_mmsi, "vessel_raw_unique_window_hours": raw_unique_window_hours, "vessel_legacy_unique_mmsi": legacy_unique_mmsi, "aisstream_connection_state": aisstream_health.connection_state if aisstream_health else None, "aisstream_last_seen_at": to_iso8601_utc(aisstream_health.last_seen_at) if aisstream_health else None, "aisstream_message_rate": aisstream_health.message_rate if aisstream_health else None, "aisstream_lag_seconds": aisstream_health.lag_seconds if aisstream_health else None, "supercomputer_count": supercomputer_count, "gpu_cluster_count": gpu_cluster_count, "bgp_event_count": active_incident_count or active_anomaly_count, "bgp_incident_count": active_incident_count, "bgp_anomaly_count": active_anomaly_count, "bgp_collector_count": bgp_collector_count, }, } @router.get("/all") async def get_all_visualization_data(db: AsyncSession = Depends(get_db)): """获取所有可视化数据的统一端点 Returns GeoJSON FeatureCollections for all data types: - satellites: 卫星 TLE 数据 - cables: 海底电缆 - landing_points: 登陆点 - supercomputers: TOP500 超算 - gpu_clusters: GPU 集群 """ records_by_source = await _load_current_collected_data_by_sources( db, [ "arcgis_cables", "arcgis_landing_points", "celestrak_tle", "top500", "epoch_ai_gpu", ], ) cables_records = records_by_source.get("arcgis_cables", []) points_records = records_by_source.get("arcgis_landing_points", []) satellites_records = _filter_known_records( records_by_source.get("celestrak_tle", []), ) supercomputers_records = _filter_known_records( records_by_source.get("top500", []), ) gpu_records = _filter_known_records( records_by_source.get("epoch_ai_gpu", []), ) cables = ( convert_cable_to_geojson(cables_records) if cables_records else {"type": "FeatureCollection", "features": []} ) landing_points = ( convert_landing_point_to_geojson(points_records) if points_records else {"type": "FeatureCollection", "features": []} ) satellites = ( convert_satellite_to_geojson(satellites_records) if satellites_records else {"type": "FeatureCollection", "features": []} ) supercomputers = ( convert_supercomputer_to_geojson(supercomputers_records) if supercomputers_records else {"type": "FeatureCollection", "features": []} ) gpu_clusters = ( convert_gpu_cluster_to_geojson(gpu_records) if gpu_records else {"type": "FeatureCollection", "features": []} ) return { "generated_at": to_iso8601_utc(datetime.now(UTC)), "version": "1.0", "data": { "satellites": satellites, "cables": cables, "landing_points": landing_points, "supercomputers": supercomputers, "gpu_clusters": gpu_clusters, }, "stats": { "total_features": ( len(satellites.get("features", [])) + len(cables.get("features", [])) + len(landing_points.get("features", [])) + len(supercomputers.get("features", [])) + len(gpu_clusters.get("features", [])) ), "satellites": len(satellites.get("features", [])), "cables": len(cables.get("features", [])), "landing_points": len(landing_points.get("features", [])), "supercomputers": len(supercomputers.get("features", [])), "gpu_clusters": len(gpu_clusters.get("features", [])), }, } # Cache for cable graph _cable_graph: Optional[CableGraph] = None async def get_cable_graph(db: AsyncSession) -> CableGraph: """Get or build cable graph (cached)""" global _cable_graph if _cable_graph is None: cables_records = await _load_current_collected_data(db, "arcgis_cables") points_records = await _load_current_collected_data(db, "arcgis_landing_points") cables_data = convert_cable_to_geojson(cables_records) points_data = convert_landing_point_to_geojson(points_records) _cable_graph = build_graph_from_data(cables_data, points_data) return _cable_graph @router.post("/geo/path") async def find_path( start: List[float], end: List[float], db: AsyncSession = Depends(get_db), ): """Find shortest path between two coordinates via cable network""" if not start or len(start) != 2: raise HTTPException(status_code=400, detail="Start must be [lon, lat]") if not end or len(end) != 2: raise HTTPException(status_code=400, detail="End must be [lon, lat]") graph = await get_cable_graph(db) result = graph.find_shortest_path(start, end) if not result: raise HTTPException( status_code=404, detail="No path found between these points. They may be too far from any landing point.", ) return result