2246 lines
79 KiB
Python
2246 lines
79 KiB
Python
"""Visualization API - GeoJSON endpoints for 3D Earth display
|
|
|
|
Unified API for all visualization data sources.
|
|
Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium.
|
|
"""
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
import math
|
|
import re
|
|
import httpx
|
|
from fastapi import APIRouter, HTTPException, Depends, Query, Response
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy import select, func
|
|
from typing import List, Dict, Any, Optional
|
|
|
|
from app.core.collected_data_fields import get_record_field
|
|
from app.core.countries import get_country_centroid
|
|
from app.core.satellite_tle import build_tle_lines_from_elements
|
|
from app.core.time import to_iso8601_utc
|
|
from app.db.session import get_db
|
|
from app.models.bgp_anomaly import BGPAnomaly
|
|
from app.models.bgp_incident import BGPIncident
|
|
from app.models.bgp_observation import BGPObservation
|
|
from app.models.collected_data import CollectedData
|
|
from app.models.vessel import AISSourceHealth, VesselPosition, VesselStatic
|
|
from app.services.bgp_collectors import build_bgp_collector_coverage
|
|
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
|
|
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
|
|
from app.services.persistent_logs import record_system_log
|
|
from app.services.vessel_ais_aggregation import (
|
|
build_field_conflict_candidates,
|
|
count_unique_raw_vessel_mmsi,
|
|
get_aggregated_vessel,
|
|
get_aggregated_vessel_track,
|
|
get_aggregated_vessels,
|
|
get_vessel_conflict_records,
|
|
get_vessel_raw_observations,
|
|
)
|
|
from app.core.logging import get_logger
|
|
|
|
router = APIRouter()
|
|
logger = get_logger(__name__, service="api")
|
|
TERRAIN_TILE_URL_TEMPLATE = (
|
|
"https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png"
|
|
)
|
|
VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE)
|
|
|
|
|
|
# ============== Converter Functions ==============
|
|
|
|
|
|
def convert_cable_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
|
|
"""Convert cable records to GeoJSON FeatureCollection"""
|
|
features = []
|
|
|
|
for record in records:
|
|
metadata = record.extra_data or {}
|
|
route_coords = metadata.get("route_coordinates", [])
|
|
|
|
if not route_coords:
|
|
continue
|
|
|
|
all_lines = []
|
|
|
|
# Handle both old format (flat array) and new format (array of arrays)
|
|
if route_coords and isinstance(route_coords[0], list):
|
|
# New format: array of arrays (MultiLineString structure)
|
|
if route_coords and isinstance(route_coords[0][0], list):
|
|
# Array of arrays of arrays - multiple lines
|
|
for line in route_coords:
|
|
line_coords = []
|
|
for point in line:
|
|
if len(point) >= 2:
|
|
try:
|
|
lon = float(point[0])
|
|
lat = float(point[1])
|
|
line_coords.append([lon, lat])
|
|
except (ValueError, TypeError):
|
|
continue
|
|
if len(line_coords) >= 2:
|
|
all_lines.append(line_coords)
|
|
else:
|
|
# Old format: flat array of points - treat as single line
|
|
line_coords = []
|
|
for point in route_coords:
|
|
if len(point) >= 2:
|
|
try:
|
|
lon = float(point[0])
|
|
lat = float(point[1])
|
|
line_coords.append([lon, lat])
|
|
except (ValueError, TypeError):
|
|
continue
|
|
if len(line_coords) >= 2:
|
|
all_lines.append(line_coords)
|
|
|
|
if not all_lines:
|
|
continue
|
|
|
|
# Use MultiLineString format to preserve cable segments
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {"type": "MultiLineString", "coordinates": all_lines},
|
|
"properties": {
|
|
"id": record.id,
|
|
"cable_id": record.name,
|
|
"source_id": record.source_id,
|
|
"Name": record.name,
|
|
"name": record.name,
|
|
"owner": metadata.get("owners"),
|
|
"owners": metadata.get("owners"),
|
|
"rfs": metadata.get("rfs"),
|
|
"RFS": metadata.get("rfs"),
|
|
"status": metadata.get("status", "active"),
|
|
"length": get_record_field(record, "value"),
|
|
"length_km": get_record_field(record, "value"),
|
|
"SHAPE__Length": get_record_field(record, "value"),
|
|
"url": metadata.get("url"),
|
|
"color": metadata.get("color"),
|
|
"year": metadata.get("year"),
|
|
},
|
|
}
|
|
)
|
|
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
def convert_landing_point_to_geojson(records: List[CollectedData], city_to_cable_ids_map: Dict[int, List[int]] = None, cable_id_to_name_map: Dict[int, str] = None) -> Dict[str, Any]:
|
|
features = []
|
|
|
|
for record in records:
|
|
try:
|
|
latitude = get_record_field(record, "latitude")
|
|
longitude = get_record_field(record, "longitude")
|
|
lat = float(latitude) if latitude else None
|
|
lon = float(longitude) if longitude else None
|
|
except (ValueError, TypeError):
|
|
continue
|
|
|
|
if lat is None or lon is None:
|
|
continue
|
|
|
|
metadata = record.extra_data or {}
|
|
city_id = metadata.get("city_id")
|
|
|
|
props = {
|
|
"id": record.id,
|
|
"source_id": record.source_id,
|
|
"name": record.name,
|
|
"country": get_record_field(record, "country"),
|
|
"city": get_record_field(record, "city"),
|
|
"is_tbd": metadata.get("is_tbd", False),
|
|
}
|
|
|
|
cable_names = []
|
|
if city_to_cable_ids_map and city_id in city_to_cable_ids_map:
|
|
for cable_id in city_to_cable_ids_map[city_id]:
|
|
if cable_id_to_name_map and cable_id in cable_id_to_name_map:
|
|
cable_names.append(cable_id_to_name_map[cable_id])
|
|
|
|
if cable_names:
|
|
props["cable_names"] = cable_names
|
|
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {"type": "Point", "coordinates": [lon, lat]},
|
|
"properties": props,
|
|
}
|
|
)
|
|
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
def convert_satellite_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
|
|
"""Convert satellite TLE records to GeoJSON"""
|
|
features = []
|
|
|
|
for record in records:
|
|
metadata = record.extra_data or {}
|
|
norad_id = metadata.get("norad_cat_id")
|
|
|
|
if not norad_id:
|
|
continue
|
|
|
|
tle_line1 = metadata.get("tle_line1")
|
|
tle_line2 = metadata.get("tle_line2")
|
|
if not tle_line1 or not tle_line2:
|
|
tle_line1, tle_line2 = build_tle_lines_from_elements(
|
|
norad_cat_id=norad_id,
|
|
epoch=metadata.get("epoch"),
|
|
inclination=metadata.get("inclination"),
|
|
raan=metadata.get("raan"),
|
|
eccentricity=metadata.get("eccentricity"),
|
|
arg_of_perigee=metadata.get("arg_of_perigee"),
|
|
mean_anomaly=metadata.get("mean_anomaly"),
|
|
mean_motion=metadata.get("mean_motion"),
|
|
)
|
|
|
|
constellation_group = _normalize_satellite_constellation_group(
|
|
metadata.get("constellation_group"),
|
|
record.name,
|
|
)
|
|
footprint_policy = _get_satellite_footprint_policy(constellation_group)
|
|
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"id": norad_id,
|
|
"geometry": {"type": "Point", "coordinates": [0, 0, 0]},
|
|
"properties": {
|
|
"id": record.id,
|
|
"norad_cat_id": norad_id,
|
|
"name": record.name,
|
|
"constellation_group": constellation_group,
|
|
"footprint_policy": footprint_policy,
|
|
"international_designator": metadata.get("international_designator"),
|
|
"epoch": metadata.get("epoch"),
|
|
"inclination": metadata.get("inclination"),
|
|
"raan": metadata.get("raan"),
|
|
"eccentricity": metadata.get("eccentricity"),
|
|
"arg_of_perigee": metadata.get("arg_of_perigee"),
|
|
"mean_anomaly": metadata.get("mean_anomaly"),
|
|
"mean_motion": metadata.get("mean_motion"),
|
|
"bstar": metadata.get("bstar"),
|
|
"classification_type": metadata.get("classification_type"),
|
|
"tle_line1": tle_line1,
|
|
"tle_line2": tle_line2,
|
|
"data_type": "satellite_tle",
|
|
},
|
|
}
|
|
)
|
|
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
def _normalize_satellite_constellation_group(
|
|
raw_group: Any,
|
|
name: Optional[str],
|
|
) -> Optional[str]:
|
|
normalized_group = str(raw_group or "").strip().lower()
|
|
if normalized_group:
|
|
return normalized_group
|
|
|
|
normalized_name = str(name or "").strip().upper()
|
|
if normalized_name.startswith("STARLINK"):
|
|
return "starlink"
|
|
if normalized_name.startswith("IRIDIUM"):
|
|
return "iridium-next"
|
|
|
|
return None
|
|
|
|
|
|
def _get_satellite_footprint_policy(constellation_group: Optional[str]) -> str:
|
|
if constellation_group == "starlink":
|
|
return "starlink_ground_footprint"
|
|
if constellation_group == "iridium-next":
|
|
return "iridium_coverage_ring"
|
|
return "none"
|
|
|
|
|
|
def _current_collected_data_stmt(source: str):
|
|
return (
|
|
select(CollectedData)
|
|
.where(CollectedData.source == source)
|
|
.where(CollectedData.is_current.is_(True))
|
|
.order_by(CollectedData.id.desc())
|
|
)
|
|
|
|
|
|
async def _load_current_collected_data(
|
|
db: AsyncSession,
|
|
source: str,
|
|
*,
|
|
exclude_unknown_name: bool = False,
|
|
limit: Optional[int] = None,
|
|
) -> List[CollectedData]:
|
|
stmt = _current_collected_data_stmt(source)
|
|
if exclude_unknown_name:
|
|
stmt = stmt.where(CollectedData.name != "Unknown")
|
|
if limit is not None:
|
|
stmt = stmt.limit(limit)
|
|
|
|
result = await db.execute(stmt)
|
|
return list(result.scalars().all())
|
|
|
|
|
|
async def _latest_task_id_for_source(
|
|
db: AsyncSession,
|
|
source: str,
|
|
*,
|
|
exclude_unknown_name: bool = False,
|
|
) -> int | None:
|
|
stmt = (
|
|
select(
|
|
CollectedData.task_id,
|
|
func.max(CollectedData.collected_at).label("latest_collected_at"),
|
|
func.max(CollectedData.id).label("latest_id"),
|
|
)
|
|
.where(CollectedData.source == source)
|
|
.where(CollectedData.task_id.isnot(None))
|
|
.group_by(CollectedData.task_id)
|
|
.order_by(func.max(CollectedData.collected_at).desc(), func.max(CollectedData.id).desc())
|
|
.limit(1)
|
|
)
|
|
if exclude_unknown_name:
|
|
stmt = stmt.where(CollectedData.name != "Unknown")
|
|
|
|
result = await db.execute(stmt)
|
|
row = result.first()
|
|
return int(row.task_id) if row and row.task_id is not None else None
|
|
|
|
|
|
async def _load_current_or_latest_task_data(
|
|
db: AsyncSession,
|
|
source: str,
|
|
*,
|
|
exclude_unknown_name: bool = False,
|
|
limit: Optional[int] = None,
|
|
) -> List[CollectedData]:
|
|
records = await _load_current_collected_data(
|
|
db,
|
|
source,
|
|
exclude_unknown_name=exclude_unknown_name,
|
|
limit=limit,
|
|
)
|
|
if records:
|
|
return records
|
|
|
|
latest_task_id = await _latest_task_id_for_source(
|
|
db,
|
|
source,
|
|
exclude_unknown_name=exclude_unknown_name,
|
|
)
|
|
if latest_task_id is None:
|
|
return []
|
|
|
|
stmt = (
|
|
select(CollectedData)
|
|
.where(CollectedData.source == source)
|
|
.where(CollectedData.task_id == latest_task_id)
|
|
.order_by(CollectedData.id.desc())
|
|
)
|
|
if exclude_unknown_name:
|
|
stmt = stmt.where(CollectedData.name != "Unknown")
|
|
if limit is not None:
|
|
stmt = stmt.limit(limit)
|
|
|
|
result = await db.execute(stmt)
|
|
return list(result.scalars().all())
|
|
|
|
|
|
async def _count_current_or_latest_task_data(
|
|
db: AsyncSession,
|
|
source: str,
|
|
*,
|
|
exclude_unknown_name: bool = False,
|
|
) -> int:
|
|
current_stmt = (
|
|
select(func.count(CollectedData.id))
|
|
.where(CollectedData.source == source)
|
|
.where(CollectedData.is_current.is_(True))
|
|
)
|
|
if exclude_unknown_name:
|
|
current_stmt = current_stmt.where(CollectedData.name != "Unknown")
|
|
|
|
current_result = await db.execute(current_stmt)
|
|
current_scalar = current_result.scalar()
|
|
if current_scalar is None and hasattr(current_result, "scalars"):
|
|
current_rows = current_result.scalars().all()
|
|
current_count = sum(
|
|
1
|
|
for row in current_rows
|
|
if getattr(row, "source", None) == source
|
|
and (not exclude_unknown_name or getattr(row, "name", None) != "Unknown")
|
|
)
|
|
else:
|
|
current_count = int(current_scalar or 0)
|
|
if current_count > 0:
|
|
return current_count
|
|
|
|
latest_task_id = await _latest_task_id_for_source(
|
|
db,
|
|
source,
|
|
exclude_unknown_name=exclude_unknown_name,
|
|
)
|
|
if latest_task_id is None:
|
|
return 0
|
|
|
|
latest_stmt = (
|
|
select(func.count(CollectedData.id))
|
|
.where(CollectedData.source == source)
|
|
.where(CollectedData.task_id == latest_task_id)
|
|
)
|
|
if exclude_unknown_name:
|
|
latest_stmt = latest_stmt.where(CollectedData.name != "Unknown")
|
|
|
|
latest_result = await db.execute(latest_stmt)
|
|
return int(latest_result.scalar() or 0)
|
|
|
|
|
|
async def _load_current_collected_data_by_sources(
|
|
db: AsyncSession,
|
|
sources: List[str],
|
|
) -> Dict[str, List[CollectedData]]:
|
|
if not sources:
|
|
return {}
|
|
|
|
stmt = (
|
|
select(CollectedData)
|
|
.where(CollectedData.source.in_(sources))
|
|
.where(CollectedData.is_current.is_(True))
|
|
.order_by(CollectedData.source.asc(), CollectedData.id.desc())
|
|
)
|
|
result = await db.execute(stmt)
|
|
|
|
grouped_records: Dict[str, List[CollectedData]] = {source: [] for source in sources}
|
|
for record in result.scalars().all():
|
|
grouped_records.setdefault(record.source, []).append(record)
|
|
|
|
return grouped_records
|
|
|
|
|
|
def _build_landing_point_cable_maps(
|
|
relation_records: List[CollectedData],
|
|
cable_records: List[CollectedData],
|
|
) -> tuple[Dict[int, List[int]], Dict[int, str]]:
|
|
city_to_cable_ids_map: Dict[int, List[int]] = {}
|
|
for relation_record in relation_records:
|
|
if not relation_record.extra_data:
|
|
continue
|
|
city_id = relation_record.extra_data.get("city_id")
|
|
cable_id = relation_record.extra_data.get("cable_id")
|
|
if city_id is None or cable_id is None:
|
|
continue
|
|
city_to_cable_ids_map.setdefault(city_id, [])
|
|
if cable_id not in city_to_cable_ids_map[city_id]:
|
|
city_to_cable_ids_map[city_id].append(cable_id)
|
|
|
|
cable_id_to_name_map: Dict[int, str] = {}
|
|
for cable_record in cable_records:
|
|
if not cable_record.extra_data:
|
|
continue
|
|
cable_id = cable_record.extra_data.get("cable_id")
|
|
cable_name = cable_record.name
|
|
if cable_id and cable_name:
|
|
cable_id_to_name_map[cable_id] = cable_name
|
|
|
|
return city_to_cable_ids_map, cable_id_to_name_map
|
|
|
|
|
|
def _filter_known_records(records: List[CollectedData]) -> List[CollectedData]:
|
|
return [record for record in records if record.name != "Unknown"]
|
|
|
|
|
|
def convert_supercomputer_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
|
|
"""Convert TOP500 supercomputer records to GeoJSON"""
|
|
features = []
|
|
|
|
for record in records:
|
|
try:
|
|
latitude = get_record_field(record, "latitude")
|
|
longitude = get_record_field(record, "longitude")
|
|
lat = float(latitude) if latitude and latitude != "0.0" else None
|
|
lon = (
|
|
float(longitude) if longitude and longitude != "0.0" else None
|
|
)
|
|
except (ValueError, TypeError):
|
|
lat, lon = None, None
|
|
|
|
metadata = record.extra_data or {}
|
|
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"id": record.id,
|
|
"geometry": {"type": "Point", "coordinates": [lon or 0, lat or 0]},
|
|
"properties": {
|
|
"id": record.id,
|
|
"name": record.name,
|
|
"rank": metadata.get("rank"),
|
|
"r_max": get_record_field(record, "rmax"),
|
|
"r_peak": get_record_field(record, "rpeak"),
|
|
"cores": get_record_field(record, "cores"),
|
|
"power": get_record_field(record, "power"),
|
|
"country": get_record_field(record, "country"),
|
|
"city": get_record_field(record, "city"),
|
|
"data_type": "supercomputer",
|
|
},
|
|
}
|
|
)
|
|
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
def convert_gpu_cluster_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
|
|
"""Convert GPU cluster records to GeoJSON"""
|
|
features = []
|
|
|
|
for record in records:
|
|
try:
|
|
latitude = get_record_field(record, "latitude")
|
|
longitude = get_record_field(record, "longitude")
|
|
lat = float(latitude) if latitude else None
|
|
lon = float(longitude) if longitude else None
|
|
except (ValueError, TypeError):
|
|
lat, lon = None, None
|
|
|
|
metadata = record.extra_data or {}
|
|
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"id": record.id,
|
|
"geometry": {"type": "Point", "coordinates": [lon or 0, lat or 0]},
|
|
"properties": {
|
|
"id": record.id,
|
|
"name": record.name,
|
|
"country": get_record_field(record, "country"),
|
|
"city": get_record_field(record, "city"),
|
|
"metadata": metadata,
|
|
"data_type": "gpu_cluster",
|
|
},
|
|
}
|
|
)
|
|
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
def _parse_float(value: Any) -> Optional[float]:
|
|
try:
|
|
if value in (None, ""):
|
|
return None
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
COMPUTE_CENTER_COORDINATE_HINTS = (
|
|
("el capitan", 37.6819, -121.7681),
|
|
("livermore", 37.6819, -121.7681),
|
|
("llnl", 37.6819, -121.7681),
|
|
("lawrence livermore", 37.6819, -121.7681),
|
|
("frontier", 35.9319, -84.3107),
|
|
("oak ridge", 35.9319, -84.3107),
|
|
("ornl", 35.9319, -84.3107),
|
|
("aurora", 41.7130, -87.9820),
|
|
("argonne", 41.7130, -87.9820),
|
|
("anl", 41.7130, -87.9820),
|
|
("fugaku", 34.6953, 135.1974),
|
|
("kobe", 34.6953, 135.1974),
|
|
("riken", 34.6953, 135.1974),
|
|
("summit", 35.9319, -84.3107),
|
|
("leonardo", 44.4949, 11.3426),
|
|
("bologna", 44.4949, 11.3426),
|
|
("alps", 46.0037, 8.9511),
|
|
("lugano", 46.0037, 8.9511),
|
|
("sunway taihulight", 31.4912, 120.3119),
|
|
("wuxi", 31.4912, 120.3119),
|
|
("tianhe-2", 23.1291, 113.2644),
|
|
("tianhe-2a", 23.1291, 113.2644),
|
|
("guangzhou", 23.1291, 113.2644),
|
|
("colossus", 35.1495, -90.0490),
|
|
("memphis", 35.1495, -90.0490),
|
|
("xai", 35.1495, -90.0490),
|
|
)
|
|
|
|
|
|
def _normalize_hint_text(*parts: Any) -> str:
|
|
return " ".join(
|
|
str(part).strip().lower()
|
|
for part in parts
|
|
if part not in (None, "")
|
|
)
|
|
|
|
|
|
def _resolve_compute_center_coordinates(
|
|
record: CollectedData,
|
|
metadata: Dict[str, Any],
|
|
) -> Dict[str, Any]:
|
|
latitude = _parse_float(get_record_field(record, "latitude"))
|
|
longitude = _parse_float(get_record_field(record, "longitude"))
|
|
if latitude not in (None, 0.0) and longitude not in (None, 0.0):
|
|
return {
|
|
"latitude": latitude,
|
|
"longitude": longitude,
|
|
"location_precision": "precise",
|
|
"geography_mode": "source_coordinates",
|
|
"is_estimated": False,
|
|
"estimated_reason": None,
|
|
}
|
|
|
|
hint_text = _normalize_hint_text(
|
|
record.name,
|
|
get_record_field(record, "city"),
|
|
get_record_field(record, "country"),
|
|
metadata.get("site"),
|
|
metadata.get("organization"),
|
|
metadata.get("operator"),
|
|
)
|
|
for needle, resolved_latitude, resolved_longitude in COMPUTE_CENTER_COORDINATE_HINTS:
|
|
if needle in hint_text:
|
|
return {
|
|
"latitude": resolved_latitude,
|
|
"longitude": resolved_longitude,
|
|
"location_precision": "estimated_site",
|
|
"geography_mode": "site_hint",
|
|
"is_estimated": True,
|
|
"estimated_reason": f"Matched known site hint: {needle}",
|
|
}
|
|
|
|
centroid = get_country_centroid(get_record_field(record, "country"))
|
|
if centroid:
|
|
return {
|
|
"latitude": centroid.get("latitude"),
|
|
"longitude": centroid.get("longitude"),
|
|
"location_precision": "estimated_country",
|
|
"geography_mode": "country_centroid",
|
|
"is_estimated": True,
|
|
"estimated_reason": "Estimated from country centroid",
|
|
}
|
|
|
|
return {
|
|
"latitude": latitude,
|
|
"longitude": longitude,
|
|
"location_precision": "unknown",
|
|
"geography_mode": "unknown",
|
|
"is_estimated": True,
|
|
"estimated_reason": "No resolvable location hints",
|
|
}
|
|
|
|
|
|
def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str) -> str:
|
|
if capacity_value is None:
|
|
return "unknown"
|
|
|
|
unit = str(capacity_unit or "").strip().lower()
|
|
if unit in {"pflop/s", "pflops", "pflop"}:
|
|
normalized_tflops = capacity_value * 1000
|
|
elif unit in {"gflop/s", "gflops", "gflop"}:
|
|
normalized_tflops = capacity_value
|
|
else:
|
|
normalized_tflops = capacity_value
|
|
|
|
if normalized_tflops >= 1_000_000:
|
|
return "exascale"
|
|
if normalized_tflops >= 100_000:
|
|
return "ultra"
|
|
if normalized_tflops >= 10_000:
|
|
return "large"
|
|
if normalized_tflops > 0:
|
|
return "regional"
|
|
return "unknown"
|
|
|
|
|
|
def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
|
|
"""Convert compute infrastructure records into a unified GeoJSON layer."""
|
|
features = []
|
|
|
|
for record in records:
|
|
metadata = record.extra_data or {}
|
|
coordinate_info = _resolve_compute_center_coordinates(record, metadata)
|
|
latitude = coordinate_info.get("latitude")
|
|
longitude = coordinate_info.get("longitude")
|
|
site_type = (
|
|
"supercomputer"
|
|
if record.source == "top500" or record.data_type == "supercomputer"
|
|
else "gpu_cluster"
|
|
)
|
|
if latitude in (None, 0.0) or longitude in (None, 0.0):
|
|
continue
|
|
|
|
if site_type == "supercomputer":
|
|
capacity_value = _parse_float(get_record_field(record, "rmax"))
|
|
capacity_unit = "GFlops"
|
|
else:
|
|
capacity_value = _parse_float(get_record_field(record, "value"))
|
|
capacity_unit = str(get_record_field(record, "unit") or "TFlop/s")
|
|
|
|
vendor = (
|
|
metadata.get("manufacturer")
|
|
or metadata.get("vendor")
|
|
or metadata.get("gpu_type")
|
|
)
|
|
operator = (
|
|
metadata.get("organization")
|
|
or metadata.get("operator")
|
|
or metadata.get("owner")
|
|
)
|
|
rank = metadata.get("rank")
|
|
if rank in (None, "") and site_type == "supercomputer":
|
|
rank = get_record_field(record, "rank")
|
|
|
|
updated_at = to_iso8601_utc(record.reference_date or record.collected_at)
|
|
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"id": record.id,
|
|
"geometry": {
|
|
"type": "Point",
|
|
"coordinates": [longitude or 0, latitude or 0],
|
|
},
|
|
"properties": {
|
|
"id": record.id,
|
|
"source_id": record.source_id,
|
|
"name": record.name,
|
|
"site_type": site_type,
|
|
"country": get_record_field(record, "country"),
|
|
"city": get_record_field(record, "city"),
|
|
"latitude": latitude,
|
|
"longitude": longitude,
|
|
"operator": operator,
|
|
"vendor": vendor,
|
|
"capacity_value": capacity_value,
|
|
"capacity_unit": capacity_unit,
|
|
"capacity_band": _normalize_capacity_band(capacity_value, capacity_unit),
|
|
"rank": rank,
|
|
"gpu_count": metadata.get("gpu_count"),
|
|
"gpu_type": metadata.get("gpu_type"),
|
|
"cores": get_record_field(record, "cores"),
|
|
"power": get_record_field(record, "power"),
|
|
"source": record.source,
|
|
"updated_at": updated_at,
|
|
"status": "observed",
|
|
"location_precision": coordinate_info.get("location_precision"),
|
|
"geography_mode": coordinate_info.get("geography_mode"),
|
|
"is_estimated": coordinate_info.get("is_estimated", False),
|
|
"estimated_reason": coordinate_info.get("estimated_reason"),
|
|
"data_type": "compute_center",
|
|
"metadata": metadata,
|
|
},
|
|
}
|
|
)
|
|
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
VESSEL_TYPE_FILTERS = {
|
|
"cargo": lambda props: str(props.get("vessel_type_name", "")).lower() == "cargo"
|
|
or 70 <= int(props.get("vessel_type") or -1) <= 79,
|
|
"tanker": lambda props: str(props.get("vessel_type_name", "")).lower() == "tanker"
|
|
or 80 <= int(props.get("vessel_type") or -1) <= 89,
|
|
"passenger": lambda props: str(props.get("vessel_type_name", "")).lower() == "passenger"
|
|
or 60 <= int(props.get("vessel_type") or -1) <= 69,
|
|
"fishing": lambda props: str(props.get("vessel_type_name", "")).lower() == "fishing"
|
|
or int(props.get("vessel_type") or -1) == 30,
|
|
"military": lambda props: str(props.get("vessel_type_name", "")).lower() == "military"
|
|
or int(props.get("vessel_type") or -1) == 35,
|
|
"other": lambda props: str(props.get("vessel_type_name", "")).lower()
|
|
not in {"cargo", "tanker", "passenger", "fishing", "military"},
|
|
}
|
|
|
|
|
|
def convert_vessels_to_geojson(rows: List[Any]) -> Dict[str, Any]:
|
|
features = []
|
|
seen_mmsi: set[int] = set()
|
|
for position, static in rows:
|
|
if position.lat is None or position.lon is None:
|
|
continue
|
|
if position.mmsi in seen_mmsi:
|
|
continue
|
|
seen_mmsi.add(position.mmsi)
|
|
props = {
|
|
"mmsi": position.mmsi,
|
|
"mmsi_display": str(position.mmsi),
|
|
"name": getattr(static, "name", None) or f"MMSI {position.mmsi}",
|
|
"name_is_fallback": _is_vessel_name_fallback(getattr(static, "name", None), position.mmsi),
|
|
"callsign": getattr(static, "callsign", None),
|
|
"imo": getattr(static, "imo", None),
|
|
"imo_display": str(getattr(static, "imo")) if getattr(static, "imo", None) else None,
|
|
"vessel_type": getattr(static, "vessel_type", None),
|
|
"vessel_type_name": getattr(static, "vessel_type_name", None) or "Other",
|
|
"flag": getattr(static, "flag", None),
|
|
"length": getattr(static, "length", None),
|
|
"width": getattr(static, "width", None),
|
|
"draught": getattr(static, "draught", None),
|
|
"sog": position.sog,
|
|
"cog": position.cog,
|
|
"heading": position.heading,
|
|
"nav_status": position.nav_status,
|
|
"received_at": to_iso8601_utc(position.received_at),
|
|
"data_type": "vessel",
|
|
}
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"id": position.mmsi,
|
|
"geometry": {
|
|
"type": "Point",
|
|
"coordinates": [position.lon, position.lat],
|
|
},
|
|
"properties": props,
|
|
}
|
|
)
|
|
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict[str, Any]:
|
|
features = []
|
|
for vessel in vessels:
|
|
if vessel.get("lat") is None or vessel.get("lon") is None:
|
|
continue
|
|
source_summary = {}
|
|
for source, summary in (vessel.get("source_summary") or {}).items():
|
|
source_summary[source] = {
|
|
**summary,
|
|
"latest_observed_at": to_iso8601_utc(summary.get("latest_observed_at")),
|
|
}
|
|
props = {
|
|
"mmsi": vessel["mmsi"],
|
|
"mmsi_display": str(vessel["mmsi"]),
|
|
"name": vessel.get("name") or f"MMSI {vessel['mmsi']}",
|
|
"name_is_fallback": _is_vessel_name_fallback(vessel.get("name"), vessel["mmsi"]),
|
|
"callsign": vessel.get("callsign"),
|
|
"imo": vessel.get("imo"),
|
|
"imo_display": str(vessel.get("imo")) if vessel.get("imo") else None,
|
|
"vessel_type": vessel.get("vessel_type"),
|
|
"vessel_type_name": vessel.get("vessel_type_name") or "Other",
|
|
"flag": vessel.get("flag"),
|
|
"length": vessel.get("length"),
|
|
"width": vessel.get("width"),
|
|
"draught": vessel.get("draught"),
|
|
"sog": vessel.get("sog"),
|
|
"cog": vessel.get("cog"),
|
|
"heading": vessel.get("heading"),
|
|
"nav_status": vessel.get("nav_status"),
|
|
"received_at": to_iso8601_utc(vessel.get("received_at")),
|
|
"field_sources": vessel.get("field_sources") or {},
|
|
"selected_reasons": vessel.get("selected_reasons") or {},
|
|
"source_summary": source_summary,
|
|
"quality_flags": vessel.get("quality_flags") or [],
|
|
"conflict_count": vessel.get("conflict_count", 0),
|
|
"aggregation_strategy_version": vessel.get("aggregation_strategy_version", 0),
|
|
"data_type": "vessel",
|
|
}
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"id": vessel["mmsi"],
|
|
"geometry": {
|
|
"type": "Point",
|
|
"coordinates": [vessel["lon"], vessel["lat"]],
|
|
},
|
|
"properties": props,
|
|
}
|
|
)
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
def _parse_bbox(value: Optional[str]) -> tuple[float, float, float, float] | None:
|
|
if not value:
|
|
return None
|
|
parts = [part.strip() for part in value.split(",")]
|
|
if len(parts) != 4:
|
|
raise HTTPException(status_code=400, detail="bbox must be lon_min,lat_min,lon_max,lat_max")
|
|
try:
|
|
lon_min, lat_min, lon_max, lat_max = [float(part) for part in parts]
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=400, detail="bbox values must be numbers") from exc
|
|
if lat_min > lat_max:
|
|
lat_min, lat_max = lat_max, lat_min
|
|
if lon_min > lon_max:
|
|
lon_min, lon_max = lon_max, lon_min
|
|
return lon_min, lat_min, lon_max, lat_max
|
|
|
|
|
|
def _is_vessel_name_fallback(name: Any, mmsi: Any) -> bool:
|
|
text = str(name or "").strip()
|
|
mmsi_text = str(mmsi or "").strip()
|
|
if not text:
|
|
return True
|
|
if mmsi_text and text == mmsi_text:
|
|
return True
|
|
return bool(VESSEL_NAME_FALLBACK_PATTERN.match(text))
|
|
|
|
|
|
def _requested_vessel_types(value: Optional[str]) -> set[str]:
|
|
return {
|
|
item.strip().lower()
|
|
for item in (value or "").split(",")
|
|
if item.strip()
|
|
}
|
|
|
|
|
|
def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bool:
|
|
if not requested_types:
|
|
return True
|
|
for requested_type in requested_types:
|
|
predicate = VESSEL_TYPE_FILTERS.get(requested_type)
|
|
if predicate and predicate(props):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _feature_mmsi_key(feature: dict[str, Any]) -> str | None:
|
|
props = feature.get("properties", {})
|
|
mmsi = props.get("mmsi") or feature.get("id")
|
|
if mmsi in (None, ""):
|
|
return None
|
|
return str(mmsi)
|
|
|
|
|
|
def _feature_in_bbox(feature: dict[str, Any], bbox: tuple[float, float, float, float] | None) -> bool:
|
|
if bbox is None:
|
|
return True
|
|
coordinates = feature.get("geometry", {}).get("coordinates") or []
|
|
if len(coordinates) < 2:
|
|
return False
|
|
try:
|
|
lon = float(coordinates[0])
|
|
lat = float(coordinates[1])
|
|
except (TypeError, ValueError):
|
|
return False
|
|
lon_min, lat_min, lon_max, lat_max = bbox
|
|
return lon_min <= lon <= lon_max and lat_min <= lat <= lat_max
|
|
|
|
|
|
def _filter_vessel_features(
|
|
features: list[dict[str, Any]],
|
|
*,
|
|
bbox: tuple[float, float, float, float] | None,
|
|
requested_types: set[str],
|
|
) -> list[dict[str, Any]]:
|
|
return [
|
|
feature
|
|
for feature in features
|
|
if _feature_in_bbox(feature, bbox)
|
|
and _matches_vessel_type(feature.get("properties", {}), requested_types)
|
|
]
|
|
|
|
|
|
def _merge_vessel_features(
|
|
raw_features: list[dict[str, Any]],
|
|
legacy_features: list[dict[str, Any]],
|
|
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
|
"""Prefer aggregated raw observations as the canonical source of truth.
|
|
|
|
Legacy `vessel_position` rows only fill MMSIs that the unified pipeline does
|
|
not yet know about, so a vessel never appears twice when both BarentsWatch
|
|
and AISStream observe it. Once the legacy table drains, this branch becomes
|
|
a no-op.
|
|
"""
|
|
|
|
merged: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
raw_keys: set[str] = set()
|
|
legacy_keys: set[str] = set()
|
|
|
|
for feature in raw_features:
|
|
key = _feature_mmsi_key(feature)
|
|
if key is None or key in seen:
|
|
continue
|
|
seen.add(key)
|
|
raw_keys.add(key)
|
|
merged.append(feature)
|
|
|
|
legacy_added = 0
|
|
for feature in legacy_features:
|
|
key = _feature_mmsi_key(feature)
|
|
if key is None:
|
|
continue
|
|
legacy_keys.add(key)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
legacy_added += 1
|
|
merged.append(feature)
|
|
|
|
return merged, {
|
|
"raw_unique_mmsi": len(raw_keys),
|
|
"legacy_unique_mmsi": len(legacy_keys),
|
|
"legacy_backfilled_mmsi": legacy_added,
|
|
"final_unique_mmsi": len(seen),
|
|
}
|
|
|
|
|
|
def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]:
|
|
by_type: dict[str, int] = {}
|
|
underway = 0
|
|
anchored_or_moored = 0
|
|
for feature in features:
|
|
props = feature.get("properties", {})
|
|
vessel_type = str(props.get("vessel_type_name") or "Other")
|
|
by_type[vessel_type] = by_type.get(vessel_type, 0) + 1
|
|
nav_status = props.get("nav_status")
|
|
if nav_status in (1, 5):
|
|
anchored_or_moored += 1
|
|
else:
|
|
underway += 1
|
|
return {
|
|
"total": len(features),
|
|
"by_type": by_type,
|
|
"underway": underway,
|
|
"anchored_or_moored": anchored_or_moored,
|
|
}
|
|
|
|
|
|
def convert_bgp_anomalies_to_geojson(
|
|
records: List[BGPAnomaly],
|
|
geography_hints: Optional[Dict[str, Dict[str, Any]]] = None,
|
|
) -> Dict[str, Any]:
|
|
features = []
|
|
geography_hints = geography_hints or {}
|
|
|
|
for record in records:
|
|
evidence = record.evidence or {}
|
|
hint = geography_hints.get(str(record.entity_key or record.id), {})
|
|
collectors = evidence.get("collectors") or record.peer_scope or []
|
|
if not collectors:
|
|
nested = evidence.get("events") or []
|
|
collectors = [
|
|
str((item or {}).get("collector") or "").strip()
|
|
for item in nested
|
|
if (item or {}).get("collector")
|
|
]
|
|
|
|
collectors = [collector for collector in collectors if collector]
|
|
if not collectors:
|
|
collectors = []
|
|
|
|
as_path = []
|
|
if isinstance(evidence.get("as_path"), list):
|
|
as_path = evidence.get("as_path") or []
|
|
if not as_path:
|
|
nested = evidence.get("events") or []
|
|
for item in nested:
|
|
candidate_path = (item or {}).get("as_path")
|
|
if isinstance(candidate_path, list) and candidate_path:
|
|
as_path = candidate_path
|
|
break
|
|
|
|
impacted_regions = []
|
|
seen_regions = set()
|
|
for collector_name in collectors:
|
|
collector_location = RIPE_RIS_COLLECTOR_COORDS.get(str(collector_name))
|
|
if not collector_location:
|
|
continue
|
|
region_key = (
|
|
collector_location.get("country"),
|
|
collector_location.get("city"),
|
|
)
|
|
if region_key in seen_regions:
|
|
continue
|
|
seen_regions.add(region_key)
|
|
impacted_regions.append(
|
|
{
|
|
"collector": collector_name,
|
|
"country": collector_location.get("country"),
|
|
"city": collector_location.get("city"),
|
|
"latitude": collector_location.get("latitude"),
|
|
"longitude": collector_location.get("longitude"),
|
|
}
|
|
)
|
|
|
|
geography_regions = _normalize_geo_regions(hint.get("regions") or [])
|
|
geography_mode = hint.get("geography_mode") or "collector_centroid"
|
|
|
|
collector = collectors[0] if collectors else None
|
|
location = geography_regions[0] if geography_regions else None
|
|
|
|
if location is None and collector:
|
|
location = RIPE_RIS_COLLECTOR_COORDS.get(str(collector))
|
|
|
|
if location is None:
|
|
nested = evidence.get("events") or []
|
|
for item in nested:
|
|
collector_name = (item or {}).get("collector")
|
|
if collector_name and collector_name in RIPE_RIS_COLLECTOR_COORDS:
|
|
location = RIPE_RIS_COLLECTOR_COORDS[collector_name]
|
|
collector = collector_name
|
|
geography_mode = "collector_centroid"
|
|
break
|
|
|
|
if location is None:
|
|
continue
|
|
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {
|
|
"type": "Point",
|
|
"coordinates": [location["longitude"], location["latitude"]],
|
|
},
|
|
"properties": {
|
|
"id": record.id,
|
|
"collector": collector,
|
|
"city": location.get("city"),
|
|
"country": location.get("country"),
|
|
"source": record.source,
|
|
"anomaly_type": record.anomaly_type,
|
|
"severity": record.severity,
|
|
"status": record.status,
|
|
"prefix": record.prefix,
|
|
"origin_asn": record.origin_asn,
|
|
"new_origin_asn": record.new_origin_asn,
|
|
"collectors": collectors,
|
|
"collector_count": len(collectors) or 1,
|
|
"as_path": as_path,
|
|
"impacted_regions": impacted_regions,
|
|
"geography_mode": geography_mode,
|
|
"confidence": record.confidence,
|
|
"summary": record.summary,
|
|
"created_at": to_iso8601_utc(record.created_at),
|
|
},
|
|
}
|
|
)
|
|
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
async def build_anomaly_geography_hints(
|
|
db: AsyncSession,
|
|
records: List[BGPAnomaly],
|
|
) -> Dict[str, Dict[str, Any]]:
|
|
hints: Dict[str, Dict[str, Any]] = {}
|
|
for record in records:
|
|
hint = _extract_evidence_geography_hint(record.evidence or {})
|
|
if hint:
|
|
hints[str(record.entity_key or record.id)] = hint
|
|
|
|
return hints
|
|
|
|
|
|
def convert_bgp_collectors_to_geojson(
|
|
coverage_by_collector: Dict[str, Dict[str, Any]] | None = None,
|
|
) -> Dict[str, Any]:
|
|
features = []
|
|
coverage_by_collector = coverage_by_collector or {}
|
|
|
|
for collector, location in sorted(RIPE_RIS_COLLECTOR_COORDS.items()):
|
|
coverage = coverage_by_collector.get(collector, {})
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {
|
|
"type": "Point",
|
|
"coordinates": [location["longitude"], location["latitude"]],
|
|
},
|
|
"properties": {
|
|
"collector": collector,
|
|
"city": coverage.get("city") or location.get("city"),
|
|
"country": coverage.get("country") or location.get("country"),
|
|
"status": "online",
|
|
"observation_count": coverage.get("observation_count", 0),
|
|
"prefix_count": coverage.get("prefix_count", 0),
|
|
"origin_asn_count": coverage.get("origin_asn_count", 0),
|
|
"peer_asn_count": coverage.get("peer_asn_count", 0),
|
|
"recent_15m_observation_count": coverage.get("recent_15m_observation_count", 0),
|
|
"recent_24h_observation_count": coverage.get("recent_24h_observation_count", 0),
|
|
"recent_7d_observation_count": coverage.get("recent_7d_observation_count", 0),
|
|
"recent_15m_prefix_count": coverage.get("recent_15m_prefix_count", 0),
|
|
"recent_24h_prefix_count": coverage.get("recent_24h_prefix_count", 0),
|
|
"recent_7d_prefix_count": coverage.get("recent_7d_prefix_count", 0),
|
|
"top_event_types": coverage.get("top_event_types", []),
|
|
"latest_observed_at": coverage.get("latest_observed_at"),
|
|
"latest_event_type": coverage.get("latest_event_type"),
|
|
"baseline_scope": coverage.get(
|
|
"baseline_scope",
|
|
{
|
|
"countries": [location.get("country")] if location.get("country") else [],
|
|
"cities": [location.get("city")] if location.get("city") else [],
|
|
},
|
|
),
|
|
},
|
|
}
|
|
)
|
|
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
def _incident_estimated_center(valid_regions: List[Dict[str, Any]]) -> Dict[str, float]:
|
|
x = 0.0
|
|
y = 0.0
|
|
z = 0.0
|
|
for region in valid_regions:
|
|
lat_rad = math.radians(float(region["latitude"]))
|
|
lon_rad = math.radians(float(region["longitude"]))
|
|
x += math.cos(lat_rad) * math.cos(lon_rad)
|
|
y += math.cos(lat_rad) * math.sin(lon_rad)
|
|
z += math.sin(lat_rad)
|
|
|
|
total = float(len(valid_regions))
|
|
if total <= 0:
|
|
return {"latitude": 0.0, "longitude": 0.0}
|
|
|
|
x /= total
|
|
y /= total
|
|
z /= total
|
|
hyp = math.sqrt((x * x) + (y * y))
|
|
if hyp == 0:
|
|
return {"latitude": 0.0, "longitude": 0.0}
|
|
|
|
return {
|
|
"latitude": math.degrees(math.atan2(z, hyp)),
|
|
"longitude": math.degrees(math.atan2(y, x)),
|
|
}
|
|
|
|
|
|
def _incident_estimated_radius_km(center: Dict[str, float], valid_regions: List[Dict[str, Any]]) -> float:
|
|
center_coords = (float(center["longitude"]), float(center["latitude"]))
|
|
distances = [
|
|
haversine_distance(
|
|
center_coords,
|
|
(float(region["longitude"]), float(region["latitude"])),
|
|
)
|
|
for region in valid_regions
|
|
]
|
|
return round(max(distances) if distances else 0.0, 1)
|
|
|
|
|
|
def _normalize_geo_regions(regions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
normalized: list[dict[str, Any]] = []
|
|
seen: set[tuple[Any, ...]] = set()
|
|
for region in regions:
|
|
if not isinstance(region, dict):
|
|
continue
|
|
latitude = region.get("latitude")
|
|
longitude = region.get("longitude")
|
|
if not isinstance(latitude, (int, float)) or not isinstance(longitude, (int, float)):
|
|
continue
|
|
item = {
|
|
"collector": region.get("collector"),
|
|
"country": region.get("country"),
|
|
"city": region.get("city"),
|
|
"latitude": float(latitude),
|
|
"longitude": float(longitude),
|
|
}
|
|
key = (
|
|
item["collector"],
|
|
item["country"],
|
|
item["city"],
|
|
item["latitude"],
|
|
item["longitude"],
|
|
)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
normalized.append(item)
|
|
return normalized
|
|
|
|
|
|
def _extract_evidence_geography_hint(evidence: Dict[str, Any]) -> Dict[str, Any] | None:
|
|
prefix_geo_regions = []
|
|
prefix_regions = []
|
|
asn_regions = []
|
|
|
|
evidence_prefix_geography = evidence.get("prefix_geography") or {}
|
|
prefix_geo_regions.extend(
|
|
_normalize_geo_regions(evidence_prefix_geography.get("regions") or [])
|
|
)
|
|
|
|
prefix_scope = evidence.get("prefix_scope") or {}
|
|
prefix_regions.extend(_normalize_geo_regions(prefix_scope.get("regions") or []))
|
|
|
|
for profile_key in ("origin_asn_profile", "new_origin_asn_profile"):
|
|
profile = evidence.get(profile_key) or {}
|
|
latitude = profile.get("latitude")
|
|
longitude = profile.get("longitude")
|
|
if isinstance(latitude, (int, float)) and isinstance(longitude, (int, float)):
|
|
asn_regions.append(
|
|
{
|
|
"country": profile.get("country"),
|
|
"city": profile.get("city"),
|
|
"latitude": float(latitude),
|
|
"longitude": float(longitude),
|
|
}
|
|
)
|
|
|
|
prefix_geo_regions = _normalize_geo_regions(prefix_geo_regions)
|
|
prefix_regions = _normalize_geo_regions(prefix_regions)
|
|
asn_regions = _normalize_geo_regions(asn_regions)
|
|
|
|
if prefix_geo_regions:
|
|
return {"regions": prefix_geo_regions, "geography_mode": "prefix_geography"}
|
|
if prefix_regions:
|
|
return {"regions": prefix_regions, "geography_mode": "prefix_scope"}
|
|
if asn_regions:
|
|
return {"regions": asn_regions, "geography_mode": "asn_region"}
|
|
return None
|
|
|
|
|
|
async def build_incident_geography_hints(
|
|
db: AsyncSession,
|
|
records: List[BGPIncident],
|
|
) -> Dict[str, Dict[str, Any]]:
|
|
evidence_refs = sorted(
|
|
{
|
|
str(ref)
|
|
for record in records
|
|
for ref in (record.evidence_refs or [])
|
|
if ref
|
|
}
|
|
)
|
|
|
|
anomalies = []
|
|
if evidence_refs:
|
|
result = await db.execute(
|
|
select(BGPAnomaly).where(BGPAnomaly.entity_key.in_(evidence_refs))
|
|
)
|
|
anomalies = result.scalars().all()
|
|
anomaly_by_key = {
|
|
str(anomaly.entity_key): anomaly
|
|
for anomaly in anomalies
|
|
if anomaly.entity_key
|
|
}
|
|
|
|
hints: Dict[str, Dict[str, Any]] = {}
|
|
for record in records:
|
|
merged_hint: Dict[str, Any] | None = None
|
|
priority = {"prefix_geography": 3, "prefix_scope": 2, "asn_region": 1}
|
|
|
|
for ref in record.evidence_refs or []:
|
|
anomaly = anomaly_by_key.get(str(ref))
|
|
if anomaly is None:
|
|
continue
|
|
hint = _extract_evidence_geography_hint(anomaly.evidence or {})
|
|
if hint is None:
|
|
continue
|
|
if merged_hint is None:
|
|
merged_hint = {
|
|
"regions": list(hint["regions"]),
|
|
"geography_mode": hint["geography_mode"],
|
|
}
|
|
continue
|
|
if priority[hint["geography_mode"]] > priority[merged_hint["geography_mode"]]:
|
|
merged_hint = {
|
|
"regions": list(hint["regions"]),
|
|
"geography_mode": hint["geography_mode"],
|
|
}
|
|
elif priority[hint["geography_mode"]] == priority[merged_hint["geography_mode"]]:
|
|
merged_hint["regions"].extend(hint["regions"])
|
|
|
|
if merged_hint:
|
|
merged_hint["regions"] = _normalize_geo_regions(merged_hint["regions"])
|
|
hints[record.incident_key] = merged_hint
|
|
|
|
return hints
|
|
|
|
|
|
def convert_bgp_incidents_to_geojson(
|
|
records: List[BGPIncident],
|
|
geography_hints: Optional[Dict[str, Dict[str, Any]]] = None,
|
|
) -> Dict[str, Any]:
|
|
features = []
|
|
|
|
for record in records:
|
|
hint = (geography_hints or {}).get(record.incident_key, {})
|
|
regions = hint.get("regions") or (record.affected_regions or [])
|
|
if not regions:
|
|
continue
|
|
|
|
valid_regions = _normalize_geo_regions(regions)
|
|
if not valid_regions:
|
|
continue
|
|
|
|
estimated_center = _incident_estimated_center(valid_regions)
|
|
estimated_radius_km = _incident_estimated_radius_km(estimated_center, valid_regions)
|
|
|
|
features.append(
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {
|
|
"type": "Point",
|
|
"coordinates": [
|
|
estimated_center["longitude"],
|
|
estimated_center["latitude"],
|
|
],
|
|
},
|
|
"properties": {
|
|
"id": record.id,
|
|
"incident_key": record.incident_key,
|
|
"incident_type": record.incident_type,
|
|
"title": record.title,
|
|
"summary": record.summary,
|
|
"severity": record.severity,
|
|
"status": record.status,
|
|
"confidence": record.confidence,
|
|
"affected_prefixes": record.affected_prefixes or [],
|
|
"affected_asns": record.affected_asns or [],
|
|
"affected_collectors": record.affected_collectors or [],
|
|
"affected_regions": valid_regions,
|
|
"estimated_center": estimated_center,
|
|
"estimated_radius_km": estimated_radius_km,
|
|
"geography_mode": hint.get("geography_mode") or "collector_centroid",
|
|
"related_cables": record.related_cables or [],
|
|
"related_ixps": record.related_ixps or [],
|
|
"created_at": to_iso8601_utc(record.created_at),
|
|
"started_at": to_iso8601_utc(record.started_at),
|
|
},
|
|
}
|
|
)
|
|
|
|
return {"type": "FeatureCollection", "features": features}
|
|
|
|
|
|
# ============== API Endpoints ==============
|
|
|
|
|
|
@router.get("/geo/cables")
|
|
async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
|
|
"""获取海底电缆 GeoJSON 数据 (LineString)"""
|
|
try:
|
|
records = await _load_current_collected_data(db, "arcgis_cables")
|
|
|
|
if not records:
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail="No cable data found. Please run the arcgis_cables collector first.",
|
|
)
|
|
|
|
return convert_cable_to_geojson(records)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.exception_event(
|
|
"Failed to build cables GeoJSON response",
|
|
event="visualization.cables.load_failed",
|
|
context={"error": str(e)},
|
|
)
|
|
await record_system_log(
|
|
source="backend",
|
|
service="api",
|
|
module=__name__,
|
|
event="visualization.cables.load_failed",
|
|
level="error",
|
|
message="Failed to build cables GeoJSON response",
|
|
category="visualization",
|
|
context={"error": str(e)},
|
|
)
|
|
raise HTTPException(status_code=500, detail=f"Internal error: {str(e)}")
|
|
|
|
|
|
@router.get("/geo/landing-points")
|
|
async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
|
|
try:
|
|
records_by_source = await _load_current_collected_data_by_sources(
|
|
db,
|
|
[
|
|
"arcgis_landing_points",
|
|
"arcgis_cable_landing_relation",
|
|
"arcgis_cables",
|
|
],
|
|
)
|
|
records = records_by_source.get("arcgis_landing_points", [])
|
|
relation_records = records_by_source.get(
|
|
"arcgis_cable_landing_relation",
|
|
[],
|
|
)
|
|
cable_records = records_by_source.get("arcgis_cables", [])
|
|
|
|
city_to_cable_ids_map, cable_id_to_name_map = _build_landing_point_cable_maps(
|
|
relation_records,
|
|
cable_records,
|
|
)
|
|
|
|
if not records:
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail="No landing point data found. Please run the arcgis_landing_points collector first.",
|
|
)
|
|
|
|
return convert_landing_point_to_geojson(records, city_to_cable_ids_map, cable_id_to_name_map)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.exception_event(
|
|
"Failed to build landing points GeoJSON response",
|
|
event="visualization.landing_points.load_failed",
|
|
context={"error": str(e)},
|
|
)
|
|
await record_system_log(
|
|
source="backend",
|
|
service="api",
|
|
module=__name__,
|
|
event="visualization.landing_points.load_failed",
|
|
level="error",
|
|
message="Failed to build landing points GeoJSON response",
|
|
category="visualization",
|
|
context={"error": str(e)},
|
|
)
|
|
raise HTTPException(status_code=500, detail=f"Internal error: {str(e)}")
|
|
|
|
|
|
@router.get("/terrain/terrarium/{z}/{x}/{y}.png")
|
|
async def get_terrarium_tile(z: int, x: int, y: int):
|
|
"""Proxy Terrarium elevation tiles through the backend to avoid browser CORS issues."""
|
|
if z < 0 or x < 0 or y < 0:
|
|
raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates")
|
|
|
|
url = TERRAIN_TILE_URL_TEMPLATE.format(z=z, x=x, y=y)
|
|
|
|
try:
|
|
async with httpx.AsyncClient(
|
|
timeout=20.0,
|
|
follow_redirects=True,
|
|
) as client:
|
|
upstream = await client.get(url)
|
|
upstream.raise_for_status()
|
|
except httpx.HTTPStatusError as exc:
|
|
raise HTTPException(
|
|
status_code=exc.response.status_code,
|
|
detail=f"Terrain tile upstream error: {exc.response.status_code}",
|
|
) from exc
|
|
except httpx.HTTPError as exc:
|
|
raise HTTPException(
|
|
status_code=502,
|
|
detail=f"Terrain tile fetch failed: {exc}",
|
|
) from exc
|
|
|
|
cache_control = upstream.headers.get("cache-control") or "public, max-age=86400"
|
|
etag = upstream.headers.get("etag")
|
|
last_modified = upstream.headers.get("last-modified")
|
|
headers = {
|
|
"Cache-Control": cache_control,
|
|
}
|
|
if etag:
|
|
headers["ETag"] = etag
|
|
if last_modified:
|
|
headers["Last-Modified"] = last_modified
|
|
|
|
return Response(
|
|
content=upstream.content,
|
|
media_type=upstream.headers.get("content-type", "image/png"),
|
|
headers=headers,
|
|
)
|
|
|
|
|
|
@router.get("/geo/all")
|
|
async def get_all_geojson(db: AsyncSession = Depends(get_db)):
|
|
records_by_source = await _load_current_collected_data_by_sources(
|
|
db,
|
|
[
|
|
"arcgis_cables",
|
|
"arcgis_landing_points",
|
|
"arcgis_cable_landing_relation",
|
|
],
|
|
)
|
|
cables_records = records_by_source.get("arcgis_cables", [])
|
|
points_records = records_by_source.get("arcgis_landing_points", [])
|
|
relation_records = records_by_source.get("arcgis_cable_landing_relation", [])
|
|
city_to_cable_ids_map, cable_id_to_name_map = _build_landing_point_cable_maps(
|
|
relation_records,
|
|
cables_records,
|
|
)
|
|
|
|
cables = (
|
|
convert_cable_to_geojson(cables_records)
|
|
if cables_records
|
|
else {"type": "FeatureCollection", "features": []}
|
|
)
|
|
points = (
|
|
convert_landing_point_to_geojson(points_records, city_to_cable_ids_map, cable_id_to_name_map)
|
|
if points_records
|
|
else {"type": "FeatureCollection", "features": []}
|
|
)
|
|
|
|
return {
|
|
"cables": cables,
|
|
"landing_points": points,
|
|
"stats": {
|
|
"cable_count": len(cables.get("features", [])) if cables else 0,
|
|
"landing_point_count": len(points.get("features", [])) if points else 0,
|
|
},
|
|
}
|
|
|
|
|
|
@router.get("/geo/satellites")
|
|
async def get_satellites_geojson(
|
|
limit: Optional[int] = Query(
|
|
None,
|
|
ge=1,
|
|
description="Maximum number of satellites to return. Omit for no limit.",
|
|
),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""获取卫星 TLE GeoJSON 数据"""
|
|
records = await _load_current_or_latest_task_data(
|
|
db,
|
|
"celestrak_tle",
|
|
exclude_unknown_name=True,
|
|
limit=limit,
|
|
)
|
|
|
|
if not records:
|
|
return {"type": "FeatureCollection", "features": [], "count": 0}
|
|
|
|
geojson = convert_satellite_to_geojson(list(records))
|
|
return {
|
|
**geojson,
|
|
"count": len(geojson.get("features", [])),
|
|
}
|
|
|
|
|
|
@router.get("/geo/supercomputers")
|
|
async def get_supercomputers_geojson(
|
|
limit: int = 500,
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""获取 TOP500 超算中心 GeoJSON 数据"""
|
|
records = await _load_current_collected_data(
|
|
db,
|
|
"top500",
|
|
exclude_unknown_name=True,
|
|
limit=limit,
|
|
)
|
|
|
|
if not records:
|
|
return {"type": "FeatureCollection", "features": [], "count": 0}
|
|
|
|
geojson = convert_supercomputer_to_geojson(list(records))
|
|
return {
|
|
**geojson,
|
|
"count": len(geojson.get("features", [])),
|
|
}
|
|
|
|
|
|
@router.get("/geo/gpu-clusters")
|
|
async def get_gpu_clusters_geojson(
|
|
limit: int = 100,
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""获取 GPU 集群 GeoJSON 数据"""
|
|
records = await _load_current_collected_data(
|
|
db,
|
|
"epoch_ai_gpu",
|
|
exclude_unknown_name=True,
|
|
limit=limit,
|
|
)
|
|
|
|
if not records:
|
|
return {"type": "FeatureCollection", "features": [], "count": 0}
|
|
|
|
geojson = convert_gpu_cluster_to_geojson(list(records))
|
|
return {
|
|
**geojson,
|
|
"count": len(geojson.get("features", [])),
|
|
}
|
|
|
|
|
|
@router.get("/geo/compute-centers")
|
|
async def get_compute_centers_geojson(
|
|
limit: int = Query(200, ge=1, le=1000),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""获取统一算力中心 GeoJSON 数据"""
|
|
records_by_source = await _load_current_collected_data_by_sources(
|
|
db,
|
|
["top500", "epoch_ai_gpu"],
|
|
)
|
|
records = _filter_known_records(
|
|
records_by_source.get("top500", []) + records_by_source.get("epoch_ai_gpu", []),
|
|
)
|
|
if limit is not None:
|
|
records = records[:limit]
|
|
|
|
if not records:
|
|
return {
|
|
"type": "FeatureCollection",
|
|
"features": [],
|
|
"count": 0,
|
|
"stats": {
|
|
"total": 0,
|
|
"supercomputers": 0,
|
|
"gpu_clusters": 0,
|
|
},
|
|
}
|
|
|
|
geojson = convert_compute_centers_to_geojson(records)
|
|
features = geojson.get("features", [])
|
|
return {
|
|
**geojson,
|
|
"count": len(features),
|
|
"stats": {
|
|
"total": len(features),
|
|
"supercomputers": sum(
|
|
1 for feature in features
|
|
if feature.get("properties", {}).get("site_type") == "supercomputer"
|
|
),
|
|
"gpu_clusters": sum(
|
|
1 for feature in features
|
|
if feature.get("properties", {}).get("site_type") == "gpu_cluster"
|
|
),
|
|
},
|
|
}
|
|
|
|
|
|
@router.get("/geo/vessels")
|
|
async def get_vessels_geojson(
|
|
bbox: Optional[str] = Query(
|
|
None,
|
|
description="Viewport bbox as lon_min,lat_min,lon_max,lat_max",
|
|
),
|
|
type: Optional[str] = Query(
|
|
None,
|
|
description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other",
|
|
),
|
|
limit: Optional[int] = Query(
|
|
None,
|
|
ge=0,
|
|
description="Maximum vessel features to return. Omit or pass 0 for no limit.",
|
|
),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""Return latest vessel positions as GeoJSON points."""
|
|
parsed_bbox = _parse_bbox(bbox)
|
|
requested_types = _requested_vessel_types(type)
|
|
merged_features, diagnostics = await _load_merged_vessel_features(db)
|
|
features = _filter_vessel_features(
|
|
merged_features,
|
|
bbox=parsed_bbox,
|
|
requested_types=requested_types,
|
|
)
|
|
if limit and limit > 0:
|
|
features = features[:limit]
|
|
return {
|
|
"type": "FeatureCollection",
|
|
"features": features,
|
|
"count": len(features),
|
|
"stats": _build_vessel_stats(features),
|
|
"diagnostics": {
|
|
**diagnostics,
|
|
"filtered_count": len(features),
|
|
},
|
|
}
|
|
|
|
|
|
async def _load_merged_vessel_features(db: AsyncSession) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
|
aggregated_vessels = await get_aggregated_vessels(db)
|
|
raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
|
|
|
|
latest_times = (
|
|
select(
|
|
VesselPosition.mmsi.label("mmsi"),
|
|
func.max(VesselPosition.received_at).label("received_at"),
|
|
)
|
|
.group_by(VesselPosition.mmsi)
|
|
.subquery()
|
|
)
|
|
stmt = (
|
|
select(VesselPosition, VesselStatic)
|
|
.join(
|
|
latest_times,
|
|
(VesselPosition.mmsi == latest_times.c.mmsi)
|
|
& (VesselPosition.received_at == latest_times.c.received_at),
|
|
)
|
|
.outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
|
|
.order_by(VesselPosition.received_at.desc())
|
|
)
|
|
|
|
result = await db.execute(stmt)
|
|
rows = list(result.all())
|
|
legacy_geojson = convert_vessels_to_geojson(rows)
|
|
merged_features, diagnostics = _merge_vessel_features(
|
|
raw_geojson.get("features", []),
|
|
legacy_geojson.get("features", []),
|
|
)
|
|
return merged_features, {
|
|
**diagnostics,
|
|
"raw_feature_count": len(raw_geojson.get("features", [])),
|
|
"legacy_feature_count": len(legacy_geojson.get("features", [])),
|
|
}
|
|
|
|
|
|
@router.get("/vessels/custom-supplements")
|
|
async def get_vessel_custom_supplements(db: AsyncSession = Depends(get_db)):
|
|
"""Group custom vessel_ais sources by their declared merge target for diagnostics."""
|
|
|
|
from app.models.datasource_config import DataSourceConfig
|
|
|
|
result = await db.execute(
|
|
select(DataSourceConfig.name, DataSourceConfig.config, DataSourceConfig.is_active)
|
|
.where(DataSourceConfig.config["target_schema"].as_string() == "vessel_ais")
|
|
)
|
|
grouped: dict[str, dict[str, Any]] = {}
|
|
for name, config, is_active in result.all():
|
|
config = config or {}
|
|
merge_target = str(config.get("merge_target_source") or "barentswatch_vessels")
|
|
bucket = grouped.setdefault(merge_target, {"merge_target": merge_target, "sources": []})
|
|
bucket["sources"].append({"name": name, "is_active": bool(is_active)})
|
|
return {"groups": list(grouped.values())}
|
|
|
|
|
|
@router.get("/vessels/name-fallbacks")
|
|
async def get_vessel_name_fallbacks(
|
|
limit: int = Query(500, ge=0, description="Maximum fallback-name vessels to return. 0 means no limit."),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""Return vessels whose display name still falls back to MMSI."""
|
|
aggregated_vessels = await get_aggregated_vessels(db)
|
|
raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
|
|
|
|
latest_times = (
|
|
select(
|
|
VesselPosition.mmsi.label("mmsi"),
|
|
func.max(VesselPosition.received_at).label("received_at"),
|
|
)
|
|
.group_by(VesselPosition.mmsi)
|
|
.subquery()
|
|
)
|
|
result = await db.execute(
|
|
select(VesselPosition, VesselStatic)
|
|
.join(
|
|
latest_times,
|
|
(VesselPosition.mmsi == latest_times.c.mmsi)
|
|
& (VesselPosition.received_at == latest_times.c.received_at),
|
|
)
|
|
.outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
|
|
.order_by(VesselPosition.received_at.desc())
|
|
)
|
|
legacy_geojson = convert_vessels_to_geojson(list(result.all()))
|
|
features, diagnostics = _merge_vessel_features(
|
|
raw_geojson.get("features", []),
|
|
legacy_geojson.get("features", []),
|
|
)
|
|
|
|
fallback_items = []
|
|
for feature in features:
|
|
props = feature.get("properties", {})
|
|
mmsi = props.get("mmsi")
|
|
name = props.get("name")
|
|
if not _is_vessel_name_fallback(name, mmsi):
|
|
continue
|
|
source_summary = props.get("source_summary") or {}
|
|
fallback_items.append(
|
|
{
|
|
"mmsi": str(mmsi),
|
|
"display_name": name or f"MMSI {mmsi}",
|
|
"reason": "missing_real_name",
|
|
"received_at": props.get("received_at"),
|
|
"sources": sorted(source_summary.keys()),
|
|
"source_summary": source_summary,
|
|
"message_types": sorted(
|
|
{
|
|
message_type
|
|
for summary in source_summary.values()
|
|
for message_type in (summary.get("message_types") or [])
|
|
}
|
|
),
|
|
"field_sources": props.get("field_sources") or {},
|
|
}
|
|
)
|
|
|
|
if limit and limit > 0:
|
|
fallback_items = fallback_items[:limit]
|
|
return {
|
|
"count": len(fallback_items),
|
|
"items": fallback_items,
|
|
"diagnostics": diagnostics,
|
|
}
|
|
|
|
|
|
@router.get("/vessels/{mmsi}")
|
|
async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)):
|
|
from app.services.vessel_enrichment import get_vessel_enrichment_bundle
|
|
|
|
aggregated = await get_aggregated_vessel(db, mmsi)
|
|
enrichment = await get_vessel_enrichment_bundle(db, mmsi)
|
|
if aggregated is not None:
|
|
return {
|
|
**aggregated,
|
|
"received_at": to_iso8601_utc(aggregated.get("received_at")),
|
|
"latitude": aggregated["lat"],
|
|
"longitude": aggregated["lon"],
|
|
"enrichment": enrichment,
|
|
}
|
|
|
|
latest_position_stmt = (
|
|
select(VesselPosition)
|
|
.where(VesselPosition.mmsi == mmsi)
|
|
.order_by(VesselPosition.received_at.desc())
|
|
.limit(1)
|
|
)
|
|
static = await db.get(VesselStatic, mmsi)
|
|
result = await db.execute(latest_position_stmt)
|
|
position = result.scalar_one_or_none()
|
|
if position is None:
|
|
raise HTTPException(status_code=404, detail="Vessel not found")
|
|
geojson = convert_vessels_to_geojson([(position, static)])
|
|
return {
|
|
**(geojson["features"][0]["properties"]),
|
|
"latitude": position.lat,
|
|
"longitude": position.lon,
|
|
"enrichment": enrichment,
|
|
}
|
|
|
|
|
|
@router.get("/vessels/{mmsi}/track")
|
|
async def get_vessel_track(
|
|
mmsi: int,
|
|
hours: int = Query(6, ge=1, le=24),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
cutoff = datetime.now(UTC) - timedelta(hours=hours)
|
|
aggregated_points = await get_aggregated_vessel_track(db, mmsi, cutoff=cutoff)
|
|
if aggregated_points:
|
|
return {
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {
|
|
"type": "LineString",
|
|
"coordinates": [[point["lon"], point["lat"]] for point in aggregated_points],
|
|
},
|
|
"properties": {
|
|
"mmsi": mmsi,
|
|
"hours": hours,
|
|
"point_count": len(aggregated_points),
|
|
"start_at": to_iso8601_utc(aggregated_points[0]["observed_at"]),
|
|
"end_at": to_iso8601_utc(aggregated_points[-1]["observed_at"]),
|
|
"point_sources": [point["source"] for point in aggregated_points],
|
|
},
|
|
}
|
|
],
|
|
"count": 1,
|
|
}
|
|
|
|
result = await db.execute(
|
|
select(VesselPosition)
|
|
.where(VesselPosition.mmsi == mmsi)
|
|
.where(VesselPosition.received_at >= cutoff)
|
|
.order_by(VesselPosition.received_at.asc())
|
|
)
|
|
positions = list(result.scalars().all())
|
|
if not positions:
|
|
return {
|
|
"type": "FeatureCollection",
|
|
"features": [],
|
|
"count": 0,
|
|
}
|
|
|
|
return {
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {
|
|
"type": "LineString",
|
|
"coordinates": [[position.lon, position.lat] for position in positions],
|
|
},
|
|
"properties": {
|
|
"mmsi": mmsi,
|
|
"hours": hours,
|
|
"point_count": len(positions),
|
|
"start_at": to_iso8601_utc(positions[0].received_at),
|
|
"end_at": to_iso8601_utc(positions[-1].received_at),
|
|
},
|
|
}
|
|
],
|
|
"count": 1,
|
|
}
|
|
|
|
|
|
@router.get("/vessels/{mmsi}/observations")
|
|
async def get_vessel_observations(
|
|
mmsi: int,
|
|
limit: int = Query(100, ge=1, le=500),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""Return raw AIS observations for debugging source-level collector facts."""
|
|
|
|
observations = await get_vessel_raw_observations(db, mmsi, limit=limit)
|
|
return {
|
|
"mmsi": mmsi,
|
|
"count": len(observations),
|
|
"observations": [item.to_dict() for item in observations],
|
|
"conflict_candidates": build_field_conflict_candidates(observations),
|
|
}
|
|
|
|
|
|
@router.get("/vessels/{mmsi}/conflicts")
|
|
async def get_vessel_conflicts(mmsi: int, db: AsyncSession = Depends(get_db)):
|
|
"""Return recorded AIS conflicts plus current raw-observation candidates."""
|
|
|
|
records = await get_vessel_conflict_records(db, mmsi)
|
|
observations = await get_vessel_raw_observations(db, mmsi, limit=500)
|
|
return {
|
|
"mmsi": mmsi,
|
|
"count": len(records),
|
|
"conflicts": [item.to_dict() for item in records],
|
|
"candidates": build_field_conflict_candidates(observations),
|
|
}
|
|
|
|
|
|
@router.get("/geo/bgp-anomalies")
|
|
async def get_bgp_anomalies_geojson(
|
|
severity: Optional[str] = Query(None),
|
|
status: Optional[str] = Query("active"),
|
|
limit: int = Query(200, ge=1, le=1000),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
stmt = select(BGPAnomaly).order_by(BGPAnomaly.created_at.desc()).limit(limit)
|
|
if severity:
|
|
stmt = stmt.where(BGPAnomaly.severity == severity)
|
|
if status:
|
|
stmt = stmt.where(BGPAnomaly.status == status)
|
|
|
|
result = await db.execute(stmt)
|
|
records = list(result.scalars().all())
|
|
geography_hints = await build_anomaly_geography_hints(db, records)
|
|
geojson = convert_bgp_anomalies_to_geojson(records, geography_hints)
|
|
return {**geojson, "count": len(geojson.get("features", []))}
|
|
|
|
|
|
@router.get("/geo/bgp-incidents")
|
|
async def get_bgp_incidents_geojson(
|
|
severity: Optional[str] = Query(None),
|
|
status: Optional[str] = Query("active"),
|
|
limit: int = Query(100, ge=1, le=500),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
stmt = select(BGPIncident).order_by(BGPIncident.created_at.desc()).limit(limit)
|
|
if severity:
|
|
stmt = stmt.where(BGPIncident.severity == severity)
|
|
if status:
|
|
stmt = stmt.where(BGPIncident.status == status)
|
|
|
|
result = await db.execute(stmt)
|
|
records = list(result.scalars().all())
|
|
geography_hints = await build_incident_geography_hints(db, records)
|
|
geojson = convert_bgp_incidents_to_geojson(records, geography_hints)
|
|
return {**geojson, "count": len(geojson.get("features", []))}
|
|
|
|
|
|
@router.get("/geo/bgp-collectors")
|
|
async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db)):
|
|
coverage = await build_bgp_collector_coverage(
|
|
db,
|
|
source_filter=("ris_live_bgp", "bgpstream_bgp"),
|
|
)
|
|
coverage_by_collector = {
|
|
item["collector"]: item
|
|
for item in coverage
|
|
if item.get("collector")
|
|
}
|
|
geojson = convert_bgp_collectors_to_geojson(coverage_by_collector)
|
|
return {**geojson, "count": len(geojson.get("features", []))}
|
|
|
|
|
|
@router.get("/geo/summary")
|
|
async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
|
|
"""Return lightweight Earth HUD counts without loading layer GeoJSON payloads."""
|
|
cable_count = await _count_current_or_latest_task_data(db, "arcgis_cables")
|
|
landing_point_count = await _count_current_or_latest_task_data(db, "arcgis_landing_points")
|
|
satellite_count = await _count_current_or_latest_task_data(
|
|
db,
|
|
"celestrak_tle",
|
|
exclude_unknown_name=True,
|
|
)
|
|
supercomputer_count = await _count_current_or_latest_task_data(db, "top500")
|
|
gpu_cluster_count = await _count_current_or_latest_task_data(db, "epoch_ai_gpu")
|
|
compute_center_count = supercomputer_count + gpu_cluster_count
|
|
|
|
active_incident_result = await db.execute(
|
|
select(func.count(BGPIncident.id)).where(BGPIncident.status == "active"),
|
|
)
|
|
active_anomaly_result = await db.execute(
|
|
select(func.count(BGPAnomaly.id)).where(BGPAnomaly.status == "active"),
|
|
)
|
|
active_incident_count = int(active_incident_result.scalar() or 0)
|
|
active_anomaly_count = int(active_anomaly_result.scalar() or 0)
|
|
bgp_collector_result = await db.execute(
|
|
select(func.count(func.distinct(BGPObservation.collector)))
|
|
.where(BGPObservation.collector.isnot(None))
|
|
.where(func.length(func.btrim(BGPObservation.collector)) > 0)
|
|
.where(BGPObservation.source.in_(("ris_live_bgp", "bgpstream_bgp")))
|
|
)
|
|
bgp_collector_scalar = bgp_collector_result.scalar()
|
|
if bgp_collector_scalar is None:
|
|
bgp_collectors = await build_bgp_collector_coverage(
|
|
db,
|
|
source_filter=("ris_live_bgp", "bgpstream_bgp"),
|
|
)
|
|
bgp_collector_count = len(
|
|
[item for item in bgp_collectors if item.get("collector")]
|
|
)
|
|
else:
|
|
bgp_collector_count = int(bgp_collector_scalar or 0)
|
|
raw_unique_window_hours = 24
|
|
raw_unique_mmsi = await count_unique_raw_vessel_mmsi(
|
|
db,
|
|
observed_since=datetime.now(UTC) - timedelta(hours=raw_unique_window_hours),
|
|
)
|
|
legacy_unique_result = await db.execute(
|
|
select(func.count(func.distinct(VesselPosition.mmsi)))
|
|
)
|
|
legacy_unique_mmsi = int(legacy_unique_result.scalar() or 0)
|
|
vessel_count = max(raw_unique_mmsi, legacy_unique_mmsi)
|
|
aisstream_health = await db.get(AISSourceHealth, "aisstream_vessels")
|
|
|
|
return {
|
|
"generated_at": to_iso8601_utc(datetime.now(UTC)),
|
|
"stats": {
|
|
"cable_count": cable_count,
|
|
"landing_point_count": landing_point_count,
|
|
"satellite_count": satellite_count,
|
|
"compute_center_count": compute_center_count,
|
|
"vessel_count": vessel_count,
|
|
"vessel_raw_unique_mmsi": raw_unique_mmsi,
|
|
"vessel_raw_unique_window_hours": raw_unique_window_hours,
|
|
"vessel_legacy_unique_mmsi": legacy_unique_mmsi,
|
|
"aisstream_connection_state": aisstream_health.connection_state if aisstream_health else None,
|
|
"aisstream_last_seen_at": to_iso8601_utc(aisstream_health.last_seen_at) if aisstream_health else None,
|
|
"aisstream_message_rate": aisstream_health.message_rate if aisstream_health else None,
|
|
"aisstream_lag_seconds": aisstream_health.lag_seconds if aisstream_health else None,
|
|
"supercomputer_count": supercomputer_count,
|
|
"gpu_cluster_count": gpu_cluster_count,
|
|
"bgp_event_count": active_incident_count or active_anomaly_count,
|
|
"bgp_incident_count": active_incident_count,
|
|
"bgp_anomaly_count": active_anomaly_count,
|
|
"bgp_collector_count": bgp_collector_count,
|
|
},
|
|
}
|
|
|
|
|
|
@router.get("/all")
|
|
async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
|
|
"""获取所有可视化数据的统一端点
|
|
|
|
Returns GeoJSON FeatureCollections for all data types:
|
|
- satellites: 卫星 TLE 数据
|
|
- cables: 海底电缆
|
|
- landing_points: 登陆点
|
|
- supercomputers: TOP500 超算
|
|
- gpu_clusters: GPU 集群
|
|
"""
|
|
records_by_source = await _load_current_collected_data_by_sources(
|
|
db,
|
|
[
|
|
"arcgis_cables",
|
|
"arcgis_landing_points",
|
|
"celestrak_tle",
|
|
"top500",
|
|
"epoch_ai_gpu",
|
|
],
|
|
)
|
|
cables_records = records_by_source.get("arcgis_cables", [])
|
|
points_records = records_by_source.get("arcgis_landing_points", [])
|
|
satellites_records = _filter_known_records(
|
|
records_by_source.get("celestrak_tle", []),
|
|
)
|
|
supercomputers_records = _filter_known_records(
|
|
records_by_source.get("top500", []),
|
|
)
|
|
gpu_records = _filter_known_records(
|
|
records_by_source.get("epoch_ai_gpu", []),
|
|
)
|
|
|
|
cables = (
|
|
convert_cable_to_geojson(cables_records)
|
|
if cables_records
|
|
else {"type": "FeatureCollection", "features": []}
|
|
)
|
|
landing_points = (
|
|
convert_landing_point_to_geojson(points_records)
|
|
if points_records
|
|
else {"type": "FeatureCollection", "features": []}
|
|
)
|
|
satellites = (
|
|
convert_satellite_to_geojson(satellites_records)
|
|
if satellites_records
|
|
else {"type": "FeatureCollection", "features": []}
|
|
)
|
|
supercomputers = (
|
|
convert_supercomputer_to_geojson(supercomputers_records)
|
|
if supercomputers_records
|
|
else {"type": "FeatureCollection", "features": []}
|
|
)
|
|
gpu_clusters = (
|
|
convert_gpu_cluster_to_geojson(gpu_records)
|
|
if gpu_records
|
|
else {"type": "FeatureCollection", "features": []}
|
|
)
|
|
|
|
return {
|
|
"generated_at": to_iso8601_utc(datetime.now(UTC)),
|
|
"version": "1.0",
|
|
"data": {
|
|
"satellites": satellites,
|
|
"cables": cables,
|
|
"landing_points": landing_points,
|
|
"supercomputers": supercomputers,
|
|
"gpu_clusters": gpu_clusters,
|
|
},
|
|
"stats": {
|
|
"total_features": (
|
|
len(satellites.get("features", []))
|
|
+ len(cables.get("features", []))
|
|
+ len(landing_points.get("features", []))
|
|
+ len(supercomputers.get("features", []))
|
|
+ len(gpu_clusters.get("features", []))
|
|
),
|
|
"satellites": len(satellites.get("features", [])),
|
|
"cables": len(cables.get("features", [])),
|
|
"landing_points": len(landing_points.get("features", [])),
|
|
"supercomputers": len(supercomputers.get("features", [])),
|
|
"gpu_clusters": len(gpu_clusters.get("features", [])),
|
|
},
|
|
}
|
|
|
|
|
|
# Cache for cable graph
|
|
_cable_graph: Optional[CableGraph] = None
|
|
|
|
|
|
async def get_cable_graph(db: AsyncSession) -> CableGraph:
|
|
"""Get or build cable graph (cached)"""
|
|
global _cable_graph
|
|
|
|
if _cable_graph is None:
|
|
cables_records = await _load_current_collected_data(db, "arcgis_cables")
|
|
points_records = await _load_current_collected_data(db, "arcgis_landing_points")
|
|
|
|
cables_data = convert_cable_to_geojson(cables_records)
|
|
points_data = convert_landing_point_to_geojson(points_records)
|
|
|
|
_cable_graph = build_graph_from_data(cables_data, points_data)
|
|
|
|
return _cable_graph
|
|
|
|
|
|
@router.post("/geo/path")
|
|
async def find_path(
|
|
start: List[float],
|
|
end: List[float],
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""Find shortest path between two coordinates via cable network"""
|
|
if not start or len(start) != 2:
|
|
raise HTTPException(status_code=400, detail="Start must be [lon, lat]")
|
|
if not end or len(end) != 2:
|
|
raise HTTPException(status_code=400, detail="End must be [lon, lat]")
|
|
|
|
graph = await get_cable_graph(db)
|
|
result = graph.find_shortest_path(start, end)
|
|
|
|
if not result:
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail="No path found between these points. They may be too far from any landing point.",
|
|
)
|
|
|
|
return result
|