linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.49.0

Browse Source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
linkong
2026-05-08 17:42:27 +08:00
parent bb9183b8a4
commit e1984c7a35
86 changed files with 9165 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

548
backend/app/api/v1/visualization.py
View File

@@ -4,17 +4,20 @@ Unified API for all visualization data sources.
Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium.
"""
import asyncio
import base64
from collections import OrderedDict
from datetime import UTC, datetime, timedelta
import math
import re
import httpx
from fastapi import APIRouter, HTTPException, Depends, Query, Response
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func
from typing import List, Dict, Any, Optional
from app.core.collected_data_fields import get_record_field
from app.core.countries import get_country_centroid
from app.core.satellite_tle import build_tle_lines_from_elements
from app.core.time import to_iso8601_utc
from app.db.session import get_db
@@ -25,6 +28,14 @@ from app.models.collected_data import CollectedData
from app.models.vessel import AISSourceHealth, VesselPosition, VesselStatic
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
from app.services.compute_center_locations import (
RENDERABLE_PRECISIONS,
ResolutionDiagnostic,
collect_location_candidates,
refresh_compute_center_location_cache,
resolve_compute_center_location_full,
upsert_compute_center_location,
)
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
from app.services.persistent_logs import record_system_log
from app.services.vessel_ais_aggregation import (
@@ -43,9 +54,23 @@ logger = get_logger(__name__, service="api")
TERRAIN_TILE_URL_TEMPLATE = (
"https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png"
)
TERRAIN_TILE_CACHE_MAX_ITEMS = 512
TERRAIN_TILE_BATCH_MAX_ITEMS = 128
TERRAIN_TILE_BATCH_CONCURRENCY = 16
_terrain_tile_cache: OrderedDict[tuple[int, int, int], tuple[bytes, str, dict[str, str]]] = OrderedDict()
VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE)
class TerrariumTileRequest(BaseModel):
z: int = Field(ge=0, le=14)
x: int = Field(ge=0)
y: int = Field(ge=0)
class TerrariumTileBatchRequest(BaseModel):
tiles: List[TerrariumTileRequest] = Field(min_length=1, max_length=TERRAIN_TILE_BATCH_MAX_ITEMS)
# ============== Converter Functions ==============
@@ -536,100 +561,6 @@ def _parse_float(value: Any) -> Optional[float]:
return None
COMPUTE_CENTER_COORDINATE_HINTS = (
("el capitan", 37.6819, -121.7681),
("livermore", 37.6819, -121.7681),
("llnl", 37.6819, -121.7681),
("lawrence livermore", 37.6819, -121.7681),
("frontier", 35.9319, -84.3107),
("oak ridge", 35.9319, -84.3107),
("ornl", 35.9319, -84.3107),
("aurora", 41.7130, -87.9820),
("argonne", 41.7130, -87.9820),
("anl", 41.7130, -87.9820),
("fugaku", 34.6953, 135.1974),
("kobe", 34.6953, 135.1974),
("riken", 34.6953, 135.1974),
("summit", 35.9319, -84.3107),
("leonardo", 44.4949, 11.3426),
("bologna", 44.4949, 11.3426),
("alps", 46.0037, 8.9511),
("lugano", 46.0037, 8.9511),
("sunway taihulight", 31.4912, 120.3119),
("wuxi", 31.4912, 120.3119),
("tianhe-2", 23.1291, 113.2644),
("tianhe-2a", 23.1291, 113.2644),
("guangzhou", 23.1291, 113.2644),
("colossus", 35.1495, -90.0490),
("memphis", 35.1495, -90.0490),
("xai", 35.1495, -90.0490),
)
def _normalize_hint_text(*parts: Any) -> str:
return " ".join(
str(part).strip().lower()
for part in parts
if part not in (None, "")
)
def _resolve_compute_center_coordinates(
record: CollectedData,
metadata: Dict[str, Any],
) -> Dict[str, Any]:
latitude = _parse_float(get_record_field(record, "latitude"))
longitude = _parse_float(get_record_field(record, "longitude"))
if latitude not in (None, 0.0) and longitude not in (None, 0.0):
return {
"latitude": latitude,
"longitude": longitude,
"location_precision": "precise",
"geography_mode": "source_coordinates",
"is_estimated": False,
"estimated_reason": None,
}
hint_text = _normalize_hint_text(
record.name,
get_record_field(record, "city"),
get_record_field(record, "country"),
metadata.get("site"),
metadata.get("organization"),
metadata.get("operator"),
)
for needle, resolved_latitude, resolved_longitude in COMPUTE_CENTER_COORDINATE_HINTS:
if needle in hint_text:
return {
"latitude": resolved_latitude,
"longitude": resolved_longitude,
"location_precision": "estimated_site",
"geography_mode": "site_hint",
"is_estimated": True,
"estimated_reason": f"Matched known site hint: {needle}",
}
centroid = get_country_centroid(get_record_field(record, "country"))
if centroid:
return {
"latitude": centroid.get("latitude"),
"longitude": centroid.get("longitude"),
"location_precision": "estimated_country",
"geography_mode": "country_centroid",
"is_estimated": True,
"estimated_reason": "Estimated from country centroid",
}
return {
"latitude": latitude,
"longitude": longitude,
"location_precision": "unknown",
"geography_mode": "unknown",
"is_estimated": True,
"estimated_reason": "No resolvable location hints",
}
def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str) -> str:
if capacity_value is None:
return "unknown"
@@ -654,22 +585,49 @@ def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str
def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
"""Convert compute infrastructure records into a unified GeoJSON layer."""
features = []
"""Convert compute infrastructure records into a unified GeoJSON layer.
Records that cannot be resolved to at least city-level precision are NOT
silently dropped: they are returned in ``unresolved`` so the UI can offer
the click-to-collect coordinate flow. The features list never contains
``[0, 0]`` placeholders or country/region/unknown precision points.
"""
features: List[Dict[str, Any]] = []
unresolved: List[Dict[str, Any]] = []
for record in records:
metadata = record.extra_data or {}
coordinate_info = _resolve_compute_center_coordinates(record, metadata)
latitude = coordinate_info.get("latitude")
longitude = coordinate_info.get("longitude")
result = resolve_compute_center_location_full(record, metadata)
site_type = (
"supercomputer"
if record.source == "top500" or record.data_type == "supercomputer"
else "gpu_cluster"
)
if latitude in (None, 0.0) or longitude in (None, 0.0):
if not result.is_resolved:
diagnostic = result.diagnostic or ResolutionDiagnostic(
failure_reason="Unknown resolver failure",
attempted_queries=(),
record_id=getattr(record, "id", None),
source=getattr(record, "source", None),
source_id=getattr(record, "source_id", None),
name=getattr(record, "name", None),
)
unresolved.append({
**diagnostic.to_dict(),
"site_type": site_type,
})
continue
location = result.location
if location is None or not location.is_renderable:
# Defensive: should not happen because is_resolved guards this.
continue
location_props = location.to_geojson_properties()
latitude = location.latitude
longitude = location.longitude
if site_type == "supercomputer":
capacity_value = _parse_float(get_record_field(record, "rmax"))
capacity_unit = "GFlops"
@@ -699,15 +657,16 @@ def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str
"id": record.id,
"geometry": {
"type": "Point",
"coordinates": [longitude or 0, latitude or 0],
"coordinates": [longitude, latitude],
},
"properties": {
"id": record.id,
"source_id": record.source_id,
"name": record.name,
"site_type": site_type,
"country": get_record_field(record, "country"),
"city": get_record_field(record, "city"),
"country": get_record_field(record, "country") or location.country,
"city": get_record_field(record, "city") or location.city,
"region": location.region,
"latitude": latitude,
"longitude": longitude,
"operator": operator,
@@ -723,17 +682,14 @@ def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str
"source": record.source,
"updated_at": updated_at,
"status": "observed",
"location_precision": coordinate_info.get("location_precision"),
"geography_mode": coordinate_info.get("geography_mode"),
"is_estimated": coordinate_info.get("is_estimated", False),
"estimated_reason": coordinate_info.get("estimated_reason"),
**location_props,
"data_type": "compute_center",
"metadata": metadata,
},
}
)
return {"type": "FeatureCollection", "features": features}
return {"type": "FeatureCollection", "features": features, "unresolved": unresolved}
VESSEL_TYPE_FILTERS = {
@@ -1486,18 +1442,12 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
@router.get("/terrain/terrarium/{z}/{x}/{y}.png")
async def get_terrarium_tile(z: int, x: int, y: int):
"""Proxy Terrarium elevation tiles through the backend to avoid browser CORS issues."""
if z < 0 or x < 0 or y < 0:
if not _is_valid_terrain_tile(z, x, y):
raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates")
url = TERRAIN_TILE_URL_TEMPLATE.format(z=z, x=x, y=y)
try:
async with httpx.AsyncClient(
timeout=20.0,
follow_redirects=True,
) as client:
upstream = await client.get(url)
upstream.raise_for_status()
async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
content, content_type, headers = await _fetch_terrain_tile(client, z, x, y)
except httpx.HTTPStatusError as exc:
raise HTTPException(
status_code=exc.response.status_code,
@@ -1509,22 +1459,140 @@ async def get_terrarium_tile(z: int, x: int, y: int):
detail=f"Terrain tile fetch failed: {exc}",
) from exc
return Response(
content=content,
media_type=content_type,
headers=headers,
)
def _is_valid_terrain_tile(z: int, x: int, y: int) -> bool:
if z < 0 or x < 0 or y < 0:
return False
max_tile = 2 ** z
return x < max_tile and y < max_tile
def _get_cached_terrain_tile(z: int, x: int, y: int) -> tuple[bytes, str, dict[str, str]] | None:
key = (z, x, y)
cached = _terrain_tile_cache.get(key)
if cached is None:
return None
_terrain_tile_cache.move_to_end(key)
content, content_type, headers = cached
return content, content_type, dict(headers)
def _cache_terrain_tile(
z: int,
x: int,
y: int,
content: bytes,
content_type: str,
headers: dict[str, str],
) -> None:
key = (z, x, y)
_terrain_tile_cache[key] = (content, content_type, dict(headers))
_terrain_tile_cache.move_to_end(key)
while len(_terrain_tile_cache) > TERRAIN_TILE_CACHE_MAX_ITEMS:
_terrain_tile_cache.popitem(last=False)
async def _fetch_terrain_tile(
client: httpx.AsyncClient,
z: int,
x: int,
y: int,
) -> tuple[bytes, str, dict[str, str]]:
cached = _get_cached_terrain_tile(z, x, y)
if cached is not None:
return cached
url = TERRAIN_TILE_URL_TEMPLATE.format(z=z, x=x, y=y)
upstream = await client.get(url)
upstream.raise_for_status()
cache_control = upstream.headers.get("cache-control") or "public, max-age=86400"
etag = upstream.headers.get("etag")
last_modified = upstream.headers.get("last-modified")
headers = {
"Cache-Control": cache_control,
}
etag = upstream.headers.get("etag")
last_modified = upstream.headers.get("last-modified")
if etag:
headers["ETag"] = etag
if last_modified:
headers["Last-Modified"] = last_modified
return Response(
content=upstream.content,
media_type=upstream.headers.get("content-type", "image/png"),
headers=headers,
)
content_type = upstream.headers.get("content-type", "image/png")
content = upstream.content
_cache_terrain_tile(z, x, y, content, content_type, headers)
return content, content_type, dict(headers)
@router.post("/terrain/terrarium/batch")
async def get_terrarium_tile_batch(payload: TerrariumTileBatchRequest):
"""Fetch Terrarium elevation tiles in batches so the browser avoids many tiny requests."""
unique_tiles: list[TerrariumTileRequest] = []
seen: set[tuple[int, int, int]] = set()
for tile in payload.tiles:
key = (tile.z, tile.x, tile.y)
if key in seen:
continue
seen.add(key)
if not _is_valid_terrain_tile(tile.z, tile.x, tile.y):
raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates")
unique_tiles.append(tile)
semaphore = asyncio.Semaphore(TERRAIN_TILE_BATCH_CONCURRENCY)
results: list[dict[str, Any]] = []
errors: list[dict[str, Any]] = []
async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
async def fetch_one(tile: TerrariumTileRequest) -> None:
async with semaphore:
try:
content, content_type, _headers = await _fetch_terrain_tile(
client,
tile.z,
tile.x,
tile.y,
)
results.append(
{
"z": tile.z,
"x": tile.x,
"y": tile.y,
"content_type": content_type,
"data": base64.b64encode(content).decode("ascii"),
},
)
except httpx.HTTPStatusError as exc:
errors.append(
{
"z": tile.z,
"x": tile.x,
"y": tile.y,
"status_code": exc.response.status_code,
"message": f"upstream error: {exc.response.status_code}",
},
)
except httpx.HTTPError as exc:
errors.append(
{
"z": tile.z,
"x": tile.x,
"y": tile.y,
"status_code": 502,
"message": str(exc),
},
)
await asyncio.gather(*(fetch_one(tile) for tile in unique_tiles))
return {
"tiles": results,
"errors": errors,
}
@router.get("/geo/all")
@@ -1659,33 +1727,253 @@ async def get_compute_centers_geojson(
return {
"type": "FeatureCollection",
"features": [],
"unresolved": [],
"count": 0,
"stats": {
"total": 0,
"supercomputers": 0,
"gpu_clusters": 0,
"unresolved": 0,
},
}
await refresh_compute_center_location_cache(db)
geojson = convert_compute_centers_to_geojson(records)
features = geojson.get("features", [])
unresolved = geojson.get("unresolved", [])
# Belt-and-suspenders: ensure no Feature ever sneaks through without
# city-or-better precision and finite, non-zero coordinates.
sanitized_features: List[Dict[str, Any]] = []
for feature in features:
coords = feature.get("geometry", {}).get("coordinates") or []
precision = feature.get("properties", {}).get("location_precision")
if precision not in RENDERABLE_PRECISIONS:
unresolved.append({
"failure_reason": f"Rejected non-renderable precision '{precision}'",
"record_id": feature.get("id"),
"source_id": feature.get("properties", {}).get("source_id"),
"name": feature.get("properties", {}).get("name"),
})
continue
if (
len(coords) != 2
or coords[0] in (None, 0, 0.0)
or coords[1] in (None, 0, 0.0)
):
unresolved.append({
"failure_reason": "Rejected feature with [0,0] or invalid coordinates",
"record_id": feature.get("id"),
"source_id": feature.get("properties", {}).get("source_id"),
"name": feature.get("properties", {}).get("name"),
})
continue
sanitized_features.append(feature)
return {
**geojson,
"count": len(features),
"type": "FeatureCollection",
"features": sanitized_features,
"unresolved": unresolved,
"count": len(sanitized_features),
"stats": {
"total": len(features),
"total": len(sanitized_features),
"supercomputers": sum(
1 for feature in features
1 for feature in sanitized_features
if feature.get("properties", {}).get("site_type") == "supercomputer"
),
"gpu_clusters": sum(
1 for feature in features
1 for feature in sanitized_features
if feature.get("properties", {}).get("site_type") == "gpu_cluster"
),
"unresolved": len(unresolved),
},
}
class CollectComputeCenterLocationRequest(BaseModel):
name: Optional[str] = None
source: Optional[str] = None
operator: Optional[str] = None
site: Optional[str] = None
organization: Optional[str] = None
city: Optional[str] = None
country: Optional[str] = None
record_id: Optional[int] = Field(default=None, alias="id")
model_config = {"populate_by_name": True}
class SaveComputeCenterLocationRequest(BaseModel):
source: Optional[str] = None
name: Optional[str] = None
operator: Optional[str] = None
site: Optional[str] = None
city: Optional[str] = None
country: Optional[str] = None
latitude: float
longitude: float
precision: str = "city"
confidence: Optional[float] = None
location_source: Optional[str] = None
source_url: Optional[str] = None
source_note: Optional[str] = None
raw_payload: Dict[str, Any] = Field(default_factory=dict)
needs_confirmation: bool = False
verification_status: Optional[str] = None
model_config = {"populate_by_name": True}
@router.post("/compute-centers/{source_id}/collect-location")
async def collect_compute_center_location(
source_id: str,
payload: CollectComputeCenterLocationRequest,
db: AsyncSession = Depends(get_db),
):
"""Run the full multi-query location collection pipeline for a record.
The endpoint accepts the source_id of a compute center plus contextual
fields (name/operator/site/city/country/...) and returns ranked candidate
locations from source coordinates, open organization lookups, and online
geocoding combinations. The caller never has to type coordinates by hand:
if any candidate is accepted it can be applied directly. If no candidate
can reach city-level precision the response includes an explicit
``failure_reason`` and the list of attempted queries.
"""
if not source_id or not source_id.strip():
raise HTTPException(status_code=400, detail="source_id is required")
record = await _load_compute_center_record(db, source_id)
name = payload.name or (record.name if record else None)
metadata = (record.extra_data or {}) if record else {}
operator = payload.operator or metadata.get("operator") or metadata.get("organization") or metadata.get("owner")
site = payload.site or metadata.get("site")
organization = payload.organization or metadata.get("organization")
city = payload.city or get_record_field(record, "city") if record else payload.city
country = payload.country or (get_record_field(record, "country") if record else None)
source = payload.source or (record.source if record else None)
record_id = payload.record_id or (record.id if record else None)
candidates, attempted_queries = collect_location_candidates(
name=name,
source=source,
source_id=source_id,
operator=operator,
site=site,
organization=organization,
city=city,
country=country,
record_id=record_id,
)
if not candidates:
return {
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": False,
"failure_reason": (
"No source coordinates, organization lookup, or online geocoding"
" result reached city-level precision."
),
"candidates": [],
"attempted_queries": list(attempted_queries),
"context": {
"name": name,
"operator": operator,
"site": site,
"city": city,
"country": country,
},
}
return {
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": True,
"candidates": [candidate.to_dict() for candidate in candidates],
"best_candidate": candidates[0].to_dict(),
"attempted_queries": list(attempted_queries),
"context": {
"name": name,
"operator": operator,
"site": site,
"city": city,
"country": country,
},
}
@router.post("/compute-centers/{source_id}/location")
async def save_compute_center_location(
source_id: str,
payload: SaveComputeCenterLocationRequest,
db: AsyncSession = Depends(get_db),
):
"""Persist the user-selected compute-center location candidate."""
if not source_id or not source_id.strip():
raise HTTPException(status_code=400, detail="source_id is required")
if payload.latitude in (0.0, None) or payload.longitude in (0.0, None):
raise HTTPException(status_code=400, detail="latitude/longitude are required")
if payload.precision not in RENDERABLE_PRECISIONS:
raise HTTPException(status_code=400, detail="precision must be precise, site, or city")
record = await _load_compute_center_record(db, source_id)
metadata = (record.extra_data or {}) if record else {}
record_source = payload.source or (record.source if record else None)
if not record_source:
raise HTTPException(status_code=400, detail="source is required for unknown compute center")
operator = (
payload.operator
or metadata.get("operator")
or metadata.get("organization")
or metadata.get("owner")
or metadata.get("manufacturer")
)
site = payload.site or metadata.get("site") or metadata.get("organization")
saved = await upsert_compute_center_location(
db,
source=record_source,
source_id=source_id,
name=payload.name or (record.name if record else None),
operator=operator,
site=site,
city=payload.city or (get_record_field(record, "city") if record else None),
country=payload.country or (get_record_field(record, "country") if record else None),
latitude=payload.latitude,
longitude=payload.longitude,
precision=payload.precision,
confidence=payload.confidence,
location_source=payload.location_source or "manual_selection",
source_url=payload.source_url,
source_note=payload.source_note,
raw_payload=payload.raw_payload,
needs_confirmation=payload.needs_confirmation,
verification_status=payload.verification_status
or ("unverified" if payload.needs_confirmation else "verified"),
)
return {
"success": True,
"source": saved.source,
"source_id": saved.source_id,
"location": saved.to_location_dict(),
}
async def _load_compute_center_record(db: AsyncSession, source_id: str) -> CollectedData | None:
stmt = (
select(CollectedData)
.where(CollectedData.source_id == source_id)
.where(CollectedData.source.in_(["top500", "epoch_ai_gpu"]))
.order_by(CollectedData.is_current.desc(), CollectedData.id.desc())
.limit(1)
)
result = await db.execute(stmt)
return result.scalars().first()
@router.get("/geo/vessels")
async def get_vessels_geojson(
bbox: Optional[str] = Query(