linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.49.0

Browse Source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
linkong
2026-05-08 17:42:27 +08:00
parent bb9183b8a4
commit e1984c7a35
86 changed files with 9165 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/app/api/v1/auth.py
View File

@@ -28,7 +28,7 @@ async def login(
):
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active FROM users WHERE username = :username"
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE username = :username"
),
{"username": form_data.username},
)
@@ -46,6 +46,7 @@ async def login(
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
if not verify_password(form_data.password, user.password_hash):
raise HTTPException(
@@ -73,6 +74,7 @@ async def login(
"id": user.id,
"username": user.username,
"role": user.role,
"gatekeeper_groups": user.gatekeeper_groups or [],
},
}
@@ -95,6 +97,7 @@ async def refresh_token(
"id": current_user.id,
"username": current_user.username,
"role": current_user.role,
"gatekeeper_groups": current_user.gatekeeper_groups or [],
},
}
@@ -111,6 +114,7 @@ async def get_me(current_user: User = Depends(get_current_user)):
"username": current_user.username,
"email": current_user.email,
"role": current_user.role,
"gatekeeper_groups": current_user.gatekeeper_groups or [],
"is_active": current_user.is_active,
"created_at": current_user.created_at,
}

77
backend/app/api/v1/bgp.py
View File

@@ -5,12 +5,18 @@ from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from pydantic import BaseModel
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.models.user import User
from app.services.bgp_collector_locations import (
collect_bgp_collector_location_candidates,
get_bgp_collector_location_dict,
)
from app.services.bgp_collectors import build_bgp_collector_coverage
router = APIRouter()
@@ -264,6 +270,77 @@ async def get_bgp_collector_summary(
}
class CollectBGPCollectorLocationRequest(BaseModel):
city: Optional[str] = None
country: Optional[str] = None
site: Optional[str] = None
operator: Optional[str] = None
@router.post("/collectors/{collector_id}/collect-location")
async def collect_bgp_collector_location(
collector_id: str,
payload: CollectBGPCollectorLocationRequest,
current_user: User = Depends(get_current_user),
):
"""Run the shared location pipeline for a BGP route collector.
Mirrors ``POST /api/v1/visualization/compute-centers/{source_id}/collect-location``.
Returns ranked candidates from source coordinates and Nominatim queries
built around the collector's stored context (IXP / city / country). Stored
collector locations provide context only; they are not emitted as
candidates.
"""
if not collector_id or not collector_id.strip():
raise HTTPException(status_code=400, detail="collector_id is required")
legacy = get_bgp_collector_location_dict(collector_id) or {}
site = payload.site or legacy.get("matched_location_name")
city = payload.city or legacy.get("city")
country = payload.country or legacy.get("country")
operator = payload.operator or "RIPE NCC"
candidates, attempted_queries = collect_bgp_collector_location_candidates(
collector=collector_id,
site=site,
city=city,
country=country,
operator=operator,
)
context = {
"collector": collector_id,
"site": site,
"city": city,
"country": country,
"operator": operator,
}
if not candidates:
return {
"collector_id": collector_id,
"name": collector_id,
"success": False,
"failure_reason": (
"No source coordinates or online geocoding result reached"
" city-level precision for this collector."
),
"candidates": [],
"attempted_queries": list(attempted_queries),
"context": context,
}
return {
"collector_id": collector_id,
"name": collector_id,
"success": True,
"candidates": [candidate.to_dict() for candidate in candidates],
"best_candidate": candidates[0].to_dict(),
"attempted_queries": list(attempted_queries),
"context": context,
}
@router.get("/overview/summary")
async def get_bgp_overview_summary(
current_user: User = Depends(get_current_user),

102
backend/app/api/v1/docs.py Normal file
View File

@@ -0,0 +1,102 @@
"""Authenticated documentation APIs."""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from sqlalchemy import text
from app.core.security import decode_token
from app.db.session import async_session_factory
from app.models.user import User
from app.services.docs_gatekeeper import (
DOCS_BY_SLUG,
VALID_DOCS_LANGS,
can_read_doc,
catalog_for_user,
doc_path_for,
title_for,
)
router = APIRouter()
optional_bearer = HTTPBearer(auto_error=False)
async def get_optional_current_user(
credentials: HTTPAuthorizationCredentials | None = Depends(optional_bearer),
) -> User | None:
if credentials is None:
return None
payload = decode_token(credentials.credentials)
if payload is None or payload.get("type") != "access" or payload.get("sub") is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid token",
)
async with async_session_factory() as db:
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": int(payload["sub"])},
)
row = result.fetchone()
if row is None or not row[5]:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="User not found or inactive",
)
user = User()
user.id = row[0]
user.username = row[1]
user.email = row[2]
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
return user
@router.get("/catalog")
async def get_docs_catalog(current_user: User | None = Depends(get_optional_current_user)):
return {
"items": catalog_for_user(current_user),
"authenticated": current_user is not None,
}
@router.get("/{lang}/{slug}")
async def get_doc_content(
lang: str,
slug: str,
current_user: User | None = Depends(get_optional_current_user),
):
if lang not in VALID_DOCS_LANGS:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
entry = DOCS_BY_SLUG.get(slug)
if entry is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
path = doc_path_for(entry, lang)
if not path.exists():
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
if not can_read_doc(entry, current_user):
if current_user is None:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authentication required")
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Insufficient Docs permissions")
return {
"slug": entry.slug,
"filename": entry.filename,
"lang": lang,
"title": title_for(entry, lang),
"group": entry.group,
"order": entry.order,
"access": entry.access,
"markdown": path.read_text(encoding="utf-8"),
}

38
backend/app/api/v1/users.py
View File

@@ -1,3 +1,4 @@
import json
from typing import List
from fastapi import APIRouter, Depends, HTTPException, status
@@ -7,10 +8,12 @@ from sqlalchemy import text
from app.core.security import get_current_user, get_password_hash
from app.db.session import get_db
from app.models.user import User
from app.schemas.user import UserCreate, UserResponse, UserUpdate
from app.schemas.user import UserCreate, UserUpdate
router = APIRouter()
VALID_GATEKEEPER_GROUPS = {"docs_user", "docs_developer", "docs_admin"}
def check_permission(current_user: User, required_roles: List[str]) -> bool:
user_role_value = (
@@ -52,7 +55,7 @@ async def list_users(
offset = (page - 1) * page_size
query = text(
f"SELECT id, username, email, role, is_active, last_login_at, created_at FROM users WHERE {where_sql} ORDER BY created_at DESC LIMIT {page_size} OFFSET {offset}"
f"SELECT id, username, email, role, is_active, last_login_at, created_at, gatekeeper_groups FROM users WHERE {where_sql} ORDER BY created_at DESC LIMIT {page_size} OFFSET {offset}"
)
count_query = text(f"SELECT COUNT(*) FROM users WHERE {where_sql}")
@@ -75,6 +78,7 @@ async def list_users(
"is_active": u[4],
"last_login_at": u[5],
"created_at": u[6],
"gatekeeper_groups": u[7] or [],
}
for u in users
],
@@ -95,7 +99,7 @@ async def get_user(
result = await db.execute(
text(
"SELECT id, username, email, role, is_active, last_login_at, created_at FROM users WHERE id = :id"
"SELECT id, username, email, role, is_active, last_login_at, created_at, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": user_id},
)
@@ -114,6 +118,7 @@ async def get_user(
"is_active": user[4],
"last_login_at": user[5],
"created_at": user[6],
"gatekeeper_groups": user[7] or [],
}
@@ -128,6 +133,12 @@ async def create_user(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can create users",
)
invalid_groups = sorted(set(user_data.gatekeeper_groups) - VALID_GATEKEEPER_GROUPS)
if invalid_groups:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unsupported Gatekeeper groups: {', '.join(invalid_groups)}",
)
result = await db.execute(
text("SELECT id FROM users WHERE username = :username OR email = :email"),
@@ -142,13 +153,14 @@ async def create_user(
hashed_password = get_password_hash(user_data.password)
await db.execute(
text("""INSERT INTO users (username, email, password_hash, role, is_active, created_at, updated_at)
VALUES (:username, :email, :password_hash, :role, :is_active, NOW(), NOW())"""),
text("""INSERT INTO users (username, email, password_hash, role, gatekeeper_groups, is_active, created_at, updated_at)
VALUES (:username, :email, :password_hash, :role, CAST(:gatekeeper_groups AS jsonb), :is_active, NOW(), NOW())"""),
{
"username": user_data.username,
"email": user_data.email,
"password_hash": hashed_password,
"role": user_data.role,
"gatekeeper_groups": json.dumps(user_data.gatekeeper_groups),
"is_active": True,
},
)
@@ -172,6 +184,7 @@ async def create_user(
"username": user_data.username,
"email": user_data.email,
"role": user_data.role,
"gatekeeper_groups": user_data.gatekeeper_groups,
"is_active": True,
}
@@ -194,6 +207,18 @@ async def update_user(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can change user role",
)
if not check_permission(current_user, ["super_admin"]) and user_data.gatekeeper_groups is not None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can change Gatekeeper groups",
)
if user_data.gatekeeper_groups is not None:
invalid_groups = sorted(set(user_data.gatekeeper_groups) - VALID_GATEKEEPER_GROUPS)
if invalid_groups:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unsupported Gatekeeper groups: {', '.join(invalid_groups)}",
)
result = await db.execute(
text("SELECT id FROM users WHERE id = :id"),
@@ -213,6 +238,9 @@ async def update_user(
if user_data.role is not None:
update_fields.append("role = :role")
params["role"] = user_data.role
if user_data.gatekeeper_groups is not None:
update_fields.append("gatekeeper_groups = CAST(:gatekeeper_groups AS jsonb)")
params["gatekeeper_groups"] = json.dumps(user_data.gatekeeper_groups)
if user_data.is_active is not None:
update_fields.append("is_active = :is_active")
params["is_active"] = user_data.is_active

548
backend/app/api/v1/visualization.py
View File

@@ -4,17 +4,20 @@ Unified API for all visualization data sources.
Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium.
"""
import asyncio
import base64
from collections import OrderedDict
from datetime import UTC, datetime, timedelta
import math
import re
import httpx
from fastapi import APIRouter, HTTPException, Depends, Query, Response
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func
from typing import List, Dict, Any, Optional
from app.core.collected_data_fields import get_record_field
from app.core.countries import get_country_centroid
from app.core.satellite_tle import build_tle_lines_from_elements
from app.core.time import to_iso8601_utc
from app.db.session import get_db
@@ -25,6 +28,14 @@ from app.models.collected_data import CollectedData
from app.models.vessel import AISSourceHealth, VesselPosition, VesselStatic
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
from app.services.compute_center_locations import (
RENDERABLE_PRECISIONS,
ResolutionDiagnostic,
collect_location_candidates,
refresh_compute_center_location_cache,
resolve_compute_center_location_full,
upsert_compute_center_location,
)
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
from app.services.persistent_logs import record_system_log
from app.services.vessel_ais_aggregation import (
@@ -43,9 +54,23 @@ logger = get_logger(__name__, service="api")
TERRAIN_TILE_URL_TEMPLATE = (
"https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png"
)
TERRAIN_TILE_CACHE_MAX_ITEMS = 512
TERRAIN_TILE_BATCH_MAX_ITEMS = 128
TERRAIN_TILE_BATCH_CONCURRENCY = 16
_terrain_tile_cache: OrderedDict[tuple[int, int, int], tuple[bytes, str, dict[str, str]]] = OrderedDict()
VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE)
class TerrariumTileRequest(BaseModel):
z: int = Field(ge=0, le=14)
x: int = Field(ge=0)
y: int = Field(ge=0)
class TerrariumTileBatchRequest(BaseModel):
tiles: List[TerrariumTileRequest] = Field(min_length=1, max_length=TERRAIN_TILE_BATCH_MAX_ITEMS)
# ============== Converter Functions ==============
@@ -536,100 +561,6 @@ def _parse_float(value: Any) -> Optional[float]:
return None
COMPUTE_CENTER_COORDINATE_HINTS = (
("el capitan", 37.6819, -121.7681),
("livermore", 37.6819, -121.7681),
("llnl", 37.6819, -121.7681),
("lawrence livermore", 37.6819, -121.7681),
("frontier", 35.9319, -84.3107),
("oak ridge", 35.9319, -84.3107),
("ornl", 35.9319, -84.3107),
("aurora", 41.7130, -87.9820),
("argonne", 41.7130, -87.9820),
("anl", 41.7130, -87.9820),
("fugaku", 34.6953, 135.1974),
("kobe", 34.6953, 135.1974),
("riken", 34.6953, 135.1974),
("summit", 35.9319, -84.3107),
("leonardo", 44.4949, 11.3426),
("bologna", 44.4949, 11.3426),
("alps", 46.0037, 8.9511),
("lugano", 46.0037, 8.9511),
("sunway taihulight", 31.4912, 120.3119),
("wuxi", 31.4912, 120.3119),
("tianhe-2", 23.1291, 113.2644),
("tianhe-2a", 23.1291, 113.2644),
("guangzhou", 23.1291, 113.2644),
("colossus", 35.1495, -90.0490),
("memphis", 35.1495, -90.0490),
("xai", 35.1495, -90.0490),
)
def _normalize_hint_text(*parts: Any) -> str:
return " ".join(
str(part).strip().lower()
for part in parts
if part not in (None, "")
)
def _resolve_compute_center_coordinates(
record: CollectedData,
metadata: Dict[str, Any],
) -> Dict[str, Any]:
latitude = _parse_float(get_record_field(record, "latitude"))
longitude = _parse_float(get_record_field(record, "longitude"))
if latitude not in (None, 0.0) and longitude not in (None, 0.0):
return {
"latitude": latitude,
"longitude": longitude,
"location_precision": "precise",
"geography_mode": "source_coordinates",
"is_estimated": False,
"estimated_reason": None,
}
hint_text = _normalize_hint_text(
record.name,
get_record_field(record, "city"),
get_record_field(record, "country"),
metadata.get("site"),
metadata.get("organization"),
metadata.get("operator"),
)
for needle, resolved_latitude, resolved_longitude in COMPUTE_CENTER_COORDINATE_HINTS:
if needle in hint_text:
return {
"latitude": resolved_latitude,
"longitude": resolved_longitude,
"location_precision": "estimated_site",
"geography_mode": "site_hint",
"is_estimated": True,
"estimated_reason": f"Matched known site hint: {needle}",
}
centroid = get_country_centroid(get_record_field(record, "country"))
if centroid:
return {
"latitude": centroid.get("latitude"),
"longitude": centroid.get("longitude"),
"location_precision": "estimated_country",
"geography_mode": "country_centroid",
"is_estimated": True,
"estimated_reason": "Estimated from country centroid",
}
return {
"latitude": latitude,
"longitude": longitude,
"location_precision": "unknown",
"geography_mode": "unknown",
"is_estimated": True,
"estimated_reason": "No resolvable location hints",
}
def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str) -> str:
if capacity_value is None:
return "unknown"
@@ -654,22 +585,49 @@ def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str
def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
"""Convert compute infrastructure records into a unified GeoJSON layer."""
features = []
"""Convert compute infrastructure records into a unified GeoJSON layer.
Records that cannot be resolved to at least city-level precision are NOT
silently dropped: they are returned in ``unresolved`` so the UI can offer
the click-to-collect coordinate flow. The features list never contains
``[0, 0]`` placeholders or country/region/unknown precision points.
"""
features: List[Dict[str, Any]] = []
unresolved: List[Dict[str, Any]] = []
for record in records:
metadata = record.extra_data or {}
coordinate_info = _resolve_compute_center_coordinates(record, metadata)
latitude = coordinate_info.get("latitude")
longitude = coordinate_info.get("longitude")
result = resolve_compute_center_location_full(record, metadata)
site_type = (
"supercomputer"
if record.source == "top500" or record.data_type == "supercomputer"
else "gpu_cluster"
)
if latitude in (None, 0.0) or longitude in (None, 0.0):
if not result.is_resolved:
diagnostic = result.diagnostic or ResolutionDiagnostic(
failure_reason="Unknown resolver failure",
attempted_queries=(),
record_id=getattr(record, "id", None),
source=getattr(record, "source", None),
source_id=getattr(record, "source_id", None),
name=getattr(record, "name", None),
)
unresolved.append({
**diagnostic.to_dict(),
"site_type": site_type,
})
continue
location = result.location
if location is None or not location.is_renderable:
# Defensive: should not happen because is_resolved guards this.
continue
location_props = location.to_geojson_properties()
latitude = location.latitude
longitude = location.longitude
if site_type == "supercomputer":
capacity_value = _parse_float(get_record_field(record, "rmax"))
capacity_unit = "GFlops"
@@ -699,15 +657,16 @@ def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str
"id": record.id,
"geometry": {
"type": "Point",
"coordinates": [longitude or 0, latitude or 0],
"coordinates": [longitude, latitude],
},
"properties": {
"id": record.id,
"source_id": record.source_id,
"name": record.name,
"site_type": site_type,
"country": get_record_field(record, "country"),
"city": get_record_field(record, "city"),
"country": get_record_field(record, "country") or location.country,
"city": get_record_field(record, "city") or location.city,
"region": location.region,
"latitude": latitude,
"longitude": longitude,
"operator": operator,
@@ -723,17 +682,14 @@ def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str
"source": record.source,
"updated_at": updated_at,
"status": "observed",
"location_precision": coordinate_info.get("location_precision"),
"geography_mode": coordinate_info.get("geography_mode"),
"is_estimated": coordinate_info.get("is_estimated", False),
"estimated_reason": coordinate_info.get("estimated_reason"),
**location_props,
"data_type": "compute_center",
"metadata": metadata,
},
}
)
return {"type": "FeatureCollection", "features": features}
return {"type": "FeatureCollection", "features": features, "unresolved": unresolved}
VESSEL_TYPE_FILTERS = {
@@ -1486,18 +1442,12 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
@router.get("/terrain/terrarium/{z}/{x}/{y}.png")
async def get_terrarium_tile(z: int, x: int, y: int):
"""Proxy Terrarium elevation tiles through the backend to avoid browser CORS issues."""
if z < 0 or x < 0 or y < 0:
if not _is_valid_terrain_tile(z, x, y):
raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates")
url = TERRAIN_TILE_URL_TEMPLATE.format(z=z, x=x, y=y)
try:
async with httpx.AsyncClient(
timeout=20.0,
follow_redirects=True,
) as client:
upstream = await client.get(url)
upstream.raise_for_status()
async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
content, content_type, headers = await _fetch_terrain_tile(client, z, x, y)
except httpx.HTTPStatusError as exc:
raise HTTPException(
status_code=exc.response.status_code,
@@ -1509,22 +1459,140 @@ async def get_terrarium_tile(z: int, x: int, y: int):
detail=f"Terrain tile fetch failed: {exc}",
) from exc
return Response(
content=content,
media_type=content_type,
headers=headers,
)
def _is_valid_terrain_tile(z: int, x: int, y: int) -> bool:
if z < 0 or x < 0 or y < 0:
return False
max_tile = 2 ** z
return x < max_tile and y < max_tile
def _get_cached_terrain_tile(z: int, x: int, y: int) -> tuple[bytes, str, dict[str, str]] | None:
key = (z, x, y)
cached = _terrain_tile_cache.get(key)
if cached is None:
return None
_terrain_tile_cache.move_to_end(key)
content, content_type, headers = cached
return content, content_type, dict(headers)
def _cache_terrain_tile(
z: int,
x: int,
y: int,
content: bytes,
content_type: str,
headers: dict[str, str],
) -> None:
key = (z, x, y)
_terrain_tile_cache[key] = (content, content_type, dict(headers))
_terrain_tile_cache.move_to_end(key)
while len(_terrain_tile_cache) > TERRAIN_TILE_CACHE_MAX_ITEMS:
_terrain_tile_cache.popitem(last=False)
async def _fetch_terrain_tile(
client: httpx.AsyncClient,
z: int,
x: int,
y: int,
) -> tuple[bytes, str, dict[str, str]]:
cached = _get_cached_terrain_tile(z, x, y)
if cached is not None:
return cached
url = TERRAIN_TILE_URL_TEMPLATE.format(z=z, x=x, y=y)
upstream = await client.get(url)
upstream.raise_for_status()
cache_control = upstream.headers.get("cache-control") or "public, max-age=86400"
etag = upstream.headers.get("etag")
last_modified = upstream.headers.get("last-modified")
headers = {
"Cache-Control": cache_control,
}
etag = upstream.headers.get("etag")
last_modified = upstream.headers.get("last-modified")
if etag:
headers["ETag"] = etag
if last_modified:
headers["Last-Modified"] = last_modified
return Response(
content=upstream.content,
media_type=upstream.headers.get("content-type", "image/png"),
headers=headers,
)
content_type = upstream.headers.get("content-type", "image/png")
content = upstream.content
_cache_terrain_tile(z, x, y, content, content_type, headers)
return content, content_type, dict(headers)
@router.post("/terrain/terrarium/batch")
async def get_terrarium_tile_batch(payload: TerrariumTileBatchRequest):
"""Fetch Terrarium elevation tiles in batches so the browser avoids many tiny requests."""
unique_tiles: list[TerrariumTileRequest] = []
seen: set[tuple[int, int, int]] = set()
for tile in payload.tiles:
key = (tile.z, tile.x, tile.y)
if key in seen:
continue
seen.add(key)
if not _is_valid_terrain_tile(tile.z, tile.x, tile.y):
raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates")
unique_tiles.append(tile)
semaphore = asyncio.Semaphore(TERRAIN_TILE_BATCH_CONCURRENCY)
results: list[dict[str, Any]] = []
errors: list[dict[str, Any]] = []
async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
async def fetch_one(tile: TerrariumTileRequest) -> None:
async with semaphore:
try:
content, content_type, _headers = await _fetch_terrain_tile(
client,
tile.z,
tile.x,
tile.y,
)
results.append(
{
"z": tile.z,
"x": tile.x,
"y": tile.y,
"content_type": content_type,
"data": base64.b64encode(content).decode("ascii"),
},
)
except httpx.HTTPStatusError as exc:
errors.append(
{
"z": tile.z,
"x": tile.x,
"y": tile.y,
"status_code": exc.response.status_code,
"message": f"upstream error: {exc.response.status_code}",
},
)
except httpx.HTTPError as exc:
errors.append(
{
"z": tile.z,
"x": tile.x,
"y": tile.y,
"status_code": 502,
"message": str(exc),
},
)
await asyncio.gather(*(fetch_one(tile) for tile in unique_tiles))
return {
"tiles": results,
"errors": errors,
}
@router.get("/geo/all")
@@ -1659,33 +1727,253 @@ async def get_compute_centers_geojson(
return {
"type": "FeatureCollection",
"features": [],
"unresolved": [],
"count": 0,
"stats": {
"total": 0,
"supercomputers": 0,
"gpu_clusters": 0,
"unresolved": 0,
},
}
await refresh_compute_center_location_cache(db)
geojson = convert_compute_centers_to_geojson(records)
features = geojson.get("features", [])
unresolved = geojson.get("unresolved", [])
# Belt-and-suspenders: ensure no Feature ever sneaks through without
# city-or-better precision and finite, non-zero coordinates.
sanitized_features: List[Dict[str, Any]] = []
for feature in features:
coords = feature.get("geometry", {}).get("coordinates") or []
precision = feature.get("properties", {}).get("location_precision")
if precision not in RENDERABLE_PRECISIONS:
unresolved.append({
"failure_reason": f"Rejected non-renderable precision '{precision}'",
"record_id": feature.get("id"),
"source_id": feature.get("properties", {}).get("source_id"),
"name": feature.get("properties", {}).get("name"),
})
continue
if (
len(coords) != 2
or coords[0] in (None, 0, 0.0)
or coords[1] in (None, 0, 0.0)
):
unresolved.append({
"failure_reason": "Rejected feature with [0,0] or invalid coordinates",
"record_id": feature.get("id"),
"source_id": feature.get("properties", {}).get("source_id"),
"name": feature.get("properties", {}).get("name"),
})
continue
sanitized_features.append(feature)
return {
**geojson,
"count": len(features),
"type": "FeatureCollection",
"features": sanitized_features,
"unresolved": unresolved,
"count": len(sanitized_features),
"stats": {
"total": len(features),
"total": len(sanitized_features),
"supercomputers": sum(
1 for feature in features
1 for feature in sanitized_features
if feature.get("properties", {}).get("site_type") == "supercomputer"
),
"gpu_clusters": sum(
1 for feature in features
1 for feature in sanitized_features
if feature.get("properties", {}).get("site_type") == "gpu_cluster"
),
"unresolved": len(unresolved),
},
}
class CollectComputeCenterLocationRequest(BaseModel):
name: Optional[str] = None
source: Optional[str] = None
operator: Optional[str] = None
site: Optional[str] = None
organization: Optional[str] = None
city: Optional[str] = None
country: Optional[str] = None
record_id: Optional[int] = Field(default=None, alias="id")
model_config = {"populate_by_name": True}
class SaveComputeCenterLocationRequest(BaseModel):
source: Optional[str] = None
name: Optional[str] = None
operator: Optional[str] = None
site: Optional[str] = None
city: Optional[str] = None
country: Optional[str] = None
latitude: float
longitude: float
precision: str = "city"
confidence: Optional[float] = None
location_source: Optional[str] = None
source_url: Optional[str] = None
source_note: Optional[str] = None
raw_payload: Dict[str, Any] = Field(default_factory=dict)
needs_confirmation: bool = False
verification_status: Optional[str] = None
model_config = {"populate_by_name": True}
@router.post("/compute-centers/{source_id}/collect-location")
async def collect_compute_center_location(
source_id: str,
payload: CollectComputeCenterLocationRequest,
db: AsyncSession = Depends(get_db),
):
"""Run the full multi-query location collection pipeline for a record.
The endpoint accepts the source_id of a compute center plus contextual
fields (name/operator/site/city/country/...) and returns ranked candidate
locations from source coordinates, open organization lookups, and online
geocoding combinations. The caller never has to type coordinates by hand:
if any candidate is accepted it can be applied directly. If no candidate
can reach city-level precision the response includes an explicit
``failure_reason`` and the list of attempted queries.
"""
if not source_id or not source_id.strip():
raise HTTPException(status_code=400, detail="source_id is required")
record = await _load_compute_center_record(db, source_id)
name = payload.name or (record.name if record else None)
metadata = (record.extra_data or {}) if record else {}
operator = payload.operator or metadata.get("operator") or metadata.get("organization") or metadata.get("owner")
site = payload.site or metadata.get("site")
organization = payload.organization or metadata.get("organization")
city = payload.city or get_record_field(record, "city") if record else payload.city
country = payload.country or (get_record_field(record, "country") if record else None)
source = payload.source or (record.source if record else None)
record_id = payload.record_id or (record.id if record else None)
candidates, attempted_queries = collect_location_candidates(
name=name,
source=source,
source_id=source_id,
operator=operator,
site=site,
organization=organization,
city=city,
country=country,
record_id=record_id,
)
if not candidates:
return {
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": False,
"failure_reason": (
"No source coordinates, organization lookup, or online geocoding"
" result reached city-level precision."
),
"candidates": [],
"attempted_queries": list(attempted_queries),
"context": {
"name": name,
"operator": operator,
"site": site,
"city": city,
"country": country,
},
}
return {
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": True,
"candidates": [candidate.to_dict() for candidate in candidates],
"best_candidate": candidates[0].to_dict(),
"attempted_queries": list(attempted_queries),
"context": {
"name": name,
"operator": operator,
"site": site,
"city": city,
"country": country,
},
}
@router.post("/compute-centers/{source_id}/location")
async def save_compute_center_location(
source_id: str,
payload: SaveComputeCenterLocationRequest,
db: AsyncSession = Depends(get_db),
):
"""Persist the user-selected compute-center location candidate."""
if not source_id or not source_id.strip():
raise HTTPException(status_code=400, detail="source_id is required")
if payload.latitude in (0.0, None) or payload.longitude in (0.0, None):
raise HTTPException(status_code=400, detail="latitude/longitude are required")
if payload.precision not in RENDERABLE_PRECISIONS:
raise HTTPException(status_code=400, detail="precision must be precise, site, or city")
record = await _load_compute_center_record(db, source_id)
metadata = (record.extra_data or {}) if record else {}
record_source = payload.source or (record.source if record else None)
if not record_source:
raise HTTPException(status_code=400, detail="source is required for unknown compute center")
operator = (
payload.operator
or metadata.get("operator")
or metadata.get("organization")
or metadata.get("owner")
or metadata.get("manufacturer")
)
site = payload.site or metadata.get("site") or metadata.get("organization")
saved = await upsert_compute_center_location(
db,
source=record_source,
source_id=source_id,
name=payload.name or (record.name if record else None),
operator=operator,
site=site,
city=payload.city or (get_record_field(record, "city") if record else None),
country=payload.country or (get_record_field(record, "country") if record else None),
latitude=payload.latitude,
longitude=payload.longitude,
precision=payload.precision,
confidence=payload.confidence,
location_source=payload.location_source or "manual_selection",
source_url=payload.source_url,
source_note=payload.source_note,
raw_payload=payload.raw_payload,
needs_confirmation=payload.needs_confirmation,
verification_status=payload.verification_status
or ("unverified" if payload.needs_confirmation else "verified"),
)
return {
"success": True,
"source": saved.source,
"source_id": saved.source_id,
"location": saved.to_location_dict(),
}
async def _load_compute_center_record(db: AsyncSession, source_id: str) -> CollectedData | None:
stmt = (
select(CollectedData)
.where(CollectedData.source_id == source_id)
.where(CollectedData.source.in_(["top500", "epoch_ai_gpu"]))
.order_by(CollectedData.is_current.desc(), CollectedData.id.desc())
.limit(1)
)
result = await db.execute(stmt)
return result.scalars().first()
@router.get("/geo/vessels")
async def get_vessels_geojson(
bbox: Optional[str] = Query(