linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.49.0

Browse Source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
linkong
2026-05-08 17:42:27 +08:00
parent bb9183b8a4
commit e1984c7a35
86 changed files with 9165 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/app/api/main.py
View File

@@ -5,6 +5,7 @@ from app.api.v1 import (
users,
datasource_config,
datasources,
docs,
tasks,
dashboard,
websocket,
@@ -29,6 +30,7 @@ api_router.include_router(
)
api_router.include_router(datasources.router, prefix="/datasources", tags=["datasources"])
api_router.include_router(collected_data.router, prefix="/collected", tags=["collected-data"])
api_router.include_router(docs.router, prefix="/docs", tags=["docs"])
api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboard"])
api_router.include_router(alerts.router, prefix="/alerts", tags=["alerts"])

6
backend/app/api/v1/auth.py
View File

@@ -28,7 +28,7 @@ async def login(
):
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active FROM users WHERE username = :username"
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE username = :username"
),
{"username": form_data.username},
)
@@ -46,6 +46,7 @@ async def login(
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
if not verify_password(form_data.password, user.password_hash):
raise HTTPException(
@@ -73,6 +74,7 @@ async def login(
"id": user.id,
"username": user.username,
"role": user.role,
"gatekeeper_groups": user.gatekeeper_groups or [],
},
}
@@ -95,6 +97,7 @@ async def refresh_token(
"id": current_user.id,
"username": current_user.username,
"role": current_user.role,
"gatekeeper_groups": current_user.gatekeeper_groups or [],
},
}
@@ -111,6 +114,7 @@ async def get_me(current_user: User = Depends(get_current_user)):
"username": current_user.username,
"email": current_user.email,
"role": current_user.role,
"gatekeeper_groups": current_user.gatekeeper_groups or [],
"is_active": current_user.is_active,
"created_at": current_user.created_at,
}

77
backend/app/api/v1/bgp.py
View File

@@ -5,12 +5,18 @@ from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from pydantic import BaseModel
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.models.user import User
from app.services.bgp_collector_locations import (
collect_bgp_collector_location_candidates,
get_bgp_collector_location_dict,
)
from app.services.bgp_collectors import build_bgp_collector_coverage
router = APIRouter()
@@ -264,6 +270,77 @@ async def get_bgp_collector_summary(
}
class CollectBGPCollectorLocationRequest(BaseModel):
city: Optional[str] = None
country: Optional[str] = None
site: Optional[str] = None
operator: Optional[str] = None
@router.post("/collectors/{collector_id}/collect-location")
async def collect_bgp_collector_location(
collector_id: str,
payload: CollectBGPCollectorLocationRequest,
current_user: User = Depends(get_current_user),
):
"""Run the shared location pipeline for a BGP route collector.
Mirrors ``POST /api/v1/visualization/compute-centers/{source_id}/collect-location``.
Returns ranked candidates from source coordinates and Nominatim queries
built around the collector's stored context (IXP / city / country). Stored
collector locations provide context only; they are not emitted as
candidates.
"""
if not collector_id or not collector_id.strip():
raise HTTPException(status_code=400, detail="collector_id is required")
legacy = get_bgp_collector_location_dict(collector_id) or {}
site = payload.site or legacy.get("matched_location_name")
city = payload.city or legacy.get("city")
country = payload.country or legacy.get("country")
operator = payload.operator or "RIPE NCC"
candidates, attempted_queries = collect_bgp_collector_location_candidates(
collector=collector_id,
site=site,
city=city,
country=country,
operator=operator,
)
context = {
"collector": collector_id,
"site": site,
"city": city,
"country": country,
"operator": operator,
}
if not candidates:
return {
"collector_id": collector_id,
"name": collector_id,
"success": False,
"failure_reason": (
"No source coordinates or online geocoding result reached"
" city-level precision for this collector."
),
"candidates": [],
"attempted_queries": list(attempted_queries),
"context": context,
}
return {
"collector_id": collector_id,
"name": collector_id,
"success": True,
"candidates": [candidate.to_dict() for candidate in candidates],
"best_candidate": candidates[0].to_dict(),
"attempted_queries": list(attempted_queries),
"context": context,
}
@router.get("/overview/summary")
async def get_bgp_overview_summary(
current_user: User = Depends(get_current_user),

102
backend/app/api/v1/docs.py Normal file
View File

@@ -0,0 +1,102 @@
"""Authenticated documentation APIs."""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from sqlalchemy import text
from app.core.security import decode_token
from app.db.session import async_session_factory
from app.models.user import User
from app.services.docs_gatekeeper import (
DOCS_BY_SLUG,
VALID_DOCS_LANGS,
can_read_doc,
catalog_for_user,
doc_path_for,
title_for,
)
router = APIRouter()
optional_bearer = HTTPBearer(auto_error=False)
async def get_optional_current_user(
credentials: HTTPAuthorizationCredentials | None = Depends(optional_bearer),
) -> User | None:
if credentials is None:
return None
payload = decode_token(credentials.credentials)
if payload is None or payload.get("type") != "access" or payload.get("sub") is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid token",
)
async with async_session_factory() as db:
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": int(payload["sub"])},
)
row = result.fetchone()
if row is None or not row[5]:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="User not found or inactive",
)
user = User()
user.id = row[0]
user.username = row[1]
user.email = row[2]
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
return user
@router.get("/catalog")
async def get_docs_catalog(current_user: User | None = Depends(get_optional_current_user)):
return {
"items": catalog_for_user(current_user),
"authenticated": current_user is not None,
}
@router.get("/{lang}/{slug}")
async def get_doc_content(
lang: str,
slug: str,
current_user: User | None = Depends(get_optional_current_user),
):
if lang not in VALID_DOCS_LANGS:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
entry = DOCS_BY_SLUG.get(slug)
if entry is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
path = doc_path_for(entry, lang)
if not path.exists():
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
if not can_read_doc(entry, current_user):
if current_user is None:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authentication required")
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Insufficient Docs permissions")
return {
"slug": entry.slug,
"filename": entry.filename,
"lang": lang,
"title": title_for(entry, lang),
"group": entry.group,
"order": entry.order,
"access": entry.access,
"markdown": path.read_text(encoding="utf-8"),
}

38
backend/app/api/v1/users.py
View File

@@ -1,3 +1,4 @@
import json
from typing import List
from fastapi import APIRouter, Depends, HTTPException, status
@@ -7,10 +8,12 @@ from sqlalchemy import text
from app.core.security import get_current_user, get_password_hash
from app.db.session import get_db
from app.models.user import User
from app.schemas.user import UserCreate, UserResponse, UserUpdate
from app.schemas.user import UserCreate, UserUpdate
router = APIRouter()
VALID_GATEKEEPER_GROUPS = {"docs_user", "docs_developer", "docs_admin"}
def check_permission(current_user: User, required_roles: List[str]) -> bool:
user_role_value = (
@@ -52,7 +55,7 @@ async def list_users(
offset = (page - 1) * page_size
query = text(
f"SELECT id, username, email, role, is_active, last_login_at, created_at FROM users WHERE {where_sql} ORDER BY created_at DESC LIMIT {page_size} OFFSET {offset}"
f"SELECT id, username, email, role, is_active, last_login_at, created_at, gatekeeper_groups FROM users WHERE {where_sql} ORDER BY created_at DESC LIMIT {page_size} OFFSET {offset}"
)
count_query = text(f"SELECT COUNT(*) FROM users WHERE {where_sql}")
@@ -75,6 +78,7 @@ async def list_users(
"is_active": u[4],
"last_login_at": u[5],
"created_at": u[6],
"gatekeeper_groups": u[7] or [],
}
for u in users
],
@@ -95,7 +99,7 @@ async def get_user(
result = await db.execute(
text(
"SELECT id, username, email, role, is_active, last_login_at, created_at FROM users WHERE id = :id"
"SELECT id, username, email, role, is_active, last_login_at, created_at, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": user_id},
)
@@ -114,6 +118,7 @@ async def get_user(
"is_active": user[4],
"last_login_at": user[5],
"created_at": user[6],
"gatekeeper_groups": user[7] or [],
}
@@ -128,6 +133,12 @@ async def create_user(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can create users",
)
invalid_groups = sorted(set(user_data.gatekeeper_groups) - VALID_GATEKEEPER_GROUPS)
if invalid_groups:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unsupported Gatekeeper groups: {', '.join(invalid_groups)}",
)
result = await db.execute(
text("SELECT id FROM users WHERE username = :username OR email = :email"),
@@ -142,13 +153,14 @@ async def create_user(
hashed_password = get_password_hash(user_data.password)
await db.execute(
text("""INSERT INTO users (username, email, password_hash, role, is_active, created_at, updated_at)
VALUES (:username, :email, :password_hash, :role, :is_active, NOW(), NOW())"""),
text("""INSERT INTO users (username, email, password_hash, role, gatekeeper_groups, is_active, created_at, updated_at)
VALUES (:username, :email, :password_hash, :role, CAST(:gatekeeper_groups AS jsonb), :is_active, NOW(), NOW())"""),
{
"username": user_data.username,
"email": user_data.email,
"password_hash": hashed_password,
"role": user_data.role,
"gatekeeper_groups": json.dumps(user_data.gatekeeper_groups),
"is_active": True,
},
)
@@ -172,6 +184,7 @@ async def create_user(
"username": user_data.username,
"email": user_data.email,
"role": user_data.role,
"gatekeeper_groups": user_data.gatekeeper_groups,
"is_active": True,
}
@@ -194,6 +207,18 @@ async def update_user(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can change user role",
)
if not check_permission(current_user, ["super_admin"]) and user_data.gatekeeper_groups is not None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can change Gatekeeper groups",
)
if user_data.gatekeeper_groups is not None:
invalid_groups = sorted(set(user_data.gatekeeper_groups) - VALID_GATEKEEPER_GROUPS)
if invalid_groups:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unsupported Gatekeeper groups: {', '.join(invalid_groups)}",
)
result = await db.execute(
text("SELECT id FROM users WHERE id = :id"),
@@ -213,6 +238,9 @@ async def update_user(
if user_data.role is not None:
update_fields.append("role = :role")
params["role"] = user_data.role
if user_data.gatekeeper_groups is not None:
update_fields.append("gatekeeper_groups = CAST(:gatekeeper_groups AS jsonb)")
params["gatekeeper_groups"] = json.dumps(user_data.gatekeeper_groups)
if user_data.is_active is not None:
update_fields.append("is_active = :is_active")
params["is_active"] = user_data.is_active

548
backend/app/api/v1/visualization.py
View File

@@ -4,17 +4,20 @@ Unified API for all visualization data sources.
Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium.
"""
import asyncio
import base64
from collections import OrderedDict
from datetime import UTC, datetime, timedelta
import math
import re
import httpx
from fastapi import APIRouter, HTTPException, Depends, Query, Response
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func
from typing import List, Dict, Any, Optional
from app.core.collected_data_fields import get_record_field
from app.core.countries import get_country_centroid
from app.core.satellite_tle import build_tle_lines_from_elements
from app.core.time import to_iso8601_utc
from app.db.session import get_db
@@ -25,6 +28,14 @@ from app.models.collected_data import CollectedData
from app.models.vessel import AISSourceHealth, VesselPosition, VesselStatic
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
from app.services.compute_center_locations import (
RENDERABLE_PRECISIONS,
ResolutionDiagnostic,
collect_location_candidates,
refresh_compute_center_location_cache,
resolve_compute_center_location_full,
upsert_compute_center_location,
)
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
from app.services.persistent_logs import record_system_log
from app.services.vessel_ais_aggregation import (
@@ -43,9 +54,23 @@ logger = get_logger(__name__, service="api")
TERRAIN_TILE_URL_TEMPLATE = (
"https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png"
)
TERRAIN_TILE_CACHE_MAX_ITEMS = 512
TERRAIN_TILE_BATCH_MAX_ITEMS = 128
TERRAIN_TILE_BATCH_CONCURRENCY = 16
_terrain_tile_cache: OrderedDict[tuple[int, int, int], tuple[bytes, str, dict[str, str]]] = OrderedDict()
VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE)
class TerrariumTileRequest(BaseModel):
z: int = Field(ge=0, le=14)
x: int = Field(ge=0)
y: int = Field(ge=0)
class TerrariumTileBatchRequest(BaseModel):
tiles: List[TerrariumTileRequest] = Field(min_length=1, max_length=TERRAIN_TILE_BATCH_MAX_ITEMS)
# ============== Converter Functions ==============
@@ -536,100 +561,6 @@ def _parse_float(value: Any) -> Optional[float]:
return None
COMPUTE_CENTER_COORDINATE_HINTS = (
("el capitan", 37.6819, -121.7681),
("livermore", 37.6819, -121.7681),
("llnl", 37.6819, -121.7681),
("lawrence livermore", 37.6819, -121.7681),
("frontier", 35.9319, -84.3107),
("oak ridge", 35.9319, -84.3107),
("ornl", 35.9319, -84.3107),
("aurora", 41.7130, -87.9820),
("argonne", 41.7130, -87.9820),
("anl", 41.7130, -87.9820),
("fugaku", 34.6953, 135.1974),
("kobe", 34.6953, 135.1974),
("riken", 34.6953, 135.1974),
("summit", 35.9319, -84.3107),
("leonardo", 44.4949, 11.3426),
("bologna", 44.4949, 11.3426),
("alps", 46.0037, 8.9511),
("lugano", 46.0037, 8.9511),
("sunway taihulight", 31.4912, 120.3119),
("wuxi", 31.4912, 120.3119),
("tianhe-2", 23.1291, 113.2644),
("tianhe-2a", 23.1291, 113.2644),
("guangzhou", 23.1291, 113.2644),
("colossus", 35.1495, -90.0490),
("memphis", 35.1495, -90.0490),
("xai", 35.1495, -90.0490),
)
def _normalize_hint_text(*parts: Any) -> str:
return " ".join(
str(part).strip().lower()
for part in parts
if part not in (None, "")
)
def _resolve_compute_center_coordinates(
record: CollectedData,
metadata: Dict[str, Any],
) -> Dict[str, Any]:
latitude = _parse_float(get_record_field(record, "latitude"))
longitude = _parse_float(get_record_field(record, "longitude"))
if latitude not in (None, 0.0) and longitude not in (None, 0.0):
return {
"latitude": latitude,
"longitude": longitude,
"location_precision": "precise",
"geography_mode": "source_coordinates",
"is_estimated": False,
"estimated_reason": None,
}
hint_text = _normalize_hint_text(
record.name,
get_record_field(record, "city"),
get_record_field(record, "country"),
metadata.get("site"),
metadata.get("organization"),
metadata.get("operator"),
)
for needle, resolved_latitude, resolved_longitude in COMPUTE_CENTER_COORDINATE_HINTS:
if needle in hint_text:
return {
"latitude": resolved_latitude,
"longitude": resolved_longitude,
"location_precision": "estimated_site",
"geography_mode": "site_hint",
"is_estimated": True,
"estimated_reason": f"Matched known site hint: {needle}",
}
centroid = get_country_centroid(get_record_field(record, "country"))
if centroid:
return {
"latitude": centroid.get("latitude"),
"longitude": centroid.get("longitude"),
"location_precision": "estimated_country",
"geography_mode": "country_centroid",
"is_estimated": True,
"estimated_reason": "Estimated from country centroid",
}
return {
"latitude": latitude,
"longitude": longitude,
"location_precision": "unknown",
"geography_mode": "unknown",
"is_estimated": True,
"estimated_reason": "No resolvable location hints",
}
def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str) -> str:
if capacity_value is None:
return "unknown"
@@ -654,22 +585,49 @@ def _normalize_capacity_band(capacity_value: Optional[float], capacity_unit: str
def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
"""Convert compute infrastructure records into a unified GeoJSON layer."""
features = []
"""Convert compute infrastructure records into a unified GeoJSON layer.
Records that cannot be resolved to at least city-level precision are NOT
silently dropped: they are returned in ``unresolved`` so the UI can offer
the click-to-collect coordinate flow. The features list never contains
``[0, 0]`` placeholders or country/region/unknown precision points.
"""
features: List[Dict[str, Any]] = []
unresolved: List[Dict[str, Any]] = []
for record in records:
metadata = record.extra_data or {}
coordinate_info = _resolve_compute_center_coordinates(record, metadata)
latitude = coordinate_info.get("latitude")
longitude = coordinate_info.get("longitude")
result = resolve_compute_center_location_full(record, metadata)
site_type = (
"supercomputer"
if record.source == "top500" or record.data_type == "supercomputer"
else "gpu_cluster"
)
if latitude in (None, 0.0) or longitude in (None, 0.0):
if not result.is_resolved:
diagnostic = result.diagnostic or ResolutionDiagnostic(
failure_reason="Unknown resolver failure",
attempted_queries=(),
record_id=getattr(record, "id", None),
source=getattr(record, "source", None),
source_id=getattr(record, "source_id", None),
name=getattr(record, "name", None),
)
unresolved.append({
**diagnostic.to_dict(),
"site_type": site_type,
})
continue
location = result.location
if location is None or not location.is_renderable:
# Defensive: should not happen because is_resolved guards this.
continue
location_props = location.to_geojson_properties()
latitude = location.latitude
longitude = location.longitude
if site_type == "supercomputer":
capacity_value = _parse_float(get_record_field(record, "rmax"))
capacity_unit = "GFlops"
@@ -699,15 +657,16 @@ def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str
"id": record.id,
"geometry": {
"type": "Point",
"coordinates": [longitude or 0, latitude or 0],
"coordinates": [longitude, latitude],
},
"properties": {
"id": record.id,
"source_id": record.source_id,
"name": record.name,
"site_type": site_type,
"country": get_record_field(record, "country"),
"city": get_record_field(record, "city"),
"country": get_record_field(record, "country") or location.country,
"city": get_record_field(record, "city") or location.city,
"region": location.region,
"latitude": latitude,
"longitude": longitude,
"operator": operator,
@@ -723,17 +682,14 @@ def convert_compute_centers_to_geojson(records: List[CollectedData]) -> Dict[str
"source": record.source,
"updated_at": updated_at,
"status": "observed",
"location_precision": coordinate_info.get("location_precision"),
"geography_mode": coordinate_info.get("geography_mode"),
"is_estimated": coordinate_info.get("is_estimated", False),
"estimated_reason": coordinate_info.get("estimated_reason"),
**location_props,
"data_type": "compute_center",
"metadata": metadata,
},
}
)
return {"type": "FeatureCollection", "features": features}
return {"type": "FeatureCollection", "features": features, "unresolved": unresolved}
VESSEL_TYPE_FILTERS = {
@@ -1486,18 +1442,12 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
@router.get("/terrain/terrarium/{z}/{x}/{y}.png")
async def get_terrarium_tile(z: int, x: int, y: int):
"""Proxy Terrarium elevation tiles through the backend to avoid browser CORS issues."""
if z < 0 or x < 0 or y < 0:
if not _is_valid_terrain_tile(z, x, y):
raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates")
url = TERRAIN_TILE_URL_TEMPLATE.format(z=z, x=x, y=y)
try:
async with httpx.AsyncClient(
timeout=20.0,
follow_redirects=True,
) as client:
upstream = await client.get(url)
upstream.raise_for_status()
async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
content, content_type, headers = await _fetch_terrain_tile(client, z, x, y)
except httpx.HTTPStatusError as exc:
raise HTTPException(
status_code=exc.response.status_code,
@@ -1509,22 +1459,140 @@ async def get_terrarium_tile(z: int, x: int, y: int):
detail=f"Terrain tile fetch failed: {exc}",
) from exc
return Response(
content=content,
media_type=content_type,
headers=headers,
)
def _is_valid_terrain_tile(z: int, x: int, y: int) -> bool:
if z < 0 or x < 0 or y < 0:
return False
max_tile = 2 ** z
return x < max_tile and y < max_tile
def _get_cached_terrain_tile(z: int, x: int, y: int) -> tuple[bytes, str, dict[str, str]] | None:
key = (z, x, y)
cached = _terrain_tile_cache.get(key)
if cached is None:
return None
_terrain_tile_cache.move_to_end(key)
content, content_type, headers = cached
return content, content_type, dict(headers)
def _cache_terrain_tile(
z: int,
x: int,
y: int,
content: bytes,
content_type: str,
headers: dict[str, str],
) -> None:
key = (z, x, y)
_terrain_tile_cache[key] = (content, content_type, dict(headers))
_terrain_tile_cache.move_to_end(key)
while len(_terrain_tile_cache) > TERRAIN_TILE_CACHE_MAX_ITEMS:
_terrain_tile_cache.popitem(last=False)
async def _fetch_terrain_tile(
client: httpx.AsyncClient,
z: int,
x: int,
y: int,
) -> tuple[bytes, str, dict[str, str]]:
cached = _get_cached_terrain_tile(z, x, y)
if cached is not None:
return cached
url = TERRAIN_TILE_URL_TEMPLATE.format(z=z, x=x, y=y)
upstream = await client.get(url)
upstream.raise_for_status()
cache_control = upstream.headers.get("cache-control") or "public, max-age=86400"
etag = upstream.headers.get("etag")
last_modified = upstream.headers.get("last-modified")
headers = {
"Cache-Control": cache_control,
}
etag = upstream.headers.get("etag")
last_modified = upstream.headers.get("last-modified")
if etag:
headers["ETag"] = etag
if last_modified:
headers["Last-Modified"] = last_modified
return Response(
content=upstream.content,
media_type=upstream.headers.get("content-type", "image/png"),
headers=headers,
)
content_type = upstream.headers.get("content-type", "image/png")
content = upstream.content
_cache_terrain_tile(z, x, y, content, content_type, headers)
return content, content_type, dict(headers)
@router.post("/terrain/terrarium/batch")
async def get_terrarium_tile_batch(payload: TerrariumTileBatchRequest):
"""Fetch Terrarium elevation tiles in batches so the browser avoids many tiny requests."""
unique_tiles: list[TerrariumTileRequest] = []
seen: set[tuple[int, int, int]] = set()
for tile in payload.tiles:
key = (tile.z, tile.x, tile.y)
if key in seen:
continue
seen.add(key)
if not _is_valid_terrain_tile(tile.z, tile.x, tile.y):
raise HTTPException(status_code=400, detail="Invalid terrain tile coordinates")
unique_tiles.append(tile)
semaphore = asyncio.Semaphore(TERRAIN_TILE_BATCH_CONCURRENCY)
results: list[dict[str, Any]] = []
errors: list[dict[str, Any]] = []
async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
async def fetch_one(tile: TerrariumTileRequest) -> None:
async with semaphore:
try:
content, content_type, _headers = await _fetch_terrain_tile(
client,
tile.z,
tile.x,
tile.y,
)
results.append(
{
"z": tile.z,
"x": tile.x,
"y": tile.y,
"content_type": content_type,
"data": base64.b64encode(content).decode("ascii"),
},
)
except httpx.HTTPStatusError as exc:
errors.append(
{
"z": tile.z,
"x": tile.x,
"y": tile.y,
"status_code": exc.response.status_code,
"message": f"upstream error: {exc.response.status_code}",
},
)
except httpx.HTTPError as exc:
errors.append(
{
"z": tile.z,
"x": tile.x,
"y": tile.y,
"status_code": 502,
"message": str(exc),
},
)
await asyncio.gather(*(fetch_one(tile) for tile in unique_tiles))
return {
"tiles": results,
"errors": errors,
}
@router.get("/geo/all")
@@ -1659,33 +1727,253 @@ async def get_compute_centers_geojson(
return {
"type": "FeatureCollection",
"features": [],
"unresolved": [],
"count": 0,
"stats": {
"total": 0,
"supercomputers": 0,
"gpu_clusters": 0,
"unresolved": 0,
},
}
await refresh_compute_center_location_cache(db)
geojson = convert_compute_centers_to_geojson(records)
features = geojson.get("features", [])
unresolved = geojson.get("unresolved", [])
# Belt-and-suspenders: ensure no Feature ever sneaks through without
# city-or-better precision and finite, non-zero coordinates.
sanitized_features: List[Dict[str, Any]] = []
for feature in features:
coords = feature.get("geometry", {}).get("coordinates") or []
precision = feature.get("properties", {}).get("location_precision")
if precision not in RENDERABLE_PRECISIONS:
unresolved.append({
"failure_reason": f"Rejected non-renderable precision '{precision}'",
"record_id": feature.get("id"),
"source_id": feature.get("properties", {}).get("source_id"),
"name": feature.get("properties", {}).get("name"),
})
continue
if (
len(coords) != 2
or coords[0] in (None, 0, 0.0)
or coords[1] in (None, 0, 0.0)
):
unresolved.append({
"failure_reason": "Rejected feature with [0,0] or invalid coordinates",
"record_id": feature.get("id"),
"source_id": feature.get("properties", {}).get("source_id"),
"name": feature.get("properties", {}).get("name"),
})
continue
sanitized_features.append(feature)
return {
**geojson,
"count": len(features),
"type": "FeatureCollection",
"features": sanitized_features,
"unresolved": unresolved,
"count": len(sanitized_features),
"stats": {
"total": len(features),
"total": len(sanitized_features),
"supercomputers": sum(
1 for feature in features
1 for feature in sanitized_features
if feature.get("properties", {}).get("site_type") == "supercomputer"
),
"gpu_clusters": sum(
1 for feature in features
1 for feature in sanitized_features
if feature.get("properties", {}).get("site_type") == "gpu_cluster"
),
"unresolved": len(unresolved),
},
}
class CollectComputeCenterLocationRequest(BaseModel):
name: Optional[str] = None
source: Optional[str] = None
operator: Optional[str] = None
site: Optional[str] = None
organization: Optional[str] = None
city: Optional[str] = None
country: Optional[str] = None
record_id: Optional[int] = Field(default=None, alias="id")
model_config = {"populate_by_name": True}
class SaveComputeCenterLocationRequest(BaseModel):
source: Optional[str] = None
name: Optional[str] = None
operator: Optional[str] = None
site: Optional[str] = None
city: Optional[str] = None
country: Optional[str] = None
latitude: float
longitude: float
precision: str = "city"
confidence: Optional[float] = None
location_source: Optional[str] = None
source_url: Optional[str] = None
source_note: Optional[str] = None
raw_payload: Dict[str, Any] = Field(default_factory=dict)
needs_confirmation: bool = False
verification_status: Optional[str] = None
model_config = {"populate_by_name": True}
@router.post("/compute-centers/{source_id}/collect-location")
async def collect_compute_center_location(
source_id: str,
payload: CollectComputeCenterLocationRequest,
db: AsyncSession = Depends(get_db),
):
"""Run the full multi-query location collection pipeline for a record.
The endpoint accepts the source_id of a compute center plus contextual
fields (name/operator/site/city/country/...) and returns ranked candidate
locations from source coordinates, open organization lookups, and online
geocoding combinations. The caller never has to type coordinates by hand:
if any candidate is accepted it can be applied directly. If no candidate
can reach city-level precision the response includes an explicit
``failure_reason`` and the list of attempted queries.
"""
if not source_id or not source_id.strip():
raise HTTPException(status_code=400, detail="source_id is required")
record = await _load_compute_center_record(db, source_id)
name = payload.name or (record.name if record else None)
metadata = (record.extra_data or {}) if record else {}
operator = payload.operator or metadata.get("operator") or metadata.get("organization") or metadata.get("owner")
site = payload.site or metadata.get("site")
organization = payload.organization or metadata.get("organization")
city = payload.city or get_record_field(record, "city") if record else payload.city
country = payload.country or (get_record_field(record, "country") if record else None)
source = payload.source or (record.source if record else None)
record_id = payload.record_id or (record.id if record else None)
candidates, attempted_queries = collect_location_candidates(
name=name,
source=source,
source_id=source_id,
operator=operator,
site=site,
organization=organization,
city=city,
country=country,
record_id=record_id,
)
if not candidates:
return {
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": False,
"failure_reason": (
"No source coordinates, organization lookup, or online geocoding"
" result reached city-level precision."
),
"candidates": [],
"attempted_queries": list(attempted_queries),
"context": {
"name": name,
"operator": operator,
"site": site,
"city": city,
"country": country,
},
}
return {
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": True,
"candidates": [candidate.to_dict() for candidate in candidates],
"best_candidate": candidates[0].to_dict(),
"attempted_queries": list(attempted_queries),
"context": {
"name": name,
"operator": operator,
"site": site,
"city": city,
"country": country,
},
}
@router.post("/compute-centers/{source_id}/location")
async def save_compute_center_location(
source_id: str,
payload: SaveComputeCenterLocationRequest,
db: AsyncSession = Depends(get_db),
):
"""Persist the user-selected compute-center location candidate."""
if not source_id or not source_id.strip():
raise HTTPException(status_code=400, detail="source_id is required")
if payload.latitude in (0.0, None) or payload.longitude in (0.0, None):
raise HTTPException(status_code=400, detail="latitude/longitude are required")
if payload.precision not in RENDERABLE_PRECISIONS:
raise HTTPException(status_code=400, detail="precision must be precise, site, or city")
record = await _load_compute_center_record(db, source_id)
metadata = (record.extra_data or {}) if record else {}
record_source = payload.source or (record.source if record else None)
if not record_source:
raise HTTPException(status_code=400, detail="source is required for unknown compute center")
operator = (
payload.operator
or metadata.get("operator")
or metadata.get("organization")
or metadata.get("owner")
or metadata.get("manufacturer")
)
site = payload.site or metadata.get("site") or metadata.get("organization")
saved = await upsert_compute_center_location(
db,
source=record_source,
source_id=source_id,
name=payload.name or (record.name if record else None),
operator=operator,
site=site,
city=payload.city or (get_record_field(record, "city") if record else None),
country=payload.country or (get_record_field(record, "country") if record else None),
latitude=payload.latitude,
longitude=payload.longitude,
precision=payload.precision,
confidence=payload.confidence,
location_source=payload.location_source or "manual_selection",
source_url=payload.source_url,
source_note=payload.source_note,
raw_payload=payload.raw_payload,
needs_confirmation=payload.needs_confirmation,
verification_status=payload.verification_status
or ("unverified" if payload.needs_confirmation else "verified"),
)
return {
"success": True,
"source": saved.source,
"source_id": saved.source_id,
"location": saved.to_location_dict(),
}
async def _load_compute_center_record(db: AsyncSession, source_id: str) -> CollectedData | None:
stmt = (
select(CollectedData)
.where(CollectedData.source_id == source_id)
.where(CollectedData.source.in_(["top500", "epoch_ai_gpu"]))
.order_by(CollectedData.is_current.desc(), CollectedData.id.desc())
.limit(1)
)
result = await db.execute(stmt)
return result.scalars().first()
@router.get("/geo/vessels")
async def get_vessels_geojson(
bbox: Optional[str] = Query(

51
backend/app/core/collected_data_fields.py
View File

@@ -4,8 +4,8 @@ from typing import Any, Dict, Optional
FIELD_ALIASES = {
"country": ("country",),
"city": ("city",),
"latitude": ("latitude",),
"longitude": ("longitude",),
"latitude": ("latitude", "lat"),
"longitude": ("longitude", "lon", "lng"),
"value": ("value",),
"unit": ("unit",),
"cores": ("cores",),
@@ -14,6 +14,28 @@ FIELD_ALIASES = {
"power": ("power",),
}
NESTED_FIELD_ALIASES = {
"latitude": (
("location", "latitude"),
("location", "lat"),
("geo", "latitude"),
("geo", "lat"),
("coordinates", "latitude"),
("coordinates", "lat"),
),
"longitude": (
("location", "longitude"),
("location", "lon"),
("location", "lng"),
("geo", "longitude"),
("geo", "lon"),
("geo", "lng"),
("coordinates", "longitude"),
("coordinates", "lon"),
("coordinates", "lng"),
),
}
def get_metadata_field(metadata: Optional[Dict[str, Any]], field: str, fallback: Any = None) -> Any:
if isinstance(metadata, dict):
@@ -21,9 +43,34 @@ def get_metadata_field(metadata: Optional[Dict[str, Any]], field: str, fallback:
value = metadata.get(key)
if value not in (None, ""):
return value
for path in NESTED_FIELD_ALIASES.get(field, ()):
current: Any = metadata
for key in path:
if not isinstance(current, dict):
current = None
break
current = current.get(key)
if current not in (None, ""):
return current
if field in {"latitude", "longitude"}:
value = _get_coordinate_sequence_value(metadata, field)
if value not in (None, ""):
return value
return fallback
def _get_coordinate_sequence_value(metadata: Dict[str, Any], field: str) -> Any:
for key in ("coordinates", "coord", "coords"):
value = metadata.get(key)
if not isinstance(value, (list, tuple)) or len(value) < 2:
continue
# GeoJSON uses [longitude, latitude]. Most raw collector tuples in this
# codebase use explicit field names, so only sequence aliases are treated
# as GeoJSON-shaped to avoid guessing.
return value[1] if field == "latitude" else value[0]
return None
def build_dynamic_metadata(
metadata: Optional[Dict[str, Any]],
*,

6
backend/app/core/security.py
View File

@@ -105,7 +105,7 @@ async def get_current_user(
)
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active FROM users WHERE id = :id"
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": int(user_id)},
)
@@ -122,6 +122,7 @@ async def get_current_user(
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
return user
@@ -144,7 +145,7 @@ async def get_current_user_refresh(
)
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active FROM users WHERE id = :id"
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": int(user_id)},
)
@@ -161,6 +162,7 @@ async def get_current_user_refresh(
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
return user

328
backend/app/data/seeds/ripe_ris_collector_locations_seed.json Normal file
View File

@@ -0,0 +1,328 @@
{
"_comment": "Seed payload for the bgp_collector_locations DB table. Coordinates were migrated from the legacy RIPE_RIS_COLLECTOR_COORDS table and default to city-center; seeded rows are unverified and should be upgraded in the database with source evidence when known.",
"locations": [
{
"canonical_name": "RIPE RIS rrc00",
"aliases": ["rrc00", "RIPE RIS rrc00", "AMS-IX"],
"operator": "RIPE NCC",
"site": "AMS-IX",
"city": "Amsterdam",
"country": "Netherlands",
"latitude": 52.3676,
"longitude": 4.9041,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc01",
"aliases": ["rrc01", "RIPE RIS rrc01", "LINX"],
"operator": "RIPE NCC",
"site": "LINX",
"city": "London",
"country": "United Kingdom",
"latitude": 51.5072,
"longitude": -0.1276,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc03",
"aliases": ["rrc03", "RIPE RIS rrc03", "AMS-IX"],
"operator": "RIPE NCC",
"site": "AMS-IX",
"city": "Amsterdam",
"country": "Netherlands",
"latitude": 52.3676,
"longitude": 4.9041,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc04",
"aliases": ["rrc04", "RIPE RIS rrc04", "CIXP", "CERN Internet Exchange Point"],
"operator": "RIPE NCC",
"site": "CIXP",
"city": "Geneva",
"country": "Switzerland",
"latitude": 46.2044,
"longitude": 6.1432,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc05",
"aliases": ["rrc05", "RIPE RIS rrc05", "VIX", "Vienna Internet Exchange"],
"operator": "RIPE NCC",
"site": "VIX",
"city": "Vienna",
"country": "Austria",
"latitude": 48.2082,
"longitude": 16.3738,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc06",
"aliases": ["rrc06", "RIPE RIS rrc06", "JPIX", "Otemachi"],
"operator": "RIPE NCC",
"site": "JPIX",
"city": "Otemachi",
"country": "Japan",
"latitude": 35.686,
"longitude": 139.7671,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc07",
"aliases": ["rrc07", "RIPE RIS rrc07", "Netnod", "Netnod Stockholm"],
"operator": "RIPE NCC",
"site": "Netnod Stockholm",
"city": "Stockholm",
"country": "Sweden",
"latitude": 59.3293,
"longitude": 18.0686,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc10",
"aliases": ["rrc10", "RIPE RIS rrc10", "MIX", "Milan Internet Exchange"],
"operator": "RIPE NCC",
"site": "MIX",
"city": "Milan",
"country": "Italy",
"latitude": 45.4642,
"longitude": 9.19,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc11",
"aliases": ["rrc11", "RIPE RIS rrc11", "NYIIX", "New York International Internet Exchange"],
"operator": "RIPE NCC",
"site": "NYIIX",
"city": "New York",
"country": "United States",
"latitude": 40.7128,
"longitude": -74.006,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc12",
"aliases": ["rrc12", "RIPE RIS rrc12", "DE-CIX", "DE-CIX Frankfurt"],
"operator": "RIPE NCC",
"site": "DE-CIX Frankfurt",
"city": "Frankfurt",
"country": "Germany",
"latitude": 50.1109,
"longitude": 8.6821,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc13",
"aliases": ["rrc13", "RIPE RIS rrc13", "MSK-IX"],
"operator": "RIPE NCC",
"site": "MSK-IX",
"city": "Moscow",
"country": "Russia",
"latitude": 55.7558,
"longitude": 37.6173,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc14",
"aliases": ["rrc14", "RIPE RIS rrc14", "PAIX", "Palo Alto Internet Exchange"],
"operator": "RIPE NCC",
"site": "PAIX",
"city": "Palo Alto",
"country": "United States",
"latitude": 37.4419,
"longitude": -122.143,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc15",
"aliases": ["rrc15", "RIPE RIS rrc15", "PTT.br Sao Paulo", "PTTMetro Sao Paulo"],
"operator": "RIPE NCC",
"site": "PTT.br",
"city": "Sao Paulo",
"country": "Brazil",
"latitude": -23.5558,
"longitude": -46.6396,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc16",
"aliases": ["rrc16", "RIPE RIS rrc16", "Equinix Miami", "NOTA Miami"],
"operator": "RIPE NCC",
"site": "Equinix Miami",
"city": "Miami",
"country": "United States",
"latitude": 25.7617,
"longitude": -80.1918,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc18",
"aliases": ["rrc18", "RIPE RIS rrc18", "CATNIX"],
"operator": "RIPE NCC",
"site": "CATNIX",
"city": "Barcelona",
"country": "Spain",
"latitude": 41.3874,
"longitude": 2.1686,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc19",
"aliases": ["rrc19", "RIPE RIS rrc19", "NAPAfrica", "JINX", "NAPAfrica Johannesburg"],
"operator": "RIPE NCC",
"site": "NAPAfrica Johannesburg",
"city": "Johannesburg",
"country": "South Africa",
"latitude": -26.2041,
"longitude": 28.0473,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc20",
"aliases": ["rrc20", "RIPE RIS rrc20", "SwissIX"],
"operator": "RIPE NCC",
"site": "SwissIX",
"city": "Zurich",
"country": "Switzerland",
"latitude": 47.3769,
"longitude": 8.5417,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc21",
"aliases": ["rrc21", "RIPE RIS rrc21", "France-IX Paris"],
"operator": "RIPE NCC",
"site": "France-IX Paris",
"city": "Paris",
"country": "France",
"latitude": 48.8566,
"longitude": 2.3522,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc22",
"aliases": ["rrc22", "RIPE RIS rrc22", "InterLAN Bucharest"],
"operator": "RIPE NCC",
"site": "InterLAN Bucharest",
"city": "Bucharest",
"country": "Romania",
"latitude": 44.4268,
"longitude": 26.1025,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc23",
"aliases": ["rrc23", "RIPE RIS rrc23", "Equinix Singapore"],
"operator": "RIPE NCC",
"site": "Equinix Singapore",
"city": "Singapore",
"country": "Singapore",
"latitude": 1.3521,
"longitude": 103.8198,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc24",
"aliases": ["rrc24", "RIPE RIS rrc24", "LACNIC Montevideo"],
"operator": "RIPE NCC",
"site": "LACNIC Montevideo",
"city": "Montevideo",
"country": "Uruguay",
"latitude": -34.9011,
"longitude": -56.1645,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc25",
"aliases": ["rrc25", "RIPE RIS rrc25", "AMS-IX"],
"operator": "RIPE NCC",
"site": "AMS-IX",
"city": "Amsterdam",
"country": "Netherlands",
"latitude": 52.3676,
"longitude": 4.9041,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
},
{
"canonical_name": "RIPE RIS rrc26",
"aliases": ["rrc26", "RIPE RIS rrc26", "UAE-IX"],
"operator": "RIPE NCC",
"site": "UAE-IX",
"city": "Dubai",
"country": "United Arab Emirates",
"latitude": 25.2048,
"longitude": 55.2708,
"precision": "city",
"confidence": 0.85,
"source_note": "Migrated from RIPE_RIS_COLLECTOR_COORDS legacy table",
"verified_at": null
}
],
"city_fallbacks": []
}

19
backend/app/db/session.py
View File

@@ -103,9 +103,11 @@ async def init_db():
import app.models.datasource_config # noqa: F401
import app.models.alert # noqa: F401
import app.models.bgp_anomaly # noqa: F401
import app.models.bgp_collector_location # noqa: F401
import app.models.bgp_incident # noqa: F401
import app.models.bgp_observation # noqa: F401
import app.models.collected_data # noqa: F401
import app.models.compute_center_location # noqa: F401
import app.models.system_setting # noqa: F401
import app.models.playground_session # noqa: F401
import app.models.playground_message # noqa: F401
@@ -128,6 +130,14 @@ async def init_db():
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
await conn.execute(
text(
"""
ALTER TABLE users
ADD COLUMN IF NOT EXISTS gatekeeper_groups JSONB DEFAULT '[]'::jsonb
"""
)
)
await conn.execute(
text(
"""
@@ -208,5 +218,14 @@ async def init_db():
)
async with async_session_factory() as session:
from app.services.bgp_collector_locations import (
seed_default_bgp_collector_locations,
)
from app.services.compute_center_locations import (
seed_compute_center_locations_from_source_coords,
)
await seed_default_bgp_collector_locations(session)
await seed_compute_center_locations_from_source_coords(session)
await seed_default_datasources(session)
await ensure_default_admin_user(session)

4
backend/app/models/__init__.py
View File

@@ -6,8 +6,10 @@ from app.models.datasource import DataSource
from app.models.datasource_config import DataSourceConfig
from app.models.alert import Alert, AlertSeverity, AlertStatus
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_collector_location import BGPCollectorLocation
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.models.compute_center_location import ComputeCenterLocationRecord
from app.models.system_setting import SystemSetting
from app.models.playground_session import PlaygroundSession
from app.models.playground_message import PlaygroundMessage
@@ -27,8 +29,10 @@ __all__ = [
"AlertSeverity",
"AlertStatus",
"BGPAnomaly",
"BGPCollectorLocation",
"BGPIncident",
"BGPObservation",
"ComputeCenterLocationRecord",
"SystemLog",
"AuditLog",
"PlaygroundSession",

52
backend/app/models/bgp_collector_location.py Normal file
View File

@@ -0,0 +1,52 @@
"""Stored BGP route-collector locations."""
from sqlalchemy import Boolean, Column, DateTime, Float, Integer, JSON, String, Text
from sqlalchemy.sql import func
from app.core.time import to_iso8601_utc
from app.db.session import Base
class BGPCollectorLocation(Base):
"""Current known location for a BGP route collector."""
__tablename__ = "bgp_collector_locations"
id = Column(Integer, primary_key=True, autoincrement=True)
collector_id = Column(String(100), nullable=False, unique=True, index=True)
operator = Column(String(255), nullable=True)
site = Column(String(255), nullable=True)
city = Column(String(255), nullable=True)
country = Column(String(255), nullable=True)
latitude = Column(Float, nullable=True)
longitude = Column(Float, nullable=True)
precision = Column(String(30), nullable=False, default="city")
confidence = Column(Float, nullable=True)
source = Column(String(80), nullable=False, default="legacy_seed", index=True)
source_url = Column(String(500), nullable=True)
source_note = Column(Text, nullable=True)
raw_payload = Column(JSON, nullable=False, default=dict)
needs_confirmation = Column(Boolean, nullable=False, default=True, index=True)
verification_status = Column(String(30), nullable=False, default="unverified", index=True)
verified_at = Column(DateTime(timezone=True), nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
def to_location_dict(self) -> dict:
return {
"city": self.city,
"country": self.country,
"latitude": self.latitude,
"longitude": self.longitude,
"precision": self.precision,
"source": self.source,
"needs_confirmation": self.needs_confirmation,
"matched_location_name": self.site or self.collector_id,
"verified_at": to_iso8601_utc(self.verified_at),
"confidence": self.confidence,
"operator": self.operator,
"site": self.site,
"verification_status": self.verification_status,
"source_note": self.source_note,
"source_url": self.source_url,
}

60
backend/app/models/compute_center_location.py Normal file
View File

@@ -0,0 +1,60 @@
"""Stored compute-center locations."""
from sqlalchemy import Boolean, Column, DateTime, Float, Integer, JSON, String, Text, UniqueConstraint
from sqlalchemy.sql import func
from app.core.time import to_iso8601_utc
from app.db.session import Base
class ComputeCenterLocationRecord(Base):
"""Current known location for a compute-center record."""
__tablename__ = "compute_center_locations"
__table_args__ = (
UniqueConstraint("source", "source_id", name="uq_compute_center_location_source_id"),
)
id = Column(Integer, primary_key=True, autoincrement=True)
source = Column(String(100), nullable=False, index=True)
source_id = Column(String(255), nullable=False, index=True)
name = Column(String(500), nullable=True)
operator = Column(String(255), nullable=True)
site = Column(String(255), nullable=True)
city = Column(String(255), nullable=True)
country = Column(String(255), nullable=True)
latitude = Column(Float, nullable=True)
longitude = Column(Float, nullable=True)
precision = Column(String(30), nullable=False, default="city")
confidence = Column(Float, nullable=True)
location_source = Column(String(80), nullable=False, default="stored_compute_center_location", index=True)
source_url = Column(String(500), nullable=True)
source_note = Column(Text, nullable=True)
raw_payload = Column(JSON, nullable=False, default=dict)
needs_confirmation = Column(Boolean, nullable=False, default=False, index=True)
verification_status = Column(String(30), nullable=False, default="verified", index=True)
verified_at = Column(DateTime(timezone=True), nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
def to_location_dict(self) -> dict:
return {
"source": self.source,
"source_id": self.source_id,
"name": self.name,
"operator": self.operator,
"site": self.site,
"city": self.city,
"country": self.country,
"latitude": self.latitude,
"longitude": self.longitude,
"precision": self.precision,
"confidence": self.confidence,
"location_source": self.location_source,
"source_url": self.source_url,
"source_note": self.source_note,
"raw_payload": self.raw_payload or {},
"needs_confirmation": self.needs_confirmation,
"verification_status": self.verification_status,
"verified_at": to_iso8601_utc(self.verified_at),
}

3
backend/app/models/user.py
View File

@@ -1,4 +1,4 @@
from sqlalchemy import Boolean, Column, Integer, String, DateTime
from sqlalchemy import Boolean, Column, DateTime, Integer, JSON, String
from sqlalchemy.sql import func
from app.db.session import Base
@@ -12,6 +12,7 @@ class User(Base):
email = Column(String(255), unique=True, index=True, nullable=False)
password_hash = Column(String(255), nullable=False)
role = Column(String(20), default="viewer")
gatekeeper_groups = Column(JSON, default=list)
is_active = Column(Boolean, default=True)
last_login_at = Column(DateTime(timezone=True))
created_at = Column(DateTime(timezone=True), server_default=func.now())

4
backend/app/schemas/user.py
View File

@@ -12,17 +12,20 @@ class UserBase(BaseModel):
class UserCreate(UserBase):
password: str = Field(..., min_length=8)
role: str = "viewer"
gatekeeper_groups: list[str] = Field(default_factory=list)
class UserUpdate(BaseModel):
email: Optional[EmailStr] = None
role: Optional[str] = None
gatekeeper_groups: Optional[list[str]] = None
is_active: Optional[bool] = None
class UserInDB(UserBase):
id: int
role: str
gatekeeper_groups: list[str] = Field(default_factory=list)
is_active: bool
last_login_at: Optional[datetime]
created_at: datetime
@@ -34,6 +37,7 @@ class UserInDB(UserBase):
class UserResponse(UserBase):
id: int
role: str
gatekeeper_groups: list[str] = Field(default_factory=list)
is_active: bool
created_at: datetime

306
backend/app/services/bgp_collector_locations.py Normal file
View File

@@ -0,0 +1,306 @@
"""BGP route-collector location resolver.
Collector positions are stored in the ``bgp_collector_locations`` database
table. The old JSON registry is now only a seed payload used during database
initialization, not a runtime resolver or candidate source.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Iterator
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.bgp_collector_location import BGPCollectorLocation
from app.services.location import (
LocationCandidate,
LocationPipeline,
LocationQuery,
NominatimResolver,
ResolutionResult,
ResolverOutput,
SourceCoordinatesResolver,
build_default_nominatim_geocoder,
coerce_str,
normalize_text,
)
SEED_PATH = (
Path(__file__).resolve().parents[1]
/ "data"
/ "seeds"
/ "ripe_ris_collector_locations_seed.json"
)
# ── Geocoder (kept at module level for monkeypatching + cache_clear) ──
_geocode_online = build_default_nominatim_geocoder()
# ── In-process compatibility cache ──────────────────────────────────
RIPE_RIS_COLLECTOR_COORDS: dict[str, dict[str, Any]] = {}
def _collector_record_to_dict(record: BGPCollectorLocation) -> dict[str, Any]:
return record.to_location_dict()
def set_bgp_collector_location_cache(
locations: dict[str, dict[str, Any]],
) -> None:
"""Replace the legacy compatibility cache in-place."""
RIPE_RIS_COLLECTOR_COORDS.clear()
RIPE_RIS_COLLECTOR_COORDS.update(
{coerce_str(key): dict(value) for key, value in locations.items()}
)
async def refresh_bgp_collector_location_cache(
session: AsyncSession,
) -> dict[str, dict[str, Any]]:
result = await session.execute(select(BGPCollectorLocation))
records = result.scalars().all()
cache = {
record.collector_id: _collector_record_to_dict(record)
for record in records
if record.collector_id
}
set_bgp_collector_location_cache(cache)
return cache
def _load_seed_payload() -> dict[str, Any]:
with SEED_PATH.open("r", encoding="utf-8") as handle:
return json.load(handle)
def _seed_entry_to_record_kwargs(entry: dict[str, Any], collector_id: str) -> dict[str, Any]:
return {
"collector_id": collector_id,
"operator": entry.get("operator") or "RIPE NCC",
"site": entry.get("site"),
"city": entry.get("city"),
"country": entry.get("country"),
"latitude": entry.get("latitude"),
"longitude": entry.get("longitude"),
"precision": entry.get("precision") or "city",
"confidence": entry.get("confidence"),
"source": "legacy_seed",
"source_url": None,
"source_note": entry.get("source_note")
or "Seeded from legacy RIPE RIS collector coordinates",
"raw_payload": entry,
"needs_confirmation": True,
"verification_status": "unverified",
"verified_at": None,
}
async def seed_default_bgp_collector_locations(session: AsyncSession) -> None:
"""Seed default RIPE RIS collector locations without overwriting users."""
payload = _load_seed_payload()
for entry in payload.get("locations", []):
aliases = entry.get("aliases") or []
collector_ids = [
coerce_str(alias)
for alias in aliases
if coerce_str(alias).startswith("rrc")
]
if not collector_ids:
continue
collector_id = collector_ids[0]
existing = await session.scalar(
select(BGPCollectorLocation).where(
BGPCollectorLocation.collector_id == collector_id
)
)
if existing:
continue
session.add(
BGPCollectorLocation(
**_seed_entry_to_record_kwargs(entry, collector_id)
)
)
await session.commit()
await refresh_bgp_collector_location_cache(session)
def get_bgp_collector_location_dict(collector_name: str) -> dict[str, Any]:
"""Return the current cached collector location dict, or ``{}`` if unknown."""
return dict(RIPE_RIS_COLLECTOR_COORDS.get(coerce_str(collector_name), {}))
def iter_known_collector_names() -> Iterator[str]:
"""Yield every collector technical name (rrcXX) known in the cache."""
return iter(sorted(RIPE_RIS_COLLECTOR_COORDS.keys()))
# ── Pipeline construction ──────────────────────────────────────────
class StoredCollectorLocationResolver:
"""Resolve a collector through the DB-backed compatibility cache."""
name = "stored_collector_location"
def resolve(self, query: LocationQuery) -> ResolverOutput:
collector = coerce_str(query.name)
if not collector:
for alias in query.aliases:
collector = coerce_str(alias)
if collector:
break
if not collector:
return ResolverOutput()
location = get_bgp_collector_location_dict(collector)
if not location:
return ResolverOutput()
latitude = location.get("latitude")
longitude = location.get("longitude")
if latitude in (None, 0.0) or longitude in (None, 0.0):
return ResolverOutput()
return ResolverOutput(
candidates=(
LocationCandidate(
latitude=float(latitude),
longitude=float(longitude),
display_name=location.get("matched_location_name") or collector,
precision=location.get("precision") or "city",
confidence=float(location.get("confidence") or 0.85),
query=f"stored_collector_location::{collector}",
source=location.get("source") or self.name,
source_note=location.get("source_note"),
matched_fields=("collector",),
needs_confirmation=bool(location.get("needs_confirmation")),
city=location.get("city"),
region=None,
country=location.get("country"),
matched_location_name=(
location.get("matched_location_name") or collector
),
location_verified_at=location.get("verified_at"),
suggested_registry_entry=None,
),
)
)
def _bgp_collector_query_plan(
query: LocationQuery,
) -> list[tuple[str, tuple[str, ...]]]:
"""Build the Nominatim query plan for a BGP collector."""
extra = query.extra or {}
site = str(extra.get("site") or "")
operator = str(extra.get("operator") or "")
city = query.city or ""
country = query.country or ""
plan: list[tuple[str, tuple[str, ...]]] = []
def add(parts: list[tuple[str, str]]) -> None:
non_empty = [(field, value) for field, value in parts if value]
if not non_empty:
return
seen: set[str] = set()
cleaned: list[str] = []
fields: list[str] = []
for field, value in non_empty:
key = normalize_text(value)
if not key or key in seen:
continue
seen.add(key)
cleaned.append(value)
fields.append(field)
if not cleaned:
return
composed = ", ".join(cleaned)
if not any(composed == existing for existing, _ in plan):
plan.append((composed, tuple(fields)))
add([("site", site), ("city", city), ("country", country)])
add([("site", site), ("country", country)])
add([("operator", operator), ("city", city), ("country", country)])
add([("city", city), ("country", country)])
return plan
BGP_COLLECTOR_PIPELINE = LocationPipeline(
[
SourceCoordinatesResolver(),
StoredCollectorLocationResolver(),
],
failure_reason=(
"Could not resolve BGP collector to renderable coordinates from"
" source coordinates or stored collector location."
),
)
BGP_COLLECTOR_COLLECTION_PIPELINE = LocationPipeline(
[
SourceCoordinatesResolver(),
NominatimResolver(
query_plan_builder=_bgp_collector_query_plan,
# Late-binding so tests can monkeypatch ``_geocode_online``.
geocoder=lambda q: _geocode_online(q),
),
],
failure_reason=(
"Could not resolve BGP collector to renderable coordinates from"
" source coordinates or online geocoding."
),
)
# ── Public API ─────────────────────────────────────────────────────
def resolve_bgp_collector_location(
collector_name: str,
*,
city: str | None = None,
country: str | None = None,
site: str | None = None,
operator: str | None = None,
) -> ResolutionResult:
"""Resolve a BGP collector to its best-known stored location."""
stored = get_bgp_collector_location_dict(collector_name)
name = coerce_str(collector_name) or None
query = LocationQuery(
name=name,
aliases=tuple(filter(None, (collector_name,))),
city=coerce_str(city or stored.get("city")) or None,
country=coerce_str(country or stored.get("country")) or None,
extra={
"site": coerce_str(site or stored.get("site")),
"operator": coerce_str(operator or stored.get("operator")) or "RIPE NCC",
},
)
return BGP_COLLECTOR_PIPELINE.resolve_best(query)
def collect_bgp_collector_location_candidates(
*,
collector: str | None = None,
city: str | None = None,
country: str | None = None,
site: str | None = None,
operator: str | None = None,
) -> tuple[list[LocationCandidate], list[str]]:
stored = get_bgp_collector_location_dict(collector or "")
name = coerce_str(collector) or None
query = LocationQuery(
name=name,
aliases=tuple(filter(None, (collector,))),
city=coerce_str(city or stored.get("city")) or None,
country=coerce_str(country or stored.get("country")) or None,
extra={
"site": coerce_str(site or stored.get("site")),
"operator": coerce_str(operator or stored.get("operator")) or "RIPE NCC",
},
)
return BGP_COLLECTOR_COLLECTION_PIPELINE.collect_candidates(query)

155
backend/app/services/bgp_event_locations.py Normal file
View File

@@ -0,0 +1,155 @@
"""BGP event location resolver.
A BGP event (announcement / withdrawal / RIB entry) is geographically tied to
the route collector that observed it. This module defines the pipeline that
turns an event payload into renderable coordinates.
Current resolver chain:
SourceCoordinates → event payload itself carries lat/lon (rare; some
enriched feeds do).
InheritFromCollector → look up the owning collector via
:func:`resolve_bgp_collector_location`.
Future plug-ins (no consumer changes required, just append to the list):
ASNFacilityResolver — origin/peer ASN → peeringdb facility.
PrefixGeoResolver — prefix → IP range geo lookup (iptoasn / opengeofeed).
"""
from __future__ import annotations
from typing import Any
from app.services.bgp_collector_locations import (
get_bgp_collector_location_dict,
)
from app.services.location import (
InheritFromAnotherEntityResolver,
LocationCandidate,
LocationPipeline,
LocationQuery,
ResolutionResult,
SourceCoordinatesResolver,
coerce_str,
)
def _inherit_from_owning_collector(
query: LocationQuery,
) -> LocationCandidate | None:
"""Look up the event's owning collector by exact name in the DB-backed cache."""
extra = query.extra or {}
collector_name = coerce_str(extra.get("collector"))
if not collector_name:
return None
legacy = get_bgp_collector_location_dict(collector_name)
if not legacy:
return None
latitude = legacy.get("latitude")
longitude = legacy.get("longitude")
if latitude in (None, 0.0) or longitude in (None, 0.0):
return None
return LocationCandidate(
latitude=float(latitude),
longitude=float(longitude),
display_name=legacy.get("matched_location_name") or collector_name,
precision=legacy.get("precision") or "city",
confidence=float(legacy.get("confidence") or 0.85),
query=f"inherit_from_collector::{collector_name}",
source="inherited_from_collector",
source_note=(
f"Inherited from owning collector {collector_name}"
),
matched_fields=("collector",),
needs_confirmation=bool(legacy.get("needs_confirmation")),
city=legacy.get("city"),
region=None,
country=legacy.get("country"),
matched_location_name=legacy.get("matched_location_name"),
location_verified_at=legacy.get("verified_at"),
suggested_registry_entry=None,
)
BGP_EVENT_PIPELINE = LocationPipeline(
[
SourceCoordinatesResolver(),
InheritFromAnotherEntityResolver(
source_lookup=_inherit_from_owning_collector,
name="inherited_from_collector",
),
# Plug new resolvers (peeringdb / ASN facility / prefix-geo) here.
],
failure_reason=(
"Could not resolve BGP event coordinates: no source coords, owning"
" collector unknown, and no fallback resolver matched."
),
)
def resolve_bgp_event_location(
*,
collector: str,
source_latitude: float | None = None,
source_longitude: float | None = None,
site: str | None = None,
operator: str | None = None,
peer_asn: int | None = None,
origin_asn: int | None = None,
prefix: str | None = None,
) -> ResolutionResult:
"""Resolve a BGP event to its renderable coordinates.
The ``peer_asn`` / ``origin_asn`` / ``prefix`` arguments are accepted
today so future resolvers (ASN→facility, prefix→geo) can consume them
without callers needing to change.
"""
query = LocationQuery(
name=collector or None,
aliases=tuple(filter(None, (collector,))),
source_latitude=source_latitude,
source_longitude=source_longitude,
extra={
"collector": collector or "",
"site": coerce_str(site),
"operator": coerce_str(operator),
"peer_asn": peer_asn,
"origin_asn": origin_asn,
"prefix": coerce_str(prefix),
},
)
return BGP_EVENT_PIPELINE.resolve_best(query)
def resolve_bgp_event_geo_dict(
collector: str,
*,
source_latitude: float | None = None,
source_longitude: float | None = None,
) -> dict[str, Any]:
"""Convenience wrapper returning the legacy ``collector_geo`` dict shape.
Preserves ``city``/``country``/``latitude``/``longitude`` keys (consumed
by existing detectors / enrichment / DB serialization) and adds
``precision``/``source``/``needs_confirmation`` for richer downstream use.
"""
result = resolve_bgp_event_location(
collector=collector,
source_latitude=source_latitude,
source_longitude=source_longitude,
)
candidate = result.location
if candidate is None:
return {}
return {
"city": candidate.city,
"country": candidate.country,
"latitude": candidate.latitude,
"longitude": candidate.longitude,
"precision": candidate.precision,
"source": candidate.source,
"needs_confirmation": candidate.needs_confirmation,
"matched_location_name": candidate.matched_location_name,
"confidence": candidate.confidence,
}

56
backend/app/services/collectors/bgp_common.py
View File

@@ -13,6 +13,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_observation import BGPObservation
from app.models.collected_data import CollectedData
from app.services.bgp_collector_locations import (
RIPE_RIS_COLLECTOR_COORDS,
get_bgp_collector_location_dict,
)
from app.services.bgp_event_locations import resolve_bgp_event_geo_dict
from app.services.bgp_incidents import create_bgp_incidents_for_anomalies
from app.services.bgp_detectors import (
detect_mass_withdrawal_anomalies,
@@ -23,32 +28,17 @@ from app.services.bgp_detectors import (
)
from app.services.bgp_enrichment import enrich_bgp_events_for_batch, extract_bgp_network_fields
RIPE_RIS_COLLECTOR_COORDS: dict[str, dict[str, Any]] = {
"rrc00": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041},
"rrc01": {"city": "London", "country": "United Kingdom", "latitude": 51.5072, "longitude": -0.1276},
"rrc03": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041},
"rrc04": {"city": "Geneva", "country": "Switzerland", "latitude": 46.2044, "longitude": 6.1432},
"rrc05": {"city": "Vienna", "country": "Austria", "latitude": 48.2082, "longitude": 16.3738},
"rrc06": {"city": "Otemachi", "country": "Japan", "latitude": 35.686, "longitude": 139.7671},
"rrc07": {"city": "Stockholm", "country": "Sweden", "latitude": 59.3293, "longitude": 18.0686},
"rrc10": {"city": "Milan", "country": "Italy", "latitude": 45.4642, "longitude": 9.19},
"rrc11": {"city": "New York", "country": "United States", "latitude": 40.7128, "longitude": -74.006},
"rrc12": {"city": "Frankfurt", "country": "Germany", "latitude": 50.1109, "longitude": 8.6821},
"rrc13": {"city": "Moscow", "country": "Russia", "latitude": 55.7558, "longitude": 37.6173},
"rrc14": {"city": "Palo Alto", "country": "United States", "latitude": 37.4419, "longitude": -122.143},
"rrc15": {"city": "Sao Paulo", "country": "Brazil", "latitude": -23.5558, "longitude": -46.6396},
"rrc16": {"city": "Miami", "country": "United States", "latitude": 25.7617, "longitude": -80.1918},
"rrc18": {"city": "Barcelona", "country": "Spain", "latitude": 41.3874, "longitude": 2.1686},
"rrc19": {"city": "Johannesburg", "country": "South Africa", "latitude": -26.2041, "longitude": 28.0473},
"rrc20": {"city": "Zurich", "country": "Switzerland", "latitude": 47.3769, "longitude": 8.5417},
"rrc21": {"city": "Paris", "country": "France", "latitude": 48.8566, "longitude": 2.3522},
"rrc22": {"city": "Bucharest", "country": "Romania", "latitude": 44.4268, "longitude": 26.1025},
"rrc23": {"city": "Singapore", "country": "Singapore", "latitude": 1.3521, "longitude": 103.8198},
"rrc24": {"city": "Montevideo", "country": "Uruguay", "latitude": -34.9011, "longitude": -56.1645},
"rrc25": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041},
"rrc26": {"city": "Dubai", "country": "United Arab Emirates", "latitude": 25.2048, "longitude": 55.2708},
}
# Re-exported for backward compatibility with anything that imports
# ``RIPE_RIS_COLLECTOR_COORDS`` from this module. New code should call
# ``app.services.bgp_collector_locations.get_bgp_collector_location_dict()``
# or ``resolve_bgp_collector_location()`` instead — those use the DB-backed
# collector-location cache.
__all__ = [
"RIPE_RIS_COLLECTOR_COORDS",
"normalize_bgp_event",
"save_bgp_observations_for_batch",
"create_bgp_anomalies_for_batch",
]
def _safe_int(value: Any) -> int | None:
@@ -131,7 +121,19 @@ def normalize_bgp_event(payload: dict[str, Any], *, project: str) -> dict[str, A
)
source_id = hashlib.sha1(source_material.encode("utf-8")).hexdigest()[:24]
collector_location = RIPE_RIS_COLLECTOR_COORDS.get(collector, {})
# Routes through the BGP event pipeline: source coords (if any) →
# collector inheritance. Returned dict keeps the legacy
# {city, country, latitude, longitude} keys plus richer
# {precision, source, needs_confirmation, matched_location_name, confidence}.
collector_location = resolve_bgp_event_geo_dict(
collector,
source_latitude=payload.get("latitude"),
source_longitude=payload.get("longitude"),
)
# Empty result (unknown collector & no source coords) — keep the
# downstream-expected dict shape so detectors / serializers don't crash.
if not collector_location:
collector_location = get_bgp_collector_location_dict(collector)
network_fields = extract_bgp_network_fields(prefix)
metadata = {
"project": project,

863
backend/app/services/compute_center_locations.py Normal file
View File

@@ -0,0 +1,863 @@
"""Compute-center location resolver, built on the shared location pipeline.
This module is a thin domain wrapper that wires up
:mod:`app.services.location` for compute centers:
SourceCoordinates
The online Nominatim step is intentionally reserved for the user-triggered
``collect-location`` flow. The regular GeoJSON endpoint runs during Earth
startup, so it must stay local and deterministic.
For the full design and the reason behind the abstraction (compute centers,
BGP collectors, BGP events, and future entities all share one pipeline),
see ``docs/plans/location-resolver-shared-pipeline-plan.md``.
The ``ComputeCenterLocation`` dataclass and the public function signatures are
preserved verbatim so existing callers and tests do not need to change.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
from functools import lru_cache
from typing import Any
import httpx
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.collected_data_fields import get_record_field
from app.models.collected_data import CollectedData
from app.models.compute_center_location import ComputeCenterLocationRecord
from app.services.location import (
LocationCandidate,
LocationPipeline,
LocationQuery,
NominatimResolver,
ResolverOutput,
SourceCoordinatesResolver,
build_default_nominatim_geocoder,
coerce_str,
normalize_country_text,
normalize_text,
parse_float,
)
ROR_SEARCH_URL = "https://api.ror.org/v2/organizations"
DEFAULT_ROR_USER_AGENT = "planet-earth-location-resolver/1.0"
DEFAULT_ROR_TIMEOUT_SECONDS = 8.0
RENDERABLE_PRECISIONS: tuple[str, ...] = ("precise", "site", "city")
FORBIDDEN_PRECISIONS: tuple[str, ...] = (
"country",
"estimated_country",
"country_major_compute_city",
"region",
"unknown",
)
# ── Public dataclasses ──────────────────────────────────────────────
@dataclass(frozen=True)
class ComputeCenterLocation:
latitude: float | None
longitude: float | None
location_precision: str
geography_mode: str
is_estimated: bool
estimated_reason: str | None = None
location_confidence: float | None = None
location_source: str | None = None
location_source_note: str | None = None
location_verified_at: str | None = None
matched_location_name: str | None = None
needs_confirmation: bool = False
city: str | None = None
region: str | None = None
country: str | None = None
@property
def is_renderable(self) -> bool:
if self.latitude in (None, 0.0) or self.longitude in (None, 0.0):
return False
return self.location_precision in RENDERABLE_PRECISIONS
def to_geojson_properties(self) -> dict[str, Any]:
return {
"latitude": self.latitude,
"longitude": self.longitude,
"location_precision": self.location_precision,
"geography_mode": self.geography_mode,
"is_estimated": self.is_estimated,
"estimated_reason": self.estimated_reason,
"location_confidence": self.location_confidence,
"location_source": self.location_source,
"location_source_note": self.location_source_note,
"location_verified_at": self.location_verified_at,
"matched_location_name": self.matched_location_name,
"needs_confirmation": self.needs_confirmation,
}
@dataclass(frozen=True)
class ResolutionDiagnostic:
failure_reason: str
attempted_queries: tuple[str, ...] = ()
record_id: int | None = None
source: str | None = None
source_id: str | None = None
name: str | None = None
country: str | None = None
city: str | None = None
site: str | None = None
operator: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"failure_reason": self.failure_reason,
"attempted_queries": list(self.attempted_queries),
"record_id": self.record_id,
"source": self.source,
"source_id": self.source_id,
"name": self.name,
"country": self.country,
"city": self.city,
"site": self.site,
"operator": self.operator,
}
@dataclass(frozen=True)
class ResolutionResult:
location: ComputeCenterLocation | None
diagnostic: ResolutionDiagnostic | None
@property
def is_resolved(self) -> bool:
return bool(self.location and self.location.is_renderable)
# ── Geocoder (kept at module level so tests can monkeypatch + cache_clear) ──
_geocode_online = build_default_nominatim_geocoder()
# ── Stored location cache ───────────────────────────────────────────
COMPUTE_CENTER_LOCATION_CACHE: dict[str, dict[str, Any]] = {}
def _cache_key(source: str | None, source_id: str | None) -> str:
return f"{coerce_str(source)}:{coerce_str(source_id)}"
def set_compute_center_location_cache(
locations: dict[str, dict[str, Any]],
) -> None:
COMPUTE_CENTER_LOCATION_CACHE.clear()
COMPUTE_CENTER_LOCATION_CACHE.update(
{coerce_str(key): dict(value) for key, value in locations.items()}
)
async def refresh_compute_center_location_cache(
session: AsyncSession,
) -> dict[str, dict[str, Any]]:
result = await session.execute(select(ComputeCenterLocationRecord))
records = result.scalars().all()
cache = {}
for record in records:
if not hasattr(record, "to_location_dict"):
continue
if not record.source or not record.source_id:
continue
cache[_cache_key(record.source, record.source_id)] = record.to_location_dict()
set_compute_center_location_cache(cache)
return cache
def get_compute_center_location_dict(
source: str | None,
source_id: str | None,
) -> dict[str, Any]:
return dict(COMPUTE_CENTER_LOCATION_CACHE.get(_cache_key(source, source_id), {}))
# ── Pipeline construction ──────────────────────────────────────────
@lru_cache(maxsize=512)
def _lookup_ror_organization(query: str) -> dict[str, Any] | None:
"""Lookup a research organization in ROR for user-triggered candidates."""
if not query:
return None
response = httpx.get(
ROR_SEARCH_URL,
params={"query": query},
headers={"User-Agent": DEFAULT_ROR_USER_AGENT},
timeout=DEFAULT_ROR_TIMEOUT_SECONDS,
)
response.raise_for_status()
payload = response.json()
items = payload.get("items") if isinstance(payload, dict) else None
if not isinstance(items, list) or not items:
return None
first = items[0]
if not isinstance(first, dict):
return None
organization = first.get("organization")
if isinstance(organization, dict):
return organization
return first
def _compute_center_ror_query_plan(
query: LocationQuery,
) -> list[tuple[str, tuple[str, ...]]]:
extra = query.extra or {}
raw_parts: list[tuple[str, str]] = [
("site", coerce_str(extra.get("site"))),
("operator", coerce_str(extra.get("operator"))),
("organization", coerce_str(extra.get("organization"))),
]
for field, value in tuple(raw_parts):
if "/" not in value:
continue
raw_parts.extend(
(field, part.strip())
for part in value.split("/")
if len(part.strip()) >= 3
)
plan: list[tuple[str, tuple[str, ...]]] = []
seen: set[str] = set()
for field, value in raw_parts:
key = normalize_text(value)
if not key or key in seen:
continue
seen.add(key)
plan.append((value, (field,)))
return plan
def _organization_label(organization: dict[str, Any], fallback: str) -> str:
names = organization.get("names")
if isinstance(names, list):
for name in names:
if not isinstance(name, dict):
continue
types = name.get("types")
if isinstance(types, list) and "ror_display" in types:
value = coerce_str(name.get("value"))
if value:
return value
for name in names:
if isinstance(name, dict):
value = coerce_str(name.get("value"))
if value:
return value
return fallback
class ROROrganizationResolver:
"""Resolve source-provided organization/site text through the open ROR API."""
name = "ror_organization_registry"
def __init__(
self,
*,
query_plan_builder=_compute_center_ror_query_plan,
lookup=lambda q: _lookup_ror_organization(q),
confidence: float = 0.68,
) -> None:
self._query_plan_builder = query_plan_builder
self._lookup = lookup
self._confidence = confidence
def resolve(self, query: LocationQuery):
from app.services.location import ResolverOutput
from app.services.location.text import parse_float
attempted: list[str] = []
candidates: list[LocationCandidate] = []
context_country = normalize_text(normalize_country_text(query.country))
for ror_query, matched_fields in self._query_plan_builder(query):
attempted.append(f"ror:{ror_query}")
try:
organization = self._lookup(ror_query)
except Exception:
continue
if not isinstance(organization, dict):
continue
locations = organization.get("locations")
if not isinstance(locations, list) or not locations:
continue
location = locations[0]
if not isinstance(location, dict):
continue
details = location.get("geonames_details")
if not isinstance(details, dict):
continue
latitude = parse_float(details.get("lat"))
longitude = parse_float(details.get("lng"))
if latitude in (None, 0.0) or longitude in (None, 0.0):
continue
country = normalize_country_text(details.get("country_name"))
if context_country and normalize_text(country) != context_country:
continue
city = coerce_str(details.get("name")) or None
region = coerce_str(details.get("country_subdivision_name")) or None
display_name = _organization_label(organization, ror_query)
ror_id = coerce_str(organization.get("id"))
geonames_id = location.get("geonames_id")
source_note = (
f"ROR organization match: {display_name}"
+ (f" ({ror_id})" if ror_id else "")
+ (f"; GeoNames {geonames_id}" if geonames_id else "")
)
candidates.append(
LocationCandidate(
latitude=latitude,
longitude=longitude,
display_name=display_name,
precision="city",
confidence=self._confidence,
query=ror_query,
source=self.name,
source_note=source_note,
matched_fields=matched_fields,
needs_confirmation=True,
city=city,
region=region,
country=country or query.country,
matched_location_name=display_name,
location_verified_at=None,
suggested_registry_entry=None,
)
)
return ResolverOutput(
candidates=tuple(candidates),
attempted_queries=tuple(attempted),
)
class StoredComputeCenterLocationResolver:
"""Resolve a compute center through the DB-backed current-location cache."""
name = "stored_compute_center_location"
def resolve(self, query: LocationQuery) -> ResolverOutput:
extra = query.extra or {}
stored = get_compute_center_location_dict(
coerce_str(extra.get("source")),
coerce_str(extra.get("source_id")),
)
if not stored:
return ResolverOutput()
latitude = parse_float(stored.get("latitude"))
longitude = parse_float(stored.get("longitude"))
if latitude in (None, 0.0) or longitude in (None, 0.0):
return ResolverOutput()
return ResolverOutput(
candidates=(
LocationCandidate(
latitude=latitude,
longitude=longitude,
display_name=stored.get("name") or query.name or "Compute center",
precision=stored.get("precision") or "city",
confidence=float(stored.get("confidence") or 0.85),
query=f"stored_compute_center_location::{stored.get('source')}:{stored.get('source_id')}",
source=self.name,
source_note=stored.get("source_note"),
matched_fields=("source", "source_id"),
needs_confirmation=bool(stored.get("needs_confirmation")),
city=stored.get("city") or query.city,
region=None,
country=stored.get("country") or query.country,
matched_location_name=stored.get("site") or stored.get("name") or query.name,
location_verified_at=stored.get("verified_at"),
suggested_registry_entry=None,
),
)
)
def _short_system_name(name: Any) -> str:
"""Strip vendor/system suffix from TOP500 names like ``"El Capitan - HPE Cray ..."``."""
text = coerce_str(name)
if not text:
return ""
head = text.split(" - ", 1)[0].strip()
return head or text
def _record_context(record: Any, metadata: dict[str, Any]) -> dict[str, str]:
name = coerce_str(getattr(record, "name", None))
return {
"source": coerce_str(getattr(record, "source", None)),
"source_id": coerce_str(getattr(record, "source_id", None)),
"name": name,
"name_short": _short_system_name(name),
"city": coerce_str(get_record_field(record, "city")),
"country": coerce_str(get_record_field(record, "country")),
"site": coerce_str(metadata.get("site") or metadata.get("organization")),
"operator": coerce_str(
metadata.get("operator")
or metadata.get("organization")
or metadata.get("owner")
or metadata.get("manufacturer")
),
"organization": coerce_str(metadata.get("organization")),
}
def _context_to_query(
context: dict[str, str],
*,
source_lat: float | None = None,
source_lon: float | None = None,
) -> LocationQuery:
name = context.get("name") or None
name_short = context.get("name_short") or ""
aliases: tuple[str, ...] = ()
if name_short and name_short != name:
aliases = (name_short,)
return LocationQuery(
name=name,
aliases=aliases,
city=context.get("city") or None,
country=context.get("country") or None,
source_latitude=source_lat,
source_longitude=source_lon,
extra={
"source": context.get("source") or "",
"source_id": context.get("source_id") or "",
"site": context.get("site") or "",
"operator": context.get("operator") or "",
"organization": context.get("organization") or "",
},
)
def _compute_center_query_plan(
query: LocationQuery,
) -> list[tuple[str, tuple[str, ...]]]:
"""Build the Nominatim query plan for a compute-center query.
Mirrors the legacy ``_build_online_query_plan`` ordering exactly.
"""
name = query.name or ""
name_short = (query.aliases[0] if query.aliases else "") or name
extra = query.extra or {}
site = str(extra.get("site") or "")
operator = str(extra.get("operator") or "")
city = query.city or ""
country = query.country or ""
plan: list[tuple[str, tuple[str, ...]]] = []
def add(parts: list[tuple[str, str]]) -> None:
non_empty = [(field, value) for field, value in parts if value]
if not non_empty:
return
seen: set[str] = set()
cleaned: list[str] = []
fields: list[str] = []
for field, value in non_empty:
key = normalize_text(value)
if not key or key in seen:
continue
seen.add(key)
cleaned.append(value)
fields.append(field)
if not cleaned:
return
composed = ", ".join(cleaned)
if not any(composed == existing for existing, _ in plan):
plan.append((composed, tuple(fields)))
add([("site", site), ("country", country)])
add([("operator", operator), ("city", city), ("country", country)])
add([("name", name_short), ("operator", operator), ("country", country)])
add([("name", name_short), ("site", site)])
add([("name", name_short), ("country", country)])
add([("name", name_short), ("city", city), ("country", country)])
add([("city", city), ("country", country)])
if name and name != name_short:
add([("name", name), ("country", country)])
return plan
COMPUTE_CENTER_PIPELINE = LocationPipeline(
[
SourceCoordinatesResolver(),
StoredComputeCenterLocationResolver(),
],
failure_reason=(
"Could not resolve to city-level coordinates from source coords"
" or stored compute-center location."
),
)
COMPUTE_CENTER_COLLECTION_PIPELINE = LocationPipeline(
[
SourceCoordinatesResolver(),
ROROrganizationResolver(),
NominatimResolver(
query_plan_builder=_compute_center_query_plan,
# Late-binding so test monkeypatching of ``_geocode_online`` works.
geocoder=lambda q: _geocode_online(q),
),
],
failure_reason=(
"Could not resolve to city-level coordinates from source coords"
", ROR organization lookup, or online geocoding."
),
)
# ── Candidate → ComputeCenterLocation conversion ───────────────────
_GEOGRAPHY_MODE_BY_SOURCE = {
"source_coordinates": "source_coordinates",
"stored_compute_center_location": "stored_compute_center_location",
"ror_organization_registry": "ror_organization",
"nominatim_online_geocode": "online_geocode",
}
def _candidate_to_location(
candidate: LocationCandidate,
*,
context: dict[str, str],
) -> ComputeCenterLocation:
geography_mode = _GEOGRAPHY_MODE_BY_SOURCE.get(candidate.source, "online_geocode")
is_estimated = candidate.needs_confirmation or candidate.source.startswith(
"nominatim"
)
estimated_reason: str | None
if candidate.source == "source_coordinates":
estimated_reason = None
elif candidate.source == "stored_compute_center_location":
estimated_reason = candidate.source_note
elif candidate.source == "ror_organization_registry":
fields_summary = ", ".join(candidate.matched_fields) or "organization"
estimated_reason = (
f"Resolved by ROR organization lookup '{candidate.query}' "
f"(matched fields: {fields_summary})"
)
elif candidate.source == "nominatim_online_geocode":
fields_summary = ", ".join(candidate.matched_fields) or "name"
estimated_reason = (
f"Resolved by online geocoding query '{candidate.query}' "
f"(matched fields: {fields_summary})"
)
else:
estimated_reason = candidate.source_note
country = (
candidate.country
or normalize_country_text(context.get("country"))
or context.get("country")
or None
)
return ComputeCenterLocation(
latitude=candidate.latitude,
longitude=candidate.longitude,
location_precision=candidate.precision,
geography_mode=geography_mode,
is_estimated=is_estimated,
estimated_reason=estimated_reason,
location_confidence=candidate.confidence,
location_source=candidate.source,
location_source_note=candidate.source_note,
location_verified_at=candidate.location_verified_at,
matched_location_name=candidate.matched_location_name
or context.get("name")
or None,
needs_confirmation=candidate.needs_confirmation,
city=candidate.city or context.get("city") or None,
region=candidate.region,
country=country,
)
def _diagnostic_for(
record: Any,
context: dict[str, str],
*,
failure_reason: str,
attempted_queries: tuple[str, ...] = (),
) -> ResolutionDiagnostic:
return ResolutionDiagnostic(
failure_reason=failure_reason,
attempted_queries=attempted_queries,
record_id=getattr(record, "id", None),
source=getattr(record, "source", None),
source_id=getattr(record, "source_id", None),
name=context.get("name") or getattr(record, "name", None),
country=context.get("country") or None,
city=context.get("city") or None,
site=context.get("site") or None,
operator=context.get("operator") or None,
)
# ── Public API ─────────────────────────────────────────────────────
def resolve_compute_center_location(
record: Any,
metadata: dict[str, Any] | None = None,
) -> ComputeCenterLocation:
"""Backwards-compatible thin wrapper returning the renderable location only.
Records that cannot be resolved to city-level get a placeholder
:class:`ComputeCenterLocation` with ``location_precision='unknown'``.
Callers should generally prefer :func:`resolve_compute_center_location_full`.
"""
full = resolve_compute_center_location_full(record, metadata)
return full.location or ComputeCenterLocation(
latitude=None,
longitude=None,
location_precision="unknown",
geography_mode="unresolved",
is_estimated=True,
estimated_reason="No resolvable location hints",
location_confidence=0.0,
location_source="unknown",
location_source_note=(
"No source coordinates, ROR organization match, or online"
" geocoding result."
),
matched_location_name=None,
needs_confirmation=False,
)
def resolve_compute_center_location_full(
record: Any,
metadata: dict[str, Any] | None = None,
*,
allow_online: bool = False,
) -> ResolutionResult:
metadata = metadata or {}
context = _record_context(record, metadata)
from app.services.location.text import parse_float as _parse_float
source_lat = _parse_float(get_record_field(record, "latitude"))
source_lon = _parse_float(get_record_field(record, "longitude"))
if source_lat in (None, 0.0):
source_lat = None
if source_lon in (None, 0.0):
source_lon = None
query = _context_to_query(
context, source_lat=source_lat, source_lon=source_lon
)
pipeline = (
COMPUTE_CENTER_COLLECTION_PIPELINE
if allow_online
else COMPUTE_CENTER_PIPELINE
)
pipeline_result = pipeline.resolve_best(query)
if pipeline_result.location and pipeline_result.location.precision in RENDERABLE_PRECISIONS:
location = _candidate_to_location(pipeline_result.location, context=context)
return ResolutionResult(location=location, diagnostic=None)
return ResolutionResult(
location=None,
diagnostic=_diagnostic_for(
record,
context,
failure_reason=(
"Could not resolve to city-level coordinates from source coords"
", ROR organization lookup, or online geocoding."
if allow_online
else (
"Could not resolve to city-level coordinates from source coords"
" or stored compute-center location."
)
),
attempted_queries=pipeline_result.attempted_queries,
),
)
def collect_location_candidates(
*,
name: str | None = None,
source: str | None = None,
source_id: str | None = None,
operator: str | None = None,
site: str | None = None,
city: str | None = None,
country: str | None = None,
organization: str | None = None,
record_id: int | None = None,
) -> tuple[list[LocationCandidate], list[str]]:
"""Run the full resolution chain and return ranked candidates with attempted queries.
The unused ``source`` / ``source_id`` / ``record_id`` arguments are kept
for backward compatibility with the API handler that calls this function.
"""
name_value = coerce_str(name)
context: dict[str, str] = {
"source": coerce_str(source),
"source_id": coerce_str(source_id),
"name": name_value,
"name_short": _short_system_name(name_value),
"city": coerce_str(city),
"country": coerce_str(country),
"site": coerce_str(site or organization),
"operator": coerce_str(operator or organization),
"organization": coerce_str(organization),
}
query = _context_to_query(context)
return COMPUTE_CENTER_COLLECTION_PIPELINE.collect_candidates(query)
def _record_operator(metadata: dict[str, Any]) -> str | None:
return coerce_str(
metadata.get("operator")
or metadata.get("organization")
or metadata.get("owner")
or metadata.get("manufacturer")
) or None
async def seed_compute_center_locations_from_source_coords(
session: AsyncSession,
) -> None:
"""Seed stored compute-center locations only from real source coordinates."""
stmt = (
select(CollectedData)
.where(CollectedData.source.in_(["top500", "epoch_ai_gpu"]))
.where(CollectedData.is_current.is_(True))
)
result = await session.execute(stmt)
records = result.scalars().all()
changed = False
for record in records:
source_value = coerce_str(getattr(record, "source", None))
source_id = coerce_str(getattr(record, "source_id", None))
if not source_value or not source_id:
continue
latitude = parse_float(get_record_field(record, "latitude"))
longitude = parse_float(get_record_field(record, "longitude"))
if latitude in (None, 0.0) or longitude in (None, 0.0):
continue
existing = await session.scalar(
select(ComputeCenterLocationRecord)
.where(ComputeCenterLocationRecord.source == source_value)
.where(ComputeCenterLocationRecord.source_id == source_id)
)
if existing:
continue
metadata = record.extra_data or {}
session.add(
ComputeCenterLocationRecord(
source=source_value,
source_id=source_id,
name=getattr(record, "name", None),
operator=_record_operator(metadata),
site=coerce_str(metadata.get("site") or metadata.get("organization")) or None,
city=coerce_str(get_record_field(record, "city")) or None,
country=coerce_str(get_record_field(record, "country")) or None,
latitude=latitude,
longitude=longitude,
precision="precise",
confidence=1.0,
location_source="source_coordinates",
source_note="Seeded from source-provided compute-center coordinates",
raw_payload={
"record_id": getattr(record, "id", None),
"source": source_value,
"source_id": source_id,
},
needs_confirmation=False,
verification_status="source_provided",
verified_at=None,
)
)
changed = True
if changed:
await session.commit()
await refresh_compute_center_location_cache(session)
async def upsert_compute_center_location(
session: AsyncSession,
*,
source: str,
source_id: str,
name: str | None = None,
operator: str | None = None,
site: str | None = None,
city: str | None = None,
country: str | None = None,
latitude: float,
longitude: float,
precision: str = "city",
confidence: float | None = None,
location_source: str = "manual_selection",
source_url: str | None = None,
source_note: str | None = None,
raw_payload: dict[str, Any] | None = None,
needs_confirmation: bool = False,
verification_status: str = "verified",
) -> ComputeCenterLocationRecord:
existing = await session.scalar(
select(ComputeCenterLocationRecord)
.where(ComputeCenterLocationRecord.source == source)
.where(ComputeCenterLocationRecord.source_id == source_id)
)
verified_at = None if needs_confirmation else datetime.now(UTC)
values = {
"name": name,
"operator": operator,
"site": site,
"city": city,
"country": country,
"latitude": latitude,
"longitude": longitude,
"precision": precision,
"confidence": confidence,
"location_source": location_source,
"source_url": source_url,
"source_note": source_note,
"raw_payload": raw_payload or {},
"needs_confirmation": needs_confirmation,
"verification_status": verification_status,
"verified_at": verified_at,
}
if existing:
for key, value in values.items():
setattr(existing, key, value)
record = existing
else:
record = ComputeCenterLocationRecord(
source=source,
source_id=source_id,
**values,
)
session.add(record)
await session.commit()
await session.refresh(record)
await refresh_compute_center_location_cache(session)
return record

119
backend/app/services/docs_gatekeeper.py Normal file
View File

@@ -0,0 +1,119 @@
"""Server-side Docs metadata and Gatekeeper authorization helpers."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Literal
from app.models.user import User
DocsAccess = Literal["public", "docs_user", "docs_developer", "docs_admin"]
DocsLang = Literal["zh", "en"]
VALID_DOCS_LANGS = {"zh", "en"}
DOCS_README_FILENAME = "README.md"
DEFAULT_DOCS_SLUG = "overview"
REPO_ROOT = Path(__file__).resolve().parents[3]
TECHNICAL_DOCS_ROOT = REPO_ROOT / "docs" / "technical"
@dataclass(frozen=True)
class DocsMetadata:
filename: str
slug: str
access: DocsAccess
group: str
order: int
zh_title: str
en_title: str
DOCS_METADATA: tuple[DocsMetadata, ...] = (
DocsMetadata(DOCS_README_FILENAME, DEFAULT_DOCS_SLUG, "public", "Overview", 0, "技术文档", "Technical Docs"),
DocsMetadata("quickstart.md", "quickstart", "public", "Manual", 1, "快速开始", "Quickstart"),
DocsMetadata("manual.md", "manual", "public", "Manual", 2, "Planet 使用手册", "Planet Manual"),
DocsMetadata("location-pipeline-user.md", "location-pipeline-user", "public", "Manual", 3, "Earth 位置候选采集使用手册", "Earth Location Candidate Collection User Guide"),
DocsMetadata("earth-frontend-context.md", "earth-frontend-context", "docs_developer", "Earth", 10, "Earth 前端结构", "Earth Frontend Context"),
DocsMetadata("earth-layer-style-reference.md", "earth-layer-style-reference", "docs_developer", "Earth", 11, "Earth 图层样式属性索引", "Earth Layer Style Reference"),
DocsMetadata("earth-render-layer-order.md", "earth-render-layer-order", "docs_developer", "Earth", 12, "Earth 渲染图层顺序", "Earth Render Layer Order"),
DocsMetadata("earth-satellite-footprint-policy.md", "earth-satellite-footprint-policy", "docs_developer", "Earth", 13, "Earth 卫星覆盖策略", "Earth Satellite Footprint Policy"),
DocsMetadata("earth-bgp-context.md", "earth-bgp-context", "docs_developer", "Earth", 14, "BGP 态势上下文", "BGP Context"),
DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Earth", 15, "新闻直播采集格式", "News Live Streams Collector Format"),
DocsMetadata("earth-interactable-usage.md", "earth-interactable-usage", "docs_developer", "Earth", 16, "Earth 可交互图标接入", "Earth Interactable Usage"),
DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 17, "Earth 工具栏与浮层协同", "Earth Toolbar and Overlay Coordination"),
DocsMetadata("frontend-admin-frontend-context.md", "frontend-admin-frontend-context", "docs_developer", "Frontend", 20, "控制台前端结构", "Admin Frontend Context"),
DocsMetadata("frontend-layout-guidelines.md", "frontend-layout-guidelines", "docs_developer", "Frontend", 21, "前端布局指南", "Frontend Layout Guidelines"),
DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Frontend", 22, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
DocsMetadata("backend-collectors.md", "backend-collectors", "docs_developer", "Backend", 30, "数据采集系统", "Data Collectors"),
DocsMetadata("backend-system-service-control.md", "backend-system-service-control", "docs_admin", "Backend", 31, "系统服务控制", "System Service Control"),
DocsMetadata("datasource-collector-settings-connectivity.md", "datasource-collector-settings-connectivity", "docs_developer", "Backend", 32, "数据源、采集器设置与连接验证", "Datasource Collector Settings and Connectivity"),
DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"),
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 34, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"),
DocsMetadata("ops-docker-compose-buildx-upgrade.md", "ops-docker-compose-buildx-upgrade", "docs_admin", "Ops", 50, "Docker + Compose + Buildx 升级", "Docker + Compose + Buildx Upgrade"),
DocsMetadata("ops-planet-sh-startup.md", "ops-planet-sh-startup", "docs_admin", "Ops", 51, "planet.sh 启动机制", "planet.sh Startup"),
)
DOCS_BY_SLUG = {entry.slug: entry for entry in DOCS_METADATA}
def get_user_gatekeeper_groups(user: User | None) -> set[str]:
if user is None:
return set()
role = user.role.value if hasattr(user.role, "value") else str(user.role or "")
if role == "super_admin":
return {"docs_user", "docs_developer", "docs_admin"}
if role == "admin":
return {"docs_user", "docs_developer", "docs_admin"}
groups = set()
raw_groups = user.gatekeeper_groups or []
if isinstance(raw_groups, list):
groups.update(str(group) for group in raw_groups)
if "docs_admin" in groups:
groups.update({"docs_developer", "docs_user"})
if "docs_developer" in groups:
groups.add("docs_user")
return groups
def can_read_doc(entry: DocsMetadata, user: User | None) -> bool:
if entry.access == "public":
return True
return entry.access in get_user_gatekeeper_groups(user)
def doc_path_for(entry: DocsMetadata, lang: str) -> Path:
if lang not in VALID_DOCS_LANGS:
raise ValueError("Unsupported docs language")
return TECHNICAL_DOCS_ROOT / lang / entry.filename
def title_for(entry: DocsMetadata, lang: str) -> str:
return entry.zh_title if lang == "zh" else entry.en_title
def catalog_for_user(user: User | None) -> list[dict]:
items: list[dict] = []
for entry in DOCS_METADATA:
if not can_read_doc(entry, user):
continue
for lang in sorted(VALID_DOCS_LANGS):
if not doc_path_for(entry, lang).exists():
continue
items.append(
{
"slug": entry.slug,
"filename": entry.filename,
"lang": lang,
"title": title_for(entry, lang),
"group": entry.group,
"order": entry.order,
"access": entry.access,
}
)
return sorted(items, key=lambda item: (item["lang"], item["order"], item["title"]))

57
backend/app/services/location/__init__.py Normal file
View File

@@ -0,0 +1,57 @@
"""Shared location-resolution pipeline.
A reusable abstraction for "given a record, decide its lat/lon" — used by
compute centers, BGP collectors, BGP events, and any future entity that needs
location estimation.
Each domain wires its own :class:`LocationPipeline` from a sequence of
:class:`LocationResolver` instances. Future algorithms (peeringdb, IXP tables,
user-confirmed coordinates, …) plug in by implementing the protocol — no
changes needed to consumers.
"""
from .models import (
LocationCandidate,
LocationQuery,
ResolutionDiagnostic,
ResolutionResult,
ResolverOutput,
)
from .pipeline import LocationPipeline, LocationResolver
from .resolvers.inherit import InheritFromAnotherEntityResolver
from .resolvers.nominatim import (
NominatimResolver,
build_default_nominatim_geocoder,
interpret_geocode_result,
)
from .resolvers.registry import RegistryResolver, default_score_alias_match
from .resolvers.source_coordinates import SourceCoordinatesResolver
from .text import (
city_key,
coerce_str,
normalize_country_text,
normalize_text,
parse_float,
)
__all__ = [
"LocationCandidate",
"LocationPipeline",
"LocationQuery",
"LocationResolver",
"ResolutionDiagnostic",
"ResolutionResult",
"ResolverOutput",
"InheritFromAnotherEntityResolver",
"NominatimResolver",
"RegistryResolver",
"SourceCoordinatesResolver",
"build_default_nominatim_geocoder",
"city_key",
"coerce_str",
"default_score_alias_match",
"interpret_geocode_result",
"normalize_country_text",
"normalize_text",
"parse_float",
]

126
backend/app/services/location/models.py Normal file
View File

@@ -0,0 +1,126 @@
"""Domain-neutral data structures for the location pipeline."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Mapping
# Renderable precision tiers, ordered from most precise to least.
RENDERABLE_PRECISIONS: tuple[str, ...] = ("precise", "site", "city")
@dataclass(frozen=True)
class LocationQuery:
"""Domain-neutral input for the resolution pipeline.
``name`` and ``aliases`` are matched against registry alias indexes;
``city`` / ``country`` / ``region`` provide geographic context for both
registry lookups and Nominatim queries; ``source_latitude`` /
``source_longitude`` short-circuit when the record already carries
coordinates; ``extra`` carries domain-specific fields (operator, site,
organization, asn, peer_ip, …) that resolvers can opt into.
"""
name: str | None = None
aliases: tuple[str, ...] = ()
city: str | None = None
country: str | None = None
region: str | None = None
source_latitude: float | None = None
source_longitude: float | None = None
extra: Mapping[str, Any] = field(default_factory=dict)
@dataclass(frozen=True)
class LocationCandidate:
"""A resolved location candidate produced by a resolver."""
latitude: float
longitude: float
display_name: str
precision: str # "precise" | "site" | "city" | (rejected: country/unknown)
confidence: float
query: str
source: str
source_note: str | None
matched_fields: tuple[str, ...]
needs_confirmation: bool
city: str | None = None
region: str | None = None
country: str | None = None
matched_location_name: str | None = None
location_verified_at: str | None = None
suggested_registry_entry: dict[str, Any] | None = None
def to_dict(self) -> dict[str, Any]:
return {
"latitude": self.latitude,
"longitude": self.longitude,
"display_name": self.display_name,
"precision": self.precision,
"confidence": self.confidence,
"query": self.query,
"source": self.source,
"source_note": self.source_note,
"matched_fields": list(self.matched_fields),
"needs_confirmation": self.needs_confirmation,
"city": self.city,
"region": self.region,
"country": self.country,
"matched_location_name": self.matched_location_name,
"location_verified_at": self.location_verified_at,
"suggested_registry_entry": self.suggested_registry_entry,
}
@dataclass(frozen=True)
class ResolverOutput:
"""What a single resolver returns from one ``resolve()`` call."""
candidates: tuple[LocationCandidate, ...] = ()
attempted_queries: tuple[str, ...] = ()
@dataclass(frozen=True)
class ResolutionDiagnostic:
"""Why we could not resolve, plus what we tried."""
failure_reason: str
attempted_queries: tuple[str, ...] = ()
record_id: int | None = None
source: str | None = None
source_id: str | None = None
name: str | None = None
country: str | None = None
city: str | None = None
site: str | None = None
operator: str | None = None
extra: Mapping[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
"failure_reason": self.failure_reason,
"attempted_queries": list(self.attempted_queries),
"record_id": self.record_id,
"source": self.source,
"source_id": self.source_id,
"name": self.name,
"country": self.country,
"city": self.city,
"site": self.site,
"operator": self.operator,
**({"extra": dict(self.extra)} if self.extra else {}),
}
@dataclass(frozen=True)
class ResolutionResult:
"""Pipeline output: best candidate (if any) + diagnostic on miss."""
location: LocationCandidate | None
diagnostic: ResolutionDiagnostic | None
attempted_queries: tuple[str, ...] = ()
@property
def is_resolved(self) -> bool:
return bool(self.location)

126
backend/app/services/location/pipeline.py Normal file
View File

@@ -0,0 +1,126 @@
"""Pipeline that runs a sequence of :class:`LocationResolver` instances."""
from __future__ import annotations
from typing import Protocol, Sequence
from .models import (
LocationCandidate,
LocationQuery,
ResolutionDiagnostic,
ResolutionResult,
ResolverOutput,
)
class LocationResolver(Protocol):
"""Pluggable location resolution step.
Implementations: ``SourceCoordinatesResolver``, ``RegistryResolver``,
``NominatimResolver``, ``InheritFromAnotherEntityResolver`` — see the
``resolvers`` subpackage. New algorithms (peeringdb / IXP / user-confirmed
coordinates) plug in by implementing this protocol; the pipeline does not
care how candidates are produced.
"""
name: str
def resolve(self, query: LocationQuery) -> ResolverOutput: ...
def default_candidate_sort_key(
candidate: LocationCandidate,
) -> tuple[int, int, float]:
precision_rank = {"precise": 0, "site": 1, "city": 2}.get(
candidate.precision, 9
)
source_rank = {
"source_coordinates": 0,
"stored_compute_center_location": 1,
"stored_collector_location": 1,
"ror_organization_registry": 2,
"inherited": 3,
"nominatim_online_geocode": 4,
"local_registry": 8,
"local_registry_city": 9,
}.get(candidate.source, 9)
return (source_rank, precision_rank, -float(candidate.confidence or 0))
class LocationPipeline:
"""Orchestrate a sequence of resolvers.
``collect_candidates`` runs every resolver and returns *all* deduped
candidates plus the queries each resolver attempted (useful for
user-facing "why didn't this work?" diagnostics).
``resolve_best`` returns the top candidate per
:func:`default_candidate_sort_key` (or a custom sort).
"""
def __init__(
self,
resolvers: Sequence[LocationResolver],
*,
sort_key=default_candidate_sort_key,
failure_reason: str = (
"Could not resolve to renderable coordinates from any configured resolver."
),
) -> None:
self._resolvers = list(resolvers)
self._sort_key = sort_key
self._failure_reason = failure_reason
@property
def resolvers(self) -> tuple[LocationResolver, ...]:
return tuple(self._resolvers)
def collect_candidates(
self, query: LocationQuery
) -> tuple[list[LocationCandidate], list[str]]:
candidates: list[LocationCandidate] = []
attempted: list[str] = []
seen_keys: set[tuple[str, str, str]] = set()
for resolver in self._resolvers:
output = resolver.resolve(query)
for q in output.attempted_queries:
if q and q not in attempted:
attempted.append(q)
for candidate in output.candidates:
key = (
candidate.source,
f"{candidate.latitude:.4f}",
f"{candidate.longitude:.4f}",
)
if key in seen_keys:
continue
seen_keys.add(key)
candidates.append(candidate)
candidates.sort(key=self._sort_key)
return candidates, attempted
def resolve_best(self, query: LocationQuery) -> ResolutionResult:
candidates, attempted = self.collect_candidates(query)
if candidates:
return ResolutionResult(
location=candidates[0],
diagnostic=None,
attempted_queries=tuple(attempted),
)
return ResolutionResult(
location=None,
diagnostic=ResolutionDiagnostic(
failure_reason=self._failure_reason,
attempted_queries=tuple(attempted),
name=query.name,
country=query.country,
city=query.city,
site=str(query.extra.get("site")) if query.extra.get("site") else None,
operator=str(query.extra.get("operator"))
if query.extra.get("operator")
else None,
),
attempted_queries=tuple(attempted),
)

20
backend/app/services/location/resolvers/__init__.py Normal file
View File

@@ -0,0 +1,20 @@
"""Built-in resolver implementations."""
from .inherit import InheritFromAnotherEntityResolver
from .nominatim import (
NominatimResolver,
build_default_nominatim_geocoder,
interpret_geocode_result,
)
from .registry import RegistryResolver, default_score_alias_match
from .source_coordinates import SourceCoordinatesResolver
__all__ = [
"InheritFromAnotherEntityResolver",
"NominatimResolver",
"RegistryResolver",
"SourceCoordinatesResolver",
"build_default_nominatim_geocoder",
"default_score_alias_match",
"interpret_geocode_result",
]

31
backend/app/services/location/resolvers/inherit.py Normal file
View File

@@ -0,0 +1,31 @@
"""Resolver that inherits a candidate from another entity's resolution.
Used by BGP events to pick up the location of their owning collector. The
``source_lookup`` callable is the only domain coupling — it receives the
incoming :class:`LocationQuery` and returns either an already-resolved
:class:`LocationCandidate` (typically by querying another pipeline) or
``None`` to signal "no parent location available".
"""
from __future__ import annotations
from typing import Callable
from ..models import LocationCandidate, LocationQuery, ResolverOutput
class InheritFromAnotherEntityResolver:
def __init__(
self,
*,
source_lookup: Callable[[LocationQuery], LocationCandidate | None],
name: str = "inherited",
) -> None:
self.name = name
self._lookup = source_lookup
def resolve(self, query: LocationQuery) -> ResolverOutput:
result = self._lookup(query)
if result is None:
return ResolverOutput()
return ResolverOutput(candidates=(result,))

292
backend/app/services/location/resolvers/nominatim.py Normal file
View File

@@ -0,0 +1,292 @@
"""Nominatim-backed online geocoder.
The actual HTTP call is encapsulated in :func:`build_default_nominatim_geocoder`
which returns an ``lru_cache``-wrapped function. Domain modules typically:
1. Build a default geocoder via :func:`build_default_nominatim_geocoder`.
2. Re-export it under a stable module-level name (e.g. ``_geocode_online``).
3. Pass a *late-binding lambda* (``lambda q: _geocode_online(q)``) to
:class:`NominatimResolver`.
This ensures tests that ``monkeypatch.setattr(module, "_geocode_online", ...)``
can swap the geocoder behavior without touching pipeline construction.
"""
from __future__ import annotations
import time
from functools import lru_cache
from typing import Any, Callable
import httpx
from ..models import LocationCandidate, LocationQuery, ResolverOutput
from ..text import (
coerce_str,
normalize_country_text,
normalize_text,
parse_float,
)
NOMINATIM_SEARCH_URL = "https://nominatim.openstreetmap.org/search"
DEFAULT_USER_AGENT = "planet-earth-location-resolver/1.0"
DEFAULT_MIN_INTERVAL_SECONDS = 1.1
DEFAULT_TIMEOUT_SECONDS = 8.0
def build_default_nominatim_geocoder(
*,
user_agent: str = DEFAULT_USER_AGENT,
min_interval_seconds: float = DEFAULT_MIN_INTERVAL_SECONDS,
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
cache_size: int = 512,
) -> Callable[[str], dict[str, Any] | None]:
"""Return a cached, rate-limited Nominatim geocoder."""
last_request_at = [0.0]
@lru_cache(maxsize=cache_size)
def geocode(query: str) -> dict[str, Any] | None:
if not query:
return None
elapsed = time.monotonic() - last_request_at[0]
if elapsed < min_interval_seconds:
time.sleep(min_interval_seconds - elapsed)
last_request_at[0] = time.monotonic()
response = httpx.get(
NOMINATIM_SEARCH_URL,
params={
"q": query,
"format": "jsonv2",
"limit": 1,
"addressdetails": 1,
},
headers={"User-Agent": user_agent},
timeout=timeout_seconds,
)
response.raise_for_status()
payload = response.json()
if not isinstance(payload, list) or not payload:
return None
result = payload[0]
if not isinstance(result, dict):
return None
return result
return geocode
_DEFAULT_SITE_CATEGORIES = frozenset(
{
"amenity",
"office",
"building",
"industrial",
"research",
"university",
"education",
"tourism",
"shop",
"man_made",
"campus",
"research_institute",
}
)
def interpret_geocode_result(
result: dict[str, Any],
*,
matched_fields: tuple[str, ...],
context_country: str | None,
site_categories: frozenset[str] = _DEFAULT_SITE_CATEGORIES,
site_promoting_match_fields: frozenset[str] = frozenset(
{"site", "operator", "name"}
),
) -> tuple[float, float, dict[str, Any], str] | None:
"""Validate a Nominatim raw result. Returns (lat, lon, address, classification)."""
latitude = parse_float(result.get("lat"))
longitude = parse_float(result.get("lon"))
if latitude in (None, 0.0) or longitude in (None, 0.0):
return None
address = result.get("address") if isinstance(result.get("address"), dict) else {}
if not isinstance(address, dict):
address = {}
has_city_level = bool(
address.get("city")
or address.get("town")
or address.get("village")
or address.get("municipality")
or address.get("hamlet")
or address.get("suburb")
)
osm_class = str(result.get("class") or "").lower()
osm_type = str(result.get("type") or "").lower()
is_site_like = osm_class in site_categories or osm_type in site_categories
if not has_city_level and not is_site_like:
return None
if context_country:
normalized_context = normalize_text(normalize_country_text(context_country))
normalized_result = normalize_text(
normalize_country_text(address.get("country"))
)
if (
normalized_context
and normalized_result
and normalized_context != normalized_result
):
return None
classification = (
"site"
if (
is_site_like
and has_city_level
and any(field in site_promoting_match_fields for field in matched_fields)
)
else "city"
)
return float(latitude), float(longitude), address, classification
def _candidate_from_geocode(
*,
query: LocationQuery,
geocode_query: str,
matched_fields: tuple[str, ...],
raw_result: dict[str, Any],
interpret: Callable[..., tuple[float, float, dict[str, Any], str] | None],
source: str,
site_confidence: float,
city_confidence: float,
) -> LocationCandidate | None:
interpreted = interpret(
raw_result,
matched_fields=matched_fields,
context_country=query.country,
)
if not interpreted:
return None
latitude, longitude, address, classification = interpreted
city = (
address.get("city")
or address.get("town")
or address.get("village")
or address.get("municipality")
or query.city
or None
)
region = address.get("state") or address.get("region")
country = address.get("country") or query.country or None
display_name = raw_result.get("display_name") or geocode_query
confidence = city_confidence if classification == "city" else site_confidence
extra = query.extra or {}
suggested_registry_entry = {
"canonical_name": (
(query.aliases[0] if query.aliases else None)
or query.name
or display_name
),
"aliases": list(
{
value
for value in [
query.name,
*query.aliases,
coerce_str(extra.get("operator")),
coerce_str(extra.get("site")),
]
if value
}
),
"operator": coerce_str(extra.get("operator")) or None,
"site": coerce_str(extra.get("site"))
or coerce_str(extra.get("organization"))
or None,
"country": country,
"city": city,
"region": region,
"latitude": latitude,
"longitude": longitude,
"precision": classification,
"confidence": confidence,
"source_note": (
f"Resolved via Nominatim query '{geocode_query}' → {display_name}"
),
}
return LocationCandidate(
latitude=latitude,
longitude=longitude,
display_name=display_name,
precision=classification,
confidence=confidence,
query=geocode_query,
source=source,
source_note=f"Nominatim search result: {display_name}",
matched_fields=matched_fields,
needs_confirmation=True,
city=city,
region=region,
country=country,
matched_location_name=display_name,
location_verified_at=None,
suggested_registry_entry=suggested_registry_entry,
)
class NominatimResolver:
"""Run a domain-specific query plan against Nominatim."""
def __init__(
self,
*,
query_plan_builder: Callable[
[LocationQuery], list[tuple[str, tuple[str, ...]]]
],
geocoder: Callable[[str], dict[str, Any] | None],
name: str = "nominatim_online_geocode",
site_confidence: float = 0.72,
city_confidence: float = 0.62,
interpret: Callable[..., tuple[float, float, dict[str, Any], str] | None] = (
interpret_geocode_result
),
) -> None:
self.name = name
self._query_plan_builder = query_plan_builder
self._geocoder = geocoder
self._site_confidence = site_confidence
self._city_confidence = city_confidence
self._interpret = interpret
def resolve(self, query: LocationQuery) -> ResolverOutput:
plan = self._query_plan_builder(query)
candidates: list[LocationCandidate] = []
attempted: list[str] = []
for geocode_query, matched_fields in plan:
attempted.append(geocode_query)
try:
raw_result = self._geocoder(geocode_query)
except Exception:
continue
if not raw_result:
continue
candidate = _candidate_from_geocode(
query=query,
geocode_query=geocode_query,
matched_fields=matched_fields,
raw_result=raw_result,
interpret=self._interpret,
source=self.name,
site_confidence=self._site_confidence,
city_confidence=self._city_confidence,
)
if candidate is not None:
candidates.append(candidate)
return ResolverOutput(
candidates=tuple(candidates),
attempted_queries=tuple(attempted),
)

323
backend/app/services/location/resolvers/registry.py Normal file
View File

@@ -0,0 +1,323 @@
"""Resolver that matches a query against a local JSON registry.
Registry schema (a single JSON file):
{
"locations": [
{
"canonical_name": "...",
"aliases": ["...", "..."],
"operator": "...",
"site": "...",
"city": "...",
"country": "...",
"region": "...",
"latitude": 0.0,
"longitude": 0.0,
"precision": "precise" | "site" | "city",
"confidence": 0.0,
"verification_status": "verified",
"source_note": "...",
"verified_at": "YYYY-MM-DD"
}
],
"city_fallbacks": [ {city, country, latitude, longitude, ...} ]
}
"""
from __future__ import annotations
import json
from functools import lru_cache
from pathlib import Path
from typing import Any, Callable, Iterable
from ..models import (
RENDERABLE_PRECISIONS,
LocationCandidate,
LocationQuery,
ResolverOutput,
)
from ..text import (
city_key,
normalize_country_text,
normalize_text,
parse_float,
)
# Field-priority weights when scoring "this query field text contains this
# alias text". Tuned to match the legacy compute-center ordering — name beats
# site beats operator beats city — which generalizes well to other domains.
_DEFAULT_FIELD_PRIORITY = {
"name": 8,
"site": 6,
"operator": 5,
"city": 3,
}
def default_score_alias_match(
alias_field: str, record_field: str, alias_text: str
) -> int:
score = max(0, len(alias_text))
score += _DEFAULT_FIELD_PRIORITY.get(alias_field, 1)
if alias_field == record_field:
score += 4
if alias_field == "name" and record_field in {"name", "name_short", "alias"}:
score += 6
if alias_field == "site" and record_field in {"site", "organization"}:
score += 4
if alias_field == "operator" and record_field in {"operator", "organization"}:
score += 4
return score
@lru_cache(maxsize=32)
def _load_registry_file(path: str) -> dict[str, Any]:
with Path(path).open("r", encoding="utf-8") as handle:
return json.load(handle)
@lru_cache(maxsize=32)
def _build_alias_index(
path: str,
) -> tuple[tuple[dict[str, Any], tuple[tuple[str, str], ...]], ...]:
index: list[tuple[dict[str, Any], tuple[tuple[str, str], ...]]] = []
for entry in _load_registry_file(path).get("locations", []):
aliases: list[tuple[str, str]] = []
seen: set[str] = set()
for alias in [entry.get("canonical_name"), *(entry.get("aliases") or [])]:
normalized = normalize_text(alias)
if normalized and normalized not in seen:
aliases.append(("name", normalized))
seen.add(normalized)
for field_name in ("operator", "site", "city"):
value = entry.get(field_name)
normalized = normalize_text(value)
if normalized and normalized not in seen:
aliases.append((field_name, normalized))
seen.add(normalized)
index.append((entry, tuple(aliases)))
return tuple(index)
def _query_corpus(query: LocationQuery) -> dict[str, str]:
"""Map a query into normalized strings keyed by source field."""
fields: dict[str, str] = {
"name": query.name or "",
"city": query.city or "",
"country": query.country or "",
}
for alias in query.aliases:
if alias and alias != query.name:
fields["name_short"] = alias
break
extra = query.extra or {}
for key in ("site", "operator", "organization"):
value = extra.get(key)
if value:
fields[key] = str(value)
return {key: normalize_text(value) for key, value in fields.items() if value}
def _country_compatible(entry: dict[str, Any], query: LocationQuery) -> bool:
record_country = normalize_country_text(query.country)
entry_country = normalize_country_text(entry.get("country"))
if not record_country or not entry_country:
return True
return normalize_text(record_country) == normalize_text(entry_country)
def _normalized_alias_matches(alias_normalized: str, record_text: str) -> bool:
alias_tokens = alias_normalized.split()
record_tokens = record_text.split()
if not alias_tokens or not record_tokens:
return False
if len(alias_tokens) == 1:
return alias_tokens[0] in record_tokens
window_size = len(alias_tokens)
return any(
record_tokens[index : index + window_size] == alias_tokens
for index in range(0, len(record_tokens) - window_size + 1)
)
def _entry_to_candidate(
entry: dict[str, Any],
*,
matched_alias: str,
matched_fields: Iterable[str],
source: str,
score_explainer: str,
confidence_floor: float,
) -> LocationCandidate:
canonical_name = entry.get("canonical_name") or matched_alias
# Registry entries are treated as candidates unless explicitly verified.
# This prevents migrated hard-coded hints from appearing as factual
# location evidence.
is_verified = entry.get("verification_status") == "verified"
precision = entry.get("precision") or "city"
if precision not in RENDERABLE_PRECISIONS:
precision = "city"
fields_summary = ", ".join(sorted(set(matched_fields))) or "name"
confidence_value = parse_float(entry.get("confidence"))
confidence = (
float(confidence_value)
if confidence_value is not None
else confidence_floor
)
return LocationCandidate(
latitude=float(parse_float(entry.get("latitude")) or 0.0),
longitude=float(parse_float(entry.get("longitude")) or 0.0),
display_name=canonical_name,
precision=precision,
confidence=confidence,
query=f"local_registry::{matched_alias or canonical_name}",
source=source,
source_note=entry.get("source_note")
or f"{score_explainer}: matched {fields_summary}",
matched_fields=tuple(sorted(set(matched_fields))) or ("name",),
needs_confirmation=bool(entry.get("needs_confirmation")) or not is_verified,
city=entry.get("city"),
region=entry.get("region"),
country=entry.get("country"),
matched_location_name=canonical_name,
location_verified_at=entry.get("verified_at") if is_verified else None,
suggested_registry_entry=None,
)
class RegistryResolver:
"""Match a query against a JSON registry (plus its city_fallbacks table)."""
def __init__(
self,
*,
registry_path: Path | str,
name: str = "local_registry",
city_fallback_source: str = "local_registry_city",
city_fallback_confidence_default: float = 0.65,
confidence_default: float = 0.85,
score_alias_match: Callable[[str, str, str], int] = default_score_alias_match,
) -> None:
self.name = name
self._registry_path = str(Path(registry_path))
self._city_fallback_source = city_fallback_source
self._city_fallback_confidence_default = city_fallback_confidence_default
self._confidence_default = confidence_default
self._score = score_alias_match
def reload(self) -> None:
"""Drop the cached registry — useful when the JSON file is edited."""
_load_registry_file.cache_clear()
_build_alias_index.cache_clear()
def resolve(self, query: LocationQuery) -> ResolverOutput:
candidates: list[LocationCandidate] = []
candidates.extend(self._registry_candidates(query))
city_candidate = self._city_fallback_candidate(query)
if city_candidate is not None:
candidates.append(city_candidate)
return ResolverOutput(candidates=tuple(candidates))
# ── internals ──────────────────────────────────────────────
def _registry_candidates(
self, query: LocationQuery
) -> list[LocationCandidate]:
corpus = _query_corpus(query)
if not corpus:
return []
# When the query carries a name (a record-specific identifier), require
# at least one alias match against a name-class field — otherwise a
# generic shared field like operator="RIPE NCC" would promote every
# registry entry that lists that operator, regardless of whether the
# name matches.
query_has_name = bool(corpus.get("name") or corpus.get("name_short"))
results: list[LocationCandidate] = []
for entry, aliases in _build_alias_index(self._registry_path):
best_alias = ""
best_score = 0
matched_fields: list[str] = []
matched_via_name_alias = False
for alias_field, alias_normalized in aliases:
for record_field, record_text in corpus.items():
if not _normalized_alias_matches(alias_normalized, record_text):
continue
score = self._score(
alias_field, record_field, alias_normalized
)
if score > best_score or (
score == best_score
and len(alias_normalized) > len(best_alias)
):
best_score = score
best_alias = alias_normalized
if record_field not in matched_fields:
matched_fields.append(record_field)
if alias_field == "name" and record_field in {"name", "name_short"}:
matched_via_name_alias = True
if not matched_fields or best_score <= 0:
continue
if query_has_name and not matched_via_name_alias:
continue
if not _country_compatible(entry, query):
continue
results.append(
_entry_to_candidate(
entry,
matched_alias=best_alias,
matched_fields=matched_fields,
source=self.name,
score_explainer="Registry alias match",
confidence_floor=self._confidence_default,
)
)
return results
def _city_fallback_candidate(
self, query: LocationQuery
) -> LocationCandidate | None:
country = normalize_country_text(query.country)
city = city_key(query.city)
if not country or not city:
return None
for fallback in _load_registry_file(self._registry_path).get(
"city_fallbacks", []
):
fallback_country = normalize_country_text(fallback.get("country"))
fallback_city = city_key(fallback.get("city"))
if fallback_country != country or fallback_city != city:
continue
confidence_value = parse_float(fallback.get("confidence"))
confidence = (
float(confidence_value)
if confidence_value is not None
else self._city_fallback_confidence_default
)
return LocationCandidate(
latitude=float(parse_float(fallback.get("latitude")) or 0.0),
longitude=float(parse_float(fallback.get("longitude")) or 0.0),
display_name=fallback.get("city") or "",
precision="city",
confidence=confidence,
query=(
f"city_fallback::{fallback.get('city')}, "
f"{fallback.get('country')}"
),
source=self._city_fallback_source,
source_note=fallback.get("source_note")
or f"City fallback for {fallback.get('city')}, {fallback.get('country')}",
matched_fields=("city", "country"),
needs_confirmation=False,
city=fallback.get("city"),
region=fallback.get("region"),
country=fallback.get("country"),
matched_location_name=fallback.get("city"),
location_verified_at=fallback.get("verified_at"),
suggested_registry_entry=None,
)
return None

42
backend/app/services/location/resolvers/source_coordinates.py Normal file
View File

@@ -0,0 +1,42 @@
"""Resolver that consumes lat/lon already present on the source record."""
from __future__ import annotations
from ..models import LocationCandidate, LocationQuery, ResolverOutput
from ..text import normalize_country_text
class SourceCoordinatesResolver:
"""Pass-through for records that already carry valid coordinates."""
name = "source_coordinates"
def __init__(self, *, source: str = "source_coordinates") -> None:
self._source = source
def resolve(self, query: LocationQuery) -> ResolverOutput:
lat = query.source_latitude
lon = query.source_longitude
if lat in (None, 0.0) or lon in (None, 0.0):
return ResolverOutput()
country = normalize_country_text(query.country) or query.country
candidate = LocationCandidate(
latitude=float(lat),
longitude=float(lon),
display_name=query.name or "",
precision="precise",
confidence=1.0,
query="source_coordinates",
source=self._source,
source_note="Source record provided valid coordinates.",
matched_fields=("source_coordinates",),
needs_confirmation=False,
city=query.city,
region=query.region,
country=country,
matched_location_name=query.name,
location_verified_at=None,
suggested_registry_entry=None,
)
return ResolverOutput(candidates=(candidate,))

41
backend/app/services/location/text.py Normal file
View File

@@ -0,0 +1,41 @@
"""Text-normalization helpers shared by every resolver."""
from __future__ import annotations
import re
from typing import Any
from app.core.countries import normalize_country
def parse_float(value: Any) -> float | None:
try:
if value in (None, ""):
return None
return float(value)
except (TypeError, ValueError):
return None
def coerce_str(value: Any) -> str:
if value in (None, ""):
return ""
return str(value).strip()
def normalize_text(value: Any) -> str:
if value in (None, ""):
return ""
normalized = str(value).casefold()
normalized = re.sub(r"[^a-z0-9一-鿿]+", " ", normalized)
return re.sub(r"\s+", " ", normalized).strip()
def normalize_country_text(value: Any) -> str:
normalized = normalize_country(value)
return normalized or coerce_str(value)
def city_key(city: Any) -> str:
text = coerce_str(city).split(",", 1)[0]
return normalize_text(text)