release: bump version to 0.49.0
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
linkong
306
backend/app/services/bgp_collector_locations.py
Normal file
306
backend/app/services/bgp_collector_locations.py
Normal file
@@ -0,0 +1,306 @@
|
||||
"""BGP route-collector location resolver.
|
||||
|
||||
Collector positions are stored in the ``bgp_collector_locations`` database
|
||||
table. The old JSON registry is now only a seed payload used during database
|
||||
initialization, not a runtime resolver or candidate source.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.bgp_collector_location import BGPCollectorLocation
|
||||
from app.services.location import (
|
||||
LocationCandidate,
|
||||
LocationPipeline,
|
||||
LocationQuery,
|
||||
NominatimResolver,
|
||||
ResolutionResult,
|
||||
ResolverOutput,
|
||||
SourceCoordinatesResolver,
|
||||
build_default_nominatim_geocoder,
|
||||
coerce_str,
|
||||
normalize_text,
|
||||
)
|
||||
|
||||
SEED_PATH = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "data"
|
||||
/ "seeds"
|
||||
/ "ripe_ris_collector_locations_seed.json"
|
||||
)
|
||||
|
||||
# ── Geocoder (kept at module level for monkeypatching + cache_clear) ──
|
||||
|
||||
_geocode_online = build_default_nominatim_geocoder()
|
||||
|
||||
|
||||
# ── In-process compatibility cache ──────────────────────────────────
|
||||
|
||||
|
||||
RIPE_RIS_COLLECTOR_COORDS: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _collector_record_to_dict(record: BGPCollectorLocation) -> dict[str, Any]:
|
||||
return record.to_location_dict()
|
||||
|
||||
|
||||
def set_bgp_collector_location_cache(
|
||||
locations: dict[str, dict[str, Any]],
|
||||
) -> None:
|
||||
"""Replace the legacy compatibility cache in-place."""
|
||||
RIPE_RIS_COLLECTOR_COORDS.clear()
|
||||
RIPE_RIS_COLLECTOR_COORDS.update(
|
||||
{coerce_str(key): dict(value) for key, value in locations.items()}
|
||||
)
|
||||
|
||||
|
||||
async def refresh_bgp_collector_location_cache(
|
||||
session: AsyncSession,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
result = await session.execute(select(BGPCollectorLocation))
|
||||
records = result.scalars().all()
|
||||
cache = {
|
||||
record.collector_id: _collector_record_to_dict(record)
|
||||
for record in records
|
||||
if record.collector_id
|
||||
}
|
||||
set_bgp_collector_location_cache(cache)
|
||||
return cache
|
||||
|
||||
|
||||
def _load_seed_payload() -> dict[str, Any]:
|
||||
with SEED_PATH.open("r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def _seed_entry_to_record_kwargs(entry: dict[str, Any], collector_id: str) -> dict[str, Any]:
|
||||
return {
|
||||
"collector_id": collector_id,
|
||||
"operator": entry.get("operator") or "RIPE NCC",
|
||||
"site": entry.get("site"),
|
||||
"city": entry.get("city"),
|
||||
"country": entry.get("country"),
|
||||
"latitude": entry.get("latitude"),
|
||||
"longitude": entry.get("longitude"),
|
||||
"precision": entry.get("precision") or "city",
|
||||
"confidence": entry.get("confidence"),
|
||||
"source": "legacy_seed",
|
||||
"source_url": None,
|
||||
"source_note": entry.get("source_note")
|
||||
or "Seeded from legacy RIPE RIS collector coordinates",
|
||||
"raw_payload": entry,
|
||||
"needs_confirmation": True,
|
||||
"verification_status": "unverified",
|
||||
"verified_at": None,
|
||||
}
|
||||
|
||||
|
||||
async def seed_default_bgp_collector_locations(session: AsyncSession) -> None:
|
||||
"""Seed default RIPE RIS collector locations without overwriting users."""
|
||||
payload = _load_seed_payload()
|
||||
for entry in payload.get("locations", []):
|
||||
aliases = entry.get("aliases") or []
|
||||
collector_ids = [
|
||||
coerce_str(alias)
|
||||
for alias in aliases
|
||||
if coerce_str(alias).startswith("rrc")
|
||||
]
|
||||
if not collector_ids:
|
||||
continue
|
||||
collector_id = collector_ids[0]
|
||||
existing = await session.scalar(
|
||||
select(BGPCollectorLocation).where(
|
||||
BGPCollectorLocation.collector_id == collector_id
|
||||
)
|
||||
)
|
||||
if existing:
|
||||
continue
|
||||
session.add(
|
||||
BGPCollectorLocation(
|
||||
**_seed_entry_to_record_kwargs(entry, collector_id)
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
await refresh_bgp_collector_location_cache(session)
|
||||
|
||||
|
||||
def get_bgp_collector_location_dict(collector_name: str) -> dict[str, Any]:
|
||||
"""Return the current cached collector location dict, or ``{}`` if unknown."""
|
||||
return dict(RIPE_RIS_COLLECTOR_COORDS.get(coerce_str(collector_name), {}))
|
||||
|
||||
|
||||
def iter_known_collector_names() -> Iterator[str]:
|
||||
"""Yield every collector technical name (rrcXX) known in the cache."""
|
||||
return iter(sorted(RIPE_RIS_COLLECTOR_COORDS.keys()))
|
||||
|
||||
|
||||
# ── Pipeline construction ──────────────────────────────────────────
|
||||
|
||||
|
||||
class StoredCollectorLocationResolver:
|
||||
"""Resolve a collector through the DB-backed compatibility cache."""
|
||||
|
||||
name = "stored_collector_location"
|
||||
|
||||
def resolve(self, query: LocationQuery) -> ResolverOutput:
|
||||
collector = coerce_str(query.name)
|
||||
if not collector:
|
||||
for alias in query.aliases:
|
||||
collector = coerce_str(alias)
|
||||
if collector:
|
||||
break
|
||||
if not collector:
|
||||
return ResolverOutput()
|
||||
location = get_bgp_collector_location_dict(collector)
|
||||
if not location:
|
||||
return ResolverOutput()
|
||||
latitude = location.get("latitude")
|
||||
longitude = location.get("longitude")
|
||||
if latitude in (None, 0.0) or longitude in (None, 0.0):
|
||||
return ResolverOutput()
|
||||
return ResolverOutput(
|
||||
candidates=(
|
||||
LocationCandidate(
|
||||
latitude=float(latitude),
|
||||
longitude=float(longitude),
|
||||
display_name=location.get("matched_location_name") or collector,
|
||||
precision=location.get("precision") or "city",
|
||||
confidence=float(location.get("confidence") or 0.85),
|
||||
query=f"stored_collector_location::{collector}",
|
||||
source=location.get("source") or self.name,
|
||||
source_note=location.get("source_note"),
|
||||
matched_fields=("collector",),
|
||||
needs_confirmation=bool(location.get("needs_confirmation")),
|
||||
city=location.get("city"),
|
||||
region=None,
|
||||
country=location.get("country"),
|
||||
matched_location_name=(
|
||||
location.get("matched_location_name") or collector
|
||||
),
|
||||
location_verified_at=location.get("verified_at"),
|
||||
suggested_registry_entry=None,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _bgp_collector_query_plan(
|
||||
query: LocationQuery,
|
||||
) -> list[tuple[str, tuple[str, ...]]]:
|
||||
"""Build the Nominatim query plan for a BGP collector."""
|
||||
extra = query.extra or {}
|
||||
site = str(extra.get("site") or "")
|
||||
operator = str(extra.get("operator") or "")
|
||||
city = query.city or ""
|
||||
country = query.country or ""
|
||||
|
||||
plan: list[tuple[str, tuple[str, ...]]] = []
|
||||
|
||||
def add(parts: list[tuple[str, str]]) -> None:
|
||||
non_empty = [(field, value) for field, value in parts if value]
|
||||
if not non_empty:
|
||||
return
|
||||
seen: set[str] = set()
|
||||
cleaned: list[str] = []
|
||||
fields: list[str] = []
|
||||
for field, value in non_empty:
|
||||
key = normalize_text(value)
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
cleaned.append(value)
|
||||
fields.append(field)
|
||||
if not cleaned:
|
||||
return
|
||||
composed = ", ".join(cleaned)
|
||||
if not any(composed == existing for existing, _ in plan):
|
||||
plan.append((composed, tuple(fields)))
|
||||
|
||||
add([("site", site), ("city", city), ("country", country)])
|
||||
add([("site", site), ("country", country)])
|
||||
add([("operator", operator), ("city", city), ("country", country)])
|
||||
add([("city", city), ("country", country)])
|
||||
return plan
|
||||
|
||||
|
||||
BGP_COLLECTOR_PIPELINE = LocationPipeline(
|
||||
[
|
||||
SourceCoordinatesResolver(),
|
||||
StoredCollectorLocationResolver(),
|
||||
],
|
||||
failure_reason=(
|
||||
"Could not resolve BGP collector to renderable coordinates from"
|
||||
" source coordinates or stored collector location."
|
||||
),
|
||||
)
|
||||
|
||||
BGP_COLLECTOR_COLLECTION_PIPELINE = LocationPipeline(
|
||||
[
|
||||
SourceCoordinatesResolver(),
|
||||
NominatimResolver(
|
||||
query_plan_builder=_bgp_collector_query_plan,
|
||||
# Late-binding so tests can monkeypatch ``_geocode_online``.
|
||||
geocoder=lambda q: _geocode_online(q),
|
||||
),
|
||||
],
|
||||
failure_reason=(
|
||||
"Could not resolve BGP collector to renderable coordinates from"
|
||||
" source coordinates or online geocoding."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ── Public API ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def resolve_bgp_collector_location(
|
||||
collector_name: str,
|
||||
*,
|
||||
city: str | None = None,
|
||||
country: str | None = None,
|
||||
site: str | None = None,
|
||||
operator: str | None = None,
|
||||
) -> ResolutionResult:
|
||||
"""Resolve a BGP collector to its best-known stored location."""
|
||||
stored = get_bgp_collector_location_dict(collector_name)
|
||||
name = coerce_str(collector_name) or None
|
||||
query = LocationQuery(
|
||||
name=name,
|
||||
aliases=tuple(filter(None, (collector_name,))),
|
||||
city=coerce_str(city or stored.get("city")) or None,
|
||||
country=coerce_str(country or stored.get("country")) or None,
|
||||
extra={
|
||||
"site": coerce_str(site or stored.get("site")),
|
||||
"operator": coerce_str(operator or stored.get("operator")) or "RIPE NCC",
|
||||
},
|
||||
)
|
||||
return BGP_COLLECTOR_PIPELINE.resolve_best(query)
|
||||
|
||||
|
||||
def collect_bgp_collector_location_candidates(
|
||||
*,
|
||||
collector: str | None = None,
|
||||
city: str | None = None,
|
||||
country: str | None = None,
|
||||
site: str | None = None,
|
||||
operator: str | None = None,
|
||||
) -> tuple[list[LocationCandidate], list[str]]:
|
||||
stored = get_bgp_collector_location_dict(collector or "")
|
||||
name = coerce_str(collector) or None
|
||||
query = LocationQuery(
|
||||
name=name,
|
||||
aliases=tuple(filter(None, (collector,))),
|
||||
city=coerce_str(city or stored.get("city")) or None,
|
||||
country=coerce_str(country or stored.get("country")) or None,
|
||||
extra={
|
||||
"site": coerce_str(site or stored.get("site")),
|
||||
"operator": coerce_str(operator or stored.get("operator")) or "RIPE NCC",
|
||||
},
|
||||
)
|
||||
return BGP_COLLECTOR_COLLECTION_PIPELINE.collect_candidates(query)
|
||||
155
backend/app/services/bgp_event_locations.py
Normal file
155
backend/app/services/bgp_event_locations.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""BGP event location resolver.
|
||||
|
||||
A BGP event (announcement / withdrawal / RIB entry) is geographically tied to
|
||||
the route collector that observed it. This module defines the pipeline that
|
||||
turns an event payload into renderable coordinates.
|
||||
|
||||
Current resolver chain:
|
||||
|
||||
SourceCoordinates → event payload itself carries lat/lon (rare; some
|
||||
enriched feeds do).
|
||||
InheritFromCollector → look up the owning collector via
|
||||
:func:`resolve_bgp_collector_location`.
|
||||
|
||||
Future plug-ins (no consumer changes required, just append to the list):
|
||||
|
||||
ASNFacilityResolver — origin/peer ASN → peeringdb facility.
|
||||
PrefixGeoResolver — prefix → IP range geo lookup (iptoasn / opengeofeed).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.services.bgp_collector_locations import (
|
||||
get_bgp_collector_location_dict,
|
||||
)
|
||||
from app.services.location import (
|
||||
InheritFromAnotherEntityResolver,
|
||||
LocationCandidate,
|
||||
LocationPipeline,
|
||||
LocationQuery,
|
||||
ResolutionResult,
|
||||
SourceCoordinatesResolver,
|
||||
coerce_str,
|
||||
)
|
||||
|
||||
|
||||
def _inherit_from_owning_collector(
|
||||
query: LocationQuery,
|
||||
) -> LocationCandidate | None:
|
||||
"""Look up the event's owning collector by exact name in the DB-backed cache."""
|
||||
extra = query.extra or {}
|
||||
collector_name = coerce_str(extra.get("collector"))
|
||||
if not collector_name:
|
||||
return None
|
||||
legacy = get_bgp_collector_location_dict(collector_name)
|
||||
if not legacy:
|
||||
return None
|
||||
latitude = legacy.get("latitude")
|
||||
longitude = legacy.get("longitude")
|
||||
if latitude in (None, 0.0) or longitude in (None, 0.0):
|
||||
return None
|
||||
return LocationCandidate(
|
||||
latitude=float(latitude),
|
||||
longitude=float(longitude),
|
||||
display_name=legacy.get("matched_location_name") or collector_name,
|
||||
precision=legacy.get("precision") or "city",
|
||||
confidence=float(legacy.get("confidence") or 0.85),
|
||||
query=f"inherit_from_collector::{collector_name}",
|
||||
source="inherited_from_collector",
|
||||
source_note=(
|
||||
f"Inherited from owning collector {collector_name}"
|
||||
),
|
||||
matched_fields=("collector",),
|
||||
needs_confirmation=bool(legacy.get("needs_confirmation")),
|
||||
city=legacy.get("city"),
|
||||
region=None,
|
||||
country=legacy.get("country"),
|
||||
matched_location_name=legacy.get("matched_location_name"),
|
||||
location_verified_at=legacy.get("verified_at"),
|
||||
suggested_registry_entry=None,
|
||||
)
|
||||
|
||||
|
||||
BGP_EVENT_PIPELINE = LocationPipeline(
|
||||
[
|
||||
SourceCoordinatesResolver(),
|
||||
InheritFromAnotherEntityResolver(
|
||||
source_lookup=_inherit_from_owning_collector,
|
||||
name="inherited_from_collector",
|
||||
),
|
||||
# Plug new resolvers (peeringdb / ASN facility / prefix-geo) here.
|
||||
],
|
||||
failure_reason=(
|
||||
"Could not resolve BGP event coordinates: no source coords, owning"
|
||||
" collector unknown, and no fallback resolver matched."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def resolve_bgp_event_location(
|
||||
*,
|
||||
collector: str,
|
||||
source_latitude: float | None = None,
|
||||
source_longitude: float | None = None,
|
||||
site: str | None = None,
|
||||
operator: str | None = None,
|
||||
peer_asn: int | None = None,
|
||||
origin_asn: int | None = None,
|
||||
prefix: str | None = None,
|
||||
) -> ResolutionResult:
|
||||
"""Resolve a BGP event to its renderable coordinates.
|
||||
|
||||
The ``peer_asn`` / ``origin_asn`` / ``prefix`` arguments are accepted
|
||||
today so future resolvers (ASN→facility, prefix→geo) can consume them
|
||||
without callers needing to change.
|
||||
"""
|
||||
query = LocationQuery(
|
||||
name=collector or None,
|
||||
aliases=tuple(filter(None, (collector,))),
|
||||
source_latitude=source_latitude,
|
||||
source_longitude=source_longitude,
|
||||
extra={
|
||||
"collector": collector or "",
|
||||
"site": coerce_str(site),
|
||||
"operator": coerce_str(operator),
|
||||
"peer_asn": peer_asn,
|
||||
"origin_asn": origin_asn,
|
||||
"prefix": coerce_str(prefix),
|
||||
},
|
||||
)
|
||||
return BGP_EVENT_PIPELINE.resolve_best(query)
|
||||
|
||||
|
||||
def resolve_bgp_event_geo_dict(
|
||||
collector: str,
|
||||
*,
|
||||
source_latitude: float | None = None,
|
||||
source_longitude: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Convenience wrapper returning the legacy ``collector_geo`` dict shape.
|
||||
|
||||
Preserves ``city``/``country``/``latitude``/``longitude`` keys (consumed
|
||||
by existing detectors / enrichment / DB serialization) and adds
|
||||
``precision``/``source``/``needs_confirmation`` for richer downstream use.
|
||||
"""
|
||||
result = resolve_bgp_event_location(
|
||||
collector=collector,
|
||||
source_latitude=source_latitude,
|
||||
source_longitude=source_longitude,
|
||||
)
|
||||
candidate = result.location
|
||||
if candidate is None:
|
||||
return {}
|
||||
return {
|
||||
"city": candidate.city,
|
||||
"country": candidate.country,
|
||||
"latitude": candidate.latitude,
|
||||
"longitude": candidate.longitude,
|
||||
"precision": candidate.precision,
|
||||
"source": candidate.source,
|
||||
"needs_confirmation": candidate.needs_confirmation,
|
||||
"matched_location_name": candidate.matched_location_name,
|
||||
"confidence": candidate.confidence,
|
||||
}
|
||||
@@ -13,6 +13,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_observation import BGPObservation
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.services.bgp_collector_locations import (
|
||||
RIPE_RIS_COLLECTOR_COORDS,
|
||||
get_bgp_collector_location_dict,
|
||||
)
|
||||
from app.services.bgp_event_locations import resolve_bgp_event_geo_dict
|
||||
from app.services.bgp_incidents import create_bgp_incidents_for_anomalies
|
||||
from app.services.bgp_detectors import (
|
||||
detect_mass_withdrawal_anomalies,
|
||||
@@ -23,32 +28,17 @@ from app.services.bgp_detectors import (
|
||||
)
|
||||
from app.services.bgp_enrichment import enrich_bgp_events_for_batch, extract_bgp_network_fields
|
||||
|
||||
|
||||
RIPE_RIS_COLLECTOR_COORDS: dict[str, dict[str, Any]] = {
|
||||
"rrc00": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041},
|
||||
"rrc01": {"city": "London", "country": "United Kingdom", "latitude": 51.5072, "longitude": -0.1276},
|
||||
"rrc03": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041},
|
||||
"rrc04": {"city": "Geneva", "country": "Switzerland", "latitude": 46.2044, "longitude": 6.1432},
|
||||
"rrc05": {"city": "Vienna", "country": "Austria", "latitude": 48.2082, "longitude": 16.3738},
|
||||
"rrc06": {"city": "Otemachi", "country": "Japan", "latitude": 35.686, "longitude": 139.7671},
|
||||
"rrc07": {"city": "Stockholm", "country": "Sweden", "latitude": 59.3293, "longitude": 18.0686},
|
||||
"rrc10": {"city": "Milan", "country": "Italy", "latitude": 45.4642, "longitude": 9.19},
|
||||
"rrc11": {"city": "New York", "country": "United States", "latitude": 40.7128, "longitude": -74.006},
|
||||
"rrc12": {"city": "Frankfurt", "country": "Germany", "latitude": 50.1109, "longitude": 8.6821},
|
||||
"rrc13": {"city": "Moscow", "country": "Russia", "latitude": 55.7558, "longitude": 37.6173},
|
||||
"rrc14": {"city": "Palo Alto", "country": "United States", "latitude": 37.4419, "longitude": -122.143},
|
||||
"rrc15": {"city": "Sao Paulo", "country": "Brazil", "latitude": -23.5558, "longitude": -46.6396},
|
||||
"rrc16": {"city": "Miami", "country": "United States", "latitude": 25.7617, "longitude": -80.1918},
|
||||
"rrc18": {"city": "Barcelona", "country": "Spain", "latitude": 41.3874, "longitude": 2.1686},
|
||||
"rrc19": {"city": "Johannesburg", "country": "South Africa", "latitude": -26.2041, "longitude": 28.0473},
|
||||
"rrc20": {"city": "Zurich", "country": "Switzerland", "latitude": 47.3769, "longitude": 8.5417},
|
||||
"rrc21": {"city": "Paris", "country": "France", "latitude": 48.8566, "longitude": 2.3522},
|
||||
"rrc22": {"city": "Bucharest", "country": "Romania", "latitude": 44.4268, "longitude": 26.1025},
|
||||
"rrc23": {"city": "Singapore", "country": "Singapore", "latitude": 1.3521, "longitude": 103.8198},
|
||||
"rrc24": {"city": "Montevideo", "country": "Uruguay", "latitude": -34.9011, "longitude": -56.1645},
|
||||
"rrc25": {"city": "Amsterdam", "country": "Netherlands", "latitude": 52.3676, "longitude": 4.9041},
|
||||
"rrc26": {"city": "Dubai", "country": "United Arab Emirates", "latitude": 25.2048, "longitude": 55.2708},
|
||||
}
|
||||
# Re-exported for backward compatibility with anything that imports
|
||||
# ``RIPE_RIS_COLLECTOR_COORDS`` from this module. New code should call
|
||||
# ``app.services.bgp_collector_locations.get_bgp_collector_location_dict()``
|
||||
# or ``resolve_bgp_collector_location()`` instead — those use the DB-backed
|
||||
# collector-location cache.
|
||||
__all__ = [
|
||||
"RIPE_RIS_COLLECTOR_COORDS",
|
||||
"normalize_bgp_event",
|
||||
"save_bgp_observations_for_batch",
|
||||
"create_bgp_anomalies_for_batch",
|
||||
]
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int | None:
|
||||
@@ -131,7 +121,19 @@ def normalize_bgp_event(payload: dict[str, Any], *, project: str) -> dict[str, A
|
||||
)
|
||||
source_id = hashlib.sha1(source_material.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
collector_location = RIPE_RIS_COLLECTOR_COORDS.get(collector, {})
|
||||
# Routes through the BGP event pipeline: source coords (if any) →
|
||||
# collector inheritance. Returned dict keeps the legacy
|
||||
# {city, country, latitude, longitude} keys plus richer
|
||||
# {precision, source, needs_confirmation, matched_location_name, confidence}.
|
||||
collector_location = resolve_bgp_event_geo_dict(
|
||||
collector,
|
||||
source_latitude=payload.get("latitude"),
|
||||
source_longitude=payload.get("longitude"),
|
||||
)
|
||||
# Empty result (unknown collector & no source coords) — keep the
|
||||
# downstream-expected dict shape so detectors / serializers don't crash.
|
||||
if not collector_location:
|
||||
collector_location = get_bgp_collector_location_dict(collector)
|
||||
network_fields = extract_bgp_network_fields(prefix)
|
||||
metadata = {
|
||||
"project": project,
|
||||
|
||||
863
backend/app/services/compute_center_locations.py
Normal file
863
backend/app/services/compute_center_locations.py
Normal file
@@ -0,0 +1,863 @@
|
||||
"""Compute-center location resolver, built on the shared location pipeline.
|
||||
|
||||
This module is a thin domain wrapper that wires up
|
||||
:mod:`app.services.location` for compute centers:
|
||||
|
||||
SourceCoordinates
|
||||
|
||||
The online Nominatim step is intentionally reserved for the user-triggered
|
||||
``collect-location`` flow. The regular GeoJSON endpoint runs during Earth
|
||||
startup, so it must stay local and deterministic.
|
||||
|
||||
For the full design and the reason behind the abstraction (compute centers,
|
||||
BGP collectors, BGP events, and future entities all share one pipeline),
|
||||
see ``docs/plans/location-resolver-shared-pipeline-plan.md``.
|
||||
|
||||
The ``ComputeCenterLocation`` dataclass and the public function signatures are
|
||||
preserved verbatim so existing callers and tests do not need to change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.collected_data_fields import get_record_field
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.compute_center_location import ComputeCenterLocationRecord
|
||||
|
||||
from app.services.location import (
|
||||
LocationCandidate,
|
||||
LocationPipeline,
|
||||
LocationQuery,
|
||||
NominatimResolver,
|
||||
ResolverOutput,
|
||||
SourceCoordinatesResolver,
|
||||
build_default_nominatim_geocoder,
|
||||
coerce_str,
|
||||
normalize_country_text,
|
||||
normalize_text,
|
||||
parse_float,
|
||||
)
|
||||
|
||||
ROR_SEARCH_URL = "https://api.ror.org/v2/organizations"
|
||||
DEFAULT_ROR_USER_AGENT = "planet-earth-location-resolver/1.0"
|
||||
DEFAULT_ROR_TIMEOUT_SECONDS = 8.0
|
||||
RENDERABLE_PRECISIONS: tuple[str, ...] = ("precise", "site", "city")
|
||||
FORBIDDEN_PRECISIONS: tuple[str, ...] = (
|
||||
"country",
|
||||
"estimated_country",
|
||||
"country_major_compute_city",
|
||||
"region",
|
||||
"unknown",
|
||||
)
|
||||
|
||||
# ── Public dataclasses ──────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ComputeCenterLocation:
|
||||
latitude: float | None
|
||||
longitude: float | None
|
||||
location_precision: str
|
||||
geography_mode: str
|
||||
is_estimated: bool
|
||||
estimated_reason: str | None = None
|
||||
location_confidence: float | None = None
|
||||
location_source: str | None = None
|
||||
location_source_note: str | None = None
|
||||
location_verified_at: str | None = None
|
||||
matched_location_name: str | None = None
|
||||
needs_confirmation: bool = False
|
||||
city: str | None = None
|
||||
region: str | None = None
|
||||
country: str | None = None
|
||||
|
||||
@property
|
||||
def is_renderable(self) -> bool:
|
||||
if self.latitude in (None, 0.0) or self.longitude in (None, 0.0):
|
||||
return False
|
||||
return self.location_precision in RENDERABLE_PRECISIONS
|
||||
|
||||
def to_geojson_properties(self) -> dict[str, Any]:
|
||||
return {
|
||||
"latitude": self.latitude,
|
||||
"longitude": self.longitude,
|
||||
"location_precision": self.location_precision,
|
||||
"geography_mode": self.geography_mode,
|
||||
"is_estimated": self.is_estimated,
|
||||
"estimated_reason": self.estimated_reason,
|
||||
"location_confidence": self.location_confidence,
|
||||
"location_source": self.location_source,
|
||||
"location_source_note": self.location_source_note,
|
||||
"location_verified_at": self.location_verified_at,
|
||||
"matched_location_name": self.matched_location_name,
|
||||
"needs_confirmation": self.needs_confirmation,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolutionDiagnostic:
|
||||
failure_reason: str
|
||||
attempted_queries: tuple[str, ...] = ()
|
||||
record_id: int | None = None
|
||||
source: str | None = None
|
||||
source_id: str | None = None
|
||||
name: str | None = None
|
||||
country: str | None = None
|
||||
city: str | None = None
|
||||
site: str | None = None
|
||||
operator: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"failure_reason": self.failure_reason,
|
||||
"attempted_queries": list(self.attempted_queries),
|
||||
"record_id": self.record_id,
|
||||
"source": self.source,
|
||||
"source_id": self.source_id,
|
||||
"name": self.name,
|
||||
"country": self.country,
|
||||
"city": self.city,
|
||||
"site": self.site,
|
||||
"operator": self.operator,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolutionResult:
|
||||
location: ComputeCenterLocation | None
|
||||
diagnostic: ResolutionDiagnostic | None
|
||||
|
||||
@property
|
||||
def is_resolved(self) -> bool:
|
||||
return bool(self.location and self.location.is_renderable)
|
||||
|
||||
|
||||
# ── Geocoder (kept at module level so tests can monkeypatch + cache_clear) ──
|
||||
|
||||
_geocode_online = build_default_nominatim_geocoder()
|
||||
|
||||
|
||||
# ── Stored location cache ───────────────────────────────────────────
|
||||
|
||||
|
||||
COMPUTE_CENTER_LOCATION_CACHE: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _cache_key(source: str | None, source_id: str | None) -> str:
|
||||
return f"{coerce_str(source)}:{coerce_str(source_id)}"
|
||||
|
||||
|
||||
def set_compute_center_location_cache(
|
||||
locations: dict[str, dict[str, Any]],
|
||||
) -> None:
|
||||
COMPUTE_CENTER_LOCATION_CACHE.clear()
|
||||
COMPUTE_CENTER_LOCATION_CACHE.update(
|
||||
{coerce_str(key): dict(value) for key, value in locations.items()}
|
||||
)
|
||||
|
||||
|
||||
async def refresh_compute_center_location_cache(
|
||||
session: AsyncSession,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
result = await session.execute(select(ComputeCenterLocationRecord))
|
||||
records = result.scalars().all()
|
||||
cache = {}
|
||||
for record in records:
|
||||
if not hasattr(record, "to_location_dict"):
|
||||
continue
|
||||
if not record.source or not record.source_id:
|
||||
continue
|
||||
cache[_cache_key(record.source, record.source_id)] = record.to_location_dict()
|
||||
set_compute_center_location_cache(cache)
|
||||
return cache
|
||||
|
||||
|
||||
def get_compute_center_location_dict(
|
||||
source: str | None,
|
||||
source_id: str | None,
|
||||
) -> dict[str, Any]:
|
||||
return dict(COMPUTE_CENTER_LOCATION_CACHE.get(_cache_key(source, source_id), {}))
|
||||
|
||||
|
||||
# ── Pipeline construction ──────────────────────────────────────────
|
||||
|
||||
|
||||
@lru_cache(maxsize=512)
|
||||
def _lookup_ror_organization(query: str) -> dict[str, Any] | None:
|
||||
"""Lookup a research organization in ROR for user-triggered candidates."""
|
||||
if not query:
|
||||
return None
|
||||
response = httpx.get(
|
||||
ROR_SEARCH_URL,
|
||||
params={"query": query},
|
||||
headers={"User-Agent": DEFAULT_ROR_USER_AGENT},
|
||||
timeout=DEFAULT_ROR_TIMEOUT_SECONDS,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
items = payload.get("items") if isinstance(payload, dict) else None
|
||||
if not isinstance(items, list) or not items:
|
||||
return None
|
||||
first = items[0]
|
||||
if not isinstance(first, dict):
|
||||
return None
|
||||
organization = first.get("organization")
|
||||
if isinstance(organization, dict):
|
||||
return organization
|
||||
return first
|
||||
|
||||
|
||||
def _compute_center_ror_query_plan(
|
||||
query: LocationQuery,
|
||||
) -> list[tuple[str, tuple[str, ...]]]:
|
||||
extra = query.extra or {}
|
||||
raw_parts: list[tuple[str, str]] = [
|
||||
("site", coerce_str(extra.get("site"))),
|
||||
("operator", coerce_str(extra.get("operator"))),
|
||||
("organization", coerce_str(extra.get("organization"))),
|
||||
]
|
||||
for field, value in tuple(raw_parts):
|
||||
if "/" not in value:
|
||||
continue
|
||||
raw_parts.extend(
|
||||
(field, part.strip())
|
||||
for part in value.split("/")
|
||||
if len(part.strip()) >= 3
|
||||
)
|
||||
|
||||
plan: list[tuple[str, tuple[str, ...]]] = []
|
||||
seen: set[str] = set()
|
||||
for field, value in raw_parts:
|
||||
key = normalize_text(value)
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
plan.append((value, (field,)))
|
||||
return plan
|
||||
|
||||
|
||||
def _organization_label(organization: dict[str, Any], fallback: str) -> str:
|
||||
names = organization.get("names")
|
||||
if isinstance(names, list):
|
||||
for name in names:
|
||||
if not isinstance(name, dict):
|
||||
continue
|
||||
types = name.get("types")
|
||||
if isinstance(types, list) and "ror_display" in types:
|
||||
value = coerce_str(name.get("value"))
|
||||
if value:
|
||||
return value
|
||||
for name in names:
|
||||
if isinstance(name, dict):
|
||||
value = coerce_str(name.get("value"))
|
||||
if value:
|
||||
return value
|
||||
return fallback
|
||||
|
||||
|
||||
class ROROrganizationResolver:
|
||||
"""Resolve source-provided organization/site text through the open ROR API."""
|
||||
|
||||
name = "ror_organization_registry"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
query_plan_builder=_compute_center_ror_query_plan,
|
||||
lookup=lambda q: _lookup_ror_organization(q),
|
||||
confidence: float = 0.68,
|
||||
) -> None:
|
||||
self._query_plan_builder = query_plan_builder
|
||||
self._lookup = lookup
|
||||
self._confidence = confidence
|
||||
|
||||
def resolve(self, query: LocationQuery):
|
||||
from app.services.location import ResolverOutput
|
||||
from app.services.location.text import parse_float
|
||||
|
||||
attempted: list[str] = []
|
||||
candidates: list[LocationCandidate] = []
|
||||
context_country = normalize_text(normalize_country_text(query.country))
|
||||
|
||||
for ror_query, matched_fields in self._query_plan_builder(query):
|
||||
attempted.append(f"ror:{ror_query}")
|
||||
try:
|
||||
organization = self._lookup(ror_query)
|
||||
except Exception:
|
||||
continue
|
||||
if not isinstance(organization, dict):
|
||||
continue
|
||||
locations = organization.get("locations")
|
||||
if not isinstance(locations, list) or not locations:
|
||||
continue
|
||||
location = locations[0]
|
||||
if not isinstance(location, dict):
|
||||
continue
|
||||
details = location.get("geonames_details")
|
||||
if not isinstance(details, dict):
|
||||
continue
|
||||
latitude = parse_float(details.get("lat"))
|
||||
longitude = parse_float(details.get("lng"))
|
||||
if latitude in (None, 0.0) or longitude in (None, 0.0):
|
||||
continue
|
||||
|
||||
country = normalize_country_text(details.get("country_name"))
|
||||
if context_country and normalize_text(country) != context_country:
|
||||
continue
|
||||
|
||||
city = coerce_str(details.get("name")) or None
|
||||
region = coerce_str(details.get("country_subdivision_name")) or None
|
||||
display_name = _organization_label(organization, ror_query)
|
||||
ror_id = coerce_str(organization.get("id"))
|
||||
geonames_id = location.get("geonames_id")
|
||||
source_note = (
|
||||
f"ROR organization match: {display_name}"
|
||||
+ (f" ({ror_id})" if ror_id else "")
|
||||
+ (f"; GeoNames {geonames_id}" if geonames_id else "")
|
||||
)
|
||||
candidates.append(
|
||||
LocationCandidate(
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
display_name=display_name,
|
||||
precision="city",
|
||||
confidence=self._confidence,
|
||||
query=ror_query,
|
||||
source=self.name,
|
||||
source_note=source_note,
|
||||
matched_fields=matched_fields,
|
||||
needs_confirmation=True,
|
||||
city=city,
|
||||
region=region,
|
||||
country=country or query.country,
|
||||
matched_location_name=display_name,
|
||||
location_verified_at=None,
|
||||
suggested_registry_entry=None,
|
||||
)
|
||||
)
|
||||
|
||||
return ResolverOutput(
|
||||
candidates=tuple(candidates),
|
||||
attempted_queries=tuple(attempted),
|
||||
)
|
||||
|
||||
|
||||
class StoredComputeCenterLocationResolver:
|
||||
"""Resolve a compute center through the DB-backed current-location cache."""
|
||||
|
||||
name = "stored_compute_center_location"
|
||||
|
||||
def resolve(self, query: LocationQuery) -> ResolverOutput:
|
||||
extra = query.extra or {}
|
||||
stored = get_compute_center_location_dict(
|
||||
coerce_str(extra.get("source")),
|
||||
coerce_str(extra.get("source_id")),
|
||||
)
|
||||
if not stored:
|
||||
return ResolverOutput()
|
||||
latitude = parse_float(stored.get("latitude"))
|
||||
longitude = parse_float(stored.get("longitude"))
|
||||
if latitude in (None, 0.0) or longitude in (None, 0.0):
|
||||
return ResolverOutput()
|
||||
return ResolverOutput(
|
||||
candidates=(
|
||||
LocationCandidate(
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
display_name=stored.get("name") or query.name or "Compute center",
|
||||
precision=stored.get("precision") or "city",
|
||||
confidence=float(stored.get("confidence") or 0.85),
|
||||
query=f"stored_compute_center_location::{stored.get('source')}:{stored.get('source_id')}",
|
||||
source=self.name,
|
||||
source_note=stored.get("source_note"),
|
||||
matched_fields=("source", "source_id"),
|
||||
needs_confirmation=bool(stored.get("needs_confirmation")),
|
||||
city=stored.get("city") or query.city,
|
||||
region=None,
|
||||
country=stored.get("country") or query.country,
|
||||
matched_location_name=stored.get("site") or stored.get("name") or query.name,
|
||||
location_verified_at=stored.get("verified_at"),
|
||||
suggested_registry_entry=None,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _short_system_name(name: Any) -> str:
|
||||
"""Strip vendor/system suffix from TOP500 names like ``"El Capitan - HPE Cray ..."``."""
|
||||
text = coerce_str(name)
|
||||
if not text:
|
||||
return ""
|
||||
head = text.split(" - ", 1)[0].strip()
|
||||
return head or text
|
||||
|
||||
|
||||
def _record_context(record: Any, metadata: dict[str, Any]) -> dict[str, str]:
|
||||
name = coerce_str(getattr(record, "name", None))
|
||||
return {
|
||||
"source": coerce_str(getattr(record, "source", None)),
|
||||
"source_id": coerce_str(getattr(record, "source_id", None)),
|
||||
"name": name,
|
||||
"name_short": _short_system_name(name),
|
||||
"city": coerce_str(get_record_field(record, "city")),
|
||||
"country": coerce_str(get_record_field(record, "country")),
|
||||
"site": coerce_str(metadata.get("site") or metadata.get("organization")),
|
||||
"operator": coerce_str(
|
||||
metadata.get("operator")
|
||||
or metadata.get("organization")
|
||||
or metadata.get("owner")
|
||||
or metadata.get("manufacturer")
|
||||
),
|
||||
"organization": coerce_str(metadata.get("organization")),
|
||||
}
|
||||
|
||||
|
||||
def _context_to_query(
|
||||
context: dict[str, str],
|
||||
*,
|
||||
source_lat: float | None = None,
|
||||
source_lon: float | None = None,
|
||||
) -> LocationQuery:
|
||||
name = context.get("name") or None
|
||||
name_short = context.get("name_short") or ""
|
||||
aliases: tuple[str, ...] = ()
|
||||
if name_short and name_short != name:
|
||||
aliases = (name_short,)
|
||||
return LocationQuery(
|
||||
name=name,
|
||||
aliases=aliases,
|
||||
city=context.get("city") or None,
|
||||
country=context.get("country") or None,
|
||||
source_latitude=source_lat,
|
||||
source_longitude=source_lon,
|
||||
extra={
|
||||
"source": context.get("source") or "",
|
||||
"source_id": context.get("source_id") or "",
|
||||
"site": context.get("site") or "",
|
||||
"operator": context.get("operator") or "",
|
||||
"organization": context.get("organization") or "",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _compute_center_query_plan(
|
||||
query: LocationQuery,
|
||||
) -> list[tuple[str, tuple[str, ...]]]:
|
||||
"""Build the Nominatim query plan for a compute-center query.
|
||||
|
||||
Mirrors the legacy ``_build_online_query_plan`` ordering exactly.
|
||||
"""
|
||||
name = query.name or ""
|
||||
name_short = (query.aliases[0] if query.aliases else "") or name
|
||||
extra = query.extra or {}
|
||||
site = str(extra.get("site") or "")
|
||||
operator = str(extra.get("operator") or "")
|
||||
city = query.city or ""
|
||||
country = query.country or ""
|
||||
|
||||
plan: list[tuple[str, tuple[str, ...]]] = []
|
||||
|
||||
def add(parts: list[tuple[str, str]]) -> None:
|
||||
non_empty = [(field, value) for field, value in parts if value]
|
||||
if not non_empty:
|
||||
return
|
||||
seen: set[str] = set()
|
||||
cleaned: list[str] = []
|
||||
fields: list[str] = []
|
||||
for field, value in non_empty:
|
||||
key = normalize_text(value)
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
cleaned.append(value)
|
||||
fields.append(field)
|
||||
if not cleaned:
|
||||
return
|
||||
composed = ", ".join(cleaned)
|
||||
if not any(composed == existing for existing, _ in plan):
|
||||
plan.append((composed, tuple(fields)))
|
||||
|
||||
add([("site", site), ("country", country)])
|
||||
add([("operator", operator), ("city", city), ("country", country)])
|
||||
add([("name", name_short), ("operator", operator), ("country", country)])
|
||||
add([("name", name_short), ("site", site)])
|
||||
add([("name", name_short), ("country", country)])
|
||||
add([("name", name_short), ("city", city), ("country", country)])
|
||||
add([("city", city), ("country", country)])
|
||||
if name and name != name_short:
|
||||
add([("name", name), ("country", country)])
|
||||
return plan
|
||||
|
||||
|
||||
COMPUTE_CENTER_PIPELINE = LocationPipeline(
|
||||
[
|
||||
SourceCoordinatesResolver(),
|
||||
StoredComputeCenterLocationResolver(),
|
||||
],
|
||||
failure_reason=(
|
||||
"Could not resolve to city-level coordinates from source coords"
|
||||
" or stored compute-center location."
|
||||
),
|
||||
)
|
||||
|
||||
COMPUTE_CENTER_COLLECTION_PIPELINE = LocationPipeline(
|
||||
[
|
||||
SourceCoordinatesResolver(),
|
||||
ROROrganizationResolver(),
|
||||
NominatimResolver(
|
||||
query_plan_builder=_compute_center_query_plan,
|
||||
# Late-binding so test monkeypatching of ``_geocode_online`` works.
|
||||
geocoder=lambda q: _geocode_online(q),
|
||||
),
|
||||
],
|
||||
failure_reason=(
|
||||
"Could not resolve to city-level coordinates from source coords"
|
||||
", ROR organization lookup, or online geocoding."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ── Candidate → ComputeCenterLocation conversion ───────────────────
|
||||
|
||||
|
||||
_GEOGRAPHY_MODE_BY_SOURCE = {
|
||||
"source_coordinates": "source_coordinates",
|
||||
"stored_compute_center_location": "stored_compute_center_location",
|
||||
"ror_organization_registry": "ror_organization",
|
||||
"nominatim_online_geocode": "online_geocode",
|
||||
}
|
||||
|
||||
|
||||
def _candidate_to_location(
|
||||
candidate: LocationCandidate,
|
||||
*,
|
||||
context: dict[str, str],
|
||||
) -> ComputeCenterLocation:
|
||||
geography_mode = _GEOGRAPHY_MODE_BY_SOURCE.get(candidate.source, "online_geocode")
|
||||
is_estimated = candidate.needs_confirmation or candidate.source.startswith(
|
||||
"nominatim"
|
||||
)
|
||||
estimated_reason: str | None
|
||||
if candidate.source == "source_coordinates":
|
||||
estimated_reason = None
|
||||
elif candidate.source == "stored_compute_center_location":
|
||||
estimated_reason = candidate.source_note
|
||||
elif candidate.source == "ror_organization_registry":
|
||||
fields_summary = ", ".join(candidate.matched_fields) or "organization"
|
||||
estimated_reason = (
|
||||
f"Resolved by ROR organization lookup '{candidate.query}' "
|
||||
f"(matched fields: {fields_summary})"
|
||||
)
|
||||
elif candidate.source == "nominatim_online_geocode":
|
||||
fields_summary = ", ".join(candidate.matched_fields) or "name"
|
||||
estimated_reason = (
|
||||
f"Resolved by online geocoding query '{candidate.query}' "
|
||||
f"(matched fields: {fields_summary})"
|
||||
)
|
||||
else:
|
||||
estimated_reason = candidate.source_note
|
||||
|
||||
country = (
|
||||
candidate.country
|
||||
or normalize_country_text(context.get("country"))
|
||||
or context.get("country")
|
||||
or None
|
||||
)
|
||||
return ComputeCenterLocation(
|
||||
latitude=candidate.latitude,
|
||||
longitude=candidate.longitude,
|
||||
location_precision=candidate.precision,
|
||||
geography_mode=geography_mode,
|
||||
is_estimated=is_estimated,
|
||||
estimated_reason=estimated_reason,
|
||||
location_confidence=candidate.confidence,
|
||||
location_source=candidate.source,
|
||||
location_source_note=candidate.source_note,
|
||||
location_verified_at=candidate.location_verified_at,
|
||||
matched_location_name=candidate.matched_location_name
|
||||
or context.get("name")
|
||||
or None,
|
||||
needs_confirmation=candidate.needs_confirmation,
|
||||
city=candidate.city or context.get("city") or None,
|
||||
region=candidate.region,
|
||||
country=country,
|
||||
)
|
||||
|
||||
|
||||
def _diagnostic_for(
|
||||
record: Any,
|
||||
context: dict[str, str],
|
||||
*,
|
||||
failure_reason: str,
|
||||
attempted_queries: tuple[str, ...] = (),
|
||||
) -> ResolutionDiagnostic:
|
||||
return ResolutionDiagnostic(
|
||||
failure_reason=failure_reason,
|
||||
attempted_queries=attempted_queries,
|
||||
record_id=getattr(record, "id", None),
|
||||
source=getattr(record, "source", None),
|
||||
source_id=getattr(record, "source_id", None),
|
||||
name=context.get("name") or getattr(record, "name", None),
|
||||
country=context.get("country") or None,
|
||||
city=context.get("city") or None,
|
||||
site=context.get("site") or None,
|
||||
operator=context.get("operator") or None,
|
||||
)
|
||||
|
||||
|
||||
# ── Public API ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def resolve_compute_center_location(
|
||||
record: Any,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> ComputeCenterLocation:
|
||||
"""Backwards-compatible thin wrapper returning the renderable location only.
|
||||
|
||||
Records that cannot be resolved to city-level get a placeholder
|
||||
:class:`ComputeCenterLocation` with ``location_precision='unknown'``.
|
||||
Callers should generally prefer :func:`resolve_compute_center_location_full`.
|
||||
"""
|
||||
full = resolve_compute_center_location_full(record, metadata)
|
||||
return full.location or ComputeCenterLocation(
|
||||
latitude=None,
|
||||
longitude=None,
|
||||
location_precision="unknown",
|
||||
geography_mode="unresolved",
|
||||
is_estimated=True,
|
||||
estimated_reason="No resolvable location hints",
|
||||
location_confidence=0.0,
|
||||
location_source="unknown",
|
||||
location_source_note=(
|
||||
"No source coordinates, ROR organization match, or online"
|
||||
" geocoding result."
|
||||
),
|
||||
matched_location_name=None,
|
||||
needs_confirmation=False,
|
||||
)
|
||||
|
||||
|
||||
def resolve_compute_center_location_full(
|
||||
record: Any,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
*,
|
||||
allow_online: bool = False,
|
||||
) -> ResolutionResult:
|
||||
metadata = metadata or {}
|
||||
context = _record_context(record, metadata)
|
||||
|
||||
from app.services.location.text import parse_float as _parse_float
|
||||
|
||||
source_lat = _parse_float(get_record_field(record, "latitude"))
|
||||
source_lon = _parse_float(get_record_field(record, "longitude"))
|
||||
if source_lat in (None, 0.0):
|
||||
source_lat = None
|
||||
if source_lon in (None, 0.0):
|
||||
source_lon = None
|
||||
|
||||
query = _context_to_query(
|
||||
context, source_lat=source_lat, source_lon=source_lon
|
||||
)
|
||||
pipeline = (
|
||||
COMPUTE_CENTER_COLLECTION_PIPELINE
|
||||
if allow_online
|
||||
else COMPUTE_CENTER_PIPELINE
|
||||
)
|
||||
pipeline_result = pipeline.resolve_best(query)
|
||||
|
||||
if pipeline_result.location and pipeline_result.location.precision in RENDERABLE_PRECISIONS:
|
||||
location = _candidate_to_location(pipeline_result.location, context=context)
|
||||
return ResolutionResult(location=location, diagnostic=None)
|
||||
|
||||
return ResolutionResult(
|
||||
location=None,
|
||||
diagnostic=_diagnostic_for(
|
||||
record,
|
||||
context,
|
||||
failure_reason=(
|
||||
"Could not resolve to city-level coordinates from source coords"
|
||||
", ROR organization lookup, or online geocoding."
|
||||
if allow_online
|
||||
else (
|
||||
"Could not resolve to city-level coordinates from source coords"
|
||||
" or stored compute-center location."
|
||||
)
|
||||
),
|
||||
attempted_queries=pipeline_result.attempted_queries,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def collect_location_candidates(
|
||||
*,
|
||||
name: str | None = None,
|
||||
source: str | None = None,
|
||||
source_id: str | None = None,
|
||||
operator: str | None = None,
|
||||
site: str | None = None,
|
||||
city: str | None = None,
|
||||
country: str | None = None,
|
||||
organization: str | None = None,
|
||||
record_id: int | None = None,
|
||||
) -> tuple[list[LocationCandidate], list[str]]:
|
||||
"""Run the full resolution chain and return ranked candidates with attempted queries.
|
||||
|
||||
The unused ``source`` / ``source_id`` / ``record_id`` arguments are kept
|
||||
for backward compatibility with the API handler that calls this function.
|
||||
"""
|
||||
name_value = coerce_str(name)
|
||||
context: dict[str, str] = {
|
||||
"source": coerce_str(source),
|
||||
"source_id": coerce_str(source_id),
|
||||
"name": name_value,
|
||||
"name_short": _short_system_name(name_value),
|
||||
"city": coerce_str(city),
|
||||
"country": coerce_str(country),
|
||||
"site": coerce_str(site or organization),
|
||||
"operator": coerce_str(operator or organization),
|
||||
"organization": coerce_str(organization),
|
||||
}
|
||||
query = _context_to_query(context)
|
||||
return COMPUTE_CENTER_COLLECTION_PIPELINE.collect_candidates(query)
|
||||
|
||||
|
||||
def _record_operator(metadata: dict[str, Any]) -> str | None:
|
||||
return coerce_str(
|
||||
metadata.get("operator")
|
||||
or metadata.get("organization")
|
||||
or metadata.get("owner")
|
||||
or metadata.get("manufacturer")
|
||||
) or None
|
||||
|
||||
|
||||
async def seed_compute_center_locations_from_source_coords(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
"""Seed stored compute-center locations only from real source coordinates."""
|
||||
stmt = (
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source.in_(["top500", "epoch_ai_gpu"]))
|
||||
.where(CollectedData.is_current.is_(True))
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
records = result.scalars().all()
|
||||
changed = False
|
||||
|
||||
for record in records:
|
||||
source_value = coerce_str(getattr(record, "source", None))
|
||||
source_id = coerce_str(getattr(record, "source_id", None))
|
||||
if not source_value or not source_id:
|
||||
continue
|
||||
latitude = parse_float(get_record_field(record, "latitude"))
|
||||
longitude = parse_float(get_record_field(record, "longitude"))
|
||||
if latitude in (None, 0.0) or longitude in (None, 0.0):
|
||||
continue
|
||||
existing = await session.scalar(
|
||||
select(ComputeCenterLocationRecord)
|
||||
.where(ComputeCenterLocationRecord.source == source_value)
|
||||
.where(ComputeCenterLocationRecord.source_id == source_id)
|
||||
)
|
||||
if existing:
|
||||
continue
|
||||
metadata = record.extra_data or {}
|
||||
session.add(
|
||||
ComputeCenterLocationRecord(
|
||||
source=source_value,
|
||||
source_id=source_id,
|
||||
name=getattr(record, "name", None),
|
||||
operator=_record_operator(metadata),
|
||||
site=coerce_str(metadata.get("site") or metadata.get("organization")) or None,
|
||||
city=coerce_str(get_record_field(record, "city")) or None,
|
||||
country=coerce_str(get_record_field(record, "country")) or None,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
precision="precise",
|
||||
confidence=1.0,
|
||||
location_source="source_coordinates",
|
||||
source_note="Seeded from source-provided compute-center coordinates",
|
||||
raw_payload={
|
||||
"record_id": getattr(record, "id", None),
|
||||
"source": source_value,
|
||||
"source_id": source_id,
|
||||
},
|
||||
needs_confirmation=False,
|
||||
verification_status="source_provided",
|
||||
verified_at=None,
|
||||
)
|
||||
)
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
await session.commit()
|
||||
await refresh_compute_center_location_cache(session)
|
||||
|
||||
|
||||
async def upsert_compute_center_location(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
source: str,
|
||||
source_id: str,
|
||||
name: str | None = None,
|
||||
operator: str | None = None,
|
||||
site: str | None = None,
|
||||
city: str | None = None,
|
||||
country: str | None = None,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
precision: str = "city",
|
||||
confidence: float | None = None,
|
||||
location_source: str = "manual_selection",
|
||||
source_url: str | None = None,
|
||||
source_note: str | None = None,
|
||||
raw_payload: dict[str, Any] | None = None,
|
||||
needs_confirmation: bool = False,
|
||||
verification_status: str = "verified",
|
||||
) -> ComputeCenterLocationRecord:
|
||||
existing = await session.scalar(
|
||||
select(ComputeCenterLocationRecord)
|
||||
.where(ComputeCenterLocationRecord.source == source)
|
||||
.where(ComputeCenterLocationRecord.source_id == source_id)
|
||||
)
|
||||
verified_at = None if needs_confirmation else datetime.now(UTC)
|
||||
values = {
|
||||
"name": name,
|
||||
"operator": operator,
|
||||
"site": site,
|
||||
"city": city,
|
||||
"country": country,
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"precision": precision,
|
||||
"confidence": confidence,
|
||||
"location_source": location_source,
|
||||
"source_url": source_url,
|
||||
"source_note": source_note,
|
||||
"raw_payload": raw_payload or {},
|
||||
"needs_confirmation": needs_confirmation,
|
||||
"verification_status": verification_status,
|
||||
"verified_at": verified_at,
|
||||
}
|
||||
if existing:
|
||||
for key, value in values.items():
|
||||
setattr(existing, key, value)
|
||||
record = existing
|
||||
else:
|
||||
record = ComputeCenterLocationRecord(
|
||||
source=source,
|
||||
source_id=source_id,
|
||||
**values,
|
||||
)
|
||||
session.add(record)
|
||||
|
||||
await session.commit()
|
||||
await session.refresh(record)
|
||||
await refresh_compute_center_location_cache(session)
|
||||
return record
|
||||
119
backend/app/services/docs_gatekeeper.py
Normal file
119
backend/app/services/docs_gatekeeper.py
Normal file
@@ -0,0 +1,119 @@
|
||||
"""Server-side Docs metadata and Gatekeeper authorization helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from app.models.user import User
|
||||
|
||||
DocsAccess = Literal["public", "docs_user", "docs_developer", "docs_admin"]
|
||||
DocsLang = Literal["zh", "en"]
|
||||
|
||||
VALID_DOCS_LANGS = {"zh", "en"}
|
||||
DOCS_README_FILENAME = "README.md"
|
||||
DEFAULT_DOCS_SLUG = "overview"
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
TECHNICAL_DOCS_ROOT = REPO_ROOT / "docs" / "technical"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DocsMetadata:
|
||||
filename: str
|
||||
slug: str
|
||||
access: DocsAccess
|
||||
group: str
|
||||
order: int
|
||||
zh_title: str
|
||||
en_title: str
|
||||
|
||||
|
||||
DOCS_METADATA: tuple[DocsMetadata, ...] = (
|
||||
DocsMetadata(DOCS_README_FILENAME, DEFAULT_DOCS_SLUG, "public", "Overview", 0, "技术文档", "Technical Docs"),
|
||||
DocsMetadata("quickstart.md", "quickstart", "public", "Manual", 1, "快速开始", "Quickstart"),
|
||||
DocsMetadata("manual.md", "manual", "public", "Manual", 2, "Planet 使用手册", "Planet Manual"),
|
||||
DocsMetadata("location-pipeline-user.md", "location-pipeline-user", "public", "Manual", 3, "Earth 位置候选采集使用手册", "Earth Location Candidate Collection User Guide"),
|
||||
DocsMetadata("earth-frontend-context.md", "earth-frontend-context", "docs_developer", "Earth", 10, "Earth 前端结构", "Earth Frontend Context"),
|
||||
DocsMetadata("earth-layer-style-reference.md", "earth-layer-style-reference", "docs_developer", "Earth", 11, "Earth 图层样式属性索引", "Earth Layer Style Reference"),
|
||||
DocsMetadata("earth-render-layer-order.md", "earth-render-layer-order", "docs_developer", "Earth", 12, "Earth 渲染图层顺序", "Earth Render Layer Order"),
|
||||
DocsMetadata("earth-satellite-footprint-policy.md", "earth-satellite-footprint-policy", "docs_developer", "Earth", 13, "Earth 卫星覆盖策略", "Earth Satellite Footprint Policy"),
|
||||
DocsMetadata("earth-bgp-context.md", "earth-bgp-context", "docs_developer", "Earth", 14, "BGP 态势上下文", "BGP Context"),
|
||||
DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Earth", 15, "新闻直播采集格式", "News Live Streams Collector Format"),
|
||||
DocsMetadata("earth-interactable-usage.md", "earth-interactable-usage", "docs_developer", "Earth", 16, "Earth 可交互图标接入", "Earth Interactable Usage"),
|
||||
DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 17, "Earth 工具栏与浮层协同", "Earth Toolbar and Overlay Coordination"),
|
||||
DocsMetadata("frontend-admin-frontend-context.md", "frontend-admin-frontend-context", "docs_developer", "Frontend", 20, "控制台前端结构", "Admin Frontend Context"),
|
||||
DocsMetadata("frontend-layout-guidelines.md", "frontend-layout-guidelines", "docs_developer", "Frontend", 21, "前端布局指南", "Frontend Layout Guidelines"),
|
||||
DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Frontend", 22, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
|
||||
DocsMetadata("backend-collectors.md", "backend-collectors", "docs_developer", "Backend", 30, "数据采集系统", "Data Collectors"),
|
||||
DocsMetadata("backend-system-service-control.md", "backend-system-service-control", "docs_admin", "Backend", 31, "系统服务控制", "System Service Control"),
|
||||
DocsMetadata("datasource-collector-settings-connectivity.md", "datasource-collector-settings-connectivity", "docs_developer", "Backend", 32, "数据源、采集器设置与连接验证", "Datasource Collector Settings and Connectivity"),
|
||||
DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"),
|
||||
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 34, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
|
||||
DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"),
|
||||
DocsMetadata("ops-docker-compose-buildx-upgrade.md", "ops-docker-compose-buildx-upgrade", "docs_admin", "Ops", 50, "Docker + Compose + Buildx 升级", "Docker + Compose + Buildx Upgrade"),
|
||||
DocsMetadata("ops-planet-sh-startup.md", "ops-planet-sh-startup", "docs_admin", "Ops", 51, "planet.sh 启动机制", "planet.sh Startup"),
|
||||
)
|
||||
|
||||
DOCS_BY_SLUG = {entry.slug: entry for entry in DOCS_METADATA}
|
||||
|
||||
|
||||
def get_user_gatekeeper_groups(user: User | None) -> set[str]:
|
||||
if user is None:
|
||||
return set()
|
||||
|
||||
role = user.role.value if hasattr(user.role, "value") else str(user.role or "")
|
||||
if role == "super_admin":
|
||||
return {"docs_user", "docs_developer", "docs_admin"}
|
||||
if role == "admin":
|
||||
return {"docs_user", "docs_developer", "docs_admin"}
|
||||
|
||||
groups = set()
|
||||
raw_groups = user.gatekeeper_groups or []
|
||||
if isinstance(raw_groups, list):
|
||||
groups.update(str(group) for group in raw_groups)
|
||||
|
||||
if "docs_admin" in groups:
|
||||
groups.update({"docs_developer", "docs_user"})
|
||||
if "docs_developer" in groups:
|
||||
groups.add("docs_user")
|
||||
return groups
|
||||
|
||||
|
||||
def can_read_doc(entry: DocsMetadata, user: User | None) -> bool:
|
||||
if entry.access == "public":
|
||||
return True
|
||||
return entry.access in get_user_gatekeeper_groups(user)
|
||||
|
||||
|
||||
def doc_path_for(entry: DocsMetadata, lang: str) -> Path:
|
||||
if lang not in VALID_DOCS_LANGS:
|
||||
raise ValueError("Unsupported docs language")
|
||||
return TECHNICAL_DOCS_ROOT / lang / entry.filename
|
||||
|
||||
|
||||
def title_for(entry: DocsMetadata, lang: str) -> str:
|
||||
return entry.zh_title if lang == "zh" else entry.en_title
|
||||
|
||||
|
||||
def catalog_for_user(user: User | None) -> list[dict]:
|
||||
items: list[dict] = []
|
||||
for entry in DOCS_METADATA:
|
||||
if not can_read_doc(entry, user):
|
||||
continue
|
||||
for lang in sorted(VALID_DOCS_LANGS):
|
||||
if not doc_path_for(entry, lang).exists():
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"slug": entry.slug,
|
||||
"filename": entry.filename,
|
||||
"lang": lang,
|
||||
"title": title_for(entry, lang),
|
||||
"group": entry.group,
|
||||
"order": entry.order,
|
||||
"access": entry.access,
|
||||
}
|
||||
)
|
||||
return sorted(items, key=lambda item: (item["lang"], item["order"], item["title"]))
|
||||
57
backend/app/services/location/__init__.py
Normal file
57
backend/app/services/location/__init__.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""Shared location-resolution pipeline.
|
||||
|
||||
A reusable abstraction for "given a record, decide its lat/lon" — used by
|
||||
compute centers, BGP collectors, BGP events, and any future entity that needs
|
||||
location estimation.
|
||||
|
||||
Each domain wires its own :class:`LocationPipeline` from a sequence of
|
||||
:class:`LocationResolver` instances. Future algorithms (peeringdb, IXP tables,
|
||||
user-confirmed coordinates, …) plug in by implementing the protocol — no
|
||||
changes needed to consumers.
|
||||
"""
|
||||
|
||||
from .models import (
|
||||
LocationCandidate,
|
||||
LocationQuery,
|
||||
ResolutionDiagnostic,
|
||||
ResolutionResult,
|
||||
ResolverOutput,
|
||||
)
|
||||
from .pipeline import LocationPipeline, LocationResolver
|
||||
from .resolvers.inherit import InheritFromAnotherEntityResolver
|
||||
from .resolvers.nominatim import (
|
||||
NominatimResolver,
|
||||
build_default_nominatim_geocoder,
|
||||
interpret_geocode_result,
|
||||
)
|
||||
from .resolvers.registry import RegistryResolver, default_score_alias_match
|
||||
from .resolvers.source_coordinates import SourceCoordinatesResolver
|
||||
from .text import (
|
||||
city_key,
|
||||
coerce_str,
|
||||
normalize_country_text,
|
||||
normalize_text,
|
||||
parse_float,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"LocationCandidate",
|
||||
"LocationPipeline",
|
||||
"LocationQuery",
|
||||
"LocationResolver",
|
||||
"ResolutionDiagnostic",
|
||||
"ResolutionResult",
|
||||
"ResolverOutput",
|
||||
"InheritFromAnotherEntityResolver",
|
||||
"NominatimResolver",
|
||||
"RegistryResolver",
|
||||
"SourceCoordinatesResolver",
|
||||
"build_default_nominatim_geocoder",
|
||||
"city_key",
|
||||
"coerce_str",
|
||||
"default_score_alias_match",
|
||||
"interpret_geocode_result",
|
||||
"normalize_country_text",
|
||||
"normalize_text",
|
||||
"parse_float",
|
||||
]
|
||||
126
backend/app/services/location/models.py
Normal file
126
backend/app/services/location/models.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""Domain-neutral data structures for the location pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Mapping
|
||||
|
||||
# Renderable precision tiers, ordered from most precise to least.
|
||||
RENDERABLE_PRECISIONS: tuple[str, ...] = ("precise", "site", "city")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LocationQuery:
|
||||
"""Domain-neutral input for the resolution pipeline.
|
||||
|
||||
``name`` and ``aliases`` are matched against registry alias indexes;
|
||||
``city`` / ``country`` / ``region`` provide geographic context for both
|
||||
registry lookups and Nominatim queries; ``source_latitude`` /
|
||||
``source_longitude`` short-circuit when the record already carries
|
||||
coordinates; ``extra`` carries domain-specific fields (operator, site,
|
||||
organization, asn, peer_ip, …) that resolvers can opt into.
|
||||
"""
|
||||
|
||||
name: str | None = None
|
||||
aliases: tuple[str, ...] = ()
|
||||
city: str | None = None
|
||||
country: str | None = None
|
||||
region: str | None = None
|
||||
source_latitude: float | None = None
|
||||
source_longitude: float | None = None
|
||||
extra: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LocationCandidate:
|
||||
"""A resolved location candidate produced by a resolver."""
|
||||
|
||||
latitude: float
|
||||
longitude: float
|
||||
display_name: str
|
||||
precision: str # "precise" | "site" | "city" | (rejected: country/unknown)
|
||||
confidence: float
|
||||
query: str
|
||||
source: str
|
||||
source_note: str | None
|
||||
matched_fields: tuple[str, ...]
|
||||
needs_confirmation: bool
|
||||
city: str | None = None
|
||||
region: str | None = None
|
||||
country: str | None = None
|
||||
matched_location_name: str | None = None
|
||||
location_verified_at: str | None = None
|
||||
suggested_registry_entry: dict[str, Any] | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"latitude": self.latitude,
|
||||
"longitude": self.longitude,
|
||||
"display_name": self.display_name,
|
||||
"precision": self.precision,
|
||||
"confidence": self.confidence,
|
||||
"query": self.query,
|
||||
"source": self.source,
|
||||
"source_note": self.source_note,
|
||||
"matched_fields": list(self.matched_fields),
|
||||
"needs_confirmation": self.needs_confirmation,
|
||||
"city": self.city,
|
||||
"region": self.region,
|
||||
"country": self.country,
|
||||
"matched_location_name": self.matched_location_name,
|
||||
"location_verified_at": self.location_verified_at,
|
||||
"suggested_registry_entry": self.suggested_registry_entry,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolverOutput:
|
||||
"""What a single resolver returns from one ``resolve()`` call."""
|
||||
|
||||
candidates: tuple[LocationCandidate, ...] = ()
|
||||
attempted_queries: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolutionDiagnostic:
|
||||
"""Why we could not resolve, plus what we tried."""
|
||||
|
||||
failure_reason: str
|
||||
attempted_queries: tuple[str, ...] = ()
|
||||
record_id: int | None = None
|
||||
source: str | None = None
|
||||
source_id: str | None = None
|
||||
name: str | None = None
|
||||
country: str | None = None
|
||||
city: str | None = None
|
||||
site: str | None = None
|
||||
operator: str | None = None
|
||||
extra: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"failure_reason": self.failure_reason,
|
||||
"attempted_queries": list(self.attempted_queries),
|
||||
"record_id": self.record_id,
|
||||
"source": self.source,
|
||||
"source_id": self.source_id,
|
||||
"name": self.name,
|
||||
"country": self.country,
|
||||
"city": self.city,
|
||||
"site": self.site,
|
||||
"operator": self.operator,
|
||||
**({"extra": dict(self.extra)} if self.extra else {}),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolutionResult:
|
||||
"""Pipeline output: best candidate (if any) + diagnostic on miss."""
|
||||
|
||||
location: LocationCandidate | None
|
||||
diagnostic: ResolutionDiagnostic | None
|
||||
attempted_queries: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def is_resolved(self) -> bool:
|
||||
return bool(self.location)
|
||||
126
backend/app/services/location/pipeline.py
Normal file
126
backend/app/services/location/pipeline.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""Pipeline that runs a sequence of :class:`LocationResolver` instances."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol, Sequence
|
||||
|
||||
from .models import (
|
||||
LocationCandidate,
|
||||
LocationQuery,
|
||||
ResolutionDiagnostic,
|
||||
ResolutionResult,
|
||||
ResolverOutput,
|
||||
)
|
||||
|
||||
|
||||
class LocationResolver(Protocol):
|
||||
"""Pluggable location resolution step.
|
||||
|
||||
Implementations: ``SourceCoordinatesResolver``, ``RegistryResolver``,
|
||||
``NominatimResolver``, ``InheritFromAnotherEntityResolver`` — see the
|
||||
``resolvers`` subpackage. New algorithms (peeringdb / IXP / user-confirmed
|
||||
coordinates) plug in by implementing this protocol; the pipeline does not
|
||||
care how candidates are produced.
|
||||
"""
|
||||
|
||||
name: str
|
||||
|
||||
def resolve(self, query: LocationQuery) -> ResolverOutput: ...
|
||||
|
||||
|
||||
def default_candidate_sort_key(
|
||||
candidate: LocationCandidate,
|
||||
) -> tuple[int, int, float]:
|
||||
precision_rank = {"precise": 0, "site": 1, "city": 2}.get(
|
||||
candidate.precision, 9
|
||||
)
|
||||
source_rank = {
|
||||
"source_coordinates": 0,
|
||||
"stored_compute_center_location": 1,
|
||||
"stored_collector_location": 1,
|
||||
"ror_organization_registry": 2,
|
||||
"inherited": 3,
|
||||
"nominatim_online_geocode": 4,
|
||||
"local_registry": 8,
|
||||
"local_registry_city": 9,
|
||||
}.get(candidate.source, 9)
|
||||
return (source_rank, precision_rank, -float(candidate.confidence or 0))
|
||||
|
||||
|
||||
class LocationPipeline:
|
||||
"""Orchestrate a sequence of resolvers.
|
||||
|
||||
``collect_candidates`` runs every resolver and returns *all* deduped
|
||||
candidates plus the queries each resolver attempted (useful for
|
||||
user-facing "why didn't this work?" diagnostics).
|
||||
|
||||
``resolve_best`` returns the top candidate per
|
||||
:func:`default_candidate_sort_key` (or a custom sort).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
resolvers: Sequence[LocationResolver],
|
||||
*,
|
||||
sort_key=default_candidate_sort_key,
|
||||
failure_reason: str = (
|
||||
"Could not resolve to renderable coordinates from any configured resolver."
|
||||
),
|
||||
) -> None:
|
||||
self._resolvers = list(resolvers)
|
||||
self._sort_key = sort_key
|
||||
self._failure_reason = failure_reason
|
||||
|
||||
@property
|
||||
def resolvers(self) -> tuple[LocationResolver, ...]:
|
||||
return tuple(self._resolvers)
|
||||
|
||||
def collect_candidates(
|
||||
self, query: LocationQuery
|
||||
) -> tuple[list[LocationCandidate], list[str]]:
|
||||
candidates: list[LocationCandidate] = []
|
||||
attempted: list[str] = []
|
||||
seen_keys: set[tuple[str, str, str]] = set()
|
||||
|
||||
for resolver in self._resolvers:
|
||||
output = resolver.resolve(query)
|
||||
for q in output.attempted_queries:
|
||||
if q and q not in attempted:
|
||||
attempted.append(q)
|
||||
for candidate in output.candidates:
|
||||
key = (
|
||||
candidate.source,
|
||||
f"{candidate.latitude:.4f}",
|
||||
f"{candidate.longitude:.4f}",
|
||||
)
|
||||
if key in seen_keys:
|
||||
continue
|
||||
seen_keys.add(key)
|
||||
candidates.append(candidate)
|
||||
|
||||
candidates.sort(key=self._sort_key)
|
||||
return candidates, attempted
|
||||
|
||||
def resolve_best(self, query: LocationQuery) -> ResolutionResult:
|
||||
candidates, attempted = self.collect_candidates(query)
|
||||
if candidates:
|
||||
return ResolutionResult(
|
||||
location=candidates[0],
|
||||
diagnostic=None,
|
||||
attempted_queries=tuple(attempted),
|
||||
)
|
||||
return ResolutionResult(
|
||||
location=None,
|
||||
diagnostic=ResolutionDiagnostic(
|
||||
failure_reason=self._failure_reason,
|
||||
attempted_queries=tuple(attempted),
|
||||
name=query.name,
|
||||
country=query.country,
|
||||
city=query.city,
|
||||
site=str(query.extra.get("site")) if query.extra.get("site") else None,
|
||||
operator=str(query.extra.get("operator"))
|
||||
if query.extra.get("operator")
|
||||
else None,
|
||||
),
|
||||
attempted_queries=tuple(attempted),
|
||||
)
|
||||
20
backend/app/services/location/resolvers/__init__.py
Normal file
20
backend/app/services/location/resolvers/__init__.py
Normal file
@@ -0,0 +1,20 @@
|
||||
"""Built-in resolver implementations."""
|
||||
|
||||
from .inherit import InheritFromAnotherEntityResolver
|
||||
from .nominatim import (
|
||||
NominatimResolver,
|
||||
build_default_nominatim_geocoder,
|
||||
interpret_geocode_result,
|
||||
)
|
||||
from .registry import RegistryResolver, default_score_alias_match
|
||||
from .source_coordinates import SourceCoordinatesResolver
|
||||
|
||||
__all__ = [
|
||||
"InheritFromAnotherEntityResolver",
|
||||
"NominatimResolver",
|
||||
"RegistryResolver",
|
||||
"SourceCoordinatesResolver",
|
||||
"build_default_nominatim_geocoder",
|
||||
"default_score_alias_match",
|
||||
"interpret_geocode_result",
|
||||
]
|
||||
31
backend/app/services/location/resolvers/inherit.py
Normal file
31
backend/app/services/location/resolvers/inherit.py
Normal file
@@ -0,0 +1,31 @@
|
||||
"""Resolver that inherits a candidate from another entity's resolution.
|
||||
|
||||
Used by BGP events to pick up the location of their owning collector. The
|
||||
``source_lookup`` callable is the only domain coupling — it receives the
|
||||
incoming :class:`LocationQuery` and returns either an already-resolved
|
||||
:class:`LocationCandidate` (typically by querying another pipeline) or
|
||||
``None`` to signal "no parent location available".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable
|
||||
|
||||
from ..models import LocationCandidate, LocationQuery, ResolverOutput
|
||||
|
||||
|
||||
class InheritFromAnotherEntityResolver:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
source_lookup: Callable[[LocationQuery], LocationCandidate | None],
|
||||
name: str = "inherited",
|
||||
) -> None:
|
||||
self.name = name
|
||||
self._lookup = source_lookup
|
||||
|
||||
def resolve(self, query: LocationQuery) -> ResolverOutput:
|
||||
result = self._lookup(query)
|
||||
if result is None:
|
||||
return ResolverOutput()
|
||||
return ResolverOutput(candidates=(result,))
|
||||
292
backend/app/services/location/resolvers/nominatim.py
Normal file
292
backend/app/services/location/resolvers/nominatim.py
Normal file
@@ -0,0 +1,292 @@
|
||||
"""Nominatim-backed online geocoder.
|
||||
|
||||
The actual HTTP call is encapsulated in :func:`build_default_nominatim_geocoder`
|
||||
which returns an ``lru_cache``-wrapped function. Domain modules typically:
|
||||
|
||||
1. Build a default geocoder via :func:`build_default_nominatim_geocoder`.
|
||||
2. Re-export it under a stable module-level name (e.g. ``_geocode_online``).
|
||||
3. Pass a *late-binding lambda* (``lambda q: _geocode_online(q)``) to
|
||||
:class:`NominatimResolver`.
|
||||
|
||||
This ensures tests that ``monkeypatch.setattr(module, "_geocode_online", ...)``
|
||||
can swap the geocoder behavior without touching pipeline construction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from functools import lru_cache
|
||||
from typing import Any, Callable
|
||||
|
||||
import httpx
|
||||
|
||||
from ..models import LocationCandidate, LocationQuery, ResolverOutput
|
||||
from ..text import (
|
||||
coerce_str,
|
||||
normalize_country_text,
|
||||
normalize_text,
|
||||
parse_float,
|
||||
)
|
||||
|
||||
NOMINATIM_SEARCH_URL = "https://nominatim.openstreetmap.org/search"
|
||||
DEFAULT_USER_AGENT = "planet-earth-location-resolver/1.0"
|
||||
DEFAULT_MIN_INTERVAL_SECONDS = 1.1
|
||||
DEFAULT_TIMEOUT_SECONDS = 8.0
|
||||
|
||||
|
||||
def build_default_nominatim_geocoder(
|
||||
*,
|
||||
user_agent: str = DEFAULT_USER_AGENT,
|
||||
min_interval_seconds: float = DEFAULT_MIN_INTERVAL_SECONDS,
|
||||
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
|
||||
cache_size: int = 512,
|
||||
) -> Callable[[str], dict[str, Any] | None]:
|
||||
"""Return a cached, rate-limited Nominatim geocoder."""
|
||||
|
||||
last_request_at = [0.0]
|
||||
|
||||
@lru_cache(maxsize=cache_size)
|
||||
def geocode(query: str) -> dict[str, Any] | None:
|
||||
if not query:
|
||||
return None
|
||||
elapsed = time.monotonic() - last_request_at[0]
|
||||
if elapsed < min_interval_seconds:
|
||||
time.sleep(min_interval_seconds - elapsed)
|
||||
last_request_at[0] = time.monotonic()
|
||||
response = httpx.get(
|
||||
NOMINATIM_SEARCH_URL,
|
||||
params={
|
||||
"q": query,
|
||||
"format": "jsonv2",
|
||||
"limit": 1,
|
||||
"addressdetails": 1,
|
||||
},
|
||||
headers={"User-Agent": user_agent},
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not isinstance(payload, list) or not payload:
|
||||
return None
|
||||
result = payload[0]
|
||||
if not isinstance(result, dict):
|
||||
return None
|
||||
return result
|
||||
|
||||
return geocode
|
||||
|
||||
|
||||
_DEFAULT_SITE_CATEGORIES = frozenset(
|
||||
{
|
||||
"amenity",
|
||||
"office",
|
||||
"building",
|
||||
"industrial",
|
||||
"research",
|
||||
"university",
|
||||
"education",
|
||||
"tourism",
|
||||
"shop",
|
||||
"man_made",
|
||||
"campus",
|
||||
"research_institute",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def interpret_geocode_result(
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
matched_fields: tuple[str, ...],
|
||||
context_country: str | None,
|
||||
site_categories: frozenset[str] = _DEFAULT_SITE_CATEGORIES,
|
||||
site_promoting_match_fields: frozenset[str] = frozenset(
|
||||
{"site", "operator", "name"}
|
||||
),
|
||||
) -> tuple[float, float, dict[str, Any], str] | None:
|
||||
"""Validate a Nominatim raw result. Returns (lat, lon, address, classification)."""
|
||||
latitude = parse_float(result.get("lat"))
|
||||
longitude = parse_float(result.get("lon"))
|
||||
if latitude in (None, 0.0) or longitude in (None, 0.0):
|
||||
return None
|
||||
|
||||
address = result.get("address") if isinstance(result.get("address"), dict) else {}
|
||||
if not isinstance(address, dict):
|
||||
address = {}
|
||||
|
||||
has_city_level = bool(
|
||||
address.get("city")
|
||||
or address.get("town")
|
||||
or address.get("village")
|
||||
or address.get("municipality")
|
||||
or address.get("hamlet")
|
||||
or address.get("suburb")
|
||||
)
|
||||
osm_class = str(result.get("class") or "").lower()
|
||||
osm_type = str(result.get("type") or "").lower()
|
||||
is_site_like = osm_class in site_categories or osm_type in site_categories
|
||||
if not has_city_level and not is_site_like:
|
||||
return None
|
||||
|
||||
if context_country:
|
||||
normalized_context = normalize_text(normalize_country_text(context_country))
|
||||
normalized_result = normalize_text(
|
||||
normalize_country_text(address.get("country"))
|
||||
)
|
||||
if (
|
||||
normalized_context
|
||||
and normalized_result
|
||||
and normalized_context != normalized_result
|
||||
):
|
||||
return None
|
||||
|
||||
classification = (
|
||||
"site"
|
||||
if (
|
||||
is_site_like
|
||||
and has_city_level
|
||||
and any(field in site_promoting_match_fields for field in matched_fields)
|
||||
)
|
||||
else "city"
|
||||
)
|
||||
return float(latitude), float(longitude), address, classification
|
||||
|
||||
|
||||
def _candidate_from_geocode(
|
||||
*,
|
||||
query: LocationQuery,
|
||||
geocode_query: str,
|
||||
matched_fields: tuple[str, ...],
|
||||
raw_result: dict[str, Any],
|
||||
interpret: Callable[..., tuple[float, float, dict[str, Any], str] | None],
|
||||
source: str,
|
||||
site_confidence: float,
|
||||
city_confidence: float,
|
||||
) -> LocationCandidate | None:
|
||||
interpreted = interpret(
|
||||
raw_result,
|
||||
matched_fields=matched_fields,
|
||||
context_country=query.country,
|
||||
)
|
||||
if not interpreted:
|
||||
return None
|
||||
latitude, longitude, address, classification = interpreted
|
||||
city = (
|
||||
address.get("city")
|
||||
or address.get("town")
|
||||
or address.get("village")
|
||||
or address.get("municipality")
|
||||
or query.city
|
||||
or None
|
||||
)
|
||||
region = address.get("state") or address.get("region")
|
||||
country = address.get("country") or query.country or None
|
||||
display_name = raw_result.get("display_name") or geocode_query
|
||||
confidence = city_confidence if classification == "city" else site_confidence
|
||||
|
||||
extra = query.extra or {}
|
||||
suggested_registry_entry = {
|
||||
"canonical_name": (
|
||||
(query.aliases[0] if query.aliases else None)
|
||||
or query.name
|
||||
or display_name
|
||||
),
|
||||
"aliases": list(
|
||||
{
|
||||
value
|
||||
for value in [
|
||||
query.name,
|
||||
*query.aliases,
|
||||
coerce_str(extra.get("operator")),
|
||||
coerce_str(extra.get("site")),
|
||||
]
|
||||
if value
|
||||
}
|
||||
),
|
||||
"operator": coerce_str(extra.get("operator")) or None,
|
||||
"site": coerce_str(extra.get("site"))
|
||||
or coerce_str(extra.get("organization"))
|
||||
or None,
|
||||
"country": country,
|
||||
"city": city,
|
||||
"region": region,
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"precision": classification,
|
||||
"confidence": confidence,
|
||||
"source_note": (
|
||||
f"Resolved via Nominatim query '{geocode_query}' → {display_name}"
|
||||
),
|
||||
}
|
||||
return LocationCandidate(
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
display_name=display_name,
|
||||
precision=classification,
|
||||
confidence=confidence,
|
||||
query=geocode_query,
|
||||
source=source,
|
||||
source_note=f"Nominatim search result: {display_name}",
|
||||
matched_fields=matched_fields,
|
||||
needs_confirmation=True,
|
||||
city=city,
|
||||
region=region,
|
||||
country=country,
|
||||
matched_location_name=display_name,
|
||||
location_verified_at=None,
|
||||
suggested_registry_entry=suggested_registry_entry,
|
||||
)
|
||||
|
||||
|
||||
class NominatimResolver:
|
||||
"""Run a domain-specific query plan against Nominatim."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
query_plan_builder: Callable[
|
||||
[LocationQuery], list[tuple[str, tuple[str, ...]]]
|
||||
],
|
||||
geocoder: Callable[[str], dict[str, Any] | None],
|
||||
name: str = "nominatim_online_geocode",
|
||||
site_confidence: float = 0.72,
|
||||
city_confidence: float = 0.62,
|
||||
interpret: Callable[..., tuple[float, float, dict[str, Any], str] | None] = (
|
||||
interpret_geocode_result
|
||||
),
|
||||
) -> None:
|
||||
self.name = name
|
||||
self._query_plan_builder = query_plan_builder
|
||||
self._geocoder = geocoder
|
||||
self._site_confidence = site_confidence
|
||||
self._city_confidence = city_confidence
|
||||
self._interpret = interpret
|
||||
|
||||
def resolve(self, query: LocationQuery) -> ResolverOutput:
|
||||
plan = self._query_plan_builder(query)
|
||||
candidates: list[LocationCandidate] = []
|
||||
attempted: list[str] = []
|
||||
for geocode_query, matched_fields in plan:
|
||||
attempted.append(geocode_query)
|
||||
try:
|
||||
raw_result = self._geocoder(geocode_query)
|
||||
except Exception:
|
||||
continue
|
||||
if not raw_result:
|
||||
continue
|
||||
candidate = _candidate_from_geocode(
|
||||
query=query,
|
||||
geocode_query=geocode_query,
|
||||
matched_fields=matched_fields,
|
||||
raw_result=raw_result,
|
||||
interpret=self._interpret,
|
||||
source=self.name,
|
||||
site_confidence=self._site_confidence,
|
||||
city_confidence=self._city_confidence,
|
||||
)
|
||||
if candidate is not None:
|
||||
candidates.append(candidate)
|
||||
return ResolverOutput(
|
||||
candidates=tuple(candidates),
|
||||
attempted_queries=tuple(attempted),
|
||||
)
|
||||
323
backend/app/services/location/resolvers/registry.py
Normal file
323
backend/app/services/location/resolvers/registry.py
Normal file
@@ -0,0 +1,323 @@
|
||||
"""Resolver that matches a query against a local JSON registry.
|
||||
|
||||
Registry schema (a single JSON file):
|
||||
|
||||
{
|
||||
"locations": [
|
||||
{
|
||||
"canonical_name": "...",
|
||||
"aliases": ["...", "..."],
|
||||
"operator": "...",
|
||||
"site": "...",
|
||||
"city": "...",
|
||||
"country": "...",
|
||||
"region": "...",
|
||||
"latitude": 0.0,
|
||||
"longitude": 0.0,
|
||||
"precision": "precise" | "site" | "city",
|
||||
"confidence": 0.0,
|
||||
"verification_status": "verified",
|
||||
"source_note": "...",
|
||||
"verified_at": "YYYY-MM-DD"
|
||||
}
|
||||
],
|
||||
"city_fallbacks": [ {city, country, latitude, longitude, ...} ]
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
from ..models import (
|
||||
RENDERABLE_PRECISIONS,
|
||||
LocationCandidate,
|
||||
LocationQuery,
|
||||
ResolverOutput,
|
||||
)
|
||||
from ..text import (
|
||||
city_key,
|
||||
normalize_country_text,
|
||||
normalize_text,
|
||||
parse_float,
|
||||
)
|
||||
|
||||
# Field-priority weights when scoring "this query field text contains this
|
||||
# alias text". Tuned to match the legacy compute-center ordering — name beats
|
||||
# site beats operator beats city — which generalizes well to other domains.
|
||||
_DEFAULT_FIELD_PRIORITY = {
|
||||
"name": 8,
|
||||
"site": 6,
|
||||
"operator": 5,
|
||||
"city": 3,
|
||||
}
|
||||
|
||||
|
||||
def default_score_alias_match(
|
||||
alias_field: str, record_field: str, alias_text: str
|
||||
) -> int:
|
||||
score = max(0, len(alias_text))
|
||||
score += _DEFAULT_FIELD_PRIORITY.get(alias_field, 1)
|
||||
if alias_field == record_field:
|
||||
score += 4
|
||||
if alias_field == "name" and record_field in {"name", "name_short", "alias"}:
|
||||
score += 6
|
||||
if alias_field == "site" and record_field in {"site", "organization"}:
|
||||
score += 4
|
||||
if alias_field == "operator" and record_field in {"operator", "organization"}:
|
||||
score += 4
|
||||
return score
|
||||
|
||||
|
||||
@lru_cache(maxsize=32)
|
||||
def _load_registry_file(path: str) -> dict[str, Any]:
|
||||
with Path(path).open("r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
@lru_cache(maxsize=32)
|
||||
def _build_alias_index(
|
||||
path: str,
|
||||
) -> tuple[tuple[dict[str, Any], tuple[tuple[str, str], ...]], ...]:
|
||||
index: list[tuple[dict[str, Any], tuple[tuple[str, str], ...]]] = []
|
||||
for entry in _load_registry_file(path).get("locations", []):
|
||||
aliases: list[tuple[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for alias in [entry.get("canonical_name"), *(entry.get("aliases") or [])]:
|
||||
normalized = normalize_text(alias)
|
||||
if normalized and normalized not in seen:
|
||||
aliases.append(("name", normalized))
|
||||
seen.add(normalized)
|
||||
for field_name in ("operator", "site", "city"):
|
||||
value = entry.get(field_name)
|
||||
normalized = normalize_text(value)
|
||||
if normalized and normalized not in seen:
|
||||
aliases.append((field_name, normalized))
|
||||
seen.add(normalized)
|
||||
index.append((entry, tuple(aliases)))
|
||||
return tuple(index)
|
||||
|
||||
|
||||
def _query_corpus(query: LocationQuery) -> dict[str, str]:
|
||||
"""Map a query into normalized strings keyed by source field."""
|
||||
fields: dict[str, str] = {
|
||||
"name": query.name or "",
|
||||
"city": query.city or "",
|
||||
"country": query.country or "",
|
||||
}
|
||||
for alias in query.aliases:
|
||||
if alias and alias != query.name:
|
||||
fields["name_short"] = alias
|
||||
break
|
||||
extra = query.extra or {}
|
||||
for key in ("site", "operator", "organization"):
|
||||
value = extra.get(key)
|
||||
if value:
|
||||
fields[key] = str(value)
|
||||
return {key: normalize_text(value) for key, value in fields.items() if value}
|
||||
|
||||
|
||||
def _country_compatible(entry: dict[str, Any], query: LocationQuery) -> bool:
|
||||
record_country = normalize_country_text(query.country)
|
||||
entry_country = normalize_country_text(entry.get("country"))
|
||||
if not record_country or not entry_country:
|
||||
return True
|
||||
return normalize_text(record_country) == normalize_text(entry_country)
|
||||
|
||||
|
||||
def _normalized_alias_matches(alias_normalized: str, record_text: str) -> bool:
|
||||
alias_tokens = alias_normalized.split()
|
||||
record_tokens = record_text.split()
|
||||
if not alias_tokens or not record_tokens:
|
||||
return False
|
||||
if len(alias_tokens) == 1:
|
||||
return alias_tokens[0] in record_tokens
|
||||
window_size = len(alias_tokens)
|
||||
return any(
|
||||
record_tokens[index : index + window_size] == alias_tokens
|
||||
for index in range(0, len(record_tokens) - window_size + 1)
|
||||
)
|
||||
|
||||
|
||||
def _entry_to_candidate(
|
||||
entry: dict[str, Any],
|
||||
*,
|
||||
matched_alias: str,
|
||||
matched_fields: Iterable[str],
|
||||
source: str,
|
||||
score_explainer: str,
|
||||
confidence_floor: float,
|
||||
) -> LocationCandidate:
|
||||
canonical_name = entry.get("canonical_name") or matched_alias
|
||||
# Registry entries are treated as candidates unless explicitly verified.
|
||||
# This prevents migrated hard-coded hints from appearing as factual
|
||||
# location evidence.
|
||||
is_verified = entry.get("verification_status") == "verified"
|
||||
precision = entry.get("precision") or "city"
|
||||
if precision not in RENDERABLE_PRECISIONS:
|
||||
precision = "city"
|
||||
fields_summary = ", ".join(sorted(set(matched_fields))) or "name"
|
||||
confidence_value = parse_float(entry.get("confidence"))
|
||||
confidence = (
|
||||
float(confidence_value)
|
||||
if confidence_value is not None
|
||||
else confidence_floor
|
||||
)
|
||||
return LocationCandidate(
|
||||
latitude=float(parse_float(entry.get("latitude")) or 0.0),
|
||||
longitude=float(parse_float(entry.get("longitude")) or 0.0),
|
||||
display_name=canonical_name,
|
||||
precision=precision,
|
||||
confidence=confidence,
|
||||
query=f"local_registry::{matched_alias or canonical_name}",
|
||||
source=source,
|
||||
source_note=entry.get("source_note")
|
||||
or f"{score_explainer}: matched {fields_summary}",
|
||||
matched_fields=tuple(sorted(set(matched_fields))) or ("name",),
|
||||
needs_confirmation=bool(entry.get("needs_confirmation")) or not is_verified,
|
||||
city=entry.get("city"),
|
||||
region=entry.get("region"),
|
||||
country=entry.get("country"),
|
||||
matched_location_name=canonical_name,
|
||||
location_verified_at=entry.get("verified_at") if is_verified else None,
|
||||
suggested_registry_entry=None,
|
||||
)
|
||||
|
||||
|
||||
class RegistryResolver:
|
||||
"""Match a query against a JSON registry (plus its city_fallbacks table)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
registry_path: Path | str,
|
||||
name: str = "local_registry",
|
||||
city_fallback_source: str = "local_registry_city",
|
||||
city_fallback_confidence_default: float = 0.65,
|
||||
confidence_default: float = 0.85,
|
||||
score_alias_match: Callable[[str, str, str], int] = default_score_alias_match,
|
||||
) -> None:
|
||||
self.name = name
|
||||
self._registry_path = str(Path(registry_path))
|
||||
self._city_fallback_source = city_fallback_source
|
||||
self._city_fallback_confidence_default = city_fallback_confidence_default
|
||||
self._confidence_default = confidence_default
|
||||
self._score = score_alias_match
|
||||
|
||||
def reload(self) -> None:
|
||||
"""Drop the cached registry — useful when the JSON file is edited."""
|
||||
_load_registry_file.cache_clear()
|
||||
_build_alias_index.cache_clear()
|
||||
|
||||
def resolve(self, query: LocationQuery) -> ResolverOutput:
|
||||
candidates: list[LocationCandidate] = []
|
||||
candidates.extend(self._registry_candidates(query))
|
||||
city_candidate = self._city_fallback_candidate(query)
|
||||
if city_candidate is not None:
|
||||
candidates.append(city_candidate)
|
||||
return ResolverOutput(candidates=tuple(candidates))
|
||||
|
||||
# ── internals ──────────────────────────────────────────────
|
||||
|
||||
def _registry_candidates(
|
||||
self, query: LocationQuery
|
||||
) -> list[LocationCandidate]:
|
||||
corpus = _query_corpus(query)
|
||||
if not corpus:
|
||||
return []
|
||||
|
||||
# When the query carries a name (a record-specific identifier), require
|
||||
# at least one alias match against a name-class field — otherwise a
|
||||
# generic shared field like operator="RIPE NCC" would promote every
|
||||
# registry entry that lists that operator, regardless of whether the
|
||||
# name matches.
|
||||
query_has_name = bool(corpus.get("name") or corpus.get("name_short"))
|
||||
|
||||
results: list[LocationCandidate] = []
|
||||
for entry, aliases in _build_alias_index(self._registry_path):
|
||||
best_alias = ""
|
||||
best_score = 0
|
||||
matched_fields: list[str] = []
|
||||
matched_via_name_alias = False
|
||||
for alias_field, alias_normalized in aliases:
|
||||
for record_field, record_text in corpus.items():
|
||||
if not _normalized_alias_matches(alias_normalized, record_text):
|
||||
continue
|
||||
score = self._score(
|
||||
alias_field, record_field, alias_normalized
|
||||
)
|
||||
if score > best_score or (
|
||||
score == best_score
|
||||
and len(alias_normalized) > len(best_alias)
|
||||
):
|
||||
best_score = score
|
||||
best_alias = alias_normalized
|
||||
if record_field not in matched_fields:
|
||||
matched_fields.append(record_field)
|
||||
if alias_field == "name" and record_field in {"name", "name_short"}:
|
||||
matched_via_name_alias = True
|
||||
if not matched_fields or best_score <= 0:
|
||||
continue
|
||||
if query_has_name and not matched_via_name_alias:
|
||||
continue
|
||||
if not _country_compatible(entry, query):
|
||||
continue
|
||||
results.append(
|
||||
_entry_to_candidate(
|
||||
entry,
|
||||
matched_alias=best_alias,
|
||||
matched_fields=matched_fields,
|
||||
source=self.name,
|
||||
score_explainer="Registry alias match",
|
||||
confidence_floor=self._confidence_default,
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
def _city_fallback_candidate(
|
||||
self, query: LocationQuery
|
||||
) -> LocationCandidate | None:
|
||||
country = normalize_country_text(query.country)
|
||||
city = city_key(query.city)
|
||||
if not country or not city:
|
||||
return None
|
||||
|
||||
for fallback in _load_registry_file(self._registry_path).get(
|
||||
"city_fallbacks", []
|
||||
):
|
||||
fallback_country = normalize_country_text(fallback.get("country"))
|
||||
fallback_city = city_key(fallback.get("city"))
|
||||
if fallback_country != country or fallback_city != city:
|
||||
continue
|
||||
confidence_value = parse_float(fallback.get("confidence"))
|
||||
confidence = (
|
||||
float(confidence_value)
|
||||
if confidence_value is not None
|
||||
else self._city_fallback_confidence_default
|
||||
)
|
||||
return LocationCandidate(
|
||||
latitude=float(parse_float(fallback.get("latitude")) or 0.0),
|
||||
longitude=float(parse_float(fallback.get("longitude")) or 0.0),
|
||||
display_name=fallback.get("city") or "",
|
||||
precision="city",
|
||||
confidence=confidence,
|
||||
query=(
|
||||
f"city_fallback::{fallback.get('city')}, "
|
||||
f"{fallback.get('country')}"
|
||||
),
|
||||
source=self._city_fallback_source,
|
||||
source_note=fallback.get("source_note")
|
||||
or f"City fallback for {fallback.get('city')}, {fallback.get('country')}",
|
||||
matched_fields=("city", "country"),
|
||||
needs_confirmation=False,
|
||||
city=fallback.get("city"),
|
||||
region=fallback.get("region"),
|
||||
country=fallback.get("country"),
|
||||
matched_location_name=fallback.get("city"),
|
||||
location_verified_at=fallback.get("verified_at"),
|
||||
suggested_registry_entry=None,
|
||||
)
|
||||
return None
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Resolver that consumes lat/lon already present on the source record."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..models import LocationCandidate, LocationQuery, ResolverOutput
|
||||
from ..text import normalize_country_text
|
||||
|
||||
|
||||
class SourceCoordinatesResolver:
|
||||
"""Pass-through for records that already carry valid coordinates."""
|
||||
|
||||
name = "source_coordinates"
|
||||
|
||||
def __init__(self, *, source: str = "source_coordinates") -> None:
|
||||
self._source = source
|
||||
|
||||
def resolve(self, query: LocationQuery) -> ResolverOutput:
|
||||
lat = query.source_latitude
|
||||
lon = query.source_longitude
|
||||
if lat in (None, 0.0) or lon in (None, 0.0):
|
||||
return ResolverOutput()
|
||||
|
||||
country = normalize_country_text(query.country) or query.country
|
||||
candidate = LocationCandidate(
|
||||
latitude=float(lat),
|
||||
longitude=float(lon),
|
||||
display_name=query.name or "",
|
||||
precision="precise",
|
||||
confidence=1.0,
|
||||
query="source_coordinates",
|
||||
source=self._source,
|
||||
source_note="Source record provided valid coordinates.",
|
||||
matched_fields=("source_coordinates",),
|
||||
needs_confirmation=False,
|
||||
city=query.city,
|
||||
region=query.region,
|
||||
country=country,
|
||||
matched_location_name=query.name,
|
||||
location_verified_at=None,
|
||||
suggested_registry_entry=None,
|
||||
)
|
||||
return ResolverOutput(candidates=(candidate,))
|
||||
41
backend/app/services/location/text.py
Normal file
41
backend/app/services/location/text.py
Normal file
@@ -0,0 +1,41 @@
|
||||
"""Text-normalization helpers shared by every resolver."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from app.core.countries import normalize_country
|
||||
|
||||
|
||||
def parse_float(value: Any) -> float | None:
|
||||
try:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def coerce_str(value: Any) -> str:
|
||||
if value in (None, ""):
|
||||
return ""
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def normalize_text(value: Any) -> str:
|
||||
if value in (None, ""):
|
||||
return ""
|
||||
normalized = str(value).casefold()
|
||||
normalized = re.sub(r"[^a-z0-9一-鿿]+", " ", normalized)
|
||||
return re.sub(r"\s+", " ", normalized).strip()
|
||||
|
||||
|
||||
def normalize_country_text(value: Any) -> str:
|
||||
normalized = normalize_country(value)
|
||||
return normalized or coerce_str(value)
|
||||
|
||||
|
||||
def city_key(city: Any) -> str:
|
||||
text = coerce_str(city).split(",", 1)[0]
|
||||
return normalize_text(text)
|
||||
Reference in New Issue
Block a user