linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.49.0

Browse Source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
linkong
2026-05-08 17:42:27 +08:00
parent bb9183b8a4
commit e1984c7a35
86 changed files with 9165 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
backend/app/services/location/resolvers/__init__.py Normal file
View File

@@ -0,0 +1,20 @@
"""Built-in resolver implementations."""
from .inherit import InheritFromAnotherEntityResolver
from .nominatim import (
NominatimResolver,
build_default_nominatim_geocoder,
interpret_geocode_result,
)
from .registry import RegistryResolver, default_score_alias_match
from .source_coordinates import SourceCoordinatesResolver
__all__ = [
"InheritFromAnotherEntityResolver",
"NominatimResolver",
"RegistryResolver",
"SourceCoordinatesResolver",
"build_default_nominatim_geocoder",
"default_score_alias_match",
"interpret_geocode_result",
]

31
backend/app/services/location/resolvers/inherit.py Normal file
View File

@@ -0,0 +1,31 @@
"""Resolver that inherits a candidate from another entity's resolution.
Used by BGP events to pick up the location of their owning collector. The
``source_lookup`` callable is the only domain coupling — it receives the
incoming :class:`LocationQuery` and returns either an already-resolved
:class:`LocationCandidate` (typically by querying another pipeline) or
``None`` to signal "no parent location available".
"""
from __future__ import annotations
from typing import Callable
from ..models import LocationCandidate, LocationQuery, ResolverOutput
class InheritFromAnotherEntityResolver:
def __init__(
self,
*,
source_lookup: Callable[[LocationQuery], LocationCandidate | None],
name: str = "inherited",
) -> None:
self.name = name
self._lookup = source_lookup
def resolve(self, query: LocationQuery) -> ResolverOutput:
result = self._lookup(query)
if result is None:
return ResolverOutput()
return ResolverOutput(candidates=(result,))

292
backend/app/services/location/resolvers/nominatim.py Normal file
View File

@@ -0,0 +1,292 @@
"""Nominatim-backed online geocoder.
The actual HTTP call is encapsulated in :func:`build_default_nominatim_geocoder`
which returns an ``lru_cache``-wrapped function. Domain modules typically:
1. Build a default geocoder via :func:`build_default_nominatim_geocoder`.
2. Re-export it under a stable module-level name (e.g. ``_geocode_online``).
3. Pass a *late-binding lambda* (``lambda q: _geocode_online(q)``) to
:class:`NominatimResolver`.
This ensures tests that ``monkeypatch.setattr(module, "_geocode_online", ...)``
can swap the geocoder behavior without touching pipeline construction.
"""
from __future__ import annotations
import time
from functools import lru_cache
from typing import Any, Callable
import httpx
from ..models import LocationCandidate, LocationQuery, ResolverOutput
from ..text import (
coerce_str,
normalize_country_text,
normalize_text,
parse_float,
)
NOMINATIM_SEARCH_URL = "https://nominatim.openstreetmap.org/search"
DEFAULT_USER_AGENT = "planet-earth-location-resolver/1.0"
DEFAULT_MIN_INTERVAL_SECONDS = 1.1
DEFAULT_TIMEOUT_SECONDS = 8.0
def build_default_nominatim_geocoder(
*,
user_agent: str = DEFAULT_USER_AGENT,
min_interval_seconds: float = DEFAULT_MIN_INTERVAL_SECONDS,
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
cache_size: int = 512,
) -> Callable[[str], dict[str, Any] | None]:
"""Return a cached, rate-limited Nominatim geocoder."""
last_request_at = [0.0]
@lru_cache(maxsize=cache_size)
def geocode(query: str) -> dict[str, Any] | None:
if not query:
return None
elapsed = time.monotonic() - last_request_at[0]
if elapsed < min_interval_seconds:
time.sleep(min_interval_seconds - elapsed)
last_request_at[0] = time.monotonic()
response = httpx.get(
NOMINATIM_SEARCH_URL,
params={
"q": query,
"format": "jsonv2",
"limit": 1,
"addressdetails": 1,
},
headers={"User-Agent": user_agent},
timeout=timeout_seconds,
)
response.raise_for_status()
payload = response.json()
if not isinstance(payload, list) or not payload:
return None
result = payload[0]
if not isinstance(result, dict):
return None
return result
return geocode
_DEFAULT_SITE_CATEGORIES = frozenset(
{
"amenity",
"office",
"building",
"industrial",
"research",
"university",
"education",
"tourism",
"shop",
"man_made",
"campus",
"research_institute",
}
)
def interpret_geocode_result(
result: dict[str, Any],
*,
matched_fields: tuple[str, ...],
context_country: str | None,
site_categories: frozenset[str] = _DEFAULT_SITE_CATEGORIES,
site_promoting_match_fields: frozenset[str] = frozenset(
{"site", "operator", "name"}
),
) -> tuple[float, float, dict[str, Any], str] | None:
"""Validate a Nominatim raw result. Returns (lat, lon, address, classification)."""
latitude = parse_float(result.get("lat"))
longitude = parse_float(result.get("lon"))
if latitude in (None, 0.0) or longitude in (None, 0.0):
return None
address = result.get("address") if isinstance(result.get("address"), dict) else {}
if not isinstance(address, dict):
address = {}
has_city_level = bool(
address.get("city")
or address.get("town")
or address.get("village")
or address.get("municipality")
or address.get("hamlet")
or address.get("suburb")
)
osm_class = str(result.get("class") or "").lower()
osm_type = str(result.get("type") or "").lower()
is_site_like = osm_class in site_categories or osm_type in site_categories
if not has_city_level and not is_site_like:
return None
if context_country:
normalized_context = normalize_text(normalize_country_text(context_country))
normalized_result = normalize_text(
normalize_country_text(address.get("country"))
)
if (
normalized_context
and normalized_result
and normalized_context != normalized_result
):
return None
classification = (
"site"
if (
is_site_like
and has_city_level
and any(field in site_promoting_match_fields for field in matched_fields)
)
else "city"
)
return float(latitude), float(longitude), address, classification
def _candidate_from_geocode(
*,
query: LocationQuery,
geocode_query: str,
matched_fields: tuple[str, ...],
raw_result: dict[str, Any],
interpret: Callable[..., tuple[float, float, dict[str, Any], str] | None],
source: str,
site_confidence: float,
city_confidence: float,
) -> LocationCandidate | None:
interpreted = interpret(
raw_result,
matched_fields=matched_fields,
context_country=query.country,
)
if not interpreted:
return None
latitude, longitude, address, classification = interpreted
city = (
address.get("city")
or address.get("town")
or address.get("village")
or address.get("municipality")
or query.city
or None
)
region = address.get("state") or address.get("region")
country = address.get("country") or query.country or None
display_name = raw_result.get("display_name") or geocode_query
confidence = city_confidence if classification == "city" else site_confidence
extra = query.extra or {}
suggested_registry_entry = {
"canonical_name": (
(query.aliases[0] if query.aliases else None)
or query.name
or display_name
),
"aliases": list(
{
value
for value in [
query.name,
*query.aliases,
coerce_str(extra.get("operator")),
coerce_str(extra.get("site")),
]
if value
}
),
"operator": coerce_str(extra.get("operator")) or None,
"site": coerce_str(extra.get("site"))
or coerce_str(extra.get("organization"))
or None,
"country": country,
"city": city,
"region": region,
"latitude": latitude,
"longitude": longitude,
"precision": classification,
"confidence": confidence,
"source_note": (
f"Resolved via Nominatim query '{geocode_query}' → {display_name}"
),
}
return LocationCandidate(
latitude=latitude,
longitude=longitude,
display_name=display_name,
precision=classification,
confidence=confidence,
query=geocode_query,
source=source,
source_note=f"Nominatim search result: {display_name}",
matched_fields=matched_fields,
needs_confirmation=True,
city=city,
region=region,
country=country,
matched_location_name=display_name,
location_verified_at=None,
suggested_registry_entry=suggested_registry_entry,
)
class NominatimResolver:
"""Run a domain-specific query plan against Nominatim."""
def __init__(
self,
*,
query_plan_builder: Callable[
[LocationQuery], list[tuple[str, tuple[str, ...]]]
],
geocoder: Callable[[str], dict[str, Any] | None],
name: str = "nominatim_online_geocode",
site_confidence: float = 0.72,
city_confidence: float = 0.62,
interpret: Callable[..., tuple[float, float, dict[str, Any], str] | None] = (
interpret_geocode_result
),
) -> None:
self.name = name
self._query_plan_builder = query_plan_builder
self._geocoder = geocoder
self._site_confidence = site_confidence
self._city_confidence = city_confidence
self._interpret = interpret
def resolve(self, query: LocationQuery) -> ResolverOutput:
plan = self._query_plan_builder(query)
candidates: list[LocationCandidate] = []
attempted: list[str] = []
for geocode_query, matched_fields in plan:
attempted.append(geocode_query)
try:
raw_result = self._geocoder(geocode_query)
except Exception:
continue
if not raw_result:
continue
candidate = _candidate_from_geocode(
query=query,
geocode_query=geocode_query,
matched_fields=matched_fields,
raw_result=raw_result,
interpret=self._interpret,
source=self.name,
site_confidence=self._site_confidence,
city_confidence=self._city_confidence,
)
if candidate is not None:
candidates.append(candidate)
return ResolverOutput(
candidates=tuple(candidates),
attempted_queries=tuple(attempted),
)

323
backend/app/services/location/resolvers/registry.py Normal file
View File

@@ -0,0 +1,323 @@
"""Resolver that matches a query against a local JSON registry.
Registry schema (a single JSON file):
{
"locations": [
{
"canonical_name": "...",
"aliases": ["...", "..."],
"operator": "...",
"site": "...",
"city": "...",
"country": "...",
"region": "...",
"latitude": 0.0,
"longitude": 0.0,
"precision": "precise" | "site" | "city",
"confidence": 0.0,
"verification_status": "verified",
"source_note": "...",
"verified_at": "YYYY-MM-DD"
}
],
"city_fallbacks": [ {city, country, latitude, longitude, ...} ]
}
"""
from __future__ import annotations
import json
from functools import lru_cache
from pathlib import Path
from typing import Any, Callable, Iterable
from ..models import (
RENDERABLE_PRECISIONS,
LocationCandidate,
LocationQuery,
ResolverOutput,
)
from ..text import (
city_key,
normalize_country_text,
normalize_text,
parse_float,
)
# Field-priority weights when scoring "this query field text contains this
# alias text". Tuned to match the legacy compute-center ordering — name beats
# site beats operator beats city — which generalizes well to other domains.
_DEFAULT_FIELD_PRIORITY = {
"name": 8,
"site": 6,
"operator": 5,
"city": 3,
}
def default_score_alias_match(
alias_field: str, record_field: str, alias_text: str
) -> int:
score = max(0, len(alias_text))
score += _DEFAULT_FIELD_PRIORITY.get(alias_field, 1)
if alias_field == record_field:
score += 4
if alias_field == "name" and record_field in {"name", "name_short", "alias"}:
score += 6
if alias_field == "site" and record_field in {"site", "organization"}:
score += 4
if alias_field == "operator" and record_field in {"operator", "organization"}:
score += 4
return score
@lru_cache(maxsize=32)
def _load_registry_file(path: str) -> dict[str, Any]:
with Path(path).open("r", encoding="utf-8") as handle:
return json.load(handle)
@lru_cache(maxsize=32)
def _build_alias_index(
path: str,
) -> tuple[tuple[dict[str, Any], tuple[tuple[str, str], ...]], ...]:
index: list[tuple[dict[str, Any], tuple[tuple[str, str], ...]]] = []
for entry in _load_registry_file(path).get("locations", []):
aliases: list[tuple[str, str]] = []
seen: set[str] = set()
for alias in [entry.get("canonical_name"), *(entry.get("aliases") or [])]:
normalized = normalize_text(alias)
if normalized and normalized not in seen:
aliases.append(("name", normalized))
seen.add(normalized)
for field_name in ("operator", "site", "city"):
value = entry.get(field_name)
normalized = normalize_text(value)
if normalized and normalized not in seen:
aliases.append((field_name, normalized))
seen.add(normalized)
index.append((entry, tuple(aliases)))
return tuple(index)
def _query_corpus(query: LocationQuery) -> dict[str, str]:
"""Map a query into normalized strings keyed by source field."""
fields: dict[str, str] = {
"name": query.name or "",
"city": query.city or "",
"country": query.country or "",
}
for alias in query.aliases:
if alias and alias != query.name:
fields["name_short"] = alias
break
extra = query.extra or {}
for key in ("site", "operator", "organization"):
value = extra.get(key)
if value:
fields[key] = str(value)
return {key: normalize_text(value) for key, value in fields.items() if value}
def _country_compatible(entry: dict[str, Any], query: LocationQuery) -> bool:
record_country = normalize_country_text(query.country)
entry_country = normalize_country_text(entry.get("country"))
if not record_country or not entry_country:
return True
return normalize_text(record_country) == normalize_text(entry_country)
def _normalized_alias_matches(alias_normalized: str, record_text: str) -> bool:
alias_tokens = alias_normalized.split()
record_tokens = record_text.split()
if not alias_tokens or not record_tokens:
return False
if len(alias_tokens) == 1:
return alias_tokens[0] in record_tokens
window_size = len(alias_tokens)
return any(
record_tokens[index : index + window_size] == alias_tokens
for index in range(0, len(record_tokens) - window_size + 1)
)
def _entry_to_candidate(
entry: dict[str, Any],
*,
matched_alias: str,
matched_fields: Iterable[str],
source: str,
score_explainer: str,
confidence_floor: float,
) -> LocationCandidate:
canonical_name = entry.get("canonical_name") or matched_alias
# Registry entries are treated as candidates unless explicitly verified.
# This prevents migrated hard-coded hints from appearing as factual
# location evidence.
is_verified = entry.get("verification_status") == "verified"
precision = entry.get("precision") or "city"
if precision not in RENDERABLE_PRECISIONS:
precision = "city"
fields_summary = ", ".join(sorted(set(matched_fields))) or "name"
confidence_value = parse_float(entry.get("confidence"))
confidence = (
float(confidence_value)
if confidence_value is not None
else confidence_floor
)
return LocationCandidate(
latitude=float(parse_float(entry.get("latitude")) or 0.0),
longitude=float(parse_float(entry.get("longitude")) or 0.0),
display_name=canonical_name,
precision=precision,
confidence=confidence,
query=f"local_registry::{matched_alias or canonical_name}",
source=source,
source_note=entry.get("source_note")
or f"{score_explainer}: matched {fields_summary}",
matched_fields=tuple(sorted(set(matched_fields))) or ("name",),
needs_confirmation=bool(entry.get("needs_confirmation")) or not is_verified,
city=entry.get("city"),
region=entry.get("region"),
country=entry.get("country"),
matched_location_name=canonical_name,
location_verified_at=entry.get("verified_at") if is_verified else None,
suggested_registry_entry=None,
)
class RegistryResolver:
"""Match a query against a JSON registry (plus its city_fallbacks table)."""
def __init__(
self,
*,
registry_path: Path | str,
name: str = "local_registry",
city_fallback_source: str = "local_registry_city",
city_fallback_confidence_default: float = 0.65,
confidence_default: float = 0.85,
score_alias_match: Callable[[str, str, str], int] = default_score_alias_match,
) -> None:
self.name = name
self._registry_path = str(Path(registry_path))
self._city_fallback_source = city_fallback_source
self._city_fallback_confidence_default = city_fallback_confidence_default
self._confidence_default = confidence_default
self._score = score_alias_match
def reload(self) -> None:
"""Drop the cached registry — useful when the JSON file is edited."""
_load_registry_file.cache_clear()
_build_alias_index.cache_clear()
def resolve(self, query: LocationQuery) -> ResolverOutput:
candidates: list[LocationCandidate] = []
candidates.extend(self._registry_candidates(query))
city_candidate = self._city_fallback_candidate(query)
if city_candidate is not None:
candidates.append(city_candidate)
return ResolverOutput(candidates=tuple(candidates))
# ── internals ──────────────────────────────────────────────
def _registry_candidates(
self, query: LocationQuery
) -> list[LocationCandidate]:
corpus = _query_corpus(query)
if not corpus:
return []
# When the query carries a name (a record-specific identifier), require
# at least one alias match against a name-class field — otherwise a
# generic shared field like operator="RIPE NCC" would promote every
# registry entry that lists that operator, regardless of whether the
# name matches.
query_has_name = bool(corpus.get("name") or corpus.get("name_short"))
results: list[LocationCandidate] = []
for entry, aliases in _build_alias_index(self._registry_path):
best_alias = ""
best_score = 0
matched_fields: list[str] = []
matched_via_name_alias = False
for alias_field, alias_normalized in aliases:
for record_field, record_text in corpus.items():
if not _normalized_alias_matches(alias_normalized, record_text):
continue
score = self._score(
alias_field, record_field, alias_normalized
)
if score > best_score or (
score == best_score
and len(alias_normalized) > len(best_alias)
):
best_score = score
best_alias = alias_normalized
if record_field not in matched_fields:
matched_fields.append(record_field)
if alias_field == "name" and record_field in {"name", "name_short"}:
matched_via_name_alias = True
if not matched_fields or best_score <= 0:
continue
if query_has_name and not matched_via_name_alias:
continue
if not _country_compatible(entry, query):
continue
results.append(
_entry_to_candidate(
entry,
matched_alias=best_alias,
matched_fields=matched_fields,
source=self.name,
score_explainer="Registry alias match",
confidence_floor=self._confidence_default,
)
)
return results
def _city_fallback_candidate(
self, query: LocationQuery
) -> LocationCandidate | None:
country = normalize_country_text(query.country)
city = city_key(query.city)
if not country or not city:
return None
for fallback in _load_registry_file(self._registry_path).get(
"city_fallbacks", []
):
fallback_country = normalize_country_text(fallback.get("country"))
fallback_city = city_key(fallback.get("city"))
if fallback_country != country or fallback_city != city:
continue
confidence_value = parse_float(fallback.get("confidence"))
confidence = (
float(confidence_value)
if confidence_value is not None
else self._city_fallback_confidence_default
)
return LocationCandidate(
latitude=float(parse_float(fallback.get("latitude")) or 0.0),
longitude=float(parse_float(fallback.get("longitude")) or 0.0),
display_name=fallback.get("city") or "",
precision="city",
confidence=confidence,
query=(
f"city_fallback::{fallback.get('city')}, "
f"{fallback.get('country')}"
),
source=self._city_fallback_source,
source_note=fallback.get("source_note")
or f"City fallback for {fallback.get('city')}, {fallback.get('country')}",
matched_fields=("city", "country"),
needs_confirmation=False,
city=fallback.get("city"),
region=fallback.get("region"),
country=fallback.get("country"),
matched_location_name=fallback.get("city"),
location_verified_at=fallback.get("verified_at"),
suggested_registry_entry=None,
)
return None

42
backend/app/services/location/resolvers/source_coordinates.py Normal file
View File

@@ -0,0 +1,42 @@
"""Resolver that consumes lat/lon already present on the source record."""
from __future__ import annotations
from ..models import LocationCandidate, LocationQuery, ResolverOutput
from ..text import normalize_country_text
class SourceCoordinatesResolver:
"""Pass-through for records that already carry valid coordinates."""
name = "source_coordinates"
def __init__(self, *, source: str = "source_coordinates") -> None:
self._source = source
def resolve(self, query: LocationQuery) -> ResolverOutput:
lat = query.source_latitude
lon = query.source_longitude
if lat in (None, 0.0) or lon in (None, 0.0):
return ResolverOutput()
country = normalize_country_text(query.country) or query.country
candidate = LocationCandidate(
latitude=float(lat),
longitude=float(lon),
display_name=query.name or "",
precision="precise",
confidence=1.0,
query="source_coordinates",
source=self._source,
source_note="Source record provided valid coordinates.",
matched_fields=("source_coordinates",),
needs_confirmation=False,
city=query.city,
region=query.region,
country=country,
matched_location_name=query.name,
location_verified_at=None,
suggested_registry_entry=None,
)
return ResolverOutput(candidates=(candidate,))