958 lines
32 KiB
Python
958 lines
32 KiB
Python
"""Tests for the shared location resolution pipeline.
|
|
|
|
Validates the abstraction itself: the protocol contract, the orchestrator,
|
|
each built-in resolver, and the pluggability promise (a custom resolver can
|
|
be slotted in without touching consumers).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.services.location import (
|
|
InheritFromAnotherEntityResolver,
|
|
LocationCandidate,
|
|
LocationPipeline,
|
|
LocationQuery,
|
|
NominatimResolver,
|
|
RegistryResolver,
|
|
ResolverOutput,
|
|
SourceCoordinatesResolver,
|
|
)
|
|
from app.schemas.ai import SituationalAnalysisResponse
|
|
import app.services.location.llm_fallback as llm_fallback
|
|
from app.services.location.llm_fallback import collect_llm_location_fallback_candidate
|
|
|
|
|
|
# ── Test fixtures ────────────────────────────────────────────────────
|
|
|
|
|
|
@pytest.fixture
|
|
def tmp_registry(tmp_path: Path) -> Path:
|
|
payload = {
|
|
"locations": [
|
|
{
|
|
"canonical_name": "Test Site Alpha",
|
|
"aliases": ["alpha", "alpha-one", "Acme HQ"],
|
|
"operator": "Acme Networks",
|
|
"site": "Acme HQ",
|
|
"city": "Lyon",
|
|
"country": "France",
|
|
"latitude": 45.764,
|
|
"longitude": 4.8357,
|
|
"precision": "site",
|
|
"confidence": 0.92,
|
|
"source_note": "Test fixture",
|
|
"verified_at": "2026-05-08",
|
|
},
|
|
{
|
|
"canonical_name": "Test Site Bravo",
|
|
"aliases": ["bravo"],
|
|
"operator": "Acme Networks",
|
|
"site": "Bravo POP",
|
|
"city": "Berlin",
|
|
"country": "Germany",
|
|
"latitude": 52.52,
|
|
"longitude": 13.405,
|
|
"precision": "city",
|
|
"confidence": 0.85,
|
|
},
|
|
],
|
|
"city_fallbacks": [
|
|
{
|
|
"city": "Bhutan-Capital",
|
|
"country": "Bhutan",
|
|
"latitude": 27.4728,
|
|
"longitude": 89.639,
|
|
"precision": "city",
|
|
"confidence": 0.5,
|
|
}
|
|
],
|
|
}
|
|
path = tmp_path / "registry.json"
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
# ── SourceCoordinatesResolver ────────────────────────────────────────
|
|
|
|
|
|
def test_source_coordinates_resolver_passes_through_valid_coordinates():
|
|
resolver = SourceCoordinatesResolver()
|
|
query = LocationQuery(
|
|
name="Acme HQ",
|
|
source_latitude=45.0,
|
|
source_longitude=4.0,
|
|
country="France",
|
|
)
|
|
output = resolver.resolve(query)
|
|
assert len(output.candidates) == 1
|
|
candidate = output.candidates[0]
|
|
assert candidate.latitude == 45.0
|
|
assert candidate.longitude == 4.0
|
|
assert candidate.precision == "precise"
|
|
assert candidate.source == "source_coordinates"
|
|
assert candidate.needs_confirmation is False
|
|
|
|
|
|
def test_source_coordinates_resolver_skips_zero_coordinates():
|
|
resolver = SourceCoordinatesResolver()
|
|
output = resolver.resolve(
|
|
LocationQuery(name="X", source_latitude=0.0, source_longitude=0.0)
|
|
)
|
|
assert output.candidates == ()
|
|
|
|
|
|
def test_source_coordinates_resolver_skips_when_missing():
|
|
resolver = SourceCoordinatesResolver()
|
|
output = resolver.resolve(LocationQuery(name="X"))
|
|
assert output.candidates == ()
|
|
|
|
|
|
# ── RegistryResolver ─────────────────────────────────────────────────
|
|
|
|
|
|
def test_registry_resolver_matches_alias(tmp_registry):
|
|
resolver = RegistryResolver(registry_path=tmp_registry)
|
|
resolver.reload()
|
|
output = resolver.resolve(
|
|
LocationQuery(name="alpha", country="France")
|
|
)
|
|
candidates = list(output.candidates)
|
|
assert candidates, "should match registry entry"
|
|
assert any(c.matched_location_name == "Test Site Alpha" for c in candidates)
|
|
alpha = next(c for c in candidates if c.matched_location_name == "Test Site Alpha")
|
|
assert alpha.precision == "site"
|
|
assert alpha.confidence == pytest.approx(0.92)
|
|
assert alpha.needs_confirmation is True
|
|
assert alpha.location_verified_at is None
|
|
|
|
|
|
def test_registry_resolver_filters_country_mismatch(tmp_registry):
|
|
resolver = RegistryResolver(registry_path=tmp_registry)
|
|
resolver.reload()
|
|
# alpha is in France; query says Spain → should reject
|
|
output = resolver.resolve(
|
|
LocationQuery(name="alpha", country="Spain")
|
|
)
|
|
assert all(
|
|
c.matched_location_name != "Test Site Alpha" for c in output.candidates
|
|
)
|
|
|
|
|
|
def test_registry_resolver_emits_city_fallback_candidate(tmp_registry):
|
|
resolver = RegistryResolver(registry_path=tmp_registry)
|
|
resolver.reload()
|
|
output = resolver.resolve(
|
|
LocationQuery(city="Bhutan-Capital", country="Bhutan")
|
|
)
|
|
candidates = list(output.candidates)
|
|
assert candidates, "city fallback should fire"
|
|
assert any(c.source == "local_registry_city" for c in candidates)
|
|
|
|
|
|
# ── NominatimResolver ───────────────────────────────────────────────
|
|
|
|
|
|
def test_nominatim_resolver_calls_geocoder_with_plan_queries():
|
|
calls = []
|
|
|
|
def fake_geocoder(query: str):
|
|
calls.append(query)
|
|
return {
|
|
"lat": "12.34",
|
|
"lon": "56.78",
|
|
"display_name": "Test City, Country",
|
|
"address": {"city": "Test City", "country": "Country"},
|
|
}
|
|
|
|
def plan(query: LocationQuery):
|
|
return [
|
|
("primary query", ("name",)),
|
|
("secondary query", ("city",)),
|
|
]
|
|
|
|
resolver = NominatimResolver(
|
|
query_plan_builder=plan,
|
|
geocoder=fake_geocoder,
|
|
)
|
|
output = resolver.resolve(LocationQuery(name="X", country="Country"))
|
|
assert calls == ["primary query", "secondary query"]
|
|
assert output.attempted_queries == ("primary query", "secondary query")
|
|
assert len(output.candidates) == 2
|
|
assert all(c.precision == "city" for c in output.candidates)
|
|
assert all(c.needs_confirmation for c in output.candidates)
|
|
|
|
|
|
def test_nominatim_resolver_skips_when_geocoder_returns_none():
|
|
resolver = NominatimResolver(
|
|
query_plan_builder=lambda q: [("only", ("name",))],
|
|
geocoder=lambda q: None,
|
|
)
|
|
output = resolver.resolve(LocationQuery(name="X"))
|
|
assert output.candidates == ()
|
|
assert output.attempted_queries == ("only",)
|
|
|
|
|
|
def test_nominatim_resolver_swallows_exceptions_per_query():
|
|
def boom(query):
|
|
raise RuntimeError("network down")
|
|
|
|
resolver = NominatimResolver(
|
|
query_plan_builder=lambda q: [("a", ()), ("b", ())],
|
|
geocoder=boom,
|
|
)
|
|
output = resolver.resolve(LocationQuery(name="X"))
|
|
assert output.candidates == ()
|
|
assert output.attempted_queries == ("a", "b")
|
|
|
|
|
|
# ── InheritFromAnotherEntityResolver ────────────────────────────────
|
|
|
|
|
|
def test_inherit_resolver_returns_provided_candidate():
|
|
sentinel = LocationCandidate(
|
|
latitude=10.0,
|
|
longitude=20.0,
|
|
display_name="Inherited",
|
|
precision="city",
|
|
confidence=0.7,
|
|
query="inherit::test",
|
|
source="inherited",
|
|
source_note=None,
|
|
matched_fields=("collector",),
|
|
needs_confirmation=False,
|
|
)
|
|
resolver = InheritFromAnotherEntityResolver(
|
|
source_lookup=lambda q: sentinel
|
|
)
|
|
output = resolver.resolve(LocationQuery(name="X"))
|
|
assert output.candidates == (sentinel,)
|
|
|
|
|
|
def test_inherit_resolver_skips_when_lookup_returns_none():
|
|
resolver = InheritFromAnotherEntityResolver(source_lookup=lambda q: None)
|
|
assert resolver.resolve(LocationQuery(name="X")).candidates == ()
|
|
|
|
|
|
# ── LocationPipeline orchestration ──────────────────────────────────
|
|
|
|
|
|
def test_pipeline_aggregates_candidates_across_resolvers(tmp_registry):
|
|
pipeline = LocationPipeline(
|
|
[
|
|
SourceCoordinatesResolver(),
|
|
RegistryResolver(registry_path=tmp_registry),
|
|
NominatimResolver(
|
|
query_plan_builder=lambda q: [("nominatim attempt", ("name",))],
|
|
geocoder=lambda q: {
|
|
"lat": "1.0",
|
|
"lon": "2.0",
|
|
"display_name": "Online City",
|
|
"address": {"city": "Online City", "country": "France"},
|
|
},
|
|
),
|
|
]
|
|
)
|
|
pipeline.resolvers[1].reload()
|
|
candidates, attempted = pipeline.collect_candidates(
|
|
LocationQuery(
|
|
name="alpha",
|
|
country="France",
|
|
source_latitude=44.0,
|
|
source_longitude=5.0,
|
|
)
|
|
)
|
|
sources = {c.source for c in candidates}
|
|
assert "source_coordinates" in sources
|
|
assert "local_registry" in sources
|
|
assert "nominatim_online_geocode" in sources
|
|
assert "nominatim attempt" in attempted
|
|
|
|
|
|
def test_pipeline_dedupes_by_source_and_coordinates():
|
|
same = LocationCandidate(
|
|
latitude=1.0,
|
|
longitude=2.0,
|
|
display_name="dup",
|
|
precision="city",
|
|
confidence=0.5,
|
|
query="x",
|
|
source="dup_source",
|
|
source_note=None,
|
|
matched_fields=(),
|
|
needs_confirmation=False,
|
|
)
|
|
|
|
class _DupResolver:
|
|
name = "dup_source"
|
|
|
|
def resolve(self, query):
|
|
return ResolverOutput(candidates=(same, same))
|
|
|
|
pipeline = LocationPipeline([_DupResolver()])
|
|
candidates, _ = pipeline.collect_candidates(LocationQuery(name="X"))
|
|
assert len(candidates) == 1
|
|
|
|
|
|
def test_registry_short_aliases_do_not_match_inside_larger_tokens(tmp_path: Path):
|
|
registry_path = tmp_path / "registry.json"
|
|
registry_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"locations": [
|
|
{
|
|
"canonical_name": "Aurora",
|
|
"aliases": ["Aurora", "ANL"],
|
|
"site": "DOE/SC/Argonne National Laboratory",
|
|
"country": "United States",
|
|
"city": "Lemont",
|
|
"latitude": 41.713,
|
|
"longitude": -87.982,
|
|
"precision": "site",
|
|
},
|
|
{
|
|
"canonical_name": "Venado",
|
|
"aliases": ["Venado"],
|
|
"site": "DOE/NNSA/LANL",
|
|
"country": "United States",
|
|
"city": "Los Alamos",
|
|
"latitude": 35.8443,
|
|
"longitude": -106.2872,
|
|
"precision": "site",
|
|
},
|
|
],
|
|
"city_fallbacks": [],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
resolver = RegistryResolver(registry_path=registry_path)
|
|
resolver.reload()
|
|
|
|
output = resolver.resolve(
|
|
LocationQuery(
|
|
name="Venado",
|
|
country="United States",
|
|
extra={"site": "DOE/NNSA/LANL"},
|
|
)
|
|
)
|
|
|
|
assert len(output.candidates) == 1
|
|
assert output.candidates[0].matched_location_name == "Venado"
|
|
|
|
|
|
def test_pipeline_resolve_best_returns_highest_priority():
|
|
online = LocationCandidate(
|
|
latitude=10.0,
|
|
longitude=20.0,
|
|
display_name="online",
|
|
precision="city",
|
|
confidence=0.9,
|
|
query="x",
|
|
source="nominatim_online_geocode",
|
|
source_note=None,
|
|
matched_fields=(),
|
|
needs_confirmation=True,
|
|
)
|
|
source = LocationCandidate(
|
|
latitude=11.0,
|
|
longitude=21.0,
|
|
display_name="src",
|
|
precision="precise",
|
|
confidence=1.0,
|
|
query="x",
|
|
source="source_coordinates",
|
|
source_note=None,
|
|
matched_fields=(),
|
|
needs_confirmation=False,
|
|
)
|
|
|
|
class _StubResolver:
|
|
def __init__(self, c, name):
|
|
self._c = c
|
|
self.name = name
|
|
|
|
def resolve(self, query):
|
|
return ResolverOutput(candidates=(self._c,))
|
|
|
|
pipeline = LocationPipeline(
|
|
[
|
|
_StubResolver(online, "online"),
|
|
_StubResolver(source, "src"),
|
|
]
|
|
)
|
|
result = pipeline.resolve_best(LocationQuery(name="X"))
|
|
assert result.location is source, "source_coordinates should beat nominatim"
|
|
|
|
|
|
def test_pipeline_returns_diagnostic_when_nothing_resolves():
|
|
pipeline = LocationPipeline([SourceCoordinatesResolver()])
|
|
result = pipeline.resolve_best(LocationQuery(name="X", country="Bhutan"))
|
|
assert result.location is None
|
|
assert result.diagnostic is not None
|
|
assert result.diagnostic.country == "Bhutan"
|
|
|
|
|
|
def test_pluggability_custom_resolver_works_without_changing_pipeline():
|
|
"""Validates the abstraction promise: a new algorithm = a new class."""
|
|
|
|
class _PeeringDBStubResolver:
|
|
name = "fake_peeringdb"
|
|
|
|
def resolve(self, query):
|
|
asn = (query.extra or {}).get("asn")
|
|
if asn != 174:
|
|
return ResolverOutput()
|
|
return ResolverOutput(
|
|
candidates=(
|
|
LocationCandidate(
|
|
latitude=1.0,
|
|
longitude=2.0,
|
|
display_name="Cogent HQ",
|
|
precision="site",
|
|
confidence=0.8,
|
|
query=f"peeringdb::{asn}",
|
|
source="peeringdb_stub",
|
|
source_note="Stub for testing",
|
|
matched_fields=("asn",),
|
|
needs_confirmation=False,
|
|
),
|
|
)
|
|
)
|
|
|
|
pipeline = LocationPipeline([_PeeringDBStubResolver()])
|
|
candidates, _ = pipeline.collect_candidates(
|
|
LocationQuery(name="X", extra={"asn": 174})
|
|
)
|
|
assert len(candidates) == 1
|
|
assert candidates[0].source == "peeringdb_stub"
|
|
|
|
|
|
# ── LLM fallback helper ─────────────────────────────────────────────
|
|
|
|
|
|
class _FakeAIProviderClient:
|
|
def __init__(self, content: str | list[str]):
|
|
self.contents = content if isinstance(content, list) else [content]
|
|
self.calls = 0
|
|
|
|
async def analyze(self, payload, request_id=None):
|
|
self.calls += 1
|
|
content = self.contents[min(self.calls - 1, len(self.contents) - 1)]
|
|
return SituationalAnalysisResponse(
|
|
provider="test",
|
|
model="test-model",
|
|
content=content,
|
|
raw_response={},
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_returns_candidate_from_strict_json():
|
|
client = _FakeAIProviderClient(
|
|
json.dumps(
|
|
{
|
|
"latitude": 45.764,
|
|
"longitude": 4.8357,
|
|
"precision": "city",
|
|
"confidence": 0.74,
|
|
"city": "Lyon",
|
|
"region": "Auvergne-Rhone-Alpes",
|
|
"country": "France",
|
|
"matched_location_name": "Lyon, France",
|
|
"evidence": ["operator and city point to Lyon"],
|
|
"reasoning_summary": "Best supported city-level match.",
|
|
}
|
|
)
|
|
)
|
|
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=client,
|
|
query=LocationQuery(
|
|
name="Mystery GPU Cluster",
|
|
city="Lyon",
|
|
country="France",
|
|
extra={"operator": "Mystery Operator"},
|
|
),
|
|
entity_type="compute_center",
|
|
attempted_queries=("Mystery Operator, Lyon, France",),
|
|
)
|
|
|
|
assert client.calls == 1
|
|
assert result.failure_reason is None
|
|
assert result.attempted_queries == ["llm_factcheck:compute_center:Mystery GPU Cluster"]
|
|
candidate = result.candidates[0]
|
|
assert candidate.source == "llm_location_factcheck"
|
|
assert candidate.needs_confirmation is True
|
|
assert candidate.precision == "city"
|
|
assert candidate.city == "Lyon"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_accepts_common_precision_aliases():
|
|
client = _FakeAIProviderClient(
|
|
json.dumps(
|
|
{
|
|
"candidate": {
|
|
"latitude": 43.2389,
|
|
"longitude": 76.8897,
|
|
"precision": "city-level",
|
|
"confidence": "0.68",
|
|
"city": "Almaty",
|
|
"country": "Kazakhstan",
|
|
"matched_location_name": "Almaty, Kazakhstan",
|
|
"evidence": ["NITEC context points to Almaty"],
|
|
"reasoning_summary": "City-level fallback.",
|
|
}
|
|
}
|
|
)
|
|
)
|
|
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=client,
|
|
query=LocationQuery(name="Alem.Cloud", country="Kazakhstan"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert result.failure_reason is None
|
|
assert result.candidates[0].precision == "city"
|
|
assert result.candidates[0].confidence >= 0.55
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_accepts_lat_lng_aliases():
|
|
client = _FakeAIProviderClient(
|
|
json.dumps(
|
|
{
|
|
"lat": 51.1694,
|
|
"lng": 71.4491,
|
|
"precision": "city",
|
|
"confidence": 0.62,
|
|
"city": "Astana",
|
|
"country": "Kazakhstan",
|
|
"matched_location_name": "Astana, Kazakhstan",
|
|
"evidence": [
|
|
{
|
|
"source": "Official source",
|
|
"source_type": "official",
|
|
"entity_match": True,
|
|
"text": "Alem.Cloud is in Astana.",
|
|
}
|
|
],
|
|
}
|
|
)
|
|
)
|
|
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=client,
|
|
query=LocationQuery(name="Alem.Cloud", country="Kazakhstan"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert result.failure_reason is None
|
|
assert result.candidates[0].latitude == pytest.approx(51.1694)
|
|
assert result.candidates[0].longitude == pytest.approx(71.4491)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_geocodes_city_when_coordinates_missing(monkeypatch):
|
|
monkeypatch.setattr(
|
|
llm_fallback,
|
|
"_geocode_llm_city",
|
|
lambda query: {
|
|
"lat": "51.1694",
|
|
"lon": "71.4491",
|
|
"display_name": "Astana, Kazakhstan",
|
|
"address": {"city": "Astana", "country": "Kazakhstan"},
|
|
},
|
|
)
|
|
client = _FakeAIProviderClient(
|
|
json.dumps(
|
|
{
|
|
"precision": "city",
|
|
"confidence": 0.62,
|
|
"city": "Astana",
|
|
"country": "Kazakhstan",
|
|
"matched_location_name": "Astana, Kazakhstan",
|
|
"evidence": [
|
|
{
|
|
"source": "Official source",
|
|
"source_type": "official",
|
|
"entity_match": True,
|
|
"text": "Alem.Cloud is in Astana.",
|
|
}
|
|
],
|
|
}
|
|
)
|
|
)
|
|
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=client,
|
|
query=LocationQuery(name="Alem.Cloud", country="Kazakhstan"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert result.failure_reason is None
|
|
candidate = result.candidates[0]
|
|
assert candidate.latitude == pytest.approx(51.1694)
|
|
assert candidate.longitude == pytest.approx(71.4491)
|
|
assert "Nominatim city fallback" in candidate.source_note
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_geocodes_matched_location_without_city(monkeypatch):
|
|
def _fake_geocode(query):
|
|
if "Falun" not in query:
|
|
return None
|
|
return {
|
|
"lat": "60.6065",
|
|
"lon": "15.6355",
|
|
"display_name": "Falun, Dalarna County, Sweden",
|
|
"address": {"city": "Falun", "state": "Dalarna County", "country": "Sweden"},
|
|
}
|
|
|
|
monkeypatch.setattr(llm_fallback, "_geocode_llm_city", _fake_geocode)
|
|
client = _FakeAIProviderClient(
|
|
json.dumps(
|
|
{
|
|
"precision": "city",
|
|
"confidence": 0.64,
|
|
"country": "Sweden",
|
|
"matched_location_name": "Falun, Sweden",
|
|
"evidence": [
|
|
{
|
|
"source": "Credible public source",
|
|
"source_type": "news",
|
|
"entity_match": True,
|
|
"text": "DeepL Mercury supercomputer is located in Falun.",
|
|
}
|
|
],
|
|
}
|
|
)
|
|
)
|
|
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=client,
|
|
query=LocationQuery(name="DeepL Mercury", country="Sweden"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert result.failure_reason is None
|
|
candidate = result.candidates[0]
|
|
assert candidate.city == "Falun"
|
|
assert candidate.country == "瑞典"
|
|
assert candidate.latitude == pytest.approx(60.6065)
|
|
assert candidate.longitude == pytest.approx(15.6355)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_repairs_non_json_answer(monkeypatch):
|
|
monkeypatch.setattr(
|
|
llm_fallback,
|
|
"_geocode_llm_city",
|
|
lambda query: {
|
|
"lat": "25.033",
|
|
"lon": "121.5654",
|
|
"display_name": "Taipei, Taiwan",
|
|
"address": {"city": "Taipei", "country": "Taiwan"},
|
|
},
|
|
)
|
|
client = _FakeAIProviderClient(
|
|
[
|
|
"TAIPEI-1 appears to be located in Taipei, Taiwan, based on NVIDIA context.",
|
|
json.dumps(
|
|
{
|
|
"latitude": None,
|
|
"longitude": None,
|
|
"precision": "city",
|
|
"confidence": 0.62,
|
|
"city": "Taipei",
|
|
"country": "Taiwan",
|
|
"matched_location_name": "Taipei, Taiwan",
|
|
"evidence": [
|
|
{
|
|
"source": "NVIDIA context",
|
|
"source_type": "generic",
|
|
"entity_match": True,
|
|
"text": "TAIPEI-1 appears to be located in Taipei.",
|
|
}
|
|
],
|
|
"reasoning_summary": "City-level location extracted from prose.",
|
|
}
|
|
),
|
|
]
|
|
)
|
|
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=client,
|
|
query=LocationQuery(name="TAIPEI-1", country="Taiwan"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert client.calls == 2
|
|
assert result.failure_reason is None
|
|
assert result.candidates[0].city == "Taipei"
|
|
assert result.candidates[0].source == "llm_location_factcheck"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_accepts_taipei_name_hint_with_weak_wording(monkeypatch):
|
|
monkeypatch.setattr(
|
|
llm_fallback,
|
|
"_geocode_llm_city",
|
|
lambda query: {
|
|
"lat": "25.033",
|
|
"lon": "121.5654",
|
|
"display_name": "Taipei, Taiwan",
|
|
"address": {"city": "Taipei", "country": "Taiwan"},
|
|
},
|
|
)
|
|
client = _FakeAIProviderClient(
|
|
json.dumps(
|
|
{
|
|
"latitude": None,
|
|
"longitude": None,
|
|
"precision": "city",
|
|
"confidence": 0.43,
|
|
"city": "Taipei",
|
|
"country": "Taiwan",
|
|
"matched_location_name": "Taipei, Taiwan",
|
|
"evidence": [
|
|
{
|
|
"source": "NVIDIA context",
|
|
"source_type": "generic",
|
|
"entity_match": True,
|
|
"text": "TAIPEI-1 points to Taipei city-level placement.",
|
|
}
|
|
],
|
|
"reasoning_summary": "Weak city-level evidence, but the entity name and geography align.",
|
|
}
|
|
)
|
|
)
|
|
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=client,
|
|
query=LocationQuery(name="TAIPEI-1", country="Taiwan"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert result.failure_reason is None
|
|
candidate = result.candidates[0]
|
|
assert candidate.city == "Taipei"
|
|
assert candidate.confidence >= 0.55
|
|
breakdown = candidate.suggested_registry_entry["llm_score_breakdown"]
|
|
assert breakdown["weak_evidence_penalty"] <= 0.15
|
|
assert breakdown["conflict_penalty"] == 0
|
|
assert breakdown["name_location_hint"] > 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_geocodes_city_from_entity_name_when_llm_unparseable(monkeypatch):
|
|
def _fake_geocode(query):
|
|
if query != "Taipei, 中国（台湾）":
|
|
return None
|
|
return {
|
|
"lat": "25.033",
|
|
"lon": "121.5654",
|
|
"display_name": "Taipei, Taiwan",
|
|
"address": {"city": "Taipei", "country": "Taiwan"},
|
|
}
|
|
|
|
monkeypatch.setattr(llm_fallback, "_geocode_llm_city", _fake_geocode)
|
|
client = _FakeAIProviderClient(["not a location answer", "still not json"])
|
|
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=client,
|
|
query=LocationQuery(name="TAIPEI-1", country="中国（台湾）"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert client.calls == 2
|
|
assert result.failure_reason is None
|
|
candidate = result.candidates[0]
|
|
assert candidate.city == "Taipei"
|
|
assert candidate.latitude == pytest.approx(25.033)
|
|
assert candidate.longitude == pytest.approx(121.5654)
|
|
assert "Entity name city hint" in candidate.source_note
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_extracts_city_from_non_json_when_repair_fails(monkeypatch):
|
|
monkeypatch.setattr(
|
|
llm_fallback,
|
|
"_geocode_llm_city",
|
|
lambda query: {
|
|
"lat": "60.6065",
|
|
"lon": "15.6355",
|
|
"display_name": "Falun, Sweden",
|
|
"address": {"city": "Falun", "country": "Sweden"},
|
|
},
|
|
)
|
|
client = _FakeAIProviderClient(
|
|
[
|
|
"DeepL Mercury 超級電腦位於瑞典的 法倫 (Falun)。",
|
|
"still not json",
|
|
]
|
|
)
|
|
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=client,
|
|
query=LocationQuery(name="DeepL Mercury", country="Sweden"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert client.calls == 2
|
|
assert result.failure_reason is None
|
|
assert result.candidates[0].city == "Falun"
|
|
assert result.candidates[0].needs_confirmation is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_combines_model_score_with_evidence_score():
|
|
client = _FakeAIProviderClient(
|
|
json.dumps(
|
|
{
|
|
"latitude": 51.1694,
|
|
"longitude": 71.4491,
|
|
"precision": "city",
|
|
"confidence": 0.38,
|
|
"city": "Astana",
|
|
"country": "Kazakhstan",
|
|
"matched_location_name": "Astana, Kazakhstan",
|
|
"evidence": [
|
|
{
|
|
"source": "Kazakhstan National Supercomputing Center",
|
|
"url": "https://example.test/alem-cloud",
|
|
"source_type": "official",
|
|
"entity_match": True,
|
|
"text": "Alem.Cloud is located in Astana.",
|
|
}
|
|
],
|
|
"reasoning_summary": "Evidence supports city-level location but not exact facility coordinates.",
|
|
}
|
|
)
|
|
)
|
|
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=client,
|
|
query=LocationQuery(name="Alem.Cloud", country="Kazakhstan"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert result.failure_reason is None
|
|
candidate = result.candidates[0]
|
|
assert candidate.city == "Astana"
|
|
assert candidate.confidence >= 0.55
|
|
assert candidate.suggested_registry_entry["llm_model_confidence"] == pytest.approx(0.38)
|
|
assert candidate.suggested_registry_entry["llm_combined_confidence"] == pytest.approx(
|
|
candidate.confidence
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_rejects_low_combined_score():
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=_FakeAIProviderClient(
|
|
json.dumps(
|
|
{
|
|
"latitude": 51.1694,
|
|
"longitude": 71.4491,
|
|
"precision": "city",
|
|
"confidence": 0.38,
|
|
"city": "Astana",
|
|
"country": "Kazakhstan",
|
|
"matched_location_name": "Astana, Kazakhstan",
|
|
"evidence": ["some page mentions Kazakhstan"],
|
|
"reasoning_summary": "Weak and ambiguous city evidence.",
|
|
"ambiguity": "weak city evidence",
|
|
}
|
|
)
|
|
),
|
|
query=LocationQuery(name="Alem.Cloud", country="Kazakhstan"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert result.candidates == []
|
|
assert "combined evidence score" in result.failure_reason
|
|
assert "below minimum 0.55" in result.failure_reason
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_rejects_explicit_conflicts():
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=_FakeAIProviderClient(
|
|
json.dumps(
|
|
{
|
|
"latitude": 25.033,
|
|
"longitude": 121.5654,
|
|
"precision": "city",
|
|
"confidence": 0.70,
|
|
"city": "Taipei",
|
|
"country": "Taiwan",
|
|
"matched_location_name": "Taipei, Taiwan",
|
|
"evidence": [
|
|
{
|
|
"source": "Conflicting source",
|
|
"source_type": "generic",
|
|
"entity_match": True,
|
|
"has_conflict": True,
|
|
"text": "One source says Taipei, another contradicts it.",
|
|
}
|
|
],
|
|
"reasoning_summary": "Conflicting evidence prevents confirmation.",
|
|
}
|
|
)
|
|
),
|
|
query=LocationQuery(name="TAIPEI-1", country="Taiwan"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert result.candidates == []
|
|
assert "conflict=" in result.failure_reason
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"content",
|
|
[
|
|
"not json",
|
|
json.dumps({"latitude": 0, "longitude": 0, "precision": "city", "confidence": 0.9}),
|
|
json.dumps({"latitude": 45, "longitude": 4, "precision": "country", "confidence": 0.9}),
|
|
json.dumps({"latitude": 45, "longitude": 4, "precision": "city", "confidence": 0.2}),
|
|
],
|
|
)
|
|
async def test_llm_location_fallback_rejects_unsafe_outputs(content):
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=_FakeAIProviderClient(content),
|
|
query=LocationQuery(name="Unsafe", country="France"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert result.candidates == []
|
|
assert result.failure_reason
|
|
assert result.attempted_queries == ["llm_factcheck:compute_center:Unsafe"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_location_fallback_failure_explains_rejection_reason():
|
|
result = await collect_llm_location_fallback_candidate(
|
|
provider_client=_FakeAIProviderClient(
|
|
json.dumps({
|
|
"latitude": 45,
|
|
"longitude": 4,
|
|
"precision": "region",
|
|
"confidence": 0.9,
|
|
})
|
|
),
|
|
query=LocationQuery(name="Unsafe", country="France"),
|
|
entity_type="compute_center",
|
|
)
|
|
|
|
assert result.candidates == []
|
|
assert "precision" in result.failure_reason
|
|
assert "region" in result.failure_reason
|