release: bump version to 0.50.0
This commit is contained in:
rayd1o
@@ -22,6 +22,9 @@ from app.services.location import (
|
||||
ResolverOutput,
|
||||
SourceCoordinatesResolver,
|
||||
)
|
||||
from app.schemas.ai import SituationalAnalysisResponse
|
||||
import app.services.location.llm_fallback as llm_fallback
|
||||
from app.services.location.llm_fallback import collect_llm_location_fallback_candidate
|
||||
|
||||
|
||||
# ── Test fixtures ────────────────────────────────────────────────────
|
||||
@@ -427,3 +430,528 @@ def test_pluggability_custom_resolver_works_without_changing_pipeline():
|
||||
)
|
||||
assert len(candidates) == 1
|
||||
assert candidates[0].source == "peeringdb_stub"
|
||||
|
||||
|
||||
# ── LLM fallback helper ─────────────────────────────────────────────
|
||||
|
||||
|
||||
class _FakeAIProviderClient:
|
||||
def __init__(self, content: str | list[str]):
|
||||
self.contents = content if isinstance(content, list) else [content]
|
||||
self.calls = 0
|
||||
|
||||
async def analyze(self, payload, request_id=None):
|
||||
self.calls += 1
|
||||
content = self.contents[min(self.calls - 1, len(self.contents) - 1)]
|
||||
return SituationalAnalysisResponse(
|
||||
provider="test",
|
||||
model="test-model",
|
||||
content=content,
|
||||
raw_response={},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_returns_candidate_from_strict_json():
|
||||
client = _FakeAIProviderClient(
|
||||
json.dumps(
|
||||
{
|
||||
"latitude": 45.764,
|
||||
"longitude": 4.8357,
|
||||
"precision": "city",
|
||||
"confidence": 0.74,
|
||||
"city": "Lyon",
|
||||
"region": "Auvergne-Rhone-Alpes",
|
||||
"country": "France",
|
||||
"matched_location_name": "Lyon, France",
|
||||
"evidence": ["operator and city point to Lyon"],
|
||||
"reasoning_summary": "Best supported city-level match.",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(
|
||||
name="Mystery GPU Cluster",
|
||||
city="Lyon",
|
||||
country="France",
|
||||
extra={"operator": "Mystery Operator"},
|
||||
),
|
||||
entity_type="compute_center",
|
||||
attempted_queries=("Mystery Operator, Lyon, France",),
|
||||
)
|
||||
|
||||
assert client.calls == 1
|
||||
assert result.failure_reason is None
|
||||
assert result.attempted_queries == ["llm_factcheck:compute_center:Mystery GPU Cluster"]
|
||||
candidate = result.candidates[0]
|
||||
assert candidate.source == "llm_location_factcheck"
|
||||
assert candidate.needs_confirmation is True
|
||||
assert candidate.precision == "city"
|
||||
assert candidate.city == "Lyon"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_accepts_common_precision_aliases():
|
||||
client = _FakeAIProviderClient(
|
||||
json.dumps(
|
||||
{
|
||||
"candidate": {
|
||||
"latitude": 43.2389,
|
||||
"longitude": 76.8897,
|
||||
"precision": "city-level",
|
||||
"confidence": "0.68",
|
||||
"city": "Almaty",
|
||||
"country": "Kazakhstan",
|
||||
"matched_location_name": "Almaty, Kazakhstan",
|
||||
"evidence": ["NITEC context points to Almaty"],
|
||||
"reasoning_summary": "City-level fallback.",
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(name="Alem.Cloud", country="Kazakhstan"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.failure_reason is None
|
||||
assert result.candidates[0].precision == "city"
|
||||
assert result.candidates[0].confidence >= 0.55
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_accepts_lat_lng_aliases():
|
||||
client = _FakeAIProviderClient(
|
||||
json.dumps(
|
||||
{
|
||||
"lat": 51.1694,
|
||||
"lng": 71.4491,
|
||||
"precision": "city",
|
||||
"confidence": 0.62,
|
||||
"city": "Astana",
|
||||
"country": "Kazakhstan",
|
||||
"matched_location_name": "Astana, Kazakhstan",
|
||||
"evidence": [
|
||||
{
|
||||
"source": "Official source",
|
||||
"source_type": "official",
|
||||
"entity_match": True,
|
||||
"text": "Alem.Cloud is in Astana.",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(name="Alem.Cloud", country="Kazakhstan"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.failure_reason is None
|
||||
assert result.candidates[0].latitude == pytest.approx(51.1694)
|
||||
assert result.candidates[0].longitude == pytest.approx(71.4491)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_geocodes_city_when_coordinates_missing(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
llm_fallback,
|
||||
"_geocode_llm_city",
|
||||
lambda query: {
|
||||
"lat": "51.1694",
|
||||
"lon": "71.4491",
|
||||
"display_name": "Astana, Kazakhstan",
|
||||
"address": {"city": "Astana", "country": "Kazakhstan"},
|
||||
},
|
||||
)
|
||||
client = _FakeAIProviderClient(
|
||||
json.dumps(
|
||||
{
|
||||
"precision": "city",
|
||||
"confidence": 0.62,
|
||||
"city": "Astana",
|
||||
"country": "Kazakhstan",
|
||||
"matched_location_name": "Astana, Kazakhstan",
|
||||
"evidence": [
|
||||
{
|
||||
"source": "Official source",
|
||||
"source_type": "official",
|
||||
"entity_match": True,
|
||||
"text": "Alem.Cloud is in Astana.",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(name="Alem.Cloud", country="Kazakhstan"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.failure_reason is None
|
||||
candidate = result.candidates[0]
|
||||
assert candidate.latitude == pytest.approx(51.1694)
|
||||
assert candidate.longitude == pytest.approx(71.4491)
|
||||
assert "Nominatim city fallback" in candidate.source_note
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_geocodes_matched_location_without_city(monkeypatch):
|
||||
def _fake_geocode(query):
|
||||
if "Falun" not in query:
|
||||
return None
|
||||
return {
|
||||
"lat": "60.6065",
|
||||
"lon": "15.6355",
|
||||
"display_name": "Falun, Dalarna County, Sweden",
|
||||
"address": {"city": "Falun", "state": "Dalarna County", "country": "Sweden"},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(llm_fallback, "_geocode_llm_city", _fake_geocode)
|
||||
client = _FakeAIProviderClient(
|
||||
json.dumps(
|
||||
{
|
||||
"precision": "city",
|
||||
"confidence": 0.64,
|
||||
"country": "Sweden",
|
||||
"matched_location_name": "Falun, Sweden",
|
||||
"evidence": [
|
||||
{
|
||||
"source": "Credible public source",
|
||||
"source_type": "news",
|
||||
"entity_match": True,
|
||||
"text": "DeepL Mercury supercomputer is located in Falun.",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(name="DeepL Mercury", country="Sweden"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.failure_reason is None
|
||||
candidate = result.candidates[0]
|
||||
assert candidate.city == "Falun"
|
||||
assert candidate.country == "瑞典"
|
||||
assert candidate.latitude == pytest.approx(60.6065)
|
||||
assert candidate.longitude == pytest.approx(15.6355)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_repairs_non_json_answer(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
llm_fallback,
|
||||
"_geocode_llm_city",
|
||||
lambda query: {
|
||||
"lat": "25.033",
|
||||
"lon": "121.5654",
|
||||
"display_name": "Taipei, Taiwan",
|
||||
"address": {"city": "Taipei", "country": "Taiwan"},
|
||||
},
|
||||
)
|
||||
client = _FakeAIProviderClient(
|
||||
[
|
||||
"TAIPEI-1 appears to be located in Taipei, Taiwan, based on NVIDIA context.",
|
||||
json.dumps(
|
||||
{
|
||||
"latitude": None,
|
||||
"longitude": None,
|
||||
"precision": "city",
|
||||
"confidence": 0.62,
|
||||
"city": "Taipei",
|
||||
"country": "Taiwan",
|
||||
"matched_location_name": "Taipei, Taiwan",
|
||||
"evidence": [
|
||||
{
|
||||
"source": "NVIDIA context",
|
||||
"source_type": "generic",
|
||||
"entity_match": True,
|
||||
"text": "TAIPEI-1 appears to be located in Taipei.",
|
||||
}
|
||||
],
|
||||
"reasoning_summary": "City-level location extracted from prose.",
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(name="TAIPEI-1", country="Taiwan"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert client.calls == 2
|
||||
assert result.failure_reason is None
|
||||
assert result.candidates[0].city == "Taipei"
|
||||
assert result.candidates[0].source == "llm_location_factcheck"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_accepts_taipei_name_hint_with_weak_wording(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
llm_fallback,
|
||||
"_geocode_llm_city",
|
||||
lambda query: {
|
||||
"lat": "25.033",
|
||||
"lon": "121.5654",
|
||||
"display_name": "Taipei, Taiwan",
|
||||
"address": {"city": "Taipei", "country": "Taiwan"},
|
||||
},
|
||||
)
|
||||
client = _FakeAIProviderClient(
|
||||
json.dumps(
|
||||
{
|
||||
"latitude": None,
|
||||
"longitude": None,
|
||||
"precision": "city",
|
||||
"confidence": 0.43,
|
||||
"city": "Taipei",
|
||||
"country": "Taiwan",
|
||||
"matched_location_name": "Taipei, Taiwan",
|
||||
"evidence": [
|
||||
{
|
||||
"source": "NVIDIA context",
|
||||
"source_type": "generic",
|
||||
"entity_match": True,
|
||||
"text": "TAIPEI-1 points to Taipei city-level placement.",
|
||||
}
|
||||
],
|
||||
"reasoning_summary": "Weak city-level evidence, but the entity name and geography align.",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(name="TAIPEI-1", country="Taiwan"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.failure_reason is None
|
||||
candidate = result.candidates[0]
|
||||
assert candidate.city == "Taipei"
|
||||
assert candidate.confidence >= 0.55
|
||||
breakdown = candidate.suggested_registry_entry["llm_score_breakdown"]
|
||||
assert breakdown["weak_evidence_penalty"] <= 0.15
|
||||
assert breakdown["conflict_penalty"] == 0
|
||||
assert breakdown["name_location_hint"] > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_geocodes_city_from_entity_name_when_llm_unparseable(monkeypatch):
|
||||
def _fake_geocode(query):
|
||||
if query != "Taipei, 中国（台湾）":
|
||||
return None
|
||||
return {
|
||||
"lat": "25.033",
|
||||
"lon": "121.5654",
|
||||
"display_name": "Taipei, Taiwan",
|
||||
"address": {"city": "Taipei", "country": "Taiwan"},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(llm_fallback, "_geocode_llm_city", _fake_geocode)
|
||||
client = _FakeAIProviderClient(["not a location answer", "still not json"])
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(name="TAIPEI-1", country="中国（台湾）"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert client.calls == 2
|
||||
assert result.failure_reason is None
|
||||
candidate = result.candidates[0]
|
||||
assert candidate.city == "Taipei"
|
||||
assert candidate.latitude == pytest.approx(25.033)
|
||||
assert candidate.longitude == pytest.approx(121.5654)
|
||||
assert "Entity name city hint" in candidate.source_note
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_extracts_city_from_non_json_when_repair_fails(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
llm_fallback,
|
||||
"_geocode_llm_city",
|
||||
lambda query: {
|
||||
"lat": "60.6065",
|
||||
"lon": "15.6355",
|
||||
"display_name": "Falun, Sweden",
|
||||
"address": {"city": "Falun", "country": "Sweden"},
|
||||
},
|
||||
)
|
||||
client = _FakeAIProviderClient(
|
||||
[
|
||||
"DeepL Mercury 超級電腦位於瑞典的 法倫 (Falun)。",
|
||||
"still not json",
|
||||
]
|
||||
)
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(name="DeepL Mercury", country="Sweden"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert client.calls == 2
|
||||
assert result.failure_reason is None
|
||||
assert result.candidates[0].city == "Falun"
|
||||
assert result.candidates[0].needs_confirmation is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_combines_model_score_with_evidence_score():
|
||||
client = _FakeAIProviderClient(
|
||||
json.dumps(
|
||||
{
|
||||
"latitude": 51.1694,
|
||||
"longitude": 71.4491,
|
||||
"precision": "city",
|
||||
"confidence": 0.38,
|
||||
"city": "Astana",
|
||||
"country": "Kazakhstan",
|
||||
"matched_location_name": "Astana, Kazakhstan",
|
||||
"evidence": [
|
||||
{
|
||||
"source": "Kazakhstan National Supercomputing Center",
|
||||
"url": "https://example.test/alem-cloud",
|
||||
"source_type": "official",
|
||||
"entity_match": True,
|
||||
"text": "Alem.Cloud is located in Astana.",
|
||||
}
|
||||
],
|
||||
"reasoning_summary": "Evidence supports city-level location but not exact facility coordinates.",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=client,
|
||||
query=LocationQuery(name="Alem.Cloud", country="Kazakhstan"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.failure_reason is None
|
||||
candidate = result.candidates[0]
|
||||
assert candidate.city == "Astana"
|
||||
assert candidate.confidence >= 0.55
|
||||
assert candidate.suggested_registry_entry["llm_model_confidence"] == pytest.approx(0.38)
|
||||
assert candidate.suggested_registry_entry["llm_combined_confidence"] == pytest.approx(
|
||||
candidate.confidence
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_rejects_low_combined_score():
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=_FakeAIProviderClient(
|
||||
json.dumps(
|
||||
{
|
||||
"latitude": 51.1694,
|
||||
"longitude": 71.4491,
|
||||
"precision": "city",
|
||||
"confidence": 0.38,
|
||||
"city": "Astana",
|
||||
"country": "Kazakhstan",
|
||||
"matched_location_name": "Astana, Kazakhstan",
|
||||
"evidence": ["some page mentions Kazakhstan"],
|
||||
"reasoning_summary": "Weak and ambiguous city evidence.",
|
||||
"ambiguity": "weak city evidence",
|
||||
}
|
||||
)
|
||||
),
|
||||
query=LocationQuery(name="Alem.Cloud", country="Kazakhstan"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.candidates == []
|
||||
assert "combined evidence score" in result.failure_reason
|
||||
assert "below minimum 0.55" in result.failure_reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_rejects_explicit_conflicts():
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=_FakeAIProviderClient(
|
||||
json.dumps(
|
||||
{
|
||||
"latitude": 25.033,
|
||||
"longitude": 121.5654,
|
||||
"precision": "city",
|
||||
"confidence": 0.70,
|
||||
"city": "Taipei",
|
||||
"country": "Taiwan",
|
||||
"matched_location_name": "Taipei, Taiwan",
|
||||
"evidence": [
|
||||
{
|
||||
"source": "Conflicting source",
|
||||
"source_type": "generic",
|
||||
"entity_match": True,
|
||||
"has_conflict": True,
|
||||
"text": "One source says Taipei, another contradicts it.",
|
||||
}
|
||||
],
|
||||
"reasoning_summary": "Conflicting evidence prevents confirmation.",
|
||||
}
|
||||
)
|
||||
),
|
||||
query=LocationQuery(name="TAIPEI-1", country="Taiwan"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.candidates == []
|
||||
assert "conflict=" in result.failure_reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
"not json",
|
||||
json.dumps({"latitude": 0, "longitude": 0, "precision": "city", "confidence": 0.9}),
|
||||
json.dumps({"latitude": 45, "longitude": 4, "precision": "country", "confidence": 0.9}),
|
||||
json.dumps({"latitude": 45, "longitude": 4, "precision": "city", "confidence": 0.2}),
|
||||
],
|
||||
)
|
||||
async def test_llm_location_fallback_rejects_unsafe_outputs(content):
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=_FakeAIProviderClient(content),
|
||||
query=LocationQuery(name="Unsafe", country="France"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.candidates == []
|
||||
assert result.failure_reason
|
||||
assert result.attempted_queries == ["llm_factcheck:compute_center:Unsafe"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_location_fallback_failure_explains_rejection_reason():
|
||||
result = await collect_llm_location_fallback_candidate(
|
||||
provider_client=_FakeAIProviderClient(
|
||||
json.dumps({
|
||||
"latitude": 45,
|
||||
"longitude": 4,
|
||||
"precision": "region",
|
||||
"confidence": 0.9,
|
||||
})
|
||||
),
|
||||
query=LocationQuery(name="Unsafe", country="France"),
|
||||
entity_type="compute_center",
|
||||
)
|
||||
|
||||
assert result.candidates == []
|
||||
assert "precision" in result.failure_reason
|
||||
assert "region" in result.failure_reason
|
||||
|
||||
Reference in New Issue
Block a user