859 lines
31 KiB
Python
859 lines
31 KiB
Python
from datetime import UTC, datetime
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from app.services.earth_news import (
|
|
NewsFeedSource,
|
|
NewsTargetLocation,
|
|
ParsedNewsItem,
|
|
_enrich_items_with_target_locations,
|
|
_extract_target_location_from_text,
|
|
_parse_feed_entries,
|
|
_serialize_item,
|
|
get_earth_news_payload,
|
|
)
|
|
from app.services.earth_news_queue import NewsTargetLocationMessage
|
|
from app.services.earth_news_worker import process_target_location_message
|
|
from app.services.collectors.media_news_archive import MediaNewsArchiveCollector
|
|
|
|
|
|
def test_serialize_item_includes_region_anchor_for_cruise():
|
|
item = ParsedNewsItem(
|
|
id="google-apac:test",
|
|
title="Example APAC story",
|
|
summary="Example summary",
|
|
url="https://example.com/story",
|
|
source="Example Source",
|
|
feed_name="Global Monitor / APAC",
|
|
feed_region="asia-pacific",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 4, 23, 2, 30, tzinfo=UTC),
|
|
)
|
|
|
|
payload = _serialize_item(item, active_region="asia-pacific")
|
|
|
|
assert payload["latitude"] == 1.3521
|
|
assert payload["longitude"] == 103.8198
|
|
assert payload["location_label"] == "亚太"
|
|
assert payload["location_source"] == "region_anchor"
|
|
assert payload["verified"] is False
|
|
assert payload["location_meta"]["target"] is None
|
|
assert payload["location_meta"]["anchor"]["region"] == "asia-pacific"
|
|
assert payload["is_focus_match"] is True
|
|
assert payload["published_at"] == "2026-04-23T02:30:00Z"
|
|
|
|
|
|
def test_serialize_item_falls_back_to_global_anchor():
|
|
item = ParsedNewsItem(
|
|
id="custom:test",
|
|
title="Fallback story",
|
|
summary="Fallback summary",
|
|
url="https://example.com/fallback",
|
|
source="Fallback Source",
|
|
feed_name="Fallback Feed",
|
|
feed_region="unknown-region",
|
|
homepage_url="https://example.com",
|
|
published_at=None,
|
|
)
|
|
|
|
payload = _serialize_item(item, active_region="americas")
|
|
|
|
assert payload["latitude"] == 20.0
|
|
assert payload["longitude"] == 0.0
|
|
assert payload["location_label"] == "全球"
|
|
assert payload["location_source"] == "region_anchor"
|
|
assert payload["verified"] is False
|
|
assert payload["location_meta"]["anchor"]["region"] == "global"
|
|
assert payload["is_focus_match"] is False
|
|
assert payload["published_at"] is None
|
|
|
|
|
|
def test_serialize_item_includes_inferred_target_location():
|
|
item = ParsedNewsItem(
|
|
id="bbc-world:f55310fb667b",
|
|
title="Watch: What happened on day one of Trump's China visit?",
|
|
summary=(
|
|
"China welcomed US President Donald Trump with cheering children "
|
|
"and a troop parade."
|
|
),
|
|
url="https://example.com/china-visit",
|
|
source="BBC World",
|
|
feed_name="BBC World",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 14, 7, 13, 24, tzinfo=UTC),
|
|
target_location=NewsTargetLocation(
|
|
latitude=39.9042,
|
|
longitude=116.4074,
|
|
label="Beijing, China",
|
|
source="ai_inferred_target",
|
|
confidence=0.88,
|
|
country="中国",
|
|
city="Beijing",
|
|
),
|
|
)
|
|
|
|
payload = _serialize_item(item, active_region="global")
|
|
|
|
assert payload["latitude"] == 39.9042
|
|
assert payload["longitude"] == 116.4074
|
|
assert payload["location_label"] == "Beijing, China"
|
|
assert payload["location_source"] == "ai_inferred_target"
|
|
assert payload["verified"] is True
|
|
assert payload["location_meta"]["target"]["confidence"] == 0.88
|
|
assert payload["location_meta"]["target"]["country"] == "中国"
|
|
assert payload["location_meta"]["target"]["city"] == "Beijing"
|
|
assert payload["location_meta"]["resolution_stage"] == "unresolved"
|
|
assert payload["location_meta"]["ai_attempted"] is False
|
|
assert payload["location_meta"]["ai_status"] == "not_attempted"
|
|
assert payload["location_meta"]["ai_error"] is None
|
|
|
|
|
|
def test_parse_plain_rss_uses_feed_name_as_source():
|
|
source = NewsFeedSource(
|
|
id="bbc-world",
|
|
name="BBC World",
|
|
region="global",
|
|
feed_url="https://feeds.bbci.co.uk/news/world/rss.xml",
|
|
homepage_url="https://www.bbc.com/news/world",
|
|
)
|
|
xml = """
|
|
<rss>
|
|
<channel>
|
|
<item>
|
|
<title>This may be the last time you hear my voice: Political executions surge in Iran since start of war</title>
|
|
<description>Story summary</description>
|
|
<link>https://www.bbc.com/news/example</link>
|
|
<pubDate>Fri, 15 May 2026 03:00:00 GMT</pubDate>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
items = _parse_feed_entries(xml, source)
|
|
|
|
assert items[0].title == "This may be the last time you hear my voice: Political executions surge in Iran since start of war"
|
|
assert items[0].source == "BBC World"
|
|
|
|
|
|
def test_parse_aggregated_rss_splits_publisher_from_title():
|
|
source = NewsFeedSource(
|
|
id="global-scan",
|
|
name="Global Monitor / World",
|
|
region="global",
|
|
feed_url="https://news.google.com/rss",
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
)
|
|
xml = """
|
|
<rss>
|
|
<channel>
|
|
<item>
|
|
<title>Example headline - Reuters</title>
|
|
<description>Story summary</description>
|
|
<link>https://news.google.com/example</link>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
items = _parse_feed_entries(xml, source)
|
|
|
|
assert items[0].title == "Example headline"
|
|
assert items[0].source == "Reuters"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_enrich_items_with_target_locations_uses_ai_and_geocode(monkeypatch):
|
|
item = ParsedNewsItem(
|
|
id="bbc-world:f55310fb667b",
|
|
title="Watch: What happened on day one of Trump's China visit?",
|
|
summary="China welcomed US President Donald Trump before a long meeting with Xi Jinping.",
|
|
url="https://example.com/china-visit",
|
|
source="BBC World",
|
|
feed_name="BBC World",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 14, 7, 13, 24, tzinfo=UTC),
|
|
)
|
|
|
|
async def fake_geocode(_query: str):
|
|
return {
|
|
"lat": "39.9042",
|
|
"lon": "116.4074",
|
|
"display_name": "Beijing, China",
|
|
}
|
|
|
|
class FakeProviderClient:
|
|
async def analyze(self, _request):
|
|
class Response:
|
|
content = (
|
|
'{"country":"China","city":"Beijing","matched_location_name":"Beijing, China",'
|
|
'"latitude":null,"longitude":null,"confidence":0.88}'
|
|
)
|
|
|
|
return Response()
|
|
|
|
monkeypatch.setattr("app.services.earth_news._geocode_target_location", fake_geocode)
|
|
|
|
enriched = await _enrich_items_with_target_locations(
|
|
[item],
|
|
provider_client=FakeProviderClient(),
|
|
)
|
|
|
|
assert len(enriched) == 1
|
|
assert enriched[0].target_location is not None
|
|
assert enriched[0].target_location.latitude == 39.9042
|
|
assert enriched[0].target_location.longitude == 116.4074
|
|
assert enriched[0].target_location.label == "Beijing, China"
|
|
assert enriched[0].target_resolution_stage == "ai_inferred_target"
|
|
assert enriched[0].target_ai_attempted is True
|
|
assert enriched[0].target_ai_status == "success"
|
|
assert enriched[0].target_ai_error is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_enrich_items_with_target_locations_adds_localizations(monkeypatch):
|
|
item = ParsedNewsItem(
|
|
id="global-scan:localized",
|
|
title="Global leaders meet to discuss energy security",
|
|
summary="Officials said the talks focused on supply chains and grid resilience.",
|
|
url="https://example.com/energy-security",
|
|
source="Example Source",
|
|
feed_name="Global Monitor / World",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 14, 7, 13, 24, tzinfo=UTC),
|
|
)
|
|
|
|
async def fake_geocode(_query: str):
|
|
return {
|
|
"lat": "50.1109",
|
|
"lon": "8.6821",
|
|
"display_name": "Frankfurt am Main, Germany",
|
|
}
|
|
|
|
class FakeProviderClient:
|
|
async def analyze(self, _request):
|
|
class Response:
|
|
content = (
|
|
'{"location":{"country":"Germany","city":"Frankfurt",'
|
|
'"matched_location_name":"Frankfurt, Germany",'
|
|
'"latitude":null,"longitude":null,"confidence":0.77},'
|
|
'"localizations":{"zh-CN":{"title":"全球领导人讨论能源安全",'
|
|
'"summary":"官员表示，会谈聚焦供应链和电网韧性。"}}}'
|
|
)
|
|
|
|
return Response()
|
|
|
|
monkeypatch.setattr("app.services.earth_news._geocode_target_location", fake_geocode)
|
|
|
|
enriched = await _enrich_items_with_target_locations(
|
|
[item],
|
|
provider_client=FakeProviderClient(),
|
|
)
|
|
payload = _serialize_item(enriched[0], active_region="global")
|
|
|
|
assert payload["title"] == "Global leaders meet to discuss energy security"
|
|
assert payload["summary"] == "Officials said the talks focused on supply chains and grid resilience."
|
|
assert payload["localizations"]["zh-CN"]["title"] == "全球领导人讨论能源安全"
|
|
assert payload["display_title"] == "全球领导人讨论能源安全"
|
|
assert payload["display_summary"] == "官员表示，会谈聚焦供应链和电网韧性。"
|
|
assert payload["enrichment_status"] == "success"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_target_location_from_text_uses_country_hint(monkeypatch):
|
|
item = ParsedNewsItem(
|
|
id="bbc-world:country-hint",
|
|
title="Giant new dinosaur identified from fossils in Thailand",
|
|
summary="The nagatitan is the largest dinosaur found in South-East Asia.",
|
|
url="https://example.com/thailand-dinosaur",
|
|
source="BBC World",
|
|
feed_name="BBC World",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 14, 17, 28, 56, tzinfo=UTC),
|
|
)
|
|
|
|
target = await _extract_target_location_from_text(item)
|
|
|
|
assert target is not None
|
|
assert target.country == "泰国"
|
|
assert target.latitude == 15.87
|
|
assert target.longitude == 100.9925
|
|
assert target.source == "headline_country_hint"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_enrich_items_with_target_locations_records_ai_provider_error():
|
|
item = ParsedNewsItem(
|
|
id="global-scan:no-hint",
|
|
title="The New Geopolitics of Power: Whoever Controls Electrons Wins the Decade",
|
|
summary="A broad analysis of industrial policy and energy systems.",
|
|
url="https://example.com/geopolitics-power",
|
|
source="Example Source",
|
|
feed_name="Global Monitor / World",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 14, 17, 3, 10, tzinfo=UTC),
|
|
)
|
|
|
|
class FailingProviderClient:
|
|
async def analyze(self, _request):
|
|
raise RuntimeError("upstream ai timeout")
|
|
|
|
enriched = await _enrich_items_with_target_locations(
|
|
[item],
|
|
provider_client=FailingProviderClient(),
|
|
)
|
|
|
|
assert len(enriched) == 1
|
|
assert enriched[0].target_location is None
|
|
assert enriched[0].target_resolution_stage == "unresolved"
|
|
assert enriched[0].target_ai_attempted is True
|
|
assert enriched[0].target_ai_status == "provider_error"
|
|
assert enriched[0].target_ai_error == "upstream ai timeout"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_earth_news_payload_returns_anchor_items_and_enqueues_location_job(monkeypatch):
|
|
source = NewsFeedSource(
|
|
id="test-feed",
|
|
name="Test Feed",
|
|
region="global",
|
|
homepage_url="https://example.com",
|
|
feed_url="https://example.com/rss.xml",
|
|
)
|
|
item = ParsedNewsItem(
|
|
id="test-feed:timeout",
|
|
title="Example story",
|
|
summary="Example summary",
|
|
url="https://example.com/story",
|
|
source="Test Feed",
|
|
feed_name="Test Feed",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
|
)
|
|
|
|
async def fake_fetch_source(_client, feed_source):
|
|
return feed_source, [item], None
|
|
|
|
async def fake_get_cached_target_location_patch(_item_id):
|
|
return None
|
|
|
|
enqueued_payloads = []
|
|
|
|
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
|
enqueued_payloads.append(payload)
|
|
return True
|
|
|
|
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
|
|
monkeypatch.setattr("app.services.earth_news._fetch_source", fake_fetch_source)
|
|
monkeypatch.setattr(
|
|
"app.services.earth_news_queue.get_cached_target_location_patch",
|
|
fake_get_cached_target_location_patch,
|
|
)
|
|
monkeypatch.setattr(
|
|
"app.services.earth_news_queue.enqueue_target_location_job",
|
|
fake_enqueue_target_location_job,
|
|
)
|
|
|
|
payload = await get_earth_news_payload(provider_client=None)
|
|
|
|
assert len(payload["items"]) == 1
|
|
assert payload["items"][0]["id"] == "test-feed:timeout"
|
|
assert payload["items"][0]["display_title"] == ""
|
|
assert payload["items"][0]["display_summary"] == ""
|
|
assert payload["items"][0]["latitude"] == 20.0
|
|
assert payload["items"][0]["longitude"] == 0.0
|
|
assert payload["items"][0]["location_source"] == "region_anchor"
|
|
assert payload["items"][0]["verified"] is False
|
|
assert payload["items"][0]["location_meta"]["ai_status"] == "queued"
|
|
assert enqueued_payloads[0]["id"] == "test-feed:timeout"
|
|
assert payload["errors"] == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_earth_news_payload_uses_fresh_database_items_without_rss(monkeypatch):
|
|
db = object()
|
|
item = ParsedNewsItem(
|
|
id="db:fresh",
|
|
title="Fresh database story",
|
|
summary="Stored summary",
|
|
url="https://example.com/fresh",
|
|
source="Stored Source",
|
|
feed_name="Stored Feed",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
|
)
|
|
item.location_patch = {
|
|
"latitude": 39.9057136,
|
|
"longitude": 116.3912972,
|
|
"location_label": "北京市, 中国",
|
|
"location_source": "headline_location_hint",
|
|
"verified": True,
|
|
"location_meta": {"target": {"city": "Beijing"}, "anchor": {"region": "global"}},
|
|
}
|
|
|
|
async def fake_get_earth_news_freshness(_db, *, active_region):
|
|
return 12, datetime.now(UTC)
|
|
|
|
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
|
assert limit == 12
|
|
return [item]
|
|
|
|
async def fail_fetch(_sources):
|
|
raise AssertionError("fresh database items should not fetch RSS")
|
|
|
|
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
|
|
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_items", fake_list_earth_news_items)
|
|
monkeypatch.setattr("app.services.earth_news._fetch_rss_items_for_sources", fail_fetch)
|
|
|
|
payload = await get_earth_news_payload(db=db)
|
|
|
|
assert payload["items"][0]["id"] == "db:fresh"
|
|
assert payload["items"][0]["verified"] is True
|
|
assert payload["items"][0]["latitude"] == 39.9057136
|
|
assert payload["stale"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_earth_news_payload_keeps_current_items_and_all_cruise_items(monkeypatch):
|
|
class FakeDb:
|
|
execute = object()
|
|
|
|
current_item = ParsedNewsItem(
|
|
id="db:current",
|
|
title="Current region story",
|
|
summary="Current summary",
|
|
url="https://example.com/current",
|
|
source="Stored Source",
|
|
feed_name="Stored Feed",
|
|
feed_region="americas",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
|
)
|
|
cruise_item = ParsedNewsItem(
|
|
id="db:apac",
|
|
title="APAC story",
|
|
summary="APAC summary",
|
|
url="https://example.com/apac",
|
|
source="Stored Source",
|
|
feed_name="Stored Feed",
|
|
feed_region="asia-pacific",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 15, 4, 0, tzinfo=UTC),
|
|
)
|
|
|
|
async def fake_get_earth_news_freshness(_db, *, active_region):
|
|
return 12, datetime.now(UTC)
|
|
|
|
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
|
return [current_item]
|
|
|
|
async def fake_list_earth_news_cruise_items(_db, *, limit):
|
|
return [current_item, cruise_item]
|
|
|
|
async def fake_enqueue_target_location_job(_payload, **_kwargs):
|
|
return True
|
|
|
|
async def fail_fetch(_sources):
|
|
raise AssertionError("fresh database items should not fetch RSS")
|
|
|
|
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
|
|
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_items", fake_list_earth_news_items)
|
|
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_cruise_items", fake_list_earth_news_cruise_items)
|
|
monkeypatch.setattr("app.services.earth_news_queue.enqueue_target_location_job", fake_enqueue_target_location_job)
|
|
monkeypatch.setattr("app.services.earth_news._fetch_rss_items_for_sources", fail_fetch)
|
|
|
|
payload = await get_earth_news_payload(lat=35.0, lon=-100.0, db=FakeDb())
|
|
|
|
assert [item["id"] for item in payload["items"]] == ["db:current"]
|
|
assert [item["id"] for item in payload["cruise_items"]] == ["db:current", "db:apac"]
|
|
assert payload["cruise_items"][1]["region"] == "asia-pacific"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_earth_news_payload_initializes_empty_database_from_rss(monkeypatch):
|
|
db = object()
|
|
source = NewsFeedSource(
|
|
id="test-feed",
|
|
name="Test Feed",
|
|
region="global",
|
|
homepage_url="https://example.com",
|
|
feed_url="https://example.com/rss.xml",
|
|
)
|
|
item = ParsedNewsItem(
|
|
id="test-feed:init",
|
|
title="Initial RSS story",
|
|
summary="Initial summary",
|
|
url="https://example.com/init",
|
|
source="Test Feed",
|
|
feed_name="Test Feed",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
|
)
|
|
item.location_patch = {
|
|
"latitude": 20.0,
|
|
"longitude": 0.0,
|
|
"location_label": "全球",
|
|
"location_source": "region_anchor",
|
|
"verified": False,
|
|
"location_meta": {"target": None, "anchor": {"region": "global"}},
|
|
}
|
|
upserted = []
|
|
enqueued = []
|
|
|
|
async def fake_get_earth_news_freshness(_db, *, active_region):
|
|
return 0, None
|
|
|
|
async def fake_fetch_rss_items_for_sources(_sources):
|
|
return [item], []
|
|
|
|
async def fake_upsert_earth_news_items(_db, items):
|
|
upserted.extend(items)
|
|
return len(items)
|
|
|
|
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
|
return [item]
|
|
|
|
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
|
enqueued.append(payload)
|
|
return True
|
|
|
|
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
|
|
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
|
|
monkeypatch.setattr("app.services.earth_news._fetch_rss_items_for_sources", fake_fetch_rss_items_for_sources)
|
|
monkeypatch.setattr("app.services.earth_news_store.upsert_earth_news_items", fake_upsert_earth_news_items)
|
|
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_items", fake_list_earth_news_items)
|
|
monkeypatch.setattr("app.services.earth_news_queue.enqueue_target_location_job", fake_enqueue_target_location_job)
|
|
|
|
payload = await get_earth_news_payload(db=db)
|
|
|
|
assert upserted[0].id == "test-feed:init"
|
|
assert payload["items"][0]["id"] == "test-feed:init"
|
|
assert payload["items"][0]["verified"] is False
|
|
assert enqueued[0]["id"] == "test-feed:init"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_earth_news_payload_supplements_stale_database_items(monkeypatch):
|
|
db = object()
|
|
old_item = ParsedNewsItem(
|
|
id="db:old",
|
|
title="Old story",
|
|
summary="Old summary",
|
|
url="https://example.com/old",
|
|
source="Stored Source",
|
|
feed_name="Stored Feed",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 14, 3, 0, tzinfo=UTC),
|
|
)
|
|
old_item.location_patch = {
|
|
"latitude": 20.0,
|
|
"longitude": 0.0,
|
|
"location_label": "全球",
|
|
"location_source": "region_anchor",
|
|
"verified": False,
|
|
"location_meta": {"target": None, "anchor": {"region": "global"}},
|
|
}
|
|
fetched = []
|
|
|
|
async def fake_get_earth_news_freshness(_db, *, active_region):
|
|
return 12, datetime(2026, 5, 14, 3, 0, tzinfo=UTC)
|
|
|
|
async def fake_fetch_rss_items_for_sources(_sources):
|
|
fetched.append(True)
|
|
return [old_item], []
|
|
|
|
async def fake_upsert_earth_news_items(_db, items):
|
|
return len(items)
|
|
|
|
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
|
return [old_item]
|
|
|
|
async def fake_enqueue_target_location_job(_payload, **_kwargs):
|
|
return True
|
|
|
|
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
|
|
monkeypatch.setattr("app.services.earth_news._fetch_rss_items_for_sources", fake_fetch_rss_items_for_sources)
|
|
monkeypatch.setattr("app.services.earth_news_store.upsert_earth_news_items", fake_upsert_earth_news_items)
|
|
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_items", fake_list_earth_news_items)
|
|
monkeypatch.setattr("app.services.earth_news_queue.enqueue_target_location_job", fake_enqueue_target_location_job)
|
|
|
|
payload = await get_earth_news_payload(db=db)
|
|
|
|
assert fetched == [True]
|
|
assert payload["items"][0]["id"] == "db:old"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_earth_news_payload_merges_cached_location_patch(monkeypatch):
|
|
source = NewsFeedSource(
|
|
id="test-feed",
|
|
name="Test Feed",
|
|
region="global",
|
|
homepage_url="https://example.com",
|
|
feed_url="https://example.com/rss.xml",
|
|
)
|
|
item = ParsedNewsItem(
|
|
id="test-feed:cached",
|
|
title="Cached story",
|
|
summary="Cached summary",
|
|
url="https://example.com/cached",
|
|
source="Test Feed",
|
|
feed_name="Test Feed",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
|
)
|
|
cached_patch = {
|
|
"latitude": 39.9057136,
|
|
"longitude": 116.3912972,
|
|
"location_label": "北京市, 中国",
|
|
"location_source": "headline_location_hint",
|
|
"verified": True,
|
|
"location_meta": {
|
|
"resolution_stage": "headline_location_hint",
|
|
"ai_attempted": False,
|
|
"ai_status": "skipped_text_hint",
|
|
"ai_error": None,
|
|
"debug_note": "text hint matched 北京市, 中国",
|
|
"target": {"city": "Beijing"},
|
|
"anchor": {"region": "global"},
|
|
},
|
|
}
|
|
|
|
async def fake_fetch_source(_client, feed_source):
|
|
return feed_source, [item], None
|
|
|
|
async def fake_get_cached_target_location_patch(_item_id):
|
|
return cached_patch
|
|
|
|
enqueued = []
|
|
|
|
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
|
enqueued.append(payload)
|
|
return True
|
|
|
|
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
|
|
monkeypatch.setattr("app.services.earth_news._fetch_source", fake_fetch_source)
|
|
monkeypatch.setattr(
|
|
"app.services.earth_news_queue.get_cached_target_location_patch",
|
|
fake_get_cached_target_location_patch,
|
|
)
|
|
monkeypatch.setattr(
|
|
"app.services.earth_news_queue.enqueue_target_location_job",
|
|
fake_enqueue_target_location_job,
|
|
)
|
|
|
|
payload = await get_earth_news_payload(provider_client=None)
|
|
|
|
assert payload["items"][0]["latitude"] == 39.9057136
|
|
assert payload["items"][0]["longitude"] == 116.3912972
|
|
assert payload["items"][0]["verified"] is True
|
|
assert payload["items"][0]["location_source"] == "headline_location_hint"
|
|
assert enqueued[0]["id"] == "test-feed:cached"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_earth_news_payload_requeues_cached_failed_localization(monkeypatch):
|
|
source = NewsFeedSource(
|
|
id="test-feed",
|
|
name="Test Feed",
|
|
region="global",
|
|
homepage_url="https://example.com",
|
|
feed_url="https://example.com/rss.xml",
|
|
)
|
|
item = ParsedNewsItem(
|
|
id="test-feed:failed-localization",
|
|
title="Failed localization story",
|
|
summary="English source summary.",
|
|
url="https://example.com/failed-localization",
|
|
source="Test Feed",
|
|
feed_name="Test Feed",
|
|
feed_region="global",
|
|
homepage_url="https://example.com",
|
|
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
|
)
|
|
cached_patch = {
|
|
"latitude": 20.0,
|
|
"longitude": 0.0,
|
|
"location_label": "全球",
|
|
"location_source": "region_anchor",
|
|
"verified": False,
|
|
"location_meta": {"target": None, "anchor": {"region": "global"}},
|
|
"content_language": "en",
|
|
"localizations": {},
|
|
"enrichment_status": "parse_error",
|
|
"enrichment_error": "AI response did not contain a parseable JSON object.",
|
|
"enriched_at": None,
|
|
}
|
|
enqueued = []
|
|
|
|
async def fake_fetch_source(_client, feed_source):
|
|
return feed_source, [item], None
|
|
|
|
async def fake_get_cached_target_location_patch(_item_id):
|
|
return cached_patch
|
|
|
|
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
|
enqueued.append(payload)
|
|
return True
|
|
|
|
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
|
|
monkeypatch.setattr("app.services.earth_news._fetch_source", fake_fetch_source)
|
|
monkeypatch.setattr(
|
|
"app.services.earth_news_queue.get_cached_target_location_patch",
|
|
fake_get_cached_target_location_patch,
|
|
)
|
|
monkeypatch.setattr(
|
|
"app.services.earth_news_queue.enqueue_target_location_job",
|
|
fake_enqueue_target_location_job,
|
|
)
|
|
|
|
payload = await get_earth_news_payload(provider_client=None)
|
|
|
|
assert enqueued[0]["id"] == "test-feed:failed-localization"
|
|
assert payload["items"][0]["display_title"] == ""
|
|
assert payload["items"][0]["enrichment_status"] == "queued"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_worker_processes_target_location_message_and_returns_patch(monkeypatch):
|
|
message = NewsTargetLocationMessage(
|
|
message_id="1-0",
|
|
item_id="bbc-world:worker",
|
|
payload={
|
|
"id": "bbc-world:worker",
|
|
"title": "Ukraine rescuers pull dead from rubble of Kyiv flats",
|
|
"summary": "Massive Russian drone and missile attacks in Ukraine's capital.",
|
|
"url": "https://example.com/kyiv",
|
|
"source": "BBC World",
|
|
"feed_name": "BBC World",
|
|
"feed_region": "global",
|
|
"homepage_url": "https://www.bbc.com/news/world",
|
|
"published_at": "2026-05-14T13:16:32Z",
|
|
},
|
|
)
|
|
|
|
async def fake_geocode(_query: str):
|
|
return {
|
|
"lat": "50.4500336",
|
|
"lon": "30.5241361",
|
|
"display_name": "Київ, Україна",
|
|
}
|
|
|
|
saved = {}
|
|
broadcasted = {}
|
|
|
|
async def fake_save_target_location_patch(item_id, patch):
|
|
saved["item_id"] = item_id
|
|
saved["patch"] = patch
|
|
|
|
async def fake_update_earth_news_item_location(_session, *, item_id, patch):
|
|
saved["db_item_id"] = item_id
|
|
saved["db_patch"] = patch
|
|
return True
|
|
|
|
async def fake_broadcast_custom(channel, data):
|
|
broadcasted["channel"] = channel
|
|
broadcasted["data"] = data
|
|
|
|
class FakeSession:
|
|
async def commit(self):
|
|
saved["committed"] = True
|
|
|
|
class FakeSessionFactory:
|
|
async def __aenter__(self):
|
|
return FakeSession()
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
monkeypatch.setattr("app.services.earth_news._geocode_target_location", fake_geocode)
|
|
monkeypatch.setattr(
|
|
"app.services.earth_news_worker.save_target_location_patch",
|
|
fake_save_target_location_patch,
|
|
)
|
|
monkeypatch.setattr(
|
|
"app.services.earth_news_worker.update_earth_news_item_location",
|
|
fake_update_earth_news_item_location,
|
|
)
|
|
monkeypatch.setattr(
|
|
"app.services.earth_news_worker.async_session_factory",
|
|
lambda: FakeSessionFactory(),
|
|
)
|
|
monkeypatch.setattr(
|
|
"app.services.earth_news_worker.broadcaster.broadcast_custom",
|
|
fake_broadcast_custom,
|
|
)
|
|
|
|
patch = await process_target_location_message(message, provider_client=None)
|
|
|
|
assert patch["latitude"] == 50.4500336
|
|
assert patch["longitude"] == 30.5241361
|
|
assert patch["location_source"] == "headline_location_hint"
|
|
assert patch["verified"] is True
|
|
assert saved["item_id"] == "bbc-world:worker"
|
|
assert saved["db_item_id"] == "bbc-world:worker"
|
|
assert saved["committed"] is True
|
|
assert broadcasted["channel"] == "earth_news"
|
|
assert broadcasted["data"]["item_id"] == "bbc-world:worker"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_media_news_archive_collector_maps_news_items(monkeypatch):
|
|
collector = MediaNewsArchiveCollector()
|
|
collector._db_session = object()
|
|
record = SimpleNamespace(
|
|
id="bbc-world:archive",
|
|
title="Archived news",
|
|
summary="Archived summary",
|
|
url="https://example.com/archive",
|
|
source="BBC World",
|
|
feed_name="BBC World",
|
|
region="global",
|
|
homepage_url="https://www.bbc.com/news/world",
|
|
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
|
latitude=39.9057136,
|
|
longitude=116.3912972,
|
|
location_label="北京市, 中国",
|
|
location_source="headline_location_hint",
|
|
verified=True,
|
|
location_meta={"target": {"country": "中国", "city": "Beijing"}},
|
|
content_language="en",
|
|
localizations={"zh-CN": {"title": "归档新闻", "summary": "归档概要"}},
|
|
enrichment_status="success",
|
|
enrichment_error=None,
|
|
enriched_at=datetime(2026, 5, 15, 3, 6, tzinfo=UTC),
|
|
first_seen_at=datetime(2026, 5, 15, 2, 0, tzinfo=UTC),
|
|
last_seen_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
|
resolved_at=datetime(2026, 5, 15, 3, 5, tzinfo=UTC),
|
|
)
|
|
|
|
async def fake_list_all_earth_news_records(_db):
|
|
return [record]
|
|
|
|
monkeypatch.setattr(
|
|
"app.services.collectors.media_news_archive.list_all_earth_news_records",
|
|
fake_list_all_earth_news_records,
|
|
)
|
|
|
|
items = await collector.fetch()
|
|
|
|
assert items[0]["source_id"] == "bbc-world:archive"
|
|
assert collector.data_type == "news_item"
|
|
assert items[0]["country"] == "中国"
|
|
assert items[0]["city"] == "Beijing"
|
|
assert items[0]["latitude"] == 39.9057136
|
|
assert items[0]["metadata"]["verified"] is True
|
|
assert "localizations" not in items[0]["metadata"]
|
|
assert "enrichment_status" not in items[0]["metadata"]
|