release: bump version to 0.59.0
Some checks failed
ci / backend (push) Has been cancelled
ci / frontend (push) Has been cancelled
ci / delivery (push) Has been cancelled
release / images (push) Has been cancelled

This commit is contained in:
rayd1o
2026-05-16 05:02:05 +08:00
parent 93eb41a9f7
commit 9b913a3b83
86 changed files with 3645 additions and 1198 deletions

View File

@@ -8,6 +8,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.models.alert import Alert, AlertSeverity, AlertStatus
from app.schemas.ai import AlertBriefRequest, SituationalAnalysisRequest
from app.ai_tasks.prompts import get_effective_prompt
ALERT_BRIEF_PROMPT_KEY = "alerts.brief"
def _format_counter(counter: Counter[str], empty_text: str = "") -> str:
@@ -84,11 +87,13 @@ async def build_alert_brief_request(
"top_datasources": dict(datasource_counts.most_common(6)),
"top_active_datasources": dict(active_datasource_counts.most_common(5)),
}
prompt = await get_effective_prompt(db, ALERT_BRIEF_PROMPT_KEY)
return (
SituationalAnalysisRequest(
title="告警态势 AI 简报",
objective="基于当前告警总量、严重度、状态、数据源分布与最近告警摘录,生成一份面向值班人员的简明告警态势简报,突出待处理风险、告警集中点和优先动作。",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
observations=facts,
constraints=[
"明确区分事实、推断与建议。",

View File

@@ -11,9 +11,12 @@ from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.schemas.ai import SituationalAnalysisRequest
from app.ai_tasks.prompts import get_effective_prompt
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.bgp_enrichment import lookup_prefix_geography
BGP_BRIEF_PROMPT_KEY = "bgp.brief"
def _format_counter(counter: dict[str, int], empty_text: str = "") -> str:
if not counter:
@@ -243,10 +246,12 @@ async def build_bgp_brief_request(
for prefix, item in list(prefix_geographies.items())[:8]
},
}
prompt = await get_effective_prompt(db, BGP_BRIEF_PROMPT_KEY)
return SituationalAnalysisRequest(
title="BGP 态势 AI 简报",
objective="基于当前 BGP incidents、anomalies、原始观测事件、观测站覆盖与 prefix geography 证据,生成一份面向操作员的简明态势简报,突出区域热点、观测偏差、当前风险、证据和优先动作。",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
observations=observations_lines,
constraints=[
"明确区分事实、推断与建议。",

View File

@@ -39,12 +39,6 @@ from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
from app.services.collectors.media_news_archive import MediaNewsArchiveCollector
from app.services.collectors.aisstream import AISStreamCollector
from app.services.collectors.vessel_ais import VesselAISCollector
from app.services.collectors.earth_boundaries import (
EarthAdmin0BoundaryCollector,
EarthBoundaryTileCollector,
EarthClaimLinesCollector,
EarthCoastlineCollector,
)
collector_registry.register(TOP500Collector())
collector_registry.register(EpochAIGPUCollector())
@@ -75,10 +69,6 @@ collector_registry.register(NewsLiveStreamsCollector())
collector_registry.register(MediaNewsArchiveCollector())
collector_registry.register(VesselAISCollector())
collector_registry.register(AISStreamCollector())
collector_registry.register(EarthAdmin0BoundaryCollector())
collector_registry.register(EarthCoastlineCollector())
collector_registry.register(EarthClaimLinesCollector())
collector_registry.register(EarthBoundaryTileCollector())
__all__ = [
"BaseCollector",
@@ -115,8 +105,4 @@ __all__ = [
"MediaNewsArchiveCollector",
"VesselAISCollector",
"AISStreamCollector",
"EarthAdmin0BoundaryCollector",
"EarthCoastlineCollector",
"EarthClaimLinesCollector",
"EarthBoundaryTileCollector",
]

View File

@@ -1,578 +0,0 @@
"""Earth boundary source and static tile collector."""
from __future__ import annotations
import asyncio
import hashlib
import json
import shutil
import sys
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import httpx
from sqlalchemy import select
from app.core.earth_boundary_defaults import EARTH_BOUNDARY_DEFAULT_SOURCES, default_earth_boundary_config
from app.models.collected_data import CollectedData
from app.models.datasource_config import DataSourceConfig
from app.services.collectors.base import BaseCollector
from app.services.custom_datasource_runtime import build_query_params, build_request_headers
REPO_ROOT = Path(__file__).resolve().parents[4]
SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
PMTILES_ARTIFACT_PATH = REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
BUILD_CONFIG = {
"builder": "scripts/build_earth_boundary_china_pov_geojson.py",
"format": "geojson-high-precision",
"production_target": "geojson-high-precision",
}
EARTH_BOUNDARY_SOURCE_COLLECTORS = {
"earth_admin0_boundaries": "admin0-boundaries",
"earth_coastline": "coastline",
"earth_claim_lines": "claim-lines",
}
def _read_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def _directory_stats(path: Path) -> dict[str, int]:
if not path.exists():
return {"file_count": 0, "size_bytes": 0}
files = [item for item in path.rglob("*") if item.is_file()]
return {
"file_count": len(files),
"size_bytes": sum(item.stat().st_size for item in files),
}
def _stable_json_hash(payload: Any) -> str:
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def _build_input_hash(source_manifest: dict[str, Any]) -> str:
return _stable_json_hash(
{
"source_manifest_schema": source_manifest.get("schema"),
"sources": [
{
"id": source.get("id"),
"sha256": source.get("sha256"),
"kind": source.get("kind"),
"pov": source.get("pov"),
}
for source in source_manifest.get("sources", [])
],
"pov_policy": source_manifest.get("povPolicy"),
"build_config": BUILD_CONFIG,
}
)
def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
if not boundary_manifest:
return False
if boundary_manifest.get("buildInputHash") != build_input_hash:
return False
if not BOUNDARY_OUTPUT_DIR.exists():
return False
if boundary_manifest.get("tileProvider") == "pmtiles-mvt":
return PMTILES_ARTIFACT_PATH.exists()
return (BOUNDARY_OUTPUT_DIR / "base.geojson").exists() and (
BOUNDARY_OUTPUT_DIR / "hover-index.geojson"
).exists()
def _sha256_bytes(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def _json_feature_count(payload: Any) -> int:
if isinstance(payload, dict) and isinstance(payload.get("features"), list):
return len(payload["features"])
if isinstance(payload, list):
return len(payload)
return 1 if payload else 0
def _json_sample_properties(payload: Any) -> dict[str, Any]:
feature = None
if isinstance(payload, dict) and isinstance(payload.get("features"), list) and payload["features"]:
feature = payload["features"][0]
elif isinstance(payload, list) and payload:
feature = payload[0]
elif isinstance(payload, dict):
feature = payload
if not isinstance(feature, dict):
return {}
props = feature.get("properties") if isinstance(feature.get("properties"), dict) else feature
return {str(key): value for key, value in list(props.items())[:20]}
def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
if suffix in {".json", ".geojson", ".zip", ".pbf"}:
return suffix
if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
return ".geojson"
if "json" in content_type:
return ".json"
return ".dat"
async def _load_datasource_config(db, name: str) -> DataSourceConfig | None:
if db is None:
return None
result = await db.execute(
select(DataSourceConfig)
.where(DataSourceConfig.name == name)
.where(DataSourceConfig.is_active.is_(True))
.order_by(DataSourceConfig.id.desc())
.limit(1)
)
return result.scalar_one_or_none()
async def _latest_boundary_source_record(db, source_name: str) -> CollectedData | None:
result = await db.execute(
select(CollectedData)
.where(CollectedData.source == source_name)
.where(CollectedData.data_type == "earth_boundary_source")
.where(CollectedData.is_current.is_(True))
.order_by(CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
.limit(1)
)
return result.scalar_one_or_none()
class EarthBoundarySourceCollector(BaseCollector):
priority = "P1"
module = "L3"
frequency_hours = 168
data_type = "earth_boundary_source"
fail_on_empty = True
source_kind = "unknown"
def _default_config(self):
source = EARTH_BOUNDARY_DEFAULT_SOURCES.get(self.name)
if not source:
return None
return SimpleNamespace(
name=self.name,
description=f"内置默认源:{self.name}",
endpoint=source["endpoint"],
source_type="http",
auth_type="none",
auth_config={},
headers={},
config=default_earth_boundary_config(self.name),
)
async def _download_payload(self, config: DataSourceConfig) -> tuple[bytes, str]:
request_config = config.config or {}
method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
if method not in {"GET", "POST"}:
raise RuntimeError("Earth boundary source collectors support GET and POST only")
endpoint = str(config.endpoint or "").strip()
if not endpoint:
raise RuntimeError(
f"{self.name} requires an endpoint in Collector Settings before it can collect data"
)
if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
path = Path(endpoint.removeprefix("file://")).expanduser()
return path.read_bytes(), "application/octet-stream"
headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {})
params = build_query_params(config.auth_type, config.auth_config or {}, request_config)
timeout = float(request_config.get("timeout", 120))
json_body = request_config.get("json_body") or request_config.get("body")
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
response = await client.request(
method,
endpoint,
headers=headers,
params=params or None,
json=json_body if isinstance(json_body, (dict, list)) else None,
)
response.raise_for_status()
return response.content, response.headers.get("content-type", "")
async def fetch(self) -> list[dict[str, Any]]:
db = getattr(self, "_db_session", None)
config = await _load_datasource_config(db, self.name) or self._default_config()
if config is None:
raise RuntimeError(
f"{self.name} has no active Collector Settings config and no built-in default source."
)
await self.set_phase("fetching_source", message=f"正在下载 {self.source_kind} 源数据")
payload, content_type = await self._download_payload(config)
sha256 = _sha256_bytes(payload)
endpoint = str(config.endpoint or "")
extension = _artifact_extension(endpoint, content_type, payload)
source_dir = SOURCE_OUTPUT_DIR / self.name
source_dir.mkdir(parents=True, exist_ok=True)
artifact_path = source_dir / f"{sha256}{extension}"
artifact_path.write_bytes(payload)
parsed: Any = None
if extension in {".json", ".geojson"}:
parsed = json.loads(payload.decode("utf-8"))
feature_count = _json_feature_count(parsed)
if feature_count <= 0:
raise RuntimeError(f"{self.name} downloaded data but found no JSON/GeoJSON features")
config_body = config.config or {}
target_schema = str(config_body.get("target_schema") or "earth_boundary_source")
if target_schema != "earth_boundary_source":
raise RuntimeError(f"{self.name} target_schema must be earth_boundary_source")
await self.update_phase_progress(
current=1,
total=1,
unit="artifact",
message=f"已保存 {feature_count}{self.source_kind} feature",
progress=100,
commit=True,
force=True,
)
relative_artifact_path = str(artifact_path.relative_to(REPO_ROOT))
return [
{
"id": f"{self.source_kind}:{sha256}",
"name": config.description or self.name,
"description": f"Earth boundary source artifact collected from configured endpoint",
"source_kind": self.source_kind,
"source_id": sha256,
"value": feature_count,
"unit": "features",
"metadata": {
"target_schema": target_schema,
"source_kind": self.source_kind,
"endpoint": endpoint,
"method": str(config_body.get("method") or config_body.get("request_method") or "GET").upper(),
"artifact_path": relative_artifact_path,
"sha256": sha256,
"feature_count": feature_count,
"size_bytes": len(payload),
"content_type": content_type,
"license": config_body.get("license"),
"mapping_json": config_body.get("mapping_json"),
"sample_properties": _json_sample_properties(parsed),
},
}
]
class EarthAdmin0BoundaryCollector(EarthBoundarySourceCollector):
name = "earth_admin0_boundaries"
source_kind = "admin0-boundaries"
class EarthCoastlineCollector(EarthBoundarySourceCollector):
name = "earth_coastline"
source_kind = "coastline"
class EarthClaimLinesCollector(EarthBoundarySourceCollector):
name = "earth_claim_lines"
source_kind = "claim-lines"
class EarthBoundaryTileCollector(BaseCollector):
name = "earth_boundary_tiles"
priority = "P1"
module = "L3"
frequency_hours = 168
data_type = "earth_boundary_tiles"
fail_on_empty = False
async def _run_step(self, args: list[str], *, allow_failure: bool = False) -> dict[str, Any]:
process = await asyncio.create_subprocess_exec(
sys.executable,
*args,
cwd=REPO_ROOT,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout_bytes, stderr_bytes = await process.communicate()
stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
if process.returncode != 0 and not allow_failure:
raise RuntimeError(stderr or stdout or f"command failed: {' '.join(args)}")
payload: dict[str, Any] = {
"stdout": stdout,
"stderr": stderr,
"returncode": process.returncode,
}
last_line = stdout.splitlines()[-1:] or []
if last_line:
try:
payload["result"] = json.loads(last_line[0])
except json.JSONDecodeError:
payload["result"] = last_line[0]
return payload
async def fetch(self) -> list[dict[str, Any]]:
db = getattr(self, "_db_session", None)
if db is None:
raise RuntimeError("Earth PMTiles builder requires an active database session")
await self.set_phase("checking_sources", message="正在检查三类 Earth 边界源")
await self.update_phase_progress(
current=0,
total=3,
unit="steps",
message="读取 admin0 / coastline / claim-lines 最新采集结果",
progress=5,
commit=True,
force=True,
)
source_records: dict[str, CollectedData] = {}
missing_sources: list[str] = []
for source_name in EARTH_BOUNDARY_SOURCE_COLLECTORS:
record = await _latest_boundary_source_record(db, source_name)
if record is None:
missing_sources.append(source_name)
else:
source_records[source_name] = record
if missing_sources:
missing_text = ", ".join(missing_sources)
raise RuntimeError(f"未就绪:缺少 {missing_text}。不会更新 Earth 国界。")
sources = []
for source_name, record in source_records.items():
metadata = record.extra_data or {}
artifact_path = metadata.get("artifact_path")
if not artifact_path or not (REPO_ROOT / str(artifact_path)).exists():
missing_sources.append(f"{source_name}: artifact missing")
continue
sources.append(
{
"id": source_name,
"kind": metadata.get("source_kind") or EARTH_BOUNDARY_SOURCE_COLLECTORS[source_name],
"path": str(artifact_path),
"sha256": metadata.get("sha256"),
"featureCount": metadata.get("feature_count"),
"license": metadata.get("license"),
}
)
if missing_sources:
missing_text = ", ".join(missing_sources)
raise RuntimeError(f"未就绪:{missing_text}。不会更新 Earth 国界。")
source_manifest = {
"schema": "planet-earth-boundary-sources/v2",
"sources": sources,
"povPolicy": _read_json(POV_POLICY_PATH),
}
SOURCE_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
with SOURCE_MANIFEST_PATH.open("w", encoding="utf-8") as f:
json.dump(source_manifest, f, ensure_ascii=False, indent=2)
f.write("\n")
build_input_hash = _build_input_hash(source_manifest)
await self.update_phase_progress(
current=1,
total=3,
unit="steps",
message="三类边界源已就绪",
progress=35,
commit=True,
force=True,
)
await self.set_phase("checking_readiness", message="正在检查 Earth 国界构建就绪状态")
missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
readiness_result = {
"returncode": 0,
"result": {
"ready": True,
"failures": [f"external tool not found in PATH: {tool}" for tool in missing_tools],
"sources": sources,
"artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"fallback_builder": "geojson-high-precision" if missing_tools else None,
},
}
await self.set_phase("building_tiles", message="正在检查 Earth 国界瓦片产物")
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
if build_skipped:
build_result = {
"result": {
"status": "unchanged",
"reason": "source manifest and build config hash unchanged",
"buildInputHash": build_input_hash,
}
}
elif shutil.which("tippecanoe") and shutil.which("pmtiles"):
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
coastline = next(source for source in sources if source["kind"] == "coastline")
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
build_result = await self._run_step(
[
"scripts/build_earth_boundary_pmtiles.py",
"--admin0-source",
admin0["path"],
"--coastline-source",
coastline["path"],
"--claims-source",
claim_lines["path"],
"--output",
str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"--manifest",
str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
"--build-input-hash",
build_input_hash,
"--pov-policy",
str(POV_POLICY_PATH.relative_to(REPO_ROOT)),
]
)
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
else:
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
coastline = next(source for source in sources if source["kind"] == "coastline")
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
build_result = await self._run_step(
[
"scripts/build_earth_boundary_china_pov_geojson.py",
"--admin0-source",
admin0["path"],
"--coastline-source",
coastline["path"],
"--claims-source",
claim_lines["path"],
"--output-dir",
str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
"--build-input-hash",
build_input_hash,
]
)
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
pmtiles_size = PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0
await self.update_phase_progress(
current=2,
total=3,
unit="steps",
message=(
"边界源未变化,已跳过瓦片重建"
if build_skipped
else f"已生成 {boundary_manifest.get('tileProvider') or 'boundary'} 国界产物"
),
progress=80,
commit=True,
force=True,
)
await self.set_phase("indexing_artifacts", message="正在登记边界瓦片产物")
tile_counts = boundary_manifest.get("tiles", {}).get("countsByZoom", {})
records = [
{
"id": "source-manifest",
"name": "Earth boundary source manifest",
"description": "Offline source collection manifest for Earth boundary tiles",
"value": len(source_manifest.get("sources", [])),
"unit": "sources",
"metadata": {
"manifest_path": str(SOURCE_MANIFEST_PATH.relative_to(REPO_ROOT)),
"manifest": source_manifest,
"collector_result": {
"status": "loaded_from_collected_data",
"sources": [source["id"] for source in sources],
},
},
},
{
"id": "production-readiness",
"name": "Earth boundary production readiness",
"description": "Checks whether all source artifacts and boundary build tooling are available",
"value": 1 if readiness_result.get("returncode") == 0 else 0,
"unit": "ready",
"metadata": {
"result": readiness_result.get("result"),
"returncode": readiness_result.get("returncode"),
},
},
{
"id": "boundary-manifest",
"name": "Earth boundary tile manifest",
"description": "Versioned static vector tile manifest for Earth country boundaries",
"value": boundary_stats["file_count"],
"unit": "files",
"metadata": {
"manifest_path": str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
"output_dir": str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
"size_bytes": boundary_stats["size_bytes"],
"manifest": boundary_manifest,
"collector_result": build_result.get("result"),
"build_skipped": build_skipped,
"production_target": BUILD_CONFIG["production_target"],
"pmtiles_artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"pmtiles_exists": pmtiles_exists,
},
},
]
if pmtiles_exists:
records.append(
{
"id": "pmtiles-artifact",
"name": "Earth boundary PMTiles artifact",
"description": "Single-file PMTiles/MVT artifact for Earth boundaries",
"value": pmtiles_size,
"unit": "bytes",
"metadata": {
"path": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"exists": pmtiles_exists,
"provider": "pmtiles-mvt",
},
}
)
for zoom, count in sorted(tile_counts.items(), key=lambda item: int(item[0])):
records.append(
{
"id": f"tile-z{zoom}",
"name": f"Earth boundary tiles z{zoom}",
"description": f"Generated Earth boundary tile count for zoom {zoom}",
"value": int(count),
"unit": "tiles",
"metadata": {
"zoom": int(zoom),
"tile_count": int(count),
"output_dir": str((BOUNDARY_OUTPUT_DIR / str(zoom)).relative_to(REPO_ROOT)),
},
}
)
await self.update_phase_progress(
current=3,
total=3,
unit="steps",
message="边界瓦片产物已登记",
progress=100,
commit=True,
force=True,
)
return records

View File

@@ -7,6 +7,7 @@ from typing import Any
from sqlalchemy import select
from app.ai_tasks.prompts import get_effective_prompt
from app.models.system_setting import SystemSetting
from app.schemas.ai import SituationalAnalysisRequest
from app.services.ai_client import AIProviderClient
@@ -15,6 +16,7 @@ from app.services.ai_tools.web_search import WebSearchClient, WebSearchError
CREDENTIAL_GUIDES_CATEGORY = "collector_credential_guides"
CREDENTIAL_GUIDE_PROMPT_KEY = "credential.guide"
@dataclass(frozen=True)
@@ -240,14 +242,12 @@ async def generate_credential_guide(
guide["sources"] = []
return guide
prompt = await get_effective_prompt(db, CREDENTIAL_GUIDE_PROMPT_KEY)
response = await ai_client.analyze(
SituationalAnalysisRequest(
title=f"Generate credential guide for {provider}",
objective=(
default.prompt
+ "\n只能根据 context.search_evidence 中的来源生成教程;"
+ "如果证据不足,明确说明需要以官方页面为准。"
),
objective=f"{default.prompt}\n{prompt.prompt}",
system_prompt=prompt.system_prompt or None,
context={
"provider": provider,
"current_default_guide": default.markdown,

View File

@@ -301,52 +301,6 @@ async def persist_mapped_records(
transport: str | None = None,
) -> int:
"""Persist validated mapped records to the destination for a target schema."""
if target_schema == "earth_boundary_source":
from app.models.collected_data import CollectedData
now = datetime.now(UTC)
written_count = 0
for index, record in enumerate(records):
source_id = record.get("source_id") or record.get("sha256") or str(index)
entity_key = f"{datasource_name}:{source_id}"
previous_result = await db.execute(
select(CollectedData)
.where(CollectedData.source == datasource_name)
.where(CollectedData.entity_key == entity_key)
.where(CollectedData.is_current.is_(True))
.order_by(CollectedData.id.desc())
.limit(1)
)
previous = previous_result.scalar_one_or_none()
if previous is not None:
previous.is_current = False
db.add(
CollectedData(
source=datasource_name,
source_id=str(source_id),
entity_key=entity_key,
data_type=target_schema,
name=record.get("name") or str(source_id),
description=f"Earth boundary source artifact: {record.get('source_kind')}",
extra_data={
**record,
"datasource_config_id": datasource_config_id,
"mapping_version": mapping_version,
"delivery_mode": delivery_mode or "polling",
"transport": transport or "http",
},
collected_at=now,
is_valid=1,
is_current=True,
previous_record_id=previous.id if previous else None,
change_type="updated" if previous else "created",
change_summary={},
)
)
written_count += 1
await db.commit()
return written_count
if target_schema == "vessel_ais":
from app.core.time import to_iso8601_utc
from app.core.websocket.broadcaster import broadcaster

View File

@@ -0,0 +1,671 @@
"""Earth boundary static asset service."""
from __future__ import annotations
import asyncio
import hashlib
import json
import shutil
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from uuid import uuid4
import httpx
REPO_ROOT = Path(__file__).resolve().parents[3]
SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
BUILD_RESULT_PATH = SOURCE_OUTPUT_DIR / "build-result.json"
BUILD_JOB_PATH = SOURCE_OUTPUT_DIR / "build-job.json"
BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
PMTILES_ARTIFACT_PATH = (
REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
)
LEGACY_GEOJSON_PATH = REPO_ROOT / "frontend/public/earth/data/countries-admin0.min.geojson"
POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
LOCAL_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.local.json"
EXAMPLE_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.example.json"
BOUNDARY_SOURCE_KINDS = {
"earth_admin0_boundaries": "admin0-boundaries",
"earth_coastline": "coastline",
"earth_claim_lines": "claim-lines",
}
DEFAULT_PUBLIC_BOUNDARY_SOURCES = {
"earth_admin0_boundaries": {
"displayName": "Natural Earth Admin-0 Countries",
"sourceKind": "admin0-boundaries",
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_countries.geojson",
"method": "GET",
"headers": {},
"auth_type": "none",
"license": "Natural Earth public domain",
},
"earth_coastline": {
"displayName": "Natural Earth Coastline",
"sourceKind": "coastline",
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_coastline.geojson",
"method": "GET",
"headers": {},
"auth_type": "none",
"license": "Natural Earth public domain",
},
"earth_claim_lines": {
"displayName": "Natural Earth Disputed Boundaries",
"sourceKind": "claim-lines",
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_boundary_lines_disputed_areas.geojson",
"method": "GET",
"headers": {},
"auth_type": "none",
"license": "Natural Earth public domain",
},
}
BUILD_CONFIG = {
"builder": "scripts/build_earth_boundary_pmtiles.py",
"format": "pmtiles+mvt",
"production_target": "pmtiles-mvt",
}
class EarthBoundaryBuildError(RuntimeError):
def __init__(self, message: str, *, code: str = "build_failed", details: Any = None) -> None:
super().__init__(message)
self.code = code
self.details = details
_build_job_lock = asyncio.Lock()
_build_task: asyncio.Task | None = None
_build_job_state: dict[str, Any] = {}
def _utc_now_iso() -> str:
return datetime.now(UTC).isoformat()
def _public_job_state() -> dict[str, Any]:
if _build_job_state:
return dict(_build_job_state)
return _read_json(BUILD_JOB_PATH)
def get_boundary_build_status() -> dict[str, Any]:
return {"job": _public_job_state()}
def _set_job_state(**updates: Any) -> dict[str, Any]:
global _build_job_state
current = dict(_build_job_state)
current.update(updates)
current["updated_at"] = _utc_now_iso()
_build_job_state = current
_write_json(BUILD_JOB_PATH, current)
return current
def _append_job_log(message: str) -> None:
logs = list(_build_job_state.get("logs") or [])
logs.append({"time": _utc_now_iso(), "message": message})
_set_job_state(logs=logs[-40:])
def _update_job_progress(progress: float, phase: str, message: str, **extra: Any) -> None:
bounded_progress = max(0, min(100, int(round(progress))))
_set_job_state(
status="running",
progress=bounded_progress,
phase=phase,
message=message,
**extra,
)
def _read_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
with path.open("r", encoding="utf-8") as f:
payload = json.load(f)
return payload if isinstance(payload, dict) else {}
def _write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as f:
json.dump(payload, f, ensure_ascii=False, indent=2)
f.write("\n")
def _sha256_bytes(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def _stable_json_hash(payload: Any) -> str:
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
if suffix in {".json", ".geojson", ".zip", ".pbf"}:
return suffix
if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
return ".geojson"
if "json" in content_type:
return ".json"
return ".dat"
def _json_feature_count(payload: Any) -> int:
if isinstance(payload, dict) and isinstance(payload.get("features"), list):
return len(payload["features"])
if isinstance(payload, list):
return len(payload)
return 1 if payload else 0
def _directory_stats(path: Path) -> dict[str, int]:
if not path.exists():
return {"file_count": 0, "size_bytes": 0}
files = [item for item in path.rglob("*") if item.is_file()]
return {"file_count": len(files), "size_bytes": sum(item.stat().st_size for item in files)}
def _load_source_feature_collection(source: dict[str, Any]) -> dict[str, Any]:
path = REPO_ROOT / source["path"]
payload = _read_json(path)
features = payload.get("features") if isinstance(payload, dict) else None
return {
"type": "FeatureCollection",
"features": features if isinstance(features, list) else [],
}
def _write_high_precision_geojson_manifest(
sources: list[dict[str, Any]],
build_input_hash: str,
missing_tools: list[str],
) -> dict[str, Any]:
BOUNDARY_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
coastline = next(source for source in sources if source["kind"] == "coastline")
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
admin0_payload = _load_source_feature_collection(admin0)
coastline_payload = _load_source_feature_collection(coastline)
claim_payload = _load_source_feature_collection(claim_lines)
for feature in coastline_payload["features"]:
props = feature.setdefault("properties", {})
if isinstance(props, dict):
props["PLANET_LAYER"] = "coastline"
base_payload = {
"type": "FeatureCollection",
"features": [*admin0_payload["features"], *coastline_payload["features"]],
}
base_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-base.geojson"
hover_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-hover.geojson"
claim_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-claims.geojson"
_write_json(base_path, base_payload)
_write_json(hover_path, admin0_payload)
_write_json(claim_path, claim_payload)
manifest = {
"version": "natural-earth-v1",
"builtAt": _utc_now_iso(),
"tileProvider": "geojson-high-precision",
"format": "geojson-directory",
"buildInputHash": build_input_hash,
"base": base_path.name,
"hoverIndex": hover_path.name,
"claimLine": claim_path.name,
"sourceFeatureCount": {
"admin0": len(admin0_payload["features"]),
"coastline": len(coastline_payload["features"]),
"claimLines": len(claim_payload["features"]),
},
"pmtiles": None,
"missingTools": missing_tools,
}
_write_json(BOUNDARY_MANIFEST_PATH, manifest)
return manifest
def _relative(path: Path) -> str:
return str(path.relative_to(REPO_ROOT))
def load_boundary_config() -> tuple[dict[str, Any], str]:
if LOCAL_CONFIG_PATH.exists():
return _read_json(LOCAL_CONFIG_PATH), "local"
return _read_json(EXAMPLE_CONFIG_PATH), "example"
def save_boundary_config(payload: dict[str, Any]) -> dict[str, Any]:
if not isinstance(payload, dict):
raise EarthBoundaryBuildError("Earth boundary config must be a JSON object", code="invalid_config")
_write_json(LOCAL_CONFIG_PATH, payload)
return get_boundary_status()
def _source_configs(payload: dict[str, Any]) -> dict[str, Any]:
raw_sources = payload.get("collectorConfigs") or payload.get("sources") or {}
return raw_sources if isinstance(raw_sources, dict) else {}
def _is_placeholder_endpoint(endpoint: Any) -> bool:
value = str(endpoint or "").strip()
return not value or "example.com" in value
def _source_configs_with_defaults(payload: dict[str, Any]) -> dict[str, Any]:
raw_sources = _source_configs(payload)
merged: dict[str, Any] = {}
for source_key, default_config in DEFAULT_PUBLIC_BOUNDARY_SOURCES.items():
configured = raw_sources.get(source_key)
if not isinstance(configured, dict) or _is_placeholder_endpoint(configured.get("endpoint")):
merged[source_key] = dict(default_config)
else:
merged[source_key] = {**default_config, **configured}
for source_key, source_config in raw_sources.items():
if source_key not in merged:
merged[source_key] = source_config
return merged
def _build_input_hash(source_manifest: dict[str, Any]) -> str:
return _stable_json_hash(
{
"source_manifest_schema": source_manifest.get("schema"),
"sources": [
{
"id": source.get("id"),
"sha256": source.get("sha256"),
"kind": source.get("kind"),
}
for source in source_manifest.get("sources", [])
],
"pov_policy": source_manifest.get("povPolicy"),
"build_config": BUILD_CONFIG,
}
)
def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
return (
bool(boundary_manifest)
and boundary_manifest.get("buildInputHash") == build_input_hash
and boundary_manifest.get("tileProvider") == "pmtiles-mvt"
and PMTILES_ARTIFACT_PATH.exists()
)
def get_boundary_status() -> dict[str, Any]:
config_payload, config_source = load_boundary_config()
effective_source_configs = _source_configs_with_defaults(config_payload)
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
manifest_exists = BOUNDARY_MANIFEST_PATH.exists()
high_precision_ready = (
manifest_exists
and (
(
boundary_manifest.get("tileProvider") == "pmtiles-mvt"
and pmtiles_exists
)
or boundary_manifest.get("tileProvider") == "geojson-high-precision"
)
)
legacy_exists = LEGACY_GEOJSON_PATH.exists()
provider = (
boundary_manifest.get("tileProvider")
if high_precision_ready
else "legacy-geojson" if legacy_exists else "missing"
)
return {
"provider": provider,
"high_precision_ready": high_precision_ready,
"fallback_available": legacy_exists,
"config_source": config_source,
"config_path": _relative(LOCAL_CONFIG_PATH),
"config_exists": LOCAL_CONFIG_PATH.exists(),
"config": config_payload,
"effective_default_sources": [
source_key
for source_key, source_config in effective_source_configs.items()
if source_key in DEFAULT_PUBLIC_BOUNDARY_SOURCES
and source_config.get("endpoint") == DEFAULT_PUBLIC_BOUNDARY_SOURCES[source_key]["endpoint"]
],
"manifest": {
"path": _relative(BOUNDARY_MANIFEST_PATH),
"exists": manifest_exists,
"tileProvider": boundary_manifest.get("tileProvider"),
"buildInputHash": boundary_manifest.get("buildInputHash"),
"builtAt": boundary_manifest.get("builtAt"),
},
"pmtiles": {
"path": _relative(PMTILES_ARTIFACT_PATH),
"exists": pmtiles_exists,
"size_bytes": PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0,
},
"legacy": {
"path": _relative(LEGACY_GEOJSON_PATH),
"exists": legacy_exists,
"size_bytes": LEGACY_GEOJSON_PATH.stat().st_size if legacy_exists else 0,
},
"source_manifest": {
"path": _relative(SOURCE_MANIFEST_PATH),
"exists": SOURCE_MANIFEST_PATH.exists(),
},
"last_build": _read_json(BUILD_RESULT_PATH),
"current_job": _public_job_state(),
}
async def _download_source(
source_key: str,
source_config: dict[str, Any],
progress_callback: Any = None,
) -> dict[str, Any]:
endpoint = str(source_config.get("endpoint") or "").strip()
if _is_placeholder_endpoint(endpoint):
raise EarthBoundaryBuildError(
f"{source_key} endpoint is not configured",
code="source_not_configured",
details={"source": source_key},
)
method = str(source_config.get("method") or "GET").upper()
if method not in {"GET", "POST"}:
raise EarthBoundaryBuildError(
f"{source_key} method must be GET or POST",
code="invalid_config",
details={"source": source_key, "method": method},
)
if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
payload = Path(endpoint.removeprefix("file://")).expanduser().read_bytes()
content_type = "application/octet-stream"
if progress_callback:
progress_callback(1, len(payload), len(payload))
else:
timeout = float(source_config.get("timeout") or 120)
headers = source_config.get("headers") if isinstance(source_config.get("headers"), dict) else {}
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
async with client.stream(method, endpoint, headers=headers) as response:
response.raise_for_status()
content_type = response.headers.get("content-type", "")
total = int(response.headers.get("content-length") or 0)
chunks = []
downloaded = 0
async for chunk in response.aiter_bytes():
if not chunk:
continue
chunks.append(chunk)
downloaded += len(chunk)
if progress_callback:
progress_callback(
(downloaded / total) if total else None,
downloaded,
total,
)
payload = b"".join(chunks)
extension = _artifact_extension(endpoint, content_type, payload)
parsed: Any = None
if extension in {".json", ".geojson"}:
parsed = json.loads(payload.decode("utf-8"))
feature_count = _json_feature_count(parsed)
if feature_count <= 0:
raise EarthBoundaryBuildError(
f"{source_key} downloaded payload contains no features",
code="empty_source",
details={"source": source_key},
)
sha256 = _sha256_bytes(payload)
source_dir = SOURCE_OUTPUT_DIR / source_key
source_dir.mkdir(parents=True, exist_ok=True)
artifact_path = source_dir / f"{sha256}{extension}"
artifact_path.write_bytes(payload)
return {
"id": source_key,
"kind": source_config.get("sourceKind") or BOUNDARY_SOURCE_KINDS[source_key],
"path": _relative(artifact_path),
"sha256": sha256,
"featureCount": feature_count,
"license": source_config.get("license"),
}
async def _run_step(args: list[str]) -> dict[str, Any]:
process = await asyncio.create_subprocess_exec(
sys.executable,
*args,
cwd=REPO_ROOT,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout_bytes, stderr_bytes = await process.communicate()
stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
payload: dict[str, Any] = {"stdout": stdout, "stderr": stderr, "returncode": process.returncode}
last_line = stdout.splitlines()[-1:] or []
if last_line:
try:
payload["result"] = json.loads(last_line[0])
except json.JSONDecodeError:
payload["result"] = last_line[0]
if process.returncode != 0:
raise EarthBoundaryBuildError(
stderr or stdout or f"command failed: {' '.join(args)}",
code="build_command_failed",
details=payload,
)
return payload
async def build_boundary_assets(progress_callback: Any = None) -> dict[str, Any]:
config_payload, config_source = load_boundary_config()
source_configs = _source_configs_with_defaults(config_payload)
missing = [source for source in BOUNDARY_SOURCE_KINDS if source not in source_configs]
if missing:
raise EarthBoundaryBuildError(
f"Missing Earth boundary source configs: {', '.join(missing)}",
code="missing_sources",
details={"missing": missing},
)
sources = []
source_keys = list(BOUNDARY_SOURCE_KINDS)
for index, source_key in enumerate(source_keys):
source_config = source_configs[source_key]
if not isinstance(source_config, dict):
raise EarthBoundaryBuildError(
f"{source_key} config must be an object",
code="invalid_config",
details={"source": source_key},
)
source_start = 8 + index * 18
source_end = source_start + 18
if progress_callback:
progress_callback(source_start, "download", f"正在下载 {source_key}")
def report_download_progress(ratio: float | None, downloaded: int, total: int) -> None:
if not progress_callback:
return
if ratio is None:
progress_callback(source_start + 8, "download", f"{source_key} 已下载 {downloaded} bytes")
return
progress_callback(
source_start + (source_end - source_start) * ratio,
"download",
f"{source_key} 下载 {int(ratio * 100)}%",
downloaded_bytes=downloaded,
total_bytes=total,
)
sources.append(await _download_source(source_key, source_config, report_download_progress))
source_manifest = {
"schema": "planet-earth-boundary-sources/v2",
"sources": sources,
"povPolicy": _read_json(POV_POLICY_PATH),
}
if progress_callback:
progress_callback(65, "manifest", "正在写入边界源 manifest")
_write_json(SOURCE_MANIFEST_PATH, source_manifest)
build_input_hash = _build_input_hash(source_manifest)
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
if missing_tools and not build_skipped:
if progress_callback:
progress_callback(82, "build", "缺少 PMTiles 工具,正在生成 GeoJSON 高清包")
boundary_manifest = _write_high_precision_geojson_manifest(
sources,
build_input_hash,
missing_tools,
)
result = {
"status": "built_geojson_fallback",
"code": "missing_tools",
"missing_tools": missing_tools,
"sources": sources,
"boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH),
"manifest": boundary_manifest,
}
_write_json(BUILD_RESULT_PATH, result)
if progress_callback:
progress_callback(96, "finalize", "GeoJSON 高清国界包已生成")
return {**get_boundary_status(), "build": result}
if build_skipped:
if progress_callback:
progress_callback(96, "unchanged", "高精国界已是最新")
build_result = {
"status": "unchanged",
"reason": "source manifest and build config hash unchanged",
"buildInputHash": build_input_hash,
}
else:
if progress_callback:
progress_callback(72, "build", "正在构建 PMTiles/MVT")
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
coastline = next(source for source in sources if source["kind"] == "coastline")
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
build_result = await _run_step(
[
"scripts/build_earth_boundary_pmtiles.py",
"--admin0-source",
admin0["path"],
"--coastline-source",
coastline["path"],
"--claims-source",
claim_lines["path"],
"--output",
_relative(PMTILES_ARTIFACT_PATH),
"--manifest",
_relative(BOUNDARY_MANIFEST_PATH),
"--build-input-hash",
build_input_hash,
"--pov-policy",
_relative(POV_POLICY_PATH),
]
)
if progress_callback:
progress_callback(95, "finalize", "正在校验构建产物")
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
result = {
"status": "unchanged" if build_skipped else "built",
"sources": sources,
"source_manifest": _relative(SOURCE_MANIFEST_PATH),
"boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH),
"pmtiles_artifact": _relative(PMTILES_ARTIFACT_PATH),
"pmtiles_exists": PMTILES_ARTIFACT_PATH.exists(),
"boundary_stats": boundary_stats,
"manifest": boundary_manifest,
"build_result": build_result,
}
_write_json(BUILD_RESULT_PATH, result)
return {**get_boundary_status(), "build": result}
async def _run_boundary_build_job(job_id: str) -> None:
def report(progress: float, phase: str, message: str, **extra: Any) -> None:
if _build_job_state.get("id") != job_id:
return
_update_job_progress(progress, phase, message, **extra)
try:
report(3, "prepare", "正在准备高精国界构建")
result = await build_boundary_assets(report)
_set_job_state(
id=job_id,
status="succeeded",
progress=100,
phase="complete",
message="高精国界构建完成",
finished_at=_utc_now_iso(),
result={
"provider": result.get("provider"),
"high_precision_ready": result.get("high_precision_ready"),
"pmtiles": result.get("pmtiles"),
"manifest": result.get("manifest"),
},
)
_append_job_log("高精国界构建完成")
except EarthBoundaryBuildError as exc:
_set_job_state(
id=job_id,
status="failed",
progress=_build_job_state.get("progress", 0),
phase="failed",
message=str(exc),
code=exc.code,
details=exc.details,
finished_at=_utc_now_iso(),
)
_append_job_log(str(exc))
except Exception as exc: # pragma: no cover - defensive guard for background task
_set_job_state(
id=job_id,
status="failed",
progress=_build_job_state.get("progress", 0),
phase="failed",
message=str(exc),
code="build_failed",
finished_at=_utc_now_iso(),
)
_append_job_log(str(exc))
async def start_boundary_build_job() -> dict[str, Any]:
global _build_task
async with _build_job_lock:
if _build_task and not _build_task.done():
return {"accepted": False, "job": _public_job_state()}
job_id = uuid4().hex
_set_job_state(
id=job_id,
status="queued",
progress=0,
phase="queued",
message="高精国界构建已加入队列",
logs=[],
started_at=_utc_now_iso(),
finished_at=None,
code=None,
details=None,
)
_append_job_log("高精国界构建已启动")
_build_task = asyncio.create_task(_run_boundary_build_job(job_id))
return {"accepted": True, "job": _public_job_state()}

View File

@@ -1,7 +1,7 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
import hashlib
@@ -18,6 +18,7 @@ from bs4 import BeautifulSoup
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt
from app.schemas.ai import SituationalAnalysisRequest
from app.services.ai_client import AIProviderClient
from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder
@@ -31,6 +32,8 @@ STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS
MAX_TARGET_INFERENCE_CONCURRENCY = 3
TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0
DEFAULT_NEWS_LOCALE = "zh-CN"
NEWS_ENRICH_PROMPT_KEY = "earth.news.enrich"
@dataclass(frozen=True)
@@ -82,6 +85,11 @@ class ParsedNewsItem:
feed_region: str
homepage_url: str
published_at: datetime | None
content_language: str = "en"
localizations: dict[str, dict[str, str]] = field(default_factory=dict)
enrichment_status: str = "pending"
enrichment_error: str | None = None
enriched_at: datetime | None = None
target_location: NewsTargetLocation | None = None
target_resolution_stage: str = "unresolved"
target_ai_attempted: bool = False
@@ -365,6 +373,65 @@ def _first_json_object(text: str) -> dict[str, Any] | None:
return None
def _normalize_localizations(value: Any) -> dict[str, dict[str, str]]:
if not isinstance(value, dict):
return {}
normalized: dict[str, dict[str, str]] = {}
for locale, payload in value.items():
locale_key = _coerce_str(locale)
if not locale_key or not isinstance(payload, dict):
continue
title = _coerce_str(payload.get("title"))
summary = _coerce_str(payload.get("summary"))
entry: dict[str, str] = {}
if title:
entry["title"] = title
if summary:
entry["summary"] = summary
if entry:
normalized[locale_key] = entry
return normalized
def _get_locale_text(
item: ParsedNewsItem,
key: str,
*,
locale: str = DEFAULT_NEWS_LOCALE,
) -> str:
localized = item.localizations.get(locale)
if isinstance(localized, dict):
value = _coerce_str(localized.get(key))
if value:
return value
return ""
def _has_default_localization(item: ParsedNewsItem) -> bool:
localized = item.localizations.get(DEFAULT_NEWS_LOCALE)
if not isinstance(localized, dict):
return False
return bool(_coerce_str(localized.get("title")) and _coerce_str(localized.get("summary")))
def apply_enrichment_patch_to_item(
item: ParsedNewsItem,
patch: dict[str, Any],
) -> ParsedNewsItem:
item.location_patch = patch
if "content_language" in patch:
item.content_language = _coerce_str(patch.get("content_language")) or item.content_language
if "localizations" in patch:
item.localizations = _normalize_localizations(patch.get("localizations"))
if "enrichment_status" in patch:
item.enrichment_status = _coerce_str(patch.get("enrichment_status")) or item.enrichment_status
if "enrichment_error" in patch:
item.enrichment_error = _coerce_str(patch.get("enrichment_error"))
if "enriched_at" in patch:
item.enriched_at = _parse_datetime(_coerce_str(patch.get("enriched_at")))
return item
async def _geocode_target_location(query: str) -> dict[str, Any] | None:
return await asyncio.to_thread(_news_target_geocode, query)
@@ -513,42 +580,60 @@ async def _infer_news_target_location(
item: ParsedNewsItem,
*,
provider_client: AIProviderClient | None,
prompt: EffectiveAIPrompt | None = None,
) -> NewsTargetLocation | None:
target, _localizations = await _infer_news_enrichment(
item,
provider_client=provider_client,
prompt=prompt,
)
return target
async def _infer_news_enrichment(
item: ParsedNewsItem,
*,
provider_client: AIProviderClient | None,
prompt: EffectiveAIPrompt | None = None,
) -> tuple[NewsTargetLocation | None, dict[str, dict[str, str]]]:
text_hint = await _extract_target_location_from_text(item)
content_error: str | None = None
if text_hint is not None and text_hint.city:
item.target_resolution_stage = text_hint.source
item.target_ai_attempted = False
item.target_ai_status = "skipped_text_hint"
item.target_ai_error = None
item.target_debug_note = f"text hint matched {text_hint.label}"
return text_hint
localizations: dict[str, dict[str, str]] = {}
if provider_client is None:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_attempted = False
item.target_ai_status = "unavailable"
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
if text_hint is None or not text_hint.city:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_attempted = False
item.target_ai_status = "unavailable"
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
item.target_debug_note = (
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
)
item.enrichment_status = "unavailable"
item.enrichment_error = "AI provider is not configured or unavailable for earth-feed."
return text_hint, localizations
if text_hint is None or not text_hint.city:
item.target_ai_attempted = True
item.target_ai_status = "attempted"
item.target_ai_error = None
item.target_debug_note = (
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
)
return text_hint
item.target_ai_attempted = True
item.target_ai_status = "attempted"
item.target_ai_error = None
item.target_debug_note = (
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
)
item.enrichment_status = "attempted"
item.enrichment_error = None
prompt = prompt or await get_effective_prompt(None, NEWS_ENRICH_PROMPT_KEY)
request = SituationalAnalysisRequest(
title="Infer likely event location for Earth news cruise",
objective=(
"Return exactly one strict JSON object for the most likely physical "
"location the news event is about. Prefer the host city when a state "
"visit, summit, meeting, attack, or disaster is clearly centered in a "
"known city. Fall back to the best-supported country only when a city "
"cannot be inferred."
),
title="Enrich Earth news item with event location and zh-CN content",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
context={
"news_item": {
"title": item.title,
@@ -564,17 +649,28 @@ async def _infer_news_target_location(
),
},
"required_json_schema": {
"country": "string|null",
"city": "string|null",
"matched_location_name": "string|null",
"latitude": "number|null",
"longitude": "number|null",
"confidence": "number from 0 to 1",
"reasoning_summary": "short string",
"location": {
"country": "string|null",
"city": "string|null",
"matched_location_name": "string|null",
"latitude": "number|null",
"longitude": "number|null",
"confidence": "number from 0 to 1",
"reasoning_summary": "short string",
},
"localizations": {
"zh-CN": {
"title": "faithful Simplified Chinese title",
"summary": "1-2 sentence faithful Simplified Chinese summary",
}
},
},
},
constraints=[
"Return only strict JSON. Do not wrap it in markdown.",
"For localizations, do not add facts that are absent from the RSS headline, description, source, or date.",
"If the RSS description is thin, write a conservative summary that says only what is supported.",
"Keep zh-CN summary concise, factual, and non-promotional.",
"Prefer the event location, not the newsroom or publisher headquarters.",
"When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.",
"Use null for unknown fields instead of inventing details.",
@@ -584,40 +680,68 @@ async def _infer_news_target_location(
try:
response = await provider_client.analyze(request)
except Exception as exc:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "provider_error"
item.target_ai_error = str(exc)
return text_hint
if text_hint is None or not text_hint.city:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "provider_error"
item.target_ai_error = str(exc)
item.enrichment_status = "provider_error"
item.enrichment_error = str(exc)
return text_hint, localizations
payload = _first_json_object(response.content)
if not isinstance(payload, dict):
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "parse_error"
item.target_ai_error = "AI response did not contain a parseable JSON object."
return text_hint
if text_hint is None or not text_hint.city:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "parse_error"
item.target_ai_error = "AI response did not contain a parseable JSON object."
item.enrichment_status = "parse_error"
item.enrichment_error = "AI response did not contain a parseable JSON object."
return text_hint, localizations
target = await _build_target_location_from_payload(payload)
localizations = _normalize_localizations(payload.get("localizations"))
if not localizations:
content_error = "AI returned no usable localizations."
location_payload = payload.get("location") if isinstance(payload.get("location"), dict) else payload
if text_hint is not None and text_hint.city:
target = text_hint
else:
target = await _build_target_location_from_payload(location_payload)
if target is None:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "no_result"
item.target_ai_error = "AI returned no usable target coordinates or geocodeable location."
return text_hint
if target.confidence is not None and target.confidence < 0.45:
target = text_hint
elif target.confidence is not None and target.confidence < 0.45:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "low_confidence"
item.target_ai_error = f"AI target confidence too low: {target.confidence:.2f}"
return text_hint
item.target_resolution_stage = target.source
item.target_ai_status = "success"
item.target_ai_error = None
item.target_debug_note = f"ai inferred {target.label}"
return target
target = text_hint
else:
item.target_resolution_stage = target.source
item.target_ai_status = "success"
item.target_ai_error = None
item.target_debug_note = f"ai inferred {target.label}"
item.localizations = localizations
if localizations and item.target_ai_status in {"success", "skipped_text_hint"}:
item.enrichment_status = "success"
item.enrichment_error = None
elif localizations:
item.enrichment_status = "content_only"
item.enrichment_error = item.target_ai_error
else:
item.enrichment_status = "location_only" if target is not None else "no_result"
item.enrichment_error = content_error or item.target_ai_error
item.enriched_at = datetime.now(UTC) if localizations else None
return target, localizations
async def _enrich_items_with_target_locations(
items: list[ParsedNewsItem],
*,
provider_client: AIProviderClient | None,
prompt: EffectiveAIPrompt | None = None,
) -> list[ParsedNewsItem]:
if not items:
return items
@@ -626,7 +750,11 @@ async def _enrich_items_with_target_locations(
async def enrich(item: ParsedNewsItem) -> ParsedNewsItem:
async with semaphore:
target = await _infer_news_target_location(item, provider_client=provider_client)
target = await _infer_news_target_location(
item,
provider_client=provider_client,
prompt=prompt,
)
item.target_location = target
return item
@@ -783,6 +911,20 @@ def _serialize_target(target: NewsTargetLocation | None) -> dict[str, Any] | Non
}
def _serialize_enriched_at(value: datetime | None) -> str | None:
return value.isoformat().replace("+00:00", "Z") if value else None
def _content_patch(item: ParsedNewsItem) -> dict[str, Any]:
return {
"content_language": item.content_language,
"localizations": item.localizations,
"enrichment_status": item.enrichment_status,
"enrichment_error": item.enrichment_error,
"enriched_at": _serialize_enriched_at(item.enriched_at),
}
def build_anchor_location_patch(
item: ParsedNewsItem,
*,
@@ -798,6 +940,9 @@ def build_anchor_location_patch(
resolution_stage = item.target_resolution_stage
ai_status = item.target_ai_status
debug_note = item.target_debug_note
content_patch = _content_patch(item)
if queued and content_patch["enrichment_status"] == "pending":
content_patch["enrichment_status"] = "queued"
return {
"latitude": anchor.latitude,
"longitude": anchor.longitude,
@@ -814,6 +959,7 @@ def build_anchor_location_patch(
"target": None,
"anchor": _serialize_anchor(anchor),
},
**content_patch,
}
@@ -836,6 +982,7 @@ def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation
"target": _serialize_target(target),
"anchor": _serialize_anchor(anchor),
},
**_content_patch(item),
}
@@ -845,6 +992,11 @@ def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
"id": item.id,
"title": item.title,
"summary": item.summary,
"content_language": item.content_language,
"localizations": item.localizations,
"enrichment_status": item.enrichment_status,
"enrichment_error": item.enrichment_error,
"enriched_at": _serialize_enriched_at(item.enriched_at),
"url": item.url,
"source": item.source,
"feed_name": item.feed_name,
@@ -859,6 +1011,11 @@ def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem
id=str(payload.get("id") or ""),
title=str(payload.get("title") or ""),
summary=str(payload.get("summary") or ""),
content_language=str(payload.get("content_language") or "en"),
localizations=_normalize_localizations(payload.get("localizations")),
enrichment_status=str(payload.get("enrichment_status") or "pending"),
enrichment_error=_coerce_str(payload.get("enrichment_error")),
enriched_at=_parse_datetime(_coerce_str(payload.get("enriched_at"))),
url=str(payload.get("url") or ""),
source=str(payload.get("source") or ""),
feed_name=str(payload.get("feed_name") or ""),
@@ -875,10 +1032,15 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An
"id": item.id,
"title": item.title,
"summary": item.summary,
"content_language": item.content_language,
"localizations": item.localizations,
"display_title": _get_locale_text(item, "title"),
"display_summary": _get_locale_text(item, "summary"),
"url": item.url,
"source": item.source,
"feed_name": item.feed_name,
"region": item.feed_region,
"display_region": get_region_anchor(item.feed_region).label,
"homepage_url": item.homepage_url,
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
"latitude": location_patch["latitude"],
@@ -887,6 +1049,9 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An
"location_source": location_patch["location_source"],
"verified": location_patch["verified"],
"location_meta": location_patch["location_meta"],
"enrichment_status": item.enrichment_status,
"enrichment_error": item.enrichment_error,
"enriched_at": _serialize_enriched_at(item.enriched_at),
"is_focus_match": item.feed_region == active_region,
}
@@ -911,6 +1076,7 @@ def _build_payload(
"lon": lon,
"region": active_region,
"label": profile.label,
"display_region": get_region_anchor(active_region).label,
"accent": profile.accent,
},
"sources": _serialize_sources(sources),
@@ -966,13 +1132,27 @@ async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> li
get_cached_target_location_patch,
)
async def enqueue_item(item: ParsedNewsItem, *, force: bool = False) -> bool:
return await enqueue_target_location_job(build_target_location_job_payload(item), force=force)
async def apply_location(item: ParsedNewsItem) -> ParsedNewsItem:
cached_patch = await get_cached_target_location_patch(item.id)
if cached_patch:
item.location_patch = cached_patch
apply_enrichment_patch_to_item(item, cached_patch)
if not _has_default_localization(item):
queued = await enqueue_item(item, force=True)
if queued and item.enrichment_status in {
"pending",
"unavailable",
"provider_error",
"parse_error",
"no_result",
"location_only",
}:
item.enrichment_status = "queued"
return item
queued = await enqueue_target_location_job(build_target_location_job_payload(item))
queued = await enqueue_item(item)
item.location_patch = build_anchor_location_patch(
item,
queued=queued,
@@ -991,9 +1171,16 @@ async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None:
await asyncio.gather(
*(
enqueue_target_location_job(build_target_location_job_payload(item))
enqueue_target_location_job(
build_target_location_job_payload(item),
force=not _has_default_localization(item),
)
for item in items
if item.location_patch is None or item.location_patch.get("verified") is False
if (
item.location_patch is None
or item.location_patch.get("verified") is False
or not _has_default_localization(item)
)
)
)

View File

@@ -32,7 +32,7 @@ class NewsTargetLocationMessage:
class NewsTargetLocationQueue(Protocol):
async def enqueue(self, *, item_id: str, payload: dict[str, Any]) -> bool:
async def enqueue(self, *, item_id: str, payload: dict[str, Any], force: bool = False) -> bool:
...
async def consume_batch(
@@ -91,9 +91,11 @@ class RedisStreamsNewsTargetLocationQueue:
raise
self._group_ready = True
async def enqueue(self, *, item_id: str, payload: dict[str, Any]) -> bool:
async def enqueue(self, *, item_id: str, payload: dict[str, Any], force: bool = False) -> bool:
await self._ensure_group()
if await self.client.exists(_result_key(item_id)):
if force:
await self.client.delete(_result_key(item_id), _queued_key(item_id))
elif await self.client.exists(_result_key(item_id)):
return False
queued = await self.client.set(
_queued_key(item_id),
@@ -187,13 +189,13 @@ def get_news_target_location_queue() -> NewsTargetLocationQueue:
return RedisStreamsNewsTargetLocationQueue()
async def enqueue_target_location_job(payload: dict[str, Any]) -> bool:
async def enqueue_target_location_job(payload: dict[str, Any], *, force: bool = False) -> bool:
item_id = str(payload.get("id") or "")
if not item_id:
return False
try:
queue = get_news_target_location_queue()
return await queue.enqueue(item_id=item_id, payload=payload)
return await queue.enqueue(item_id=item_id, payload=payload, force=force)
except Exception as exc:
logger.warning_event(
"Failed to enqueue Earth news target location job",

View File

@@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.models.earth_news import EarthNewsItem
from app.services.earth_news import (
ParsedNewsItem,
apply_enrichment_patch_to_item,
build_anchor_location_patch,
)
@@ -33,7 +34,7 @@ def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]:
def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
return ParsedNewsItem(
item = ParsedNewsItem(
id=record.id,
title=record.title,
summary=record.summary or "",
@@ -43,8 +44,13 @@ def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
feed_region=record.region or "global",
homepage_url=record.homepage_url or "",
published_at=_coerce_datetime(record.published_at),
location_patch=_location_patch_from_record(record),
content_language=record.content_language or "en",
localizations=dict(record.localizations or {}),
enrichment_status=record.enrichment_status or "pending",
enrichment_error=record.enrichment_error,
enriched_at=_coerce_datetime(record.enriched_at),
)
return apply_enrichment_patch_to_item(item, _location_patch_from_record(record))
def _query_sort_key(active_region: str):
@@ -108,6 +114,8 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
id=item.id,
title=item.title,
summary=item.summary,
content_language=item.content_language,
localizations=dict(item.localizations or {}),
url=item.url,
source=item.source,
feed_name=item.feed_name,
@@ -122,6 +130,9 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
location_meta=patch["location_meta"],
first_seen_at=now,
last_seen_at=now,
enrichment_status=item.enrichment_status,
enrichment_error=item.enrichment_error,
enriched_at=item.enriched_at,
)
db.add(record)
changed += 1
@@ -136,6 +147,12 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
record.homepage_url = item.homepage_url
record.published_at = item.published_at
record.last_seen_at = now
if item.localizations:
record.content_language = item.content_language
record.localizations = dict(item.localizations or {})
record.enrichment_status = item.enrichment_status
record.enrichment_error = item.enrichment_error
record.enriched_at = item.enriched_at
changed += 1
await db.flush()
return changed
@@ -161,6 +178,45 @@ async def update_earth_news_item_location(
return True
async def update_earth_news_item_enrichment(
db: AsyncSession,
*,
item_id: str,
patch: dict[str, Any],
) -> bool:
record = await db.get(EarthNewsItem, item_id)
if record is None:
return False
if "latitude" in patch:
record.latitude = float(patch["latitude"])
record.longitude = float(patch["longitude"])
record.location_label = str(patch["location_label"])
record.location_source = str(patch["location_source"])
record.verified = bool(patch["verified"])
record.location_meta = dict(patch.get("location_meta") or {})
record.resolved_at = datetime.now(UTC) if record.verified else None
if "content_language" in patch:
record.content_language = str(patch.get("content_language") or "en")
if "localizations" in patch:
record.localizations = dict(patch.get("localizations") or {})
if "enrichment_status" in patch:
record.enrichment_status = str(patch.get("enrichment_status") or "pending")
if "enrichment_error" in patch:
record.enrichment_error = patch.get("enrichment_error")
if patch.get("enriched_at"):
try:
parsed_enriched_at = datetime.fromisoformat(
str(patch["enriched_at"]).replace("Z", "+00:00")
)
except ValueError:
parsed_enriched_at = datetime.now(UTC)
record.enriched_at = _coerce_datetime(parsed_enriched_at)
elif patch.get("localizations"):
record.enriched_at = datetime.now(UTC)
await db.flush()
return True
async def list_unverified_earth_news_items(
db: AsyncSession,
*,

View File

@@ -9,8 +9,10 @@ from app.core.logging import get_logger
from app.core.websocket.broadcaster import broadcaster
from app.db.session import async_session_factory
from app.services.ai_client import AIProviderClient
from app.ai_tasks.prompts import get_effective_prompt
from app.services.earth_news import (
_infer_news_target_location,
NEWS_ENRICH_PROMPT_KEY,
_infer_news_enrichment,
build_target_location_patch,
parsed_news_item_from_job_payload,
)
@@ -19,7 +21,7 @@ from app.services.earth_news_queue import (
get_news_target_location_queue,
save_target_location_patch,
)
from app.services.earth_news_store import update_earth_news_item_location
from app.services.earth_news_store import update_earth_news_item_enrichment as update_earth_news_item_location
logger = get_logger(__name__, service="earth_news")
@@ -59,8 +61,15 @@ async def process_target_location_message(
provider_client: AIProviderClient | None,
) -> dict[str, Any]:
item = parsed_news_item_from_job_payload(message.payload)
target = await _infer_news_target_location(item, provider_client=provider_client)
async with async_session_factory() as session:
prompt = await get_effective_prompt(session, NEWS_ENRICH_PROMPT_KEY)
target, localizations = await _infer_news_enrichment(
item,
provider_client=provider_client,
prompt=prompt,
)
item.target_location = target
item.localizations = localizations or item.localizations
patch = build_target_location_patch(item, target)
await save_target_location_patch(item.id, patch)
async with async_session_factory() as session:

View File

@@ -7,8 +7,11 @@ import re
from dataclasses import dataclass
from typing import Any, Iterable
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.countries import COUNTRY_ENTRIES, normalize_country
from app.schemas.ai import SituationalAnalysisRequest
from app.ai_tasks.prompts import get_effective_prompt
from app.services.ai_client import AIProviderClient
from app.services.ai_tools.evidence_store import normalize_search_evidence
from app.services.ai_tools.web_search import WebSearchClient, WebSearchError
@@ -23,6 +26,8 @@ from app.services.location.text import (
VALID_LLM_PRECISIONS = {"precise", "site", "city"}
DEFAULT_MIN_CONFIDENCE = 0.55
LOCATION_NORMALIZE_PROMPT_KEY = "location.factcheck.normalize"
LOCATION_RESOLVE_PROMPT_KEY = "location.factcheck.resolve"
MODEL_CONFIDENCE_WEIGHT = 0.25
_geocode_llm_city = build_default_nominatim_geocoder()
_LLM_LOCATION_NAME_KEYS = (
@@ -876,6 +881,7 @@ async def _repair_location_payload_from_text(
raw_text: str,
query: LocationQuery,
entity_type: str,
db: AsyncSession | None = None,
) -> dict[str, Any] | None:
"""Second-pass structure repair for models that answer in prose.
@@ -884,12 +890,11 @@ async def _repair_location_payload_from_text(
"""
if not coerce_str(raw_text):
return None
prompt = await get_effective_prompt(db, LOCATION_NORMALIZE_PROMPT_KEY)
request = SituationalAnalysisRequest(
title=f"Normalize location factcheck for {entity_type}",
objective=(
"Convert the supplied location factcheck text into exactly one strict "
"JSON object. Extract only facts present in the text or original query."
),
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
context={
"entity_type": entity_type,
"location_query": _query_context(query),
@@ -929,6 +934,7 @@ async def collect_llm_location_fallback_candidate(
provider_client: AIProviderClient,
query: LocationQuery,
entity_type: str,
db: AsyncSession | None = None,
attempted_queries: Iterable[str] = (),
search_evidence: list[dict[str, Any]] | None = None,
min_confidence: float = DEFAULT_MIN_CONFIDENCE,
@@ -946,13 +952,11 @@ async def collect_llm_location_fallback_candidate(
attempted_queries=[attempt],
failure_reason="LLM location factcheck skipped: no WebSearch evidence.",
)
prompt = await get_effective_prompt(db, LOCATION_RESOLVE_PROMPT_KEY)
request = SituationalAnalysisRequest(
title=f"Location factcheck fallback for {entity_type}",
objective=(
"Return exactly one JSON object for the most likely physical location. "
"Use only fact-checkable public knowledge; return null fields rather "
"than guessing when evidence is weak."
),
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
context={
"entity_type": entity_type,
"location_query": _query_context(query),
@@ -1001,6 +1005,7 @@ async def collect_llm_location_fallback_candidate(
raw_text=response.content,
query=query,
entity_type=entity_type,
db=db,
)
if payload is None:
payload = _payload_from_free_text(response.content, query=query)

View File

@@ -10,8 +10,11 @@ from app.models.alert import Alert, AlertSeverity, AlertStatus
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.schemas.ai import SituationalAnalysisRequest
from app.ai_tasks.prompts import get_effective_prompt
from app.services.bgp_ai_brief_store import get_latest_bgp_brief_record
SITUATIONAL_ALERT_BRIEF_PROMPT_KEY = "alerts.situational.brief"
def _format_pairs(pairs: list[tuple[str, int]], empty_text: str = "") -> str:
if not pairs:
@@ -158,10 +161,12 @@ async def build_situational_alert_brief_request(
"latest_bgp_brief_id": latest_bgp_brief.id if latest_bgp_brief else None,
"latest_bgp_brief_generated_at": latest_bgp_brief.generated_at if latest_bgp_brief else None,
}
prompt = await get_effective_prompt(db, SITUATIONAL_ALERT_BRIEF_PROMPT_KEY)
request = SituationalAnalysisRequest(
title="态势告警 AI 简报",
objective="综合系统告警、BGP incidents、BGP anomalies 与近期 BGP AI 简报,生成一份面向值班人员的态势告警简报,指出当前最需要关注的风险域、跨模块联动迹象和优先动作。",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
observations=facts,
constraints=[
"明确区分事实、推断与建议。",