release: bump version to 0.59.0
This commit is contained in:
@@ -8,6 +8,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.alert import Alert, AlertSeverity, AlertStatus
|
||||
from app.schemas.ai import AlertBriefRequest, SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
|
||||
ALERT_BRIEF_PROMPT_KEY = "alerts.brief"
|
||||
|
||||
|
||||
def _format_counter(counter: Counter[str], empty_text: str = "无") -> str:
|
||||
@@ -84,11 +87,13 @@ async def build_alert_brief_request(
|
||||
"top_datasources": dict(datasource_counts.most_common(6)),
|
||||
"top_active_datasources": dict(active_datasource_counts.most_common(5)),
|
||||
}
|
||||
prompt = await get_effective_prompt(db, ALERT_BRIEF_PROMPT_KEY)
|
||||
|
||||
return (
|
||||
SituationalAnalysisRequest(
|
||||
title="告警态势 AI 简报",
|
||||
objective="基于当前告警总量、严重度、状态、数据源分布与最近告警摘录,生成一份面向值班人员的简明告警态势简报,突出待处理风险、告警集中点和优先动作。",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=facts,
|
||||
constraints=[
|
||||
"明确区分事实、推断与建议。",
|
||||
|
||||
@@ -11,9 +11,12 @@ from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.models.bgp_observation import BGPObservation
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.bgp_collectors import build_bgp_collector_coverage
|
||||
from app.services.bgp_enrichment import lookup_prefix_geography
|
||||
|
||||
BGP_BRIEF_PROMPT_KEY = "bgp.brief"
|
||||
|
||||
|
||||
def _format_counter(counter: dict[str, int], empty_text: str = "无") -> str:
|
||||
if not counter:
|
||||
@@ -243,10 +246,12 @@ async def build_bgp_brief_request(
|
||||
for prefix, item in list(prefix_geographies.items())[:8]
|
||||
},
|
||||
}
|
||||
prompt = await get_effective_prompt(db, BGP_BRIEF_PROMPT_KEY)
|
||||
|
||||
return SituationalAnalysisRequest(
|
||||
title="BGP 态势 AI 简报",
|
||||
objective="基于当前 BGP incidents、anomalies、原始观测事件、观测站覆盖与 prefix geography 证据,生成一份面向操作员的简明态势简报,突出区域热点、观测偏差、当前风险、证据和优先动作。",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=observations_lines,
|
||||
constraints=[
|
||||
"明确区分事实、推断与建议。",
|
||||
|
||||
@@ -39,12 +39,6 @@ from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
|
||||
from app.services.collectors.media_news_archive import MediaNewsArchiveCollector
|
||||
from app.services.collectors.aisstream import AISStreamCollector
|
||||
from app.services.collectors.vessel_ais import VesselAISCollector
|
||||
from app.services.collectors.earth_boundaries import (
|
||||
EarthAdmin0BoundaryCollector,
|
||||
EarthBoundaryTileCollector,
|
||||
EarthClaimLinesCollector,
|
||||
EarthCoastlineCollector,
|
||||
)
|
||||
|
||||
collector_registry.register(TOP500Collector())
|
||||
collector_registry.register(EpochAIGPUCollector())
|
||||
@@ -75,10 +69,6 @@ collector_registry.register(NewsLiveStreamsCollector())
|
||||
collector_registry.register(MediaNewsArchiveCollector())
|
||||
collector_registry.register(VesselAISCollector())
|
||||
collector_registry.register(AISStreamCollector())
|
||||
collector_registry.register(EarthAdmin0BoundaryCollector())
|
||||
collector_registry.register(EarthCoastlineCollector())
|
||||
collector_registry.register(EarthClaimLinesCollector())
|
||||
collector_registry.register(EarthBoundaryTileCollector())
|
||||
|
||||
__all__ = [
|
||||
"BaseCollector",
|
||||
@@ -115,8 +105,4 @@ __all__ = [
|
||||
"MediaNewsArchiveCollector",
|
||||
"VesselAISCollector",
|
||||
"AISStreamCollector",
|
||||
"EarthAdmin0BoundaryCollector",
|
||||
"EarthCoastlineCollector",
|
||||
"EarthClaimLinesCollector",
|
||||
"EarthBoundaryTileCollector",
|
||||
]
|
||||
|
||||
@@ -1,578 +0,0 @@
|
||||
"""Earth boundary source and static tile collector."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.earth_boundary_defaults import EARTH_BOUNDARY_DEFAULT_SOURCES, default_earth_boundary_config
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.services.collectors.base import BaseCollector
|
||||
from app.services.custom_datasource_runtime import build_query_params, build_request_headers
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
|
||||
SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
|
||||
BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
|
||||
BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
|
||||
PMTILES_ARTIFACT_PATH = REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
|
||||
POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
|
||||
BUILD_CONFIG = {
|
||||
"builder": "scripts/build_earth_boundary_china_pov_geojson.py",
|
||||
"format": "geojson-high-precision",
|
||||
"production_target": "geojson-high-precision",
|
||||
}
|
||||
|
||||
EARTH_BOUNDARY_SOURCE_COLLECTORS = {
|
||||
"earth_admin0_boundaries": "admin0-boundaries",
|
||||
"earth_coastline": "coastline",
|
||||
"earth_claim_lines": "claim-lines",
|
||||
}
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _directory_stats(path: Path) -> dict[str, int]:
|
||||
if not path.exists():
|
||||
return {"file_count": 0, "size_bytes": 0}
|
||||
files = [item for item in path.rglob("*") if item.is_file()]
|
||||
return {
|
||||
"file_count": len(files),
|
||||
"size_bytes": sum(item.stat().st_size for item in files),
|
||||
}
|
||||
|
||||
|
||||
def _stable_json_hash(payload: Any) -> str:
|
||||
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _build_input_hash(source_manifest: dict[str, Any]) -> str:
|
||||
return _stable_json_hash(
|
||||
{
|
||||
"source_manifest_schema": source_manifest.get("schema"),
|
||||
"sources": [
|
||||
{
|
||||
"id": source.get("id"),
|
||||
"sha256": source.get("sha256"),
|
||||
"kind": source.get("kind"),
|
||||
"pov": source.get("pov"),
|
||||
}
|
||||
for source in source_manifest.get("sources", [])
|
||||
],
|
||||
"pov_policy": source_manifest.get("povPolicy"),
|
||||
"build_config": BUILD_CONFIG,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
|
||||
if not boundary_manifest:
|
||||
return False
|
||||
if boundary_manifest.get("buildInputHash") != build_input_hash:
|
||||
return False
|
||||
if not BOUNDARY_OUTPUT_DIR.exists():
|
||||
return False
|
||||
if boundary_manifest.get("tileProvider") == "pmtiles-mvt":
|
||||
return PMTILES_ARTIFACT_PATH.exists()
|
||||
return (BOUNDARY_OUTPUT_DIR / "base.geojson").exists() and (
|
||||
BOUNDARY_OUTPUT_DIR / "hover-index.geojson"
|
||||
).exists()
|
||||
|
||||
|
||||
def _sha256_bytes(payload: bytes) -> str:
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _json_feature_count(payload: Any) -> int:
|
||||
if isinstance(payload, dict) and isinstance(payload.get("features"), list):
|
||||
return len(payload["features"])
|
||||
if isinstance(payload, list):
|
||||
return len(payload)
|
||||
return 1 if payload else 0
|
||||
|
||||
|
||||
def _json_sample_properties(payload: Any) -> dict[str, Any]:
|
||||
feature = None
|
||||
if isinstance(payload, dict) and isinstance(payload.get("features"), list) and payload["features"]:
|
||||
feature = payload["features"][0]
|
||||
elif isinstance(payload, list) and payload:
|
||||
feature = payload[0]
|
||||
elif isinstance(payload, dict):
|
||||
feature = payload
|
||||
if not isinstance(feature, dict):
|
||||
return {}
|
||||
props = feature.get("properties") if isinstance(feature.get("properties"), dict) else feature
|
||||
return {str(key): value for key, value in list(props.items())[:20]}
|
||||
|
||||
|
||||
def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
|
||||
suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
|
||||
if suffix in {".json", ".geojson", ".zip", ".pbf"}:
|
||||
return suffix
|
||||
if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
|
||||
return ".geojson"
|
||||
if "json" in content_type:
|
||||
return ".json"
|
||||
return ".dat"
|
||||
|
||||
|
||||
async def _load_datasource_config(db, name: str) -> DataSourceConfig | None:
|
||||
if db is None:
|
||||
return None
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == name)
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
.order_by(DataSourceConfig.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _latest_boundary_source_record(db, source_name: str) -> CollectedData | None:
|
||||
result = await db.execute(
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == source_name)
|
||||
.where(CollectedData.data_type == "earth_boundary_source")
|
||||
.where(CollectedData.is_current.is_(True))
|
||||
.order_by(CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
class EarthBoundarySourceCollector(BaseCollector):
|
||||
priority = "P1"
|
||||
module = "L3"
|
||||
frequency_hours = 168
|
||||
data_type = "earth_boundary_source"
|
||||
fail_on_empty = True
|
||||
source_kind = "unknown"
|
||||
|
||||
def _default_config(self):
|
||||
source = EARTH_BOUNDARY_DEFAULT_SOURCES.get(self.name)
|
||||
if not source:
|
||||
return None
|
||||
return SimpleNamespace(
|
||||
name=self.name,
|
||||
description=f"内置默认源:{self.name}",
|
||||
endpoint=source["endpoint"],
|
||||
source_type="http",
|
||||
auth_type="none",
|
||||
auth_config={},
|
||||
headers={},
|
||||
config=default_earth_boundary_config(self.name),
|
||||
)
|
||||
|
||||
async def _download_payload(self, config: DataSourceConfig) -> tuple[bytes, str]:
|
||||
request_config = config.config or {}
|
||||
method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
|
||||
if method not in {"GET", "POST"}:
|
||||
raise RuntimeError("Earth boundary source collectors support GET and POST only")
|
||||
|
||||
endpoint = str(config.endpoint or "").strip()
|
||||
if not endpoint:
|
||||
raise RuntimeError(
|
||||
f"{self.name} requires an endpoint in Collector Settings before it can collect data"
|
||||
)
|
||||
|
||||
if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
|
||||
path = Path(endpoint.removeprefix("file://")).expanduser()
|
||||
return path.read_bytes(), "application/octet-stream"
|
||||
|
||||
headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {})
|
||||
params = build_query_params(config.auth_type, config.auth_config or {}, request_config)
|
||||
timeout = float(request_config.get("timeout", 120))
|
||||
json_body = request_config.get("json_body") or request_config.get("body")
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
response = await client.request(
|
||||
method,
|
||||
endpoint,
|
||||
headers=headers,
|
||||
params=params or None,
|
||||
json=json_body if isinstance(json_body, (dict, list)) else None,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.content, response.headers.get("content-type", "")
|
||||
|
||||
async def fetch(self) -> list[dict[str, Any]]:
|
||||
db = getattr(self, "_db_session", None)
|
||||
config = await _load_datasource_config(db, self.name) or self._default_config()
|
||||
if config is None:
|
||||
raise RuntimeError(
|
||||
f"{self.name} has no active Collector Settings config and no built-in default source."
|
||||
)
|
||||
|
||||
await self.set_phase("fetching_source", message=f"正在下载 {self.source_kind} 源数据")
|
||||
payload, content_type = await self._download_payload(config)
|
||||
sha256 = _sha256_bytes(payload)
|
||||
endpoint = str(config.endpoint or "")
|
||||
extension = _artifact_extension(endpoint, content_type, payload)
|
||||
source_dir = SOURCE_OUTPUT_DIR / self.name
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact_path = source_dir / f"{sha256}{extension}"
|
||||
artifact_path.write_bytes(payload)
|
||||
|
||||
parsed: Any = None
|
||||
if extension in {".json", ".geojson"}:
|
||||
parsed = json.loads(payload.decode("utf-8"))
|
||||
|
||||
feature_count = _json_feature_count(parsed)
|
||||
if feature_count <= 0:
|
||||
raise RuntimeError(f"{self.name} downloaded data but found no JSON/GeoJSON features")
|
||||
|
||||
config_body = config.config or {}
|
||||
target_schema = str(config_body.get("target_schema") or "earth_boundary_source")
|
||||
if target_schema != "earth_boundary_source":
|
||||
raise RuntimeError(f"{self.name} target_schema must be earth_boundary_source")
|
||||
|
||||
await self.update_phase_progress(
|
||||
current=1,
|
||||
total=1,
|
||||
unit="artifact",
|
||||
message=f"已保存 {feature_count} 个 {self.source_kind} feature",
|
||||
progress=100,
|
||||
commit=True,
|
||||
force=True,
|
||||
)
|
||||
|
||||
relative_artifact_path = str(artifact_path.relative_to(REPO_ROOT))
|
||||
return [
|
||||
{
|
||||
"id": f"{self.source_kind}:{sha256}",
|
||||
"name": config.description or self.name,
|
||||
"description": f"Earth boundary source artifact collected from configured endpoint",
|
||||
"source_kind": self.source_kind,
|
||||
"source_id": sha256,
|
||||
"value": feature_count,
|
||||
"unit": "features",
|
||||
"metadata": {
|
||||
"target_schema": target_schema,
|
||||
"source_kind": self.source_kind,
|
||||
"endpoint": endpoint,
|
||||
"method": str(config_body.get("method") or config_body.get("request_method") or "GET").upper(),
|
||||
"artifact_path": relative_artifact_path,
|
||||
"sha256": sha256,
|
||||
"feature_count": feature_count,
|
||||
"size_bytes": len(payload),
|
||||
"content_type": content_type,
|
||||
"license": config_body.get("license"),
|
||||
"mapping_json": config_body.get("mapping_json"),
|
||||
"sample_properties": _json_sample_properties(parsed),
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class EarthAdmin0BoundaryCollector(EarthBoundarySourceCollector):
|
||||
name = "earth_admin0_boundaries"
|
||||
source_kind = "admin0-boundaries"
|
||||
|
||||
|
||||
class EarthCoastlineCollector(EarthBoundarySourceCollector):
|
||||
name = "earth_coastline"
|
||||
source_kind = "coastline"
|
||||
|
||||
|
||||
class EarthClaimLinesCollector(EarthBoundarySourceCollector):
|
||||
name = "earth_claim_lines"
|
||||
source_kind = "claim-lines"
|
||||
|
||||
|
||||
class EarthBoundaryTileCollector(BaseCollector):
|
||||
name = "earth_boundary_tiles"
|
||||
priority = "P1"
|
||||
module = "L3"
|
||||
frequency_hours = 168
|
||||
data_type = "earth_boundary_tiles"
|
||||
fail_on_empty = False
|
||||
|
||||
async def _run_step(self, args: list[str], *, allow_failure: bool = False) -> dict[str, Any]:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
*args,
|
||||
cwd=REPO_ROOT,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout_bytes, stderr_bytes = await process.communicate()
|
||||
stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
|
||||
stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
|
||||
if process.returncode != 0 and not allow_failure:
|
||||
raise RuntimeError(stderr or stdout or f"command failed: {' '.join(args)}")
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
"returncode": process.returncode,
|
||||
}
|
||||
last_line = stdout.splitlines()[-1:] or []
|
||||
if last_line:
|
||||
try:
|
||||
payload["result"] = json.loads(last_line[0])
|
||||
except json.JSONDecodeError:
|
||||
payload["result"] = last_line[0]
|
||||
return payload
|
||||
|
||||
async def fetch(self) -> list[dict[str, Any]]:
|
||||
db = getattr(self, "_db_session", None)
|
||||
if db is None:
|
||||
raise RuntimeError("Earth PMTiles builder requires an active database session")
|
||||
|
||||
await self.set_phase("checking_sources", message="正在检查三类 Earth 边界源")
|
||||
await self.update_phase_progress(
|
||||
current=0,
|
||||
total=3,
|
||||
unit="steps",
|
||||
message="读取 admin0 / coastline / claim-lines 最新采集结果",
|
||||
progress=5,
|
||||
commit=True,
|
||||
force=True,
|
||||
)
|
||||
|
||||
source_records: dict[str, CollectedData] = {}
|
||||
missing_sources: list[str] = []
|
||||
for source_name in EARTH_BOUNDARY_SOURCE_COLLECTORS:
|
||||
record = await _latest_boundary_source_record(db, source_name)
|
||||
if record is None:
|
||||
missing_sources.append(source_name)
|
||||
else:
|
||||
source_records[source_name] = record
|
||||
|
||||
if missing_sources:
|
||||
missing_text = ", ".join(missing_sources)
|
||||
raise RuntimeError(f"未就绪:缺少 {missing_text}。不会更新 Earth 国界。")
|
||||
|
||||
sources = []
|
||||
for source_name, record in source_records.items():
|
||||
metadata = record.extra_data or {}
|
||||
artifact_path = metadata.get("artifact_path")
|
||||
if not artifact_path or not (REPO_ROOT / str(artifact_path)).exists():
|
||||
missing_sources.append(f"{source_name}: artifact missing")
|
||||
continue
|
||||
sources.append(
|
||||
{
|
||||
"id": source_name,
|
||||
"kind": metadata.get("source_kind") or EARTH_BOUNDARY_SOURCE_COLLECTORS[source_name],
|
||||
"path": str(artifact_path),
|
||||
"sha256": metadata.get("sha256"),
|
||||
"featureCount": metadata.get("feature_count"),
|
||||
"license": metadata.get("license"),
|
||||
}
|
||||
)
|
||||
|
||||
if missing_sources:
|
||||
missing_text = ", ".join(missing_sources)
|
||||
raise RuntimeError(f"未就绪:{missing_text}。不会更新 Earth 国界。")
|
||||
|
||||
source_manifest = {
|
||||
"schema": "planet-earth-boundary-sources/v2",
|
||||
"sources": sources,
|
||||
"povPolicy": _read_json(POV_POLICY_PATH),
|
||||
}
|
||||
SOURCE_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with SOURCE_MANIFEST_PATH.open("w", encoding="utf-8") as f:
|
||||
json.dump(source_manifest, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
build_input_hash = _build_input_hash(source_manifest)
|
||||
await self.update_phase_progress(
|
||||
current=1,
|
||||
total=3,
|
||||
unit="steps",
|
||||
message="三类边界源已就绪",
|
||||
progress=35,
|
||||
commit=True,
|
||||
force=True,
|
||||
)
|
||||
|
||||
await self.set_phase("checking_readiness", message="正在检查 Earth 国界构建就绪状态")
|
||||
missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
|
||||
readiness_result = {
|
||||
"returncode": 0,
|
||||
"result": {
|
||||
"ready": True,
|
||||
"failures": [f"external tool not found in PATH: {tool}" for tool in missing_tools],
|
||||
"sources": sources,
|
||||
"artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
|
||||
"fallback_builder": "geojson-high-precision" if missing_tools else None,
|
||||
},
|
||||
}
|
||||
|
||||
await self.set_phase("building_tiles", message="正在检查 Earth 国界瓦片产物")
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
|
||||
if build_skipped:
|
||||
build_result = {
|
||||
"result": {
|
||||
"status": "unchanged",
|
||||
"reason": "source manifest and build config hash unchanged",
|
||||
"buildInputHash": build_input_hash,
|
||||
}
|
||||
}
|
||||
elif shutil.which("tippecanoe") and shutil.which("pmtiles"):
|
||||
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
|
||||
coastline = next(source for source in sources if source["kind"] == "coastline")
|
||||
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
|
||||
build_result = await self._run_step(
|
||||
[
|
||||
"scripts/build_earth_boundary_pmtiles.py",
|
||||
"--admin0-source",
|
||||
admin0["path"],
|
||||
"--coastline-source",
|
||||
coastline["path"],
|
||||
"--claims-source",
|
||||
claim_lines["path"],
|
||||
"--output",
|
||||
str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
|
||||
"--manifest",
|
||||
str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
|
||||
"--build-input-hash",
|
||||
build_input_hash,
|
||||
"--pov-policy",
|
||||
str(POV_POLICY_PATH.relative_to(REPO_ROOT)),
|
||||
]
|
||||
)
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
else:
|
||||
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
|
||||
coastline = next(source for source in sources if source["kind"] == "coastline")
|
||||
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
|
||||
build_result = await self._run_step(
|
||||
[
|
||||
"scripts/build_earth_boundary_china_pov_geojson.py",
|
||||
"--admin0-source",
|
||||
admin0["path"],
|
||||
"--coastline-source",
|
||||
coastline["path"],
|
||||
"--claims-source",
|
||||
claim_lines["path"],
|
||||
"--output-dir",
|
||||
str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
|
||||
"--build-input-hash",
|
||||
build_input_hash,
|
||||
]
|
||||
)
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
|
||||
pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
|
||||
pmtiles_size = PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0
|
||||
await self.update_phase_progress(
|
||||
current=2,
|
||||
total=3,
|
||||
unit="steps",
|
||||
message=(
|
||||
"边界源未变化,已跳过瓦片重建"
|
||||
if build_skipped
|
||||
else f"已生成 {boundary_manifest.get('tileProvider') or 'boundary'} 国界产物"
|
||||
),
|
||||
progress=80,
|
||||
commit=True,
|
||||
force=True,
|
||||
)
|
||||
|
||||
await self.set_phase("indexing_artifacts", message="正在登记边界瓦片产物")
|
||||
tile_counts = boundary_manifest.get("tiles", {}).get("countsByZoom", {})
|
||||
records = [
|
||||
{
|
||||
"id": "source-manifest",
|
||||
"name": "Earth boundary source manifest",
|
||||
"description": "Offline source collection manifest for Earth boundary tiles",
|
||||
"value": len(source_manifest.get("sources", [])),
|
||||
"unit": "sources",
|
||||
"metadata": {
|
||||
"manifest_path": str(SOURCE_MANIFEST_PATH.relative_to(REPO_ROOT)),
|
||||
"manifest": source_manifest,
|
||||
"collector_result": {
|
||||
"status": "loaded_from_collected_data",
|
||||
"sources": [source["id"] for source in sources],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "production-readiness",
|
||||
"name": "Earth boundary production readiness",
|
||||
"description": "Checks whether all source artifacts and boundary build tooling are available",
|
||||
"value": 1 if readiness_result.get("returncode") == 0 else 0,
|
||||
"unit": "ready",
|
||||
"metadata": {
|
||||
"result": readiness_result.get("result"),
|
||||
"returncode": readiness_result.get("returncode"),
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "boundary-manifest",
|
||||
"name": "Earth boundary tile manifest",
|
||||
"description": "Versioned static vector tile manifest for Earth country boundaries",
|
||||
"value": boundary_stats["file_count"],
|
||||
"unit": "files",
|
||||
"metadata": {
|
||||
"manifest_path": str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
|
||||
"output_dir": str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
|
||||
"size_bytes": boundary_stats["size_bytes"],
|
||||
"manifest": boundary_manifest,
|
||||
"collector_result": build_result.get("result"),
|
||||
"build_skipped": build_skipped,
|
||||
"production_target": BUILD_CONFIG["production_target"],
|
||||
"pmtiles_artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
|
||||
"pmtiles_exists": pmtiles_exists,
|
||||
},
|
||||
},
|
||||
]
|
||||
if pmtiles_exists:
|
||||
records.append(
|
||||
{
|
||||
"id": "pmtiles-artifact",
|
||||
"name": "Earth boundary PMTiles artifact",
|
||||
"description": "Single-file PMTiles/MVT artifact for Earth boundaries",
|
||||
"value": pmtiles_size,
|
||||
"unit": "bytes",
|
||||
"metadata": {
|
||||
"path": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
|
||||
"exists": pmtiles_exists,
|
||||
"provider": "pmtiles-mvt",
|
||||
},
|
||||
}
|
||||
)
|
||||
for zoom, count in sorted(tile_counts.items(), key=lambda item: int(item[0])):
|
||||
records.append(
|
||||
{
|
||||
"id": f"tile-z{zoom}",
|
||||
"name": f"Earth boundary tiles z{zoom}",
|
||||
"description": f"Generated Earth boundary tile count for zoom {zoom}",
|
||||
"value": int(count),
|
||||
"unit": "tiles",
|
||||
"metadata": {
|
||||
"zoom": int(zoom),
|
||||
"tile_count": int(count),
|
||||
"output_dir": str((BOUNDARY_OUTPUT_DIR / str(zoom)).relative_to(REPO_ROOT)),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
await self.update_phase_progress(
|
||||
current=3,
|
||||
total=3,
|
||||
unit="steps",
|
||||
message="边界瓦片产物已登记",
|
||||
progress=100,
|
||||
commit=True,
|
||||
force=True,
|
||||
)
|
||||
return records
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient
|
||||
@@ -15,6 +16,7 @@ from app.services.ai_tools.web_search import WebSearchClient, WebSearchError
|
||||
|
||||
|
||||
CREDENTIAL_GUIDES_CATEGORY = "collector_credential_guides"
|
||||
CREDENTIAL_GUIDE_PROMPT_KEY = "credential.guide"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -240,14 +242,12 @@ async def generate_credential_guide(
|
||||
guide["sources"] = []
|
||||
return guide
|
||||
|
||||
prompt = await get_effective_prompt(db, CREDENTIAL_GUIDE_PROMPT_KEY)
|
||||
response = await ai_client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title=f"Generate credential guide for {provider}",
|
||||
objective=(
|
||||
default.prompt
|
||||
+ "\n只能根据 context.search_evidence 中的来源生成教程;"
|
||||
+ "如果证据不足,明确说明需要以官方页面为准。"
|
||||
),
|
||||
objective=f"{default.prompt}\n{prompt.prompt}",
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"provider": provider,
|
||||
"current_default_guide": default.markdown,
|
||||
|
||||
@@ -301,52 +301,6 @@ async def persist_mapped_records(
|
||||
transport: str | None = None,
|
||||
) -> int:
|
||||
"""Persist validated mapped records to the destination for a target schema."""
|
||||
if target_schema == "earth_boundary_source":
|
||||
from app.models.collected_data import CollectedData
|
||||
|
||||
now = datetime.now(UTC)
|
||||
written_count = 0
|
||||
for index, record in enumerate(records):
|
||||
source_id = record.get("source_id") or record.get("sha256") or str(index)
|
||||
entity_key = f"{datasource_name}:{source_id}"
|
||||
previous_result = await db.execute(
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == datasource_name)
|
||||
.where(CollectedData.entity_key == entity_key)
|
||||
.where(CollectedData.is_current.is_(True))
|
||||
.order_by(CollectedData.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
previous = previous_result.scalar_one_or_none()
|
||||
if previous is not None:
|
||||
previous.is_current = False
|
||||
db.add(
|
||||
CollectedData(
|
||||
source=datasource_name,
|
||||
source_id=str(source_id),
|
||||
entity_key=entity_key,
|
||||
data_type=target_schema,
|
||||
name=record.get("name") or str(source_id),
|
||||
description=f"Earth boundary source artifact: {record.get('source_kind')}",
|
||||
extra_data={
|
||||
**record,
|
||||
"datasource_config_id": datasource_config_id,
|
||||
"mapping_version": mapping_version,
|
||||
"delivery_mode": delivery_mode or "polling",
|
||||
"transport": transport or "http",
|
||||
},
|
||||
collected_at=now,
|
||||
is_valid=1,
|
||||
is_current=True,
|
||||
previous_record_id=previous.id if previous else None,
|
||||
change_type="updated" if previous else "created",
|
||||
change_summary={},
|
||||
)
|
||||
)
|
||||
written_count += 1
|
||||
await db.commit()
|
||||
return written_count
|
||||
|
||||
if target_schema == "vessel_ais":
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
|
||||
671
backend/app/services/earth_boundaries.py
Normal file
671
backend/app/services/earth_boundaries.py
Normal file
@@ -0,0 +1,671 @@
|
||||
"""Earth boundary static asset service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
|
||||
SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
|
||||
BUILD_RESULT_PATH = SOURCE_OUTPUT_DIR / "build-result.json"
|
||||
BUILD_JOB_PATH = SOURCE_OUTPUT_DIR / "build-job.json"
|
||||
BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
|
||||
BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
|
||||
PMTILES_ARTIFACT_PATH = (
|
||||
REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
|
||||
)
|
||||
LEGACY_GEOJSON_PATH = REPO_ROOT / "frontend/public/earth/data/countries-admin0.min.geojson"
|
||||
POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
|
||||
LOCAL_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.local.json"
|
||||
EXAMPLE_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.example.json"
|
||||
|
||||
BOUNDARY_SOURCE_KINDS = {
|
||||
"earth_admin0_boundaries": "admin0-boundaries",
|
||||
"earth_coastline": "coastline",
|
||||
"earth_claim_lines": "claim-lines",
|
||||
}
|
||||
|
||||
DEFAULT_PUBLIC_BOUNDARY_SOURCES = {
|
||||
"earth_admin0_boundaries": {
|
||||
"displayName": "Natural Earth Admin-0 Countries",
|
||||
"sourceKind": "admin0-boundaries",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_countries.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
"earth_coastline": {
|
||||
"displayName": "Natural Earth Coastline",
|
||||
"sourceKind": "coastline",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_coastline.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
"earth_claim_lines": {
|
||||
"displayName": "Natural Earth Disputed Boundaries",
|
||||
"sourceKind": "claim-lines",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_boundary_lines_disputed_areas.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
}
|
||||
|
||||
BUILD_CONFIG = {
|
||||
"builder": "scripts/build_earth_boundary_pmtiles.py",
|
||||
"format": "pmtiles+mvt",
|
||||
"production_target": "pmtiles-mvt",
|
||||
}
|
||||
|
||||
|
||||
class EarthBoundaryBuildError(RuntimeError):
|
||||
def __init__(self, message: str, *, code: str = "build_failed", details: Any = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.details = details
|
||||
|
||||
|
||||
_build_job_lock = asyncio.Lock()
|
||||
_build_task: asyncio.Task | None = None
|
||||
_build_job_state: dict[str, Any] = {}
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _public_job_state() -> dict[str, Any]:
|
||||
if _build_job_state:
|
||||
return dict(_build_job_state)
|
||||
return _read_json(BUILD_JOB_PATH)
|
||||
|
||||
|
||||
def get_boundary_build_status() -> dict[str, Any]:
|
||||
return {"job": _public_job_state()}
|
||||
|
||||
|
||||
def _set_job_state(**updates: Any) -> dict[str, Any]:
|
||||
global _build_job_state
|
||||
current = dict(_build_job_state)
|
||||
current.update(updates)
|
||||
current["updated_at"] = _utc_now_iso()
|
||||
_build_job_state = current
|
||||
_write_json(BUILD_JOB_PATH, current)
|
||||
return current
|
||||
|
||||
|
||||
def _append_job_log(message: str) -> None:
|
||||
logs = list(_build_job_state.get("logs") or [])
|
||||
logs.append({"time": _utc_now_iso(), "message": message})
|
||||
_set_job_state(logs=logs[-40:])
|
||||
|
||||
|
||||
def _update_job_progress(progress: float, phase: str, message: str, **extra: Any) -> None:
|
||||
bounded_progress = max(0, min(100, int(round(progress))))
|
||||
_set_job_state(
|
||||
status="running",
|
||||
progress=bounded_progress,
|
||||
phase=phase,
|
||||
message=message,
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def _sha256_bytes(payload: bytes) -> str:
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _stable_json_hash(payload: Any) -> str:
|
||||
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
|
||||
suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
|
||||
if suffix in {".json", ".geojson", ".zip", ".pbf"}:
|
||||
return suffix
|
||||
if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
|
||||
return ".geojson"
|
||||
if "json" in content_type:
|
||||
return ".json"
|
||||
return ".dat"
|
||||
|
||||
|
||||
def _json_feature_count(payload: Any) -> int:
|
||||
if isinstance(payload, dict) and isinstance(payload.get("features"), list):
|
||||
return len(payload["features"])
|
||||
if isinstance(payload, list):
|
||||
return len(payload)
|
||||
return 1 if payload else 0
|
||||
|
||||
|
||||
def _directory_stats(path: Path) -> dict[str, int]:
|
||||
if not path.exists():
|
||||
return {"file_count": 0, "size_bytes": 0}
|
||||
files = [item for item in path.rglob("*") if item.is_file()]
|
||||
return {"file_count": len(files), "size_bytes": sum(item.stat().st_size for item in files)}
|
||||
|
||||
|
||||
def _load_source_feature_collection(source: dict[str, Any]) -> dict[str, Any]:
|
||||
path = REPO_ROOT / source["path"]
|
||||
payload = _read_json(path)
|
||||
features = payload.get("features") if isinstance(payload, dict) else None
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": features if isinstance(features, list) else [],
|
||||
}
|
||||
|
||||
|
||||
def _write_high_precision_geojson_manifest(
|
||||
sources: list[dict[str, Any]],
|
||||
build_input_hash: str,
|
||||
missing_tools: list[str],
|
||||
) -> dict[str, Any]:
|
||||
BOUNDARY_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
|
||||
coastline = next(source for source in sources if source["kind"] == "coastline")
|
||||
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
|
||||
|
||||
admin0_payload = _load_source_feature_collection(admin0)
|
||||
coastline_payload = _load_source_feature_collection(coastline)
|
||||
claim_payload = _load_source_feature_collection(claim_lines)
|
||||
for feature in coastline_payload["features"]:
|
||||
props = feature.setdefault("properties", {})
|
||||
if isinstance(props, dict):
|
||||
props["PLANET_LAYER"] = "coastline"
|
||||
|
||||
base_payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [*admin0_payload["features"], *coastline_payload["features"]],
|
||||
}
|
||||
base_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-base.geojson"
|
||||
hover_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-hover.geojson"
|
||||
claim_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-claims.geojson"
|
||||
_write_json(base_path, base_payload)
|
||||
_write_json(hover_path, admin0_payload)
|
||||
_write_json(claim_path, claim_payload)
|
||||
|
||||
manifest = {
|
||||
"version": "natural-earth-v1",
|
||||
"builtAt": _utc_now_iso(),
|
||||
"tileProvider": "geojson-high-precision",
|
||||
"format": "geojson-directory",
|
||||
"buildInputHash": build_input_hash,
|
||||
"base": base_path.name,
|
||||
"hoverIndex": hover_path.name,
|
||||
"claimLine": claim_path.name,
|
||||
"sourceFeatureCount": {
|
||||
"admin0": len(admin0_payload["features"]),
|
||||
"coastline": len(coastline_payload["features"]),
|
||||
"claimLines": len(claim_payload["features"]),
|
||||
},
|
||||
"pmtiles": None,
|
||||
"missingTools": missing_tools,
|
||||
}
|
||||
_write_json(BOUNDARY_MANIFEST_PATH, manifest)
|
||||
return manifest
|
||||
|
||||
|
||||
def _relative(path: Path) -> str:
|
||||
return str(path.relative_to(REPO_ROOT))
|
||||
|
||||
|
||||
def load_boundary_config() -> tuple[dict[str, Any], str]:
|
||||
if LOCAL_CONFIG_PATH.exists():
|
||||
return _read_json(LOCAL_CONFIG_PATH), "local"
|
||||
return _read_json(EXAMPLE_CONFIG_PATH), "example"
|
||||
|
||||
|
||||
def save_boundary_config(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
if not isinstance(payload, dict):
|
||||
raise EarthBoundaryBuildError("Earth boundary config must be a JSON object", code="invalid_config")
|
||||
_write_json(LOCAL_CONFIG_PATH, payload)
|
||||
return get_boundary_status()
|
||||
|
||||
|
||||
def _source_configs(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
raw_sources = payload.get("collectorConfigs") or payload.get("sources") or {}
|
||||
return raw_sources if isinstance(raw_sources, dict) else {}
|
||||
|
||||
|
||||
def _is_placeholder_endpoint(endpoint: Any) -> bool:
|
||||
value = str(endpoint or "").strip()
|
||||
return not value or "example.com" in value
|
||||
|
||||
|
||||
def _source_configs_with_defaults(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
raw_sources = _source_configs(payload)
|
||||
merged: dict[str, Any] = {}
|
||||
for source_key, default_config in DEFAULT_PUBLIC_BOUNDARY_SOURCES.items():
|
||||
configured = raw_sources.get(source_key)
|
||||
if not isinstance(configured, dict) or _is_placeholder_endpoint(configured.get("endpoint")):
|
||||
merged[source_key] = dict(default_config)
|
||||
else:
|
||||
merged[source_key] = {**default_config, **configured}
|
||||
for source_key, source_config in raw_sources.items():
|
||||
if source_key not in merged:
|
||||
merged[source_key] = source_config
|
||||
return merged
|
||||
|
||||
|
||||
def _build_input_hash(source_manifest: dict[str, Any]) -> str:
|
||||
return _stable_json_hash(
|
||||
{
|
||||
"source_manifest_schema": source_manifest.get("schema"),
|
||||
"sources": [
|
||||
{
|
||||
"id": source.get("id"),
|
||||
"sha256": source.get("sha256"),
|
||||
"kind": source.get("kind"),
|
||||
}
|
||||
for source in source_manifest.get("sources", [])
|
||||
],
|
||||
"pov_policy": source_manifest.get("povPolicy"),
|
||||
"build_config": BUILD_CONFIG,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
|
||||
return (
|
||||
bool(boundary_manifest)
|
||||
and boundary_manifest.get("buildInputHash") == build_input_hash
|
||||
and boundary_manifest.get("tileProvider") == "pmtiles-mvt"
|
||||
and PMTILES_ARTIFACT_PATH.exists()
|
||||
)
|
||||
|
||||
|
||||
def get_boundary_status() -> dict[str, Any]:
|
||||
config_payload, config_source = load_boundary_config()
|
||||
effective_source_configs = _source_configs_with_defaults(config_payload)
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
|
||||
manifest_exists = BOUNDARY_MANIFEST_PATH.exists()
|
||||
high_precision_ready = (
|
||||
manifest_exists
|
||||
and (
|
||||
(
|
||||
boundary_manifest.get("tileProvider") == "pmtiles-mvt"
|
||||
and pmtiles_exists
|
||||
)
|
||||
or boundary_manifest.get("tileProvider") == "geojson-high-precision"
|
||||
)
|
||||
)
|
||||
legacy_exists = LEGACY_GEOJSON_PATH.exists()
|
||||
provider = (
|
||||
boundary_manifest.get("tileProvider")
|
||||
if high_precision_ready
|
||||
else "legacy-geojson" if legacy_exists else "missing"
|
||||
)
|
||||
return {
|
||||
"provider": provider,
|
||||
"high_precision_ready": high_precision_ready,
|
||||
"fallback_available": legacy_exists,
|
||||
"config_source": config_source,
|
||||
"config_path": _relative(LOCAL_CONFIG_PATH),
|
||||
"config_exists": LOCAL_CONFIG_PATH.exists(),
|
||||
"config": config_payload,
|
||||
"effective_default_sources": [
|
||||
source_key
|
||||
for source_key, source_config in effective_source_configs.items()
|
||||
if source_key in DEFAULT_PUBLIC_BOUNDARY_SOURCES
|
||||
and source_config.get("endpoint") == DEFAULT_PUBLIC_BOUNDARY_SOURCES[source_key]["endpoint"]
|
||||
],
|
||||
"manifest": {
|
||||
"path": _relative(BOUNDARY_MANIFEST_PATH),
|
||||
"exists": manifest_exists,
|
||||
"tileProvider": boundary_manifest.get("tileProvider"),
|
||||
"buildInputHash": boundary_manifest.get("buildInputHash"),
|
||||
"builtAt": boundary_manifest.get("builtAt"),
|
||||
},
|
||||
"pmtiles": {
|
||||
"path": _relative(PMTILES_ARTIFACT_PATH),
|
||||
"exists": pmtiles_exists,
|
||||
"size_bytes": PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0,
|
||||
},
|
||||
"legacy": {
|
||||
"path": _relative(LEGACY_GEOJSON_PATH),
|
||||
"exists": legacy_exists,
|
||||
"size_bytes": LEGACY_GEOJSON_PATH.stat().st_size if legacy_exists else 0,
|
||||
},
|
||||
"source_manifest": {
|
||||
"path": _relative(SOURCE_MANIFEST_PATH),
|
||||
"exists": SOURCE_MANIFEST_PATH.exists(),
|
||||
},
|
||||
"last_build": _read_json(BUILD_RESULT_PATH),
|
||||
"current_job": _public_job_state(),
|
||||
}
|
||||
|
||||
|
||||
async def _download_source(
|
||||
source_key: str,
|
||||
source_config: dict[str, Any],
|
||||
progress_callback: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
endpoint = str(source_config.get("endpoint") or "").strip()
|
||||
if _is_placeholder_endpoint(endpoint):
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} endpoint is not configured",
|
||||
code="source_not_configured",
|
||||
details={"source": source_key},
|
||||
)
|
||||
method = str(source_config.get("method") or "GET").upper()
|
||||
if method not in {"GET", "POST"}:
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} method must be GET or POST",
|
||||
code="invalid_config",
|
||||
details={"source": source_key, "method": method},
|
||||
)
|
||||
|
||||
if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
|
||||
payload = Path(endpoint.removeprefix("file://")).expanduser().read_bytes()
|
||||
content_type = "application/octet-stream"
|
||||
if progress_callback:
|
||||
progress_callback(1, len(payload), len(payload))
|
||||
else:
|
||||
timeout = float(source_config.get("timeout") or 120)
|
||||
headers = source_config.get("headers") if isinstance(source_config.get("headers"), dict) else {}
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
async with client.stream(method, endpoint, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
content_type = response.headers.get("content-type", "")
|
||||
total = int(response.headers.get("content-length") or 0)
|
||||
chunks = []
|
||||
downloaded = 0
|
||||
async for chunk in response.aiter_bytes():
|
||||
if not chunk:
|
||||
continue
|
||||
chunks.append(chunk)
|
||||
downloaded += len(chunk)
|
||||
if progress_callback:
|
||||
progress_callback(
|
||||
(downloaded / total) if total else None,
|
||||
downloaded,
|
||||
total,
|
||||
)
|
||||
payload = b"".join(chunks)
|
||||
|
||||
extension = _artifact_extension(endpoint, content_type, payload)
|
||||
parsed: Any = None
|
||||
if extension in {".json", ".geojson"}:
|
||||
parsed = json.loads(payload.decode("utf-8"))
|
||||
feature_count = _json_feature_count(parsed)
|
||||
if feature_count <= 0:
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} downloaded payload contains no features",
|
||||
code="empty_source",
|
||||
details={"source": source_key},
|
||||
)
|
||||
|
||||
sha256 = _sha256_bytes(payload)
|
||||
source_dir = SOURCE_OUTPUT_DIR / source_key
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact_path = source_dir / f"{sha256}{extension}"
|
||||
artifact_path.write_bytes(payload)
|
||||
return {
|
||||
"id": source_key,
|
||||
"kind": source_config.get("sourceKind") or BOUNDARY_SOURCE_KINDS[source_key],
|
||||
"path": _relative(artifact_path),
|
||||
"sha256": sha256,
|
||||
"featureCount": feature_count,
|
||||
"license": source_config.get("license"),
|
||||
}
|
||||
|
||||
|
||||
async def _run_step(args: list[str]) -> dict[str, Any]:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
*args,
|
||||
cwd=REPO_ROOT,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout_bytes, stderr_bytes = await process.communicate()
|
||||
stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
|
||||
stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
|
||||
payload: dict[str, Any] = {"stdout": stdout, "stderr": stderr, "returncode": process.returncode}
|
||||
last_line = stdout.splitlines()[-1:] or []
|
||||
if last_line:
|
||||
try:
|
||||
payload["result"] = json.loads(last_line[0])
|
||||
except json.JSONDecodeError:
|
||||
payload["result"] = last_line[0]
|
||||
if process.returncode != 0:
|
||||
raise EarthBoundaryBuildError(
|
||||
stderr or stdout or f"command failed: {' '.join(args)}",
|
||||
code="build_command_failed",
|
||||
details=payload,
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
async def build_boundary_assets(progress_callback: Any = None) -> dict[str, Any]:
|
||||
config_payload, config_source = load_boundary_config()
|
||||
|
||||
source_configs = _source_configs_with_defaults(config_payload)
|
||||
missing = [source for source in BOUNDARY_SOURCE_KINDS if source not in source_configs]
|
||||
if missing:
|
||||
raise EarthBoundaryBuildError(
|
||||
f"Missing Earth boundary source configs: {', '.join(missing)}",
|
||||
code="missing_sources",
|
||||
details={"missing": missing},
|
||||
)
|
||||
|
||||
sources = []
|
||||
source_keys = list(BOUNDARY_SOURCE_KINDS)
|
||||
for index, source_key in enumerate(source_keys):
|
||||
source_config = source_configs[source_key]
|
||||
if not isinstance(source_config, dict):
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} config must be an object",
|
||||
code="invalid_config",
|
||||
details={"source": source_key},
|
||||
)
|
||||
source_start = 8 + index * 18
|
||||
source_end = source_start + 18
|
||||
if progress_callback:
|
||||
progress_callback(source_start, "download", f"正在下载 {source_key}")
|
||||
|
||||
def report_download_progress(ratio: float | None, downloaded: int, total: int) -> None:
|
||||
if not progress_callback:
|
||||
return
|
||||
if ratio is None:
|
||||
progress_callback(source_start + 8, "download", f"{source_key} 已下载 {downloaded} bytes")
|
||||
return
|
||||
progress_callback(
|
||||
source_start + (source_end - source_start) * ratio,
|
||||
"download",
|
||||
f"{source_key} 下载 {int(ratio * 100)}%",
|
||||
downloaded_bytes=downloaded,
|
||||
total_bytes=total,
|
||||
)
|
||||
|
||||
sources.append(await _download_source(source_key, source_config, report_download_progress))
|
||||
|
||||
source_manifest = {
|
||||
"schema": "planet-earth-boundary-sources/v2",
|
||||
"sources": sources,
|
||||
"povPolicy": _read_json(POV_POLICY_PATH),
|
||||
}
|
||||
if progress_callback:
|
||||
progress_callback(65, "manifest", "正在写入边界源 manifest")
|
||||
_write_json(SOURCE_MANIFEST_PATH, source_manifest)
|
||||
build_input_hash = _build_input_hash(source_manifest)
|
||||
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
|
||||
missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
|
||||
if missing_tools and not build_skipped:
|
||||
if progress_callback:
|
||||
progress_callback(82, "build", "缺少 PMTiles 工具,正在生成 GeoJSON 高清包")
|
||||
boundary_manifest = _write_high_precision_geojson_manifest(
|
||||
sources,
|
||||
build_input_hash,
|
||||
missing_tools,
|
||||
)
|
||||
result = {
|
||||
"status": "built_geojson_fallback",
|
||||
"code": "missing_tools",
|
||||
"missing_tools": missing_tools,
|
||||
"sources": sources,
|
||||
"boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH),
|
||||
"manifest": boundary_manifest,
|
||||
}
|
||||
_write_json(BUILD_RESULT_PATH, result)
|
||||
if progress_callback:
|
||||
progress_callback(96, "finalize", "GeoJSON 高清国界包已生成")
|
||||
return {**get_boundary_status(), "build": result}
|
||||
|
||||
if build_skipped:
|
||||
if progress_callback:
|
||||
progress_callback(96, "unchanged", "高精国界已是最新")
|
||||
build_result = {
|
||||
"status": "unchanged",
|
||||
"reason": "source manifest and build config hash unchanged",
|
||||
"buildInputHash": build_input_hash,
|
||||
}
|
||||
else:
|
||||
if progress_callback:
|
||||
progress_callback(72, "build", "正在构建 PMTiles/MVT")
|
||||
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
|
||||
coastline = next(source for source in sources if source["kind"] == "coastline")
|
||||
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
|
||||
build_result = await _run_step(
|
||||
[
|
||||
"scripts/build_earth_boundary_pmtiles.py",
|
||||
"--admin0-source",
|
||||
admin0["path"],
|
||||
"--coastline-source",
|
||||
coastline["path"],
|
||||
"--claims-source",
|
||||
claim_lines["path"],
|
||||
"--output",
|
||||
_relative(PMTILES_ARTIFACT_PATH),
|
||||
"--manifest",
|
||||
_relative(BOUNDARY_MANIFEST_PATH),
|
||||
"--build-input-hash",
|
||||
build_input_hash,
|
||||
"--pov-policy",
|
||||
_relative(POV_POLICY_PATH),
|
||||
]
|
||||
)
|
||||
if progress_callback:
|
||||
progress_callback(95, "finalize", "正在校验构建产物")
|
||||
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
|
||||
result = {
|
||||
"status": "unchanged" if build_skipped else "built",
|
||||
"sources": sources,
|
||||
"source_manifest": _relative(SOURCE_MANIFEST_PATH),
|
||||
"boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH),
|
||||
"pmtiles_artifact": _relative(PMTILES_ARTIFACT_PATH),
|
||||
"pmtiles_exists": PMTILES_ARTIFACT_PATH.exists(),
|
||||
"boundary_stats": boundary_stats,
|
||||
"manifest": boundary_manifest,
|
||||
"build_result": build_result,
|
||||
}
|
||||
_write_json(BUILD_RESULT_PATH, result)
|
||||
return {**get_boundary_status(), "build": result}
|
||||
|
||||
|
||||
async def _run_boundary_build_job(job_id: str) -> None:
|
||||
def report(progress: float, phase: str, message: str, **extra: Any) -> None:
|
||||
if _build_job_state.get("id") != job_id:
|
||||
return
|
||||
_update_job_progress(progress, phase, message, **extra)
|
||||
|
||||
try:
|
||||
report(3, "prepare", "正在准备高精国界构建")
|
||||
result = await build_boundary_assets(report)
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="succeeded",
|
||||
progress=100,
|
||||
phase="complete",
|
||||
message="高精国界构建完成",
|
||||
finished_at=_utc_now_iso(),
|
||||
result={
|
||||
"provider": result.get("provider"),
|
||||
"high_precision_ready": result.get("high_precision_ready"),
|
||||
"pmtiles": result.get("pmtiles"),
|
||||
"manifest": result.get("manifest"),
|
||||
},
|
||||
)
|
||||
_append_job_log("高精国界构建完成")
|
||||
except EarthBoundaryBuildError as exc:
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="failed",
|
||||
progress=_build_job_state.get("progress", 0),
|
||||
phase="failed",
|
||||
message=str(exc),
|
||||
code=exc.code,
|
||||
details=exc.details,
|
||||
finished_at=_utc_now_iso(),
|
||||
)
|
||||
_append_job_log(str(exc))
|
||||
except Exception as exc: # pragma: no cover - defensive guard for background task
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="failed",
|
||||
progress=_build_job_state.get("progress", 0),
|
||||
phase="failed",
|
||||
message=str(exc),
|
||||
code="build_failed",
|
||||
finished_at=_utc_now_iso(),
|
||||
)
|
||||
_append_job_log(str(exc))
|
||||
|
||||
|
||||
async def start_boundary_build_job() -> dict[str, Any]:
|
||||
global _build_task
|
||||
async with _build_job_lock:
|
||||
if _build_task and not _build_task.done():
|
||||
return {"accepted": False, "job": _public_job_state()}
|
||||
job_id = uuid4().hex
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="queued",
|
||||
progress=0,
|
||||
phase="queued",
|
||||
message="高精国界构建已加入队列",
|
||||
logs=[],
|
||||
started_at=_utc_now_iso(),
|
||||
finished_at=None,
|
||||
code=None,
|
||||
details=None,
|
||||
)
|
||||
_append_job_log("高精国界构建已启动")
|
||||
_build_task = asyncio.create_task(_run_boundary_build_job(job_id))
|
||||
return {"accepted": True, "job": _public_job_state()}
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
import hashlib
|
||||
@@ -18,6 +18,7 @@ from bs4 import BeautifulSoup
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
|
||||
from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder
|
||||
@@ -31,6 +32,8 @@ STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
|
||||
RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS
|
||||
MAX_TARGET_INFERENCE_CONCURRENCY = 3
|
||||
TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0
|
||||
DEFAULT_NEWS_LOCALE = "zh-CN"
|
||||
NEWS_ENRICH_PROMPT_KEY = "earth.news.enrich"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -82,6 +85,11 @@ class ParsedNewsItem:
|
||||
feed_region: str
|
||||
homepage_url: str
|
||||
published_at: datetime | None
|
||||
content_language: str = "en"
|
||||
localizations: dict[str, dict[str, str]] = field(default_factory=dict)
|
||||
enrichment_status: str = "pending"
|
||||
enrichment_error: str | None = None
|
||||
enriched_at: datetime | None = None
|
||||
target_location: NewsTargetLocation | None = None
|
||||
target_resolution_stage: str = "unresolved"
|
||||
target_ai_attempted: bool = False
|
||||
@@ -365,6 +373,65 @@ def _first_json_object(text: str) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_localizations(value: Any) -> dict[str, dict[str, str]]:
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
normalized: dict[str, dict[str, str]] = {}
|
||||
for locale, payload in value.items():
|
||||
locale_key = _coerce_str(locale)
|
||||
if not locale_key or not isinstance(payload, dict):
|
||||
continue
|
||||
title = _coerce_str(payload.get("title"))
|
||||
summary = _coerce_str(payload.get("summary"))
|
||||
entry: dict[str, str] = {}
|
||||
if title:
|
||||
entry["title"] = title
|
||||
if summary:
|
||||
entry["summary"] = summary
|
||||
if entry:
|
||||
normalized[locale_key] = entry
|
||||
return normalized
|
||||
|
||||
|
||||
def _get_locale_text(
|
||||
item: ParsedNewsItem,
|
||||
key: str,
|
||||
*,
|
||||
locale: str = DEFAULT_NEWS_LOCALE,
|
||||
) -> str:
|
||||
localized = item.localizations.get(locale)
|
||||
if isinstance(localized, dict):
|
||||
value = _coerce_str(localized.get(key))
|
||||
if value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def _has_default_localization(item: ParsedNewsItem) -> bool:
|
||||
localized = item.localizations.get(DEFAULT_NEWS_LOCALE)
|
||||
if not isinstance(localized, dict):
|
||||
return False
|
||||
return bool(_coerce_str(localized.get("title")) and _coerce_str(localized.get("summary")))
|
||||
|
||||
|
||||
def apply_enrichment_patch_to_item(
|
||||
item: ParsedNewsItem,
|
||||
patch: dict[str, Any],
|
||||
) -> ParsedNewsItem:
|
||||
item.location_patch = patch
|
||||
if "content_language" in patch:
|
||||
item.content_language = _coerce_str(patch.get("content_language")) or item.content_language
|
||||
if "localizations" in patch:
|
||||
item.localizations = _normalize_localizations(patch.get("localizations"))
|
||||
if "enrichment_status" in patch:
|
||||
item.enrichment_status = _coerce_str(patch.get("enrichment_status")) or item.enrichment_status
|
||||
if "enrichment_error" in patch:
|
||||
item.enrichment_error = _coerce_str(patch.get("enrichment_error"))
|
||||
if "enriched_at" in patch:
|
||||
item.enriched_at = _parse_datetime(_coerce_str(patch.get("enriched_at")))
|
||||
return item
|
||||
|
||||
|
||||
async def _geocode_target_location(query: str) -> dict[str, Any] | None:
|
||||
return await asyncio.to_thread(_news_target_geocode, query)
|
||||
|
||||
@@ -513,42 +580,60 @@ async def _infer_news_target_location(
|
||||
item: ParsedNewsItem,
|
||||
*,
|
||||
provider_client: AIProviderClient | None,
|
||||
prompt: EffectiveAIPrompt | None = None,
|
||||
) -> NewsTargetLocation | None:
|
||||
target, _localizations = await _infer_news_enrichment(
|
||||
item,
|
||||
provider_client=provider_client,
|
||||
prompt=prompt,
|
||||
)
|
||||
return target
|
||||
|
||||
|
||||
async def _infer_news_enrichment(
|
||||
item: ParsedNewsItem,
|
||||
*,
|
||||
provider_client: AIProviderClient | None,
|
||||
prompt: EffectiveAIPrompt | None = None,
|
||||
) -> tuple[NewsTargetLocation | None, dict[str, dict[str, str]]]:
|
||||
text_hint = await _extract_target_location_from_text(item)
|
||||
content_error: str | None = None
|
||||
if text_hint is not None and text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source
|
||||
item.target_ai_attempted = False
|
||||
item.target_ai_status = "skipped_text_hint"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = f"text hint matched {text_hint.label}"
|
||||
return text_hint
|
||||
localizations: dict[str, dict[str, str]] = {}
|
||||
|
||||
if provider_client is None:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_attempted = False
|
||||
item.target_ai_status = "unavailable"
|
||||
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_attempted = False
|
||||
item.target_ai_status = "unavailable"
|
||||
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
|
||||
item.target_debug_note = (
|
||||
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
||||
)
|
||||
item.enrichment_status = "unavailable"
|
||||
item.enrichment_error = "AI provider is not configured or unavailable for earth-feed."
|
||||
return text_hint, localizations
|
||||
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_ai_attempted = True
|
||||
item.target_ai_status = "attempted"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = (
|
||||
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
||||
)
|
||||
return text_hint
|
||||
|
||||
item.target_ai_attempted = True
|
||||
item.target_ai_status = "attempted"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = (
|
||||
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
||||
)
|
||||
item.enrichment_status = "attempted"
|
||||
item.enrichment_error = None
|
||||
|
||||
prompt = prompt or await get_effective_prompt(None, NEWS_ENRICH_PROMPT_KEY)
|
||||
request = SituationalAnalysisRequest(
|
||||
title="Infer likely event location for Earth news cruise",
|
||||
objective=(
|
||||
"Return exactly one strict JSON object for the most likely physical "
|
||||
"location the news event is about. Prefer the host city when a state "
|
||||
"visit, summit, meeting, attack, or disaster is clearly centered in a "
|
||||
"known city. Fall back to the best-supported country only when a city "
|
||||
"cannot be inferred."
|
||||
),
|
||||
title="Enrich Earth news item with event location and zh-CN content",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"news_item": {
|
||||
"title": item.title,
|
||||
@@ -564,17 +649,28 @@ async def _infer_news_target_location(
|
||||
),
|
||||
},
|
||||
"required_json_schema": {
|
||||
"country": "string|null",
|
||||
"city": "string|null",
|
||||
"matched_location_name": "string|null",
|
||||
"latitude": "number|null",
|
||||
"longitude": "number|null",
|
||||
"confidence": "number from 0 to 1",
|
||||
"reasoning_summary": "short string",
|
||||
"location": {
|
||||
"country": "string|null",
|
||||
"city": "string|null",
|
||||
"matched_location_name": "string|null",
|
||||
"latitude": "number|null",
|
||||
"longitude": "number|null",
|
||||
"confidence": "number from 0 to 1",
|
||||
"reasoning_summary": "short string",
|
||||
},
|
||||
"localizations": {
|
||||
"zh-CN": {
|
||||
"title": "faithful Simplified Chinese title",
|
||||
"summary": "1-2 sentence faithful Simplified Chinese summary",
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
constraints=[
|
||||
"Return only strict JSON. Do not wrap it in markdown.",
|
||||
"For localizations, do not add facts that are absent from the RSS headline, description, source, or date.",
|
||||
"If the RSS description is thin, write a conservative summary that says only what is supported.",
|
||||
"Keep zh-CN summary concise, factual, and non-promotional.",
|
||||
"Prefer the event location, not the newsroom or publisher headquarters.",
|
||||
"When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.",
|
||||
"Use null for unknown fields instead of inventing details.",
|
||||
@@ -584,40 +680,68 @@ async def _infer_news_target_location(
|
||||
try:
|
||||
response = await provider_client.analyze(request)
|
||||
except Exception as exc:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "provider_error"
|
||||
item.target_ai_error = str(exc)
|
||||
return text_hint
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "provider_error"
|
||||
item.target_ai_error = str(exc)
|
||||
item.enrichment_status = "provider_error"
|
||||
item.enrichment_error = str(exc)
|
||||
return text_hint, localizations
|
||||
|
||||
payload = _first_json_object(response.content)
|
||||
if not isinstance(payload, dict):
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "parse_error"
|
||||
item.target_ai_error = "AI response did not contain a parseable JSON object."
|
||||
return text_hint
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "parse_error"
|
||||
item.target_ai_error = "AI response did not contain a parseable JSON object."
|
||||
item.enrichment_status = "parse_error"
|
||||
item.enrichment_error = "AI response did not contain a parseable JSON object."
|
||||
return text_hint, localizations
|
||||
|
||||
target = await _build_target_location_from_payload(payload)
|
||||
localizations = _normalize_localizations(payload.get("localizations"))
|
||||
if not localizations:
|
||||
content_error = "AI returned no usable localizations."
|
||||
|
||||
location_payload = payload.get("location") if isinstance(payload.get("location"), dict) else payload
|
||||
if text_hint is not None and text_hint.city:
|
||||
target = text_hint
|
||||
else:
|
||||
target = await _build_target_location_from_payload(location_payload)
|
||||
if target is None:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "no_result"
|
||||
item.target_ai_error = "AI returned no usable target coordinates or geocodeable location."
|
||||
return text_hint
|
||||
if target.confidence is not None and target.confidence < 0.45:
|
||||
target = text_hint
|
||||
elif target.confidence is not None and target.confidence < 0.45:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "low_confidence"
|
||||
item.target_ai_error = f"AI target confidence too low: {target.confidence:.2f}"
|
||||
return text_hint
|
||||
item.target_resolution_stage = target.source
|
||||
item.target_ai_status = "success"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = f"ai inferred {target.label}"
|
||||
return target
|
||||
target = text_hint
|
||||
else:
|
||||
item.target_resolution_stage = target.source
|
||||
item.target_ai_status = "success"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = f"ai inferred {target.label}"
|
||||
|
||||
item.localizations = localizations
|
||||
if localizations and item.target_ai_status in {"success", "skipped_text_hint"}:
|
||||
item.enrichment_status = "success"
|
||||
item.enrichment_error = None
|
||||
elif localizations:
|
||||
item.enrichment_status = "content_only"
|
||||
item.enrichment_error = item.target_ai_error
|
||||
else:
|
||||
item.enrichment_status = "location_only" if target is not None else "no_result"
|
||||
item.enrichment_error = content_error or item.target_ai_error
|
||||
item.enriched_at = datetime.now(UTC) if localizations else None
|
||||
return target, localizations
|
||||
|
||||
|
||||
async def _enrich_items_with_target_locations(
|
||||
items: list[ParsedNewsItem],
|
||||
*,
|
||||
provider_client: AIProviderClient | None,
|
||||
prompt: EffectiveAIPrompt | None = None,
|
||||
) -> list[ParsedNewsItem]:
|
||||
if not items:
|
||||
return items
|
||||
@@ -626,7 +750,11 @@ async def _enrich_items_with_target_locations(
|
||||
|
||||
async def enrich(item: ParsedNewsItem) -> ParsedNewsItem:
|
||||
async with semaphore:
|
||||
target = await _infer_news_target_location(item, provider_client=provider_client)
|
||||
target = await _infer_news_target_location(
|
||||
item,
|
||||
provider_client=provider_client,
|
||||
prompt=prompt,
|
||||
)
|
||||
item.target_location = target
|
||||
return item
|
||||
|
||||
@@ -783,6 +911,20 @@ def _serialize_target(target: NewsTargetLocation | None) -> dict[str, Any] | Non
|
||||
}
|
||||
|
||||
|
||||
def _serialize_enriched_at(value: datetime | None) -> str | None:
|
||||
return value.isoformat().replace("+00:00", "Z") if value else None
|
||||
|
||||
|
||||
def _content_patch(item: ParsedNewsItem) -> dict[str, Any]:
|
||||
return {
|
||||
"content_language": item.content_language,
|
||||
"localizations": item.localizations,
|
||||
"enrichment_status": item.enrichment_status,
|
||||
"enrichment_error": item.enrichment_error,
|
||||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||||
}
|
||||
|
||||
|
||||
def build_anchor_location_patch(
|
||||
item: ParsedNewsItem,
|
||||
*,
|
||||
@@ -798,6 +940,9 @@ def build_anchor_location_patch(
|
||||
resolution_stage = item.target_resolution_stage
|
||||
ai_status = item.target_ai_status
|
||||
debug_note = item.target_debug_note
|
||||
content_patch = _content_patch(item)
|
||||
if queued and content_patch["enrichment_status"] == "pending":
|
||||
content_patch["enrichment_status"] = "queued"
|
||||
return {
|
||||
"latitude": anchor.latitude,
|
||||
"longitude": anchor.longitude,
|
||||
@@ -814,6 +959,7 @@ def build_anchor_location_patch(
|
||||
"target": None,
|
||||
"anchor": _serialize_anchor(anchor),
|
||||
},
|
||||
**content_patch,
|
||||
}
|
||||
|
||||
|
||||
@@ -836,6 +982,7 @@ def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation
|
||||
"target": _serialize_target(target),
|
||||
"anchor": _serialize_anchor(anchor),
|
||||
},
|
||||
**_content_patch(item),
|
||||
}
|
||||
|
||||
|
||||
@@ -845,6 +992,11 @@ def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
|
||||
"id": item.id,
|
||||
"title": item.title,
|
||||
"summary": item.summary,
|
||||
"content_language": item.content_language,
|
||||
"localizations": item.localizations,
|
||||
"enrichment_status": item.enrichment_status,
|
||||
"enrichment_error": item.enrichment_error,
|
||||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||||
"url": item.url,
|
||||
"source": item.source,
|
||||
"feed_name": item.feed_name,
|
||||
@@ -859,6 +1011,11 @@ def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem
|
||||
id=str(payload.get("id") or ""),
|
||||
title=str(payload.get("title") or ""),
|
||||
summary=str(payload.get("summary") or ""),
|
||||
content_language=str(payload.get("content_language") or "en"),
|
||||
localizations=_normalize_localizations(payload.get("localizations")),
|
||||
enrichment_status=str(payload.get("enrichment_status") or "pending"),
|
||||
enrichment_error=_coerce_str(payload.get("enrichment_error")),
|
||||
enriched_at=_parse_datetime(_coerce_str(payload.get("enriched_at"))),
|
||||
url=str(payload.get("url") or ""),
|
||||
source=str(payload.get("source") or ""),
|
||||
feed_name=str(payload.get("feed_name") or ""),
|
||||
@@ -875,10 +1032,15 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An
|
||||
"id": item.id,
|
||||
"title": item.title,
|
||||
"summary": item.summary,
|
||||
"content_language": item.content_language,
|
||||
"localizations": item.localizations,
|
||||
"display_title": _get_locale_text(item, "title"),
|
||||
"display_summary": _get_locale_text(item, "summary"),
|
||||
"url": item.url,
|
||||
"source": item.source,
|
||||
"feed_name": item.feed_name,
|
||||
"region": item.feed_region,
|
||||
"display_region": get_region_anchor(item.feed_region).label,
|
||||
"homepage_url": item.homepage_url,
|
||||
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
|
||||
"latitude": location_patch["latitude"],
|
||||
@@ -887,6 +1049,9 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An
|
||||
"location_source": location_patch["location_source"],
|
||||
"verified": location_patch["verified"],
|
||||
"location_meta": location_patch["location_meta"],
|
||||
"enrichment_status": item.enrichment_status,
|
||||
"enrichment_error": item.enrichment_error,
|
||||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||||
"is_focus_match": item.feed_region == active_region,
|
||||
}
|
||||
|
||||
@@ -911,6 +1076,7 @@ def _build_payload(
|
||||
"lon": lon,
|
||||
"region": active_region,
|
||||
"label": profile.label,
|
||||
"display_region": get_region_anchor(active_region).label,
|
||||
"accent": profile.accent,
|
||||
},
|
||||
"sources": _serialize_sources(sources),
|
||||
@@ -966,13 +1132,27 @@ async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> li
|
||||
get_cached_target_location_patch,
|
||||
)
|
||||
|
||||
async def enqueue_item(item: ParsedNewsItem, *, force: bool = False) -> bool:
|
||||
return await enqueue_target_location_job(build_target_location_job_payload(item), force=force)
|
||||
|
||||
async def apply_location(item: ParsedNewsItem) -> ParsedNewsItem:
|
||||
cached_patch = await get_cached_target_location_patch(item.id)
|
||||
if cached_patch:
|
||||
item.location_patch = cached_patch
|
||||
apply_enrichment_patch_to_item(item, cached_patch)
|
||||
if not _has_default_localization(item):
|
||||
queued = await enqueue_item(item, force=True)
|
||||
if queued and item.enrichment_status in {
|
||||
"pending",
|
||||
"unavailable",
|
||||
"provider_error",
|
||||
"parse_error",
|
||||
"no_result",
|
||||
"location_only",
|
||||
}:
|
||||
item.enrichment_status = "queued"
|
||||
return item
|
||||
|
||||
queued = await enqueue_target_location_job(build_target_location_job_payload(item))
|
||||
queued = await enqueue_item(item)
|
||||
item.location_patch = build_anchor_location_patch(
|
||||
item,
|
||||
queued=queued,
|
||||
@@ -991,9 +1171,16 @@ async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None:
|
||||
|
||||
await asyncio.gather(
|
||||
*(
|
||||
enqueue_target_location_job(build_target_location_job_payload(item))
|
||||
enqueue_target_location_job(
|
||||
build_target_location_job_payload(item),
|
||||
force=not _has_default_localization(item),
|
||||
)
|
||||
for item in items
|
||||
if item.location_patch is None or item.location_patch.get("verified") is False
|
||||
if (
|
||||
item.location_patch is None
|
||||
or item.location_patch.get("verified") is False
|
||||
or not _has_default_localization(item)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ class NewsTargetLocationMessage:
|
||||
|
||||
|
||||
class NewsTargetLocationQueue(Protocol):
|
||||
async def enqueue(self, *, item_id: str, payload: dict[str, Any]) -> bool:
|
||||
async def enqueue(self, *, item_id: str, payload: dict[str, Any], force: bool = False) -> bool:
|
||||
...
|
||||
|
||||
async def consume_batch(
|
||||
@@ -91,9 +91,11 @@ class RedisStreamsNewsTargetLocationQueue:
|
||||
raise
|
||||
self._group_ready = True
|
||||
|
||||
async def enqueue(self, *, item_id: str, payload: dict[str, Any]) -> bool:
|
||||
async def enqueue(self, *, item_id: str, payload: dict[str, Any], force: bool = False) -> bool:
|
||||
await self._ensure_group()
|
||||
if await self.client.exists(_result_key(item_id)):
|
||||
if force:
|
||||
await self.client.delete(_result_key(item_id), _queued_key(item_id))
|
||||
elif await self.client.exists(_result_key(item_id)):
|
||||
return False
|
||||
queued = await self.client.set(
|
||||
_queued_key(item_id),
|
||||
@@ -187,13 +189,13 @@ def get_news_target_location_queue() -> NewsTargetLocationQueue:
|
||||
return RedisStreamsNewsTargetLocationQueue()
|
||||
|
||||
|
||||
async def enqueue_target_location_job(payload: dict[str, Any]) -> bool:
|
||||
async def enqueue_target_location_job(payload: dict[str, Any], *, force: bool = False) -> bool:
|
||||
item_id = str(payload.get("id") or "")
|
||||
if not item_id:
|
||||
return False
|
||||
try:
|
||||
queue = get_news_target_location_queue()
|
||||
return await queue.enqueue(item_id=item_id, payload=payload)
|
||||
return await queue.enqueue(item_id=item_id, payload=payload, force=force)
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"Failed to enqueue Earth news target location job",
|
||||
|
||||
@@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.models.earth_news import EarthNewsItem
|
||||
from app.services.earth_news import (
|
||||
ParsedNewsItem,
|
||||
apply_enrichment_patch_to_item,
|
||||
build_anchor_location_patch,
|
||||
)
|
||||
|
||||
@@ -33,7 +34,7 @@ def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]:
|
||||
|
||||
|
||||
def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
|
||||
return ParsedNewsItem(
|
||||
item = ParsedNewsItem(
|
||||
id=record.id,
|
||||
title=record.title,
|
||||
summary=record.summary or "",
|
||||
@@ -43,8 +44,13 @@ def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
|
||||
feed_region=record.region or "global",
|
||||
homepage_url=record.homepage_url or "",
|
||||
published_at=_coerce_datetime(record.published_at),
|
||||
location_patch=_location_patch_from_record(record),
|
||||
content_language=record.content_language or "en",
|
||||
localizations=dict(record.localizations or {}),
|
||||
enrichment_status=record.enrichment_status or "pending",
|
||||
enrichment_error=record.enrichment_error,
|
||||
enriched_at=_coerce_datetime(record.enriched_at),
|
||||
)
|
||||
return apply_enrichment_patch_to_item(item, _location_patch_from_record(record))
|
||||
|
||||
|
||||
def _query_sort_key(active_region: str):
|
||||
@@ -108,6 +114,8 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
|
||||
id=item.id,
|
||||
title=item.title,
|
||||
summary=item.summary,
|
||||
content_language=item.content_language,
|
||||
localizations=dict(item.localizations or {}),
|
||||
url=item.url,
|
||||
source=item.source,
|
||||
feed_name=item.feed_name,
|
||||
@@ -122,6 +130,9 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
|
||||
location_meta=patch["location_meta"],
|
||||
first_seen_at=now,
|
||||
last_seen_at=now,
|
||||
enrichment_status=item.enrichment_status,
|
||||
enrichment_error=item.enrichment_error,
|
||||
enriched_at=item.enriched_at,
|
||||
)
|
||||
db.add(record)
|
||||
changed += 1
|
||||
@@ -136,6 +147,12 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
|
||||
record.homepage_url = item.homepage_url
|
||||
record.published_at = item.published_at
|
||||
record.last_seen_at = now
|
||||
if item.localizations:
|
||||
record.content_language = item.content_language
|
||||
record.localizations = dict(item.localizations or {})
|
||||
record.enrichment_status = item.enrichment_status
|
||||
record.enrichment_error = item.enrichment_error
|
||||
record.enriched_at = item.enriched_at
|
||||
changed += 1
|
||||
await db.flush()
|
||||
return changed
|
||||
@@ -161,6 +178,45 @@ async def update_earth_news_item_location(
|
||||
return True
|
||||
|
||||
|
||||
async def update_earth_news_item_enrichment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
item_id: str,
|
||||
patch: dict[str, Any],
|
||||
) -> bool:
|
||||
record = await db.get(EarthNewsItem, item_id)
|
||||
if record is None:
|
||||
return False
|
||||
if "latitude" in patch:
|
||||
record.latitude = float(patch["latitude"])
|
||||
record.longitude = float(patch["longitude"])
|
||||
record.location_label = str(patch["location_label"])
|
||||
record.location_source = str(patch["location_source"])
|
||||
record.verified = bool(patch["verified"])
|
||||
record.location_meta = dict(patch.get("location_meta") or {})
|
||||
record.resolved_at = datetime.now(UTC) if record.verified else None
|
||||
if "content_language" in patch:
|
||||
record.content_language = str(patch.get("content_language") or "en")
|
||||
if "localizations" in patch:
|
||||
record.localizations = dict(patch.get("localizations") or {})
|
||||
if "enrichment_status" in patch:
|
||||
record.enrichment_status = str(patch.get("enrichment_status") or "pending")
|
||||
if "enrichment_error" in patch:
|
||||
record.enrichment_error = patch.get("enrichment_error")
|
||||
if patch.get("enriched_at"):
|
||||
try:
|
||||
parsed_enriched_at = datetime.fromisoformat(
|
||||
str(patch["enriched_at"]).replace("Z", "+00:00")
|
||||
)
|
||||
except ValueError:
|
||||
parsed_enriched_at = datetime.now(UTC)
|
||||
record.enriched_at = _coerce_datetime(parsed_enriched_at)
|
||||
elif patch.get("localizations"):
|
||||
record.enriched_at = datetime.now(UTC)
|
||||
await db.flush()
|
||||
return True
|
||||
|
||||
|
||||
async def list_unverified_earth_news_items(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
|
||||
@@ -9,8 +9,10 @@ from app.core.logging import get_logger
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.db.session import async_session_factory
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.earth_news import (
|
||||
_infer_news_target_location,
|
||||
NEWS_ENRICH_PROMPT_KEY,
|
||||
_infer_news_enrichment,
|
||||
build_target_location_patch,
|
||||
parsed_news_item_from_job_payload,
|
||||
)
|
||||
@@ -19,7 +21,7 @@ from app.services.earth_news_queue import (
|
||||
get_news_target_location_queue,
|
||||
save_target_location_patch,
|
||||
)
|
||||
from app.services.earth_news_store import update_earth_news_item_location
|
||||
from app.services.earth_news_store import update_earth_news_item_enrichment as update_earth_news_item_location
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="earth_news")
|
||||
@@ -59,8 +61,15 @@ async def process_target_location_message(
|
||||
provider_client: AIProviderClient | None,
|
||||
) -> dict[str, Any]:
|
||||
item = parsed_news_item_from_job_payload(message.payload)
|
||||
target = await _infer_news_target_location(item, provider_client=provider_client)
|
||||
async with async_session_factory() as session:
|
||||
prompt = await get_effective_prompt(session, NEWS_ENRICH_PROMPT_KEY)
|
||||
target, localizations = await _infer_news_enrichment(
|
||||
item,
|
||||
provider_client=provider_client,
|
||||
prompt=prompt,
|
||||
)
|
||||
item.target_location = target
|
||||
item.localizations = localizations or item.localizations
|
||||
patch = build_target_location_patch(item, target)
|
||||
await save_target_location_patch(item.id, patch)
|
||||
async with async_session_factory() as session:
|
||||
|
||||
@@ -7,8 +7,11 @@ import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.countries import COUNTRY_ENTRIES, normalize_country
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.services.ai_tools.evidence_store import normalize_search_evidence
|
||||
from app.services.ai_tools.web_search import WebSearchClient, WebSearchError
|
||||
@@ -23,6 +26,8 @@ from app.services.location.text import (
|
||||
|
||||
VALID_LLM_PRECISIONS = {"precise", "site", "city"}
|
||||
DEFAULT_MIN_CONFIDENCE = 0.55
|
||||
LOCATION_NORMALIZE_PROMPT_KEY = "location.factcheck.normalize"
|
||||
LOCATION_RESOLVE_PROMPT_KEY = "location.factcheck.resolve"
|
||||
MODEL_CONFIDENCE_WEIGHT = 0.25
|
||||
_geocode_llm_city = build_default_nominatim_geocoder()
|
||||
_LLM_LOCATION_NAME_KEYS = (
|
||||
@@ -876,6 +881,7 @@ async def _repair_location_payload_from_text(
|
||||
raw_text: str,
|
||||
query: LocationQuery,
|
||||
entity_type: str,
|
||||
db: AsyncSession | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Second-pass structure repair for models that answer in prose.
|
||||
|
||||
@@ -884,12 +890,11 @@ async def _repair_location_payload_from_text(
|
||||
"""
|
||||
if not coerce_str(raw_text):
|
||||
return None
|
||||
prompt = await get_effective_prompt(db, LOCATION_NORMALIZE_PROMPT_KEY)
|
||||
request = SituationalAnalysisRequest(
|
||||
title=f"Normalize location factcheck for {entity_type}",
|
||||
objective=(
|
||||
"Convert the supplied location factcheck text into exactly one strict "
|
||||
"JSON object. Extract only facts present in the text or original query."
|
||||
),
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"location_query": _query_context(query),
|
||||
@@ -929,6 +934,7 @@ async def collect_llm_location_fallback_candidate(
|
||||
provider_client: AIProviderClient,
|
||||
query: LocationQuery,
|
||||
entity_type: str,
|
||||
db: AsyncSession | None = None,
|
||||
attempted_queries: Iterable[str] = (),
|
||||
search_evidence: list[dict[str, Any]] | None = None,
|
||||
min_confidence: float = DEFAULT_MIN_CONFIDENCE,
|
||||
@@ -946,13 +952,11 @@ async def collect_llm_location_fallback_candidate(
|
||||
attempted_queries=[attempt],
|
||||
failure_reason="LLM location factcheck skipped: no WebSearch evidence.",
|
||||
)
|
||||
prompt = await get_effective_prompt(db, LOCATION_RESOLVE_PROMPT_KEY)
|
||||
request = SituationalAnalysisRequest(
|
||||
title=f"Location factcheck fallback for {entity_type}",
|
||||
objective=(
|
||||
"Return exactly one JSON object for the most likely physical location. "
|
||||
"Use only fact-checkable public knowledge; return null fields rather "
|
||||
"than guessing when evidence is weak."
|
||||
),
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"location_query": _query_context(query),
|
||||
@@ -1001,6 +1005,7 @@ async def collect_llm_location_fallback_candidate(
|
||||
raw_text=response.content,
|
||||
query=query,
|
||||
entity_type=entity_type,
|
||||
db=db,
|
||||
)
|
||||
if payload is None:
|
||||
payload = _payload_from_free_text(response.content, query=query)
|
||||
|
||||
@@ -10,8 +10,11 @@ from app.models.alert import Alert, AlertSeverity, AlertStatus
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.bgp_ai_brief_store import get_latest_bgp_brief_record
|
||||
|
||||
SITUATIONAL_ALERT_BRIEF_PROMPT_KEY = "alerts.situational.brief"
|
||||
|
||||
|
||||
def _format_pairs(pairs: list[tuple[str, int]], empty_text: str = "无") -> str:
|
||||
if not pairs:
|
||||
@@ -158,10 +161,12 @@ async def build_situational_alert_brief_request(
|
||||
"latest_bgp_brief_id": latest_bgp_brief.id if latest_bgp_brief else None,
|
||||
"latest_bgp_brief_generated_at": latest_bgp_brief.generated_at if latest_bgp_brief else None,
|
||||
}
|
||||
prompt = await get_effective_prompt(db, SITUATIONAL_ALERT_BRIEF_PROMPT_KEY)
|
||||
|
||||
request = SituationalAnalysisRequest(
|
||||
title="态势告警 AI 简报",
|
||||
objective="综合系统告警、BGP incidents、BGP anomalies 与近期 BGP AI 简报,生成一份面向值班人员的态势告警简报,指出当前最需要关注的风险域、跨模块联动迹象和优先动作。",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=facts,
|
||||
constraints=[
|
||||
"明确区分事实、推断与建议。",
|
||||
|
||||
Reference in New Issue
Block a user