release: bump version to 0.59.0
Some checks failed
ci / backend (push) Has been cancelled
ci / frontend (push) Has been cancelled
ci / delivery (push) Has been cancelled
release / images (push) Has been cancelled

This commit is contained in:
rayd1o
2026-05-16 05:02:05 +08:00
parent 93eb41a9f7
commit 9b913a3b83
86 changed files with 3645 additions and 1198 deletions

View File

@@ -0,0 +1,2 @@
"""AI task prompt registry and runtime helpers."""

View File

@@ -0,0 +1,74 @@
[
{
"key": "earth.news.enrich",
"label": "Earth 新闻汉化与定位",
"group": "Earth 新闻",
"version": "2026-05-16.1",
"system_prompt": "",
"prompt": "Return exactly one strict JSON object with a location object and a localizations object. Infer the most likely physical event location and produce a faithful Simplified Chinese title and summary based only on the supplied RSS headline, description, source, and date."
},
{
"key": "alerts.brief",
"label": "系统告警研判",
"group": "告警研判",
"version": "2026-05-16.1",
"system_prompt": "你是告警研判助手。请基于输入的告警事实、上下文与约束,输出结构化、克制、可执行的值班研判;明确区分事实、推断与建议,不要夸大证据不足的风险。",
"prompt": "基于当前告警总量、严重度、状态、数据源分布与最近告警摘录,生成一份面向值班人员的简明告警态势简报,突出待处理风险、告警集中点和优先动作。"
},
{
"key": "alerts.situational.brief",
"label": "跨模块态势告警研判",
"group": "告警研判",
"version": "2026-05-16.1",
"system_prompt": "你是告警研判助手。请基于输入的告警事实、上下文与约束,输出结构化、克制、可执行的值班研判;明确区分事实、推断与建议,不要夸大证据不足的风险。",
"prompt": "综合系统告警、BGP incidents、BGP anomalies 与近期 BGP AI 简报,生成一份面向值班人员的态势告警简报,指出当前最需要关注的风险域、跨模块联动迹象和优先动作。"
},
{
"key": "bgp.brief",
"label": "BGP 态势简报",
"group": "BGP",
"version": "2026-05-16.1",
"system_prompt": "",
"prompt": "基于当前 BGP incidents、anomalies、原始观测事件、观测站覆盖与 prefix geography 证据,生成一份面向操作员的简明态势简报,突出区域热点、观测偏差、当前风险、证据和优先动作。"
},
{
"key": "location.factcheck.normalize",
"label": "位置事实核查结构化",
"group": "位置解析",
"version": "2026-05-16.1",
"system_prompt": "",
"prompt": "Convert the supplied location factcheck text into exactly one strict JSON object. Extract only facts present in the text or original query."
},
{
"key": "location.factcheck.resolve",
"label": "位置事实核查兜底",
"group": "位置解析",
"version": "2026-05-16.1",
"system_prompt": "",
"prompt": "Return exactly one JSON object for the most likely physical location. Use only fact-checkable public knowledge; return null fields rather than guessing when evidence is weak."
},
{
"key": "datasource.mapping",
"label": "数据源映射生成",
"group": "采集配置",
"version": "2026-05-16.1",
"system_prompt": "",
"prompt": "Return only JSON for a deterministic mapping DSL. The JSON must contain source.items_path and fields. Do not include prose or code."
},
{
"key": "credential.guide",
"label": "采集器凭据教程",
"group": "采集配置",
"version": "2026-05-16.1",
"system_prompt": "",
"prompt": "生成一份中文采集器凭据配置教程。只能根据 context.search_evidence 中的来源生成教程;如果证据不足,明确说明需要以官方页面为准。"
},
{
"key": "ai.connection_test",
"label": "AI Provider 连接测试",
"group": "运维测试",
"version": "2026-05-16.1",
"system_prompt": "",
"prompt": "Reply OK."
}
]

View File

@@ -0,0 +1,182 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
import json
from functools import lru_cache
from pathlib import Path
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.system_setting import SystemSetting
AI_PROMPTS_CATEGORY = "ai_prompts"
DEFAULT_PROMPTS_PATH = Path(__file__).with_name("default_prompts.json")
@dataclass(frozen=True)
class AIPromptDefinition:
key: str
label: str
group: str
version: str
system_prompt: str
prompt: str
@dataclass(frozen=True)
class EffectiveAIPrompt:
key: str
label: str
group: str
version: str
default_system_prompt: str
default_prompt: str
system_prompt: str
prompt: str
is_custom: bool
updated_at: str | None = None
@lru_cache(maxsize=1)
def list_prompt_definitions() -> tuple[AIPromptDefinition, ...]:
raw_items = json.loads(DEFAULT_PROMPTS_PATH.read_text(encoding="utf-8"))
return tuple(
AIPromptDefinition(
key=str(item["key"]),
label=str(item["label"]),
group=str(item["group"]),
version=str(item["version"]),
system_prompt=str(item.get("system_prompt") or ""),
prompt=str(item.get("prompt") or ""),
)
for item in raw_items
)
def get_prompt_definition(task_key: str) -> AIPromptDefinition:
for definition in list_prompt_definitions():
if definition.key == task_key:
return definition
raise KeyError(task_key)
async def _get_prompt_setting(db: AsyncSession) -> SystemSetting | None:
result = await db.execute(
select(SystemSetting).where(SystemSetting.category == AI_PROMPTS_CATEGORY)
)
return result.scalar_one_or_none()
def _normalize_overrides(payload: dict[str, Any] | None) -> dict[str, dict[str, Any]]:
raw = (payload or {}).get("overrides")
if not isinstance(raw, dict):
return {}
return {
str(key): dict(value)
for key, value in raw.items()
if isinstance(value, dict)
}
async def get_prompt_overrides(db: AsyncSession) -> dict[str, dict[str, Any]]:
if not hasattr(db, "execute"):
return {}
setting = await _get_prompt_setting(db)
return _normalize_overrides(setting.payload if setting else None)
def _effective_prompt(
definition: AIPromptDefinition,
override: dict[str, Any] | None,
) -> EffectiveAIPrompt:
override = override or {}
custom_system = override.get("system_prompt")
custom_prompt = override.get("prompt")
has_custom_system = isinstance(custom_system, str)
has_custom_prompt = isinstance(custom_prompt, str)
return EffectiveAIPrompt(
key=definition.key,
label=definition.label,
group=definition.group,
version=definition.version,
default_system_prompt=definition.system_prompt,
default_prompt=definition.prompt,
system_prompt=custom_system if has_custom_system else definition.system_prompt,
prompt=custom_prompt if has_custom_prompt else definition.prompt,
is_custom=has_custom_system or has_custom_prompt,
updated_at=str(override.get("updated_at") or "") or None,
)
async def list_effective_prompts(db: AsyncSession) -> list[EffectiveAIPrompt]:
overrides = await get_prompt_overrides(db)
return [
_effective_prompt(definition, overrides.get(definition.key))
for definition in list_prompt_definitions()
]
async def get_effective_prompt(db: AsyncSession | None, task_key: str) -> EffectiveAIPrompt:
definition = get_prompt_definition(task_key)
if db is None:
return _effective_prompt(definition, None)
overrides = await get_prompt_overrides(db)
return _effective_prompt(definition, overrides.get(task_key))
async def save_prompt_override(
db: AsyncSession,
task_key: str,
*,
system_prompt: str,
prompt: str,
) -> EffectiveAIPrompt:
definition = get_prompt_definition(task_key)
setting = await _get_prompt_setting(db)
payload = dict(setting.payload or {}) if setting else {}
overrides = _normalize_overrides(payload)
overrides[definition.key] = {
"system_prompt": system_prompt,
"prompt": prompt,
"updated_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
}
payload["overrides"] = overrides
if setting is None:
setting = SystemSetting(category=AI_PROMPTS_CATEGORY, payload=payload)
db.add(setting)
else:
setting.payload = payload
await db.commit()
return _effective_prompt(definition, overrides[definition.key])
async def reset_prompt_override(db: AsyncSession, task_key: str) -> EffectiveAIPrompt:
definition = get_prompt_definition(task_key)
setting = await _get_prompt_setting(db)
if setting is None:
return _effective_prompt(definition, None)
payload = dict(setting.payload or {})
overrides = _normalize_overrides(payload)
overrides.pop(definition.key, None)
payload["overrides"] = overrides
setting.payload = payload
await db.commit()
return _effective_prompt(definition, None)
def serialize_effective_prompt(prompt: EffectiveAIPrompt) -> dict[str, Any]:
return {
"key": prompt.key,
"label": prompt.label,
"group": prompt.group,
"version": prompt.version,
"default_system_prompt": prompt.default_system_prompt,
"default_prompt": prompt.default_prompt,
"system_prompt": prompt.system_prompt,
"prompt": prompt.prompt,
"is_custom": prompt.is_custom,
"updated_at": prompt.updated_at,
}

View File

@@ -6,6 +6,7 @@ from app.api.v1 import (
datasource_config,
datasources,
docs,
earth,
tasks,
dashboard,
alerts,
@@ -34,6 +35,7 @@ api_router.include_router(
api_router.include_router(datasources.router, prefix="/datasources", tags=["datasources"])
api_router.include_router(collected_data.router, prefix="/collected", tags=["collected-data"])
api_router.include_router(docs.router, prefix="/docs", tags=["docs"])
api_router.include_router(earth.router, prefix="/earth", tags=["earth"])
api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboard"])
api_router.include_router(alerts.router, prefix="/alerts", tags=["alerts"])

View File

@@ -341,6 +341,7 @@ async def collect_bgp_collector_location(
provider_client=provider_client,
query=query,
entity_type="bgp_collector",
db=db,
attempted_queries=attempted_queries,
search_evidence=search_result.evidence,
)

View File

@@ -13,11 +13,6 @@ import httpx
from app.core.target_schema_registry import get_target_schema, list_target_schemas
from app.core.datasource_defaults import DEFAULT_DATASOURCES
from app.core.earth_boundary_defaults import (
EARTH_BOUNDARY_DEFAULT_SOURCES,
EARTH_BOUNDARY_SOURCE_MAPPING,
default_earth_boundary_config,
)
from app.db.session import get_db
from app.models.user import User
from app.models.datasource_config import DataSourceConfig
@@ -27,6 +22,7 @@ from app.models.vessel import AISRawObservation, AISSourceHealth
from app.core.security import get_current_user
from app.core.cache import cache
from app.core.time import to_iso8601_utc
from app.ai_tasks.prompts import get_effective_prompt
from app.schemas.ai import SituationalAnalysisRequest
from app.services.ai_client import AIProviderClient, get_ai_provider_client
from app.services.datasource_mapping import (
@@ -46,6 +42,8 @@ from app.services.custom_datasource_runtime import (
stop_custom_stream,
test_websocket_config,
)
DATASOURCE_MAPPING_PROMPT_KEY = "datasource.mapping"
from app.services.datasource_connectivity import (
get_builtin_connection_status,
save_connectivity_success,
@@ -56,22 +54,12 @@ from app.services.datasource_connectivity import (
router = APIRouter()
def _default_builtin_config(name: str) -> dict[str, Any]:
if name in {"earth_admin0_boundaries", "earth_coastline", "earth_claim_lines"}:
return default_earth_boundary_config(name)
if name == "earth_boundary_tiles":
return {
"timeout": 30,
"retry": 0,
"input_mode": "latest_collected_earth_boundary_sources",
}
return {"timeout": 30, "retry": 3}
def _default_builtin_source_type(name: str) -> str:
if name == "aisstream_vessels":
return "websocket"
if name == "earth_boundary_tiles":
return "internal"
return "http"
@@ -400,7 +388,7 @@ async def list_all_datasources(
yaml_url = config.get_yaml_url(name)
db_config = db_configs.get(name)
default_config = _default_builtin_config(name)
default_url = EARTH_BOUNDARY_DEFAULT_SOURCES.get(name, {}).get("endpoint") or yaml_url
default_url = yaml_url
result.append(
{
@@ -783,14 +771,12 @@ async def propose_datasource_mapping(
generated_by = "heuristic"
if payload.use_ai:
try:
prompt = await get_effective_prompt(db, DATASOURCE_MAPPING_PROMPT_KEY)
response = await ai_client.analyze(
SituationalAnalysisRequest(
title=f"Generate datasource mapping for {schema.key}",
objective=(
"Return only JSON for a deterministic mapping DSL. "
"The JSON must contain source.items_path and fields. "
"Do not include prose or code."
),
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
context={
"target_schema": schema.to_dict(),
"sample_payload": redacted_sample,

View File

@@ -35,7 +35,6 @@ PRODUCT_SOURCE_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = (
("cables", ("cable", "landing", "telegeography", "arcgis", "fao")),
("satellites", ("tle", "satellite", "spacetrack", "celestrak")),
("bgp", ("bgp", "asn", "prefix_geo", "opengeofeed", "nro")),
("earth", ("earth_", "earth_boundary", "boundary", "country boundary", "coastline", "claim")),
("compute", ("top500", "gpu", "supercomputer", "compute")),
("ai", ("huggingface", "epoch_ai")),
("media", ("news", "tv", "live_stream")),

118
backend/app/api/v1/earth.py Normal file
View File

@@ -0,0 +1,118 @@
"""Earth asset management APIs."""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, Request, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from pydantic import BaseModel, Field
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.security import decode_token, get_current_user, redis_client
from app.db.session import get_db
from app.models.user import User
from app.services.earth_boundaries import (
EarthBoundaryBuildError,
get_boundary_build_status,
get_boundary_status,
save_boundary_config,
start_boundary_build_job,
)
router = APIRouter()
optional_bearer = HTTPBearer(auto_error=False)
class EarthBoundaryConfigPayload(BaseModel):
config: dict[str, Any] = Field(default_factory=dict)
@router.get("/boundaries/status")
async def get_earth_boundary_status():
return get_boundary_status()
async def _get_optional_current_user(
credentials: HTTPAuthorizationCredentials | None = Depends(optional_bearer),
db: AsyncSession = Depends(get_db),
) -> User | None:
if credentials is None:
return None
token = credentials.credentials
if redis_client.sismember("blacklisted_tokens", token):
return None
payload = decode_token(token)
if payload is None or payload.get("type") != "access":
return None
user_id = payload.get("sub")
if user_id is None:
return None
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id"
),
{"id": int(user_id)},
)
row = result.fetchone()
if row is None or not row[5]:
return None
user = User()
user.id = row[0]
user.username = row[1]
user.email = row[2]
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
return user
def _is_loopback_request(request: Request) -> bool:
host = request.client.host if request.client else ""
return host in {"127.0.0.1", "::1", "localhost"} or host.startswith("127.")
def _require_local_or_user(request: Request, user: User | None) -> None:
if user is not None or _is_loopback_request(request):
return
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Authentication required outside localhost",
)
@router.put("/boundaries/config")
async def update_earth_boundary_config(
payload: EarthBoundaryConfigPayload,
_current_user: User = Depends(get_current_user),
):
try:
return save_boundary_config(payload.config)
except EarthBoundaryBuildError as exc:
raise HTTPException(
status_code=400,
detail={"code": exc.code, "message": str(exc), "details": exc.details},
) from exc
@router.post("/boundaries/build")
async def build_earth_boundary_assets(
request: Request,
current_user: User | None = Depends(_get_optional_current_user),
):
_require_local_or_user(request, current_user)
try:
return await start_boundary_build_job()
except EarthBoundaryBuildError as exc:
raise HTTPException(
status_code=400,
detail={"code": exc.code, "message": str(exc), "details": exc.details},
) from exc
@router.get("/boundaries/build/status")
async def get_earth_boundary_build_status():
return get_boundary_build_status()

View File

@@ -15,6 +15,13 @@ from app.core.time import to_iso8601_utc
from app.core.config import settings as app_settings
from app.core.data_sources import get_data_sources_config
from app.core.datasource_defaults import DEFAULT_DATASOURCES
from app.ai_tasks.prompts import (
get_effective_prompt,
list_effective_prompts,
reset_prompt_override,
save_prompt_override,
serialize_effective_prompt,
)
from app.db.session import get_db
from app.models.datasource import DataSource
from app.models.datasource_config import DataSourceConfig
@@ -60,6 +67,7 @@ from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload
router = APIRouter()
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5
AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test"
DEFAULT_SETTINGS = {
"system": {
@@ -251,6 +259,11 @@ class OCRIntegrationUpdate(BaseModel):
output_format: str = Field(default="markdown", pattern="^(markdown|text|json)$")
class AIPromptUpdate(BaseModel):
system_prompt: str = Field(default="", max_length=8000)
prompt: str = Field(min_length=1, max_length=20000)
class ExternalIntegrationsUpdate(BaseModel):
ai_provider: AIProviderIntegrationUpdate
barentswatch: BarentsWatchIntegrationUpdate
@@ -487,6 +500,10 @@ def _is_secret_placeholder(value: Optional[str], current_preview: str = "") -> b
return True
if text == current_preview or text.startswith("••••"):
return True
if "-" in text:
_prefix, masked = text.split("-", 1)
if masked and all(char in {"*", "", " ", "\t"} for char in masked):
return True
return all(char in {"*", "", " ", "\t"} for char in text)
@@ -597,10 +614,12 @@ async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict:
status_code=400,
detail="AI Provider 可访问,但当前 provider/model/key 未完整配置。",
)
prompt = await get_effective_prompt(None, AI_CONNECTION_TEST_PROMPT_KEY)
analysis_result = await client.analyze(
SituationalAnalysisRequest(
title="保存前完整连接测试",
objective="请用一句话回复连接可用。",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
observations=["这是保存 AI Provider 配置前的完整 LLM 调用测试。"],
constraints=["回复尽量简短。"],
)
@@ -1212,6 +1231,47 @@ async def get_external_integrations(
return {"integrations": await serialize_external_integrations(db)}
@router.get("/ai-prompts")
async def get_ai_prompts(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
prompts = await list_effective_prompts(db)
return {"data": [serialize_effective_prompt(prompt) for prompt in prompts]}
@router.put("/ai-prompts/{task_key}")
async def update_ai_prompt(
task_key: str,
payload: AIPromptUpdate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
prompt = await save_prompt_override(
db,
task_key,
system_prompt=payload.system_prompt,
prompt=payload.prompt,
)
except KeyError:
raise HTTPException(status_code=404, detail="未知 AI 提示词任务") from None
return {"data": serialize_effective_prompt(prompt)}
@router.post("/ai-prompts/{task_key}/reset")
async def reset_ai_prompt(
task_key: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
prompt = await reset_prompt_override(db, task_key)
except KeyError:
raise HTTPException(status_code=404, detail="未知 AI 提示词任务") from None
return {"data": serialize_effective_prompt(prompt)}
@router.get("/integrations/barentswatch/connectivity")
async def get_barentswatch_connectivity(
current_user: User = Depends(get_current_user),
@@ -1294,10 +1354,12 @@ async def connect_ai_provider_integration(
"message": "AI Provider 可访问,但当前 provider/model/key 未完整配置。",
"status": status_result.model_dump(),
}
prompt = await get_effective_prompt(db, AI_CONNECTION_TEST_PROMPT_KEY)
probe_result = await client.analyze(
SituationalAnalysisRequest(
title="快速连接测试",
objective="Reply OK.",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
observations=[],
constraints=["Output only OK."],
)

View File

@@ -1982,6 +1982,7 @@ async def collect_compute_center_location(
provider_client=provider_client,
query=query,
entity_type="compute_center",
db=db,
attempted_queries=attempted_queries,
search_evidence=search_result.evidence,
)

View File

@@ -257,46 +257,6 @@ DEFAULT_DATASOURCES = {
"credential_provider": "aisstream",
"credential_status": "supported",
},
"earth_admin0_boundaries": {
"id": 29,
"name": "Earth Admin-0 Boundaries",
"display_name": "Earth Admin-0 国界源",
"module": "L3",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"earth_coastline": {
"id": 30,
"name": "Earth Coastline",
"display_name": "Earth 海岸线源",
"module": "L3",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"earth_claim_lines": {
"id": 31,
"name": "Earth Claim Lines",
"display_name": "Earth 主张线源",
"module": "L3",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"earth_boundary_tiles": {
"id": 32,
"name": "Earth PMTiles Builder",
"display_name": "Earth PMTiles 构建器",
"module": "L3",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"media_news_archive": {
"id": 33,
"name": "Media News Archive",

View File

@@ -1,49 +0,0 @@
"""Default Earth boundary source configuration."""
from __future__ import annotations
from typing import Any
EARTH_BOUNDARY_SOURCE_MAPPING: dict[str, Any] = {
"source": {"items_path": "$.features[*]"},
"fields": {
"source_id": {"path": "$.properties.id", "type": "string"},
"name": {"path": "$.properties.name", "type": "string"},
"geometry": {"path": "$.geometry", "type": "object"},
"properties": {"path": "$.properties", "type": "object"},
},
}
EARTH_BOUNDARY_DEFAULT_SOURCES: dict[str, dict[str, Any]] = {
"earth_admin0_boundaries": {
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_countries.geojson",
"source_kind": "admin0-boundaries",
"license": "Natural Earth public domain",
},
"earth_coastline": {
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_coastline.geojson",
"source_kind": "coastline",
"license": "Natural Earth public domain",
},
"earth_claim_lines": {
"endpoint": "https://www.arcgis.com/sharing/rest/content/items/faaa1908c3ab43f0823c6fde9f18389c/data",
"source_kind": "claim-lines",
"license": "CC BY 4.0; source item owner mapmakersami",
},
}
def default_earth_boundary_config(name: str) -> dict[str, Any]:
source = EARTH_BOUNDARY_DEFAULT_SOURCES.get(name)
if not source:
return {}
return {
"method": "GET",
"timeout": 120,
"retry": 3,
"target_schema": "earth_boundary_source",
"license": source["license"],
"mapping_json": EARTH_BOUNDARY_SOURCE_MAPPING,
}

View File

@@ -47,18 +47,6 @@ class GenericRecord(BaseModel):
return value
class EarthBoundarySourceRecord(BaseModel):
source_id: str | None = None
source_kind: str = Field(pattern="^(admin0-boundaries|coastline|claim-lines)$")
name: str | None = None
geometry: dict[str, Any] | None = None
properties: dict[str, Any] = Field(default_factory=dict)
artifact_path: str | None = None
sha256: str | None = None
feature_count: int | None = Field(default=None, ge=0)
license: str | None = None
@dataclass(frozen=True)
class TargetField:
name: str
@@ -155,24 +143,6 @@ TARGET_SCHEMAS: dict[str, TargetSchema] = {
TargetField("observed_at", "datetime", False, "观测时间", "2026-04-28T00:00:00Z"),
),
),
"earth_boundary_source": TargetSchema(
key="earth_boundary_source",
label="Earth 边界矢量源",
description="Admin-0 国界、海岸线、主张线等 Earth 边界源数据 artifact。",
destination="collected_data",
model=EarthBoundarySourceRecord,
fields=(
TargetField("source_id", "string", False, "来源侧 ID", "feature-1"),
TargetField("source_kind", "string", True, "边界源类型", "admin0-boundaries"),
TargetField("name", "string", False, "记录名称", "China"),
TargetField("geometry", "object", False, "GeoJSON geometry", {"type": "Polygon", "coordinates": []}),
TargetField("properties", "object", False, "GeoJSON properties", {"ISO_A3": "CHN"}),
TargetField("artifact_path", "string", False, "完整源数据本地 artifact 路径", "data/earth-boundary-sources/earth_admin0_boundaries/abc.geojson"),
TargetField("sha256", "string", False, "源数据 artifact SHA-256", "abc123"),
TargetField("feature_count", "integer", False, "源数据 feature 数", 1),
TargetField("license", "string", False, "源数据许可", "ODbL"),
),
),
}

View File

@@ -1,6 +1,6 @@
from typing import AsyncGenerator
from sqlalchemy import text
from sqlalchemy import bindparam, text
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
from sqlalchemy.orm import declarative_base
@@ -72,6 +72,74 @@ async def seed_default_datasources(session: AsyncSession):
await session.commit()
LEGACY_EARTH_BOUNDARY_SOURCES = (
"earth_admin0_boundaries",
"earth_coastline",
"earth_claim_lines",
"earth_boundary_tiles",
)
LEGACY_EARTH_BOUNDARY_DATATYPES = (
"earth_boundary_source",
"earth_boundary_tiles",
)
LEGACY_EARTH_BOUNDARY_IDS = (29, 30, 31, 32)
async def purge_legacy_earth_boundary_datasources(session: AsyncSession) -> None:
source_names = tuple(LEGACY_EARTH_BOUNDARY_SOURCES)
source_ids = tuple(LEGACY_EARTH_BOUNDARY_IDS)
data_types = tuple(LEGACY_EARTH_BOUNDARY_DATATYPES)
await session.execute(
text(
"""
DELETE FROM datasource_mapping_templates
WHERE target_schema IN :data_types
OR datasource_config_id IN (
SELECT id FROM datasource_configs WHERE name IN :source_names
)
"""
).bindparams(bindparam("source_names", expanding=True), bindparam("data_types", expanding=True)),
{"source_names": list(source_names), "data_types": list(data_types)},
)
await session.execute(
text("DELETE FROM datasource_configs WHERE name IN :source_names").bindparams(
bindparam("source_names", expanding=True)
),
{"source_names": list(source_names)},
)
await session.execute(
text(
"""
DELETE FROM collected_data
WHERE source IN :source_names OR data_type IN :data_types
"""
).bindparams(bindparam("source_names", expanding=True), bindparam("data_types", expanding=True)),
{"source_names": list(source_names), "data_types": list(data_types)},
)
await session.execute(
text(
"""
DELETE FROM data_snapshots
WHERE source IN :source_names OR datasource_id IN :source_ids
"""
).bindparams(bindparam("source_names", expanding=True), bindparam("source_ids", expanding=True)),
{"source_names": list(source_names), "source_ids": list(source_ids)},
)
await session.execute(
text("DELETE FROM collection_tasks WHERE datasource_id IN :source_ids").bindparams(
bindparam("source_ids", expanding=True)
),
{"source_ids": list(source_ids)},
)
await session.execute(
text("DELETE FROM data_sources WHERE source IN :source_names OR id IN :source_ids").bindparams(
bindparam("source_names", expanding=True), bindparam("source_ids", expanding=True)
),
{"source_names": list(source_names), "source_ids": list(source_ids)},
)
await session.commit()
DEFAULT_LOGIN_USERS = (
{
"username": "admin",
@@ -203,6 +271,18 @@ async def init_db():
"""
)
)
await conn.execute(
text(
"""
ALTER TABLE earth_news_items
ADD COLUMN IF NOT EXISTS content_language VARCHAR(32) NOT NULL DEFAULT 'en',
ADD COLUMN IF NOT EXISTS localizations JSONB NOT NULL DEFAULT '{}'::jsonb,
ADD COLUMN IF NOT EXISTS enrichment_status VARCHAR(80) NOT NULL DEFAULT 'pending',
ADD COLUMN IF NOT EXISTS enrichment_error TEXT,
ADD COLUMN IF NOT EXISTS enriched_at TIMESTAMPTZ
"""
)
)
await conn.execute(
text(
"""
@@ -211,6 +291,22 @@ async def init_db():
"""
)
)
await conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS idx_earth_news_enrichment_status
ON earth_news_items (enrichment_status)
"""
)
)
await conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS idx_earth_news_enriched_at
ON earth_news_items (enriched_at)
"""
)
)
await conn.execute(
text(
"""
@@ -285,4 +381,5 @@ async def init_db():
await seed_default_bgp_collector_locations(session)
await seed_compute_center_locations_from_source_coords(session)
await seed_default_datasources(session)
await purge_legacy_earth_boundary_datasources(session)
await ensure_default_admin_user(session)

View File

@@ -10,6 +10,8 @@ class EarthNewsItem(Base):
id = Column(String(160), primary_key=True)
title = Column(String(500), nullable=False)
summary = Column(Text, nullable=False, default="")
content_language = Column(String(32), nullable=False, default="en")
localizations = Column(JSON, nullable=False, default=dict)
url = Column(Text, nullable=False)
source = Column(String(255), nullable=False, default="")
feed_name = Column(String(255), nullable=False, default="")
@@ -27,6 +29,9 @@ class EarthNewsItem(Base):
first_seen_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
last_seen_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False, index=True)
resolved_at = Column(DateTime(timezone=True), nullable=True, index=True)
enrichment_status = Column(String(80), nullable=False, default="pending", index=True)
enrichment_error = Column(Text, nullable=True)
enriched_at = Column(DateTime(timezone=True), nullable=True, index=True)
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
__table_args__ = (

View File

@@ -13,10 +13,11 @@ class AIContentBlock(BaseModel):
class SituationalAnalysisRequest(BaseModel):
title: str = Field(..., min_length=1, max_length=200)
objective: str = Field(..., min_length=1, max_length=1000)
objective: str = Field(..., min_length=1, max_length=20000)
context: dict[str, Any] = Field(default_factory=dict)
observations: list[str] = Field(default_factory=list)
constraints: list[str] = Field(default_factory=list)
system_prompt: str | None = Field(default=None, max_length=8000)
preferred_model: str | None = Field(default=None, max_length=200)
thinking: dict[str, Any] | None = None

View File

@@ -8,6 +8,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.models.alert import Alert, AlertSeverity, AlertStatus
from app.schemas.ai import AlertBriefRequest, SituationalAnalysisRequest
from app.ai_tasks.prompts import get_effective_prompt
ALERT_BRIEF_PROMPT_KEY = "alerts.brief"
def _format_counter(counter: Counter[str], empty_text: str = "") -> str:
@@ -84,11 +87,13 @@ async def build_alert_brief_request(
"top_datasources": dict(datasource_counts.most_common(6)),
"top_active_datasources": dict(active_datasource_counts.most_common(5)),
}
prompt = await get_effective_prompt(db, ALERT_BRIEF_PROMPT_KEY)
return (
SituationalAnalysisRequest(
title="告警态势 AI 简报",
objective="基于当前告警总量、严重度、状态、数据源分布与最近告警摘录,生成一份面向值班人员的简明告警态势简报,突出待处理风险、告警集中点和优先动作。",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
observations=facts,
constraints=[
"明确区分事实、推断与建议。",

View File

@@ -11,9 +11,12 @@ from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.schemas.ai import SituationalAnalysisRequest
from app.ai_tasks.prompts import get_effective_prompt
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.bgp_enrichment import lookup_prefix_geography
BGP_BRIEF_PROMPT_KEY = "bgp.brief"
def _format_counter(counter: dict[str, int], empty_text: str = "") -> str:
if not counter:
@@ -243,10 +246,12 @@ async def build_bgp_brief_request(
for prefix, item in list(prefix_geographies.items())[:8]
},
}
prompt = await get_effective_prompt(db, BGP_BRIEF_PROMPT_KEY)
return SituationalAnalysisRequest(
title="BGP 态势 AI 简报",
objective="基于当前 BGP incidents、anomalies、原始观测事件、观测站覆盖与 prefix geography 证据,生成一份面向操作员的简明态势简报,突出区域热点、观测偏差、当前风险、证据和优先动作。",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
observations=observations_lines,
constraints=[
"明确区分事实、推断与建议。",

View File

@@ -39,12 +39,6 @@ from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
from app.services.collectors.media_news_archive import MediaNewsArchiveCollector
from app.services.collectors.aisstream import AISStreamCollector
from app.services.collectors.vessel_ais import VesselAISCollector
from app.services.collectors.earth_boundaries import (
EarthAdmin0BoundaryCollector,
EarthBoundaryTileCollector,
EarthClaimLinesCollector,
EarthCoastlineCollector,
)
collector_registry.register(TOP500Collector())
collector_registry.register(EpochAIGPUCollector())
@@ -75,10 +69,6 @@ collector_registry.register(NewsLiveStreamsCollector())
collector_registry.register(MediaNewsArchiveCollector())
collector_registry.register(VesselAISCollector())
collector_registry.register(AISStreamCollector())
collector_registry.register(EarthAdmin0BoundaryCollector())
collector_registry.register(EarthCoastlineCollector())
collector_registry.register(EarthClaimLinesCollector())
collector_registry.register(EarthBoundaryTileCollector())
__all__ = [
"BaseCollector",
@@ -115,8 +105,4 @@ __all__ = [
"MediaNewsArchiveCollector",
"VesselAISCollector",
"AISStreamCollector",
"EarthAdmin0BoundaryCollector",
"EarthCoastlineCollector",
"EarthClaimLinesCollector",
"EarthBoundaryTileCollector",
]

View File

@@ -1,578 +0,0 @@
"""Earth boundary source and static tile collector."""
from __future__ import annotations
import asyncio
import hashlib
import json
import shutil
import sys
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import httpx
from sqlalchemy import select
from app.core.earth_boundary_defaults import EARTH_BOUNDARY_DEFAULT_SOURCES, default_earth_boundary_config
from app.models.collected_data import CollectedData
from app.models.datasource_config import DataSourceConfig
from app.services.collectors.base import BaseCollector
from app.services.custom_datasource_runtime import build_query_params, build_request_headers
REPO_ROOT = Path(__file__).resolve().parents[4]
SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
PMTILES_ARTIFACT_PATH = REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
BUILD_CONFIG = {
"builder": "scripts/build_earth_boundary_china_pov_geojson.py",
"format": "geojson-high-precision",
"production_target": "geojson-high-precision",
}
EARTH_BOUNDARY_SOURCE_COLLECTORS = {
"earth_admin0_boundaries": "admin0-boundaries",
"earth_coastline": "coastline",
"earth_claim_lines": "claim-lines",
}
def _read_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def _directory_stats(path: Path) -> dict[str, int]:
if not path.exists():
return {"file_count": 0, "size_bytes": 0}
files = [item for item in path.rglob("*") if item.is_file()]
return {
"file_count": len(files),
"size_bytes": sum(item.stat().st_size for item in files),
}
def _stable_json_hash(payload: Any) -> str:
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def _build_input_hash(source_manifest: dict[str, Any]) -> str:
return _stable_json_hash(
{
"source_manifest_schema": source_manifest.get("schema"),
"sources": [
{
"id": source.get("id"),
"sha256": source.get("sha256"),
"kind": source.get("kind"),
"pov": source.get("pov"),
}
for source in source_manifest.get("sources", [])
],
"pov_policy": source_manifest.get("povPolicy"),
"build_config": BUILD_CONFIG,
}
)
def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
if not boundary_manifest:
return False
if boundary_manifest.get("buildInputHash") != build_input_hash:
return False
if not BOUNDARY_OUTPUT_DIR.exists():
return False
if boundary_manifest.get("tileProvider") == "pmtiles-mvt":
return PMTILES_ARTIFACT_PATH.exists()
return (BOUNDARY_OUTPUT_DIR / "base.geojson").exists() and (
BOUNDARY_OUTPUT_DIR / "hover-index.geojson"
).exists()
def _sha256_bytes(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def _json_feature_count(payload: Any) -> int:
if isinstance(payload, dict) and isinstance(payload.get("features"), list):
return len(payload["features"])
if isinstance(payload, list):
return len(payload)
return 1 if payload else 0
def _json_sample_properties(payload: Any) -> dict[str, Any]:
feature = None
if isinstance(payload, dict) and isinstance(payload.get("features"), list) and payload["features"]:
feature = payload["features"][0]
elif isinstance(payload, list) and payload:
feature = payload[0]
elif isinstance(payload, dict):
feature = payload
if not isinstance(feature, dict):
return {}
props = feature.get("properties") if isinstance(feature.get("properties"), dict) else feature
return {str(key): value for key, value in list(props.items())[:20]}
def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
if suffix in {".json", ".geojson", ".zip", ".pbf"}:
return suffix
if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
return ".geojson"
if "json" in content_type:
return ".json"
return ".dat"
async def _load_datasource_config(db, name: str) -> DataSourceConfig | None:
if db is None:
return None
result = await db.execute(
select(DataSourceConfig)
.where(DataSourceConfig.name == name)
.where(DataSourceConfig.is_active.is_(True))
.order_by(DataSourceConfig.id.desc())
.limit(1)
)
return result.scalar_one_or_none()
async def _latest_boundary_source_record(db, source_name: str) -> CollectedData | None:
result = await db.execute(
select(CollectedData)
.where(CollectedData.source == source_name)
.where(CollectedData.data_type == "earth_boundary_source")
.where(CollectedData.is_current.is_(True))
.order_by(CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
.limit(1)
)
return result.scalar_one_or_none()
class EarthBoundarySourceCollector(BaseCollector):
priority = "P1"
module = "L3"
frequency_hours = 168
data_type = "earth_boundary_source"
fail_on_empty = True
source_kind = "unknown"
def _default_config(self):
source = EARTH_BOUNDARY_DEFAULT_SOURCES.get(self.name)
if not source:
return None
return SimpleNamespace(
name=self.name,
description=f"内置默认源:{self.name}",
endpoint=source["endpoint"],
source_type="http",
auth_type="none",
auth_config={},
headers={},
config=default_earth_boundary_config(self.name),
)
async def _download_payload(self, config: DataSourceConfig) -> tuple[bytes, str]:
request_config = config.config or {}
method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
if method not in {"GET", "POST"}:
raise RuntimeError("Earth boundary source collectors support GET and POST only")
endpoint = str(config.endpoint or "").strip()
if not endpoint:
raise RuntimeError(
f"{self.name} requires an endpoint in Collector Settings before it can collect data"
)
if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
path = Path(endpoint.removeprefix("file://")).expanduser()
return path.read_bytes(), "application/octet-stream"
headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {})
params = build_query_params(config.auth_type, config.auth_config or {}, request_config)
timeout = float(request_config.get("timeout", 120))
json_body = request_config.get("json_body") or request_config.get("body")
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
response = await client.request(
method,
endpoint,
headers=headers,
params=params or None,
json=json_body if isinstance(json_body, (dict, list)) else None,
)
response.raise_for_status()
return response.content, response.headers.get("content-type", "")
async def fetch(self) -> list[dict[str, Any]]:
db = getattr(self, "_db_session", None)
config = await _load_datasource_config(db, self.name) or self._default_config()
if config is None:
raise RuntimeError(
f"{self.name} has no active Collector Settings config and no built-in default source."
)
await self.set_phase("fetching_source", message=f"正在下载 {self.source_kind} 源数据")
payload, content_type = await self._download_payload(config)
sha256 = _sha256_bytes(payload)
endpoint = str(config.endpoint or "")
extension = _artifact_extension(endpoint, content_type, payload)
source_dir = SOURCE_OUTPUT_DIR / self.name
source_dir.mkdir(parents=True, exist_ok=True)
artifact_path = source_dir / f"{sha256}{extension}"
artifact_path.write_bytes(payload)
parsed: Any = None
if extension in {".json", ".geojson"}:
parsed = json.loads(payload.decode("utf-8"))
feature_count = _json_feature_count(parsed)
if feature_count <= 0:
raise RuntimeError(f"{self.name} downloaded data but found no JSON/GeoJSON features")
config_body = config.config or {}
target_schema = str(config_body.get("target_schema") or "earth_boundary_source")
if target_schema != "earth_boundary_source":
raise RuntimeError(f"{self.name} target_schema must be earth_boundary_source")
await self.update_phase_progress(
current=1,
total=1,
unit="artifact",
message=f"已保存 {feature_count}{self.source_kind} feature",
progress=100,
commit=True,
force=True,
)
relative_artifact_path = str(artifact_path.relative_to(REPO_ROOT))
return [
{
"id": f"{self.source_kind}:{sha256}",
"name": config.description or self.name,
"description": f"Earth boundary source artifact collected from configured endpoint",
"source_kind": self.source_kind,
"source_id": sha256,
"value": feature_count,
"unit": "features",
"metadata": {
"target_schema": target_schema,
"source_kind": self.source_kind,
"endpoint": endpoint,
"method": str(config_body.get("method") or config_body.get("request_method") or "GET").upper(),
"artifact_path": relative_artifact_path,
"sha256": sha256,
"feature_count": feature_count,
"size_bytes": len(payload),
"content_type": content_type,
"license": config_body.get("license"),
"mapping_json": config_body.get("mapping_json"),
"sample_properties": _json_sample_properties(parsed),
},
}
]
class EarthAdmin0BoundaryCollector(EarthBoundarySourceCollector):
name = "earth_admin0_boundaries"
source_kind = "admin0-boundaries"
class EarthCoastlineCollector(EarthBoundarySourceCollector):
name = "earth_coastline"
source_kind = "coastline"
class EarthClaimLinesCollector(EarthBoundarySourceCollector):
name = "earth_claim_lines"
source_kind = "claim-lines"
class EarthBoundaryTileCollector(BaseCollector):
name = "earth_boundary_tiles"
priority = "P1"
module = "L3"
frequency_hours = 168
data_type = "earth_boundary_tiles"
fail_on_empty = False
async def _run_step(self, args: list[str], *, allow_failure: bool = False) -> dict[str, Any]:
process = await asyncio.create_subprocess_exec(
sys.executable,
*args,
cwd=REPO_ROOT,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout_bytes, stderr_bytes = await process.communicate()
stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
if process.returncode != 0 and not allow_failure:
raise RuntimeError(stderr or stdout or f"command failed: {' '.join(args)}")
payload: dict[str, Any] = {
"stdout": stdout,
"stderr": stderr,
"returncode": process.returncode,
}
last_line = stdout.splitlines()[-1:] or []
if last_line:
try:
payload["result"] = json.loads(last_line[0])
except json.JSONDecodeError:
payload["result"] = last_line[0]
return payload
async def fetch(self) -> list[dict[str, Any]]:
db = getattr(self, "_db_session", None)
if db is None:
raise RuntimeError("Earth PMTiles builder requires an active database session")
await self.set_phase("checking_sources", message="正在检查三类 Earth 边界源")
await self.update_phase_progress(
current=0,
total=3,
unit="steps",
message="读取 admin0 / coastline / claim-lines 最新采集结果",
progress=5,
commit=True,
force=True,
)
source_records: dict[str, CollectedData] = {}
missing_sources: list[str] = []
for source_name in EARTH_BOUNDARY_SOURCE_COLLECTORS:
record = await _latest_boundary_source_record(db, source_name)
if record is None:
missing_sources.append(source_name)
else:
source_records[source_name] = record
if missing_sources:
missing_text = ", ".join(missing_sources)
raise RuntimeError(f"未就绪:缺少 {missing_text}。不会更新 Earth 国界。")
sources = []
for source_name, record in source_records.items():
metadata = record.extra_data or {}
artifact_path = metadata.get("artifact_path")
if not artifact_path or not (REPO_ROOT / str(artifact_path)).exists():
missing_sources.append(f"{source_name}: artifact missing")
continue
sources.append(
{
"id": source_name,
"kind": metadata.get("source_kind") or EARTH_BOUNDARY_SOURCE_COLLECTORS[source_name],
"path": str(artifact_path),
"sha256": metadata.get("sha256"),
"featureCount": metadata.get("feature_count"),
"license": metadata.get("license"),
}
)
if missing_sources:
missing_text = ", ".join(missing_sources)
raise RuntimeError(f"未就绪:{missing_text}。不会更新 Earth 国界。")
source_manifest = {
"schema": "planet-earth-boundary-sources/v2",
"sources": sources,
"povPolicy": _read_json(POV_POLICY_PATH),
}
SOURCE_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
with SOURCE_MANIFEST_PATH.open("w", encoding="utf-8") as f:
json.dump(source_manifest, f, ensure_ascii=False, indent=2)
f.write("\n")
build_input_hash = _build_input_hash(source_manifest)
await self.update_phase_progress(
current=1,
total=3,
unit="steps",
message="三类边界源已就绪",
progress=35,
commit=True,
force=True,
)
await self.set_phase("checking_readiness", message="正在检查 Earth 国界构建就绪状态")
missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
readiness_result = {
"returncode": 0,
"result": {
"ready": True,
"failures": [f"external tool not found in PATH: {tool}" for tool in missing_tools],
"sources": sources,
"artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"fallback_builder": "geojson-high-precision" if missing_tools else None,
},
}
await self.set_phase("building_tiles", message="正在检查 Earth 国界瓦片产物")
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
if build_skipped:
build_result = {
"result": {
"status": "unchanged",
"reason": "source manifest and build config hash unchanged",
"buildInputHash": build_input_hash,
}
}
elif shutil.which("tippecanoe") and shutil.which("pmtiles"):
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
coastline = next(source for source in sources if source["kind"] == "coastline")
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
build_result = await self._run_step(
[
"scripts/build_earth_boundary_pmtiles.py",
"--admin0-source",
admin0["path"],
"--coastline-source",
coastline["path"],
"--claims-source",
claim_lines["path"],
"--output",
str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"--manifest",
str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
"--build-input-hash",
build_input_hash,
"--pov-policy",
str(POV_POLICY_PATH.relative_to(REPO_ROOT)),
]
)
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
else:
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
coastline = next(source for source in sources if source["kind"] == "coastline")
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
build_result = await self._run_step(
[
"scripts/build_earth_boundary_china_pov_geojson.py",
"--admin0-source",
admin0["path"],
"--coastline-source",
coastline["path"],
"--claims-source",
claim_lines["path"],
"--output-dir",
str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
"--build-input-hash",
build_input_hash,
]
)
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
pmtiles_size = PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0
await self.update_phase_progress(
current=2,
total=3,
unit="steps",
message=(
"边界源未变化,已跳过瓦片重建"
if build_skipped
else f"已生成 {boundary_manifest.get('tileProvider') or 'boundary'} 国界产物"
),
progress=80,
commit=True,
force=True,
)
await self.set_phase("indexing_artifacts", message="正在登记边界瓦片产物")
tile_counts = boundary_manifest.get("tiles", {}).get("countsByZoom", {})
records = [
{
"id": "source-manifest",
"name": "Earth boundary source manifest",
"description": "Offline source collection manifest for Earth boundary tiles",
"value": len(source_manifest.get("sources", [])),
"unit": "sources",
"metadata": {
"manifest_path": str(SOURCE_MANIFEST_PATH.relative_to(REPO_ROOT)),
"manifest": source_manifest,
"collector_result": {
"status": "loaded_from_collected_data",
"sources": [source["id"] for source in sources],
},
},
},
{
"id": "production-readiness",
"name": "Earth boundary production readiness",
"description": "Checks whether all source artifacts and boundary build tooling are available",
"value": 1 if readiness_result.get("returncode") == 0 else 0,
"unit": "ready",
"metadata": {
"result": readiness_result.get("result"),
"returncode": readiness_result.get("returncode"),
},
},
{
"id": "boundary-manifest",
"name": "Earth boundary tile manifest",
"description": "Versioned static vector tile manifest for Earth country boundaries",
"value": boundary_stats["file_count"],
"unit": "files",
"metadata": {
"manifest_path": str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
"output_dir": str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
"size_bytes": boundary_stats["size_bytes"],
"manifest": boundary_manifest,
"collector_result": build_result.get("result"),
"build_skipped": build_skipped,
"production_target": BUILD_CONFIG["production_target"],
"pmtiles_artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"pmtiles_exists": pmtiles_exists,
},
},
]
if pmtiles_exists:
records.append(
{
"id": "pmtiles-artifact",
"name": "Earth boundary PMTiles artifact",
"description": "Single-file PMTiles/MVT artifact for Earth boundaries",
"value": pmtiles_size,
"unit": "bytes",
"metadata": {
"path": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"exists": pmtiles_exists,
"provider": "pmtiles-mvt",
},
}
)
for zoom, count in sorted(tile_counts.items(), key=lambda item: int(item[0])):
records.append(
{
"id": f"tile-z{zoom}",
"name": f"Earth boundary tiles z{zoom}",
"description": f"Generated Earth boundary tile count for zoom {zoom}",
"value": int(count),
"unit": "tiles",
"metadata": {
"zoom": int(zoom),
"tile_count": int(count),
"output_dir": str((BOUNDARY_OUTPUT_DIR / str(zoom)).relative_to(REPO_ROOT)),
},
}
)
await self.update_phase_progress(
current=3,
total=3,
unit="steps",
message="边界瓦片产物已登记",
progress=100,
commit=True,
force=True,
)
return records

View File

@@ -7,6 +7,7 @@ from typing import Any
from sqlalchemy import select
from app.ai_tasks.prompts import get_effective_prompt
from app.models.system_setting import SystemSetting
from app.schemas.ai import SituationalAnalysisRequest
from app.services.ai_client import AIProviderClient
@@ -15,6 +16,7 @@ from app.services.ai_tools.web_search import WebSearchClient, WebSearchError
CREDENTIAL_GUIDES_CATEGORY = "collector_credential_guides"
CREDENTIAL_GUIDE_PROMPT_KEY = "credential.guide"
@dataclass(frozen=True)
@@ -240,14 +242,12 @@ async def generate_credential_guide(
guide["sources"] = []
return guide
prompt = await get_effective_prompt(db, CREDENTIAL_GUIDE_PROMPT_KEY)
response = await ai_client.analyze(
SituationalAnalysisRequest(
title=f"Generate credential guide for {provider}",
objective=(
default.prompt
+ "\n只能根据 context.search_evidence 中的来源生成教程;"
+ "如果证据不足,明确说明需要以官方页面为准。"
),
objective=f"{default.prompt}\n{prompt.prompt}",
system_prompt=prompt.system_prompt or None,
context={
"provider": provider,
"current_default_guide": default.markdown,

View File

@@ -301,52 +301,6 @@ async def persist_mapped_records(
transport: str | None = None,
) -> int:
"""Persist validated mapped records to the destination for a target schema."""
if target_schema == "earth_boundary_source":
from app.models.collected_data import CollectedData
now = datetime.now(UTC)
written_count = 0
for index, record in enumerate(records):
source_id = record.get("source_id") or record.get("sha256") or str(index)
entity_key = f"{datasource_name}:{source_id}"
previous_result = await db.execute(
select(CollectedData)
.where(CollectedData.source == datasource_name)
.where(CollectedData.entity_key == entity_key)
.where(CollectedData.is_current.is_(True))
.order_by(CollectedData.id.desc())
.limit(1)
)
previous = previous_result.scalar_one_or_none()
if previous is not None:
previous.is_current = False
db.add(
CollectedData(
source=datasource_name,
source_id=str(source_id),
entity_key=entity_key,
data_type=target_schema,
name=record.get("name") or str(source_id),
description=f"Earth boundary source artifact: {record.get('source_kind')}",
extra_data={
**record,
"datasource_config_id": datasource_config_id,
"mapping_version": mapping_version,
"delivery_mode": delivery_mode or "polling",
"transport": transport or "http",
},
collected_at=now,
is_valid=1,
is_current=True,
previous_record_id=previous.id if previous else None,
change_type="updated" if previous else "created",
change_summary={},
)
)
written_count += 1
await db.commit()
return written_count
if target_schema == "vessel_ais":
from app.core.time import to_iso8601_utc
from app.core.websocket.broadcaster import broadcaster

View File

@@ -0,0 +1,671 @@
"""Earth boundary static asset service."""
from __future__ import annotations
import asyncio
import hashlib
import json
import shutil
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from uuid import uuid4
import httpx
REPO_ROOT = Path(__file__).resolve().parents[3]
SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
BUILD_RESULT_PATH = SOURCE_OUTPUT_DIR / "build-result.json"
BUILD_JOB_PATH = SOURCE_OUTPUT_DIR / "build-job.json"
BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
PMTILES_ARTIFACT_PATH = (
REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
)
LEGACY_GEOJSON_PATH = REPO_ROOT / "frontend/public/earth/data/countries-admin0.min.geojson"
POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
LOCAL_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.local.json"
EXAMPLE_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.example.json"
BOUNDARY_SOURCE_KINDS = {
"earth_admin0_boundaries": "admin0-boundaries",
"earth_coastline": "coastline",
"earth_claim_lines": "claim-lines",
}
DEFAULT_PUBLIC_BOUNDARY_SOURCES = {
"earth_admin0_boundaries": {
"displayName": "Natural Earth Admin-0 Countries",
"sourceKind": "admin0-boundaries",
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_countries.geojson",
"method": "GET",
"headers": {},
"auth_type": "none",
"license": "Natural Earth public domain",
},
"earth_coastline": {
"displayName": "Natural Earth Coastline",
"sourceKind": "coastline",
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_coastline.geojson",
"method": "GET",
"headers": {},
"auth_type": "none",
"license": "Natural Earth public domain",
},
"earth_claim_lines": {
"displayName": "Natural Earth Disputed Boundaries",
"sourceKind": "claim-lines",
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_boundary_lines_disputed_areas.geojson",
"method": "GET",
"headers": {},
"auth_type": "none",
"license": "Natural Earth public domain",
},
}
BUILD_CONFIG = {
"builder": "scripts/build_earth_boundary_pmtiles.py",
"format": "pmtiles+mvt",
"production_target": "pmtiles-mvt",
}
class EarthBoundaryBuildError(RuntimeError):
def __init__(self, message: str, *, code: str = "build_failed", details: Any = None) -> None:
super().__init__(message)
self.code = code
self.details = details
_build_job_lock = asyncio.Lock()
_build_task: asyncio.Task | None = None
_build_job_state: dict[str, Any] = {}
def _utc_now_iso() -> str:
return datetime.now(UTC).isoformat()
def _public_job_state() -> dict[str, Any]:
if _build_job_state:
return dict(_build_job_state)
return _read_json(BUILD_JOB_PATH)
def get_boundary_build_status() -> dict[str, Any]:
return {"job": _public_job_state()}
def _set_job_state(**updates: Any) -> dict[str, Any]:
global _build_job_state
current = dict(_build_job_state)
current.update(updates)
current["updated_at"] = _utc_now_iso()
_build_job_state = current
_write_json(BUILD_JOB_PATH, current)
return current
def _append_job_log(message: str) -> None:
logs = list(_build_job_state.get("logs") or [])
logs.append({"time": _utc_now_iso(), "message": message})
_set_job_state(logs=logs[-40:])
def _update_job_progress(progress: float, phase: str, message: str, **extra: Any) -> None:
bounded_progress = max(0, min(100, int(round(progress))))
_set_job_state(
status="running",
progress=bounded_progress,
phase=phase,
message=message,
**extra,
)
def _read_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
with path.open("r", encoding="utf-8") as f:
payload = json.load(f)
return payload if isinstance(payload, dict) else {}
def _write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as f:
json.dump(payload, f, ensure_ascii=False, indent=2)
f.write("\n")
def _sha256_bytes(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def _stable_json_hash(payload: Any) -> str:
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
if suffix in {".json", ".geojson", ".zip", ".pbf"}:
return suffix
if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
return ".geojson"
if "json" in content_type:
return ".json"
return ".dat"
def _json_feature_count(payload: Any) -> int:
if isinstance(payload, dict) and isinstance(payload.get("features"), list):
return len(payload["features"])
if isinstance(payload, list):
return len(payload)
return 1 if payload else 0
def _directory_stats(path: Path) -> dict[str, int]:
if not path.exists():
return {"file_count": 0, "size_bytes": 0}
files = [item for item in path.rglob("*") if item.is_file()]
return {"file_count": len(files), "size_bytes": sum(item.stat().st_size for item in files)}
def _load_source_feature_collection(source: dict[str, Any]) -> dict[str, Any]:
path = REPO_ROOT / source["path"]
payload = _read_json(path)
features = payload.get("features") if isinstance(payload, dict) else None
return {
"type": "FeatureCollection",
"features": features if isinstance(features, list) else [],
}
def _write_high_precision_geojson_manifest(
sources: list[dict[str, Any]],
build_input_hash: str,
missing_tools: list[str],
) -> dict[str, Any]:
BOUNDARY_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
coastline = next(source for source in sources if source["kind"] == "coastline")
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
admin0_payload = _load_source_feature_collection(admin0)
coastline_payload = _load_source_feature_collection(coastline)
claim_payload = _load_source_feature_collection(claim_lines)
for feature in coastline_payload["features"]:
props = feature.setdefault("properties", {})
if isinstance(props, dict):
props["PLANET_LAYER"] = "coastline"
base_payload = {
"type": "FeatureCollection",
"features": [*admin0_payload["features"], *coastline_payload["features"]],
}
base_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-base.geojson"
hover_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-hover.geojson"
claim_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-claims.geojson"
_write_json(base_path, base_payload)
_write_json(hover_path, admin0_payload)
_write_json(claim_path, claim_payload)
manifest = {
"version": "natural-earth-v1",
"builtAt": _utc_now_iso(),
"tileProvider": "geojson-high-precision",
"format": "geojson-directory",
"buildInputHash": build_input_hash,
"base": base_path.name,
"hoverIndex": hover_path.name,
"claimLine": claim_path.name,
"sourceFeatureCount": {
"admin0": len(admin0_payload["features"]),
"coastline": len(coastline_payload["features"]),
"claimLines": len(claim_payload["features"]),
},
"pmtiles": None,
"missingTools": missing_tools,
}
_write_json(BOUNDARY_MANIFEST_PATH, manifest)
return manifest
def _relative(path: Path) -> str:
return str(path.relative_to(REPO_ROOT))
def load_boundary_config() -> tuple[dict[str, Any], str]:
if LOCAL_CONFIG_PATH.exists():
return _read_json(LOCAL_CONFIG_PATH), "local"
return _read_json(EXAMPLE_CONFIG_PATH), "example"
def save_boundary_config(payload: dict[str, Any]) -> dict[str, Any]:
if not isinstance(payload, dict):
raise EarthBoundaryBuildError("Earth boundary config must be a JSON object", code="invalid_config")
_write_json(LOCAL_CONFIG_PATH, payload)
return get_boundary_status()
def _source_configs(payload: dict[str, Any]) -> dict[str, Any]:
raw_sources = payload.get("collectorConfigs") or payload.get("sources") or {}
return raw_sources if isinstance(raw_sources, dict) else {}
def _is_placeholder_endpoint(endpoint: Any) -> bool:
value = str(endpoint or "").strip()
return not value or "example.com" in value
def _source_configs_with_defaults(payload: dict[str, Any]) -> dict[str, Any]:
raw_sources = _source_configs(payload)
merged: dict[str, Any] = {}
for source_key, default_config in DEFAULT_PUBLIC_BOUNDARY_SOURCES.items():
configured = raw_sources.get(source_key)
if not isinstance(configured, dict) or _is_placeholder_endpoint(configured.get("endpoint")):
merged[source_key] = dict(default_config)
else:
merged[source_key] = {**default_config, **configured}
for source_key, source_config in raw_sources.items():
if source_key not in merged:
merged[source_key] = source_config
return merged
def _build_input_hash(source_manifest: dict[str, Any]) -> str:
return _stable_json_hash(
{
"source_manifest_schema": source_manifest.get("schema"),
"sources": [
{
"id": source.get("id"),
"sha256": source.get("sha256"),
"kind": source.get("kind"),
}
for source in source_manifest.get("sources", [])
],
"pov_policy": source_manifest.get("povPolicy"),
"build_config": BUILD_CONFIG,
}
)
def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
return (
bool(boundary_manifest)
and boundary_manifest.get("buildInputHash") == build_input_hash
and boundary_manifest.get("tileProvider") == "pmtiles-mvt"
and PMTILES_ARTIFACT_PATH.exists()
)
def get_boundary_status() -> dict[str, Any]:
config_payload, config_source = load_boundary_config()
effective_source_configs = _source_configs_with_defaults(config_payload)
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
manifest_exists = BOUNDARY_MANIFEST_PATH.exists()
high_precision_ready = (
manifest_exists
and (
(
boundary_manifest.get("tileProvider") == "pmtiles-mvt"
and pmtiles_exists
)
or boundary_manifest.get("tileProvider") == "geojson-high-precision"
)
)
legacy_exists = LEGACY_GEOJSON_PATH.exists()
provider = (
boundary_manifest.get("tileProvider")
if high_precision_ready
else "legacy-geojson" if legacy_exists else "missing"
)
return {
"provider": provider,
"high_precision_ready": high_precision_ready,
"fallback_available": legacy_exists,
"config_source": config_source,
"config_path": _relative(LOCAL_CONFIG_PATH),
"config_exists": LOCAL_CONFIG_PATH.exists(),
"config": config_payload,
"effective_default_sources": [
source_key
for source_key, source_config in effective_source_configs.items()
if source_key in DEFAULT_PUBLIC_BOUNDARY_SOURCES
and source_config.get("endpoint") == DEFAULT_PUBLIC_BOUNDARY_SOURCES[source_key]["endpoint"]
],
"manifest": {
"path": _relative(BOUNDARY_MANIFEST_PATH),
"exists": manifest_exists,
"tileProvider": boundary_manifest.get("tileProvider"),
"buildInputHash": boundary_manifest.get("buildInputHash"),
"builtAt": boundary_manifest.get("builtAt"),
},
"pmtiles": {
"path": _relative(PMTILES_ARTIFACT_PATH),
"exists": pmtiles_exists,
"size_bytes": PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0,
},
"legacy": {
"path": _relative(LEGACY_GEOJSON_PATH),
"exists": legacy_exists,
"size_bytes": LEGACY_GEOJSON_PATH.stat().st_size if legacy_exists else 0,
},
"source_manifest": {
"path": _relative(SOURCE_MANIFEST_PATH),
"exists": SOURCE_MANIFEST_PATH.exists(),
},
"last_build": _read_json(BUILD_RESULT_PATH),
"current_job": _public_job_state(),
}
async def _download_source(
source_key: str,
source_config: dict[str, Any],
progress_callback: Any = None,
) -> dict[str, Any]:
endpoint = str(source_config.get("endpoint") or "").strip()
if _is_placeholder_endpoint(endpoint):
raise EarthBoundaryBuildError(
f"{source_key} endpoint is not configured",
code="source_not_configured",
details={"source": source_key},
)
method = str(source_config.get("method") or "GET").upper()
if method not in {"GET", "POST"}:
raise EarthBoundaryBuildError(
f"{source_key} method must be GET or POST",
code="invalid_config",
details={"source": source_key, "method": method},
)
if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
payload = Path(endpoint.removeprefix("file://")).expanduser().read_bytes()
content_type = "application/octet-stream"
if progress_callback:
progress_callback(1, len(payload), len(payload))
else:
timeout = float(source_config.get("timeout") or 120)
headers = source_config.get("headers") if isinstance(source_config.get("headers"), dict) else {}
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
async with client.stream(method, endpoint, headers=headers) as response:
response.raise_for_status()
content_type = response.headers.get("content-type", "")
total = int(response.headers.get("content-length") or 0)
chunks = []
downloaded = 0
async for chunk in response.aiter_bytes():
if not chunk:
continue
chunks.append(chunk)
downloaded += len(chunk)
if progress_callback:
progress_callback(
(downloaded / total) if total else None,
downloaded,
total,
)
payload = b"".join(chunks)
extension = _artifact_extension(endpoint, content_type, payload)
parsed: Any = None
if extension in {".json", ".geojson"}:
parsed = json.loads(payload.decode("utf-8"))
feature_count = _json_feature_count(parsed)
if feature_count <= 0:
raise EarthBoundaryBuildError(
f"{source_key} downloaded payload contains no features",
code="empty_source",
details={"source": source_key},
)
sha256 = _sha256_bytes(payload)
source_dir = SOURCE_OUTPUT_DIR / source_key
source_dir.mkdir(parents=True, exist_ok=True)
artifact_path = source_dir / f"{sha256}{extension}"
artifact_path.write_bytes(payload)
return {
"id": source_key,
"kind": source_config.get("sourceKind") or BOUNDARY_SOURCE_KINDS[source_key],
"path": _relative(artifact_path),
"sha256": sha256,
"featureCount": feature_count,
"license": source_config.get("license"),
}
async def _run_step(args: list[str]) -> dict[str, Any]:
process = await asyncio.create_subprocess_exec(
sys.executable,
*args,
cwd=REPO_ROOT,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout_bytes, stderr_bytes = await process.communicate()
stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
payload: dict[str, Any] = {"stdout": stdout, "stderr": stderr, "returncode": process.returncode}
last_line = stdout.splitlines()[-1:] or []
if last_line:
try:
payload["result"] = json.loads(last_line[0])
except json.JSONDecodeError:
payload["result"] = last_line[0]
if process.returncode != 0:
raise EarthBoundaryBuildError(
stderr or stdout or f"command failed: {' '.join(args)}",
code="build_command_failed",
details=payload,
)
return payload
async def build_boundary_assets(progress_callback: Any = None) -> dict[str, Any]:
config_payload, config_source = load_boundary_config()
source_configs = _source_configs_with_defaults(config_payload)
missing = [source for source in BOUNDARY_SOURCE_KINDS if source not in source_configs]
if missing:
raise EarthBoundaryBuildError(
f"Missing Earth boundary source configs: {', '.join(missing)}",
code="missing_sources",
details={"missing": missing},
)
sources = []
source_keys = list(BOUNDARY_SOURCE_KINDS)
for index, source_key in enumerate(source_keys):
source_config = source_configs[source_key]
if not isinstance(source_config, dict):
raise EarthBoundaryBuildError(
f"{source_key} config must be an object",
code="invalid_config",
details={"source": source_key},
)
source_start = 8 + index * 18
source_end = source_start + 18
if progress_callback:
progress_callback(source_start, "download", f"正在下载 {source_key}")
def report_download_progress(ratio: float | None, downloaded: int, total: int) -> None:
if not progress_callback:
return
if ratio is None:
progress_callback(source_start + 8, "download", f"{source_key} 已下载 {downloaded} bytes")
return
progress_callback(
source_start + (source_end - source_start) * ratio,
"download",
f"{source_key} 下载 {int(ratio * 100)}%",
downloaded_bytes=downloaded,
total_bytes=total,
)
sources.append(await _download_source(source_key, source_config, report_download_progress))
source_manifest = {
"schema": "planet-earth-boundary-sources/v2",
"sources": sources,
"povPolicy": _read_json(POV_POLICY_PATH),
}
if progress_callback:
progress_callback(65, "manifest", "正在写入边界源 manifest")
_write_json(SOURCE_MANIFEST_PATH, source_manifest)
build_input_hash = _build_input_hash(source_manifest)
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
if missing_tools and not build_skipped:
if progress_callback:
progress_callback(82, "build", "缺少 PMTiles 工具,正在生成 GeoJSON 高清包")
boundary_manifest = _write_high_precision_geojson_manifest(
sources,
build_input_hash,
missing_tools,
)
result = {
"status": "built_geojson_fallback",
"code": "missing_tools",
"missing_tools": missing_tools,
"sources": sources,
"boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH),
"manifest": boundary_manifest,
}
_write_json(BUILD_RESULT_PATH, result)
if progress_callback:
progress_callback(96, "finalize", "GeoJSON 高清国界包已生成")
return {**get_boundary_status(), "build": result}
if build_skipped:
if progress_callback:
progress_callback(96, "unchanged", "高精国界已是最新")
build_result = {
"status": "unchanged",
"reason": "source manifest and build config hash unchanged",
"buildInputHash": build_input_hash,
}
else:
if progress_callback:
progress_callback(72, "build", "正在构建 PMTiles/MVT")
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
coastline = next(source for source in sources if source["kind"] == "coastline")
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
build_result = await _run_step(
[
"scripts/build_earth_boundary_pmtiles.py",
"--admin0-source",
admin0["path"],
"--coastline-source",
coastline["path"],
"--claims-source",
claim_lines["path"],
"--output",
_relative(PMTILES_ARTIFACT_PATH),
"--manifest",
_relative(BOUNDARY_MANIFEST_PATH),
"--build-input-hash",
build_input_hash,
"--pov-policy",
_relative(POV_POLICY_PATH),
]
)
if progress_callback:
progress_callback(95, "finalize", "正在校验构建产物")
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
result = {
"status": "unchanged" if build_skipped else "built",
"sources": sources,
"source_manifest": _relative(SOURCE_MANIFEST_PATH),
"boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH),
"pmtiles_artifact": _relative(PMTILES_ARTIFACT_PATH),
"pmtiles_exists": PMTILES_ARTIFACT_PATH.exists(),
"boundary_stats": boundary_stats,
"manifest": boundary_manifest,
"build_result": build_result,
}
_write_json(BUILD_RESULT_PATH, result)
return {**get_boundary_status(), "build": result}
async def _run_boundary_build_job(job_id: str) -> None:
def report(progress: float, phase: str, message: str, **extra: Any) -> None:
if _build_job_state.get("id") != job_id:
return
_update_job_progress(progress, phase, message, **extra)
try:
report(3, "prepare", "正在准备高精国界构建")
result = await build_boundary_assets(report)
_set_job_state(
id=job_id,
status="succeeded",
progress=100,
phase="complete",
message="高精国界构建完成",
finished_at=_utc_now_iso(),
result={
"provider": result.get("provider"),
"high_precision_ready": result.get("high_precision_ready"),
"pmtiles": result.get("pmtiles"),
"manifest": result.get("manifest"),
},
)
_append_job_log("高精国界构建完成")
except EarthBoundaryBuildError as exc:
_set_job_state(
id=job_id,
status="failed",
progress=_build_job_state.get("progress", 0),
phase="failed",
message=str(exc),
code=exc.code,
details=exc.details,
finished_at=_utc_now_iso(),
)
_append_job_log(str(exc))
except Exception as exc: # pragma: no cover - defensive guard for background task
_set_job_state(
id=job_id,
status="failed",
progress=_build_job_state.get("progress", 0),
phase="failed",
message=str(exc),
code="build_failed",
finished_at=_utc_now_iso(),
)
_append_job_log(str(exc))
async def start_boundary_build_job() -> dict[str, Any]:
global _build_task
async with _build_job_lock:
if _build_task and not _build_task.done():
return {"accepted": False, "job": _public_job_state()}
job_id = uuid4().hex
_set_job_state(
id=job_id,
status="queued",
progress=0,
phase="queued",
message="高精国界构建已加入队列",
logs=[],
started_at=_utc_now_iso(),
finished_at=None,
code=None,
details=None,
)
_append_job_log("高精国界构建已启动")
_build_task = asyncio.create_task(_run_boundary_build_job(job_id))
return {"accepted": True, "job": _public_job_state()}

View File

@@ -1,7 +1,7 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
import hashlib
@@ -18,6 +18,7 @@ from bs4 import BeautifulSoup
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt
from app.schemas.ai import SituationalAnalysisRequest
from app.services.ai_client import AIProviderClient
from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder
@@ -31,6 +32,8 @@ STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS
MAX_TARGET_INFERENCE_CONCURRENCY = 3
TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0
DEFAULT_NEWS_LOCALE = "zh-CN"
NEWS_ENRICH_PROMPT_KEY = "earth.news.enrich"
@dataclass(frozen=True)
@@ -82,6 +85,11 @@ class ParsedNewsItem:
feed_region: str
homepage_url: str
published_at: datetime | None
content_language: str = "en"
localizations: dict[str, dict[str, str]] = field(default_factory=dict)
enrichment_status: str = "pending"
enrichment_error: str | None = None
enriched_at: datetime | None = None
target_location: NewsTargetLocation | None = None
target_resolution_stage: str = "unresolved"
target_ai_attempted: bool = False
@@ -365,6 +373,65 @@ def _first_json_object(text: str) -> dict[str, Any] | None:
return None
def _normalize_localizations(value: Any) -> dict[str, dict[str, str]]:
if not isinstance(value, dict):
return {}
normalized: dict[str, dict[str, str]] = {}
for locale, payload in value.items():
locale_key = _coerce_str(locale)
if not locale_key or not isinstance(payload, dict):
continue
title = _coerce_str(payload.get("title"))
summary = _coerce_str(payload.get("summary"))
entry: dict[str, str] = {}
if title:
entry["title"] = title
if summary:
entry["summary"] = summary
if entry:
normalized[locale_key] = entry
return normalized
def _get_locale_text(
item: ParsedNewsItem,
key: str,
*,
locale: str = DEFAULT_NEWS_LOCALE,
) -> str:
localized = item.localizations.get(locale)
if isinstance(localized, dict):
value = _coerce_str(localized.get(key))
if value:
return value
return ""
def _has_default_localization(item: ParsedNewsItem) -> bool:
localized = item.localizations.get(DEFAULT_NEWS_LOCALE)
if not isinstance(localized, dict):
return False
return bool(_coerce_str(localized.get("title")) and _coerce_str(localized.get("summary")))
def apply_enrichment_patch_to_item(
item: ParsedNewsItem,
patch: dict[str, Any],
) -> ParsedNewsItem:
item.location_patch = patch
if "content_language" in patch:
item.content_language = _coerce_str(patch.get("content_language")) or item.content_language
if "localizations" in patch:
item.localizations = _normalize_localizations(patch.get("localizations"))
if "enrichment_status" in patch:
item.enrichment_status = _coerce_str(patch.get("enrichment_status")) or item.enrichment_status
if "enrichment_error" in patch:
item.enrichment_error = _coerce_str(patch.get("enrichment_error"))
if "enriched_at" in patch:
item.enriched_at = _parse_datetime(_coerce_str(patch.get("enriched_at")))
return item
async def _geocode_target_location(query: str) -> dict[str, Any] | None:
return await asyncio.to_thread(_news_target_geocode, query)
@@ -513,42 +580,60 @@ async def _infer_news_target_location(
item: ParsedNewsItem,
*,
provider_client: AIProviderClient | None,
prompt: EffectiveAIPrompt | None = None,
) -> NewsTargetLocation | None:
target, _localizations = await _infer_news_enrichment(
item,
provider_client=provider_client,
prompt=prompt,
)
return target
async def _infer_news_enrichment(
item: ParsedNewsItem,
*,
provider_client: AIProviderClient | None,
prompt: EffectiveAIPrompt | None = None,
) -> tuple[NewsTargetLocation | None, dict[str, dict[str, str]]]:
text_hint = await _extract_target_location_from_text(item)
content_error: str | None = None
if text_hint is not None and text_hint.city:
item.target_resolution_stage = text_hint.source
item.target_ai_attempted = False
item.target_ai_status = "skipped_text_hint"
item.target_ai_error = None
item.target_debug_note = f"text hint matched {text_hint.label}"
return text_hint
localizations: dict[str, dict[str, str]] = {}
if provider_client is None:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_attempted = False
item.target_ai_status = "unavailable"
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
if text_hint is None or not text_hint.city:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_attempted = False
item.target_ai_status = "unavailable"
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
item.target_debug_note = (
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
)
item.enrichment_status = "unavailable"
item.enrichment_error = "AI provider is not configured or unavailable for earth-feed."
return text_hint, localizations
if text_hint is None or not text_hint.city:
item.target_ai_attempted = True
item.target_ai_status = "attempted"
item.target_ai_error = None
item.target_debug_note = (
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
)
return text_hint
item.target_ai_attempted = True
item.target_ai_status = "attempted"
item.target_ai_error = None
item.target_debug_note = (
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
)
item.enrichment_status = "attempted"
item.enrichment_error = None
prompt = prompt or await get_effective_prompt(None, NEWS_ENRICH_PROMPT_KEY)
request = SituationalAnalysisRequest(
title="Infer likely event location for Earth news cruise",
objective=(
"Return exactly one strict JSON object for the most likely physical "
"location the news event is about. Prefer the host city when a state "
"visit, summit, meeting, attack, or disaster is clearly centered in a "
"known city. Fall back to the best-supported country only when a city "
"cannot be inferred."
),
title="Enrich Earth news item with event location and zh-CN content",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
context={
"news_item": {
"title": item.title,
@@ -564,17 +649,28 @@ async def _infer_news_target_location(
),
},
"required_json_schema": {
"country": "string|null",
"city": "string|null",
"matched_location_name": "string|null",
"latitude": "number|null",
"longitude": "number|null",
"confidence": "number from 0 to 1",
"reasoning_summary": "short string",
"location": {
"country": "string|null",
"city": "string|null",
"matched_location_name": "string|null",
"latitude": "number|null",
"longitude": "number|null",
"confidence": "number from 0 to 1",
"reasoning_summary": "short string",
},
"localizations": {
"zh-CN": {
"title": "faithful Simplified Chinese title",
"summary": "1-2 sentence faithful Simplified Chinese summary",
}
},
},
},
constraints=[
"Return only strict JSON. Do not wrap it in markdown.",
"For localizations, do not add facts that are absent from the RSS headline, description, source, or date.",
"If the RSS description is thin, write a conservative summary that says only what is supported.",
"Keep zh-CN summary concise, factual, and non-promotional.",
"Prefer the event location, not the newsroom or publisher headquarters.",
"When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.",
"Use null for unknown fields instead of inventing details.",
@@ -584,40 +680,68 @@ async def _infer_news_target_location(
try:
response = await provider_client.analyze(request)
except Exception as exc:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "provider_error"
item.target_ai_error = str(exc)
return text_hint
if text_hint is None or not text_hint.city:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "provider_error"
item.target_ai_error = str(exc)
item.enrichment_status = "provider_error"
item.enrichment_error = str(exc)
return text_hint, localizations
payload = _first_json_object(response.content)
if not isinstance(payload, dict):
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "parse_error"
item.target_ai_error = "AI response did not contain a parseable JSON object."
return text_hint
if text_hint is None or not text_hint.city:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "parse_error"
item.target_ai_error = "AI response did not contain a parseable JSON object."
item.enrichment_status = "parse_error"
item.enrichment_error = "AI response did not contain a parseable JSON object."
return text_hint, localizations
target = await _build_target_location_from_payload(payload)
localizations = _normalize_localizations(payload.get("localizations"))
if not localizations:
content_error = "AI returned no usable localizations."
location_payload = payload.get("location") if isinstance(payload.get("location"), dict) else payload
if text_hint is not None and text_hint.city:
target = text_hint
else:
target = await _build_target_location_from_payload(location_payload)
if target is None:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "no_result"
item.target_ai_error = "AI returned no usable target coordinates or geocodeable location."
return text_hint
if target.confidence is not None and target.confidence < 0.45:
target = text_hint
elif target.confidence is not None and target.confidence < 0.45:
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
item.target_ai_status = "low_confidence"
item.target_ai_error = f"AI target confidence too low: {target.confidence:.2f}"
return text_hint
item.target_resolution_stage = target.source
item.target_ai_status = "success"
item.target_ai_error = None
item.target_debug_note = f"ai inferred {target.label}"
return target
target = text_hint
else:
item.target_resolution_stage = target.source
item.target_ai_status = "success"
item.target_ai_error = None
item.target_debug_note = f"ai inferred {target.label}"
item.localizations = localizations
if localizations and item.target_ai_status in {"success", "skipped_text_hint"}:
item.enrichment_status = "success"
item.enrichment_error = None
elif localizations:
item.enrichment_status = "content_only"
item.enrichment_error = item.target_ai_error
else:
item.enrichment_status = "location_only" if target is not None else "no_result"
item.enrichment_error = content_error or item.target_ai_error
item.enriched_at = datetime.now(UTC) if localizations else None
return target, localizations
async def _enrich_items_with_target_locations(
items: list[ParsedNewsItem],
*,
provider_client: AIProviderClient | None,
prompt: EffectiveAIPrompt | None = None,
) -> list[ParsedNewsItem]:
if not items:
return items
@@ -626,7 +750,11 @@ async def _enrich_items_with_target_locations(
async def enrich(item: ParsedNewsItem) -> ParsedNewsItem:
async with semaphore:
target = await _infer_news_target_location(item, provider_client=provider_client)
target = await _infer_news_target_location(
item,
provider_client=provider_client,
prompt=prompt,
)
item.target_location = target
return item
@@ -783,6 +911,20 @@ def _serialize_target(target: NewsTargetLocation | None) -> dict[str, Any] | Non
}
def _serialize_enriched_at(value: datetime | None) -> str | None:
return value.isoformat().replace("+00:00", "Z") if value else None
def _content_patch(item: ParsedNewsItem) -> dict[str, Any]:
return {
"content_language": item.content_language,
"localizations": item.localizations,
"enrichment_status": item.enrichment_status,
"enrichment_error": item.enrichment_error,
"enriched_at": _serialize_enriched_at(item.enriched_at),
}
def build_anchor_location_patch(
item: ParsedNewsItem,
*,
@@ -798,6 +940,9 @@ def build_anchor_location_patch(
resolution_stage = item.target_resolution_stage
ai_status = item.target_ai_status
debug_note = item.target_debug_note
content_patch = _content_patch(item)
if queued and content_patch["enrichment_status"] == "pending":
content_patch["enrichment_status"] = "queued"
return {
"latitude": anchor.latitude,
"longitude": anchor.longitude,
@@ -814,6 +959,7 @@ def build_anchor_location_patch(
"target": None,
"anchor": _serialize_anchor(anchor),
},
**content_patch,
}
@@ -836,6 +982,7 @@ def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation
"target": _serialize_target(target),
"anchor": _serialize_anchor(anchor),
},
**_content_patch(item),
}
@@ -845,6 +992,11 @@ def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
"id": item.id,
"title": item.title,
"summary": item.summary,
"content_language": item.content_language,
"localizations": item.localizations,
"enrichment_status": item.enrichment_status,
"enrichment_error": item.enrichment_error,
"enriched_at": _serialize_enriched_at(item.enriched_at),
"url": item.url,
"source": item.source,
"feed_name": item.feed_name,
@@ -859,6 +1011,11 @@ def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem
id=str(payload.get("id") or ""),
title=str(payload.get("title") or ""),
summary=str(payload.get("summary") or ""),
content_language=str(payload.get("content_language") or "en"),
localizations=_normalize_localizations(payload.get("localizations")),
enrichment_status=str(payload.get("enrichment_status") or "pending"),
enrichment_error=_coerce_str(payload.get("enrichment_error")),
enriched_at=_parse_datetime(_coerce_str(payload.get("enriched_at"))),
url=str(payload.get("url") or ""),
source=str(payload.get("source") or ""),
feed_name=str(payload.get("feed_name") or ""),
@@ -875,10 +1032,15 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An
"id": item.id,
"title": item.title,
"summary": item.summary,
"content_language": item.content_language,
"localizations": item.localizations,
"display_title": _get_locale_text(item, "title"),
"display_summary": _get_locale_text(item, "summary"),
"url": item.url,
"source": item.source,
"feed_name": item.feed_name,
"region": item.feed_region,
"display_region": get_region_anchor(item.feed_region).label,
"homepage_url": item.homepage_url,
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
"latitude": location_patch["latitude"],
@@ -887,6 +1049,9 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An
"location_source": location_patch["location_source"],
"verified": location_patch["verified"],
"location_meta": location_patch["location_meta"],
"enrichment_status": item.enrichment_status,
"enrichment_error": item.enrichment_error,
"enriched_at": _serialize_enriched_at(item.enriched_at),
"is_focus_match": item.feed_region == active_region,
}
@@ -911,6 +1076,7 @@ def _build_payload(
"lon": lon,
"region": active_region,
"label": profile.label,
"display_region": get_region_anchor(active_region).label,
"accent": profile.accent,
},
"sources": _serialize_sources(sources),
@@ -966,13 +1132,27 @@ async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> li
get_cached_target_location_patch,
)
async def enqueue_item(item: ParsedNewsItem, *, force: bool = False) -> bool:
return await enqueue_target_location_job(build_target_location_job_payload(item), force=force)
async def apply_location(item: ParsedNewsItem) -> ParsedNewsItem:
cached_patch = await get_cached_target_location_patch(item.id)
if cached_patch:
item.location_patch = cached_patch
apply_enrichment_patch_to_item(item, cached_patch)
if not _has_default_localization(item):
queued = await enqueue_item(item, force=True)
if queued and item.enrichment_status in {
"pending",
"unavailable",
"provider_error",
"parse_error",
"no_result",
"location_only",
}:
item.enrichment_status = "queued"
return item
queued = await enqueue_target_location_job(build_target_location_job_payload(item))
queued = await enqueue_item(item)
item.location_patch = build_anchor_location_patch(
item,
queued=queued,
@@ -991,9 +1171,16 @@ async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None:
await asyncio.gather(
*(
enqueue_target_location_job(build_target_location_job_payload(item))
enqueue_target_location_job(
build_target_location_job_payload(item),
force=not _has_default_localization(item),
)
for item in items
if item.location_patch is None or item.location_patch.get("verified") is False
if (
item.location_patch is None
or item.location_patch.get("verified") is False
or not _has_default_localization(item)
)
)
)

View File

@@ -32,7 +32,7 @@ class NewsTargetLocationMessage:
class NewsTargetLocationQueue(Protocol):
async def enqueue(self, *, item_id: str, payload: dict[str, Any]) -> bool:
async def enqueue(self, *, item_id: str, payload: dict[str, Any], force: bool = False) -> bool:
...
async def consume_batch(
@@ -91,9 +91,11 @@ class RedisStreamsNewsTargetLocationQueue:
raise
self._group_ready = True
async def enqueue(self, *, item_id: str, payload: dict[str, Any]) -> bool:
async def enqueue(self, *, item_id: str, payload: dict[str, Any], force: bool = False) -> bool:
await self._ensure_group()
if await self.client.exists(_result_key(item_id)):
if force:
await self.client.delete(_result_key(item_id), _queued_key(item_id))
elif await self.client.exists(_result_key(item_id)):
return False
queued = await self.client.set(
_queued_key(item_id),
@@ -187,13 +189,13 @@ def get_news_target_location_queue() -> NewsTargetLocationQueue:
return RedisStreamsNewsTargetLocationQueue()
async def enqueue_target_location_job(payload: dict[str, Any]) -> bool:
async def enqueue_target_location_job(payload: dict[str, Any], *, force: bool = False) -> bool:
item_id = str(payload.get("id") or "")
if not item_id:
return False
try:
queue = get_news_target_location_queue()
return await queue.enqueue(item_id=item_id, payload=payload)
return await queue.enqueue(item_id=item_id, payload=payload, force=force)
except Exception as exc:
logger.warning_event(
"Failed to enqueue Earth news target location job",

View File

@@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.models.earth_news import EarthNewsItem
from app.services.earth_news import (
ParsedNewsItem,
apply_enrichment_patch_to_item,
build_anchor_location_patch,
)
@@ -33,7 +34,7 @@ def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]:
def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
return ParsedNewsItem(
item = ParsedNewsItem(
id=record.id,
title=record.title,
summary=record.summary or "",
@@ -43,8 +44,13 @@ def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
feed_region=record.region or "global",
homepage_url=record.homepage_url or "",
published_at=_coerce_datetime(record.published_at),
location_patch=_location_patch_from_record(record),
content_language=record.content_language or "en",
localizations=dict(record.localizations or {}),
enrichment_status=record.enrichment_status or "pending",
enrichment_error=record.enrichment_error,
enriched_at=_coerce_datetime(record.enriched_at),
)
return apply_enrichment_patch_to_item(item, _location_patch_from_record(record))
def _query_sort_key(active_region: str):
@@ -108,6 +114,8 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
id=item.id,
title=item.title,
summary=item.summary,
content_language=item.content_language,
localizations=dict(item.localizations or {}),
url=item.url,
source=item.source,
feed_name=item.feed_name,
@@ -122,6 +130,9 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
location_meta=patch["location_meta"],
first_seen_at=now,
last_seen_at=now,
enrichment_status=item.enrichment_status,
enrichment_error=item.enrichment_error,
enriched_at=item.enriched_at,
)
db.add(record)
changed += 1
@@ -136,6 +147,12 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
record.homepage_url = item.homepage_url
record.published_at = item.published_at
record.last_seen_at = now
if item.localizations:
record.content_language = item.content_language
record.localizations = dict(item.localizations or {})
record.enrichment_status = item.enrichment_status
record.enrichment_error = item.enrichment_error
record.enriched_at = item.enriched_at
changed += 1
await db.flush()
return changed
@@ -161,6 +178,45 @@ async def update_earth_news_item_location(
return True
async def update_earth_news_item_enrichment(
db: AsyncSession,
*,
item_id: str,
patch: dict[str, Any],
) -> bool:
record = await db.get(EarthNewsItem, item_id)
if record is None:
return False
if "latitude" in patch:
record.latitude = float(patch["latitude"])
record.longitude = float(patch["longitude"])
record.location_label = str(patch["location_label"])
record.location_source = str(patch["location_source"])
record.verified = bool(patch["verified"])
record.location_meta = dict(patch.get("location_meta") or {})
record.resolved_at = datetime.now(UTC) if record.verified else None
if "content_language" in patch:
record.content_language = str(patch.get("content_language") or "en")
if "localizations" in patch:
record.localizations = dict(patch.get("localizations") or {})
if "enrichment_status" in patch:
record.enrichment_status = str(patch.get("enrichment_status") or "pending")
if "enrichment_error" in patch:
record.enrichment_error = patch.get("enrichment_error")
if patch.get("enriched_at"):
try:
parsed_enriched_at = datetime.fromisoformat(
str(patch["enriched_at"]).replace("Z", "+00:00")
)
except ValueError:
parsed_enriched_at = datetime.now(UTC)
record.enriched_at = _coerce_datetime(parsed_enriched_at)
elif patch.get("localizations"):
record.enriched_at = datetime.now(UTC)
await db.flush()
return True
async def list_unverified_earth_news_items(
db: AsyncSession,
*,

View File

@@ -9,8 +9,10 @@ from app.core.logging import get_logger
from app.core.websocket.broadcaster import broadcaster
from app.db.session import async_session_factory
from app.services.ai_client import AIProviderClient
from app.ai_tasks.prompts import get_effective_prompt
from app.services.earth_news import (
_infer_news_target_location,
NEWS_ENRICH_PROMPT_KEY,
_infer_news_enrichment,
build_target_location_patch,
parsed_news_item_from_job_payload,
)
@@ -19,7 +21,7 @@ from app.services.earth_news_queue import (
get_news_target_location_queue,
save_target_location_patch,
)
from app.services.earth_news_store import update_earth_news_item_location
from app.services.earth_news_store import update_earth_news_item_enrichment as update_earth_news_item_location
logger = get_logger(__name__, service="earth_news")
@@ -59,8 +61,15 @@ async def process_target_location_message(
provider_client: AIProviderClient | None,
) -> dict[str, Any]:
item = parsed_news_item_from_job_payload(message.payload)
target = await _infer_news_target_location(item, provider_client=provider_client)
async with async_session_factory() as session:
prompt = await get_effective_prompt(session, NEWS_ENRICH_PROMPT_KEY)
target, localizations = await _infer_news_enrichment(
item,
provider_client=provider_client,
prompt=prompt,
)
item.target_location = target
item.localizations = localizations or item.localizations
patch = build_target_location_patch(item, target)
await save_target_location_patch(item.id, patch)
async with async_session_factory() as session:

View File

@@ -7,8 +7,11 @@ import re
from dataclasses import dataclass
from typing import Any, Iterable
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.countries import COUNTRY_ENTRIES, normalize_country
from app.schemas.ai import SituationalAnalysisRequest
from app.ai_tasks.prompts import get_effective_prompt
from app.services.ai_client import AIProviderClient
from app.services.ai_tools.evidence_store import normalize_search_evidence
from app.services.ai_tools.web_search import WebSearchClient, WebSearchError
@@ -23,6 +26,8 @@ from app.services.location.text import (
VALID_LLM_PRECISIONS = {"precise", "site", "city"}
DEFAULT_MIN_CONFIDENCE = 0.55
LOCATION_NORMALIZE_PROMPT_KEY = "location.factcheck.normalize"
LOCATION_RESOLVE_PROMPT_KEY = "location.factcheck.resolve"
MODEL_CONFIDENCE_WEIGHT = 0.25
_geocode_llm_city = build_default_nominatim_geocoder()
_LLM_LOCATION_NAME_KEYS = (
@@ -876,6 +881,7 @@ async def _repair_location_payload_from_text(
raw_text: str,
query: LocationQuery,
entity_type: str,
db: AsyncSession | None = None,
) -> dict[str, Any] | None:
"""Second-pass structure repair for models that answer in prose.
@@ -884,12 +890,11 @@ async def _repair_location_payload_from_text(
"""
if not coerce_str(raw_text):
return None
prompt = await get_effective_prompt(db, LOCATION_NORMALIZE_PROMPT_KEY)
request = SituationalAnalysisRequest(
title=f"Normalize location factcheck for {entity_type}",
objective=(
"Convert the supplied location factcheck text into exactly one strict "
"JSON object. Extract only facts present in the text or original query."
),
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
context={
"entity_type": entity_type,
"location_query": _query_context(query),
@@ -929,6 +934,7 @@ async def collect_llm_location_fallback_candidate(
provider_client: AIProviderClient,
query: LocationQuery,
entity_type: str,
db: AsyncSession | None = None,
attempted_queries: Iterable[str] = (),
search_evidence: list[dict[str, Any]] | None = None,
min_confidence: float = DEFAULT_MIN_CONFIDENCE,
@@ -946,13 +952,11 @@ async def collect_llm_location_fallback_candidate(
attempted_queries=[attempt],
failure_reason="LLM location factcheck skipped: no WebSearch evidence.",
)
prompt = await get_effective_prompt(db, LOCATION_RESOLVE_PROMPT_KEY)
request = SituationalAnalysisRequest(
title=f"Location factcheck fallback for {entity_type}",
objective=(
"Return exactly one JSON object for the most likely physical location. "
"Use only fact-checkable public knowledge; return null fields rather "
"than guessing when evidence is weak."
),
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
context={
"entity_type": entity_type,
"location_query": _query_context(query),
@@ -1001,6 +1005,7 @@ async def collect_llm_location_fallback_candidate(
raw_text=response.content,
query=query,
entity_type=entity_type,
db=db,
)
if payload is None:
payload = _payload_from_free_text(response.content, query=query)

View File

@@ -10,8 +10,11 @@ from app.models.alert import Alert, AlertSeverity, AlertStatus
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.schemas.ai import SituationalAnalysisRequest
from app.ai_tasks.prompts import get_effective_prompt
from app.services.bgp_ai_brief_store import get_latest_bgp_brief_record
SITUATIONAL_ALERT_BRIEF_PROMPT_KEY = "alerts.situational.brief"
def _format_pairs(pairs: list[tuple[str, int]], empty_text: str = "") -> str:
if not pairs:
@@ -158,10 +161,12 @@ async def build_situational_alert_brief_request(
"latest_bgp_brief_id": latest_bgp_brief.id if latest_bgp_brief else None,
"latest_bgp_brief_generated_at": latest_bgp_brief.generated_at if latest_bgp_brief else None,
}
prompt = await get_effective_prompt(db, SITUATIONAL_ALERT_BRIEF_PROMPT_KEY)
request = SituationalAnalysisRequest(
title="态势告警 AI 简报",
objective="综合系统告警、BGP incidents、BGP anomalies 与近期 BGP AI 简报,生成一份面向值班人员的态势告警简报,指出当前最需要关注的风险域、跨模块联动迹象和优先动作。",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
observations=facts,
constraints=[
"明确区分事实、推断与建议。",

View File

@@ -0,0 +1,177 @@
import json
import pytest
from app.core.datasource_defaults import DEFAULT_DATASOURCES
from app.services.collectors.registry import collector_registry
from app.services import earth_boundaries
def write_geojson(path, name="Test"):
path.write_text(
json.dumps(
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {"name": name},
"geometry": {
"type": "Polygon",
"coordinates": [[[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]],
},
}
],
}
),
encoding="utf-8",
)
def patch_paths(monkeypatch, tmp_path):
repo = tmp_path
source_dir = repo / "data/earth-boundary-sources"
boundary_dir = repo / "frontend/public/earth/data/boundaries/v1"
pmtiles = repo / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
legacy = repo / "frontend/public/earth/data/countries-admin0.min.geojson"
config = repo / "config/earth-boundary-sources.local.json"
example = repo / "config/earth-boundary-sources.example.json"
policy = repo / "config/earth-boundary-pov-policy.china-v1.json"
for path in (source_dir, boundary_dir, pmtiles.parent, legacy.parent, config.parent):
path.mkdir(parents=True, exist_ok=True)
policy.write_text('{"productionTileFormat":"pmtiles+mvt"}\n', encoding="utf-8")
example.write_text('{"collectorConfigs":{}}\n', encoding="utf-8")
monkeypatch.setattr(earth_boundaries, "REPO_ROOT", repo)
monkeypatch.setattr(earth_boundaries, "SOURCE_OUTPUT_DIR", source_dir)
monkeypatch.setattr(earth_boundaries, "SOURCE_MANIFEST_PATH", source_dir / "manifest.json")
monkeypatch.setattr(earth_boundaries, "BUILD_RESULT_PATH", source_dir / "build-result.json")
monkeypatch.setattr(earth_boundaries, "BUILD_JOB_PATH", source_dir / "build-job.json")
monkeypatch.setattr(earth_boundaries, "BOUNDARY_OUTPUT_DIR", boundary_dir)
monkeypatch.setattr(earth_boundaries, "BOUNDARY_MANIFEST_PATH", boundary_dir / "manifest.json")
monkeypatch.setattr(earth_boundaries, "PMTILES_ARTIFACT_PATH", pmtiles)
monkeypatch.setattr(earth_boundaries, "LEGACY_GEOJSON_PATH", legacy)
monkeypatch.setattr(earth_boundaries, "LOCAL_CONFIG_PATH", config)
monkeypatch.setattr(earth_boundaries, "EXAMPLE_CONFIG_PATH", example)
monkeypatch.setattr(earth_boundaries, "POV_POLICY_PATH", policy)
return {
"repo": repo,
"config": config,
"legacy": legacy,
"pmtiles": pmtiles,
"manifest": boundary_dir / "manifest.json",
}
def test_boundary_status_uses_legacy_provider_when_pmtiles_missing(monkeypatch, tmp_path):
paths = patch_paths(monkeypatch, tmp_path)
write_geojson(paths["legacy"])
status = earth_boundaries.get_boundary_status()
assert status["provider"] == "legacy-geojson"
assert status["fallback_available"] is True
assert status["high_precision_ready"] is False
def test_boundary_status_prefers_high_precision_when_manifest_and_pmtiles_exist(monkeypatch, tmp_path):
paths = patch_paths(monkeypatch, tmp_path)
write_geojson(paths["legacy"])
paths["pmtiles"].write_bytes(b"pmtiles")
paths["manifest"].write_text('{"tileProvider":"pmtiles-mvt"}\n', encoding="utf-8")
status = earth_boundaries.get_boundary_status()
assert status["provider"] == "pmtiles-mvt"
assert status["high_precision_ready"] is True
def test_save_boundary_config_writes_local_config(monkeypatch, tmp_path):
paths = patch_paths(monkeypatch, tmp_path)
payload = {"collectorConfigs": {"earth_admin0_boundaries": {"endpoint": "file:///tmp/a.geojson"}}}
status = earth_boundaries.save_boundary_config(payload)
assert paths["config"].exists()
assert status["config_source"] == "local"
assert status["config"] == payload
@pytest.mark.asyncio
async def test_build_reports_missing_tools_after_source_artifacts(monkeypatch, tmp_path):
paths = patch_paths(monkeypatch, tmp_path)
source_files = {}
for source in earth_boundaries.BOUNDARY_SOURCE_KINDS:
source_path = paths["repo"] / f"{source}.geojson"
write_geojson(source_path, name=source)
source_files[source] = source_path
paths["config"].write_text(
json.dumps(
{
"collectorConfigs": {
source: {
"sourceKind": kind,
"endpoint": str(source_files[source]),
"method": "GET",
}
for source, kind in earth_boundaries.BOUNDARY_SOURCE_KINDS.items()
}
}
),
encoding="utf-8",
)
monkeypatch.setattr(earth_boundaries.shutil, "which", lambda _tool: None)
progress_events = []
status = await earth_boundaries.build_boundary_assets(
lambda progress, phase, message, **_extra: progress_events.append((progress, phase, message))
)
assert status["provider"] == "geojson-high-precision"
assert status["high_precision_ready"] is True
assert (paths["repo"] / "data/earth-boundary-sources/manifest.json").exists()
assert paths["manifest"].exists()
assert any(phase == "download" for _progress, phase, _message in progress_events)
@pytest.mark.asyncio
async def test_start_boundary_build_job_records_geojson_fallback_success(monkeypatch, tmp_path):
paths = patch_paths(monkeypatch, tmp_path)
monkeypatch.setattr(earth_boundaries, "_build_task", None)
monkeypatch.setattr(earth_boundaries, "_build_job_state", {})
source_files = {}
for source in earth_boundaries.BOUNDARY_SOURCE_KINDS:
source_path = paths["repo"] / f"{source}.geojson"
write_geojson(source_path, name=source)
source_files[source] = source_path
paths["config"].write_text(
json.dumps(
{
"collectorConfigs": {
source: {
"sourceKind": kind,
"endpoint": str(source_files[source]),
"method": "GET",
}
for source, kind in earth_boundaries.BOUNDARY_SOURCE_KINDS.items()
}
}
),
encoding="utf-8",
)
monkeypatch.setattr(earth_boundaries.shutil, "which", lambda _tool: None)
response = await earth_boundaries.start_boundary_build_job()
await earth_boundaries._build_task
status = earth_boundaries.get_boundary_build_status()
assert response["accepted"] is True
assert status["job"]["status"] == "succeeded"
assert status["job"]["result"]["provider"] == "geojson-high-precision"
def test_earth_boundary_collectors_are_not_registered_as_datasources():
removed = set(earth_boundaries.BOUNDARY_SOURCE_KINDS) | {"earth_boundary_tiles"}
assert removed.isdisjoint(DEFAULT_DATASOURCES)
for source in removed:
assert collector_registry.get(source) is None

View File

@@ -158,6 +158,56 @@ async def test_enrich_items_with_target_locations_uses_ai_and_geocode(monkeypatc
assert enriched[0].target_ai_error is None
@pytest.mark.asyncio
async def test_enrich_items_with_target_locations_adds_localizations(monkeypatch):
item = ParsedNewsItem(
id="global-scan:localized",
title="Global leaders meet to discuss energy security",
summary="Officials said the talks focused on supply chains and grid resilience.",
url="https://example.com/energy-security",
source="Example Source",
feed_name="Global Monitor / World",
feed_region="global",
homepage_url="https://example.com",
published_at=datetime(2026, 5, 14, 7, 13, 24, tzinfo=UTC),
)
async def fake_geocode(_query: str):
return {
"lat": "50.1109",
"lon": "8.6821",
"display_name": "Frankfurt am Main, Germany",
}
class FakeProviderClient:
async def analyze(self, _request):
class Response:
content = (
'{"location":{"country":"Germany","city":"Frankfurt",'
'"matched_location_name":"Frankfurt, Germany",'
'"latitude":null,"longitude":null,"confidence":0.77},'
'"localizations":{"zh-CN":{"title":"全球领导人讨论能源安全",'
'"summary":"官员表示,会谈聚焦供应链和电网韧性。"}}}'
)
return Response()
monkeypatch.setattr("app.services.earth_news._geocode_target_location", fake_geocode)
enriched = await _enrich_items_with_target_locations(
[item],
provider_client=FakeProviderClient(),
)
payload = _serialize_item(enriched[0], active_region="global")
assert payload["title"] == "Global leaders meet to discuss energy security"
assert payload["summary"] == "Officials said the talks focused on supply chains and grid resilience."
assert payload["localizations"]["zh-CN"]["title"] == "全球领导人讨论能源安全"
assert payload["display_title"] == "全球领导人讨论能源安全"
assert payload["display_summary"] == "官员表示,会谈聚焦供应链和电网韧性。"
assert payload["enrichment_status"] == "success"
@pytest.mark.asyncio
async def test_extract_target_location_from_text_uses_country_hint(monkeypatch):
item = ParsedNewsItem(
@@ -241,7 +291,7 @@ async def test_earth_news_payload_returns_anchor_items_and_enqueues_location_job
enqueued_payloads = []
async def fake_enqueue_target_location_job(payload):
async def fake_enqueue_target_location_job(payload, **_kwargs):
enqueued_payloads.append(payload)
return True
@@ -260,6 +310,8 @@ async def test_earth_news_payload_returns_anchor_items_and_enqueues_location_job
assert len(payload["items"]) == 1
assert payload["items"][0]["id"] == "test-feed:timeout"
assert payload["items"][0]["display_title"] == ""
assert payload["items"][0]["display_summary"] == ""
assert payload["items"][0]["latitude"] == 20.0
assert payload["items"][0]["longitude"] == 0.0
assert payload["items"][0]["location_source"] == "region_anchor"
@@ -359,7 +411,7 @@ async def test_earth_news_payload_initializes_empty_database_from_rss(monkeypatc
async def fake_list_earth_news_items(_db, *, active_region, limit):
return [item]
async def fake_enqueue_target_location_job(payload):
async def fake_enqueue_target_location_job(payload, **_kwargs):
enqueued.append(payload)
return True
@@ -415,7 +467,7 @@ async def test_earth_news_payload_supplements_stale_database_items(monkeypatch):
async def fake_list_earth_news_items(_db, *, active_region, limit):
return [old_item]
async def fake_enqueue_target_location_job(_payload):
async def fake_enqueue_target_location_job(_payload, **_kwargs):
return True
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
@@ -473,8 +525,11 @@ async def test_earth_news_payload_merges_cached_location_patch(monkeypatch):
async def fake_get_cached_target_location_patch(_item_id):
return cached_patch
async def fake_enqueue_target_location_job(_payload):
raise AssertionError("cached items should not be enqueued")
enqueued = []
async def fake_enqueue_target_location_job(payload, **_kwargs):
enqueued.append(payload)
return True
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
monkeypatch.setattr("app.services.earth_news._fetch_source", fake_fetch_source)
@@ -493,6 +548,70 @@ async def test_earth_news_payload_merges_cached_location_patch(monkeypatch):
assert payload["items"][0]["longitude"] == 116.3912972
assert payload["items"][0]["verified"] is True
assert payload["items"][0]["location_source"] == "headline_location_hint"
assert enqueued[0]["id"] == "test-feed:cached"
@pytest.mark.asyncio
async def test_earth_news_payload_requeues_cached_failed_localization(monkeypatch):
source = NewsFeedSource(
id="test-feed",
name="Test Feed",
region="global",
homepage_url="https://example.com",
feed_url="https://example.com/rss.xml",
)
item = ParsedNewsItem(
id="test-feed:failed-localization",
title="Failed localization story",
summary="English source summary.",
url="https://example.com/failed-localization",
source="Test Feed",
feed_name="Test Feed",
feed_region="global",
homepage_url="https://example.com",
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
)
cached_patch = {
"latitude": 20.0,
"longitude": 0.0,
"location_label": "全球",
"location_source": "region_anchor",
"verified": False,
"location_meta": {"target": None, "anchor": {"region": "global"}},
"content_language": "en",
"localizations": {},
"enrichment_status": "parse_error",
"enrichment_error": "AI response did not contain a parseable JSON object.",
"enriched_at": None,
}
enqueued = []
async def fake_fetch_source(_client, feed_source):
return feed_source, [item], None
async def fake_get_cached_target_location_patch(_item_id):
return cached_patch
async def fake_enqueue_target_location_job(payload, **_kwargs):
enqueued.append(payload)
return True
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
monkeypatch.setattr("app.services.earth_news._fetch_source", fake_fetch_source)
monkeypatch.setattr(
"app.services.earth_news_queue.get_cached_target_location_patch",
fake_get_cached_target_location_patch,
)
monkeypatch.setattr(
"app.services.earth_news_queue.enqueue_target_location_job",
fake_enqueue_target_location_job,
)
payload = await get_earth_news_payload(provider_client=None)
assert enqueued[0]["id"] == "test-feed:failed-localization"
assert payload["items"][0]["display_title"] == ""
assert payload["items"][0]["enrichment_status"] == "queued"
@pytest.mark.asyncio
@@ -598,6 +717,11 @@ async def test_media_news_archive_collector_maps_news_items(monkeypatch):
location_source="headline_location_hint",
verified=True,
location_meta={"target": {"country": "中国", "city": "Beijing"}},
content_language="en",
localizations={"zh-CN": {"title": "归档新闻", "summary": "归档概要"}},
enrichment_status="success",
enrichment_error=None,
enriched_at=datetime(2026, 5, 15, 3, 6, tzinfo=UTC),
first_seen_at=datetime(2026, 5, 15, 2, 0, tzinfo=UTC),
last_seen_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
resolved_at=datetime(2026, 5, 15, 3, 5, tzinfo=UTC),
@@ -619,3 +743,5 @@ async def test_media_news_archive_collector_maps_news_items(monkeypatch):
assert items[0]["city"] == "Beijing"
assert items[0]["latitude"] == 39.9057136
assert items[0]["metadata"]["verified"] is True
assert "localizations" not in items[0]["metadata"]
assert "enrichment_status" not in items[0]["metadata"]

View File

@@ -0,0 +1,92 @@
from types import SimpleNamespace
import pytest
from app.ai_tasks.prompts import (
get_effective_prompt,
list_effective_prompts,
reset_prompt_override,
save_prompt_override,
)
class _ScalarResult:
def __init__(self, value):
self._value = value
def scalar_one_or_none(self):
return self._value
class _PromptSettingsDB:
def __init__(self, payload=None):
self.record = SimpleNamespace(category="ai_prompts", payload=payload) if payload is not None else None
self.added = None
self.commits = 0
async def execute(self, _statement):
return _ScalarResult(self.record)
def add(self, record):
self.record = record
self.added = record
async def commit(self):
self.commits += 1
@pytest.mark.asyncio
async def test_prompt_defaults_are_loaded_without_override():
db = _PromptSettingsDB()
prompt = await get_effective_prompt(db, "earth.news.enrich")
assert prompt.key == "earth.news.enrich"
assert prompt.is_custom is False
assert "strict JSON" in prompt.prompt
@pytest.mark.asyncio
async def test_prompt_override_save_and_reset():
db = _PromptSettingsDB()
saved = await save_prompt_override(
db,
"alerts.brief",
system_prompt="system custom",
prompt="prompt custom",
)
assert saved.is_custom is True
assert saved.system_prompt == "system custom"
assert saved.prompt == "prompt custom"
assert db.commits == 1
effective = await get_effective_prompt(db, "alerts.brief")
assert effective.prompt == "prompt custom"
reset = await reset_prompt_override(db, "alerts.brief")
assert reset.is_custom is False
assert reset.prompt != "prompt custom"
@pytest.mark.asyncio
async def test_prompt_list_marks_custom_items():
db = _PromptSettingsDB(
{
"overrides": {
"bgp.brief": {
"system_prompt": "",
"prompt": "custom bgp prompt",
"updated_at": "2026-05-16T00:00:00Z",
}
}
}
)
prompts = await list_effective_prompts(db)
by_key = {prompt.key: prompt for prompt in prompts}
assert by_key["bgp.brief"].is_custom is True
assert by_key["bgp.brief"].prompt == "custom bgp prompt"
assert by_key["earth.news.enrich"].is_custom is False