release: bump version to 0.59.0
This commit is contained in:
2
backend/app/ai_tasks/__init__.py
Normal file
2
backend/app/ai_tasks/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""AI task prompt registry and runtime helpers."""
|
||||
|
||||
74
backend/app/ai_tasks/default_prompts.json
Normal file
74
backend/app/ai_tasks/default_prompts.json
Normal file
@@ -0,0 +1,74 @@
|
||||
[
|
||||
{
|
||||
"key": "earth.news.enrich",
|
||||
"label": "Earth 新闻汉化与定位",
|
||||
"group": "Earth 新闻",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "Return exactly one strict JSON object with a location object and a localizations object. Infer the most likely physical event location and produce a faithful Simplified Chinese title and summary based only on the supplied RSS headline, description, source, and date."
|
||||
},
|
||||
{
|
||||
"key": "alerts.brief",
|
||||
"label": "系统告警研判",
|
||||
"group": "告警研判",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "你是告警研判助手。请基于输入的告警事实、上下文与约束,输出结构化、克制、可执行的值班研判;明确区分事实、推断与建议,不要夸大证据不足的风险。",
|
||||
"prompt": "基于当前告警总量、严重度、状态、数据源分布与最近告警摘录,生成一份面向值班人员的简明告警态势简报,突出待处理风险、告警集中点和优先动作。"
|
||||
},
|
||||
{
|
||||
"key": "alerts.situational.brief",
|
||||
"label": "跨模块态势告警研判",
|
||||
"group": "告警研判",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "你是告警研判助手。请基于输入的告警事实、上下文与约束,输出结构化、克制、可执行的值班研判;明确区分事实、推断与建议,不要夸大证据不足的风险。",
|
||||
"prompt": "综合系统告警、BGP incidents、BGP anomalies 与近期 BGP AI 简报,生成一份面向值班人员的态势告警简报,指出当前最需要关注的风险域、跨模块联动迹象和优先动作。"
|
||||
},
|
||||
{
|
||||
"key": "bgp.brief",
|
||||
"label": "BGP 态势简报",
|
||||
"group": "BGP",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "基于当前 BGP incidents、anomalies、原始观测事件、观测站覆盖与 prefix geography 证据,生成一份面向操作员的简明态势简报,突出区域热点、观测偏差、当前风险、证据和优先动作。"
|
||||
},
|
||||
{
|
||||
"key": "location.factcheck.normalize",
|
||||
"label": "位置事实核查结构化",
|
||||
"group": "位置解析",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "Convert the supplied location factcheck text into exactly one strict JSON object. Extract only facts present in the text or original query."
|
||||
},
|
||||
{
|
||||
"key": "location.factcheck.resolve",
|
||||
"label": "位置事实核查兜底",
|
||||
"group": "位置解析",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "Return exactly one JSON object for the most likely physical location. Use only fact-checkable public knowledge; return null fields rather than guessing when evidence is weak."
|
||||
},
|
||||
{
|
||||
"key": "datasource.mapping",
|
||||
"label": "数据源映射生成",
|
||||
"group": "采集配置",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "Return only JSON for a deterministic mapping DSL. The JSON must contain source.items_path and fields. Do not include prose or code."
|
||||
},
|
||||
{
|
||||
"key": "credential.guide",
|
||||
"label": "采集器凭据教程",
|
||||
"group": "采集配置",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "生成一份中文采集器凭据配置教程。只能根据 context.search_evidence 中的来源生成教程;如果证据不足,明确说明需要以官方页面为准。"
|
||||
},
|
||||
{
|
||||
"key": "ai.connection_test",
|
||||
"label": "AI Provider 连接测试",
|
||||
"group": "运维测试",
|
||||
"version": "2026-05-16.1",
|
||||
"system_prompt": "",
|
||||
"prompt": "Reply OK."
|
||||
}
|
||||
]
|
||||
182
backend/app/ai_tasks/prompts.py
Normal file
182
backend/app/ai_tasks/prompts.py
Normal file
@@ -0,0 +1,182 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.system_setting import SystemSetting
|
||||
|
||||
AI_PROMPTS_CATEGORY = "ai_prompts"
|
||||
DEFAULT_PROMPTS_PATH = Path(__file__).with_name("default_prompts.json")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AIPromptDefinition:
|
||||
key: str
|
||||
label: str
|
||||
group: str
|
||||
version: str
|
||||
system_prompt: str
|
||||
prompt: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EffectiveAIPrompt:
|
||||
key: str
|
||||
label: str
|
||||
group: str
|
||||
version: str
|
||||
default_system_prompt: str
|
||||
default_prompt: str
|
||||
system_prompt: str
|
||||
prompt: str
|
||||
is_custom: bool
|
||||
updated_at: str | None = None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def list_prompt_definitions() -> tuple[AIPromptDefinition, ...]:
|
||||
raw_items = json.loads(DEFAULT_PROMPTS_PATH.read_text(encoding="utf-8"))
|
||||
return tuple(
|
||||
AIPromptDefinition(
|
||||
key=str(item["key"]),
|
||||
label=str(item["label"]),
|
||||
group=str(item["group"]),
|
||||
version=str(item["version"]),
|
||||
system_prompt=str(item.get("system_prompt") or ""),
|
||||
prompt=str(item.get("prompt") or ""),
|
||||
)
|
||||
for item in raw_items
|
||||
)
|
||||
|
||||
|
||||
def get_prompt_definition(task_key: str) -> AIPromptDefinition:
|
||||
for definition in list_prompt_definitions():
|
||||
if definition.key == task_key:
|
||||
return definition
|
||||
raise KeyError(task_key)
|
||||
|
||||
|
||||
async def _get_prompt_setting(db: AsyncSession) -> SystemSetting | None:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == AI_PROMPTS_CATEGORY)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _normalize_overrides(payload: dict[str, Any] | None) -> dict[str, dict[str, Any]]:
|
||||
raw = (payload or {}).get("overrides")
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
return {
|
||||
str(key): dict(value)
|
||||
for key, value in raw.items()
|
||||
if isinstance(value, dict)
|
||||
}
|
||||
|
||||
|
||||
async def get_prompt_overrides(db: AsyncSession) -> dict[str, dict[str, Any]]:
|
||||
if not hasattr(db, "execute"):
|
||||
return {}
|
||||
setting = await _get_prompt_setting(db)
|
||||
return _normalize_overrides(setting.payload if setting else None)
|
||||
|
||||
|
||||
def _effective_prompt(
|
||||
definition: AIPromptDefinition,
|
||||
override: dict[str, Any] | None,
|
||||
) -> EffectiveAIPrompt:
|
||||
override = override or {}
|
||||
custom_system = override.get("system_prompt")
|
||||
custom_prompt = override.get("prompt")
|
||||
has_custom_system = isinstance(custom_system, str)
|
||||
has_custom_prompt = isinstance(custom_prompt, str)
|
||||
return EffectiveAIPrompt(
|
||||
key=definition.key,
|
||||
label=definition.label,
|
||||
group=definition.group,
|
||||
version=definition.version,
|
||||
default_system_prompt=definition.system_prompt,
|
||||
default_prompt=definition.prompt,
|
||||
system_prompt=custom_system if has_custom_system else definition.system_prompt,
|
||||
prompt=custom_prompt if has_custom_prompt else definition.prompt,
|
||||
is_custom=has_custom_system or has_custom_prompt,
|
||||
updated_at=str(override.get("updated_at") or "") or None,
|
||||
)
|
||||
|
||||
|
||||
async def list_effective_prompts(db: AsyncSession) -> list[EffectiveAIPrompt]:
|
||||
overrides = await get_prompt_overrides(db)
|
||||
return [
|
||||
_effective_prompt(definition, overrides.get(definition.key))
|
||||
for definition in list_prompt_definitions()
|
||||
]
|
||||
|
||||
|
||||
async def get_effective_prompt(db: AsyncSession | None, task_key: str) -> EffectiveAIPrompt:
|
||||
definition = get_prompt_definition(task_key)
|
||||
if db is None:
|
||||
return _effective_prompt(definition, None)
|
||||
overrides = await get_prompt_overrides(db)
|
||||
return _effective_prompt(definition, overrides.get(task_key))
|
||||
|
||||
|
||||
async def save_prompt_override(
|
||||
db: AsyncSession,
|
||||
task_key: str,
|
||||
*,
|
||||
system_prompt: str,
|
||||
prompt: str,
|
||||
) -> EffectiveAIPrompt:
|
||||
definition = get_prompt_definition(task_key)
|
||||
setting = await _get_prompt_setting(db)
|
||||
payload = dict(setting.payload or {}) if setting else {}
|
||||
overrides = _normalize_overrides(payload)
|
||||
overrides[definition.key] = {
|
||||
"system_prompt": system_prompt,
|
||||
"prompt": prompt,
|
||||
"updated_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
payload["overrides"] = overrides
|
||||
if setting is None:
|
||||
setting = SystemSetting(category=AI_PROMPTS_CATEGORY, payload=payload)
|
||||
db.add(setting)
|
||||
else:
|
||||
setting.payload = payload
|
||||
await db.commit()
|
||||
return _effective_prompt(definition, overrides[definition.key])
|
||||
|
||||
|
||||
async def reset_prompt_override(db: AsyncSession, task_key: str) -> EffectiveAIPrompt:
|
||||
definition = get_prompt_definition(task_key)
|
||||
setting = await _get_prompt_setting(db)
|
||||
if setting is None:
|
||||
return _effective_prompt(definition, None)
|
||||
payload = dict(setting.payload or {})
|
||||
overrides = _normalize_overrides(payload)
|
||||
overrides.pop(definition.key, None)
|
||||
payload["overrides"] = overrides
|
||||
setting.payload = payload
|
||||
await db.commit()
|
||||
return _effective_prompt(definition, None)
|
||||
|
||||
|
||||
def serialize_effective_prompt(prompt: EffectiveAIPrompt) -> dict[str, Any]:
|
||||
return {
|
||||
"key": prompt.key,
|
||||
"label": prompt.label,
|
||||
"group": prompt.group,
|
||||
"version": prompt.version,
|
||||
"default_system_prompt": prompt.default_system_prompt,
|
||||
"default_prompt": prompt.default_prompt,
|
||||
"system_prompt": prompt.system_prompt,
|
||||
"prompt": prompt.prompt,
|
||||
"is_custom": prompt.is_custom,
|
||||
"updated_at": prompt.updated_at,
|
||||
}
|
||||
@@ -6,6 +6,7 @@ from app.api.v1 import (
|
||||
datasource_config,
|
||||
datasources,
|
||||
docs,
|
||||
earth,
|
||||
tasks,
|
||||
dashboard,
|
||||
alerts,
|
||||
@@ -34,6 +35,7 @@ api_router.include_router(
|
||||
api_router.include_router(datasources.router, prefix="/datasources", tags=["datasources"])
|
||||
api_router.include_router(collected_data.router, prefix="/collected", tags=["collected-data"])
|
||||
api_router.include_router(docs.router, prefix="/docs", tags=["docs"])
|
||||
api_router.include_router(earth.router, prefix="/earth", tags=["earth"])
|
||||
api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
|
||||
api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboard"])
|
||||
api_router.include_router(alerts.router, prefix="/alerts", tags=["alerts"])
|
||||
|
||||
@@ -341,6 +341,7 @@ async def collect_bgp_collector_location(
|
||||
provider_client=provider_client,
|
||||
query=query,
|
||||
entity_type="bgp_collector",
|
||||
db=db,
|
||||
attempted_queries=attempted_queries,
|
||||
search_evidence=search_result.evidence,
|
||||
)
|
||||
|
||||
@@ -13,11 +13,6 @@ import httpx
|
||||
|
||||
from app.core.target_schema_registry import get_target_schema, list_target_schemas
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.core.earth_boundary_defaults import (
|
||||
EARTH_BOUNDARY_DEFAULT_SOURCES,
|
||||
EARTH_BOUNDARY_SOURCE_MAPPING,
|
||||
default_earth_boundary_config,
|
||||
)
|
||||
from app.db.session import get_db
|
||||
from app.models.user import User
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
@@ -27,6 +22,7 @@ from app.models.vessel import AISRawObservation, AISSourceHealth
|
||||
from app.core.security import get_current_user
|
||||
from app.core.cache import cache
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient, get_ai_provider_client
|
||||
from app.services.datasource_mapping import (
|
||||
@@ -46,6 +42,8 @@ from app.services.custom_datasource_runtime import (
|
||||
stop_custom_stream,
|
||||
test_websocket_config,
|
||||
)
|
||||
|
||||
DATASOURCE_MAPPING_PROMPT_KEY = "datasource.mapping"
|
||||
from app.services.datasource_connectivity import (
|
||||
get_builtin_connection_status,
|
||||
save_connectivity_success,
|
||||
@@ -56,22 +54,12 @@ from app.services.datasource_connectivity import (
|
||||
router = APIRouter()
|
||||
|
||||
def _default_builtin_config(name: str) -> dict[str, Any]:
|
||||
if name in {"earth_admin0_boundaries", "earth_coastline", "earth_claim_lines"}:
|
||||
return default_earth_boundary_config(name)
|
||||
if name == "earth_boundary_tiles":
|
||||
return {
|
||||
"timeout": 30,
|
||||
"retry": 0,
|
||||
"input_mode": "latest_collected_earth_boundary_sources",
|
||||
}
|
||||
return {"timeout": 30, "retry": 3}
|
||||
|
||||
|
||||
def _default_builtin_source_type(name: str) -> str:
|
||||
if name == "aisstream_vessels":
|
||||
return "websocket"
|
||||
if name == "earth_boundary_tiles":
|
||||
return "internal"
|
||||
return "http"
|
||||
|
||||
|
||||
@@ -400,7 +388,7 @@ async def list_all_datasources(
|
||||
yaml_url = config.get_yaml_url(name)
|
||||
db_config = db_configs.get(name)
|
||||
default_config = _default_builtin_config(name)
|
||||
default_url = EARTH_BOUNDARY_DEFAULT_SOURCES.get(name, {}).get("endpoint") or yaml_url
|
||||
default_url = yaml_url
|
||||
|
||||
result.append(
|
||||
{
|
||||
@@ -783,14 +771,12 @@ async def propose_datasource_mapping(
|
||||
generated_by = "heuristic"
|
||||
if payload.use_ai:
|
||||
try:
|
||||
prompt = await get_effective_prompt(db, DATASOURCE_MAPPING_PROMPT_KEY)
|
||||
response = await ai_client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title=f"Generate datasource mapping for {schema.key}",
|
||||
objective=(
|
||||
"Return only JSON for a deterministic mapping DSL. "
|
||||
"The JSON must contain source.items_path and fields. "
|
||||
"Do not include prose or code."
|
||||
),
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"target_schema": schema.to_dict(),
|
||||
"sample_payload": redacted_sample,
|
||||
|
||||
@@ -35,7 +35,6 @@ PRODUCT_SOURCE_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
("cables", ("cable", "landing", "telegeography", "arcgis", "fao")),
|
||||
("satellites", ("tle", "satellite", "spacetrack", "celestrak")),
|
||||
("bgp", ("bgp", "asn", "prefix_geo", "opengeofeed", "nro")),
|
||||
("earth", ("earth_", "earth_boundary", "boundary", "country boundary", "coastline", "claim")),
|
||||
("compute", ("top500", "gpu", "supercomputer", "compute")),
|
||||
("ai", ("huggingface", "epoch_ai")),
|
||||
("media", ("news", "tv", "live_stream")),
|
||||
|
||||
118
backend/app/api/v1/earth.py
Normal file
118
backend/app/api/v1/earth.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""Earth asset management APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.security import decode_token, get_current_user, redis_client
|
||||
from app.db.session import get_db
|
||||
from app.models.user import User
|
||||
from app.services.earth_boundaries import (
|
||||
EarthBoundaryBuildError,
|
||||
get_boundary_build_status,
|
||||
get_boundary_status,
|
||||
save_boundary_config,
|
||||
start_boundary_build_job,
|
||||
)
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
optional_bearer = HTTPBearer(auto_error=False)
|
||||
|
||||
|
||||
class EarthBoundaryConfigPayload(BaseModel):
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
@router.get("/boundaries/status")
|
||||
async def get_earth_boundary_status():
|
||||
return get_boundary_status()
|
||||
|
||||
|
||||
async def _get_optional_current_user(
|
||||
credentials: HTTPAuthorizationCredentials | None = Depends(optional_bearer),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> User | None:
|
||||
if credentials is None:
|
||||
return None
|
||||
token = credentials.credentials
|
||||
if redis_client.sismember("blacklisted_tokens", token):
|
||||
return None
|
||||
payload = decode_token(token)
|
||||
if payload is None or payload.get("type") != "access":
|
||||
return None
|
||||
user_id = payload.get("sub")
|
||||
if user_id is None:
|
||||
return None
|
||||
result = await db.execute(
|
||||
text(
|
||||
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE id = :id"
|
||||
),
|
||||
{"id": int(user_id)},
|
||||
)
|
||||
row = result.fetchone()
|
||||
if row is None or not row[5]:
|
||||
return None
|
||||
user = User()
|
||||
user.id = row[0]
|
||||
user.username = row[1]
|
||||
user.email = row[2]
|
||||
user.password_hash = row[3]
|
||||
user.role = row[4]
|
||||
user.is_active = row[5]
|
||||
user.gatekeeper_groups = row[6] or []
|
||||
return user
|
||||
|
||||
|
||||
def _is_loopback_request(request: Request) -> bool:
|
||||
host = request.client.host if request.client else ""
|
||||
return host in {"127.0.0.1", "::1", "localhost"} or host.startswith("127.")
|
||||
|
||||
|
||||
def _require_local_or_user(request: Request, user: User | None) -> None:
|
||||
if user is not None or _is_loopback_request(request):
|
||||
return
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Authentication required outside localhost",
|
||||
)
|
||||
|
||||
|
||||
@router.put("/boundaries/config")
|
||||
async def update_earth_boundary_config(
|
||||
payload: EarthBoundaryConfigPayload,
|
||||
_current_user: User = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
return save_boundary_config(payload.config)
|
||||
except EarthBoundaryBuildError as exc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": exc.code, "message": str(exc), "details": exc.details},
|
||||
) from exc
|
||||
|
||||
|
||||
@router.post("/boundaries/build")
|
||||
async def build_earth_boundary_assets(
|
||||
request: Request,
|
||||
current_user: User | None = Depends(_get_optional_current_user),
|
||||
):
|
||||
_require_local_or_user(request, current_user)
|
||||
try:
|
||||
return await start_boundary_build_job()
|
||||
except EarthBoundaryBuildError as exc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": exc.code, "message": str(exc), "details": exc.details},
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get("/boundaries/build/status")
|
||||
async def get_earth_boundary_build_status():
|
||||
return get_boundary_build_status()
|
||||
@@ -15,6 +15,13 @@ from app.core.time import to_iso8601_utc
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.ai_tasks.prompts import (
|
||||
get_effective_prompt,
|
||||
list_effective_prompts,
|
||||
reset_prompt_override,
|
||||
save_prompt_override,
|
||||
serialize_effective_prompt,
|
||||
)
|
||||
from app.db.session import get_db
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
@@ -60,6 +67,7 @@ from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload
|
||||
|
||||
router = APIRouter()
|
||||
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5
|
||||
AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test"
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
"system": {
|
||||
@@ -251,6 +259,11 @@ class OCRIntegrationUpdate(BaseModel):
|
||||
output_format: str = Field(default="markdown", pattern="^(markdown|text|json)$")
|
||||
|
||||
|
||||
class AIPromptUpdate(BaseModel):
|
||||
system_prompt: str = Field(default="", max_length=8000)
|
||||
prompt: str = Field(min_length=1, max_length=20000)
|
||||
|
||||
|
||||
class ExternalIntegrationsUpdate(BaseModel):
|
||||
ai_provider: AIProviderIntegrationUpdate
|
||||
barentswatch: BarentsWatchIntegrationUpdate
|
||||
@@ -487,6 +500,10 @@ def _is_secret_placeholder(value: Optional[str], current_preview: str = "") -> b
|
||||
return True
|
||||
if text == current_preview or text.startswith("••••"):
|
||||
return True
|
||||
if "-" in text:
|
||||
_prefix, masked = text.split("-", 1)
|
||||
if masked and all(char in {"*", "•", " ", "\t"} for char in masked):
|
||||
return True
|
||||
return all(char in {"*", "•", " ", "\t"} for char in text)
|
||||
|
||||
|
||||
@@ -597,10 +614,12 @@ async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict:
|
||||
status_code=400,
|
||||
detail="AI Provider 可访问,但当前 provider/model/key 未完整配置。",
|
||||
)
|
||||
prompt = await get_effective_prompt(None, AI_CONNECTION_TEST_PROMPT_KEY)
|
||||
analysis_result = await client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title="保存前完整连接测试",
|
||||
objective="请用一句话回复连接可用。",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=["这是保存 AI Provider 配置前的完整 LLM 调用测试。"],
|
||||
constraints=["回复尽量简短。"],
|
||||
)
|
||||
@@ -1212,6 +1231,47 @@ async def get_external_integrations(
|
||||
return {"integrations": await serialize_external_integrations(db)}
|
||||
|
||||
|
||||
@router.get("/ai-prompts")
|
||||
async def get_ai_prompts(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
prompts = await list_effective_prompts(db)
|
||||
return {"data": [serialize_effective_prompt(prompt) for prompt in prompts]}
|
||||
|
||||
|
||||
@router.put("/ai-prompts/{task_key}")
|
||||
async def update_ai_prompt(
|
||||
task_key: str,
|
||||
payload: AIPromptUpdate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
try:
|
||||
prompt = await save_prompt_override(
|
||||
db,
|
||||
task_key,
|
||||
system_prompt=payload.system_prompt,
|
||||
prompt=payload.prompt,
|
||||
)
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=404, detail="未知 AI 提示词任务") from None
|
||||
return {"data": serialize_effective_prompt(prompt)}
|
||||
|
||||
|
||||
@router.post("/ai-prompts/{task_key}/reset")
|
||||
async def reset_ai_prompt(
|
||||
task_key: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
try:
|
||||
prompt = await reset_prompt_override(db, task_key)
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=404, detail="未知 AI 提示词任务") from None
|
||||
return {"data": serialize_effective_prompt(prompt)}
|
||||
|
||||
|
||||
@router.get("/integrations/barentswatch/connectivity")
|
||||
async def get_barentswatch_connectivity(
|
||||
current_user: User = Depends(get_current_user),
|
||||
@@ -1294,10 +1354,12 @@ async def connect_ai_provider_integration(
|
||||
"message": "AI Provider 可访问,但当前 provider/model/key 未完整配置。",
|
||||
"status": status_result.model_dump(),
|
||||
}
|
||||
prompt = await get_effective_prompt(db, AI_CONNECTION_TEST_PROMPT_KEY)
|
||||
probe_result = await client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title="快速连接测试",
|
||||
objective="Reply OK.",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=[],
|
||||
constraints=["Output only OK."],
|
||||
)
|
||||
|
||||
@@ -1982,6 +1982,7 @@ async def collect_compute_center_location(
|
||||
provider_client=provider_client,
|
||||
query=query,
|
||||
entity_type="compute_center",
|
||||
db=db,
|
||||
attempted_queries=attempted_queries,
|
||||
search_evidence=search_result.evidence,
|
||||
)
|
||||
|
||||
@@ -257,46 +257,6 @@ DEFAULT_DATASOURCES = {
|
||||
"credential_provider": "aisstream",
|
||||
"credential_status": "supported",
|
||||
},
|
||||
"earth_admin0_boundaries": {
|
||||
"id": 29,
|
||||
"name": "Earth Admin-0 Boundaries",
|
||||
"display_name": "Earth Admin-0 国界源",
|
||||
"module": "L3",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"earth_coastline": {
|
||||
"id": 30,
|
||||
"name": "Earth Coastline",
|
||||
"display_name": "Earth 海岸线源",
|
||||
"module": "L3",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"earth_claim_lines": {
|
||||
"id": 31,
|
||||
"name": "Earth Claim Lines",
|
||||
"display_name": "Earth 主张线源",
|
||||
"module": "L3",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"earth_boundary_tiles": {
|
||||
"id": 32,
|
||||
"name": "Earth PMTiles Builder",
|
||||
"display_name": "Earth PMTiles 构建器",
|
||||
"module": "L3",
|
||||
"priority": "P1",
|
||||
"frequency_minutes": 10080,
|
||||
"is_free": True,
|
||||
"requires_credentials": False,
|
||||
},
|
||||
"media_news_archive": {
|
||||
"id": 33,
|
||||
"name": "Media News Archive",
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
"""Default Earth boundary source configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
EARTH_BOUNDARY_SOURCE_MAPPING: dict[str, Any] = {
|
||||
"source": {"items_path": "$.features[*]"},
|
||||
"fields": {
|
||||
"source_id": {"path": "$.properties.id", "type": "string"},
|
||||
"name": {"path": "$.properties.name", "type": "string"},
|
||||
"geometry": {"path": "$.geometry", "type": "object"},
|
||||
"properties": {"path": "$.properties", "type": "object"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
EARTH_BOUNDARY_DEFAULT_SOURCES: dict[str, dict[str, Any]] = {
|
||||
"earth_admin0_boundaries": {
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_countries.geojson",
|
||||
"source_kind": "admin0-boundaries",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
"earth_coastline": {
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_coastline.geojson",
|
||||
"source_kind": "coastline",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
"earth_claim_lines": {
|
||||
"endpoint": "https://www.arcgis.com/sharing/rest/content/items/faaa1908c3ab43f0823c6fde9f18389c/data",
|
||||
"source_kind": "claim-lines",
|
||||
"license": "CC BY 4.0; source item owner mapmakersami",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def default_earth_boundary_config(name: str) -> dict[str, Any]:
|
||||
source = EARTH_BOUNDARY_DEFAULT_SOURCES.get(name)
|
||||
if not source:
|
||||
return {}
|
||||
return {
|
||||
"method": "GET",
|
||||
"timeout": 120,
|
||||
"retry": 3,
|
||||
"target_schema": "earth_boundary_source",
|
||||
"license": source["license"],
|
||||
"mapping_json": EARTH_BOUNDARY_SOURCE_MAPPING,
|
||||
}
|
||||
@@ -47,18 +47,6 @@ class GenericRecord(BaseModel):
|
||||
return value
|
||||
|
||||
|
||||
class EarthBoundarySourceRecord(BaseModel):
|
||||
source_id: str | None = None
|
||||
source_kind: str = Field(pattern="^(admin0-boundaries|coastline|claim-lines)$")
|
||||
name: str | None = None
|
||||
geometry: dict[str, Any] | None = None
|
||||
properties: dict[str, Any] = Field(default_factory=dict)
|
||||
artifact_path: str | None = None
|
||||
sha256: str | None = None
|
||||
feature_count: int | None = Field(default=None, ge=0)
|
||||
license: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TargetField:
|
||||
name: str
|
||||
@@ -155,24 +143,6 @@ TARGET_SCHEMAS: dict[str, TargetSchema] = {
|
||||
TargetField("observed_at", "datetime", False, "观测时间", "2026-04-28T00:00:00Z"),
|
||||
),
|
||||
),
|
||||
"earth_boundary_source": TargetSchema(
|
||||
key="earth_boundary_source",
|
||||
label="Earth 边界矢量源",
|
||||
description="Admin-0 国界、海岸线、主张线等 Earth 边界源数据 artifact。",
|
||||
destination="collected_data",
|
||||
model=EarthBoundarySourceRecord,
|
||||
fields=(
|
||||
TargetField("source_id", "string", False, "来源侧 ID", "feature-1"),
|
||||
TargetField("source_kind", "string", True, "边界源类型", "admin0-boundaries"),
|
||||
TargetField("name", "string", False, "记录名称", "China"),
|
||||
TargetField("geometry", "object", False, "GeoJSON geometry", {"type": "Polygon", "coordinates": []}),
|
||||
TargetField("properties", "object", False, "GeoJSON properties", {"ISO_A3": "CHN"}),
|
||||
TargetField("artifact_path", "string", False, "完整源数据本地 artifact 路径", "data/earth-boundary-sources/earth_admin0_boundaries/abc.geojson"),
|
||||
TargetField("sha256", "string", False, "源数据 artifact SHA-256", "abc123"),
|
||||
TargetField("feature_count", "integer", False, "源数据 feature 数", 1),
|
||||
TargetField("license", "string", False, "源数据许可", "ODbL"),
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy import bindparam, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
from sqlalchemy.orm import declarative_base
|
||||
|
||||
@@ -72,6 +72,74 @@ async def seed_default_datasources(session: AsyncSession):
|
||||
await session.commit()
|
||||
|
||||
|
||||
LEGACY_EARTH_BOUNDARY_SOURCES = (
|
||||
"earth_admin0_boundaries",
|
||||
"earth_coastline",
|
||||
"earth_claim_lines",
|
||||
"earth_boundary_tiles",
|
||||
)
|
||||
LEGACY_EARTH_BOUNDARY_DATATYPES = (
|
||||
"earth_boundary_source",
|
||||
"earth_boundary_tiles",
|
||||
)
|
||||
LEGACY_EARTH_BOUNDARY_IDS = (29, 30, 31, 32)
|
||||
|
||||
|
||||
async def purge_legacy_earth_boundary_datasources(session: AsyncSession) -> None:
|
||||
source_names = tuple(LEGACY_EARTH_BOUNDARY_SOURCES)
|
||||
source_ids = tuple(LEGACY_EARTH_BOUNDARY_IDS)
|
||||
data_types = tuple(LEGACY_EARTH_BOUNDARY_DATATYPES)
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
DELETE FROM datasource_mapping_templates
|
||||
WHERE target_schema IN :data_types
|
||||
OR datasource_config_id IN (
|
||||
SELECT id FROM datasource_configs WHERE name IN :source_names
|
||||
)
|
||||
"""
|
||||
).bindparams(bindparam("source_names", expanding=True), bindparam("data_types", expanding=True)),
|
||||
{"source_names": list(source_names), "data_types": list(data_types)},
|
||||
)
|
||||
await session.execute(
|
||||
text("DELETE FROM datasource_configs WHERE name IN :source_names").bindparams(
|
||||
bindparam("source_names", expanding=True)
|
||||
),
|
||||
{"source_names": list(source_names)},
|
||||
)
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
DELETE FROM collected_data
|
||||
WHERE source IN :source_names OR data_type IN :data_types
|
||||
"""
|
||||
).bindparams(bindparam("source_names", expanding=True), bindparam("data_types", expanding=True)),
|
||||
{"source_names": list(source_names), "data_types": list(data_types)},
|
||||
)
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
DELETE FROM data_snapshots
|
||||
WHERE source IN :source_names OR datasource_id IN :source_ids
|
||||
"""
|
||||
).bindparams(bindparam("source_names", expanding=True), bindparam("source_ids", expanding=True)),
|
||||
{"source_names": list(source_names), "source_ids": list(source_ids)},
|
||||
)
|
||||
await session.execute(
|
||||
text("DELETE FROM collection_tasks WHERE datasource_id IN :source_ids").bindparams(
|
||||
bindparam("source_ids", expanding=True)
|
||||
),
|
||||
{"source_ids": list(source_ids)},
|
||||
)
|
||||
await session.execute(
|
||||
text("DELETE FROM data_sources WHERE source IN :source_names OR id IN :source_ids").bindparams(
|
||||
bindparam("source_names", expanding=True), bindparam("source_ids", expanding=True)
|
||||
),
|
||||
{"source_names": list(source_names), "source_ids": list(source_ids)},
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
DEFAULT_LOGIN_USERS = (
|
||||
{
|
||||
"username": "admin",
|
||||
@@ -203,6 +271,18 @@ async def init_db():
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
ALTER TABLE earth_news_items
|
||||
ADD COLUMN IF NOT EXISTS content_language VARCHAR(32) NOT NULL DEFAULT 'en',
|
||||
ADD COLUMN IF NOT EXISTS localizations JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
ADD COLUMN IF NOT EXISTS enrichment_status VARCHAR(80) NOT NULL DEFAULT 'pending',
|
||||
ADD COLUMN IF NOT EXISTS enrichment_error TEXT,
|
||||
ADD COLUMN IF NOT EXISTS enriched_at TIMESTAMPTZ
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -211,6 +291,22 @@ async def init_db():
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_news_enrichment_status
|
||||
ON earth_news_items (enrichment_status)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_news_enriched_at
|
||||
ON earth_news_items (enriched_at)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -285,4 +381,5 @@ async def init_db():
|
||||
await seed_default_bgp_collector_locations(session)
|
||||
await seed_compute_center_locations_from_source_coords(session)
|
||||
await seed_default_datasources(session)
|
||||
await purge_legacy_earth_boundary_datasources(session)
|
||||
await ensure_default_admin_user(session)
|
||||
|
||||
@@ -10,6 +10,8 @@ class EarthNewsItem(Base):
|
||||
id = Column(String(160), primary_key=True)
|
||||
title = Column(String(500), nullable=False)
|
||||
summary = Column(Text, nullable=False, default="")
|
||||
content_language = Column(String(32), nullable=False, default="en")
|
||||
localizations = Column(JSON, nullable=False, default=dict)
|
||||
url = Column(Text, nullable=False)
|
||||
source = Column(String(255), nullable=False, default="")
|
||||
feed_name = Column(String(255), nullable=False, default="")
|
||||
@@ -27,6 +29,9 @@ class EarthNewsItem(Base):
|
||||
first_seen_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
|
||||
last_seen_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False, index=True)
|
||||
resolved_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
enrichment_status = Column(String(80), nullable=False, default="pending", index=True)
|
||||
enrichment_error = Column(Text, nullable=True)
|
||||
enriched_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
|
||||
@@ -13,10 +13,11 @@ class AIContentBlock(BaseModel):
|
||||
|
||||
class SituationalAnalysisRequest(BaseModel):
|
||||
title: str = Field(..., min_length=1, max_length=200)
|
||||
objective: str = Field(..., min_length=1, max_length=1000)
|
||||
objective: str = Field(..., min_length=1, max_length=20000)
|
||||
context: dict[str, Any] = Field(default_factory=dict)
|
||||
observations: list[str] = Field(default_factory=list)
|
||||
constraints: list[str] = Field(default_factory=list)
|
||||
system_prompt: str | None = Field(default=None, max_length=8000)
|
||||
preferred_model: str | None = Field(default=None, max_length=200)
|
||||
thinking: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@@ -8,6 +8,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.alert import Alert, AlertSeverity, AlertStatus
|
||||
from app.schemas.ai import AlertBriefRequest, SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
|
||||
ALERT_BRIEF_PROMPT_KEY = "alerts.brief"
|
||||
|
||||
|
||||
def _format_counter(counter: Counter[str], empty_text: str = "无") -> str:
|
||||
@@ -84,11 +87,13 @@ async def build_alert_brief_request(
|
||||
"top_datasources": dict(datasource_counts.most_common(6)),
|
||||
"top_active_datasources": dict(active_datasource_counts.most_common(5)),
|
||||
}
|
||||
prompt = await get_effective_prompt(db, ALERT_BRIEF_PROMPT_KEY)
|
||||
|
||||
return (
|
||||
SituationalAnalysisRequest(
|
||||
title="告警态势 AI 简报",
|
||||
objective="基于当前告警总量、严重度、状态、数据源分布与最近告警摘录,生成一份面向值班人员的简明告警态势简报,突出待处理风险、告警集中点和优先动作。",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=facts,
|
||||
constraints=[
|
||||
"明确区分事实、推断与建议。",
|
||||
|
||||
@@ -11,9 +11,12 @@ from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.models.bgp_observation import BGPObservation
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.bgp_collectors import build_bgp_collector_coverage
|
||||
from app.services.bgp_enrichment import lookup_prefix_geography
|
||||
|
||||
BGP_BRIEF_PROMPT_KEY = "bgp.brief"
|
||||
|
||||
|
||||
def _format_counter(counter: dict[str, int], empty_text: str = "无") -> str:
|
||||
if not counter:
|
||||
@@ -243,10 +246,12 @@ async def build_bgp_brief_request(
|
||||
for prefix, item in list(prefix_geographies.items())[:8]
|
||||
},
|
||||
}
|
||||
prompt = await get_effective_prompt(db, BGP_BRIEF_PROMPT_KEY)
|
||||
|
||||
return SituationalAnalysisRequest(
|
||||
title="BGP 态势 AI 简报",
|
||||
objective="基于当前 BGP incidents、anomalies、原始观测事件、观测站覆盖与 prefix geography 证据,生成一份面向操作员的简明态势简报,突出区域热点、观测偏差、当前风险、证据和优先动作。",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=observations_lines,
|
||||
constraints=[
|
||||
"明确区分事实、推断与建议。",
|
||||
|
||||
@@ -39,12 +39,6 @@ from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
|
||||
from app.services.collectors.media_news_archive import MediaNewsArchiveCollector
|
||||
from app.services.collectors.aisstream import AISStreamCollector
|
||||
from app.services.collectors.vessel_ais import VesselAISCollector
|
||||
from app.services.collectors.earth_boundaries import (
|
||||
EarthAdmin0BoundaryCollector,
|
||||
EarthBoundaryTileCollector,
|
||||
EarthClaimLinesCollector,
|
||||
EarthCoastlineCollector,
|
||||
)
|
||||
|
||||
collector_registry.register(TOP500Collector())
|
||||
collector_registry.register(EpochAIGPUCollector())
|
||||
@@ -75,10 +69,6 @@ collector_registry.register(NewsLiveStreamsCollector())
|
||||
collector_registry.register(MediaNewsArchiveCollector())
|
||||
collector_registry.register(VesselAISCollector())
|
||||
collector_registry.register(AISStreamCollector())
|
||||
collector_registry.register(EarthAdmin0BoundaryCollector())
|
||||
collector_registry.register(EarthCoastlineCollector())
|
||||
collector_registry.register(EarthClaimLinesCollector())
|
||||
collector_registry.register(EarthBoundaryTileCollector())
|
||||
|
||||
__all__ = [
|
||||
"BaseCollector",
|
||||
@@ -115,8 +105,4 @@ __all__ = [
|
||||
"MediaNewsArchiveCollector",
|
||||
"VesselAISCollector",
|
||||
"AISStreamCollector",
|
||||
"EarthAdmin0BoundaryCollector",
|
||||
"EarthCoastlineCollector",
|
||||
"EarthClaimLinesCollector",
|
||||
"EarthBoundaryTileCollector",
|
||||
]
|
||||
|
||||
@@ -1,578 +0,0 @@
|
||||
"""Earth boundary source and static tile collector."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.earth_boundary_defaults import EARTH_BOUNDARY_DEFAULT_SOURCES, default_earth_boundary_config
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.services.collectors.base import BaseCollector
|
||||
from app.services.custom_datasource_runtime import build_query_params, build_request_headers
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
|
||||
SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
|
||||
BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
|
||||
BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
|
||||
PMTILES_ARTIFACT_PATH = REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
|
||||
POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
|
||||
BUILD_CONFIG = {
|
||||
"builder": "scripts/build_earth_boundary_china_pov_geojson.py",
|
||||
"format": "geojson-high-precision",
|
||||
"production_target": "geojson-high-precision",
|
||||
}
|
||||
|
||||
EARTH_BOUNDARY_SOURCE_COLLECTORS = {
|
||||
"earth_admin0_boundaries": "admin0-boundaries",
|
||||
"earth_coastline": "coastline",
|
||||
"earth_claim_lines": "claim-lines",
|
||||
}
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _directory_stats(path: Path) -> dict[str, int]:
|
||||
if not path.exists():
|
||||
return {"file_count": 0, "size_bytes": 0}
|
||||
files = [item for item in path.rglob("*") if item.is_file()]
|
||||
return {
|
||||
"file_count": len(files),
|
||||
"size_bytes": sum(item.stat().st_size for item in files),
|
||||
}
|
||||
|
||||
|
||||
def _stable_json_hash(payload: Any) -> str:
|
||||
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _build_input_hash(source_manifest: dict[str, Any]) -> str:
|
||||
return _stable_json_hash(
|
||||
{
|
||||
"source_manifest_schema": source_manifest.get("schema"),
|
||||
"sources": [
|
||||
{
|
||||
"id": source.get("id"),
|
||||
"sha256": source.get("sha256"),
|
||||
"kind": source.get("kind"),
|
||||
"pov": source.get("pov"),
|
||||
}
|
||||
for source in source_manifest.get("sources", [])
|
||||
],
|
||||
"pov_policy": source_manifest.get("povPolicy"),
|
||||
"build_config": BUILD_CONFIG,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
|
||||
if not boundary_manifest:
|
||||
return False
|
||||
if boundary_manifest.get("buildInputHash") != build_input_hash:
|
||||
return False
|
||||
if not BOUNDARY_OUTPUT_DIR.exists():
|
||||
return False
|
||||
if boundary_manifest.get("tileProvider") == "pmtiles-mvt":
|
||||
return PMTILES_ARTIFACT_PATH.exists()
|
||||
return (BOUNDARY_OUTPUT_DIR / "base.geojson").exists() and (
|
||||
BOUNDARY_OUTPUT_DIR / "hover-index.geojson"
|
||||
).exists()
|
||||
|
||||
|
||||
def _sha256_bytes(payload: bytes) -> str:
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _json_feature_count(payload: Any) -> int:
|
||||
if isinstance(payload, dict) and isinstance(payload.get("features"), list):
|
||||
return len(payload["features"])
|
||||
if isinstance(payload, list):
|
||||
return len(payload)
|
||||
return 1 if payload else 0
|
||||
|
||||
|
||||
def _json_sample_properties(payload: Any) -> dict[str, Any]:
|
||||
feature = None
|
||||
if isinstance(payload, dict) and isinstance(payload.get("features"), list) and payload["features"]:
|
||||
feature = payload["features"][0]
|
||||
elif isinstance(payload, list) and payload:
|
||||
feature = payload[0]
|
||||
elif isinstance(payload, dict):
|
||||
feature = payload
|
||||
if not isinstance(feature, dict):
|
||||
return {}
|
||||
props = feature.get("properties") if isinstance(feature.get("properties"), dict) else feature
|
||||
return {str(key): value for key, value in list(props.items())[:20]}
|
||||
|
||||
|
||||
def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
|
||||
suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
|
||||
if suffix in {".json", ".geojson", ".zip", ".pbf"}:
|
||||
return suffix
|
||||
if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
|
||||
return ".geojson"
|
||||
if "json" in content_type:
|
||||
return ".json"
|
||||
return ".dat"
|
||||
|
||||
|
||||
async def _load_datasource_config(db, name: str) -> DataSourceConfig | None:
|
||||
if db is None:
|
||||
return None
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig)
|
||||
.where(DataSourceConfig.name == name)
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
.order_by(DataSourceConfig.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _latest_boundary_source_record(db, source_name: str) -> CollectedData | None:
|
||||
result = await db.execute(
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == source_name)
|
||||
.where(CollectedData.data_type == "earth_boundary_source")
|
||||
.where(CollectedData.is_current.is_(True))
|
||||
.order_by(CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
class EarthBoundarySourceCollector(BaseCollector):
|
||||
priority = "P1"
|
||||
module = "L3"
|
||||
frequency_hours = 168
|
||||
data_type = "earth_boundary_source"
|
||||
fail_on_empty = True
|
||||
source_kind = "unknown"
|
||||
|
||||
def _default_config(self):
|
||||
source = EARTH_BOUNDARY_DEFAULT_SOURCES.get(self.name)
|
||||
if not source:
|
||||
return None
|
||||
return SimpleNamespace(
|
||||
name=self.name,
|
||||
description=f"内置默认源:{self.name}",
|
||||
endpoint=source["endpoint"],
|
||||
source_type="http",
|
||||
auth_type="none",
|
||||
auth_config={},
|
||||
headers={},
|
||||
config=default_earth_boundary_config(self.name),
|
||||
)
|
||||
|
||||
async def _download_payload(self, config: DataSourceConfig) -> tuple[bytes, str]:
|
||||
request_config = config.config or {}
|
||||
method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
|
||||
if method not in {"GET", "POST"}:
|
||||
raise RuntimeError("Earth boundary source collectors support GET and POST only")
|
||||
|
||||
endpoint = str(config.endpoint or "").strip()
|
||||
if not endpoint:
|
||||
raise RuntimeError(
|
||||
f"{self.name} requires an endpoint in Collector Settings before it can collect data"
|
||||
)
|
||||
|
||||
if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
|
||||
path = Path(endpoint.removeprefix("file://")).expanduser()
|
||||
return path.read_bytes(), "application/octet-stream"
|
||||
|
||||
headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {})
|
||||
params = build_query_params(config.auth_type, config.auth_config or {}, request_config)
|
||||
timeout = float(request_config.get("timeout", 120))
|
||||
json_body = request_config.get("json_body") or request_config.get("body")
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
response = await client.request(
|
||||
method,
|
||||
endpoint,
|
||||
headers=headers,
|
||||
params=params or None,
|
||||
json=json_body if isinstance(json_body, (dict, list)) else None,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.content, response.headers.get("content-type", "")
|
||||
|
||||
async def fetch(self) -> list[dict[str, Any]]:
|
||||
db = getattr(self, "_db_session", None)
|
||||
config = await _load_datasource_config(db, self.name) or self._default_config()
|
||||
if config is None:
|
||||
raise RuntimeError(
|
||||
f"{self.name} has no active Collector Settings config and no built-in default source."
|
||||
)
|
||||
|
||||
await self.set_phase("fetching_source", message=f"正在下载 {self.source_kind} 源数据")
|
||||
payload, content_type = await self._download_payload(config)
|
||||
sha256 = _sha256_bytes(payload)
|
||||
endpoint = str(config.endpoint or "")
|
||||
extension = _artifact_extension(endpoint, content_type, payload)
|
||||
source_dir = SOURCE_OUTPUT_DIR / self.name
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact_path = source_dir / f"{sha256}{extension}"
|
||||
artifact_path.write_bytes(payload)
|
||||
|
||||
parsed: Any = None
|
||||
if extension in {".json", ".geojson"}:
|
||||
parsed = json.loads(payload.decode("utf-8"))
|
||||
|
||||
feature_count = _json_feature_count(parsed)
|
||||
if feature_count <= 0:
|
||||
raise RuntimeError(f"{self.name} downloaded data but found no JSON/GeoJSON features")
|
||||
|
||||
config_body = config.config or {}
|
||||
target_schema = str(config_body.get("target_schema") or "earth_boundary_source")
|
||||
if target_schema != "earth_boundary_source":
|
||||
raise RuntimeError(f"{self.name} target_schema must be earth_boundary_source")
|
||||
|
||||
await self.update_phase_progress(
|
||||
current=1,
|
||||
total=1,
|
||||
unit="artifact",
|
||||
message=f"已保存 {feature_count} 个 {self.source_kind} feature",
|
||||
progress=100,
|
||||
commit=True,
|
||||
force=True,
|
||||
)
|
||||
|
||||
relative_artifact_path = str(artifact_path.relative_to(REPO_ROOT))
|
||||
return [
|
||||
{
|
||||
"id": f"{self.source_kind}:{sha256}",
|
||||
"name": config.description or self.name,
|
||||
"description": f"Earth boundary source artifact collected from configured endpoint",
|
||||
"source_kind": self.source_kind,
|
||||
"source_id": sha256,
|
||||
"value": feature_count,
|
||||
"unit": "features",
|
||||
"metadata": {
|
||||
"target_schema": target_schema,
|
||||
"source_kind": self.source_kind,
|
||||
"endpoint": endpoint,
|
||||
"method": str(config_body.get("method") or config_body.get("request_method") or "GET").upper(),
|
||||
"artifact_path": relative_artifact_path,
|
||||
"sha256": sha256,
|
||||
"feature_count": feature_count,
|
||||
"size_bytes": len(payload),
|
||||
"content_type": content_type,
|
||||
"license": config_body.get("license"),
|
||||
"mapping_json": config_body.get("mapping_json"),
|
||||
"sample_properties": _json_sample_properties(parsed),
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class EarthAdmin0BoundaryCollector(EarthBoundarySourceCollector):
|
||||
name = "earth_admin0_boundaries"
|
||||
source_kind = "admin0-boundaries"
|
||||
|
||||
|
||||
class EarthCoastlineCollector(EarthBoundarySourceCollector):
|
||||
name = "earth_coastline"
|
||||
source_kind = "coastline"
|
||||
|
||||
|
||||
class EarthClaimLinesCollector(EarthBoundarySourceCollector):
|
||||
name = "earth_claim_lines"
|
||||
source_kind = "claim-lines"
|
||||
|
||||
|
||||
class EarthBoundaryTileCollector(BaseCollector):
|
||||
name = "earth_boundary_tiles"
|
||||
priority = "P1"
|
||||
module = "L3"
|
||||
frequency_hours = 168
|
||||
data_type = "earth_boundary_tiles"
|
||||
fail_on_empty = False
|
||||
|
||||
async def _run_step(self, args: list[str], *, allow_failure: bool = False) -> dict[str, Any]:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
*args,
|
||||
cwd=REPO_ROOT,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout_bytes, stderr_bytes = await process.communicate()
|
||||
stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
|
||||
stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
|
||||
if process.returncode != 0 and not allow_failure:
|
||||
raise RuntimeError(stderr or stdout or f"command failed: {' '.join(args)}")
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
"returncode": process.returncode,
|
||||
}
|
||||
last_line = stdout.splitlines()[-1:] or []
|
||||
if last_line:
|
||||
try:
|
||||
payload["result"] = json.loads(last_line[0])
|
||||
except json.JSONDecodeError:
|
||||
payload["result"] = last_line[0]
|
||||
return payload
|
||||
|
||||
async def fetch(self) -> list[dict[str, Any]]:
|
||||
db = getattr(self, "_db_session", None)
|
||||
if db is None:
|
||||
raise RuntimeError("Earth PMTiles builder requires an active database session")
|
||||
|
||||
await self.set_phase("checking_sources", message="正在检查三类 Earth 边界源")
|
||||
await self.update_phase_progress(
|
||||
current=0,
|
||||
total=3,
|
||||
unit="steps",
|
||||
message="读取 admin0 / coastline / claim-lines 最新采集结果",
|
||||
progress=5,
|
||||
commit=True,
|
||||
force=True,
|
||||
)
|
||||
|
||||
source_records: dict[str, CollectedData] = {}
|
||||
missing_sources: list[str] = []
|
||||
for source_name in EARTH_BOUNDARY_SOURCE_COLLECTORS:
|
||||
record = await _latest_boundary_source_record(db, source_name)
|
||||
if record is None:
|
||||
missing_sources.append(source_name)
|
||||
else:
|
||||
source_records[source_name] = record
|
||||
|
||||
if missing_sources:
|
||||
missing_text = ", ".join(missing_sources)
|
||||
raise RuntimeError(f"未就绪:缺少 {missing_text}。不会更新 Earth 国界。")
|
||||
|
||||
sources = []
|
||||
for source_name, record in source_records.items():
|
||||
metadata = record.extra_data or {}
|
||||
artifact_path = metadata.get("artifact_path")
|
||||
if not artifact_path or not (REPO_ROOT / str(artifact_path)).exists():
|
||||
missing_sources.append(f"{source_name}: artifact missing")
|
||||
continue
|
||||
sources.append(
|
||||
{
|
||||
"id": source_name,
|
||||
"kind": metadata.get("source_kind") or EARTH_BOUNDARY_SOURCE_COLLECTORS[source_name],
|
||||
"path": str(artifact_path),
|
||||
"sha256": metadata.get("sha256"),
|
||||
"featureCount": metadata.get("feature_count"),
|
||||
"license": metadata.get("license"),
|
||||
}
|
||||
)
|
||||
|
||||
if missing_sources:
|
||||
missing_text = ", ".join(missing_sources)
|
||||
raise RuntimeError(f"未就绪:{missing_text}。不会更新 Earth 国界。")
|
||||
|
||||
source_manifest = {
|
||||
"schema": "planet-earth-boundary-sources/v2",
|
||||
"sources": sources,
|
||||
"povPolicy": _read_json(POV_POLICY_PATH),
|
||||
}
|
||||
SOURCE_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with SOURCE_MANIFEST_PATH.open("w", encoding="utf-8") as f:
|
||||
json.dump(source_manifest, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
build_input_hash = _build_input_hash(source_manifest)
|
||||
await self.update_phase_progress(
|
||||
current=1,
|
||||
total=3,
|
||||
unit="steps",
|
||||
message="三类边界源已就绪",
|
||||
progress=35,
|
||||
commit=True,
|
||||
force=True,
|
||||
)
|
||||
|
||||
await self.set_phase("checking_readiness", message="正在检查 Earth 国界构建就绪状态")
|
||||
missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
|
||||
readiness_result = {
|
||||
"returncode": 0,
|
||||
"result": {
|
||||
"ready": True,
|
||||
"failures": [f"external tool not found in PATH: {tool}" for tool in missing_tools],
|
||||
"sources": sources,
|
||||
"artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
|
||||
"fallback_builder": "geojson-high-precision" if missing_tools else None,
|
||||
},
|
||||
}
|
||||
|
||||
await self.set_phase("building_tiles", message="正在检查 Earth 国界瓦片产物")
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
|
||||
if build_skipped:
|
||||
build_result = {
|
||||
"result": {
|
||||
"status": "unchanged",
|
||||
"reason": "source manifest and build config hash unchanged",
|
||||
"buildInputHash": build_input_hash,
|
||||
}
|
||||
}
|
||||
elif shutil.which("tippecanoe") and shutil.which("pmtiles"):
|
||||
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
|
||||
coastline = next(source for source in sources if source["kind"] == "coastline")
|
||||
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
|
||||
build_result = await self._run_step(
|
||||
[
|
||||
"scripts/build_earth_boundary_pmtiles.py",
|
||||
"--admin0-source",
|
||||
admin0["path"],
|
||||
"--coastline-source",
|
||||
coastline["path"],
|
||||
"--claims-source",
|
||||
claim_lines["path"],
|
||||
"--output",
|
||||
str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
|
||||
"--manifest",
|
||||
str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
|
||||
"--build-input-hash",
|
||||
build_input_hash,
|
||||
"--pov-policy",
|
||||
str(POV_POLICY_PATH.relative_to(REPO_ROOT)),
|
||||
]
|
||||
)
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
else:
|
||||
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
|
||||
coastline = next(source for source in sources if source["kind"] == "coastline")
|
||||
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
|
||||
build_result = await self._run_step(
|
||||
[
|
||||
"scripts/build_earth_boundary_china_pov_geojson.py",
|
||||
"--admin0-source",
|
||||
admin0["path"],
|
||||
"--coastline-source",
|
||||
coastline["path"],
|
||||
"--claims-source",
|
||||
claim_lines["path"],
|
||||
"--output-dir",
|
||||
str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
|
||||
"--build-input-hash",
|
||||
build_input_hash,
|
||||
]
|
||||
)
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
|
||||
pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
|
||||
pmtiles_size = PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0
|
||||
await self.update_phase_progress(
|
||||
current=2,
|
||||
total=3,
|
||||
unit="steps",
|
||||
message=(
|
||||
"边界源未变化,已跳过瓦片重建"
|
||||
if build_skipped
|
||||
else f"已生成 {boundary_manifest.get('tileProvider') or 'boundary'} 国界产物"
|
||||
),
|
||||
progress=80,
|
||||
commit=True,
|
||||
force=True,
|
||||
)
|
||||
|
||||
await self.set_phase("indexing_artifacts", message="正在登记边界瓦片产物")
|
||||
tile_counts = boundary_manifest.get("tiles", {}).get("countsByZoom", {})
|
||||
records = [
|
||||
{
|
||||
"id": "source-manifest",
|
||||
"name": "Earth boundary source manifest",
|
||||
"description": "Offline source collection manifest for Earth boundary tiles",
|
||||
"value": len(source_manifest.get("sources", [])),
|
||||
"unit": "sources",
|
||||
"metadata": {
|
||||
"manifest_path": str(SOURCE_MANIFEST_PATH.relative_to(REPO_ROOT)),
|
||||
"manifest": source_manifest,
|
||||
"collector_result": {
|
||||
"status": "loaded_from_collected_data",
|
||||
"sources": [source["id"] for source in sources],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "production-readiness",
|
||||
"name": "Earth boundary production readiness",
|
||||
"description": "Checks whether all source artifacts and boundary build tooling are available",
|
||||
"value": 1 if readiness_result.get("returncode") == 0 else 0,
|
||||
"unit": "ready",
|
||||
"metadata": {
|
||||
"result": readiness_result.get("result"),
|
||||
"returncode": readiness_result.get("returncode"),
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "boundary-manifest",
|
||||
"name": "Earth boundary tile manifest",
|
||||
"description": "Versioned static vector tile manifest for Earth country boundaries",
|
||||
"value": boundary_stats["file_count"],
|
||||
"unit": "files",
|
||||
"metadata": {
|
||||
"manifest_path": str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
|
||||
"output_dir": str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
|
||||
"size_bytes": boundary_stats["size_bytes"],
|
||||
"manifest": boundary_manifest,
|
||||
"collector_result": build_result.get("result"),
|
||||
"build_skipped": build_skipped,
|
||||
"production_target": BUILD_CONFIG["production_target"],
|
||||
"pmtiles_artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
|
||||
"pmtiles_exists": pmtiles_exists,
|
||||
},
|
||||
},
|
||||
]
|
||||
if pmtiles_exists:
|
||||
records.append(
|
||||
{
|
||||
"id": "pmtiles-artifact",
|
||||
"name": "Earth boundary PMTiles artifact",
|
||||
"description": "Single-file PMTiles/MVT artifact for Earth boundaries",
|
||||
"value": pmtiles_size,
|
||||
"unit": "bytes",
|
||||
"metadata": {
|
||||
"path": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
|
||||
"exists": pmtiles_exists,
|
||||
"provider": "pmtiles-mvt",
|
||||
},
|
||||
}
|
||||
)
|
||||
for zoom, count in sorted(tile_counts.items(), key=lambda item: int(item[0])):
|
||||
records.append(
|
||||
{
|
||||
"id": f"tile-z{zoom}",
|
||||
"name": f"Earth boundary tiles z{zoom}",
|
||||
"description": f"Generated Earth boundary tile count for zoom {zoom}",
|
||||
"value": int(count),
|
||||
"unit": "tiles",
|
||||
"metadata": {
|
||||
"zoom": int(zoom),
|
||||
"tile_count": int(count),
|
||||
"output_dir": str((BOUNDARY_OUTPUT_DIR / str(zoom)).relative_to(REPO_ROOT)),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
await self.update_phase_progress(
|
||||
current=3,
|
||||
total=3,
|
||||
unit="steps",
|
||||
message="边界瓦片产物已登记",
|
||||
progress=100,
|
||||
commit=True,
|
||||
force=True,
|
||||
)
|
||||
return records
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient
|
||||
@@ -15,6 +16,7 @@ from app.services.ai_tools.web_search import WebSearchClient, WebSearchError
|
||||
|
||||
|
||||
CREDENTIAL_GUIDES_CATEGORY = "collector_credential_guides"
|
||||
CREDENTIAL_GUIDE_PROMPT_KEY = "credential.guide"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -240,14 +242,12 @@ async def generate_credential_guide(
|
||||
guide["sources"] = []
|
||||
return guide
|
||||
|
||||
prompt = await get_effective_prompt(db, CREDENTIAL_GUIDE_PROMPT_KEY)
|
||||
response = await ai_client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title=f"Generate credential guide for {provider}",
|
||||
objective=(
|
||||
default.prompt
|
||||
+ "\n只能根据 context.search_evidence 中的来源生成教程;"
|
||||
+ "如果证据不足,明确说明需要以官方页面为准。"
|
||||
),
|
||||
objective=f"{default.prompt}\n{prompt.prompt}",
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"provider": provider,
|
||||
"current_default_guide": default.markdown,
|
||||
|
||||
@@ -301,52 +301,6 @@ async def persist_mapped_records(
|
||||
transport: str | None = None,
|
||||
) -> int:
|
||||
"""Persist validated mapped records to the destination for a target schema."""
|
||||
if target_schema == "earth_boundary_source":
|
||||
from app.models.collected_data import CollectedData
|
||||
|
||||
now = datetime.now(UTC)
|
||||
written_count = 0
|
||||
for index, record in enumerate(records):
|
||||
source_id = record.get("source_id") or record.get("sha256") or str(index)
|
||||
entity_key = f"{datasource_name}:{source_id}"
|
||||
previous_result = await db.execute(
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == datasource_name)
|
||||
.where(CollectedData.entity_key == entity_key)
|
||||
.where(CollectedData.is_current.is_(True))
|
||||
.order_by(CollectedData.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
previous = previous_result.scalar_one_or_none()
|
||||
if previous is not None:
|
||||
previous.is_current = False
|
||||
db.add(
|
||||
CollectedData(
|
||||
source=datasource_name,
|
||||
source_id=str(source_id),
|
||||
entity_key=entity_key,
|
||||
data_type=target_schema,
|
||||
name=record.get("name") or str(source_id),
|
||||
description=f"Earth boundary source artifact: {record.get('source_kind')}",
|
||||
extra_data={
|
||||
**record,
|
||||
"datasource_config_id": datasource_config_id,
|
||||
"mapping_version": mapping_version,
|
||||
"delivery_mode": delivery_mode or "polling",
|
||||
"transport": transport or "http",
|
||||
},
|
||||
collected_at=now,
|
||||
is_valid=1,
|
||||
is_current=True,
|
||||
previous_record_id=previous.id if previous else None,
|
||||
change_type="updated" if previous else "created",
|
||||
change_summary={},
|
||||
)
|
||||
)
|
||||
written_count += 1
|
||||
await db.commit()
|
||||
return written_count
|
||||
|
||||
if target_schema == "vessel_ais":
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
|
||||
671
backend/app/services/earth_boundaries.py
Normal file
671
backend/app/services/earth_boundaries.py
Normal file
@@ -0,0 +1,671 @@
|
||||
"""Earth boundary static asset service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
|
||||
SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
|
||||
BUILD_RESULT_PATH = SOURCE_OUTPUT_DIR / "build-result.json"
|
||||
BUILD_JOB_PATH = SOURCE_OUTPUT_DIR / "build-job.json"
|
||||
BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
|
||||
BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
|
||||
PMTILES_ARTIFACT_PATH = (
|
||||
REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
|
||||
)
|
||||
LEGACY_GEOJSON_PATH = REPO_ROOT / "frontend/public/earth/data/countries-admin0.min.geojson"
|
||||
POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
|
||||
LOCAL_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.local.json"
|
||||
EXAMPLE_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.example.json"
|
||||
|
||||
BOUNDARY_SOURCE_KINDS = {
|
||||
"earth_admin0_boundaries": "admin0-boundaries",
|
||||
"earth_coastline": "coastline",
|
||||
"earth_claim_lines": "claim-lines",
|
||||
}
|
||||
|
||||
DEFAULT_PUBLIC_BOUNDARY_SOURCES = {
|
||||
"earth_admin0_boundaries": {
|
||||
"displayName": "Natural Earth Admin-0 Countries",
|
||||
"sourceKind": "admin0-boundaries",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_countries.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
"earth_coastline": {
|
||||
"displayName": "Natural Earth Coastline",
|
||||
"sourceKind": "coastline",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_coastline.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
"earth_claim_lines": {
|
||||
"displayName": "Natural Earth Disputed Boundaries",
|
||||
"sourceKind": "claim-lines",
|
||||
"endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_boundary_lines_disputed_areas.geojson",
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"auth_type": "none",
|
||||
"license": "Natural Earth public domain",
|
||||
},
|
||||
}
|
||||
|
||||
BUILD_CONFIG = {
|
||||
"builder": "scripts/build_earth_boundary_pmtiles.py",
|
||||
"format": "pmtiles+mvt",
|
||||
"production_target": "pmtiles-mvt",
|
||||
}
|
||||
|
||||
|
||||
class EarthBoundaryBuildError(RuntimeError):
|
||||
def __init__(self, message: str, *, code: str = "build_failed", details: Any = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.details = details
|
||||
|
||||
|
||||
_build_job_lock = asyncio.Lock()
|
||||
_build_task: asyncio.Task | None = None
|
||||
_build_job_state: dict[str, Any] = {}
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _public_job_state() -> dict[str, Any]:
|
||||
if _build_job_state:
|
||||
return dict(_build_job_state)
|
||||
return _read_json(BUILD_JOB_PATH)
|
||||
|
||||
|
||||
def get_boundary_build_status() -> dict[str, Any]:
|
||||
return {"job": _public_job_state()}
|
||||
|
||||
|
||||
def _set_job_state(**updates: Any) -> dict[str, Any]:
|
||||
global _build_job_state
|
||||
current = dict(_build_job_state)
|
||||
current.update(updates)
|
||||
current["updated_at"] = _utc_now_iso()
|
||||
_build_job_state = current
|
||||
_write_json(BUILD_JOB_PATH, current)
|
||||
return current
|
||||
|
||||
|
||||
def _append_job_log(message: str) -> None:
|
||||
logs = list(_build_job_state.get("logs") or [])
|
||||
logs.append({"time": _utc_now_iso(), "message": message})
|
||||
_set_job_state(logs=logs[-40:])
|
||||
|
||||
|
||||
def _update_job_progress(progress: float, phase: str, message: str, **extra: Any) -> None:
|
||||
bounded_progress = max(0, min(100, int(round(progress))))
|
||||
_set_job_state(
|
||||
status="running",
|
||||
progress=bounded_progress,
|
||||
phase=phase,
|
||||
message=message,
|
||||
**extra,
|
||||
)
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def _sha256_bytes(payload: bytes) -> str:
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _stable_json_hash(payload: Any) -> str:
|
||||
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
|
||||
suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
|
||||
if suffix in {".json", ".geojson", ".zip", ".pbf"}:
|
||||
return suffix
|
||||
if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
|
||||
return ".geojson"
|
||||
if "json" in content_type:
|
||||
return ".json"
|
||||
return ".dat"
|
||||
|
||||
|
||||
def _json_feature_count(payload: Any) -> int:
|
||||
if isinstance(payload, dict) and isinstance(payload.get("features"), list):
|
||||
return len(payload["features"])
|
||||
if isinstance(payload, list):
|
||||
return len(payload)
|
||||
return 1 if payload else 0
|
||||
|
||||
|
||||
def _directory_stats(path: Path) -> dict[str, int]:
|
||||
if not path.exists():
|
||||
return {"file_count": 0, "size_bytes": 0}
|
||||
files = [item for item in path.rglob("*") if item.is_file()]
|
||||
return {"file_count": len(files), "size_bytes": sum(item.stat().st_size for item in files)}
|
||||
|
||||
|
||||
def _load_source_feature_collection(source: dict[str, Any]) -> dict[str, Any]:
|
||||
path = REPO_ROOT / source["path"]
|
||||
payload = _read_json(path)
|
||||
features = payload.get("features") if isinstance(payload, dict) else None
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": features if isinstance(features, list) else [],
|
||||
}
|
||||
|
||||
|
||||
def _write_high_precision_geojson_manifest(
|
||||
sources: list[dict[str, Any]],
|
||||
build_input_hash: str,
|
||||
missing_tools: list[str],
|
||||
) -> dict[str, Any]:
|
||||
BOUNDARY_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
|
||||
coastline = next(source for source in sources if source["kind"] == "coastline")
|
||||
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
|
||||
|
||||
admin0_payload = _load_source_feature_collection(admin0)
|
||||
coastline_payload = _load_source_feature_collection(coastline)
|
||||
claim_payload = _load_source_feature_collection(claim_lines)
|
||||
for feature in coastline_payload["features"]:
|
||||
props = feature.setdefault("properties", {})
|
||||
if isinstance(props, dict):
|
||||
props["PLANET_LAYER"] = "coastline"
|
||||
|
||||
base_payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [*admin0_payload["features"], *coastline_payload["features"]],
|
||||
}
|
||||
base_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-base.geojson"
|
||||
hover_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-hover.geojson"
|
||||
claim_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-claims.geojson"
|
||||
_write_json(base_path, base_payload)
|
||||
_write_json(hover_path, admin0_payload)
|
||||
_write_json(claim_path, claim_payload)
|
||||
|
||||
manifest = {
|
||||
"version": "natural-earth-v1",
|
||||
"builtAt": _utc_now_iso(),
|
||||
"tileProvider": "geojson-high-precision",
|
||||
"format": "geojson-directory",
|
||||
"buildInputHash": build_input_hash,
|
||||
"base": base_path.name,
|
||||
"hoverIndex": hover_path.name,
|
||||
"claimLine": claim_path.name,
|
||||
"sourceFeatureCount": {
|
||||
"admin0": len(admin0_payload["features"]),
|
||||
"coastline": len(coastline_payload["features"]),
|
||||
"claimLines": len(claim_payload["features"]),
|
||||
},
|
||||
"pmtiles": None,
|
||||
"missingTools": missing_tools,
|
||||
}
|
||||
_write_json(BOUNDARY_MANIFEST_PATH, manifest)
|
||||
return manifest
|
||||
|
||||
|
||||
def _relative(path: Path) -> str:
|
||||
return str(path.relative_to(REPO_ROOT))
|
||||
|
||||
|
||||
def load_boundary_config() -> tuple[dict[str, Any], str]:
|
||||
if LOCAL_CONFIG_PATH.exists():
|
||||
return _read_json(LOCAL_CONFIG_PATH), "local"
|
||||
return _read_json(EXAMPLE_CONFIG_PATH), "example"
|
||||
|
||||
|
||||
def save_boundary_config(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
if not isinstance(payload, dict):
|
||||
raise EarthBoundaryBuildError("Earth boundary config must be a JSON object", code="invalid_config")
|
||||
_write_json(LOCAL_CONFIG_PATH, payload)
|
||||
return get_boundary_status()
|
||||
|
||||
|
||||
def _source_configs(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
raw_sources = payload.get("collectorConfigs") or payload.get("sources") or {}
|
||||
return raw_sources if isinstance(raw_sources, dict) else {}
|
||||
|
||||
|
||||
def _is_placeholder_endpoint(endpoint: Any) -> bool:
|
||||
value = str(endpoint or "").strip()
|
||||
return not value or "example.com" in value
|
||||
|
||||
|
||||
def _source_configs_with_defaults(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
raw_sources = _source_configs(payload)
|
||||
merged: dict[str, Any] = {}
|
||||
for source_key, default_config in DEFAULT_PUBLIC_BOUNDARY_SOURCES.items():
|
||||
configured = raw_sources.get(source_key)
|
||||
if not isinstance(configured, dict) or _is_placeholder_endpoint(configured.get("endpoint")):
|
||||
merged[source_key] = dict(default_config)
|
||||
else:
|
||||
merged[source_key] = {**default_config, **configured}
|
||||
for source_key, source_config in raw_sources.items():
|
||||
if source_key not in merged:
|
||||
merged[source_key] = source_config
|
||||
return merged
|
||||
|
||||
|
||||
def _build_input_hash(source_manifest: dict[str, Any]) -> str:
|
||||
return _stable_json_hash(
|
||||
{
|
||||
"source_manifest_schema": source_manifest.get("schema"),
|
||||
"sources": [
|
||||
{
|
||||
"id": source.get("id"),
|
||||
"sha256": source.get("sha256"),
|
||||
"kind": source.get("kind"),
|
||||
}
|
||||
for source in source_manifest.get("sources", [])
|
||||
],
|
||||
"pov_policy": source_manifest.get("povPolicy"),
|
||||
"build_config": BUILD_CONFIG,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
|
||||
return (
|
||||
bool(boundary_manifest)
|
||||
and boundary_manifest.get("buildInputHash") == build_input_hash
|
||||
and boundary_manifest.get("tileProvider") == "pmtiles-mvt"
|
||||
and PMTILES_ARTIFACT_PATH.exists()
|
||||
)
|
||||
|
||||
|
||||
def get_boundary_status() -> dict[str, Any]:
|
||||
config_payload, config_source = load_boundary_config()
|
||||
effective_source_configs = _source_configs_with_defaults(config_payload)
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
|
||||
manifest_exists = BOUNDARY_MANIFEST_PATH.exists()
|
||||
high_precision_ready = (
|
||||
manifest_exists
|
||||
and (
|
||||
(
|
||||
boundary_manifest.get("tileProvider") == "pmtiles-mvt"
|
||||
and pmtiles_exists
|
||||
)
|
||||
or boundary_manifest.get("tileProvider") == "geojson-high-precision"
|
||||
)
|
||||
)
|
||||
legacy_exists = LEGACY_GEOJSON_PATH.exists()
|
||||
provider = (
|
||||
boundary_manifest.get("tileProvider")
|
||||
if high_precision_ready
|
||||
else "legacy-geojson" if legacy_exists else "missing"
|
||||
)
|
||||
return {
|
||||
"provider": provider,
|
||||
"high_precision_ready": high_precision_ready,
|
||||
"fallback_available": legacy_exists,
|
||||
"config_source": config_source,
|
||||
"config_path": _relative(LOCAL_CONFIG_PATH),
|
||||
"config_exists": LOCAL_CONFIG_PATH.exists(),
|
||||
"config": config_payload,
|
||||
"effective_default_sources": [
|
||||
source_key
|
||||
for source_key, source_config in effective_source_configs.items()
|
||||
if source_key in DEFAULT_PUBLIC_BOUNDARY_SOURCES
|
||||
and source_config.get("endpoint") == DEFAULT_PUBLIC_BOUNDARY_SOURCES[source_key]["endpoint"]
|
||||
],
|
||||
"manifest": {
|
||||
"path": _relative(BOUNDARY_MANIFEST_PATH),
|
||||
"exists": manifest_exists,
|
||||
"tileProvider": boundary_manifest.get("tileProvider"),
|
||||
"buildInputHash": boundary_manifest.get("buildInputHash"),
|
||||
"builtAt": boundary_manifest.get("builtAt"),
|
||||
},
|
||||
"pmtiles": {
|
||||
"path": _relative(PMTILES_ARTIFACT_PATH),
|
||||
"exists": pmtiles_exists,
|
||||
"size_bytes": PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0,
|
||||
},
|
||||
"legacy": {
|
||||
"path": _relative(LEGACY_GEOJSON_PATH),
|
||||
"exists": legacy_exists,
|
||||
"size_bytes": LEGACY_GEOJSON_PATH.stat().st_size if legacy_exists else 0,
|
||||
},
|
||||
"source_manifest": {
|
||||
"path": _relative(SOURCE_MANIFEST_PATH),
|
||||
"exists": SOURCE_MANIFEST_PATH.exists(),
|
||||
},
|
||||
"last_build": _read_json(BUILD_RESULT_PATH),
|
||||
"current_job": _public_job_state(),
|
||||
}
|
||||
|
||||
|
||||
async def _download_source(
|
||||
source_key: str,
|
||||
source_config: dict[str, Any],
|
||||
progress_callback: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
endpoint = str(source_config.get("endpoint") or "").strip()
|
||||
if _is_placeholder_endpoint(endpoint):
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} endpoint is not configured",
|
||||
code="source_not_configured",
|
||||
details={"source": source_key},
|
||||
)
|
||||
method = str(source_config.get("method") or "GET").upper()
|
||||
if method not in {"GET", "POST"}:
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} method must be GET or POST",
|
||||
code="invalid_config",
|
||||
details={"source": source_key, "method": method},
|
||||
)
|
||||
|
||||
if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
|
||||
payload = Path(endpoint.removeprefix("file://")).expanduser().read_bytes()
|
||||
content_type = "application/octet-stream"
|
||||
if progress_callback:
|
||||
progress_callback(1, len(payload), len(payload))
|
||||
else:
|
||||
timeout = float(source_config.get("timeout") or 120)
|
||||
headers = source_config.get("headers") if isinstance(source_config.get("headers"), dict) else {}
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
async with client.stream(method, endpoint, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
content_type = response.headers.get("content-type", "")
|
||||
total = int(response.headers.get("content-length") or 0)
|
||||
chunks = []
|
||||
downloaded = 0
|
||||
async for chunk in response.aiter_bytes():
|
||||
if not chunk:
|
||||
continue
|
||||
chunks.append(chunk)
|
||||
downloaded += len(chunk)
|
||||
if progress_callback:
|
||||
progress_callback(
|
||||
(downloaded / total) if total else None,
|
||||
downloaded,
|
||||
total,
|
||||
)
|
||||
payload = b"".join(chunks)
|
||||
|
||||
extension = _artifact_extension(endpoint, content_type, payload)
|
||||
parsed: Any = None
|
||||
if extension in {".json", ".geojson"}:
|
||||
parsed = json.loads(payload.decode("utf-8"))
|
||||
feature_count = _json_feature_count(parsed)
|
||||
if feature_count <= 0:
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} downloaded payload contains no features",
|
||||
code="empty_source",
|
||||
details={"source": source_key},
|
||||
)
|
||||
|
||||
sha256 = _sha256_bytes(payload)
|
||||
source_dir = SOURCE_OUTPUT_DIR / source_key
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact_path = source_dir / f"{sha256}{extension}"
|
||||
artifact_path.write_bytes(payload)
|
||||
return {
|
||||
"id": source_key,
|
||||
"kind": source_config.get("sourceKind") or BOUNDARY_SOURCE_KINDS[source_key],
|
||||
"path": _relative(artifact_path),
|
||||
"sha256": sha256,
|
||||
"featureCount": feature_count,
|
||||
"license": source_config.get("license"),
|
||||
}
|
||||
|
||||
|
||||
async def _run_step(args: list[str]) -> dict[str, Any]:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
*args,
|
||||
cwd=REPO_ROOT,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout_bytes, stderr_bytes = await process.communicate()
|
||||
stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
|
||||
stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
|
||||
payload: dict[str, Any] = {"stdout": stdout, "stderr": stderr, "returncode": process.returncode}
|
||||
last_line = stdout.splitlines()[-1:] or []
|
||||
if last_line:
|
||||
try:
|
||||
payload["result"] = json.loads(last_line[0])
|
||||
except json.JSONDecodeError:
|
||||
payload["result"] = last_line[0]
|
||||
if process.returncode != 0:
|
||||
raise EarthBoundaryBuildError(
|
||||
stderr or stdout or f"command failed: {' '.join(args)}",
|
||||
code="build_command_failed",
|
||||
details=payload,
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
async def build_boundary_assets(progress_callback: Any = None) -> dict[str, Any]:
|
||||
config_payload, config_source = load_boundary_config()
|
||||
|
||||
source_configs = _source_configs_with_defaults(config_payload)
|
||||
missing = [source for source in BOUNDARY_SOURCE_KINDS if source not in source_configs]
|
||||
if missing:
|
||||
raise EarthBoundaryBuildError(
|
||||
f"Missing Earth boundary source configs: {', '.join(missing)}",
|
||||
code="missing_sources",
|
||||
details={"missing": missing},
|
||||
)
|
||||
|
||||
sources = []
|
||||
source_keys = list(BOUNDARY_SOURCE_KINDS)
|
||||
for index, source_key in enumerate(source_keys):
|
||||
source_config = source_configs[source_key]
|
||||
if not isinstance(source_config, dict):
|
||||
raise EarthBoundaryBuildError(
|
||||
f"{source_key} config must be an object",
|
||||
code="invalid_config",
|
||||
details={"source": source_key},
|
||||
)
|
||||
source_start = 8 + index * 18
|
||||
source_end = source_start + 18
|
||||
if progress_callback:
|
||||
progress_callback(source_start, "download", f"正在下载 {source_key}")
|
||||
|
||||
def report_download_progress(ratio: float | None, downloaded: int, total: int) -> None:
|
||||
if not progress_callback:
|
||||
return
|
||||
if ratio is None:
|
||||
progress_callback(source_start + 8, "download", f"{source_key} 已下载 {downloaded} bytes")
|
||||
return
|
||||
progress_callback(
|
||||
source_start + (source_end - source_start) * ratio,
|
||||
"download",
|
||||
f"{source_key} 下载 {int(ratio * 100)}%",
|
||||
downloaded_bytes=downloaded,
|
||||
total_bytes=total,
|
||||
)
|
||||
|
||||
sources.append(await _download_source(source_key, source_config, report_download_progress))
|
||||
|
||||
source_manifest = {
|
||||
"schema": "planet-earth-boundary-sources/v2",
|
||||
"sources": sources,
|
||||
"povPolicy": _read_json(POV_POLICY_PATH),
|
||||
}
|
||||
if progress_callback:
|
||||
progress_callback(65, "manifest", "正在写入边界源 manifest")
|
||||
_write_json(SOURCE_MANIFEST_PATH, source_manifest)
|
||||
build_input_hash = _build_input_hash(source_manifest)
|
||||
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
|
||||
missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
|
||||
if missing_tools and not build_skipped:
|
||||
if progress_callback:
|
||||
progress_callback(82, "build", "缺少 PMTiles 工具,正在生成 GeoJSON 高清包")
|
||||
boundary_manifest = _write_high_precision_geojson_manifest(
|
||||
sources,
|
||||
build_input_hash,
|
||||
missing_tools,
|
||||
)
|
||||
result = {
|
||||
"status": "built_geojson_fallback",
|
||||
"code": "missing_tools",
|
||||
"missing_tools": missing_tools,
|
||||
"sources": sources,
|
||||
"boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH),
|
||||
"manifest": boundary_manifest,
|
||||
}
|
||||
_write_json(BUILD_RESULT_PATH, result)
|
||||
if progress_callback:
|
||||
progress_callback(96, "finalize", "GeoJSON 高清国界包已生成")
|
||||
return {**get_boundary_status(), "build": result}
|
||||
|
||||
if build_skipped:
|
||||
if progress_callback:
|
||||
progress_callback(96, "unchanged", "高精国界已是最新")
|
||||
build_result = {
|
||||
"status": "unchanged",
|
||||
"reason": "source manifest and build config hash unchanged",
|
||||
"buildInputHash": build_input_hash,
|
||||
}
|
||||
else:
|
||||
if progress_callback:
|
||||
progress_callback(72, "build", "正在构建 PMTiles/MVT")
|
||||
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
|
||||
coastline = next(source for source in sources if source["kind"] == "coastline")
|
||||
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
|
||||
build_result = await _run_step(
|
||||
[
|
||||
"scripts/build_earth_boundary_pmtiles.py",
|
||||
"--admin0-source",
|
||||
admin0["path"],
|
||||
"--coastline-source",
|
||||
coastline["path"],
|
||||
"--claims-source",
|
||||
claim_lines["path"],
|
||||
"--output",
|
||||
_relative(PMTILES_ARTIFACT_PATH),
|
||||
"--manifest",
|
||||
_relative(BOUNDARY_MANIFEST_PATH),
|
||||
"--build-input-hash",
|
||||
build_input_hash,
|
||||
"--pov-policy",
|
||||
_relative(POV_POLICY_PATH),
|
||||
]
|
||||
)
|
||||
if progress_callback:
|
||||
progress_callback(95, "finalize", "正在校验构建产物")
|
||||
|
||||
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
|
||||
boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
|
||||
result = {
|
||||
"status": "unchanged" if build_skipped else "built",
|
||||
"sources": sources,
|
||||
"source_manifest": _relative(SOURCE_MANIFEST_PATH),
|
||||
"boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH),
|
||||
"pmtiles_artifact": _relative(PMTILES_ARTIFACT_PATH),
|
||||
"pmtiles_exists": PMTILES_ARTIFACT_PATH.exists(),
|
||||
"boundary_stats": boundary_stats,
|
||||
"manifest": boundary_manifest,
|
||||
"build_result": build_result,
|
||||
}
|
||||
_write_json(BUILD_RESULT_PATH, result)
|
||||
return {**get_boundary_status(), "build": result}
|
||||
|
||||
|
||||
async def _run_boundary_build_job(job_id: str) -> None:
|
||||
def report(progress: float, phase: str, message: str, **extra: Any) -> None:
|
||||
if _build_job_state.get("id") != job_id:
|
||||
return
|
||||
_update_job_progress(progress, phase, message, **extra)
|
||||
|
||||
try:
|
||||
report(3, "prepare", "正在准备高精国界构建")
|
||||
result = await build_boundary_assets(report)
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="succeeded",
|
||||
progress=100,
|
||||
phase="complete",
|
||||
message="高精国界构建完成",
|
||||
finished_at=_utc_now_iso(),
|
||||
result={
|
||||
"provider": result.get("provider"),
|
||||
"high_precision_ready": result.get("high_precision_ready"),
|
||||
"pmtiles": result.get("pmtiles"),
|
||||
"manifest": result.get("manifest"),
|
||||
},
|
||||
)
|
||||
_append_job_log("高精国界构建完成")
|
||||
except EarthBoundaryBuildError as exc:
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="failed",
|
||||
progress=_build_job_state.get("progress", 0),
|
||||
phase="failed",
|
||||
message=str(exc),
|
||||
code=exc.code,
|
||||
details=exc.details,
|
||||
finished_at=_utc_now_iso(),
|
||||
)
|
||||
_append_job_log(str(exc))
|
||||
except Exception as exc: # pragma: no cover - defensive guard for background task
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="failed",
|
||||
progress=_build_job_state.get("progress", 0),
|
||||
phase="failed",
|
||||
message=str(exc),
|
||||
code="build_failed",
|
||||
finished_at=_utc_now_iso(),
|
||||
)
|
||||
_append_job_log(str(exc))
|
||||
|
||||
|
||||
async def start_boundary_build_job() -> dict[str, Any]:
|
||||
global _build_task
|
||||
async with _build_job_lock:
|
||||
if _build_task and not _build_task.done():
|
||||
return {"accepted": False, "job": _public_job_state()}
|
||||
job_id = uuid4().hex
|
||||
_set_job_state(
|
||||
id=job_id,
|
||||
status="queued",
|
||||
progress=0,
|
||||
phase="queued",
|
||||
message="高精国界构建已加入队列",
|
||||
logs=[],
|
||||
started_at=_utc_now_iso(),
|
||||
finished_at=None,
|
||||
code=None,
|
||||
details=None,
|
||||
)
|
||||
_append_job_log("高精国界构建已启动")
|
||||
_build_task = asyncio.create_task(_run_boundary_build_job(job_id))
|
||||
return {"accepted": True, "job": _public_job_state()}
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
import hashlib
|
||||
@@ -18,6 +18,7 @@ from bs4 import BeautifulSoup
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
|
||||
from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder
|
||||
@@ -31,6 +32,8 @@ STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
|
||||
RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS
|
||||
MAX_TARGET_INFERENCE_CONCURRENCY = 3
|
||||
TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0
|
||||
DEFAULT_NEWS_LOCALE = "zh-CN"
|
||||
NEWS_ENRICH_PROMPT_KEY = "earth.news.enrich"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -82,6 +85,11 @@ class ParsedNewsItem:
|
||||
feed_region: str
|
||||
homepage_url: str
|
||||
published_at: datetime | None
|
||||
content_language: str = "en"
|
||||
localizations: dict[str, dict[str, str]] = field(default_factory=dict)
|
||||
enrichment_status: str = "pending"
|
||||
enrichment_error: str | None = None
|
||||
enriched_at: datetime | None = None
|
||||
target_location: NewsTargetLocation | None = None
|
||||
target_resolution_stage: str = "unresolved"
|
||||
target_ai_attempted: bool = False
|
||||
@@ -365,6 +373,65 @@ def _first_json_object(text: str) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_localizations(value: Any) -> dict[str, dict[str, str]]:
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
normalized: dict[str, dict[str, str]] = {}
|
||||
for locale, payload in value.items():
|
||||
locale_key = _coerce_str(locale)
|
||||
if not locale_key or not isinstance(payload, dict):
|
||||
continue
|
||||
title = _coerce_str(payload.get("title"))
|
||||
summary = _coerce_str(payload.get("summary"))
|
||||
entry: dict[str, str] = {}
|
||||
if title:
|
||||
entry["title"] = title
|
||||
if summary:
|
||||
entry["summary"] = summary
|
||||
if entry:
|
||||
normalized[locale_key] = entry
|
||||
return normalized
|
||||
|
||||
|
||||
def _get_locale_text(
|
||||
item: ParsedNewsItem,
|
||||
key: str,
|
||||
*,
|
||||
locale: str = DEFAULT_NEWS_LOCALE,
|
||||
) -> str:
|
||||
localized = item.localizations.get(locale)
|
||||
if isinstance(localized, dict):
|
||||
value = _coerce_str(localized.get(key))
|
||||
if value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def _has_default_localization(item: ParsedNewsItem) -> bool:
|
||||
localized = item.localizations.get(DEFAULT_NEWS_LOCALE)
|
||||
if not isinstance(localized, dict):
|
||||
return False
|
||||
return bool(_coerce_str(localized.get("title")) and _coerce_str(localized.get("summary")))
|
||||
|
||||
|
||||
def apply_enrichment_patch_to_item(
|
||||
item: ParsedNewsItem,
|
||||
patch: dict[str, Any],
|
||||
) -> ParsedNewsItem:
|
||||
item.location_patch = patch
|
||||
if "content_language" in patch:
|
||||
item.content_language = _coerce_str(patch.get("content_language")) or item.content_language
|
||||
if "localizations" in patch:
|
||||
item.localizations = _normalize_localizations(patch.get("localizations"))
|
||||
if "enrichment_status" in patch:
|
||||
item.enrichment_status = _coerce_str(patch.get("enrichment_status")) or item.enrichment_status
|
||||
if "enrichment_error" in patch:
|
||||
item.enrichment_error = _coerce_str(patch.get("enrichment_error"))
|
||||
if "enriched_at" in patch:
|
||||
item.enriched_at = _parse_datetime(_coerce_str(patch.get("enriched_at")))
|
||||
return item
|
||||
|
||||
|
||||
async def _geocode_target_location(query: str) -> dict[str, Any] | None:
|
||||
return await asyncio.to_thread(_news_target_geocode, query)
|
||||
|
||||
@@ -513,42 +580,60 @@ async def _infer_news_target_location(
|
||||
item: ParsedNewsItem,
|
||||
*,
|
||||
provider_client: AIProviderClient | None,
|
||||
prompt: EffectiveAIPrompt | None = None,
|
||||
) -> NewsTargetLocation | None:
|
||||
target, _localizations = await _infer_news_enrichment(
|
||||
item,
|
||||
provider_client=provider_client,
|
||||
prompt=prompt,
|
||||
)
|
||||
return target
|
||||
|
||||
|
||||
async def _infer_news_enrichment(
|
||||
item: ParsedNewsItem,
|
||||
*,
|
||||
provider_client: AIProviderClient | None,
|
||||
prompt: EffectiveAIPrompt | None = None,
|
||||
) -> tuple[NewsTargetLocation | None, dict[str, dict[str, str]]]:
|
||||
text_hint = await _extract_target_location_from_text(item)
|
||||
content_error: str | None = None
|
||||
if text_hint is not None and text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source
|
||||
item.target_ai_attempted = False
|
||||
item.target_ai_status = "skipped_text_hint"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = f"text hint matched {text_hint.label}"
|
||||
return text_hint
|
||||
localizations: dict[str, dict[str, str]] = {}
|
||||
|
||||
if provider_client is None:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_attempted = False
|
||||
item.target_ai_status = "unavailable"
|
||||
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_attempted = False
|
||||
item.target_ai_status = "unavailable"
|
||||
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
|
||||
item.target_debug_note = (
|
||||
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
||||
)
|
||||
item.enrichment_status = "unavailable"
|
||||
item.enrichment_error = "AI provider is not configured or unavailable for earth-feed."
|
||||
return text_hint, localizations
|
||||
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_ai_attempted = True
|
||||
item.target_ai_status = "attempted"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = (
|
||||
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
||||
)
|
||||
return text_hint
|
||||
|
||||
item.target_ai_attempted = True
|
||||
item.target_ai_status = "attempted"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = (
|
||||
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
||||
)
|
||||
item.enrichment_status = "attempted"
|
||||
item.enrichment_error = None
|
||||
|
||||
prompt = prompt or await get_effective_prompt(None, NEWS_ENRICH_PROMPT_KEY)
|
||||
request = SituationalAnalysisRequest(
|
||||
title="Infer likely event location for Earth news cruise",
|
||||
objective=(
|
||||
"Return exactly one strict JSON object for the most likely physical "
|
||||
"location the news event is about. Prefer the host city when a state "
|
||||
"visit, summit, meeting, attack, or disaster is clearly centered in a "
|
||||
"known city. Fall back to the best-supported country only when a city "
|
||||
"cannot be inferred."
|
||||
),
|
||||
title="Enrich Earth news item with event location and zh-CN content",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"news_item": {
|
||||
"title": item.title,
|
||||
@@ -564,17 +649,28 @@ async def _infer_news_target_location(
|
||||
),
|
||||
},
|
||||
"required_json_schema": {
|
||||
"country": "string|null",
|
||||
"city": "string|null",
|
||||
"matched_location_name": "string|null",
|
||||
"latitude": "number|null",
|
||||
"longitude": "number|null",
|
||||
"confidence": "number from 0 to 1",
|
||||
"reasoning_summary": "short string",
|
||||
"location": {
|
||||
"country": "string|null",
|
||||
"city": "string|null",
|
||||
"matched_location_name": "string|null",
|
||||
"latitude": "number|null",
|
||||
"longitude": "number|null",
|
||||
"confidence": "number from 0 to 1",
|
||||
"reasoning_summary": "short string",
|
||||
},
|
||||
"localizations": {
|
||||
"zh-CN": {
|
||||
"title": "faithful Simplified Chinese title",
|
||||
"summary": "1-2 sentence faithful Simplified Chinese summary",
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
constraints=[
|
||||
"Return only strict JSON. Do not wrap it in markdown.",
|
||||
"For localizations, do not add facts that are absent from the RSS headline, description, source, or date.",
|
||||
"If the RSS description is thin, write a conservative summary that says only what is supported.",
|
||||
"Keep zh-CN summary concise, factual, and non-promotional.",
|
||||
"Prefer the event location, not the newsroom or publisher headquarters.",
|
||||
"When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.",
|
||||
"Use null for unknown fields instead of inventing details.",
|
||||
@@ -584,40 +680,68 @@ async def _infer_news_target_location(
|
||||
try:
|
||||
response = await provider_client.analyze(request)
|
||||
except Exception as exc:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "provider_error"
|
||||
item.target_ai_error = str(exc)
|
||||
return text_hint
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "provider_error"
|
||||
item.target_ai_error = str(exc)
|
||||
item.enrichment_status = "provider_error"
|
||||
item.enrichment_error = str(exc)
|
||||
return text_hint, localizations
|
||||
|
||||
payload = _first_json_object(response.content)
|
||||
if not isinstance(payload, dict):
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "parse_error"
|
||||
item.target_ai_error = "AI response did not contain a parseable JSON object."
|
||||
return text_hint
|
||||
if text_hint is None or not text_hint.city:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "parse_error"
|
||||
item.target_ai_error = "AI response did not contain a parseable JSON object."
|
||||
item.enrichment_status = "parse_error"
|
||||
item.enrichment_error = "AI response did not contain a parseable JSON object."
|
||||
return text_hint, localizations
|
||||
|
||||
target = await _build_target_location_from_payload(payload)
|
||||
localizations = _normalize_localizations(payload.get("localizations"))
|
||||
if not localizations:
|
||||
content_error = "AI returned no usable localizations."
|
||||
|
||||
location_payload = payload.get("location") if isinstance(payload.get("location"), dict) else payload
|
||||
if text_hint is not None and text_hint.city:
|
||||
target = text_hint
|
||||
else:
|
||||
target = await _build_target_location_from_payload(location_payload)
|
||||
if target is None:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "no_result"
|
||||
item.target_ai_error = "AI returned no usable target coordinates or geocodeable location."
|
||||
return text_hint
|
||||
if target.confidence is not None and target.confidence < 0.45:
|
||||
target = text_hint
|
||||
elif target.confidence is not None and target.confidence < 0.45:
|
||||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||||
item.target_ai_status = "low_confidence"
|
||||
item.target_ai_error = f"AI target confidence too low: {target.confidence:.2f}"
|
||||
return text_hint
|
||||
item.target_resolution_stage = target.source
|
||||
item.target_ai_status = "success"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = f"ai inferred {target.label}"
|
||||
return target
|
||||
target = text_hint
|
||||
else:
|
||||
item.target_resolution_stage = target.source
|
||||
item.target_ai_status = "success"
|
||||
item.target_ai_error = None
|
||||
item.target_debug_note = f"ai inferred {target.label}"
|
||||
|
||||
item.localizations = localizations
|
||||
if localizations and item.target_ai_status in {"success", "skipped_text_hint"}:
|
||||
item.enrichment_status = "success"
|
||||
item.enrichment_error = None
|
||||
elif localizations:
|
||||
item.enrichment_status = "content_only"
|
||||
item.enrichment_error = item.target_ai_error
|
||||
else:
|
||||
item.enrichment_status = "location_only" if target is not None else "no_result"
|
||||
item.enrichment_error = content_error or item.target_ai_error
|
||||
item.enriched_at = datetime.now(UTC) if localizations else None
|
||||
return target, localizations
|
||||
|
||||
|
||||
async def _enrich_items_with_target_locations(
|
||||
items: list[ParsedNewsItem],
|
||||
*,
|
||||
provider_client: AIProviderClient | None,
|
||||
prompt: EffectiveAIPrompt | None = None,
|
||||
) -> list[ParsedNewsItem]:
|
||||
if not items:
|
||||
return items
|
||||
@@ -626,7 +750,11 @@ async def _enrich_items_with_target_locations(
|
||||
|
||||
async def enrich(item: ParsedNewsItem) -> ParsedNewsItem:
|
||||
async with semaphore:
|
||||
target = await _infer_news_target_location(item, provider_client=provider_client)
|
||||
target = await _infer_news_target_location(
|
||||
item,
|
||||
provider_client=provider_client,
|
||||
prompt=prompt,
|
||||
)
|
||||
item.target_location = target
|
||||
return item
|
||||
|
||||
@@ -783,6 +911,20 @@ def _serialize_target(target: NewsTargetLocation | None) -> dict[str, Any] | Non
|
||||
}
|
||||
|
||||
|
||||
def _serialize_enriched_at(value: datetime | None) -> str | None:
|
||||
return value.isoformat().replace("+00:00", "Z") if value else None
|
||||
|
||||
|
||||
def _content_patch(item: ParsedNewsItem) -> dict[str, Any]:
|
||||
return {
|
||||
"content_language": item.content_language,
|
||||
"localizations": item.localizations,
|
||||
"enrichment_status": item.enrichment_status,
|
||||
"enrichment_error": item.enrichment_error,
|
||||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||||
}
|
||||
|
||||
|
||||
def build_anchor_location_patch(
|
||||
item: ParsedNewsItem,
|
||||
*,
|
||||
@@ -798,6 +940,9 @@ def build_anchor_location_patch(
|
||||
resolution_stage = item.target_resolution_stage
|
||||
ai_status = item.target_ai_status
|
||||
debug_note = item.target_debug_note
|
||||
content_patch = _content_patch(item)
|
||||
if queued and content_patch["enrichment_status"] == "pending":
|
||||
content_patch["enrichment_status"] = "queued"
|
||||
return {
|
||||
"latitude": anchor.latitude,
|
||||
"longitude": anchor.longitude,
|
||||
@@ -814,6 +959,7 @@ def build_anchor_location_patch(
|
||||
"target": None,
|
||||
"anchor": _serialize_anchor(anchor),
|
||||
},
|
||||
**content_patch,
|
||||
}
|
||||
|
||||
|
||||
@@ -836,6 +982,7 @@ def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation
|
||||
"target": _serialize_target(target),
|
||||
"anchor": _serialize_anchor(anchor),
|
||||
},
|
||||
**_content_patch(item),
|
||||
}
|
||||
|
||||
|
||||
@@ -845,6 +992,11 @@ def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
|
||||
"id": item.id,
|
||||
"title": item.title,
|
||||
"summary": item.summary,
|
||||
"content_language": item.content_language,
|
||||
"localizations": item.localizations,
|
||||
"enrichment_status": item.enrichment_status,
|
||||
"enrichment_error": item.enrichment_error,
|
||||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||||
"url": item.url,
|
||||
"source": item.source,
|
||||
"feed_name": item.feed_name,
|
||||
@@ -859,6 +1011,11 @@ def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem
|
||||
id=str(payload.get("id") or ""),
|
||||
title=str(payload.get("title") or ""),
|
||||
summary=str(payload.get("summary") or ""),
|
||||
content_language=str(payload.get("content_language") or "en"),
|
||||
localizations=_normalize_localizations(payload.get("localizations")),
|
||||
enrichment_status=str(payload.get("enrichment_status") or "pending"),
|
||||
enrichment_error=_coerce_str(payload.get("enrichment_error")),
|
||||
enriched_at=_parse_datetime(_coerce_str(payload.get("enriched_at"))),
|
||||
url=str(payload.get("url") or ""),
|
||||
source=str(payload.get("source") or ""),
|
||||
feed_name=str(payload.get("feed_name") or ""),
|
||||
@@ -875,10 +1032,15 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An
|
||||
"id": item.id,
|
||||
"title": item.title,
|
||||
"summary": item.summary,
|
||||
"content_language": item.content_language,
|
||||
"localizations": item.localizations,
|
||||
"display_title": _get_locale_text(item, "title"),
|
||||
"display_summary": _get_locale_text(item, "summary"),
|
||||
"url": item.url,
|
||||
"source": item.source,
|
||||
"feed_name": item.feed_name,
|
||||
"region": item.feed_region,
|
||||
"display_region": get_region_anchor(item.feed_region).label,
|
||||
"homepage_url": item.homepage_url,
|
||||
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
|
||||
"latitude": location_patch["latitude"],
|
||||
@@ -887,6 +1049,9 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An
|
||||
"location_source": location_patch["location_source"],
|
||||
"verified": location_patch["verified"],
|
||||
"location_meta": location_patch["location_meta"],
|
||||
"enrichment_status": item.enrichment_status,
|
||||
"enrichment_error": item.enrichment_error,
|
||||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||||
"is_focus_match": item.feed_region == active_region,
|
||||
}
|
||||
|
||||
@@ -911,6 +1076,7 @@ def _build_payload(
|
||||
"lon": lon,
|
||||
"region": active_region,
|
||||
"label": profile.label,
|
||||
"display_region": get_region_anchor(active_region).label,
|
||||
"accent": profile.accent,
|
||||
},
|
||||
"sources": _serialize_sources(sources),
|
||||
@@ -966,13 +1132,27 @@ async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> li
|
||||
get_cached_target_location_patch,
|
||||
)
|
||||
|
||||
async def enqueue_item(item: ParsedNewsItem, *, force: bool = False) -> bool:
|
||||
return await enqueue_target_location_job(build_target_location_job_payload(item), force=force)
|
||||
|
||||
async def apply_location(item: ParsedNewsItem) -> ParsedNewsItem:
|
||||
cached_patch = await get_cached_target_location_patch(item.id)
|
||||
if cached_patch:
|
||||
item.location_patch = cached_patch
|
||||
apply_enrichment_patch_to_item(item, cached_patch)
|
||||
if not _has_default_localization(item):
|
||||
queued = await enqueue_item(item, force=True)
|
||||
if queued and item.enrichment_status in {
|
||||
"pending",
|
||||
"unavailable",
|
||||
"provider_error",
|
||||
"parse_error",
|
||||
"no_result",
|
||||
"location_only",
|
||||
}:
|
||||
item.enrichment_status = "queued"
|
||||
return item
|
||||
|
||||
queued = await enqueue_target_location_job(build_target_location_job_payload(item))
|
||||
queued = await enqueue_item(item)
|
||||
item.location_patch = build_anchor_location_patch(
|
||||
item,
|
||||
queued=queued,
|
||||
@@ -991,9 +1171,16 @@ async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None:
|
||||
|
||||
await asyncio.gather(
|
||||
*(
|
||||
enqueue_target_location_job(build_target_location_job_payload(item))
|
||||
enqueue_target_location_job(
|
||||
build_target_location_job_payload(item),
|
||||
force=not _has_default_localization(item),
|
||||
)
|
||||
for item in items
|
||||
if item.location_patch is None or item.location_patch.get("verified") is False
|
||||
if (
|
||||
item.location_patch is None
|
||||
or item.location_patch.get("verified") is False
|
||||
or not _has_default_localization(item)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ class NewsTargetLocationMessage:
|
||||
|
||||
|
||||
class NewsTargetLocationQueue(Protocol):
|
||||
async def enqueue(self, *, item_id: str, payload: dict[str, Any]) -> bool:
|
||||
async def enqueue(self, *, item_id: str, payload: dict[str, Any], force: bool = False) -> bool:
|
||||
...
|
||||
|
||||
async def consume_batch(
|
||||
@@ -91,9 +91,11 @@ class RedisStreamsNewsTargetLocationQueue:
|
||||
raise
|
||||
self._group_ready = True
|
||||
|
||||
async def enqueue(self, *, item_id: str, payload: dict[str, Any]) -> bool:
|
||||
async def enqueue(self, *, item_id: str, payload: dict[str, Any], force: bool = False) -> bool:
|
||||
await self._ensure_group()
|
||||
if await self.client.exists(_result_key(item_id)):
|
||||
if force:
|
||||
await self.client.delete(_result_key(item_id), _queued_key(item_id))
|
||||
elif await self.client.exists(_result_key(item_id)):
|
||||
return False
|
||||
queued = await self.client.set(
|
||||
_queued_key(item_id),
|
||||
@@ -187,13 +189,13 @@ def get_news_target_location_queue() -> NewsTargetLocationQueue:
|
||||
return RedisStreamsNewsTargetLocationQueue()
|
||||
|
||||
|
||||
async def enqueue_target_location_job(payload: dict[str, Any]) -> bool:
|
||||
async def enqueue_target_location_job(payload: dict[str, Any], *, force: bool = False) -> bool:
|
||||
item_id = str(payload.get("id") or "")
|
||||
if not item_id:
|
||||
return False
|
||||
try:
|
||||
queue = get_news_target_location_queue()
|
||||
return await queue.enqueue(item_id=item_id, payload=payload)
|
||||
return await queue.enqueue(item_id=item_id, payload=payload, force=force)
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"Failed to enqueue Earth news target location job",
|
||||
|
||||
@@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.models.earth_news import EarthNewsItem
|
||||
from app.services.earth_news import (
|
||||
ParsedNewsItem,
|
||||
apply_enrichment_patch_to_item,
|
||||
build_anchor_location_patch,
|
||||
)
|
||||
|
||||
@@ -33,7 +34,7 @@ def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]:
|
||||
|
||||
|
||||
def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
|
||||
return ParsedNewsItem(
|
||||
item = ParsedNewsItem(
|
||||
id=record.id,
|
||||
title=record.title,
|
||||
summary=record.summary or "",
|
||||
@@ -43,8 +44,13 @@ def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
|
||||
feed_region=record.region or "global",
|
||||
homepage_url=record.homepage_url or "",
|
||||
published_at=_coerce_datetime(record.published_at),
|
||||
location_patch=_location_patch_from_record(record),
|
||||
content_language=record.content_language or "en",
|
||||
localizations=dict(record.localizations or {}),
|
||||
enrichment_status=record.enrichment_status or "pending",
|
||||
enrichment_error=record.enrichment_error,
|
||||
enriched_at=_coerce_datetime(record.enriched_at),
|
||||
)
|
||||
return apply_enrichment_patch_to_item(item, _location_patch_from_record(record))
|
||||
|
||||
|
||||
def _query_sort_key(active_region: str):
|
||||
@@ -108,6 +114,8 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
|
||||
id=item.id,
|
||||
title=item.title,
|
||||
summary=item.summary,
|
||||
content_language=item.content_language,
|
||||
localizations=dict(item.localizations or {}),
|
||||
url=item.url,
|
||||
source=item.source,
|
||||
feed_name=item.feed_name,
|
||||
@@ -122,6 +130,9 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
|
||||
location_meta=patch["location_meta"],
|
||||
first_seen_at=now,
|
||||
last_seen_at=now,
|
||||
enrichment_status=item.enrichment_status,
|
||||
enrichment_error=item.enrichment_error,
|
||||
enriched_at=item.enriched_at,
|
||||
)
|
||||
db.add(record)
|
||||
changed += 1
|
||||
@@ -136,6 +147,12 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem])
|
||||
record.homepage_url = item.homepage_url
|
||||
record.published_at = item.published_at
|
||||
record.last_seen_at = now
|
||||
if item.localizations:
|
||||
record.content_language = item.content_language
|
||||
record.localizations = dict(item.localizations or {})
|
||||
record.enrichment_status = item.enrichment_status
|
||||
record.enrichment_error = item.enrichment_error
|
||||
record.enriched_at = item.enriched_at
|
||||
changed += 1
|
||||
await db.flush()
|
||||
return changed
|
||||
@@ -161,6 +178,45 @@ async def update_earth_news_item_location(
|
||||
return True
|
||||
|
||||
|
||||
async def update_earth_news_item_enrichment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
item_id: str,
|
||||
patch: dict[str, Any],
|
||||
) -> bool:
|
||||
record = await db.get(EarthNewsItem, item_id)
|
||||
if record is None:
|
||||
return False
|
||||
if "latitude" in patch:
|
||||
record.latitude = float(patch["latitude"])
|
||||
record.longitude = float(patch["longitude"])
|
||||
record.location_label = str(patch["location_label"])
|
||||
record.location_source = str(patch["location_source"])
|
||||
record.verified = bool(patch["verified"])
|
||||
record.location_meta = dict(patch.get("location_meta") or {})
|
||||
record.resolved_at = datetime.now(UTC) if record.verified else None
|
||||
if "content_language" in patch:
|
||||
record.content_language = str(patch.get("content_language") or "en")
|
||||
if "localizations" in patch:
|
||||
record.localizations = dict(patch.get("localizations") or {})
|
||||
if "enrichment_status" in patch:
|
||||
record.enrichment_status = str(patch.get("enrichment_status") or "pending")
|
||||
if "enrichment_error" in patch:
|
||||
record.enrichment_error = patch.get("enrichment_error")
|
||||
if patch.get("enriched_at"):
|
||||
try:
|
||||
parsed_enriched_at = datetime.fromisoformat(
|
||||
str(patch["enriched_at"]).replace("Z", "+00:00")
|
||||
)
|
||||
except ValueError:
|
||||
parsed_enriched_at = datetime.now(UTC)
|
||||
record.enriched_at = _coerce_datetime(parsed_enriched_at)
|
||||
elif patch.get("localizations"):
|
||||
record.enriched_at = datetime.now(UTC)
|
||||
await db.flush()
|
||||
return True
|
||||
|
||||
|
||||
async def list_unverified_earth_news_items(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
|
||||
@@ -9,8 +9,10 @@ from app.core.logging import get_logger
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.db.session import async_session_factory
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.earth_news import (
|
||||
_infer_news_target_location,
|
||||
NEWS_ENRICH_PROMPT_KEY,
|
||||
_infer_news_enrichment,
|
||||
build_target_location_patch,
|
||||
parsed_news_item_from_job_payload,
|
||||
)
|
||||
@@ -19,7 +21,7 @@ from app.services.earth_news_queue import (
|
||||
get_news_target_location_queue,
|
||||
save_target_location_patch,
|
||||
)
|
||||
from app.services.earth_news_store import update_earth_news_item_location
|
||||
from app.services.earth_news_store import update_earth_news_item_enrichment as update_earth_news_item_location
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="earth_news")
|
||||
@@ -59,8 +61,15 @@ async def process_target_location_message(
|
||||
provider_client: AIProviderClient | None,
|
||||
) -> dict[str, Any]:
|
||||
item = parsed_news_item_from_job_payload(message.payload)
|
||||
target = await _infer_news_target_location(item, provider_client=provider_client)
|
||||
async with async_session_factory() as session:
|
||||
prompt = await get_effective_prompt(session, NEWS_ENRICH_PROMPT_KEY)
|
||||
target, localizations = await _infer_news_enrichment(
|
||||
item,
|
||||
provider_client=provider_client,
|
||||
prompt=prompt,
|
||||
)
|
||||
item.target_location = target
|
||||
item.localizations = localizations or item.localizations
|
||||
patch = build_target_location_patch(item, target)
|
||||
await save_target_location_patch(item.id, patch)
|
||||
async with async_session_factory() as session:
|
||||
|
||||
@@ -7,8 +7,11 @@ import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.countries import COUNTRY_ENTRIES, normalize_country
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.services.ai_tools.evidence_store import normalize_search_evidence
|
||||
from app.services.ai_tools.web_search import WebSearchClient, WebSearchError
|
||||
@@ -23,6 +26,8 @@ from app.services.location.text import (
|
||||
|
||||
VALID_LLM_PRECISIONS = {"precise", "site", "city"}
|
||||
DEFAULT_MIN_CONFIDENCE = 0.55
|
||||
LOCATION_NORMALIZE_PROMPT_KEY = "location.factcheck.normalize"
|
||||
LOCATION_RESOLVE_PROMPT_KEY = "location.factcheck.resolve"
|
||||
MODEL_CONFIDENCE_WEIGHT = 0.25
|
||||
_geocode_llm_city = build_default_nominatim_geocoder()
|
||||
_LLM_LOCATION_NAME_KEYS = (
|
||||
@@ -876,6 +881,7 @@ async def _repair_location_payload_from_text(
|
||||
raw_text: str,
|
||||
query: LocationQuery,
|
||||
entity_type: str,
|
||||
db: AsyncSession | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Second-pass structure repair for models that answer in prose.
|
||||
|
||||
@@ -884,12 +890,11 @@ async def _repair_location_payload_from_text(
|
||||
"""
|
||||
if not coerce_str(raw_text):
|
||||
return None
|
||||
prompt = await get_effective_prompt(db, LOCATION_NORMALIZE_PROMPT_KEY)
|
||||
request = SituationalAnalysisRequest(
|
||||
title=f"Normalize location factcheck for {entity_type}",
|
||||
objective=(
|
||||
"Convert the supplied location factcheck text into exactly one strict "
|
||||
"JSON object. Extract only facts present in the text or original query."
|
||||
),
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"location_query": _query_context(query),
|
||||
@@ -929,6 +934,7 @@ async def collect_llm_location_fallback_candidate(
|
||||
provider_client: AIProviderClient,
|
||||
query: LocationQuery,
|
||||
entity_type: str,
|
||||
db: AsyncSession | None = None,
|
||||
attempted_queries: Iterable[str] = (),
|
||||
search_evidence: list[dict[str, Any]] | None = None,
|
||||
min_confidence: float = DEFAULT_MIN_CONFIDENCE,
|
||||
@@ -946,13 +952,11 @@ async def collect_llm_location_fallback_candidate(
|
||||
attempted_queries=[attempt],
|
||||
failure_reason="LLM location factcheck skipped: no WebSearch evidence.",
|
||||
)
|
||||
prompt = await get_effective_prompt(db, LOCATION_RESOLVE_PROMPT_KEY)
|
||||
request = SituationalAnalysisRequest(
|
||||
title=f"Location factcheck fallback for {entity_type}",
|
||||
objective=(
|
||||
"Return exactly one JSON object for the most likely physical location. "
|
||||
"Use only fact-checkable public knowledge; return null fields rather "
|
||||
"than guessing when evidence is weak."
|
||||
),
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"location_query": _query_context(query),
|
||||
@@ -1001,6 +1005,7 @@ async def collect_llm_location_fallback_candidate(
|
||||
raw_text=response.content,
|
||||
query=query,
|
||||
entity_type=entity_type,
|
||||
db=db,
|
||||
)
|
||||
if payload is None:
|
||||
payload = _payload_from_free_text(response.content, query=query)
|
||||
|
||||
@@ -10,8 +10,11 @@ from app.models.alert import Alert, AlertSeverity, AlertStatus
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.services.bgp_ai_brief_store import get_latest_bgp_brief_record
|
||||
|
||||
SITUATIONAL_ALERT_BRIEF_PROMPT_KEY = "alerts.situational.brief"
|
||||
|
||||
|
||||
def _format_pairs(pairs: list[tuple[str, int]], empty_text: str = "无") -> str:
|
||||
if not pairs:
|
||||
@@ -158,10 +161,12 @@ async def build_situational_alert_brief_request(
|
||||
"latest_bgp_brief_id": latest_bgp_brief.id if latest_bgp_brief else None,
|
||||
"latest_bgp_brief_generated_at": latest_bgp_brief.generated_at if latest_bgp_brief else None,
|
||||
}
|
||||
prompt = await get_effective_prompt(db, SITUATIONAL_ALERT_BRIEF_PROMPT_KEY)
|
||||
|
||||
request = SituationalAnalysisRequest(
|
||||
title="态势告警 AI 简报",
|
||||
objective="综合系统告警、BGP incidents、BGP anomalies 与近期 BGP AI 简报,生成一份面向值班人员的态势告警简报,指出当前最需要关注的风险域、跨模块联动迹象和优先动作。",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=facts,
|
||||
constraints=[
|
||||
"明确区分事实、推断与建议。",
|
||||
|
||||
177
backend/tests/test_earth_boundaries.py
Normal file
177
backend/tests/test_earth_boundaries.py
Normal file
@@ -0,0 +1,177 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.services.collectors.registry import collector_registry
|
||||
from app.services import earth_boundaries
|
||||
|
||||
|
||||
def write_geojson(path, name="Test"):
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"name": name},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def patch_paths(monkeypatch, tmp_path):
|
||||
repo = tmp_path
|
||||
source_dir = repo / "data/earth-boundary-sources"
|
||||
boundary_dir = repo / "frontend/public/earth/data/boundaries/v1"
|
||||
pmtiles = repo / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
|
||||
legacy = repo / "frontend/public/earth/data/countries-admin0.min.geojson"
|
||||
config = repo / "config/earth-boundary-sources.local.json"
|
||||
example = repo / "config/earth-boundary-sources.example.json"
|
||||
policy = repo / "config/earth-boundary-pov-policy.china-v1.json"
|
||||
for path in (source_dir, boundary_dir, pmtiles.parent, legacy.parent, config.parent):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
policy.write_text('{"productionTileFormat":"pmtiles+mvt"}\n', encoding="utf-8")
|
||||
example.write_text('{"collectorConfigs":{}}\n', encoding="utf-8")
|
||||
monkeypatch.setattr(earth_boundaries, "REPO_ROOT", repo)
|
||||
monkeypatch.setattr(earth_boundaries, "SOURCE_OUTPUT_DIR", source_dir)
|
||||
monkeypatch.setattr(earth_boundaries, "SOURCE_MANIFEST_PATH", source_dir / "manifest.json")
|
||||
monkeypatch.setattr(earth_boundaries, "BUILD_RESULT_PATH", source_dir / "build-result.json")
|
||||
monkeypatch.setattr(earth_boundaries, "BUILD_JOB_PATH", source_dir / "build-job.json")
|
||||
monkeypatch.setattr(earth_boundaries, "BOUNDARY_OUTPUT_DIR", boundary_dir)
|
||||
monkeypatch.setattr(earth_boundaries, "BOUNDARY_MANIFEST_PATH", boundary_dir / "manifest.json")
|
||||
monkeypatch.setattr(earth_boundaries, "PMTILES_ARTIFACT_PATH", pmtiles)
|
||||
monkeypatch.setattr(earth_boundaries, "LEGACY_GEOJSON_PATH", legacy)
|
||||
monkeypatch.setattr(earth_boundaries, "LOCAL_CONFIG_PATH", config)
|
||||
monkeypatch.setattr(earth_boundaries, "EXAMPLE_CONFIG_PATH", example)
|
||||
monkeypatch.setattr(earth_boundaries, "POV_POLICY_PATH", policy)
|
||||
return {
|
||||
"repo": repo,
|
||||
"config": config,
|
||||
"legacy": legacy,
|
||||
"pmtiles": pmtiles,
|
||||
"manifest": boundary_dir / "manifest.json",
|
||||
}
|
||||
|
||||
|
||||
def test_boundary_status_uses_legacy_provider_when_pmtiles_missing(monkeypatch, tmp_path):
|
||||
paths = patch_paths(monkeypatch, tmp_path)
|
||||
write_geojson(paths["legacy"])
|
||||
|
||||
status = earth_boundaries.get_boundary_status()
|
||||
|
||||
assert status["provider"] == "legacy-geojson"
|
||||
assert status["fallback_available"] is True
|
||||
assert status["high_precision_ready"] is False
|
||||
|
||||
|
||||
def test_boundary_status_prefers_high_precision_when_manifest_and_pmtiles_exist(monkeypatch, tmp_path):
|
||||
paths = patch_paths(monkeypatch, tmp_path)
|
||||
write_geojson(paths["legacy"])
|
||||
paths["pmtiles"].write_bytes(b"pmtiles")
|
||||
paths["manifest"].write_text('{"tileProvider":"pmtiles-mvt"}\n', encoding="utf-8")
|
||||
|
||||
status = earth_boundaries.get_boundary_status()
|
||||
|
||||
assert status["provider"] == "pmtiles-mvt"
|
||||
assert status["high_precision_ready"] is True
|
||||
|
||||
|
||||
def test_save_boundary_config_writes_local_config(monkeypatch, tmp_path):
|
||||
paths = patch_paths(monkeypatch, tmp_path)
|
||||
payload = {"collectorConfigs": {"earth_admin0_boundaries": {"endpoint": "file:///tmp/a.geojson"}}}
|
||||
|
||||
status = earth_boundaries.save_boundary_config(payload)
|
||||
|
||||
assert paths["config"].exists()
|
||||
assert status["config_source"] == "local"
|
||||
assert status["config"] == payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_reports_missing_tools_after_source_artifacts(monkeypatch, tmp_path):
|
||||
paths = patch_paths(monkeypatch, tmp_path)
|
||||
source_files = {}
|
||||
for source in earth_boundaries.BOUNDARY_SOURCE_KINDS:
|
||||
source_path = paths["repo"] / f"{source}.geojson"
|
||||
write_geojson(source_path, name=source)
|
||||
source_files[source] = source_path
|
||||
paths["config"].write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collectorConfigs": {
|
||||
source: {
|
||||
"sourceKind": kind,
|
||||
"endpoint": str(source_files[source]),
|
||||
"method": "GET",
|
||||
}
|
||||
for source, kind in earth_boundaries.BOUNDARY_SOURCE_KINDS.items()
|
||||
}
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(earth_boundaries.shutil, "which", lambda _tool: None)
|
||||
progress_events = []
|
||||
|
||||
status = await earth_boundaries.build_boundary_assets(
|
||||
lambda progress, phase, message, **_extra: progress_events.append((progress, phase, message))
|
||||
)
|
||||
|
||||
assert status["provider"] == "geojson-high-precision"
|
||||
assert status["high_precision_ready"] is True
|
||||
assert (paths["repo"] / "data/earth-boundary-sources/manifest.json").exists()
|
||||
assert paths["manifest"].exists()
|
||||
assert any(phase == "download" for _progress, phase, _message in progress_events)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_boundary_build_job_records_geojson_fallback_success(monkeypatch, tmp_path):
|
||||
paths = patch_paths(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(earth_boundaries, "_build_task", None)
|
||||
monkeypatch.setattr(earth_boundaries, "_build_job_state", {})
|
||||
source_files = {}
|
||||
for source in earth_boundaries.BOUNDARY_SOURCE_KINDS:
|
||||
source_path = paths["repo"] / f"{source}.geojson"
|
||||
write_geojson(source_path, name=source)
|
||||
source_files[source] = source_path
|
||||
paths["config"].write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collectorConfigs": {
|
||||
source: {
|
||||
"sourceKind": kind,
|
||||
"endpoint": str(source_files[source]),
|
||||
"method": "GET",
|
||||
}
|
||||
for source, kind in earth_boundaries.BOUNDARY_SOURCE_KINDS.items()
|
||||
}
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(earth_boundaries.shutil, "which", lambda _tool: None)
|
||||
|
||||
response = await earth_boundaries.start_boundary_build_job()
|
||||
await earth_boundaries._build_task
|
||||
status = earth_boundaries.get_boundary_build_status()
|
||||
|
||||
assert response["accepted"] is True
|
||||
assert status["job"]["status"] == "succeeded"
|
||||
assert status["job"]["result"]["provider"] == "geojson-high-precision"
|
||||
|
||||
|
||||
def test_earth_boundary_collectors_are_not_registered_as_datasources():
|
||||
removed = set(earth_boundaries.BOUNDARY_SOURCE_KINDS) | {"earth_boundary_tiles"}
|
||||
|
||||
assert removed.isdisjoint(DEFAULT_DATASOURCES)
|
||||
for source in removed:
|
||||
assert collector_registry.get(source) is None
|
||||
@@ -158,6 +158,56 @@ async def test_enrich_items_with_target_locations_uses_ai_and_geocode(monkeypatc
|
||||
assert enriched[0].target_ai_error is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enrich_items_with_target_locations_adds_localizations(monkeypatch):
|
||||
item = ParsedNewsItem(
|
||||
id="global-scan:localized",
|
||||
title="Global leaders meet to discuss energy security",
|
||||
summary="Officials said the talks focused on supply chains and grid resilience.",
|
||||
url="https://example.com/energy-security",
|
||||
source="Example Source",
|
||||
feed_name="Global Monitor / World",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 14, 7, 13, 24, tzinfo=UTC),
|
||||
)
|
||||
|
||||
async def fake_geocode(_query: str):
|
||||
return {
|
||||
"lat": "50.1109",
|
||||
"lon": "8.6821",
|
||||
"display_name": "Frankfurt am Main, Germany",
|
||||
}
|
||||
|
||||
class FakeProviderClient:
|
||||
async def analyze(self, _request):
|
||||
class Response:
|
||||
content = (
|
||||
'{"location":{"country":"Germany","city":"Frankfurt",'
|
||||
'"matched_location_name":"Frankfurt, Germany",'
|
||||
'"latitude":null,"longitude":null,"confidence":0.77},'
|
||||
'"localizations":{"zh-CN":{"title":"全球领导人讨论能源安全",'
|
||||
'"summary":"官员表示,会谈聚焦供应链和电网韧性。"}}}'
|
||||
)
|
||||
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news._geocode_target_location", fake_geocode)
|
||||
|
||||
enriched = await _enrich_items_with_target_locations(
|
||||
[item],
|
||||
provider_client=FakeProviderClient(),
|
||||
)
|
||||
payload = _serialize_item(enriched[0], active_region="global")
|
||||
|
||||
assert payload["title"] == "Global leaders meet to discuss energy security"
|
||||
assert payload["summary"] == "Officials said the talks focused on supply chains and grid resilience."
|
||||
assert payload["localizations"]["zh-CN"]["title"] == "全球领导人讨论能源安全"
|
||||
assert payload["display_title"] == "全球领导人讨论能源安全"
|
||||
assert payload["display_summary"] == "官员表示,会谈聚焦供应链和电网韧性。"
|
||||
assert payload["enrichment_status"] == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_target_location_from_text_uses_country_hint(monkeypatch):
|
||||
item = ParsedNewsItem(
|
||||
@@ -241,7 +291,7 @@ async def test_earth_news_payload_returns_anchor_items_and_enqueues_location_job
|
||||
|
||||
enqueued_payloads = []
|
||||
|
||||
async def fake_enqueue_target_location_job(payload):
|
||||
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
||||
enqueued_payloads.append(payload)
|
||||
return True
|
||||
|
||||
@@ -260,6 +310,8 @@ async def test_earth_news_payload_returns_anchor_items_and_enqueues_location_job
|
||||
|
||||
assert len(payload["items"]) == 1
|
||||
assert payload["items"][0]["id"] == "test-feed:timeout"
|
||||
assert payload["items"][0]["display_title"] == ""
|
||||
assert payload["items"][0]["display_summary"] == ""
|
||||
assert payload["items"][0]["latitude"] == 20.0
|
||||
assert payload["items"][0]["longitude"] == 0.0
|
||||
assert payload["items"][0]["location_source"] == "region_anchor"
|
||||
@@ -359,7 +411,7 @@ async def test_earth_news_payload_initializes_empty_database_from_rss(monkeypatc
|
||||
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
||||
return [item]
|
||||
|
||||
async def fake_enqueue_target_location_job(payload):
|
||||
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
||||
enqueued.append(payload)
|
||||
return True
|
||||
|
||||
@@ -415,7 +467,7 @@ async def test_earth_news_payload_supplements_stale_database_items(monkeypatch):
|
||||
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
||||
return [old_item]
|
||||
|
||||
async def fake_enqueue_target_location_job(_payload):
|
||||
async def fake_enqueue_target_location_job(_payload, **_kwargs):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
|
||||
@@ -473,8 +525,11 @@ async def test_earth_news_payload_merges_cached_location_patch(monkeypatch):
|
||||
async def fake_get_cached_target_location_patch(_item_id):
|
||||
return cached_patch
|
||||
|
||||
async def fake_enqueue_target_location_job(_payload):
|
||||
raise AssertionError("cached items should not be enqueued")
|
||||
enqueued = []
|
||||
|
||||
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
||||
enqueued.append(payload)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
|
||||
monkeypatch.setattr("app.services.earth_news._fetch_source", fake_fetch_source)
|
||||
@@ -493,6 +548,70 @@ async def test_earth_news_payload_merges_cached_location_patch(monkeypatch):
|
||||
assert payload["items"][0]["longitude"] == 116.3912972
|
||||
assert payload["items"][0]["verified"] is True
|
||||
assert payload["items"][0]["location_source"] == "headline_location_hint"
|
||||
assert enqueued[0]["id"] == "test-feed:cached"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_news_payload_requeues_cached_failed_localization(monkeypatch):
|
||||
source = NewsFeedSource(
|
||||
id="test-feed",
|
||||
name="Test Feed",
|
||||
region="global",
|
||||
homepage_url="https://example.com",
|
||||
feed_url="https://example.com/rss.xml",
|
||||
)
|
||||
item = ParsedNewsItem(
|
||||
id="test-feed:failed-localization",
|
||||
title="Failed localization story",
|
||||
summary="English source summary.",
|
||||
url="https://example.com/failed-localization",
|
||||
source="Test Feed",
|
||||
feed_name="Test Feed",
|
||||
feed_region="global",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
)
|
||||
cached_patch = {
|
||||
"latitude": 20.0,
|
||||
"longitude": 0.0,
|
||||
"location_label": "全球",
|
||||
"location_source": "region_anchor",
|
||||
"verified": False,
|
||||
"location_meta": {"target": None, "anchor": {"region": "global"}},
|
||||
"content_language": "en",
|
||||
"localizations": {},
|
||||
"enrichment_status": "parse_error",
|
||||
"enrichment_error": "AI response did not contain a parseable JSON object.",
|
||||
"enriched_at": None,
|
||||
}
|
||||
enqueued = []
|
||||
|
||||
async def fake_fetch_source(_client, feed_source):
|
||||
return feed_source, [item], None
|
||||
|
||||
async def fake_get_cached_target_location_patch(_item_id):
|
||||
return cached_patch
|
||||
|
||||
async def fake_enqueue_target_location_job(payload, **_kwargs):
|
||||
enqueued.append(payload)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news.get_sources_for_region", lambda _region: [source])
|
||||
monkeypatch.setattr("app.services.earth_news._fetch_source", fake_fetch_source)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_queue.get_cached_target_location_patch",
|
||||
fake_get_cached_target_location_patch,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.earth_news_queue.enqueue_target_location_job",
|
||||
fake_enqueue_target_location_job,
|
||||
)
|
||||
|
||||
payload = await get_earth_news_payload(provider_client=None)
|
||||
|
||||
assert enqueued[0]["id"] == "test-feed:failed-localization"
|
||||
assert payload["items"][0]["display_title"] == ""
|
||||
assert payload["items"][0]["enrichment_status"] == "queued"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -598,6 +717,11 @@ async def test_media_news_archive_collector_maps_news_items(monkeypatch):
|
||||
location_source="headline_location_hint",
|
||||
verified=True,
|
||||
location_meta={"target": {"country": "中国", "city": "Beijing"}},
|
||||
content_language="en",
|
||||
localizations={"zh-CN": {"title": "归档新闻", "summary": "归档概要"}},
|
||||
enrichment_status="success",
|
||||
enrichment_error=None,
|
||||
enriched_at=datetime(2026, 5, 15, 3, 6, tzinfo=UTC),
|
||||
first_seen_at=datetime(2026, 5, 15, 2, 0, tzinfo=UTC),
|
||||
last_seen_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
resolved_at=datetime(2026, 5, 15, 3, 5, tzinfo=UTC),
|
||||
@@ -619,3 +743,5 @@ async def test_media_news_archive_collector_maps_news_items(monkeypatch):
|
||||
assert items[0]["city"] == "Beijing"
|
||||
assert items[0]["latitude"] == 39.9057136
|
||||
assert items[0]["metadata"]["verified"] is True
|
||||
assert "localizations" not in items[0]["metadata"]
|
||||
assert "enrichment_status" not in items[0]["metadata"]
|
||||
|
||||
92
backend/tests/test_settings_ai_prompts.py
Normal file
92
backend/tests/test_settings_ai_prompts.py
Normal file
@@ -0,0 +1,92 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.ai_tasks.prompts import (
|
||||
get_effective_prompt,
|
||||
list_effective_prompts,
|
||||
reset_prompt_override,
|
||||
save_prompt_override,
|
||||
)
|
||||
|
||||
|
||||
class _ScalarResult:
|
||||
def __init__(self, value):
|
||||
self._value = value
|
||||
|
||||
def scalar_one_or_none(self):
|
||||
return self._value
|
||||
|
||||
|
||||
class _PromptSettingsDB:
|
||||
def __init__(self, payload=None):
|
||||
self.record = SimpleNamespace(category="ai_prompts", payload=payload) if payload is not None else None
|
||||
self.added = None
|
||||
self.commits = 0
|
||||
|
||||
async def execute(self, _statement):
|
||||
return _ScalarResult(self.record)
|
||||
|
||||
def add(self, record):
|
||||
self.record = record
|
||||
self.added = record
|
||||
|
||||
async def commit(self):
|
||||
self.commits += 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_defaults_are_loaded_without_override():
|
||||
db = _PromptSettingsDB()
|
||||
|
||||
prompt = await get_effective_prompt(db, "earth.news.enrich")
|
||||
|
||||
assert prompt.key == "earth.news.enrich"
|
||||
assert prompt.is_custom is False
|
||||
assert "strict JSON" in prompt.prompt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_override_save_and_reset():
|
||||
db = _PromptSettingsDB()
|
||||
|
||||
saved = await save_prompt_override(
|
||||
db,
|
||||
"alerts.brief",
|
||||
system_prompt="system custom",
|
||||
prompt="prompt custom",
|
||||
)
|
||||
|
||||
assert saved.is_custom is True
|
||||
assert saved.system_prompt == "system custom"
|
||||
assert saved.prompt == "prompt custom"
|
||||
assert db.commits == 1
|
||||
|
||||
effective = await get_effective_prompt(db, "alerts.brief")
|
||||
assert effective.prompt == "prompt custom"
|
||||
|
||||
reset = await reset_prompt_override(db, "alerts.brief")
|
||||
assert reset.is_custom is False
|
||||
assert reset.prompt != "prompt custom"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_list_marks_custom_items():
|
||||
db = _PromptSettingsDB(
|
||||
{
|
||||
"overrides": {
|
||||
"bgp.brief": {
|
||||
"system_prompt": "",
|
||||
"prompt": "custom bgp prompt",
|
||||
"updated_at": "2026-05-16T00:00:00Z",
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
prompts = await list_effective_prompts(db)
|
||||
by_key = {prompt.key: prompt for prompt in prompts}
|
||||
|
||||
assert by_key["bgp.brief"].is_custom is True
|
||||
assert by_key["bgp.brief"].prompt == "custom bgp prompt"
|
||||
assert by_key["earth.news.enrich"].is_custom is False
|
||||
Reference in New Issue
Block a user