linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.51.0

Browse Source
This commit is contained in:
rayd1o
2026-05-11 09:49:08 +08:00
parent 455b8360d0
commit 1cb51b1172
52 changed files with 4440 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
backend/app/api/v1/bgp.py
View File

@@ -20,7 +20,11 @@ from app.services.bgp_collector_locations import (
)
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.ai_client import get_ai_provider_client
from app.services.location.llm_fallback import collect_llm_location_fallback_candidate
from app.api.v1.settings import get_web_search_client
from app.services.location.llm_fallback import (
collect_llm_location_fallback_candidate,
collect_location_search_evidence,
)
router = APIRouter()
@@ -320,21 +324,33 @@ async def collect_bgp_collector_location(
country=country,
operator=operator,
)
llm_result = None
try:
web_search_client = await get_web_search_client(db)
search_result = await collect_location_search_evidence(
web_search_client=web_search_client,
query=query,
entity_type="bgp_collector",
)
attempted_queries = [*attempted_queries, *search_result.attempted_queries]
if not search_result.evidence:
llm_failure_reason = search_result.failure_reason
raise RuntimeError(search_result.failure_reason or "no WebSearch evidence")
provider_client = await get_ai_provider_client(db)
llm_result = await collect_llm_location_fallback_candidate(
provider_client=provider_client,
query=query,
entity_type="bgp_collector",
attempted_queries=attempted_queries,
search_evidence=search_result.evidence,
)
except Exception as exc:
llm_result = None
llm_failure_reason = f"LLM location factcheck unavailable: {exc}"
attempted_queries = [
*attempted_queries,
f"llm_factcheck:bgp_collector:{collector_id or 'unknown'}",
]
if llm_failure_reason is None:
llm_failure_reason = f"LLM location factcheck unavailable: {exc}"
attempted_queries = [
*attempted_queries,
f"llm_factcheck:bgp_collector:{collector_id or 'unknown'}",
]
if llm_result is not None:
attempted_queries = [*attempted_queries, *llm_result.attempted_queries]
candidates = llm_result.candidates

367
backend/app/api/v1/settings.py
View File

@@ -1,5 +1,6 @@
from copy import deepcopy
from datetime import UTC, datetime
import os
from pathlib import Path
from typing import Optional
@@ -38,6 +39,16 @@ from app.services.datasource_connectivity import (
save_connectivity_success,
)
from app.services.ai_client import AIProviderClient, get_ai_provider_client
from app.services.ai_tools.schemas import WebSearchConfig, WebSearchProviderConfig
from app.services.ai_tools.web_search import (
WebSearchClient,
WebSearchConfigurationError,
WebSearchError,
get_web_search_provider_preset,
list_web_search_provider_presets,
normalize_web_search_provider,
provider_defaults as web_search_provider_defaults,
)
from app.services.llm_provider_catalog import (
FALLBACK_LLM_PROVIDER_PRESETS,
get_fallback_llm_provider_preset,
@@ -78,7 +89,12 @@ DEFAULT_SETTINGS = {
"providers": {},
"timeout_seconds": 60,
"retry_attempts": 2,
}
},
"web_search": {
"enabled": False,
"default_provider": "tavily",
"providers": {},
},
},
}
@@ -161,9 +177,31 @@ class BarentsWatchIntegrationUpdate(BaseModel):
clear_client_secret: bool = False
class WebSearchIntegrationUpdate(BaseModel):
enabled: bool = False
default_provider: Optional[str] = None
provider: str = Field(default="tavily", max_length=80)
base_url: str = Field(default="", max_length=500)
api_key: Optional[str] = None
max_results: int = Field(default=5, ge=1, le=20)
timeout_seconds: int = Field(default=20, ge=3, le=120)
endpoint_path: str = Field(default="", max_length=200)
search_depth: str = Field(default="basic", max_length=40)
engine: str = Field(default="google", max_length=80)
include_answer: bool = False
include_raw_content: bool = False
include_text: bool = False
categories: str = Field(default="general", max_length=120)
engines: list[str] = Field(default_factory=list)
search_path: str = Field(default="", max_length=200)
scrape_path: str = Field(default="", max_length=200)
scrape_formats: list[str] = Field(default_factory=lambda: ["markdown"])
class ExternalIntegrationsUpdate(BaseModel):
ai_provider: AIProviderIntegrationUpdate
barentswatch: BarentsWatchIntegrationUpdate
web_search: WebSearchIntegrationUpdate | None = None
def merge_with_defaults(category: str, payload: Optional[dict]) -> dict:
@@ -217,6 +255,11 @@ async def save_setting_payload(db: AsyncSession, category: str, payload: dict) -
AI_PROVIDER_ENV_FILE = Path(__file__).resolve().parents[4] / "aiprovider" / ".env"
WEB_SEARCH_ENV_FILES = (
Path(__file__).resolve().parents[4] / ".env",
Path(__file__).resolve().parents[3] / ".env",
AI_PROVIDER_ENV_FILE,
)
def _mask_secret(value: Optional[str], source: str = "") -> dict:
@@ -270,6 +313,33 @@ def _resolve_env_secret(*names: str) -> tuple[str, str]:
return "", ""
def _read_web_search_env_files() -> dict[str, str]:
values: dict[str, str] = {}
for path in WEB_SEARCH_ENV_FILES:
if not path.exists():
continue
values.update({
key: str(value)
for key, value in dotenv_values(path).items()
if value is not None
})
return values
def _resolve_web_search_env_secret(*names: str) -> tuple[str, str]:
env_file_values = _read_web_search_env_files()
for name in names:
if not name:
continue
value = env_file_values.get(name)
if value:
return value, "env_file"
value = os.environ.get(name)
if value:
return value, "env"
return "", ""
def _provider_defaults(provider: str) -> dict:
preset = _get_provider_preset(provider)
return {
@@ -438,6 +508,160 @@ def _runtime_config_from_ai_payload(ai_payload: dict) -> dict:
}
def _web_search_provider_defaults(provider: str) -> dict:
return web_search_provider_defaults(provider).model_dump()
def _normalize_web_search_payload(web_search_payload: dict | None) -> dict:
raw = dict(web_search_payload or {})
default_provider = normalize_web_search_provider(
raw.get("default_provider") or raw.get("provider")
)
providers = {
normalize_web_search_provider(provider): dict(config or {})
for provider, config in (raw.get("providers") or {}).items()
if provider
}
legacy_fields = {
key: raw.get(key)
for key in (
"base_url",
"api_key",
"max_results",
"timeout_seconds",
"endpoint_path",
"search_depth",
"engine",
"include_answer",
"include_raw_content",
"include_text",
"categories",
"engines",
"search_path",
"scrape_path",
"scrape_formats",
)
if raw.get(key) not in (None, "")
}
if legacy_fields:
providers[default_provider] = {
**providers.get(default_provider, {}),
**legacy_fields,
}
normalized_providers: dict[str, dict] = {}
for provider, config in providers.items():
provider_id = normalize_web_search_provider(provider)
normalized_providers[provider_id] = {
**_web_search_provider_defaults(provider_id),
**dict(config or {}),
"provider": provider_id,
}
if default_provider not in normalized_providers:
normalized_providers[default_provider] = _web_search_provider_defaults(default_provider)
return {
"enabled": bool(raw.get("enabled", False)),
"default_provider": default_provider,
"providers": normalized_providers,
}
def _resolve_web_search_api_key(provider: str, provider_config: dict) -> tuple[str, str]:
saved_key = provider_config.get("api_key") or ""
if saved_key:
return str(saved_key), "runtime"
preset = get_web_search_provider_preset(provider)
return _resolve_web_search_env_secret(preset.get("api_key_env") or "", "WEB_SEARCH_API_KEY")
def _build_web_search_payload(
current_payload: dict,
update: WebSearchIntegrationUpdate | None,
) -> dict:
current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {})
if update is None:
return current_web_search
provider_id = normalize_web_search_provider(update.default_provider or update.provider)
current_providers = {
provider: dict(config or {})
for provider, config in current_web_search.get("providers", {}).items()
}
current_provider = current_providers.get(provider_id) or _web_search_provider_defaults(provider_id)
current_key, current_key_source = _resolve_web_search_api_key(provider_id, current_provider)
current_key_preview = _mask_secret(current_key, current_key_source)["preview"]
provider_payload = {
**_web_search_provider_defaults(provider_id),
**current_provider,
"provider": provider_id,
"base_url": update.base_url.strip()
or current_provider.get("base_url")
or _web_search_provider_defaults(provider_id).get("base_url")
or "",
"max_results": update.max_results,
"timeout_seconds": update.timeout_seconds,
"endpoint_path": update.endpoint_path.strip() or current_provider.get("endpoint_path") or "",
"search_depth": update.search_depth.strip() or "basic",
"engine": update.engine.strip() or "google",
"include_answer": update.include_answer,
"include_raw_content": update.include_raw_content,
"include_text": update.include_text,
"categories": update.categories.strip() or "general",
"engines": update.engines,
"search_path": update.search_path.strip() or current_provider.get("search_path") or "",
"scrape_path": update.scrape_path.strip() or current_provider.get("scrape_path") or "",
"scrape_formats": update.scrape_formats or ["markdown"],
}
if not _is_secret_placeholder(update.api_key, current_key_preview):
provider_payload["api_key"] = str(update.api_key).strip()
elif current_provider.get("api_key"):
provider_payload["api_key"] = current_provider.get("api_key") or ""
else:
provider_payload["api_key"] = ""
current_providers[provider_id] = provider_payload
return {
"enabled": update.enabled,
"default_provider": provider_id,
"providers": current_providers,
}
def _runtime_config_from_web_search_payload(web_search_payload: dict) -> WebSearchConfig:
normalized = _normalize_web_search_payload(web_search_payload)
provider_id = normalized["default_provider"]
provider_config = normalized["providers"].get(provider_id) or _web_search_provider_defaults(provider_id)
api_key, _source = _resolve_web_search_api_key(provider_id, provider_config)
provider_models = {
provider: WebSearchProviderConfig(**{
**config,
"api_key": (
api_key if provider == provider_id else _resolve_web_search_api_key(provider, config)[0]
),
})
for provider, config in normalized["providers"].items()
}
return WebSearchConfig(
enabled=normalized["enabled"],
default_provider=provider_id,
provider=provider_id,
providers=provider_models,
)
async def get_runtime_web_search_config(db: AsyncSession) -> WebSearchConfig:
runtime_record = await get_setting_record(db, "external_integrations")
payload = merge_with_defaults(
"external_integrations",
runtime_record.payload if runtime_record else None,
)
return _runtime_config_from_web_search_payload(payload.get("web_search") or {})
async def get_web_search_client(db: AsyncSession) -> WebSearchClient:
return WebSearchClient(await get_runtime_web_search_config(db))
async def get_runtime_ai_provider_config(db: AsyncSession) -> dict:
runtime_record = await get_setting_record(db, "external_integrations")
payload = merge_with_defaults(
@@ -459,6 +683,7 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
runtime_setting.payload if runtime_setting else None,
)
normalized_ai = _normalize_ai_provider_payload(raw_payload.get("ai_provider") or {})
normalized_web_search = _normalize_web_search_payload(raw_payload.get("web_search") or {})
default_provider = normalized_ai["default_provider"]
providers_payload: dict[str, dict] = {}
for provider in sorted({
@@ -480,6 +705,32 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
"source": "runtime" if provider_config.get("api_key") else (api_key_source or "preset"),
}
display_llm_config = providers_payload.get(default_provider) or _provider_defaults(default_provider)
web_search_providers_payload: dict[str, dict] = {}
for provider in sorted({
*[preset["provider"] for preset in list_web_search_provider_presets()],
*normalized_web_search["providers"].keys(),
normalized_web_search["default_provider"],
}):
provider_id = normalize_web_search_provider(provider)
provider_config = (
normalized_web_search["providers"].get(provider_id)
or _web_search_provider_defaults(provider_id)
)
api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config)
web_search_providers_payload[provider_id] = {
**{
key: value
for key, value in provider_config.items()
if key != "api_key"
},
"provider": provider_id,
"api_key": _mask_secret(api_key, api_key_source),
"source": "runtime" if provider_config.get("api_key") else (api_key_source or "preset"),
}
display_web_search_config = (
web_search_providers_payload.get(normalized_web_search["default_provider"])
or _web_search_provider_defaults(normalized_web_search["default_provider"])
)
barentswatch_record = await get_barentswatch_config_record(db)
barentswatch_auth = barentswatch_record.auth_config if barentswatch_record else {}
barentswatch_auth = barentswatch_auth or {}
@@ -509,6 +760,28 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
),
"source": resolved_barentswatch.credential_source,
},
"web_search": {
"enabled": normalized_web_search["enabled"],
"default_provider": normalized_web_search["default_provider"],
"provider": normalized_web_search["default_provider"],
"base_url": display_web_search_config.get("base_url") or "",
"api_key": display_web_search_config.get("api_key") or _mask_secret(None),
"providers": web_search_providers_payload,
"max_results": int(display_web_search_config.get("max_results") or 5),
"timeout_seconds": int(display_web_search_config.get("timeout_seconds") or 20),
"endpoint_path": display_web_search_config.get("endpoint_path") or "",
"search_depth": display_web_search_config.get("search_depth") or "basic",
"engine": display_web_search_config.get("engine") or "google",
"include_answer": bool(display_web_search_config.get("include_answer", False)),
"include_raw_content": bool(display_web_search_config.get("include_raw_content", False)),
"include_text": bool(display_web_search_config.get("include_text", False)),
"categories": display_web_search_config.get("categories") or "general",
"engines": display_web_search_config.get("engines") or [],
"search_path": display_web_search_config.get("search_path") or "",
"scrape_path": display_web_search_config.get("scrape_path") or "",
"scrape_formats": display_web_search_config.get("scrape_formats") or ["markdown"],
"source": "runtime" if runtime_setting else "env",
},
}
@@ -518,8 +791,13 @@ async def save_external_integrations_payload(
) -> dict:
current_payload = await get_setting_payload(db, "external_integrations")
ai_payload = _build_ai_provider_payload(current_payload, update.ai_provider)
web_search_payload = _build_web_search_payload(current_payload, update.web_search)
await save_setting_payload(db, "external_integrations", {"ai_provider": ai_payload})
await save_setting_payload(
db,
"external_integrations",
{"ai_provider": ai_payload, "web_search": web_search_payload},
)
default_endpoint = get_data_sources_config().get_yaml_url("barentswatch_vessels")
barentswatch_record = await get_barentswatch_config_record(db)
@@ -754,7 +1032,12 @@ async def connect_ai_provider_integration(
constraints=["回复尽量简短。"],
)
)
await save_setting_payload(db, "external_integrations", {"ai_provider": draft_ai_payload})
current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {})
await save_setting_payload(
db,
"external_integrations",
{"ai_provider": draft_ai_payload, "web_search": current_web_search},
)
return {
"success": True,
"connected": True,
@@ -799,6 +1082,74 @@ async def reveal_ai_provider_secrets(
}
@router.get("/integrations/web-search/presets")
async def get_web_search_presets(
current_user: User = Depends(get_current_user),
):
return {"data": list_web_search_provider_presets()}
@router.get("/integrations/web-search/secrets")
async def reveal_web_search_secrets(
provider: str = Query(default=""),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
current_payload = await get_setting_payload(db, "external_integrations")
web_search_payload = _normalize_web_search_payload(current_payload.get("web_search") or {})
provider_id = normalize_web_search_provider(provider or web_search_payload["default_provider"])
provider_config = (
web_search_payload["providers"].get(provider_id)
or _web_search_provider_defaults(provider_id)
)
api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config)
return {
"provider": provider_id,
"api_key": api_key,
"api_key_source": api_key_source,
}
@router.post("/integrations/web-search/connect")
async def connect_web_search_integration(
payload: WebSearchIntegrationUpdate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
current_payload = await get_setting_payload(db, "external_integrations")
draft_web_search_payload = _build_web_search_payload(current_payload, payload)
runtime_config = _runtime_config_from_web_search_payload(draft_web_search_payload)
client = WebSearchClient(runtime_config)
try:
results = await client.test_connection()
return {
"success": True,
"connected": True,
"message": "WebSearch 连接成功。",
"provider": runtime_config.default_provider,
"results": [item.model_dump(mode="json") for item in results[:3]],
}
except WebSearchConfigurationError as exc:
return {
"success": False,
"connected": False,
"message": str(exc),
}
except WebSearchError as exc:
return {
"success": False,
"connected": False,
"message": str(exc),
}
except Exception as exc:
return {
"success": False,
"connected": False,
"message": f"WebSearch 连接测试失败: {exc}",
}
@router.get("/credential-guides/{provider}")
async def read_credential_guide(
provider: str,
@@ -819,7 +1170,15 @@ async def generate_provider_credential_guide(
ai_client: AIProviderClient = Depends(get_ai_provider_client),
):
try:
return {"guide": await generate_credential_guide(db, provider, ai_client)}
web_search_client = await get_web_search_client(db)
return {
"guide": await generate_credential_guide(
db,
provider,
ai_client,
web_search_client,
)
}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc

30
backend/app/api/v1/visualization.py
View File

@@ -38,8 +38,12 @@ from app.services.compute_center_locations import (
upsert_compute_center_location,
)
from app.services.ai_client import get_ai_provider_client
from app.api.v1.settings import get_web_search_client
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
from app.services.location.llm_fallback import collect_llm_location_fallback_candidate
from app.services.location.llm_fallback import (
collect_llm_location_fallback_candidate,
collect_location_search_evidence,
)
from app.services.persistent_logs import record_system_log
from app.services.vessel_ais_aggregation import (
build_field_conflict_candidates,
@@ -1879,21 +1883,33 @@ async def collect_compute_center_location(
city=city,
country=country,
)
llm_result = None
try:
web_search_client = await get_web_search_client(db)
search_result = await collect_location_search_evidence(
web_search_client=web_search_client,
query=query,
entity_type="compute_center",
)
attempted_queries = [*attempted_queries, *search_result.attempted_queries]
if not search_result.evidence:
llm_failure_reason = search_result.failure_reason
raise RuntimeError(search_result.failure_reason or "no WebSearch evidence")
provider_client = await get_ai_provider_client(db)
llm_result = await collect_llm_location_fallback_candidate(
provider_client=provider_client,
query=query,
entity_type="compute_center",
attempted_queries=attempted_queries,
search_evidence=search_result.evidence,
)
except Exception as exc:
llm_result = None
llm_failure_reason = f"LLM location factcheck unavailable: {exc}"
attempted_queries = [
*attempted_queries,
f"llm_factcheck:compute_center:{name or source_id or 'unknown'}",
]
if llm_failure_reason is None:
llm_failure_reason = f"LLM location factcheck unavailable: {exc}"
attempted_queries = [
*attempted_queries,
f"llm_factcheck:compute_center:{name or source_id or 'unknown'}",
]
if llm_result is not None:
attempted_queries = [*attempted_queries, *llm_result.attempted_queries]
candidates = llm_result.candidates