release: bump version to 0.62.0
Some checks failed
ci / backend (push) Has been cancelled
ci / frontend (push) Has been cancelled
ci / delivery (push) Has been cancelled
release / images (push) Has been cancelled

This commit is contained in:
linkong
2026-05-21 01:37:32 +08:00
parent 5c65ee24d6
commit fbca381512
138 changed files with 21303 additions and 5721 deletions

View File

@@ -5,7 +5,7 @@ from datetime import datetime
import base64
import json
import re
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from sqlalchemy import delete, select, func
from sqlalchemy.ext.asyncio import AsyncSession
from pydantic import BaseModel, Field
@@ -45,14 +45,70 @@ from app.services.custom_datasource_runtime import (
DATASOURCE_MAPPING_PROMPT_KEY = "datasource.mapping"
from app.services.datasource_connectivity import (
_resolve_aisstream_api_key,
_resolve_spacetrack_credentials_with_override,
get_builtin_connection_status,
save_connectivity_success,
strip_connectivity_validation,
test_builtin_connectivity,
)
from app.services.barentswatch import resolve_barentswatch_config
from app.services.persistent_logs import record_audit_log
router = APIRouter()
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
def _user_role_value(user: User) -> str:
role = getattr(user, "role", "")
return str(getattr(role, "value", role) or "").lower()
def _user_display_name(user: User) -> str:
return str(getattr(user, "username", None) or getattr(user, "email", None) or getattr(user, "id", ""))
async def _record_datasource_secret_reveal(
*,
current_user: User,
request: Request,
target_id: str,
result: str,
details: dict[str, Any],
) -> None:
await record_audit_log(
action="datasource_config.secret.reveal",
actor_id=getattr(current_user, "id", None),
actor_name=_user_display_name(current_user),
target_type="datasource_config_secret",
target_id=target_id,
result=result,
ip=request.client.host if request.client else None,
details=details,
)
async def _ensure_datasource_secret_reveal_allowed(
current_user: User,
request: Request,
target_id: str,
details: dict[str, Any],
) -> None:
if _user_role_value(current_user) in SECRET_REVEAL_ROLES:
return
await _record_datasource_secret_reveal(
current_user=current_user,
request=request,
target_id=target_id,
result="denied",
details={**details, "role": _user_role_value(current_user)},
)
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only administrators can reveal datasource credentials",
)
def _default_builtin_config(name: str) -> dict[str, Any]:
return {"timeout": 30, "retry": 3}
@@ -389,10 +445,14 @@ async def list_all_datasources(
db_config = db_configs.get(name)
default_config = _default_builtin_config(name)
default_url = yaml_url
db_auth_config = db_config.auth_config or {} if db_config else {}
result.append(
{
"name": name,
"requires_credentials": bool(metadata.get("requires_credentials", False)),
"credential_provider": metadata.get("credential_provider"),
"credential_status": metadata.get("credential_status", "none"),
"default_url": default_url,
"endpoint": db_config.endpoint if db_config else default_url,
"is_overridden": db_config is not None and db_config.endpoint != yaml_url
@@ -401,10 +461,20 @@ async def list_all_datasources(
"is_active": db_config.is_active if db_config else True,
"source_type": db_config.source_type if db_config else _default_builtin_source_type(name),
"auth_type": db_config.auth_type if db_config else "none",
"auth_config": {
"client_id": db_auth_config.get("client_id") or "",
"username": db_auth_config.get("username") or "",
"key_name": db_auth_config.get("key_name") or db_auth_config.get("param_name") or "",
"param_name": db_auth_config.get("param_name") or db_auth_config.get("key_name") or "",
"location": db_auth_config.get("location") or db_auth_config.get("in") or "",
"in": db_auth_config.get("in") or db_auth_config.get("location") or "",
},
"auth_configured": {
"api_key": bool((db_config.auth_config or {}).get("api_key"))
if db_config
else False,
"api_key": bool(db_auth_config.get("api_key")),
"client_id": bool(db_auth_config.get("client_id")),
"client_secret": bool(db_auth_config.get("client_secret")),
"username": bool(db_auth_config.get("username")),
"password": bool(db_auth_config.get("password")),
},
"headers": db_config.headers if db_config else {},
"config": strip_connectivity_validation(db_config.config if db_config else default_config),
@@ -418,6 +488,96 @@ async def list_all_datasources(
return {"total": len(result), "data": result}
@router.get("/configs/secrets")
async def reveal_builtin_config_secrets(
request: Request,
name: str = Query(..., min_length=1),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Reveal configured built-in datasource credentials for admin editing."""
source = name.strip()
metadata = DEFAULT_DATASOURCES.get(source)
if not metadata or not metadata.get("requires_credentials"):
raise HTTPException(status_code=404, detail="Credentialed datasource config not found")
provider = str(metadata.get("credential_provider") or "")
target_id = f"datasource_config:{source}"
await _ensure_datasource_secret_reveal_allowed(
current_user,
request,
target_id,
{"source": source, "provider": provider},
)
result = await db.execute(select(DataSourceConfig).where(DataSourceConfig.name == source))
record = result.scalar_one_or_none()
auth_config = dict(record.auth_config or {}) if record else {}
payload: dict[str, Any] = {
"name": source,
"provider": provider,
}
details: dict[str, Any] = {"source": source, "provider": provider}
if provider == "barentswatch":
resolved = await resolve_barentswatch_config(db)
client_id = str(auth_config.get("client_id") or resolved.client_id or "")
client_secret = str(auth_config.get("client_secret") or resolved.client_secret or "")
source_label = "datasource_config" if auth_config.get("client_id") or auth_config.get("client_secret") else resolved.credential_source
payload.update(
{
"client_id": client_id,
"client_secret": client_secret,
"client_id_source": source_label if client_id else "missing",
"client_secret_source": source_label if client_secret else "missing",
}
)
details.update(
{
"client_id_configured": bool(client_id),
"client_secret_configured": bool(client_secret),
"credential_source": source_label,
}
)
elif provider == "aisstream":
api_key, api_key_source = await _resolve_aisstream_api_key(db)
payload.update({"api_key": api_key, "api_key_source": api_key_source})
details.update({"api_key_configured": bool(api_key), "api_key_source": api_key_source})
elif provider == "spacetrack":
if auth_config.get("username") or auth_config.get("password"):
username = str(auth_config.get("username") or "")
password = str(auth_config.get("password") or "")
credential_source = "datasource_config"
else:
username, password, credential_source = _resolve_spacetrack_credentials_with_override()
payload.update(
{
"username": username,
"password": password,
"username_source": credential_source if username else "missing",
"password_source": credential_source if password else "missing",
}
)
details.update(
{
"username_configured": bool(username),
"password_configured": bool(password),
"credential_source": credential_source,
}
)
else:
raise HTTPException(status_code=400, detail="Datasource credential provider is not supported")
await _record_datasource_secret_reveal(
current_user=current_user,
request=request,
target_id=target_id,
result="success",
details=details,
)
return payload
@router.get("/configs/{config_id}")
async def get_config(
config_id: int,

View File

@@ -7,6 +7,7 @@ from pydantic import BaseModel, Field
from sqlalchemy import func, or_, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.cache import cache
from app.core.time import to_iso8601_utc
from app.core.security import get_current_user
from app.core.data_sources import get_data_sources_config
@@ -26,6 +27,7 @@ from app.services.scheduler import (
run_collector_now,
sync_datasource_job,
)
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
router = APIRouter()
STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90
@@ -235,6 +237,8 @@ async def _load_datasource_endpoint_overrides(
async def _load_datasource_list_context(
db: AsyncSession,
datasources: list[DataSource],
*,
include_endpoint: bool = True,
) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, str]]:
datasource_ids = [datasource.id for datasource in datasources]
sources = [datasource.source for datasource in datasources]
@@ -260,10 +264,65 @@ async def _load_datasource_list_context(
running_tasks = await _load_latest_running_tasks(db, datasource_ids)
latest_tasks = await _load_latest_tasks(db, datasource_ids)
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources) if include_endpoint else {}
return running_tasks, latest_tasks, endpoint_overrides
def serialize_datasource_row(
datasource: DataSource,
*,
running_tasks: dict[int, CollectionTask],
latest_tasks: dict[int, CollectionTask],
record_counts: dict[str, int],
endpoint_overrides: dict[str, str],
config,
include_endpoint: bool,
) -> dict:
running_task = running_tasks.get(datasource.id)
latest_task = latest_tasks.get(datasource.id)
display_task = running_task or latest_task
endpoint = None
if include_endpoint:
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None)
last_status = datasource.last_status or (latest_task.status if latest_task else None)
collected_records = record_counts.get(datasource.source, 0)
row = {
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
**datasource_metadata(datasource.source),
"product": datasource_product_key(datasource),
"module": datasource.module,
"priority": datasource.priority,
"frequency": format_frequency_label(datasource.frequency_minutes),
"frequency_minutes": datasource.frequency_minutes,
"is_active": datasource.is_active,
"collector_class": datasource.collector_class,
"last_run": to_iso8601_utc(last_run_at),
"last_run_at": to_iso8601_utc(last_run_at),
"last_status": last_status,
"is_running": running_task is not None,
"task_id": display_task.id if display_task else None,
"progress": display_task.progress if display_task else None,
"phase": display_task.phase if display_task else None,
"phase_progress": display_task.phase_progress if display_task else None,
"phase_message": display_task.phase_message if display_task else None,
"phase_current": display_task.phase_current if display_task else None,
"phase_total": display_task.phase_total if display_task else None,
"phase_unit": display_task.phase_unit if display_task else None,
"records_processed": display_task.records_processed if display_task else None,
"total_records": display_task.total_records if display_task else None,
"error_message": display_task.error_message if display_task else None,
"collected_records": collected_records,
"has_collected_data": collected_records > 0,
}
if include_endpoint:
row["endpoint"] = endpoint
return row
def _apply_datasource_query_filters(
query,
*,
@@ -658,6 +717,7 @@ async def list_datasources(
collected: Optional[bool] = None,
credential_status: Optional[str] = None,
q: Optional[str] = None,
include_endpoint: bool = True,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
@@ -676,7 +736,11 @@ async def list_datasources(
collector_list = []
config = get_data_sources_config()
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(
db,
datasources,
include_endpoint=include_endpoint,
)
record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
datasources = _filter_datasources_in_memory(
datasources,
@@ -689,46 +753,16 @@ async def list_datasources(
credential_status=credential_status,
)
for datasource in datasources:
running_task = running_tasks.get(datasource.id)
latest_task = latest_tasks.get(datasource.id)
display_task = running_task or latest_task
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None)
last_status = datasource.last_status or (latest_task.status if latest_task else None)
collected_records = record_counts.get(datasource.source, 0)
collector_list.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
**datasource_metadata(datasource.source),
"product": datasource_product_key(datasource),
"module": datasource.module,
"priority": datasource.priority,
"frequency": format_frequency_label(datasource.frequency_minutes),
"frequency_minutes": datasource.frequency_minutes,
"is_active": datasource.is_active,
"collector_class": datasource.collector_class,
"endpoint": endpoint,
"last_run": to_iso8601_utc(last_run_at),
"last_run_at": to_iso8601_utc(last_run_at),
"last_status": last_status,
"is_running": running_task is not None,
"task_id": display_task.id if display_task else None,
"progress": display_task.progress if display_task else None,
"phase": display_task.phase if display_task else None,
"phase_progress": display_task.phase_progress if display_task else None,
"phase_message": display_task.phase_message if display_task else None,
"phase_current": display_task.phase_current if display_task else None,
"phase_total": display_task.phase_total if display_task else None,
"phase_unit": display_task.phase_unit if display_task else None,
"records_processed": display_task.records_processed if display_task else None,
"total_records": display_task.total_records if display_task else None,
"error_message": display_task.error_message if display_task else None,
"collected_records": collected_records,
"has_collected_data": collected_records > 0,
}
serialize_datasource_row(
datasource,
running_tasks=running_tasks,
latest_tasks=latest_tasks,
record_counts=record_counts,
endpoint_overrides=endpoint_overrides,
config=config,
include_endpoint=include_endpoint,
)
)
return {"total": len(collector_list), "data": collector_list}
@@ -785,6 +819,51 @@ async def trigger_datasource_batch(
return await _trigger_datasource_batch(db, datasources, force=payload.force)
@router.get("/snapshots")
async def list_datasource_snapshots(
source_id: Optional[str] = None,
current_only: Optional[bool] = None,
limit: int = Query(default=100, ge=1, le=500),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
query = (
select(DataSnapshot, DataSource.name, DataSource.module)
.outerjoin(DataSource, DataSource.id == DataSnapshot.datasource_id)
.order_by(DataSnapshot.created_at.desc().nullslast(), DataSnapshot.id.desc())
.limit(limit)
)
if source_id:
query = query.where(DataSnapshot.source == source_id)
if current_only is not None:
query = query.where(DataSnapshot.is_current.is_(current_only))
result = await db.execute(query)
rows = []
for snapshot, datasource_name, datasource_module in result.all():
rows.append(
{
"id": snapshot.id,
"datasource_id": snapshot.datasource_id,
"datasource_name": datasource_name,
"module": datasource_module,
"task_id": snapshot.task_id,
"source": snapshot.source,
"snapshot_key": snapshot.snapshot_key,
"reference_date": to_iso8601_utc(snapshot.reference_date),
"started_at": to_iso8601_utc(snapshot.started_at),
"completed_at": to_iso8601_utc(snapshot.completed_at),
"record_count": snapshot.record_count,
"status": snapshot.status,
"is_current": snapshot.is_current,
"parent_snapshot_id": snapshot.parent_snapshot_id,
"summary": snapshot.summary or {},
"created_at": to_iso8601_utc(snapshot.created_at),
}
)
return {"total": len(rows), "data": rows}
@router.get("/{source_id}")
async def get_datasource(
source_id: str,
@@ -813,6 +892,37 @@ async def get_datasource(
}
@router.get("/{source_id}/row")
async def get_datasource_row(
source_id: str,
include_endpoint: bool = True,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
datasource = await get_datasource_record(db, source_id)
if not datasource:
raise HTTPException(status_code=404, detail="Data source not found")
config = get_data_sources_config()
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(
db,
[datasource],
include_endpoint=include_endpoint,
)
record_counts = await _load_collected_record_counts(db, [datasource.source])
return {
"data": serialize_datasource_row(
datasource,
running_tasks=running_tasks,
latest_tasks=latest_tasks,
record_counts=record_counts,
endpoint_overrides=endpoint_overrides,
config=config,
include_endpoint=include_endpoint,
)
}
@router.post("/{source_id}/enable")
async def enable_datasource(
source_id: str,
@@ -960,6 +1070,29 @@ async def clear_datasource_data(
}
@router.delete("/{source_id}/cache")
async def clear_datasource_cache(
source_id: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
datasource = await get_datasource_record(db, source_id)
if not datasource:
raise HTTPException(status_code=404, detail="Data source not found")
earth_deleted_count = invalidate_earth_layer_cache_for_source(datasource.source)
dashboard_deleted_count = int(cache.delete("dashboard:stats")) + int(cache.delete("dashboard:summary"))
deleted_count = earth_deleted_count + dashboard_deleted_count
return {
"status": "success",
"message": f"Cleared {deleted_count} cache keys for data source '{datasource.name}'",
"deleted_count": deleted_count,
"earth_layer_deleted_count": earth_deleted_count,
"dashboard_deleted_count": dashboard_deleted_count,
}
@router.get("/{source_id}/task-status")
async def get_task_status(
source_id: str,

View File

@@ -4,7 +4,8 @@ import os
from pathlib import Path
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
import httpx
from pydantic import BaseModel, EmailStr, Field
from dotenv import dotenv_values
from sqlalchemy import select
@@ -64,10 +65,12 @@ from app.services.llm_provider_catalog import (
)
from app.services.scheduler import sync_datasource_job
from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings
from app.services.persistent_logs import record_audit_log
router = APIRouter()
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5
AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test"
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
DEFAULT_SETTINGS = {
"system": {
@@ -130,6 +133,70 @@ DEFAULT_SETTINGS = {
}
def _user_role_value(user: User) -> str:
role = getattr(user, "role", "")
return role.value if hasattr(role, "value") else str(role or "")
def _user_display_name(user: User) -> str | None:
return getattr(user, "username", None) or getattr(user, "email", None)
def _request_client_ip(request: Request | None) -> str | None:
if request is None or request.client is None:
return None
return request.client.host
def _can_reveal_integration_secrets(user: User) -> bool:
return _user_role_value(user) in SECRET_REVEAL_ROLES
async def _record_integration_secret_reveal(
*,
current_user: User,
request: Request | None,
target_id: str,
result: str,
details: dict,
) -> None:
await record_audit_log(
action="settings.integration_secret.reveal",
actor_id=getattr(current_user, "id", None),
actor_name=_user_display_name(current_user),
target_type="integration_secret",
target_id=target_id,
result=result,
ip=_request_client_ip(request),
details=details,
)
async def _ensure_secret_reveal_allowed(
*,
current_user: User,
request: Request | None,
target_id: str,
details: dict | None = None,
) -> None:
if _can_reveal_integration_secrets(current_user):
return
await _record_integration_secret_reveal(
current_user=current_user,
request=request,
target_id=target_id,
result="denied",
details={
**(details or {}),
"role": _user_role_value(current_user),
},
)
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only administrators can reveal integration secrets",
)
class SystemSettingsUpdate(BaseModel):
system_name: str = "智能星球"
refresh_interval: int = Field(default=60, ge=10, le=3600)
@@ -353,9 +420,10 @@ def _get_provider_preset(provider: str) -> dict:
"provider": provider,
"provider_api": "openai-completions",
"base_url": "",
"model": "",
"models": [],
"api_key_env": "",
"model": "",
"models": [],
"model_provider_apis": {},
"api_key_env": "",
}
@@ -377,6 +445,9 @@ def _resolve_env_secret(*names: str) -> tuple[str, str]:
value = env_file_values.get(name)
if value:
return value, "env_file"
value = os.environ.get(name)
if value:
return value, "env"
return "", ""
@@ -419,9 +490,16 @@ def _provider_defaults(provider: str) -> dict:
1200 if preset.get("provider_api") == "anthropic-messages" else 4096
),
"anthropic_version": "2023-06-01",
"model_provider_apis": preset.get("model_provider_apis") or {},
}
def _selected_ai_env_provider() -> str:
env_file_values = _read_ai_provider_env_file()
provider = env_file_values.get("AI_PROVIDER") or os.environ.get("AI_PROVIDER") or "minimax"
return _normalize_provider_id(provider)
def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict:
raw = dict(ai_payload or {})
default_provider = _normalize_provider_id(raw.get("default_provider") or raw.get("provider"))
@@ -440,6 +518,7 @@ def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict:
"api_key",
"max_tokens",
"anthropic_version",
"model_provider_apis",
)
if raw.get(key) not in (None, "")
}
@@ -477,7 +556,12 @@ def _resolve_provider_api_key(provider: str, provider_config: dict) -> tuple[str
return str(saved_key), "runtime"
preset = _get_provider_preset(provider)
api_key_env = preset.get("api_key_env") or ""
return _resolve_env_secret(api_key_env, "AI_API_KEY")
value, source = _resolve_env_secret(api_key_env)
if value:
return value, source
if _normalize_provider_id(provider) == _selected_ai_env_provider():
return _resolve_env_secret("AI_API_KEY")
return "", ""
def _resolve_service_token(ai_payload: dict) -> tuple[str, str]:
@@ -509,7 +593,12 @@ def _is_secret_placeholder(value: Optional[str], current_preview: str = "") -> b
def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrationUpdate) -> dict:
current_ai = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
provider_id = _normalize_provider_id(update.default_provider or update.provider)
provider_id = _normalize_provider_id(update.provider)
default_provider = (
_normalize_provider_id(update.default_provider)
if update.default_provider is not None
else current_ai["default_provider"]
)
current_providers = {
provider: dict(config or {})
for provider, config in current_ai.get("providers", {}).items()
@@ -543,7 +632,7 @@ def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrat
"service_url": update.service_url.strip()
or app_settings.AI_PROVIDER_SERVICE_URL,
"service_token": current_ai.get("service_token") or "",
"default_provider": provider_id,
"default_provider": default_provider,
"providers": current_providers,
"timeout_seconds": update.timeout_seconds,
"retry_attempts": update.retry_attempts,
@@ -577,6 +666,8 @@ def _runtime_config_from_ai_payload(ai_payload: dict) -> dict:
"api_key": api_key,
"max_tokens": int(provider_config.get("max_tokens") or 1200),
"anthropic_version": provider_config.get("anthropic_version") or "2023-06-01",
"model_provider_apis": provider_config.get("model_provider_apis") or {},
"preset_models": _get_provider_preset(default_provider).get("models") or [],
},
}
@@ -631,6 +722,159 @@ async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict:
}
def _join_provider_url(base_url: str, path: str) -> str:
return f"{base_url.rstrip('/')}/{path.lstrip('/')}"
def _extract_model_ids(payload: dict) -> list[str]:
data = payload.get("data") if isinstance(payload, dict) else None
if isinstance(data, list):
return [
str(item.get("id"))
for item in data
if isinstance(item, dict) and item.get("id")
]
models = payload.get("models") if isinstance(payload, dict) else None
if isinstance(models, list):
return [
str(item.get("name") or item.get("model") or item.get("id") or item)
for item in models
if item
]
return []
def _contains_model(model_ids: list[str], model: str) -> bool:
normalized_model = model.strip().lower()
return any(str(item).strip().lower() == normalized_model for item in model_ids)
async def _check_ai_provider_lightweight(llm_config: dict, timeout_seconds: int) -> dict:
provider = _normalize_provider_id(llm_config.get("provider") or "")
configured_api = str(llm_config.get("provider_api") or "").strip() or "openai-completions"
model = str(llm_config.get("model") or "").strip()
base_url = str(llm_config.get("base_url") or "").strip().rstrip("/")
api_key = str(llm_config.get("api_key") or "").strip()
provider_api = configured_api
model_provider_apis = llm_config.get("model_provider_apis")
if isinstance(model_provider_apis, dict):
provider_api = str(model_provider_apis.get(model) or provider_api)
preset_models = [
str(item)
for item in (llm_config.get("preset_models") or [])
if str(item).strip()
]
if not provider or not base_url or not model:
return {
"success": False,
"connected": False,
"message": "当前 provider/base_url/model 未完整配置。",
"mode": "lightweight_config",
}
if provider_api != "ollama-generate" and not api_key:
return {
"success": False,
"connected": False,
"message": "当前 provider 未配置 API Key。",
"mode": "lightweight_config",
}
if provider == "opencode-go":
url = _join_provider_url(base_url, "/models")
headers = {"Authorization": f"Bearer {api_key}"}
elif provider_api == "ollama-generate":
url = _join_provider_url(base_url, "/api/tags")
headers: dict[str, str] = {}
elif provider_api == "openai-completions":
url = _join_provider_url(base_url, "/models")
headers = {"Authorization": f"Bearer {api_key}"}
elif provider_api == "anthropic-messages":
url = _join_provider_url(base_url, "/models")
headers = {
"x-api-key": api_key,
"anthropic-version": str(llm_config.get("anthropic_version") or "2023-06-01"),
}
else:
return {
"success": False,
"connected": False,
"message": f"当前 provider_api 不支持轻量连通性测试: {provider_api}",
"mode": "lightweight_unsupported",
}
try:
async with httpx.AsyncClient(timeout=min(timeout_seconds, AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS)) as client:
response = await client.get(url, headers=headers)
response.raise_for_status()
payload = response.json()
except httpx.HTTPStatusError as exc:
detail = exc.response.text or exc.response.reason_phrase
if exc.response.status_code == 404 and _contains_model(preset_models, model):
return {
"success": True,
"connected": True,
"message": "轻量连通性测试通过;当前 provider 不提供可用的模型目录,已按内置模型预设确认。",
"mode": "lightweight_preset",
"provider": provider,
"provider_api": provider_api,
"model": model,
"url": url,
}
return {
"success": False,
"connected": False,
"message": f"轻量连通性测试失败: HTTP {exc.response.status_code} {detail}",
"mode": "lightweight_models",
"url": url,
}
except Exception as exc:
return {
"success": False,
"connected": False,
"message": f"轻量连通性测试失败: {exc}",
"mode": "lightweight_models",
"url": url,
}
model_ids = _extract_model_ids(payload)
if model_ids and not _contains_model(model_ids, model):
if _contains_model(preset_models, model):
return {
"success": True,
"connected": True,
"message": "轻量连通性测试通过provider 模型目录未返回当前别名,已按内置模型预设确认。",
"mode": "lightweight_models_with_preset_alias",
"provider": provider,
"provider_api": provider_api,
"model": model,
"models_count": len(model_ids),
"url": url,
}
return {
"success": False,
"connected": False,
"message": f"连接可用,但模型目录中没有当前模型: {model}",
"mode": "lightweight_models",
"provider": provider,
"model": model,
"models_count": len(model_ids),
"url": url,
}
return {
"success": True,
"connected": True,
"message": "轻量连通性测试通过",
"mode": "lightweight_models",
"provider": provider,
"provider_api": provider_api,
"model": model,
"models_count": len(model_ids),
"url": url,
}
def _web_search_provider_defaults(provider: str) -> dict:
return web_search_provider_defaults(provider).model_dump()
@@ -691,12 +935,21 @@ def _normalize_web_search_payload(web_search_payload: dict | None) -> dict:
}
def _resolve_web_search_api_key(provider: str, provider_config: dict) -> tuple[str, str]:
def _resolve_web_search_api_key(
provider: str,
provider_config: dict,
default_provider: str | None = None,
) -> tuple[str, str]:
saved_key = provider_config.get("api_key") or ""
if saved_key:
return str(saved_key), "runtime"
preset = get_web_search_provider_preset(provider)
return _resolve_web_search_env_secret(preset.get("api_key_env") or "", "WEB_SEARCH_API_KEY")
value, source = _resolve_web_search_env_secret(preset.get("api_key_env") or "")
if value:
return value, source
if normalize_web_search_provider(provider) == normalize_web_search_provider(default_provider or "tavily"):
return _resolve_web_search_env_secret("WEB_SEARCH_API_KEY")
return "", ""
def _build_web_search_payload(
@@ -706,13 +959,22 @@ def _build_web_search_payload(
current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {})
if update is None:
return current_web_search
provider_id = normalize_web_search_provider(update.default_provider or update.provider)
provider_id = normalize_web_search_provider(update.provider)
default_provider = (
normalize_web_search_provider(update.default_provider)
if update.default_provider is not None
else current_web_search["default_provider"]
)
current_providers = {
provider: dict(config or {})
for provider, config in current_web_search.get("providers", {}).items()
}
current_provider = current_providers.get(provider_id) or _web_search_provider_defaults(provider_id)
current_key, current_key_source = _resolve_web_search_api_key(provider_id, current_provider)
current_key, current_key_source = _resolve_web_search_api_key(
provider_id,
current_provider,
current_web_search["default_provider"],
)
current_key_preview = _mask_secret(current_key, current_key_source)["preview"]
provider_payload = {
**_web_search_provider_defaults(provider_id),
@@ -745,7 +1007,7 @@ def _build_web_search_payload(
current_providers[provider_id] = provider_payload
return {
"enabled": update.enabled,
"default_provider": provider_id,
"default_provider": default_provider,
"providers": current_providers,
}
@@ -754,12 +1016,12 @@ def _runtime_config_from_web_search_payload(web_search_payload: dict) -> WebSear
normalized = _normalize_web_search_payload(web_search_payload)
provider_id = normalized["default_provider"]
provider_config = normalized["providers"].get(provider_id) or _web_search_provider_defaults(provider_id)
api_key, _source = _resolve_web_search_api_key(provider_id, provider_config)
api_key, _source = _resolve_web_search_api_key(provider_id, provider_config, provider_id)
provider_models = {
provider: WebSearchProviderConfig(**{
**config,
"api_key": (
api_key if provider == provider_id else _resolve_web_search_api_key(provider, config)[0]
api_key if provider == provider_id else _resolve_web_search_api_key(provider, config, provider_id)[0]
),
})
for provider, config in normalized["providers"].items()
@@ -893,7 +1155,11 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
normalized_web_search["providers"].get(provider_id)
or _web_search_provider_defaults(provider_id)
)
api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config)
api_key, api_key_source = _resolve_web_search_api_key(
provider_id,
provider_config,
normalized_web_search["default_provider"],
)
web_search_providers_payload[provider_id] = {
**{
key: value
@@ -980,10 +1246,7 @@ async def save_external_integrations_payload(
update: ExternalIntegrationsUpdate,
) -> dict:
current_payload = await get_setting_payload(db, "external_integrations")
current_ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
ai_payload = _build_ai_provider_payload(current_payload, update.ai_provider)
if _ai_provider_runtime_fingerprint(ai_payload) != _ai_provider_runtime_fingerprint(current_ai_payload):
await _validate_ai_provider_full_connection(ai_payload)
web_search_payload = _build_web_search_payload(current_payload, update.web_search)
ocr_payload = _build_ocr_payload(current_payload, update.ocr)
@@ -1331,6 +1594,10 @@ async def connect_ai_provider_integration(
db: AsyncSession = Depends(get_db),
):
current_payload = await get_setting_payload(db, "external_integrations")
# Connection testing should validate the provider being edited, not the
# currently saved default provider. This is a transient draft only and is
# intentionally not persisted.
payload = payload.model_copy(update={"default_provider": payload.provider})
draft_ai_payload = _build_ai_provider_payload(current_payload, payload)
runtime_config = _runtime_config_from_ai_payload(draft_ai_payload)
quick_llm_config = {
@@ -1354,24 +1621,16 @@ async def connect_ai_provider_integration(
"message": "AI Provider 可访问,但当前 provider/model/key 未完整配置。",
"status": status_result.model_dump(),
}
prompt = await get_effective_prompt(db, AI_CONNECTION_TEST_PROMPT_KEY)
probe_result = await client.analyze(
SituationalAnalysisRequest(
title="快速连接测试",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
observations=[],
constraints=["Output only OK."],
)
lightweight_result = await _check_ai_provider_lightweight(
quick_llm_config,
timeout_seconds=min(
int(runtime_config["timeout_seconds"] or 60),
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS,
),
)
return {
"success": True,
"connected": True,
"message": "连接测试通过",
**lightweight_result,
"status": status_result.model_dump(),
"provider": probe_result.provider,
"model": probe_result.model,
"mode": "quick_probe",
}
except HTTPException as exc:
return {
@@ -1389,16 +1648,38 @@ async def connect_ai_provider_integration(
@router.get("/integrations/ai-provider/secrets")
async def reveal_ai_provider_secrets(
request: Request,
provider: str = Query(default=""),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
requested_provider = _normalize_provider_id(provider) if provider else "default"
await _ensure_secret_reveal_allowed(
current_user=current_user,
request=request,
target_id=f"ai_provider:{requested_provider}",
details={"kind": "ai_provider", "provider": requested_provider},
)
current_payload = await get_setting_payload(db, "external_integrations")
ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
provider_id = _normalize_provider_id(provider or ai_payload["default_provider"])
provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id)
api_key, api_key_source = _resolve_provider_api_key(provider_id, provider_config)
service_token, service_token_source = _resolve_service_token(ai_payload)
await _record_integration_secret_reveal(
current_user=current_user,
request=request,
target_id=f"ai_provider:{provider_id}",
result="success",
details={
"kind": "ai_provider",
"provider": provider_id,
"api_key_configured": bool(api_key),
"api_key_source": api_key_source,
"service_token_configured": bool(service_token),
"service_token_source": service_token_source,
},
)
return {
"provider": provider_id,
"api_key": api_key,
@@ -1417,10 +1698,18 @@ async def get_web_search_presets(
@router.get("/integrations/web-search/secrets")
async def reveal_web_search_secrets(
request: Request,
provider: str = Query(default=""),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
requested_provider = normalize_web_search_provider(provider) if provider else "default"
await _ensure_secret_reveal_allowed(
current_user=current_user,
request=request,
target_id=f"web_search:{requested_provider}",
details={"kind": "web_search", "provider": requested_provider},
)
current_payload = await get_setting_payload(db, "external_integrations")
web_search_payload = _normalize_web_search_payload(current_payload.get("web_search") or {})
provider_id = normalize_web_search_provider(provider or web_search_payload["default_provider"])
@@ -1428,7 +1717,23 @@ async def reveal_web_search_secrets(
web_search_payload["providers"].get(provider_id)
or _web_search_provider_defaults(provider_id)
)
api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config)
api_key, api_key_source = _resolve_web_search_api_key(
provider_id,
provider_config,
web_search_payload["default_provider"],
)
await _record_integration_secret_reveal(
current_user=current_user,
request=request,
target_id=f"web_search:{provider_id}",
result="success",
details={
"kind": "web_search",
"provider": provider_id,
"api_key_configured": bool(api_key),
"api_key_source": api_key_source,
},
)
return {
"provider": provider_id,
"api_key": api_key,
@@ -1438,12 +1743,31 @@ async def reveal_web_search_secrets(
@router.get("/integrations/ocr/secrets")
async def reveal_ocr_secrets(
request: Request,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
await _ensure_secret_reveal_allowed(
current_user=current_user,
request=request,
target_id="ocr:default",
details={"kind": "ocr", "provider": "default"},
)
current_payload = await get_setting_payload(db, "external_integrations")
ocr_payload = _normalize_ocr_payload(current_payload.get("ocr") or {})
api_key, api_key_source = _resolve_ocr_api_key(ocr_payload)
await _record_integration_secret_reveal(
current_user=current_user,
request=request,
target_id=f"ocr:{ocr_payload['provider']}",
result="success",
details={
"kind": "ocr",
"provider": ocr_payload["provider"],
"api_key_configured": bool(api_key),
"api_key_source": api_key_source,
},
)
return {
"provider": ocr_payload["provider"],
"api_key": api_key,
@@ -1547,9 +1871,17 @@ async def get_ai_provider_presets(
async def refresh_ai_provider_preset(
provider: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
return {"data": await refresh_llm_provider_preset(provider)}
provider_id = _normalize_provider_id(provider)
api_key = None
if provider_id == "opencode-go":
current_payload = await get_setting_payload(db, "external_integrations")
ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id)
api_key, _api_key_source = _resolve_provider_api_key(provider_id, provider_config)
return {"data": await refresh_llm_provider_preset(provider_id, api_key=api_key)}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except Exception as exc:

View File

@@ -8,9 +8,13 @@ from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import ROOT_DIR
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.system_log import AuditLog, SystemLog
from app.models.user import User
from app.services.persistent_logs import record_audit_log, record_system_log
from app.services.system_control import (
@@ -310,7 +314,120 @@ async def get_system_log_sources(
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
return {"items": list_log_sources()}
return {
"items": [
*list_log_sources(),
{
"source_id": "system-db",
"name": "系统事件",
"kind": "database",
"location": "table://system_logs",
"description": "后端持久化系统事件、AI 和采集器操作日志。",
"category": "database",
"status": "ok",
},
{
"source_id": "audit-db",
"name": "审计事件",
"kind": "database",
"location": "table://audit_logs",
"description": "管理员敏感操作和密钥 reveal 审计记录。",
"category": "audit",
"status": "ok",
},
]
}
async def read_database_log_snapshot(
source_id: str,
*,
limit: int,
level: str,
levels: str | None,
start_date: str | None,
end_date: str | None,
search: str | None,
db: AsyncSession,
) -> dict | None:
selected_levels = set(normalize_log_level(item) for item in (levels or level).split(",") if item.strip())
selected_levels.discard("all")
search_query = (search or "").strip().lower()
lines: list[str] = []
if source_id == "system-db":
query = select(SystemLog).order_by(SystemLog.occurred_at.desc().nullslast(), SystemLog.id.desc()).limit(limit * 5)
result = await db.execute(query)
records = result.scalars().all()
for record in records:
record_level = normalize_log_level(record.level)
if selected_levels and record_level not in selected_levels:
continue
occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else ""
if start_date and occurred_at and occurred_at < start_date:
continue
if end_date and occurred_at and occurred_at > end_date:
continue
line = " ".join(
part
for part in [
record.occurred_at.isoformat() if record.occurred_at else "",
record_level.upper(),
record.source,
record.event or "",
record.message,
]
if part
)
if search_query and search_query not in line.lower():
continue
lines.append(line)
elif source_id == "audit-db":
query = select(AuditLog).order_by(AuditLog.occurred_at.desc().nullslast(), AuditLog.id.desc()).limit(limit * 5)
result = await db.execute(query)
records = result.scalars().all()
for record in records:
occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else ""
if start_date and occurred_at and occurred_at < start_date:
continue
if end_date and occurred_at and occurred_at > end_date:
continue
line = " ".join(
part
for part in [
record.occurred_at.isoformat() if record.occurred_at else "",
"INFO",
record.action,
record.target_type or "",
record.target_id or "",
record.result or "",
]
if part
)
if search_query and search_query not in line.lower():
continue
lines.append(line)
else:
return None
lines = list(reversed(lines[:limit]))
return {
"source_id": source_id,
"name": "系统事件" if source_id == "system-db" else "审计事件",
"kind": "database",
"location": "table://system_logs" if source_id == "system-db" else "table://audit_logs",
"description": "数据库持久化日志",
"category": "database" if source_id == "system-db" else "audit",
"status": "ok" if lines else "empty",
"level": level,
"selected_levels": sorted(selected_levels),
"search_query": search or "",
"available_levels": ["all", "error", "warning", "info", "debug"],
"daily_markers": [],
"line_limit": limit,
"line_count": len(lines),
"lines": lines,
}
@router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse)
@@ -323,6 +440,7 @@ async def get_system_log_snapshot(
end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"),
search: str | None = Query(None, description="Case-insensitive substring search"),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
ensure_super_admin(current_user)
@@ -345,15 +463,26 @@ async def get_system_log_snapshot(
if normalized_start_date and normalized_end_date and normalized_start_date > normalized_end_date:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="start_date must be earlier than or equal to end_date")
snapshot = read_log_snapshot(
snapshot = await read_database_log_snapshot(
source_id,
limit,
limit=limit,
level=level,
levels=levels,
start_date=normalized_start_date,
end_date=normalized_end_date,
search=search,
db=db,
)
if snapshot is None:
snapshot = read_log_snapshot(
source_id,
limit,
level=level,
levels=levels,
start_date=normalized_start_date,
end_date=normalized_end_date,
search=search,
)
if snapshot is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Log source not found")
return snapshot

View File

@@ -2159,6 +2159,24 @@ async def collect_compute_center_location(
llm_failure_reason = llm_result.failure_reason
if not candidates:
logger.warning_event(
"Compute center location collection returned no candidates",
event="visualization.compute_center.location_collect.completed",
context={
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": False,
"llm_failure_reason": llm_failure_reason,
"attempted_queries": list(attempted_queries),
"context": {
"operator": operator,
"site": site,
"city": city,
"country": country,
},
},
)
return {
"source_id": source_id,
"record_id": record_id,
@@ -2180,13 +2198,34 @@ async def collect_compute_center_location(
},
}
best_candidate = candidates[0].to_dict()
logger.info_event(
"Compute center location collection returned candidates",
event="visualization.compute_center.location_collect.completed",
context={
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": True,
"candidate_count": len(candidates),
"best_candidate": best_candidate,
"llm_failure_reason": llm_failure_reason,
"attempted_queries": list(attempted_queries),
"context": {
"operator": operator,
"site": site,
"city": city,
"country": country,
},
},
)
return {
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": True,
"candidates": [candidate.to_dict() for candidate in candidates],
"best_candidate": candidates[0].to_dict(),
"best_candidate": best_candidate,
"attempted_queries": list(attempted_queries),
"context": {
"name": name,

View File

@@ -14,6 +14,7 @@ from app.core.websocket.manager import manager
logger = get_logger(__name__, service="api")
router = APIRouter()
EARTH_UPDATES_CHANNEL = "earth_updates"
async def authenticate_token(token: str) -> Optional[dict]:
@@ -58,7 +59,7 @@ async def websocket_endpoint(
is_anonymous = payload is None
user_id = str(payload.get("sub")) if payload else f"anonymous:{id(websocket)}"
supported_channels = ["vessels", "earth_news"] if is_anonymous else [
supported_channels = ["vessels", "earth_news", EARTH_UPDATES_CHANNEL] if is_anonymous else [
"gpu_clusters",
"submarine_cables",
"ixp_nodes",
@@ -67,6 +68,7 @@ async def websocket_endpoint(
"datasource_tasks",
"vessels",
"earth_news",
EARTH_UPDATES_CHANNEL,
]
await manager.connect(websocket, user_id)

View File

@@ -8,6 +8,8 @@ from app.core.time import to_iso8601_utc
from app.core.websocket.manager import manager
EARTH_UPDATES_CHANNEL = "earth_updates"
class DataBroadcaster:
"""Periodically broadcasts data to connected WebSocket clients"""
@@ -83,6 +85,10 @@ class DataBroadcaster:
channel=channel,
)
async def broadcast_earth_update(self, data: Dict[str, Any]):
"""Broadcast Earth visualization refresh hints to connected clients."""
await self.broadcast_custom(EARTH_UPDATES_CHANNEL, data)
def enqueue_vessel_update(self, data: Dict[str, Any]):
vessels = data.get("vessels") if isinstance(data, dict) else None
if not isinstance(vessels, list):

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import json
import httpx
from fastapi import Depends, HTTPException, status
@@ -57,6 +58,9 @@ class AIProviderClient:
value = self.llm_config.get(key)
if value not in (None, ""):
headers[header_name] = str(value)
model_provider_apis = self.llm_config.get("model_provider_apis")
if isinstance(model_provider_apis, dict) and model_provider_apis:
headers["X-AI-Model-Provider-APIs"] = json.dumps(model_provider_apis)
return headers
async def get_status(self, request_id: str | None = None) -> AIProviderStatusResponse:

View File

@@ -9,12 +9,38 @@ from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.collected_data_fields import build_dynamic_metadata, get_record_field
from app.core.config import settings
from app.core.countries import normalize_country
from app.core.time import to_iso8601_utc
from app.core.websocket.broadcaster import broadcaster
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
EARTH_UPDATE_LAYER_HINTS: dict[str, list[str]] = {
"ris_live_bgp": ["bgp"],
"bgpstream_bgp": ["bgp"],
"top500_supercomputers": ["computeCenters"],
"epoch_ai_gpu": ["computeCenters"],
"huggingface_models": ["computeCenters"],
"huggingface_datasets": ["computeCenters"],
"huggingface_spaces": ["computeCenters"],
"telegeography_cables": ["cables"],
"telegeography_landing_points": ["cables"],
"telegeography_cable_systems": ["cables"],
"arcgis_cables": ["cables"],
"fao_landing_points": ["cables"],
"arcgis_landing_points": ["cables"],
"arcgis_cable_landing_relations": ["cables"],
"spacetrack_tle": ["satellites"],
"celestrak_tle": ["satellites"],
"barentswatch_vessels": ["vessels"],
"aisstream_vessels": ["vessels"],
"news_live_streams": ["media"],
"media_news_archive": ["news"],
}
def get_earth_update_layers_for_source(source: str) -> list[str]:
return EARTH_UPDATE_LAYER_HINTS.get(source, [])
class BaseCollector(ABC):
"""Abstract base class for data collectors"""
@@ -70,6 +96,29 @@ class BaseCollector(ABC):
)
self._last_broadcast_progress = rounded_progress
async def _publish_earth_update(
self,
*,
action: str,
records_processed: int,
task_id: int | None = None,
) -> None:
layers = get_earth_update_layers_for_source(self.name)
if not layers:
return
await broadcaster.broadcast_earth_update(
{
"action": action,
"source": self.name,
"data_type": self.data_type,
"layers": layers,
"datasource_id": getattr(self, "_datasource_id", None),
"task_id": task_id,
"records_processed": records_processed,
"timestamp": to_iso8601_utc(datetime.now(UTC)),
}
)
async def update_progress(self, records_processed: int, *, commit: bool = False, force: bool = False):
"""Update task progress - call this during data processing"""
if self._current_task and self._db_session:
@@ -187,7 +236,7 @@ class BaseCollector(ABC):
result = await db.execute(
select(DataSnapshot)
.where(DataSnapshot.source == self.name, DataSnapshot.is_current == True)
.where(DataSnapshot.source == self.name, DataSnapshot.is_current.is_(True))
.order_by(DataSnapshot.completed_at.desc().nullslast(), DataSnapshot.id.desc())
.limit(1)
)
@@ -325,6 +374,11 @@ class BaseCollector(ABC):
task.completed_at = datetime.now(UTC)
await db.commit()
await self._publish_task_update(force=True)
await self._publish_earth_update(
action="collector_completed",
records_processed=records_count,
task_id=task_id,
)
return {
"status": "success",
@@ -406,7 +460,7 @@ class BaseCollector(ABC):
select(CollectedData)
.where(
CollectedData.source == self.name,
CollectedData.is_current == True,
CollectedData.is_current.is_(True),
)
.order_by(CollectedData.entity_key.asc(), CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
)

View File

@@ -3,9 +3,11 @@
from __future__ import annotations
from dataclasses import dataclass
from copy import deepcopy
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm.attributes import flag_modified
from app.ai_tasks.prompts import get_effective_prompt
from app.models.system_setting import SystemSetting
@@ -135,33 +137,57 @@ DEFAULT_CREDENTIAL_GUIDES = {
}
def _normalize_provider(provider: str) -> str:
return provider.strip().lower().replace(" ", "_")
def _credential_guide_default(provider: str) -> CredentialGuideDefault:
normalized = _normalize_provider(provider)
known = DEFAULT_CREDENTIAL_GUIDES.get(normalized)
if known is not None:
return known
title = f"{normalized or 'collector'} 凭证配置教程"
return CredentialGuideDefault(
provider=normalized,
title=title,
prompt=(
f"请生成一份中文教程,指导开发者为 Planet 采集器配置 {normalized} 凭证。"
"教程要面向已经有本地开发环境的人,包含官方入口或文档查找方式、"
"获取 API Key / Token / Client credentials 的通用步骤、在 Planet 采集器配置中"
"填写凭证字段、连接测试、保存、常见失败排查。不要编造具体页面按钮文案;"
"如果公开资料不足,必须明确提醒以 provider 官方文档和当前控制台页面为准。"
),
markdown="",
)
async def _get_guide_store(db) -> tuple[SystemSetting | None, dict[str, Any]]:
result = await db.execute(
select(SystemSetting).where(SystemSetting.category == CREDENTIAL_GUIDES_CATEGORY)
)
record = result.scalar_one_or_none()
payload = dict(record.payload or {}) if record and isinstance(record.payload, dict) else {}
payload = deepcopy(record.payload) if record and isinstance(record.payload, dict) else {}
return record, payload
async def get_credential_guide(db, provider: str) -> dict[str, Any]:
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
if default is None:
raise ValueError(f"Unsupported credential guide provider: {provider}")
provider = _normalize_provider(provider)
default = _credential_guide_default(provider)
_record, store = await _get_guide_store(db)
custom = store.get(provider) if isinstance(store.get(provider), dict) else None
has_default_markdown = bool(default.markdown.strip())
return {
"provider": provider,
"title": custom.get("title") if custom else default.title,
"markdown": custom.get("markdown") if custom else default.markdown,
"prompt": default.prompt,
"source": "ai" if custom else "default",
"source": "ai" if custom else "default" if has_default_markdown else "missing",
"sources": custom.get("sources", []) if custom else [],
"verification_status": (
custom.get("verification_status", "verified_with_search_evidence")
if custom
else "default_unverified"
else "default_unverified" if has_default_markdown else "missing"
),
"verification_error": custom.get("verification_error") if custom else None,
}
@@ -177,9 +203,8 @@ async def save_credential_guide(
verification_status: str = "verified_with_search_evidence",
verification_error: str | None = None,
) -> dict[str, Any]:
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
if default is None:
raise ValueError(f"Unsupported credential guide provider: {provider}")
provider = _normalize_provider(provider)
default = _credential_guide_default(provider)
record, store = await _get_guide_store(db)
store[provider] = {
@@ -192,21 +217,21 @@ async def save_credential_guide(
if record is None:
db.add(SystemSetting(category=CREDENTIAL_GUIDES_CATEGORY, payload=store))
else:
record.payload = store
record.payload = deepcopy(store)
flag_modified(record, "payload")
await db.commit()
return await get_credential_guide(db, provider)
async def reset_credential_guide(db, provider: str) -> dict[str, Any]:
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
if default is None:
raise ValueError(f"Unsupported credential guide provider: {provider}")
provider = _normalize_provider(provider)
record, store = await _get_guide_store(db)
if provider in store:
store.pop(provider, None)
if record is not None:
record.payload = store
record.payload = deepcopy(store)
flag_modified(record, "payload")
await db.commit()
return await get_credential_guide(db, provider)
@@ -217,9 +242,8 @@ async def generate_credential_guide(
ai_client: AIProviderClient,
web_search_client: WebSearchClient | None = None,
) -> dict[str, Any]:
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
if default is None:
raise ValueError(f"Unsupported credential guide provider: {provider}")
provider = _normalize_provider(provider)
default = _credential_guide_default(provider)
search_evidence: list[dict[str, Any]] = []
search_error: str | None = None

View File

@@ -18,6 +18,7 @@ from app.models.datasource import DataSource
from app.models.datasource_config import DataSourceConfig
from app.models.system_setting import SystemSetting
from app.services.barentswatch import (
BarentsWatchConfig,
_read_zshrc_env,
fetch_barentswatch_access_token,
resolve_barentswatch_config,
@@ -44,6 +45,20 @@ def _resolve_spacetrack_credentials() -> tuple[str, str, str]:
return username, password, source or "missing"
def _resolve_spacetrack_credentials_with_override(
credential_override: dict[str, str] | None = None,
) -> tuple[str, str, str]:
if credential_override and (
credential_override.get("username") or credential_override.get("password")
):
return (
str(credential_override.get("username") or ""),
str(credential_override.get("password") or ""),
"draft",
)
return _resolve_spacetrack_credentials()
async def _resolve_aisstream_api_key(
db=None,
credential_override: dict[str, str] | None = None,
@@ -126,7 +141,9 @@ async def build_builtin_connectivity_checksum(
}
)
elif credential_provider == "spacetrack":
username, password, credential_source = _resolve_spacetrack_credentials()
username, password, credential_source = _resolve_spacetrack_credentials_with_override(
credential_override
)
has_credentials = bool(username and password)
credential_fingerprint = _sha256_json(
{
@@ -230,7 +247,16 @@ async def test_builtin_connectivity(
try:
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
if credential_context["credential_provider"] == "barentswatch":
barentswatch_config = await resolve_barentswatch_config(db)
if credential_override:
barentswatch_config = BarentsWatchConfig(
endpoint=str(request_endpoint or ""),
client_id=str(credential_override.get("client_id") or ""),
client_secret=str(credential_override.get("client_secret") or ""),
credential_source="draft",
endpoint_source="draft",
)
else:
barentswatch_config = await resolve_barentswatch_config(db)
token = await fetch_barentswatch_access_token(client, barentswatch_config)
if not token:
return {
@@ -243,7 +269,9 @@ async def test_builtin_connectivity(
}
request_headers["Authorization"] = f"Bearer {token}"
elif credential_context["credential_provider"] == "spacetrack":
username, password, _source = _resolve_spacetrack_credentials()
username, password, _source = _resolve_spacetrack_credentials_with_override(
credential_override
)
login_url = "https://www.space-track.org/ajaxauth/login"
login_response = await client.post(
login_url,

View File

@@ -270,15 +270,31 @@ def apply_payload_budget(payload: dict[str, Any], policy: EarthLayerCachePolicy)
def invalidate_earth_layer_cache_for_source(source: str) -> int:
source_key = str(source or "").strip()
patterns = {
"barentswatch_vessels": ["vessels*", "summary*"],
"aisstream_vessels": ["vessels*", "summary*"],
"telegeography_cables": ["cables*", "landing-points*", "summary*"],
"telegeography_landing": ["landing-points*", "summary*"],
"telegeography_landing_points": ["landing-points*", "summary*"],
"telegeography_systems": ["cables*", "summary*"],
"telegeography_cable_systems": ["cables*", "summary*"],
"arcgis_cables": ["cables*", "landing-points*", "summary*"],
"arcgis_landing_points": ["landing-points*", "summary*"],
"arcgis_cable_landing_relation": ["landing-points*", "summary*"],
"arcgis_cable_landing_relations": ["landing-points*", "summary*"],
"fao_landing_points": ["landing-points*", "summary*"],
"celestrak_tle": ["satellites*", "summary*"],
"spacetrack_tle": ["satellites*", "summary*"],
"top500": ["compute-centers*", "summary*"],
"top500_supercomputers": ["compute-centers*", "summary*"],
"epoch_ai_gpu": ["compute-centers*", "summary*"],
"huggingface_models": ["compute-centers*", "summary*"],
"huggingface_datasets": ["compute-centers*", "summary*"],
"huggingface_spaces": ["compute-centers*", "summary*"],
"ris_live_bgp": ["bgp*", "summary*"],
"bgpstream_bgp": ["bgp*", "summary*"],
"iptoasn_prefix_geo": ["bgp*", "summary*"],
"opengeofeed_prefix_geo": ["bgp*", "summary*"],
"nro_delegated_prefix_geo": ["bgp*", "summary*"],
}.get(source_key, [])
deleted = 0
for layer_pattern in patterns:

View File

@@ -853,7 +853,7 @@ def _parse_feed_entries(xml_text: str, source: NewsFeedSource) -> list[ParsedNew
if not clean_title or not link:
continue
item_source = _normalize_source_name(clean_title, source.name)
item_source = source.name
display_title = clean_title
if source.source_type == "aggregated" and " - " in clean_title:
parts = clean_title.rsplit(" - ", 1)

View File

@@ -7,6 +7,26 @@ from typing import Any
import httpx
MODELS_DEV_URL = "https://models.dev/api.json"
OPENCODE_GO_MODELS_URL = "https://opencode.ai/zen/go/v1/models"
OPENCODE_GO_MODEL_PROVIDER_APIS = {
"minimax-m2.7": "anthropic-messages",
"minimax-m2.5": "anthropic-messages",
}
OPENCODE_GO_FALLBACK_MODELS = [
"minimax-m2.7",
"minimax-m2.5",
"kimi-k2.6",
"kimi-k2.5",
"glm-5.1",
"glm-5",
"deepseek-v4-pro",
"deepseek-v4-flash",
"qwen3.6-plus",
"qwen3.5-plus",
"mimo-v2.5-pro",
"mimo-v2.5",
]
FALLBACK_LLM_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
@@ -80,6 +100,17 @@ FALLBACK_LLM_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
"api_key_env": "OPENROUTER_API_KEY",
"source": "fallback",
},
"opencode-go": {
"provider": "opencode-go",
"label": "OpenCode Go",
"provider_api": "openai-completions",
"base_url": "https://opencode.ai/zen/go/v1",
"model": "glm-5.1",
"models": OPENCODE_GO_FALLBACK_MODELS,
"model_provider_apis": OPENCODE_GO_MODEL_PROVIDER_APIS,
"api_key_env": "OPENCODE_GO_API_KEY",
"source": "fallback",
},
"ollama": {
"provider": "ollama",
"label": "Ollama Local",
@@ -114,8 +145,43 @@ def get_fallback_llm_provider_preset(provider: str) -> dict[str, Any]:
return dict(FALLBACK_LLM_PROVIDER_PRESETS[key])
async def refresh_llm_provider_preset(provider: str) -> dict[str, Any]:
def _opencode_go_model_provider_apis(model_ids: list[str]) -> dict[str, str]:
return {
model_id: OPENCODE_GO_MODEL_PROVIDER_APIS.get(model_id, "openai-completions")
for model_id in model_ids
}
async def refresh_llm_provider_preset(provider: str, api_key: str | None = None) -> dict[str, Any]:
fallback = get_fallback_llm_provider_preset(provider)
if fallback["provider"] == "opencode-go":
headers = {"User-Agent": "Planet/1.0"}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
response = await client.get(
OPENCODE_GO_MODELS_URL,
headers=headers,
)
response.raise_for_status()
payload = response.json()
data = payload.get("data") if isinstance(payload, dict) else []
model_ids = [
str(item.get("id"))
for item in data
if isinstance(item, dict) and item.get("id")
][:120]
if not model_ids:
model_ids = fallback["models"]
return {
**fallback,
"model": fallback["model"] if fallback["model"] in model_ids else model_ids[0],
"models": model_ids,
"model_provider_apis": _opencode_go_model_provider_apis(model_ids),
"source": OPENCODE_GO_MODELS_URL,
}
models_dev_key = MODELS_DEV_PROVIDER_KEYS.get(fallback["provider"])
if not models_dev_key:
return fallback

View File

@@ -9,6 +9,7 @@ from typing import Any, Iterable
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.logging import get_logger
from app.core.countries import COUNTRY_ENTRIES, normalize_country
from app.schemas.ai import SituationalAnalysisRequest
from app.ai_tasks.prompts import get_effective_prompt
@@ -29,6 +30,9 @@ DEFAULT_MIN_CONFIDENCE = 0.55
LOCATION_NORMALIZE_PROMPT_KEY = "location.factcheck.normalize"
LOCATION_RESOLVE_PROMPT_KEY = "location.factcheck.resolve"
MODEL_CONFIDENCE_WEIGHT = 0.25
LOG_TEXT_LIMIT = 1200
LOG_EVIDENCE_LIMIT = 5
logger = get_logger(__name__, service="location")
_geocode_llm_city = build_default_nominatim_geocoder()
_LLM_LOCATION_NAME_KEYS = (
"matched_location_name",
@@ -97,6 +101,35 @@ class LocationEvidenceScore:
summary: str
def _truncate_log_text(value: Any, limit: int = LOG_TEXT_LIMIT) -> str:
text = coerce_str(value)
if len(text) <= limit:
return text
return f"{text[:limit]}"
def _summarize_search_evidence(evidence: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
for item in (evidence or [])[:LOG_EVIDENCE_LIMIT]:
if not isinstance(item, dict):
continue
items.append(
{
"title": _truncate_log_text(item.get("title"), 180),
"source": _truncate_log_text(item.get("source") or item.get("name"), 120),
"url": _truncate_log_text(item.get("url"), 240),
"snippet": _truncate_log_text(
item.get("snippet")
or item.get("content")
or item.get("text")
or item.get("summary"),
360,
),
}
)
return items
def _first_json_object(text: str) -> dict[str, Any] | None:
stripped = text.strip()
if not stripped:
@@ -162,6 +195,64 @@ def _evidence_label(item: Any) -> str:
return coerce_str(item)
def _evidence_text(item: dict[str, Any]) -> str:
return " ".join(
coerce_str(item.get(key))
for key in ("title", "source", "name", "url", "snippet", "content", "text", "quote", "summary")
if coerce_str(item.get(key))
)
def _search_evidence_entity_match(item: dict[str, Any], query: LocationQuery) -> bool:
haystack = normalize_text(_evidence_text(item))
if not haystack:
return False
needles = [
coerce_str(query.name),
*[coerce_str(alias) for alias in query.aliases],
]
return any(normalize_text(needle) and normalize_text(needle) in haystack for needle in needles)
def _evidence_has_location_assertion(item: dict[str, Any], city: str) -> bool:
normalized_city = normalize_text(city)
text = normalize_text(_evidence_text(item))
if not normalized_city or normalized_city not in text:
return False
assertion_terms = (
"located",
"situated",
"built",
"hosted",
"deployed",
"installed",
"facility",
"campus",
"site",
"data center",
"datacenter",
"supercomputer center",
"位于",
"位於",
"坐落",
"建置",
"設置",
"设置",
)
return any(term in text for term in assertion_terms)
def _city_is_unsupported_name_hint(payload: dict[str, Any], query: LocationQuery, evidence_items: list[dict[str, Any]]) -> bool:
city = coerce_str(payload.get("city") or query.city)
if not city:
return False
normalized_city = normalize_text(city)
normalized_name = normalize_text(query.name)
if not normalized_city or not normalized_name or normalized_city not in normalized_name:
return False
return not any(_evidence_has_location_assertion(item, city) for item in evidence_items)
def _normalize_llm_precision(value: Any) -> str:
text = coerce_str(value).lower()
return LLM_PRECISION_ALIASES.get(text, text)
@@ -588,6 +679,7 @@ def _weak_evidence_penalty(
payload: dict[str, Any],
evidence_items: list[dict[str, Any]],
*,
query: LocationQuery,
entity_match: float,
geography_match: float,
conflict_penalty: float,
@@ -598,6 +690,8 @@ def _weak_evidence_penalty(
penalty += 0.20
if any(_truthy_evidence_field(item, "ambiguous") for item in evidence_items):
penalty += 0.15
if _city_is_unsupported_name_hint(payload, query, evidence_items):
penalty += 0.10
if conflict_penalty == 0.0 and entity_match > 0 and geography_match >= 0.20:
return min(penalty, 0.15)
return min(penalty, 0.30)
@@ -620,6 +714,7 @@ def _score_llm_location_payload(
weak_evidence_penalty = _weak_evidence_penalty(
payload,
evidence_items,
query=query,
entity_match=entity_match,
geography_match=geography_match,
conflict_penalty=conflict_penalty,
@@ -636,6 +731,8 @@ def _score_llm_location_payload(
- weak_evidence_penalty
)
score = min(max(score, 0.0), 1.0)
if _city_is_unsupported_name_hint(payload, query, evidence_items):
score = min(score, 0.54)
summary = (
f"combined={score:.2f}; model={model_confidence:.2f}; "
f"source={source_quality:.2f}; entity={entity_match:.2f}; "
@@ -847,15 +944,43 @@ async def collect_location_search_evidence(
) -> LocationSearchEvidenceResult:
search_query = _location_search_query(query, entity_type)
attempt = f"web_search:{entity_type}:{search_query}"
logger.info_event(
"Collecting location search evidence",
event="location.factcheck.web_search.start",
context={
"entity_type": entity_type,
"search_query": search_query,
"location_query": _query_context(query),
"max_results": max_results,
},
)
try:
evidence = await web_search_client.search(search_query, max_results=max_results)
except WebSearchError as exc:
logger.warning_event(
"Location search evidence failed",
event="location.factcheck.web_search.failed",
context={
"entity_type": entity_type,
"search_query": search_query,
"error": str(exc),
},
)
return LocationSearchEvidenceResult(
evidence=[],
attempted_queries=[attempt],
failure_reason=f"WebSearch location evidence failed: {exc}",
)
except Exception as exc:
logger.warning_event(
"Location search evidence unavailable",
event="location.factcheck.web_search.unavailable",
context={
"entity_type": entity_type,
"search_query": search_query,
"error": str(exc),
},
)
return LocationSearchEvidenceResult(
evidence=[],
attempted_queries=[attempt],
@@ -863,11 +988,29 @@ async def collect_location_search_evidence(
)
normalized = normalize_search_evidence(evidence, limit=max_results)
if not normalized:
logger.warning_event(
"Location search returned no usable evidence",
event="location.factcheck.web_search.empty",
context={
"entity_type": entity_type,
"search_query": search_query,
},
)
return LocationSearchEvidenceResult(
evidence=[],
attempted_queries=[attempt],
failure_reason="WebSearch returned no usable location evidence.",
)
logger.info_event(
"Collected location search evidence",
event="location.factcheck.web_search.result",
context={
"entity_type": entity_type,
"search_query": search_query,
"evidence_count": len(normalized),
"evidence": _summarize_search_evidence(normalized),
},
)
return LocationSearchEvidenceResult(
evidence=normalized,
attempted_queries=[attempt],
@@ -947,6 +1090,15 @@ async def collect_llm_location_fallback_candidate(
"""
attempt = f"llm_factcheck:{entity_type}:{coerce_str(query.name) or 'unknown'}"
if search_evidence is not None and not search_evidence:
logger.warning_event(
"Skipping LLM location factcheck because search evidence is empty",
event="location.factcheck.llm.skipped_no_evidence",
context={
"entity_type": entity_type,
"attempt": attempt,
"location_query": _query_context(query),
},
)
return LocationLLMFallbackResult(
candidates=[],
attempted_queries=[attempt],
@@ -986,20 +1138,66 @@ async def collect_llm_location_fallback_candidate(
"Return evidence as objects when possible, including source, url, source_type, and entity_match.",
"Include source names or URLs in evidence when known. The backend will recompute the final confidence from model confidence plus evidence quality.",
"If search_evidence is provided, use only that evidence as factual support.",
"Do not treat a website footer, office address, publisher address, or contact address as the entity's physical location.",
"If the entity name contains a city name, do not choose that city unless evidence explicitly says the entity/facility/supercomputer is located, hosted, built, deployed, or installed there.",
"Prefer the facility/site if known; otherwise use the best supported city.",
],
)
logger.info_event(
"Sending location factcheck request to LLM",
event="location.factcheck.llm.request",
context={
"entity_type": entity_type,
"attempt": attempt,
"title": request.title,
"objective": request.objective,
"location_query": request.context.get("location_query"),
"observations": request.observations,
"constraints": request.constraints,
"search_evidence_count": len(search_evidence or []),
"search_evidence": _summarize_search_evidence(search_evidence),
},
)
try:
response = await provider_client.analyze(request)
except Exception as exc:
logger.warning_event(
"LLM location factcheck failed",
event="location.factcheck.llm.failed",
context={
"entity_type": entity_type,
"attempt": attempt,
"error": str(exc),
},
)
return LocationLLMFallbackResult(
candidates=[],
attempted_queries=[attempt],
failure_reason=f"LLM location factcheck failed: {exc}",
)
logger.info_event(
"Received location factcheck response from LLM",
event="location.factcheck.llm.response",
context={
"entity_type": entity_type,
"attempt": attempt,
"provider": response.provider,
"model": response.model,
"content": _truncate_log_text(response.content, 2000),
},
)
payload = _first_json_object(response.content)
if payload is None:
logger.warning_event(
"LLM location factcheck response was not strict JSON; attempting repair",
event="location.factcheck.llm.non_json",
context={
"entity_type": entity_type,
"attempt": attempt,
"content": _truncate_log_text(response.content, 1200),
},
)
payload = await _repair_location_payload_from_text(
provider_client=provider_client,
raw_text=response.content,
@@ -1009,9 +1207,17 @@ async def collect_llm_location_fallback_candidate(
)
if payload is None:
payload = _payload_from_free_text(response.content, query=query)
if payload is None:
if payload is None and entity_type != "compute_center":
payload = _payload_from_query_name_geocode(query)
if payload is None:
logger.warning_event(
"LLM location factcheck produced no parseable payload",
event="location.factcheck.llm.unparseable",
context={
"entity_type": entity_type,
"attempt": attempt,
},
)
return LocationLLMFallbackResult(
candidates=[],
attempted_queries=[attempt],
@@ -1032,7 +1238,7 @@ async def collect_llm_location_fallback_candidate(
"url": item.get("url"),
"text": item.get("snippet") or item.get("content"),
"source_type": "web_search",
"entity_match": True,
"entity_match": _search_evidence_entity_match(item, query),
}
for item in search_evidence
if isinstance(item, dict)
@@ -1054,6 +1260,18 @@ async def collect_llm_location_fallback_candidate(
if candidate is None:
if city_geocode_failure and rejection_reason == "missing, invalid, or zero latitude/longitude":
rejection_reason = f"{rejection_reason}; {city_geocode_failure}"
logger.warning_event(
"Rejected LLM location factcheck candidate",
event="location.factcheck.llm.rejected",
context={
"entity_type": entity_type,
"attempt": attempt,
"reason": rejection_reason,
"payload": payload,
"search_evidence_count": len(search_evidence or []),
"search_evidence": _summarize_search_evidence(search_evidence),
},
)
return LocationLLMFallbackResult(
candidates=[],
attempted_queries=[attempt],
@@ -1062,6 +1280,18 @@ async def collect_llm_location_fallback_candidate(
+ (f": {rejection_reason}." if rejection_reason else ".")
),
)
logger.info_event(
"Accepted LLM location factcheck candidate",
event="location.factcheck.llm.accepted",
context={
"entity_type": entity_type,
"attempt": attempt,
"candidate": candidate.to_dict(),
"payload": payload,
"search_evidence_count": len(search_evidence or []),
"search_evidence": _summarize_search_evidence(search_evidence),
},
)
return LocationLLMFallbackResult(
candidates=[candidate],
attempted_queries=[attempt],

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import json
import os
import re
import shutil
import subprocess
@@ -86,6 +87,7 @@ class LogSource:
status: str = "ok"
buffer_key: str | None = None
container_name: str | None = None
fallback_locations: tuple[str, ...] = ()
@dataclass
@@ -104,22 +106,38 @@ class DailyLogMarker:
dominant_level: str
def _planet_state_dir() -> Path:
configured = os.getenv("PLANET_STATE_DIR")
if configured:
return Path(configured).expanduser()
xdg_state = os.getenv("XDG_STATE_HOME")
if xdg_state:
return Path(xdg_state).expanduser() / "planet"
return Path.home() / ".local" / "state" / "planet"
def _state_log_path(filename: str) -> str:
return str(_planet_state_dir() / filename)
LOG_SOURCES: dict[str, LogSource] = {
"backend": LogSource(
source_id="backend",
name="后端服务",
kind="file",
location="/tmp/planet_backend.log",
location=_state_log_path("backend.log"),
description="FastAPI 后端、调度器和采集任务共享日志。",
category="service",
fallback_locations=("/tmp/planet_backend.log",),
),
"frontend": LogSource(
source_id="frontend",
name="前端开发服务",
kind="file",
location="/tmp/planet_frontend.log",
location=_state_log_path("frontend.log"),
description="控制台与 Earth 前端开发服务输出。",
category="service",
fallback_locations=("/tmp/planet_frontend.log",),
),
"ai-provider": LogSource(
source_id="ai-provider",
@@ -164,9 +182,18 @@ def normalize_log_levels(level: str | None = None, levels: str | None = None) ->
return tuple(normalized_levels)
def resolve_file_log_path(source: LogSource) -> Path:
primary = Path(source.location).expanduser()
candidates = (primary, *(Path(item).expanduser() for item in source.fallback_locations))
for candidate in candidates:
if candidate.exists():
return candidate
return primary
def get_source_status(source: LogSource) -> str:
if source.kind == "file":
path = Path(source.location)
path = resolve_file_log_path(source)
if not path.exists():
return "missing"
return "ok" if path.stat().st_size > 0 else "empty"
@@ -190,7 +217,7 @@ def list_log_sources() -> list[dict[str, str]]:
"source_id": source.source_id,
"name": source.name,
"kind": source.kind,
"location": source.location,
"location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location,
"description": source.description,
"category": source.category,
"status": get_source_status(source),
@@ -339,7 +366,7 @@ def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry:
def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
path = Path(source.location)
path = resolve_file_log_path(source)
if not path.exists():
return []
with path.open("r", encoding="utf-8", errors="replace") as handle:
@@ -511,7 +538,7 @@ def read_log_snapshot(
"source_id": source.source_id,
"name": source.name,
"kind": source.kind,
"location": source.location,
"location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location,
"description": source.description,
"category": source.category,
"status": get_source_status(source),

View File

@@ -385,13 +385,18 @@ def build_public_tv_payload(
settings_payload: dict[str, Any],
collected_sources: list[dict[str, Any]],
) -> dict[str, Any]:
configured_by_id = {
source["id"]: source
for source in settings_payload["sources"]
if source.get("id")
}
configured_sources = [
source for source in settings_payload["sources"] if source["is_enabled"]
]
merged_by_id = {source["id"]: source for source in configured_sources}
for source in collected_sources:
if source["id"] in merged_by_id or not source["is_enabled"]:
if source["id"] in configured_by_id or not source["is_enabled"]:
continue
merged_by_id[source["id"]] = source

View File

@@ -4,6 +4,7 @@ import pytest
from app.api.v1 import datasources as datasources_api
from app.models.datasource import DataSource
from app.services import earth_layer_cache as earth_cache
def make_datasource(
@@ -53,6 +54,71 @@ def test_filter_datasources_by_product_status_and_collected_state():
assert filtered == [vessels]
def test_serialize_datasource_row_can_skip_endpoint_resolution():
datasource = make_datasource(7, "arcgis_cables", last_status="success", module="L2")
class ExplodingConfig:
def get_yaml_url(self, _source):
raise AssertionError("endpoint resolution should be skipped")
row = datasources_api.serialize_datasource_row(
datasource,
running_tasks={},
latest_tasks={},
record_counts={"arcgis_cables": 343},
endpoint_overrides={},
config=ExplodingConfig(),
include_endpoint=False,
)
assert row["id"] == 7
assert row["source"] == "arcgis_cables"
assert row["module"] == "L2"
assert row["last_status"] == "success"
assert row["collected_records"] == 343
assert row["has_collected_data"] is True
assert "endpoint" not in row
def test_serialize_datasource_row_includes_endpoint_when_requested():
datasource = make_datasource(8, "arcgis_landing_points")
class Config:
def get_yaml_url(self, source):
return f"https://example.test/{source}"
row = datasources_api.serialize_datasource_row(
datasource,
running_tasks={},
latest_tasks={},
record_counts={},
endpoint_overrides={},
config=Config(),
include_endpoint=True,
)
assert row["endpoint"] == "https://example.test/arcgis_landing_points"
def test_invalidate_earth_layer_cache_for_source_covers_datasource_aliases(monkeypatch):
patterns: list[str] = []
def fake_delete_pattern(pattern: str) -> int:
patterns.append(pattern)
return 1
monkeypatch.setattr(earth_cache.earth_layer_cache, "delete_pattern", fake_delete_pattern)
deleted = earth_cache.invalidate_earth_layer_cache_for_source("telegeography_cables")
assert deleted == 3
assert patterns == [
"earth:layer:v1:cables*",
"earth:layer:v1:landing-points*",
"earth:layer:v1:summary*",
]
@pytest.mark.asyncio
async def test_trigger_datasource_batch_skips_disabled_and_frequency_window(monkeypatch):
now = datetime.now(timezone.utc)

View File

@@ -9,6 +9,7 @@ from app.services.earth_news import (
ParsedNewsItem,
_enrich_items_with_target_locations,
_extract_target_location_from_text,
_parse_feed_entries,
_serialize_item,
get_earth_news_payload,
)
@@ -109,6 +110,60 @@ def test_serialize_item_includes_inferred_target_location():
assert payload["location_meta"]["ai_error"] is None
def test_parse_plain_rss_uses_feed_name_as_source():
source = NewsFeedSource(
id="bbc-world",
name="BBC World",
region="global",
feed_url="https://feeds.bbci.co.uk/news/world/rss.xml",
homepage_url="https://www.bbc.com/news/world",
)
xml = """
<rss>
<channel>
<item>
<title>This may be the last time you hear my voice: Political executions surge in Iran since start of war</title>
<description>Story summary</description>
<link>https://www.bbc.com/news/example</link>
<pubDate>Fri, 15 May 2026 03:00:00 GMT</pubDate>
</item>
</channel>
</rss>
"""
items = _parse_feed_entries(xml, source)
assert items[0].title == "This may be the last time you hear my voice: Political executions surge in Iran since start of war"
assert items[0].source == "BBC World"
def test_parse_aggregated_rss_splits_publisher_from_title():
source = NewsFeedSource(
id="global-scan",
name="Global Monitor / World",
region="global",
feed_url="https://news.google.com/rss",
homepage_url="https://news.google.com/",
source_type="aggregated",
)
xml = """
<rss>
<channel>
<item>
<title>Example headline - Reuters</title>
<description>Story summary</description>
<link>https://news.google.com/example</link>
</item>
</channel>
</rss>
"""
items = _parse_feed_entries(xml, source)
assert items[0].title == "Example headline"
assert items[0].source == "Reuters"
@pytest.mark.asyncio
async def test_enrich_items_with_target_locations_uses_ai_and_geocode(monkeypatch):
item = ParsedNewsItem(

View File

@@ -700,7 +700,7 @@ async def test_llm_location_fallback_repairs_non_json_answer(monkeypatch):
@pytest.mark.asyncio
async def test_llm_location_fallback_accepts_taipei_name_hint_with_weak_wording(monkeypatch):
async def test_llm_location_fallback_rejects_city_from_name_without_location_evidence(monkeypatch):
monkeypatch.setattr(
llm_fallback,
"_geocode_llm_city",
@@ -740,18 +740,60 @@ async def test_llm_location_fallback_accepts_taipei_name_hint_with_weak_wording(
entity_type="compute_center",
)
assert result.failure_reason is None
candidate = result.candidates[0]
assert candidate.city == "Taipei"
assert candidate.confidence >= 0.55
breakdown = candidate.suggested_registry_entry["llm_score_breakdown"]
assert breakdown["weak_evidence_penalty"] <= 0.15
assert breakdown["conflict_penalty"] == 0
assert breakdown["name_location_hint"] > 0
assert result.candidates == []
assert result.failure_reason is not None
assert "below minimum" in result.failure_reason
@pytest.mark.asyncio
async def test_llm_location_fallback_geocodes_city_from_entity_name_when_llm_unparseable(monkeypatch):
async def test_llm_location_fallback_accepts_explicit_facility_location_for_name_city_conflict(monkeypatch):
monkeypatch.setattr(
llm_fallback,
"_geocode_llm_city",
lambda query: {
"lat": "22.6048",
"lon": "120.3000",
"display_name": "Kaohsiung, Taiwan",
"address": {"city": "Kaohsiung", "country": "Taiwan"},
},
)
client = _FakeAIProviderClient(
json.dumps(
{
"latitude": None,
"longitude": None,
"precision": "city",
"confidence": 0.72,
"city": "Kaohsiung",
"country": "Taiwan",
"matched_location_name": "Kaohsiung, Taiwan",
"evidence": [
{
"source": "Taiwan News",
"source_type": "news",
"entity_match": True,
"text": "Nvidia's first AI supercomputer center, Taipei-1, is located in Kaohsiung.",
}
],
"reasoning_summary": "Explicit facility location evidence overrides the city-like system name.",
}
)
)
result = await collect_llm_location_fallback_candidate(
provider_client=client,
query=LocationQuery(name="TAIPEI-1", country="Taiwan"),
entity_type="compute_center",
)
assert result.failure_reason is None
candidate = result.candidates[0]
assert candidate.city == "Kaohsiung"
assert candidate.confidence >= 0.55
@pytest.mark.asyncio
async def test_llm_location_fallback_rejects_compute_center_city_from_entity_name_when_llm_unparseable(monkeypatch):
def _fake_geocode(query):
if query != "Taipei, 中国(台湾)":
return None
@@ -772,12 +814,9 @@ async def test_llm_location_fallback_geocodes_city_from_entity_name_when_llm_unp
)
assert client.calls == 2
assert result.failure_reason is None
candidate = result.candidates[0]
assert candidate.city == "Taipei"
assert candidate.latitude == pytest.approx(25.033)
assert candidate.longitude == pytest.approx(121.5654)
assert "Entity name city hint" in candidate.source_note
assert result.candidates == []
assert result.failure_reason is not None
assert "parseable city-level location fact" in result.failure_reason
@pytest.mark.asyncio

View File

@@ -5,21 +5,42 @@ import pytest
from app.api.v1 import settings as settings_api
from app.api.v1.settings import (
AIProviderIntegrationUpdate,
BarentsWatchIntegrationUpdate,
ExternalIntegrationsUpdate,
OCRIntegrationUpdate,
WebSearchIntegrationUpdate,
_build_ai_provider_payload,
_build_ocr_payload,
_can_reveal_integration_secrets,
_ensure_secret_reveal_allowed,
_mask_secret,
_normalize_ai_provider_payload,
_normalize_ocr_payload,
_record_integration_secret_reveal,
_resolve_provider_api_key,
get_runtime_ai_provider_config,
save_external_integrations_payload,
)
from app.services.llm_provider_catalog import get_fallback_llm_provider_preset
@pytest.fixture(autouse=True)
def isolated_ai_provider_env_file(monkeypatch, tmp_path):
env_file = tmp_path / ".env"
monkeypatch.setattr(settings_api, "AI_PROVIDER_ENV_FILE", env_file)
for name in (
"AI_PROVIDER",
"AI_API_KEY",
"MINIMAX_API_KEY",
"OPENAI_API_KEY",
"ANTHROPIC_API_KEY",
"DEEPSEEK_API_KEY",
"DASHSCOPE_API_KEY",
"MOONSHOT_API_KEY",
"OPENROUTER_API_KEY",
"OPENCODE_GO_API_KEY",
):
monkeypatch.delenv(name, raising=False)
return env_file
@@ -56,7 +77,7 @@ def test_provider_key_prefers_specific_env_file_key(isolated_ai_provider_env_fil
def test_provider_key_falls_back_to_generic_ai_api_key(isolated_ai_provider_env_file):
isolated_ai_provider_env_file.write_text(
"AI_API_KEY=generic-env-file-key\n",
"AI_PROVIDER=openai\nAI_API_KEY=generic-env-file-key\n",
encoding="utf-8",
)
@@ -66,11 +87,101 @@ def test_provider_key_falls_back_to_generic_ai_api_key(isolated_ai_provider_env_
assert source == "env_file"
def test_generic_ai_api_key_only_applies_to_selected_env_provider(isolated_ai_provider_env_file):
isolated_ai_provider_env_file.write_text(
"AI_PROVIDER=minimax\nAI_API_KEY=generic-env-file-key\n",
encoding="utf-8",
)
openai_value, openai_source = _resolve_provider_api_key("openai", {"api_key": ""})
minimax_value, minimax_source = _resolve_provider_api_key("minimax", {"api_key": ""})
assert openai_value == ""
assert openai_source == ""
assert minimax_value == "generic-env-file-key"
assert minimax_source == "env_file"
def test_opencode_go_provider_preset_is_openai_compatible():
preset = get_fallback_llm_provider_preset("opencode-go")
assert preset["label"] == "OpenCode Go"
assert preset["provider_api"] == "openai-completions"
assert preset["base_url"] == "https://opencode.ai/zen/go/v1"
assert preset["model"] == "glm-5.1"
assert "glm-5.1" in preset["models"]
assert "deepseek-v4-flash" in preset["models"]
assert preset["model_provider_apis"]["minimax-m2.7"] == "anthropic-messages"
assert preset["api_key_env"] == "OPENCODE_GO_API_KEY"
def test_mask_secret_without_prefix_is_fully_masked():
assert _mask_secret("plainsecret")["preview"] == "***********"
assert _mask_secret("sk-prefixed")["preview"] == "sk-********"
def test_secret_reveal_permission_is_admin_only():
assert _can_reveal_integration_secrets(SimpleNamespace(role="super_admin")) is True
assert _can_reveal_integration_secrets(SimpleNamespace(role="admin")) is True
assert _can_reveal_integration_secrets(SimpleNamespace(role="viewer")) is False
@pytest.mark.asyncio
async def test_secret_reveal_denial_is_audited(monkeypatch):
events = []
async def fake_record_audit_log(**payload):
events.append(payload)
monkeypatch.setattr(settings_api, "record_audit_log", fake_record_audit_log)
user = SimpleNamespace(id=7, username="viewer", email="viewer@example.test", role="viewer")
with pytest.raises(settings_api.HTTPException) as exc:
await _ensure_secret_reveal_allowed(
current_user=user,
request=None,
target_id="ai_provider:openai",
details={"kind": "ai_provider", "provider": "openai"},
)
assert exc.value.status_code == 403
assert events[0]["action"] == "settings.integration_secret.reveal"
assert events[0]["result"] == "denied"
assert events[0]["actor_id"] == 7
assert events[0]["target_id"] == "ai_provider:openai"
@pytest.mark.asyncio
async def test_secret_reveal_audit_does_not_store_plaintext(monkeypatch):
events = []
async def fake_record_audit_log(**payload):
events.append(payload)
monkeypatch.setattr(settings_api, "record_audit_log", fake_record_audit_log)
user = SimpleNamespace(id=1, username="admin", email="admin@example.test", role="admin")
await _record_integration_secret_reveal(
current_user=user,
request=None,
target_id="ai_provider:minimax",
result="success",
details={
"kind": "ai_provider",
"provider": "minimax",
"api_key_configured": True,
"api_key_source": "env_file",
"service_token_configured": True,
"service_token_source": "runtime",
},
)
serialized = str(events[0])
assert "secret-value" not in serialized
assert events[0]["details"]["api_key_source"] == "env_file"
assert events[0]["details"]["service_token_configured"] is True
def test_build_payload_updates_only_selected_provider_key():
current = {
"ai_provider": {
@@ -109,6 +220,30 @@ def test_build_payload_updates_only_selected_provider_key():
assert payload["providers"]["minimax"]["api_key"] == "minimax-old-key"
def test_build_payload_saves_provider_without_changing_default():
current = {
"ai_provider": {
"default_provider": "minimax",
"providers": {
"minimax": {"provider": "minimax", "api_key": "minimax-key"},
"openai": {"provider": "openai", "api_key": ""},
},
}
}
update = AIProviderIntegrationUpdate(
provider="openai",
provider_api="openai-completions",
base_url="https://api.openai.com/v1",
model="gpt-test",
api_key="openai-new-key",
)
payload = _build_ai_provider_payload(current, update)
assert payload["default_provider"] == "minimax"
assert payload["providers"]["openai"]["api_key"] == "openai-new-key"
def test_build_payload_keeps_saved_key_when_preview_submitted():
current = {
"ai_provider": {
@@ -207,3 +342,78 @@ async def test_runtime_config_uses_default_provider_specific_key(monkeypatch):
assert runtime_config["llm_config"]["provider"] == "minimax"
assert runtime_config["llm_config"]["api_key"] == "minimax-key"
assert runtime_config["llm_config"]["model"] == "MiniMax-test"
@pytest.mark.asyncio
async def test_saving_ai_provider_does_not_run_connection_validation(monkeypatch):
calls = {"validated": 0, "saved": 0}
async def fake_get_setting_payload(_db, category):
assert category == "external_integrations"
return {
"ai_provider": {
"default_provider": "minimax",
"providers": {
"minimax": {
"provider": "minimax",
"api_key": "minimax-key",
},
},
},
"web_search": {},
"ocr": {},
}
async def fake_validate(_payload):
calls["validated"] += 1
raise AssertionError("save should not run AI provider connection validation")
async def fake_save_setting_payload(_db, category, payload):
assert category == "external_integrations"
calls["saved"] += 1
return payload
async def fake_get_barentswatch_config_record(_db):
return SimpleNamespace(
endpoint="",
auth_config={},
)
async def fake_serialize_external_integrations(_db):
return {"ai_provider": {"default_provider": "minimax"}}
monkeypatch.setattr(settings_api, "get_setting_payload", fake_get_setting_payload)
monkeypatch.setattr(settings_api, "_validate_ai_provider_full_connection", fake_validate)
monkeypatch.setattr(settings_api, "save_setting_payload", fake_save_setting_payload)
monkeypatch.setattr(settings_api, "get_barentswatch_config_record", fake_get_barentswatch_config_record)
monkeypatch.setattr(settings_api, "serialize_external_integrations", fake_serialize_external_integrations)
update = ExternalIntegrationsUpdate(
ai_provider=AIProviderIntegrationUpdate(
provider="minimax",
default_provider="minimax",
provider_api="anthropic-messages",
base_url="https://api.minimaxi.com/anthropic",
model="MiniMax-M2.7",
api_key="sk-new-key",
),
barentswatch=BarentsWatchIntegrationUpdate(),
web_search=WebSearchIntegrationUpdate(),
ocr=OCRIntegrationUpdate(),
)
async def fake_commit():
return None
async def fake_refresh(_record):
return None
db = SimpleNamespace(
add=lambda _record: None,
commit=fake_commit,
refresh=fake_refresh,
)
await save_external_integrations_payload(db, update)
assert calls == {"validated": 0, "saved": 1}

View File

@@ -46,6 +46,21 @@ def test_web_search_key_prefers_provider_env(isolated_web_search_env_files):
assert source == "env_file"
def test_web_search_generic_key_only_applies_to_default_provider(isolated_web_search_env_files):
isolated_web_search_env_files.write_text(
"WEB_SEARCH_API_KEY=generic-search-key\n",
encoding="utf-8",
)
default_value, default_source = _resolve_web_search_api_key("tavily", {"api_key": ""}, "tavily")
other_value, other_source = _resolve_web_search_api_key("brave", {"api_key": ""}, "tavily")
assert default_value == "generic-search-key"
assert default_source == "env_file"
assert other_value == ""
assert other_source == ""
def test_build_web_search_payload_keeps_saved_key_when_preview_submitted():
current = {
"web_search": {
@@ -71,6 +86,35 @@ def test_build_web_search_payload_keeps_saved_key_when_preview_submitted():
assert payload["providers"]["tavily"]["api_key"] == "tvly-old-secret"
def test_build_web_search_payload_saves_provider_without_changing_default():
current = {
"web_search": {
"default_provider": "tavily",
"providers": {
"tavily": {
"provider": "tavily",
"api_key": "tvly-key",
},
"brave": {
"provider": "brave",
"api_key": "",
},
},
}
}
update = WebSearchIntegrationUpdate(
enabled=True,
provider="brave",
base_url="https://api.search.brave.com/res/v1/web/search",
api_key="brave-new-key",
)
payload = _build_web_search_payload(current, update)
assert payload["default_provider"] == "tavily"
assert payload["providers"]["brave"]["api_key"] == "brave-new-key"
@pytest.mark.asyncio
async def test_tavily_adapter_normalizes_results(monkeypatch):
config = WebSearchConfig(