release: bump version to 0.62.0
Some checks failed
ci / backend (push) Has been cancelled
ci / frontend (push) Has been cancelled
ci / delivery (push) Has been cancelled
release / images (push) Has been cancelled

This commit is contained in:
linkong
2026-05-21 01:37:32 +08:00
parent 5c65ee24d6
commit fbca381512
138 changed files with 21303 additions and 5721 deletions

View File

@@ -5,7 +5,7 @@ from datetime import datetime
import base64
import json
import re
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from sqlalchemy import delete, select, func
from sqlalchemy.ext.asyncio import AsyncSession
from pydantic import BaseModel, Field
@@ -45,14 +45,70 @@ from app.services.custom_datasource_runtime import (
DATASOURCE_MAPPING_PROMPT_KEY = "datasource.mapping"
from app.services.datasource_connectivity import (
_resolve_aisstream_api_key,
_resolve_spacetrack_credentials_with_override,
get_builtin_connection_status,
save_connectivity_success,
strip_connectivity_validation,
test_builtin_connectivity,
)
from app.services.barentswatch import resolve_barentswatch_config
from app.services.persistent_logs import record_audit_log
router = APIRouter()
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
def _user_role_value(user: User) -> str:
role = getattr(user, "role", "")
return str(getattr(role, "value", role) or "").lower()
def _user_display_name(user: User) -> str:
return str(getattr(user, "username", None) or getattr(user, "email", None) or getattr(user, "id", ""))
async def _record_datasource_secret_reveal(
*,
current_user: User,
request: Request,
target_id: str,
result: str,
details: dict[str, Any],
) -> None:
await record_audit_log(
action="datasource_config.secret.reveal",
actor_id=getattr(current_user, "id", None),
actor_name=_user_display_name(current_user),
target_type="datasource_config_secret",
target_id=target_id,
result=result,
ip=request.client.host if request.client else None,
details=details,
)
async def _ensure_datasource_secret_reveal_allowed(
current_user: User,
request: Request,
target_id: str,
details: dict[str, Any],
) -> None:
if _user_role_value(current_user) in SECRET_REVEAL_ROLES:
return
await _record_datasource_secret_reveal(
current_user=current_user,
request=request,
target_id=target_id,
result="denied",
details={**details, "role": _user_role_value(current_user)},
)
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only administrators can reveal datasource credentials",
)
def _default_builtin_config(name: str) -> dict[str, Any]:
return {"timeout": 30, "retry": 3}
@@ -389,10 +445,14 @@ async def list_all_datasources(
db_config = db_configs.get(name)
default_config = _default_builtin_config(name)
default_url = yaml_url
db_auth_config = db_config.auth_config or {} if db_config else {}
result.append(
{
"name": name,
"requires_credentials": bool(metadata.get("requires_credentials", False)),
"credential_provider": metadata.get("credential_provider"),
"credential_status": metadata.get("credential_status", "none"),
"default_url": default_url,
"endpoint": db_config.endpoint if db_config else default_url,
"is_overridden": db_config is not None and db_config.endpoint != yaml_url
@@ -401,10 +461,20 @@ async def list_all_datasources(
"is_active": db_config.is_active if db_config else True,
"source_type": db_config.source_type if db_config else _default_builtin_source_type(name),
"auth_type": db_config.auth_type if db_config else "none",
"auth_config": {
"client_id": db_auth_config.get("client_id") or "",
"username": db_auth_config.get("username") or "",
"key_name": db_auth_config.get("key_name") or db_auth_config.get("param_name") or "",
"param_name": db_auth_config.get("param_name") or db_auth_config.get("key_name") or "",
"location": db_auth_config.get("location") or db_auth_config.get("in") or "",
"in": db_auth_config.get("in") or db_auth_config.get("location") or "",
},
"auth_configured": {
"api_key": bool((db_config.auth_config or {}).get("api_key"))
if db_config
else False,
"api_key": bool(db_auth_config.get("api_key")),
"client_id": bool(db_auth_config.get("client_id")),
"client_secret": bool(db_auth_config.get("client_secret")),
"username": bool(db_auth_config.get("username")),
"password": bool(db_auth_config.get("password")),
},
"headers": db_config.headers if db_config else {},
"config": strip_connectivity_validation(db_config.config if db_config else default_config),
@@ -418,6 +488,96 @@ async def list_all_datasources(
return {"total": len(result), "data": result}
@router.get("/configs/secrets")
async def reveal_builtin_config_secrets(
request: Request,
name: str = Query(..., min_length=1),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Reveal configured built-in datasource credentials for admin editing."""
source = name.strip()
metadata = DEFAULT_DATASOURCES.get(source)
if not metadata or not metadata.get("requires_credentials"):
raise HTTPException(status_code=404, detail="Credentialed datasource config not found")
provider = str(metadata.get("credential_provider") or "")
target_id = f"datasource_config:{source}"
await _ensure_datasource_secret_reveal_allowed(
current_user,
request,
target_id,
{"source": source, "provider": provider},
)
result = await db.execute(select(DataSourceConfig).where(DataSourceConfig.name == source))
record = result.scalar_one_or_none()
auth_config = dict(record.auth_config or {}) if record else {}
payload: dict[str, Any] = {
"name": source,
"provider": provider,
}
details: dict[str, Any] = {"source": source, "provider": provider}
if provider == "barentswatch":
resolved = await resolve_barentswatch_config(db)
client_id = str(auth_config.get("client_id") or resolved.client_id or "")
client_secret = str(auth_config.get("client_secret") or resolved.client_secret or "")
source_label = "datasource_config" if auth_config.get("client_id") or auth_config.get("client_secret") else resolved.credential_source
payload.update(
{
"client_id": client_id,
"client_secret": client_secret,
"client_id_source": source_label if client_id else "missing",
"client_secret_source": source_label if client_secret else "missing",
}
)
details.update(
{
"client_id_configured": bool(client_id),
"client_secret_configured": bool(client_secret),
"credential_source": source_label,
}
)
elif provider == "aisstream":
api_key, api_key_source = await _resolve_aisstream_api_key(db)
payload.update({"api_key": api_key, "api_key_source": api_key_source})
details.update({"api_key_configured": bool(api_key), "api_key_source": api_key_source})
elif provider == "spacetrack":
if auth_config.get("username") or auth_config.get("password"):
username = str(auth_config.get("username") or "")
password = str(auth_config.get("password") or "")
credential_source = "datasource_config"
else:
username, password, credential_source = _resolve_spacetrack_credentials_with_override()
payload.update(
{
"username": username,
"password": password,
"username_source": credential_source if username else "missing",
"password_source": credential_source if password else "missing",
}
)
details.update(
{
"username_configured": bool(username),
"password_configured": bool(password),
"credential_source": credential_source,
}
)
else:
raise HTTPException(status_code=400, detail="Datasource credential provider is not supported")
await _record_datasource_secret_reveal(
current_user=current_user,
request=request,
target_id=target_id,
result="success",
details=details,
)
return payload
@router.get("/configs/{config_id}")
async def get_config(
config_id: int,

View File

@@ -7,6 +7,7 @@ from pydantic import BaseModel, Field
from sqlalchemy import func, or_, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.cache import cache
from app.core.time import to_iso8601_utc
from app.core.security import get_current_user
from app.core.data_sources import get_data_sources_config
@@ -26,6 +27,7 @@ from app.services.scheduler import (
run_collector_now,
sync_datasource_job,
)
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
router = APIRouter()
STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90
@@ -235,6 +237,8 @@ async def _load_datasource_endpoint_overrides(
async def _load_datasource_list_context(
db: AsyncSession,
datasources: list[DataSource],
*,
include_endpoint: bool = True,
) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, str]]:
datasource_ids = [datasource.id for datasource in datasources]
sources = [datasource.source for datasource in datasources]
@@ -260,10 +264,65 @@ async def _load_datasource_list_context(
running_tasks = await _load_latest_running_tasks(db, datasource_ids)
latest_tasks = await _load_latest_tasks(db, datasource_ids)
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources) if include_endpoint else {}
return running_tasks, latest_tasks, endpoint_overrides
def serialize_datasource_row(
datasource: DataSource,
*,
running_tasks: dict[int, CollectionTask],
latest_tasks: dict[int, CollectionTask],
record_counts: dict[str, int],
endpoint_overrides: dict[str, str],
config,
include_endpoint: bool,
) -> dict:
running_task = running_tasks.get(datasource.id)
latest_task = latest_tasks.get(datasource.id)
display_task = running_task or latest_task
endpoint = None
if include_endpoint:
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None)
last_status = datasource.last_status or (latest_task.status if latest_task else None)
collected_records = record_counts.get(datasource.source, 0)
row = {
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
**datasource_metadata(datasource.source),
"product": datasource_product_key(datasource),
"module": datasource.module,
"priority": datasource.priority,
"frequency": format_frequency_label(datasource.frequency_minutes),
"frequency_minutes": datasource.frequency_minutes,
"is_active": datasource.is_active,
"collector_class": datasource.collector_class,
"last_run": to_iso8601_utc(last_run_at),
"last_run_at": to_iso8601_utc(last_run_at),
"last_status": last_status,
"is_running": running_task is not None,
"task_id": display_task.id if display_task else None,
"progress": display_task.progress if display_task else None,
"phase": display_task.phase if display_task else None,
"phase_progress": display_task.phase_progress if display_task else None,
"phase_message": display_task.phase_message if display_task else None,
"phase_current": display_task.phase_current if display_task else None,
"phase_total": display_task.phase_total if display_task else None,
"phase_unit": display_task.phase_unit if display_task else None,
"records_processed": display_task.records_processed if display_task else None,
"total_records": display_task.total_records if display_task else None,
"error_message": display_task.error_message if display_task else None,
"collected_records": collected_records,
"has_collected_data": collected_records > 0,
}
if include_endpoint:
row["endpoint"] = endpoint
return row
def _apply_datasource_query_filters(
query,
*,
@@ -658,6 +717,7 @@ async def list_datasources(
collected: Optional[bool] = None,
credential_status: Optional[str] = None,
q: Optional[str] = None,
include_endpoint: bool = True,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
@@ -676,7 +736,11 @@ async def list_datasources(
collector_list = []
config = get_data_sources_config()
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(
db,
datasources,
include_endpoint=include_endpoint,
)
record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
datasources = _filter_datasources_in_memory(
datasources,
@@ -689,46 +753,16 @@ async def list_datasources(
credential_status=credential_status,
)
for datasource in datasources:
running_task = running_tasks.get(datasource.id)
latest_task = latest_tasks.get(datasource.id)
display_task = running_task or latest_task
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None)
last_status = datasource.last_status or (latest_task.status if latest_task else None)
collected_records = record_counts.get(datasource.source, 0)
collector_list.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
**datasource_metadata(datasource.source),
"product": datasource_product_key(datasource),
"module": datasource.module,
"priority": datasource.priority,
"frequency": format_frequency_label(datasource.frequency_minutes),
"frequency_minutes": datasource.frequency_minutes,
"is_active": datasource.is_active,
"collector_class": datasource.collector_class,
"endpoint": endpoint,
"last_run": to_iso8601_utc(last_run_at),
"last_run_at": to_iso8601_utc(last_run_at),
"last_status": last_status,
"is_running": running_task is not None,
"task_id": display_task.id if display_task else None,
"progress": display_task.progress if display_task else None,
"phase": display_task.phase if display_task else None,
"phase_progress": display_task.phase_progress if display_task else None,
"phase_message": display_task.phase_message if display_task else None,
"phase_current": display_task.phase_current if display_task else None,
"phase_total": display_task.phase_total if display_task else None,
"phase_unit": display_task.phase_unit if display_task else None,
"records_processed": display_task.records_processed if display_task else None,
"total_records": display_task.total_records if display_task else None,
"error_message": display_task.error_message if display_task else None,
"collected_records": collected_records,
"has_collected_data": collected_records > 0,
}
serialize_datasource_row(
datasource,
running_tasks=running_tasks,
latest_tasks=latest_tasks,
record_counts=record_counts,
endpoint_overrides=endpoint_overrides,
config=config,
include_endpoint=include_endpoint,
)
)
return {"total": len(collector_list), "data": collector_list}
@@ -785,6 +819,51 @@ async def trigger_datasource_batch(
return await _trigger_datasource_batch(db, datasources, force=payload.force)
@router.get("/snapshots")
async def list_datasource_snapshots(
source_id: Optional[str] = None,
current_only: Optional[bool] = None,
limit: int = Query(default=100, ge=1, le=500),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
query = (
select(DataSnapshot, DataSource.name, DataSource.module)
.outerjoin(DataSource, DataSource.id == DataSnapshot.datasource_id)
.order_by(DataSnapshot.created_at.desc().nullslast(), DataSnapshot.id.desc())
.limit(limit)
)
if source_id:
query = query.where(DataSnapshot.source == source_id)
if current_only is not None:
query = query.where(DataSnapshot.is_current.is_(current_only))
result = await db.execute(query)
rows = []
for snapshot, datasource_name, datasource_module in result.all():
rows.append(
{
"id": snapshot.id,
"datasource_id": snapshot.datasource_id,
"datasource_name": datasource_name,
"module": datasource_module,
"task_id": snapshot.task_id,
"source": snapshot.source,
"snapshot_key": snapshot.snapshot_key,
"reference_date": to_iso8601_utc(snapshot.reference_date),
"started_at": to_iso8601_utc(snapshot.started_at),
"completed_at": to_iso8601_utc(snapshot.completed_at),
"record_count": snapshot.record_count,
"status": snapshot.status,
"is_current": snapshot.is_current,
"parent_snapshot_id": snapshot.parent_snapshot_id,
"summary": snapshot.summary or {},
"created_at": to_iso8601_utc(snapshot.created_at),
}
)
return {"total": len(rows), "data": rows}
@router.get("/{source_id}")
async def get_datasource(
source_id: str,
@@ -813,6 +892,37 @@ async def get_datasource(
}
@router.get("/{source_id}/row")
async def get_datasource_row(
source_id: str,
include_endpoint: bool = True,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
datasource = await get_datasource_record(db, source_id)
if not datasource:
raise HTTPException(status_code=404, detail="Data source not found")
config = get_data_sources_config()
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(
db,
[datasource],
include_endpoint=include_endpoint,
)
record_counts = await _load_collected_record_counts(db, [datasource.source])
return {
"data": serialize_datasource_row(
datasource,
running_tasks=running_tasks,
latest_tasks=latest_tasks,
record_counts=record_counts,
endpoint_overrides=endpoint_overrides,
config=config,
include_endpoint=include_endpoint,
)
}
@router.post("/{source_id}/enable")
async def enable_datasource(
source_id: str,
@@ -960,6 +1070,29 @@ async def clear_datasource_data(
}
@router.delete("/{source_id}/cache")
async def clear_datasource_cache(
source_id: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
datasource = await get_datasource_record(db, source_id)
if not datasource:
raise HTTPException(status_code=404, detail="Data source not found")
earth_deleted_count = invalidate_earth_layer_cache_for_source(datasource.source)
dashboard_deleted_count = int(cache.delete("dashboard:stats")) + int(cache.delete("dashboard:summary"))
deleted_count = earth_deleted_count + dashboard_deleted_count
return {
"status": "success",
"message": f"Cleared {deleted_count} cache keys for data source '{datasource.name}'",
"deleted_count": deleted_count,
"earth_layer_deleted_count": earth_deleted_count,
"dashboard_deleted_count": dashboard_deleted_count,
}
@router.get("/{source_id}/task-status")
async def get_task_status(
source_id: str,

View File

@@ -4,7 +4,8 @@ import os
from pathlib import Path
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
import httpx
from pydantic import BaseModel, EmailStr, Field
from dotenv import dotenv_values
from sqlalchemy import select
@@ -64,10 +65,12 @@ from app.services.llm_provider_catalog import (
)
from app.services.scheduler import sync_datasource_job
from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings
from app.services.persistent_logs import record_audit_log
router = APIRouter()
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5
AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test"
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
DEFAULT_SETTINGS = {
"system": {
@@ -130,6 +133,70 @@ DEFAULT_SETTINGS = {
}
def _user_role_value(user: User) -> str:
role = getattr(user, "role", "")
return role.value if hasattr(role, "value") else str(role or "")
def _user_display_name(user: User) -> str | None:
return getattr(user, "username", None) or getattr(user, "email", None)
def _request_client_ip(request: Request | None) -> str | None:
if request is None or request.client is None:
return None
return request.client.host
def _can_reveal_integration_secrets(user: User) -> bool:
return _user_role_value(user) in SECRET_REVEAL_ROLES
async def _record_integration_secret_reveal(
*,
current_user: User,
request: Request | None,
target_id: str,
result: str,
details: dict,
) -> None:
await record_audit_log(
action="settings.integration_secret.reveal",
actor_id=getattr(current_user, "id", None),
actor_name=_user_display_name(current_user),
target_type="integration_secret",
target_id=target_id,
result=result,
ip=_request_client_ip(request),
details=details,
)
async def _ensure_secret_reveal_allowed(
*,
current_user: User,
request: Request | None,
target_id: str,
details: dict | None = None,
) -> None:
if _can_reveal_integration_secrets(current_user):
return
await _record_integration_secret_reveal(
current_user=current_user,
request=request,
target_id=target_id,
result="denied",
details={
**(details or {}),
"role": _user_role_value(current_user),
},
)
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only administrators can reveal integration secrets",
)
class SystemSettingsUpdate(BaseModel):
system_name: str = "智能星球"
refresh_interval: int = Field(default=60, ge=10, le=3600)
@@ -353,9 +420,10 @@ def _get_provider_preset(provider: str) -> dict:
"provider": provider,
"provider_api": "openai-completions",
"base_url": "",
"model": "",
"models": [],
"api_key_env": "",
"model": "",
"models": [],
"model_provider_apis": {},
"api_key_env": "",
}
@@ -377,6 +445,9 @@ def _resolve_env_secret(*names: str) -> tuple[str, str]:
value = env_file_values.get(name)
if value:
return value, "env_file"
value = os.environ.get(name)
if value:
return value, "env"
return "", ""
@@ -419,9 +490,16 @@ def _provider_defaults(provider: str) -> dict:
1200 if preset.get("provider_api") == "anthropic-messages" else 4096
),
"anthropic_version": "2023-06-01",
"model_provider_apis": preset.get("model_provider_apis") or {},
}
def _selected_ai_env_provider() -> str:
env_file_values = _read_ai_provider_env_file()
provider = env_file_values.get("AI_PROVIDER") or os.environ.get("AI_PROVIDER") or "minimax"
return _normalize_provider_id(provider)
def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict:
raw = dict(ai_payload or {})
default_provider = _normalize_provider_id(raw.get("default_provider") or raw.get("provider"))
@@ -440,6 +518,7 @@ def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict:
"api_key",
"max_tokens",
"anthropic_version",
"model_provider_apis",
)
if raw.get(key) not in (None, "")
}
@@ -477,7 +556,12 @@ def _resolve_provider_api_key(provider: str, provider_config: dict) -> tuple[str
return str(saved_key), "runtime"
preset = _get_provider_preset(provider)
api_key_env = preset.get("api_key_env") or ""
return _resolve_env_secret(api_key_env, "AI_API_KEY")
value, source = _resolve_env_secret(api_key_env)
if value:
return value, source
if _normalize_provider_id(provider) == _selected_ai_env_provider():
return _resolve_env_secret("AI_API_KEY")
return "", ""
def _resolve_service_token(ai_payload: dict) -> tuple[str, str]:
@@ -509,7 +593,12 @@ def _is_secret_placeholder(value: Optional[str], current_preview: str = "") -> b
def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrationUpdate) -> dict:
current_ai = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
provider_id = _normalize_provider_id(update.default_provider or update.provider)
provider_id = _normalize_provider_id(update.provider)
default_provider = (
_normalize_provider_id(update.default_provider)
if update.default_provider is not None
else current_ai["default_provider"]
)
current_providers = {
provider: dict(config or {})
for provider, config in current_ai.get("providers", {}).items()
@@ -543,7 +632,7 @@ def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrat
"service_url": update.service_url.strip()
or app_settings.AI_PROVIDER_SERVICE_URL,
"service_token": current_ai.get("service_token") or "",
"default_provider": provider_id,
"default_provider": default_provider,
"providers": current_providers,
"timeout_seconds": update.timeout_seconds,
"retry_attempts": update.retry_attempts,
@@ -577,6 +666,8 @@ def _runtime_config_from_ai_payload(ai_payload: dict) -> dict:
"api_key": api_key,
"max_tokens": int(provider_config.get("max_tokens") or 1200),
"anthropic_version": provider_config.get("anthropic_version") or "2023-06-01",
"model_provider_apis": provider_config.get("model_provider_apis") or {},
"preset_models": _get_provider_preset(default_provider).get("models") or [],
},
}
@@ -631,6 +722,159 @@ async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict:
}
def _join_provider_url(base_url: str, path: str) -> str:
return f"{base_url.rstrip('/')}/{path.lstrip('/')}"
def _extract_model_ids(payload: dict) -> list[str]:
data = payload.get("data") if isinstance(payload, dict) else None
if isinstance(data, list):
return [
str(item.get("id"))
for item in data
if isinstance(item, dict) and item.get("id")
]
models = payload.get("models") if isinstance(payload, dict) else None
if isinstance(models, list):
return [
str(item.get("name") or item.get("model") or item.get("id") or item)
for item in models
if item
]
return []
def _contains_model(model_ids: list[str], model: str) -> bool:
normalized_model = model.strip().lower()
return any(str(item).strip().lower() == normalized_model for item in model_ids)
async def _check_ai_provider_lightweight(llm_config: dict, timeout_seconds: int) -> dict:
provider = _normalize_provider_id(llm_config.get("provider") or "")
configured_api = str(llm_config.get("provider_api") or "").strip() or "openai-completions"
model = str(llm_config.get("model") or "").strip()
base_url = str(llm_config.get("base_url") or "").strip().rstrip("/")
api_key = str(llm_config.get("api_key") or "").strip()
provider_api = configured_api
model_provider_apis = llm_config.get("model_provider_apis")
if isinstance(model_provider_apis, dict):
provider_api = str(model_provider_apis.get(model) or provider_api)
preset_models = [
str(item)
for item in (llm_config.get("preset_models") or [])
if str(item).strip()
]
if not provider or not base_url or not model:
return {
"success": False,
"connected": False,
"message": "当前 provider/base_url/model 未完整配置。",
"mode": "lightweight_config",
}
if provider_api != "ollama-generate" and not api_key:
return {
"success": False,
"connected": False,
"message": "当前 provider 未配置 API Key。",
"mode": "lightweight_config",
}
if provider == "opencode-go":
url = _join_provider_url(base_url, "/models")
headers = {"Authorization": f"Bearer {api_key}"}
elif provider_api == "ollama-generate":
url = _join_provider_url(base_url, "/api/tags")
headers: dict[str, str] = {}
elif provider_api == "openai-completions":
url = _join_provider_url(base_url, "/models")
headers = {"Authorization": f"Bearer {api_key}"}
elif provider_api == "anthropic-messages":
url = _join_provider_url(base_url, "/models")
headers = {
"x-api-key": api_key,
"anthropic-version": str(llm_config.get("anthropic_version") or "2023-06-01"),
}
else:
return {
"success": False,
"connected": False,
"message": f"当前 provider_api 不支持轻量连通性测试: {provider_api}",
"mode": "lightweight_unsupported",
}
try:
async with httpx.AsyncClient(timeout=min(timeout_seconds, AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS)) as client:
response = await client.get(url, headers=headers)
response.raise_for_status()
payload = response.json()
except httpx.HTTPStatusError as exc:
detail = exc.response.text or exc.response.reason_phrase
if exc.response.status_code == 404 and _contains_model(preset_models, model):
return {
"success": True,
"connected": True,
"message": "轻量连通性测试通过;当前 provider 不提供可用的模型目录,已按内置模型预设确认。",
"mode": "lightweight_preset",
"provider": provider,
"provider_api": provider_api,
"model": model,
"url": url,
}
return {
"success": False,
"connected": False,
"message": f"轻量连通性测试失败: HTTP {exc.response.status_code} {detail}",
"mode": "lightweight_models",
"url": url,
}
except Exception as exc:
return {
"success": False,
"connected": False,
"message": f"轻量连通性测试失败: {exc}",
"mode": "lightweight_models",
"url": url,
}
model_ids = _extract_model_ids(payload)
if model_ids and not _contains_model(model_ids, model):
if _contains_model(preset_models, model):
return {
"success": True,
"connected": True,
"message": "轻量连通性测试通过provider 模型目录未返回当前别名,已按内置模型预设确认。",
"mode": "lightweight_models_with_preset_alias",
"provider": provider,
"provider_api": provider_api,
"model": model,
"models_count": len(model_ids),
"url": url,
}
return {
"success": False,
"connected": False,
"message": f"连接可用,但模型目录中没有当前模型: {model}",
"mode": "lightweight_models",
"provider": provider,
"model": model,
"models_count": len(model_ids),
"url": url,
}
return {
"success": True,
"connected": True,
"message": "轻量连通性测试通过",
"mode": "lightweight_models",
"provider": provider,
"provider_api": provider_api,
"model": model,
"models_count": len(model_ids),
"url": url,
}
def _web_search_provider_defaults(provider: str) -> dict:
return web_search_provider_defaults(provider).model_dump()
@@ -691,12 +935,21 @@ def _normalize_web_search_payload(web_search_payload: dict | None) -> dict:
}
def _resolve_web_search_api_key(provider: str, provider_config: dict) -> tuple[str, str]:
def _resolve_web_search_api_key(
provider: str,
provider_config: dict,
default_provider: str | None = None,
) -> tuple[str, str]:
saved_key = provider_config.get("api_key") or ""
if saved_key:
return str(saved_key), "runtime"
preset = get_web_search_provider_preset(provider)
return _resolve_web_search_env_secret(preset.get("api_key_env") or "", "WEB_SEARCH_API_KEY")
value, source = _resolve_web_search_env_secret(preset.get("api_key_env") or "")
if value:
return value, source
if normalize_web_search_provider(provider) == normalize_web_search_provider(default_provider or "tavily"):
return _resolve_web_search_env_secret("WEB_SEARCH_API_KEY")
return "", ""
def _build_web_search_payload(
@@ -706,13 +959,22 @@ def _build_web_search_payload(
current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {})
if update is None:
return current_web_search
provider_id = normalize_web_search_provider(update.default_provider or update.provider)
provider_id = normalize_web_search_provider(update.provider)
default_provider = (
normalize_web_search_provider(update.default_provider)
if update.default_provider is not None
else current_web_search["default_provider"]
)
current_providers = {
provider: dict(config or {})
for provider, config in current_web_search.get("providers", {}).items()
}
current_provider = current_providers.get(provider_id) or _web_search_provider_defaults(provider_id)
current_key, current_key_source = _resolve_web_search_api_key(provider_id, current_provider)
current_key, current_key_source = _resolve_web_search_api_key(
provider_id,
current_provider,
current_web_search["default_provider"],
)
current_key_preview = _mask_secret(current_key, current_key_source)["preview"]
provider_payload = {
**_web_search_provider_defaults(provider_id),
@@ -745,7 +1007,7 @@ def _build_web_search_payload(
current_providers[provider_id] = provider_payload
return {
"enabled": update.enabled,
"default_provider": provider_id,
"default_provider": default_provider,
"providers": current_providers,
}
@@ -754,12 +1016,12 @@ def _runtime_config_from_web_search_payload(web_search_payload: dict) -> WebSear
normalized = _normalize_web_search_payload(web_search_payload)
provider_id = normalized["default_provider"]
provider_config = normalized["providers"].get(provider_id) or _web_search_provider_defaults(provider_id)
api_key, _source = _resolve_web_search_api_key(provider_id, provider_config)
api_key, _source = _resolve_web_search_api_key(provider_id, provider_config, provider_id)
provider_models = {
provider: WebSearchProviderConfig(**{
**config,
"api_key": (
api_key if provider == provider_id else _resolve_web_search_api_key(provider, config)[0]
api_key if provider == provider_id else _resolve_web_search_api_key(provider, config, provider_id)[0]
),
})
for provider, config in normalized["providers"].items()
@@ -893,7 +1155,11 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
normalized_web_search["providers"].get(provider_id)
or _web_search_provider_defaults(provider_id)
)
api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config)
api_key, api_key_source = _resolve_web_search_api_key(
provider_id,
provider_config,
normalized_web_search["default_provider"],
)
web_search_providers_payload[provider_id] = {
**{
key: value
@@ -980,10 +1246,7 @@ async def save_external_integrations_payload(
update: ExternalIntegrationsUpdate,
) -> dict:
current_payload = await get_setting_payload(db, "external_integrations")
current_ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
ai_payload = _build_ai_provider_payload(current_payload, update.ai_provider)
if _ai_provider_runtime_fingerprint(ai_payload) != _ai_provider_runtime_fingerprint(current_ai_payload):
await _validate_ai_provider_full_connection(ai_payload)
web_search_payload = _build_web_search_payload(current_payload, update.web_search)
ocr_payload = _build_ocr_payload(current_payload, update.ocr)
@@ -1331,6 +1594,10 @@ async def connect_ai_provider_integration(
db: AsyncSession = Depends(get_db),
):
current_payload = await get_setting_payload(db, "external_integrations")
# Connection testing should validate the provider being edited, not the
# currently saved default provider. This is a transient draft only and is
# intentionally not persisted.
payload = payload.model_copy(update={"default_provider": payload.provider})
draft_ai_payload = _build_ai_provider_payload(current_payload, payload)
runtime_config = _runtime_config_from_ai_payload(draft_ai_payload)
quick_llm_config = {
@@ -1354,24 +1621,16 @@ async def connect_ai_provider_integration(
"message": "AI Provider 可访问,但当前 provider/model/key 未完整配置。",
"status": status_result.model_dump(),
}
prompt = await get_effective_prompt(db, AI_CONNECTION_TEST_PROMPT_KEY)
probe_result = await client.analyze(
SituationalAnalysisRequest(
title="快速连接测试",
objective=prompt.prompt,
system_prompt=prompt.system_prompt or None,
observations=[],
constraints=["Output only OK."],
)
lightweight_result = await _check_ai_provider_lightweight(
quick_llm_config,
timeout_seconds=min(
int(runtime_config["timeout_seconds"] or 60),
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS,
),
)
return {
"success": True,
"connected": True,
"message": "连接测试通过",
**lightweight_result,
"status": status_result.model_dump(),
"provider": probe_result.provider,
"model": probe_result.model,
"mode": "quick_probe",
}
except HTTPException as exc:
return {
@@ -1389,16 +1648,38 @@ async def connect_ai_provider_integration(
@router.get("/integrations/ai-provider/secrets")
async def reveal_ai_provider_secrets(
request: Request,
provider: str = Query(default=""),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
requested_provider = _normalize_provider_id(provider) if provider else "default"
await _ensure_secret_reveal_allowed(
current_user=current_user,
request=request,
target_id=f"ai_provider:{requested_provider}",
details={"kind": "ai_provider", "provider": requested_provider},
)
current_payload = await get_setting_payload(db, "external_integrations")
ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
provider_id = _normalize_provider_id(provider or ai_payload["default_provider"])
provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id)
api_key, api_key_source = _resolve_provider_api_key(provider_id, provider_config)
service_token, service_token_source = _resolve_service_token(ai_payload)
await _record_integration_secret_reveal(
current_user=current_user,
request=request,
target_id=f"ai_provider:{provider_id}",
result="success",
details={
"kind": "ai_provider",
"provider": provider_id,
"api_key_configured": bool(api_key),
"api_key_source": api_key_source,
"service_token_configured": bool(service_token),
"service_token_source": service_token_source,
},
)
return {
"provider": provider_id,
"api_key": api_key,
@@ -1417,10 +1698,18 @@ async def get_web_search_presets(
@router.get("/integrations/web-search/secrets")
async def reveal_web_search_secrets(
request: Request,
provider: str = Query(default=""),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
requested_provider = normalize_web_search_provider(provider) if provider else "default"
await _ensure_secret_reveal_allowed(
current_user=current_user,
request=request,
target_id=f"web_search:{requested_provider}",
details={"kind": "web_search", "provider": requested_provider},
)
current_payload = await get_setting_payload(db, "external_integrations")
web_search_payload = _normalize_web_search_payload(current_payload.get("web_search") or {})
provider_id = normalize_web_search_provider(provider or web_search_payload["default_provider"])
@@ -1428,7 +1717,23 @@ async def reveal_web_search_secrets(
web_search_payload["providers"].get(provider_id)
or _web_search_provider_defaults(provider_id)
)
api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config)
api_key, api_key_source = _resolve_web_search_api_key(
provider_id,
provider_config,
web_search_payload["default_provider"],
)
await _record_integration_secret_reveal(
current_user=current_user,
request=request,
target_id=f"web_search:{provider_id}",
result="success",
details={
"kind": "web_search",
"provider": provider_id,
"api_key_configured": bool(api_key),
"api_key_source": api_key_source,
},
)
return {
"provider": provider_id,
"api_key": api_key,
@@ -1438,12 +1743,31 @@ async def reveal_web_search_secrets(
@router.get("/integrations/ocr/secrets")
async def reveal_ocr_secrets(
request: Request,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
await _ensure_secret_reveal_allowed(
current_user=current_user,
request=request,
target_id="ocr:default",
details={"kind": "ocr", "provider": "default"},
)
current_payload = await get_setting_payload(db, "external_integrations")
ocr_payload = _normalize_ocr_payload(current_payload.get("ocr") or {})
api_key, api_key_source = _resolve_ocr_api_key(ocr_payload)
await _record_integration_secret_reveal(
current_user=current_user,
request=request,
target_id=f"ocr:{ocr_payload['provider']}",
result="success",
details={
"kind": "ocr",
"provider": ocr_payload["provider"],
"api_key_configured": bool(api_key),
"api_key_source": api_key_source,
},
)
return {
"provider": ocr_payload["provider"],
"api_key": api_key,
@@ -1547,9 +1871,17 @@ async def get_ai_provider_presets(
async def refresh_ai_provider_preset(
provider: str,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
try:
return {"data": await refresh_llm_provider_preset(provider)}
provider_id = _normalize_provider_id(provider)
api_key = None
if provider_id == "opencode-go":
current_payload = await get_setting_payload(db, "external_integrations")
ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id)
api_key, _api_key_source = _resolve_provider_api_key(provider_id, provider_config)
return {"data": await refresh_llm_provider_preset(provider_id, api_key=api_key)}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except Exception as exc:

View File

@@ -8,9 +8,13 @@ from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import ROOT_DIR
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.system_log import AuditLog, SystemLog
from app.models.user import User
from app.services.persistent_logs import record_audit_log, record_system_log
from app.services.system_control import (
@@ -310,7 +314,120 @@ async def get_system_log_sources(
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
return {"items": list_log_sources()}
return {
"items": [
*list_log_sources(),
{
"source_id": "system-db",
"name": "系统事件",
"kind": "database",
"location": "table://system_logs",
"description": "后端持久化系统事件、AI 和采集器操作日志。",
"category": "database",
"status": "ok",
},
{
"source_id": "audit-db",
"name": "审计事件",
"kind": "database",
"location": "table://audit_logs",
"description": "管理员敏感操作和密钥 reveal 审计记录。",
"category": "audit",
"status": "ok",
},
]
}
async def read_database_log_snapshot(
source_id: str,
*,
limit: int,
level: str,
levels: str | None,
start_date: str | None,
end_date: str | None,
search: str | None,
db: AsyncSession,
) -> dict | None:
selected_levels = set(normalize_log_level(item) for item in (levels or level).split(",") if item.strip())
selected_levels.discard("all")
search_query = (search or "").strip().lower()
lines: list[str] = []
if source_id == "system-db":
query = select(SystemLog).order_by(SystemLog.occurred_at.desc().nullslast(), SystemLog.id.desc()).limit(limit * 5)
result = await db.execute(query)
records = result.scalars().all()
for record in records:
record_level = normalize_log_level(record.level)
if selected_levels and record_level not in selected_levels:
continue
occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else ""
if start_date and occurred_at and occurred_at < start_date:
continue
if end_date and occurred_at and occurred_at > end_date:
continue
line = " ".join(
part
for part in [
record.occurred_at.isoformat() if record.occurred_at else "",
record_level.upper(),
record.source,
record.event or "",
record.message,
]
if part
)
if search_query and search_query not in line.lower():
continue
lines.append(line)
elif source_id == "audit-db":
query = select(AuditLog).order_by(AuditLog.occurred_at.desc().nullslast(), AuditLog.id.desc()).limit(limit * 5)
result = await db.execute(query)
records = result.scalars().all()
for record in records:
occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else ""
if start_date and occurred_at and occurred_at < start_date:
continue
if end_date and occurred_at and occurred_at > end_date:
continue
line = " ".join(
part
for part in [
record.occurred_at.isoformat() if record.occurred_at else "",
"INFO",
record.action,
record.target_type or "",
record.target_id or "",
record.result or "",
]
if part
)
if search_query and search_query not in line.lower():
continue
lines.append(line)
else:
return None
lines = list(reversed(lines[:limit]))
return {
"source_id": source_id,
"name": "系统事件" if source_id == "system-db" else "审计事件",
"kind": "database",
"location": "table://system_logs" if source_id == "system-db" else "table://audit_logs",
"description": "数据库持久化日志",
"category": "database" if source_id == "system-db" else "audit",
"status": "ok" if lines else "empty",
"level": level,
"selected_levels": sorted(selected_levels),
"search_query": search or "",
"available_levels": ["all", "error", "warning", "info", "debug"],
"daily_markers": [],
"line_limit": limit,
"line_count": len(lines),
"lines": lines,
}
@router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse)
@@ -323,6 +440,7 @@ async def get_system_log_snapshot(
end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"),
search: str | None = Query(None, description="Case-insensitive substring search"),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
ensure_super_admin(current_user)
@@ -345,15 +463,26 @@ async def get_system_log_snapshot(
if normalized_start_date and normalized_end_date and normalized_start_date > normalized_end_date:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="start_date must be earlier than or equal to end_date")
snapshot = read_log_snapshot(
snapshot = await read_database_log_snapshot(
source_id,
limit,
limit=limit,
level=level,
levels=levels,
start_date=normalized_start_date,
end_date=normalized_end_date,
search=search,
db=db,
)
if snapshot is None:
snapshot = read_log_snapshot(
source_id,
limit,
level=level,
levels=levels,
start_date=normalized_start_date,
end_date=normalized_end_date,
search=search,
)
if snapshot is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Log source not found")
return snapshot

View File

@@ -2159,6 +2159,24 @@ async def collect_compute_center_location(
llm_failure_reason = llm_result.failure_reason
if not candidates:
logger.warning_event(
"Compute center location collection returned no candidates",
event="visualization.compute_center.location_collect.completed",
context={
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": False,
"llm_failure_reason": llm_failure_reason,
"attempted_queries": list(attempted_queries),
"context": {
"operator": operator,
"site": site,
"city": city,
"country": country,
},
},
)
return {
"source_id": source_id,
"record_id": record_id,
@@ -2180,13 +2198,34 @@ async def collect_compute_center_location(
},
}
best_candidate = candidates[0].to_dict()
logger.info_event(
"Compute center location collection returned candidates",
event="visualization.compute_center.location_collect.completed",
context={
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": True,
"candidate_count": len(candidates),
"best_candidate": best_candidate,
"llm_failure_reason": llm_failure_reason,
"attempted_queries": list(attempted_queries),
"context": {
"operator": operator,
"site": site,
"city": city,
"country": country,
},
},
)
return {
"source_id": source_id,
"record_id": record_id,
"name": name,
"success": True,
"candidates": [candidate.to_dict() for candidate in candidates],
"best_candidate": candidates[0].to_dict(),
"best_candidate": best_candidate,
"attempted_queries": list(attempted_queries),
"context": {
"name": name,

View File

@@ -14,6 +14,7 @@ from app.core.websocket.manager import manager
logger = get_logger(__name__, service="api")
router = APIRouter()
EARTH_UPDATES_CHANNEL = "earth_updates"
async def authenticate_token(token: str) -> Optional[dict]:
@@ -58,7 +59,7 @@ async def websocket_endpoint(
is_anonymous = payload is None
user_id = str(payload.get("sub")) if payload else f"anonymous:{id(websocket)}"
supported_channels = ["vessels", "earth_news"] if is_anonymous else [
supported_channels = ["vessels", "earth_news", EARTH_UPDATES_CHANNEL] if is_anonymous else [
"gpu_clusters",
"submarine_cables",
"ixp_nodes",
@@ -67,6 +68,7 @@ async def websocket_endpoint(
"datasource_tasks",
"vessels",
"earth_news",
EARTH_UPDATES_CHANNEL,
]
await manager.connect(websocket, user_id)