release: bump version to 0.62.0
This commit is contained in:
@@ -5,7 +5,7 @@ from datetime import datetime
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from sqlalchemy import delete, select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from pydantic import BaseModel, Field
|
||||
@@ -45,14 +45,70 @@ from app.services.custom_datasource_runtime import (
|
||||
|
||||
DATASOURCE_MAPPING_PROMPT_KEY = "datasource.mapping"
|
||||
from app.services.datasource_connectivity import (
|
||||
_resolve_aisstream_api_key,
|
||||
_resolve_spacetrack_credentials_with_override,
|
||||
get_builtin_connection_status,
|
||||
save_connectivity_success,
|
||||
strip_connectivity_validation,
|
||||
test_builtin_connectivity,
|
||||
)
|
||||
from app.services.barentswatch import resolve_barentswatch_config
|
||||
from app.services.persistent_logs import record_audit_log
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
|
||||
|
||||
|
||||
def _user_role_value(user: User) -> str:
|
||||
role = getattr(user, "role", "")
|
||||
return str(getattr(role, "value", role) or "").lower()
|
||||
|
||||
|
||||
def _user_display_name(user: User) -> str:
|
||||
return str(getattr(user, "username", None) or getattr(user, "email", None) or getattr(user, "id", ""))
|
||||
|
||||
|
||||
async def _record_datasource_secret_reveal(
|
||||
*,
|
||||
current_user: User,
|
||||
request: Request,
|
||||
target_id: str,
|
||||
result: str,
|
||||
details: dict[str, Any],
|
||||
) -> None:
|
||||
await record_audit_log(
|
||||
action="datasource_config.secret.reveal",
|
||||
actor_id=getattr(current_user, "id", None),
|
||||
actor_name=_user_display_name(current_user),
|
||||
target_type="datasource_config_secret",
|
||||
target_id=target_id,
|
||||
result=result,
|
||||
ip=request.client.host if request.client else None,
|
||||
details=details,
|
||||
)
|
||||
|
||||
|
||||
async def _ensure_datasource_secret_reveal_allowed(
|
||||
current_user: User,
|
||||
request: Request,
|
||||
target_id: str,
|
||||
details: dict[str, Any],
|
||||
) -> None:
|
||||
if _user_role_value(current_user) in SECRET_REVEAL_ROLES:
|
||||
return
|
||||
await _record_datasource_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=target_id,
|
||||
result="denied",
|
||||
details={**details, "role": _user_role_value(current_user)},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Only administrators can reveal datasource credentials",
|
||||
)
|
||||
|
||||
def _default_builtin_config(name: str) -> dict[str, Any]:
|
||||
return {"timeout": 30, "retry": 3}
|
||||
|
||||
@@ -389,10 +445,14 @@ async def list_all_datasources(
|
||||
db_config = db_configs.get(name)
|
||||
default_config = _default_builtin_config(name)
|
||||
default_url = yaml_url
|
||||
db_auth_config = db_config.auth_config or {} if db_config else {}
|
||||
|
||||
result.append(
|
||||
{
|
||||
"name": name,
|
||||
"requires_credentials": bool(metadata.get("requires_credentials", False)),
|
||||
"credential_provider": metadata.get("credential_provider"),
|
||||
"credential_status": metadata.get("credential_status", "none"),
|
||||
"default_url": default_url,
|
||||
"endpoint": db_config.endpoint if db_config else default_url,
|
||||
"is_overridden": db_config is not None and db_config.endpoint != yaml_url
|
||||
@@ -401,10 +461,20 @@ async def list_all_datasources(
|
||||
"is_active": db_config.is_active if db_config else True,
|
||||
"source_type": db_config.source_type if db_config else _default_builtin_source_type(name),
|
||||
"auth_type": db_config.auth_type if db_config else "none",
|
||||
"auth_config": {
|
||||
"client_id": db_auth_config.get("client_id") or "",
|
||||
"username": db_auth_config.get("username") or "",
|
||||
"key_name": db_auth_config.get("key_name") or db_auth_config.get("param_name") or "",
|
||||
"param_name": db_auth_config.get("param_name") or db_auth_config.get("key_name") or "",
|
||||
"location": db_auth_config.get("location") or db_auth_config.get("in") or "",
|
||||
"in": db_auth_config.get("in") or db_auth_config.get("location") or "",
|
||||
},
|
||||
"auth_configured": {
|
||||
"api_key": bool((db_config.auth_config or {}).get("api_key"))
|
||||
if db_config
|
||||
else False,
|
||||
"api_key": bool(db_auth_config.get("api_key")),
|
||||
"client_id": bool(db_auth_config.get("client_id")),
|
||||
"client_secret": bool(db_auth_config.get("client_secret")),
|
||||
"username": bool(db_auth_config.get("username")),
|
||||
"password": bool(db_auth_config.get("password")),
|
||||
},
|
||||
"headers": db_config.headers if db_config else {},
|
||||
"config": strip_connectivity_validation(db_config.config if db_config else default_config),
|
||||
@@ -418,6 +488,96 @@ async def list_all_datasources(
|
||||
return {"total": len(result), "data": result}
|
||||
|
||||
|
||||
@router.get("/configs/secrets")
|
||||
async def reveal_builtin_config_secrets(
|
||||
request: Request,
|
||||
name: str = Query(..., min_length=1),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Reveal configured built-in datasource credentials for admin editing."""
|
||||
source = name.strip()
|
||||
metadata = DEFAULT_DATASOURCES.get(source)
|
||||
if not metadata or not metadata.get("requires_credentials"):
|
||||
raise HTTPException(status_code=404, detail="Credentialed datasource config not found")
|
||||
|
||||
provider = str(metadata.get("credential_provider") or "")
|
||||
target_id = f"datasource_config:{source}"
|
||||
await _ensure_datasource_secret_reveal_allowed(
|
||||
current_user,
|
||||
request,
|
||||
target_id,
|
||||
{"source": source, "provider": provider},
|
||||
)
|
||||
|
||||
result = await db.execute(select(DataSourceConfig).where(DataSourceConfig.name == source))
|
||||
record = result.scalar_one_or_none()
|
||||
auth_config = dict(record.auth_config or {}) if record else {}
|
||||
payload: dict[str, Any] = {
|
||||
"name": source,
|
||||
"provider": provider,
|
||||
}
|
||||
details: dict[str, Any] = {"source": source, "provider": provider}
|
||||
|
||||
if provider == "barentswatch":
|
||||
resolved = await resolve_barentswatch_config(db)
|
||||
client_id = str(auth_config.get("client_id") or resolved.client_id or "")
|
||||
client_secret = str(auth_config.get("client_secret") or resolved.client_secret or "")
|
||||
source_label = "datasource_config" if auth_config.get("client_id") or auth_config.get("client_secret") else resolved.credential_source
|
||||
payload.update(
|
||||
{
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"client_id_source": source_label if client_id else "missing",
|
||||
"client_secret_source": source_label if client_secret else "missing",
|
||||
}
|
||||
)
|
||||
details.update(
|
||||
{
|
||||
"client_id_configured": bool(client_id),
|
||||
"client_secret_configured": bool(client_secret),
|
||||
"credential_source": source_label,
|
||||
}
|
||||
)
|
||||
elif provider == "aisstream":
|
||||
api_key, api_key_source = await _resolve_aisstream_api_key(db)
|
||||
payload.update({"api_key": api_key, "api_key_source": api_key_source})
|
||||
details.update({"api_key_configured": bool(api_key), "api_key_source": api_key_source})
|
||||
elif provider == "spacetrack":
|
||||
if auth_config.get("username") or auth_config.get("password"):
|
||||
username = str(auth_config.get("username") or "")
|
||||
password = str(auth_config.get("password") or "")
|
||||
credential_source = "datasource_config"
|
||||
else:
|
||||
username, password, credential_source = _resolve_spacetrack_credentials_with_override()
|
||||
payload.update(
|
||||
{
|
||||
"username": username,
|
||||
"password": password,
|
||||
"username_source": credential_source if username else "missing",
|
||||
"password_source": credential_source if password else "missing",
|
||||
}
|
||||
)
|
||||
details.update(
|
||||
{
|
||||
"username_configured": bool(username),
|
||||
"password_configured": bool(password),
|
||||
"credential_source": credential_source,
|
||||
}
|
||||
)
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="Datasource credential provider is not supported")
|
||||
|
||||
await _record_datasource_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=target_id,
|
||||
result="success",
|
||||
details=details,
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
@router.get("/configs/{config_id}")
|
||||
async def get_config(
|
||||
config_id: int,
|
||||
|
||||
@@ -7,6 +7,7 @@ from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, or_, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.cache import cache
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.security import get_current_user
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
@@ -26,6 +27,7 @@ from app.services.scheduler import (
|
||||
run_collector_now,
|
||||
sync_datasource_job,
|
||||
)
|
||||
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
|
||||
|
||||
router = APIRouter()
|
||||
STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90
|
||||
@@ -235,6 +237,8 @@ async def _load_datasource_endpoint_overrides(
|
||||
async def _load_datasource_list_context(
|
||||
db: AsyncSession,
|
||||
datasources: list[DataSource],
|
||||
*,
|
||||
include_endpoint: bool = True,
|
||||
) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, str]]:
|
||||
datasource_ids = [datasource.id for datasource in datasources]
|
||||
sources = [datasource.source for datasource in datasources]
|
||||
@@ -260,10 +264,65 @@ async def _load_datasource_list_context(
|
||||
running_tasks = await _load_latest_running_tasks(db, datasource_ids)
|
||||
|
||||
latest_tasks = await _load_latest_tasks(db, datasource_ids)
|
||||
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
|
||||
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources) if include_endpoint else {}
|
||||
return running_tasks, latest_tasks, endpoint_overrides
|
||||
|
||||
|
||||
def serialize_datasource_row(
|
||||
datasource: DataSource,
|
||||
*,
|
||||
running_tasks: dict[int, CollectionTask],
|
||||
latest_tasks: dict[int, CollectionTask],
|
||||
record_counts: dict[str, int],
|
||||
endpoint_overrides: dict[str, str],
|
||||
config,
|
||||
include_endpoint: bool,
|
||||
) -> dict:
|
||||
running_task = running_tasks.get(datasource.id)
|
||||
latest_task = latest_tasks.get(datasource.id)
|
||||
display_task = running_task or latest_task
|
||||
endpoint = None
|
||||
if include_endpoint:
|
||||
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
|
||||
last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None)
|
||||
last_status = datasource.last_status or (latest_task.status if latest_task else None)
|
||||
collected_records = record_counts.get(datasource.source, 0)
|
||||
|
||||
row = {
|
||||
"id": datasource.id,
|
||||
"source": datasource.source,
|
||||
"name": datasource.name,
|
||||
**datasource_metadata(datasource.source),
|
||||
"product": datasource_product_key(datasource),
|
||||
"module": datasource.module,
|
||||
"priority": datasource.priority,
|
||||
"frequency": format_frequency_label(datasource.frequency_minutes),
|
||||
"frequency_minutes": datasource.frequency_minutes,
|
||||
"is_active": datasource.is_active,
|
||||
"collector_class": datasource.collector_class,
|
||||
"last_run": to_iso8601_utc(last_run_at),
|
||||
"last_run_at": to_iso8601_utc(last_run_at),
|
||||
"last_status": last_status,
|
||||
"is_running": running_task is not None,
|
||||
"task_id": display_task.id if display_task else None,
|
||||
"progress": display_task.progress if display_task else None,
|
||||
"phase": display_task.phase if display_task else None,
|
||||
"phase_progress": display_task.phase_progress if display_task else None,
|
||||
"phase_message": display_task.phase_message if display_task else None,
|
||||
"phase_current": display_task.phase_current if display_task else None,
|
||||
"phase_total": display_task.phase_total if display_task else None,
|
||||
"phase_unit": display_task.phase_unit if display_task else None,
|
||||
"records_processed": display_task.records_processed if display_task else None,
|
||||
"total_records": display_task.total_records if display_task else None,
|
||||
"error_message": display_task.error_message if display_task else None,
|
||||
"collected_records": collected_records,
|
||||
"has_collected_data": collected_records > 0,
|
||||
}
|
||||
if include_endpoint:
|
||||
row["endpoint"] = endpoint
|
||||
return row
|
||||
|
||||
|
||||
def _apply_datasource_query_filters(
|
||||
query,
|
||||
*,
|
||||
@@ -658,6 +717,7 @@ async def list_datasources(
|
||||
collected: Optional[bool] = None,
|
||||
credential_status: Optional[str] = None,
|
||||
q: Optional[str] = None,
|
||||
include_endpoint: bool = True,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
@@ -676,7 +736,11 @@ async def list_datasources(
|
||||
|
||||
collector_list = []
|
||||
config = get_data_sources_config()
|
||||
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
|
||||
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(
|
||||
db,
|
||||
datasources,
|
||||
include_endpoint=include_endpoint,
|
||||
)
|
||||
record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
|
||||
datasources = _filter_datasources_in_memory(
|
||||
datasources,
|
||||
@@ -689,46 +753,16 @@ async def list_datasources(
|
||||
credential_status=credential_status,
|
||||
)
|
||||
for datasource in datasources:
|
||||
running_task = running_tasks.get(datasource.id)
|
||||
latest_task = latest_tasks.get(datasource.id)
|
||||
display_task = running_task or latest_task
|
||||
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
|
||||
last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None)
|
||||
last_status = datasource.last_status or (latest_task.status if latest_task else None)
|
||||
collected_records = record_counts.get(datasource.source, 0)
|
||||
|
||||
collector_list.append(
|
||||
{
|
||||
"id": datasource.id,
|
||||
"source": datasource.source,
|
||||
"name": datasource.name,
|
||||
**datasource_metadata(datasource.source),
|
||||
"product": datasource_product_key(datasource),
|
||||
"module": datasource.module,
|
||||
"priority": datasource.priority,
|
||||
"frequency": format_frequency_label(datasource.frequency_minutes),
|
||||
"frequency_minutes": datasource.frequency_minutes,
|
||||
"is_active": datasource.is_active,
|
||||
"collector_class": datasource.collector_class,
|
||||
"endpoint": endpoint,
|
||||
"last_run": to_iso8601_utc(last_run_at),
|
||||
"last_run_at": to_iso8601_utc(last_run_at),
|
||||
"last_status": last_status,
|
||||
"is_running": running_task is not None,
|
||||
"task_id": display_task.id if display_task else None,
|
||||
"progress": display_task.progress if display_task else None,
|
||||
"phase": display_task.phase if display_task else None,
|
||||
"phase_progress": display_task.phase_progress if display_task else None,
|
||||
"phase_message": display_task.phase_message if display_task else None,
|
||||
"phase_current": display_task.phase_current if display_task else None,
|
||||
"phase_total": display_task.phase_total if display_task else None,
|
||||
"phase_unit": display_task.phase_unit if display_task else None,
|
||||
"records_processed": display_task.records_processed if display_task else None,
|
||||
"total_records": display_task.total_records if display_task else None,
|
||||
"error_message": display_task.error_message if display_task else None,
|
||||
"collected_records": collected_records,
|
||||
"has_collected_data": collected_records > 0,
|
||||
}
|
||||
serialize_datasource_row(
|
||||
datasource,
|
||||
running_tasks=running_tasks,
|
||||
latest_tasks=latest_tasks,
|
||||
record_counts=record_counts,
|
||||
endpoint_overrides=endpoint_overrides,
|
||||
config=config,
|
||||
include_endpoint=include_endpoint,
|
||||
)
|
||||
)
|
||||
|
||||
return {"total": len(collector_list), "data": collector_list}
|
||||
@@ -785,6 +819,51 @@ async def trigger_datasource_batch(
|
||||
return await _trigger_datasource_batch(db, datasources, force=payload.force)
|
||||
|
||||
|
||||
@router.get("/snapshots")
|
||||
async def list_datasource_snapshots(
|
||||
source_id: Optional[str] = None,
|
||||
current_only: Optional[bool] = None,
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
query = (
|
||||
select(DataSnapshot, DataSource.name, DataSource.module)
|
||||
.outerjoin(DataSource, DataSource.id == DataSnapshot.datasource_id)
|
||||
.order_by(DataSnapshot.created_at.desc().nullslast(), DataSnapshot.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if source_id:
|
||||
query = query.where(DataSnapshot.source == source_id)
|
||||
if current_only is not None:
|
||||
query = query.where(DataSnapshot.is_current.is_(current_only))
|
||||
|
||||
result = await db.execute(query)
|
||||
rows = []
|
||||
for snapshot, datasource_name, datasource_module in result.all():
|
||||
rows.append(
|
||||
{
|
||||
"id": snapshot.id,
|
||||
"datasource_id": snapshot.datasource_id,
|
||||
"datasource_name": datasource_name,
|
||||
"module": datasource_module,
|
||||
"task_id": snapshot.task_id,
|
||||
"source": snapshot.source,
|
||||
"snapshot_key": snapshot.snapshot_key,
|
||||
"reference_date": to_iso8601_utc(snapshot.reference_date),
|
||||
"started_at": to_iso8601_utc(snapshot.started_at),
|
||||
"completed_at": to_iso8601_utc(snapshot.completed_at),
|
||||
"record_count": snapshot.record_count,
|
||||
"status": snapshot.status,
|
||||
"is_current": snapshot.is_current,
|
||||
"parent_snapshot_id": snapshot.parent_snapshot_id,
|
||||
"summary": snapshot.summary or {},
|
||||
"created_at": to_iso8601_utc(snapshot.created_at),
|
||||
}
|
||||
)
|
||||
return {"total": len(rows), "data": rows}
|
||||
|
||||
|
||||
@router.get("/{source_id}")
|
||||
async def get_datasource(
|
||||
source_id: str,
|
||||
@@ -813,6 +892,37 @@ async def get_datasource(
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{source_id}/row")
|
||||
async def get_datasource_row(
|
||||
source_id: str,
|
||||
include_endpoint: bool = True,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
datasource = await get_datasource_record(db, source_id)
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
config = get_data_sources_config()
|
||||
running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(
|
||||
db,
|
||||
[datasource],
|
||||
include_endpoint=include_endpoint,
|
||||
)
|
||||
record_counts = await _load_collected_record_counts(db, [datasource.source])
|
||||
return {
|
||||
"data": serialize_datasource_row(
|
||||
datasource,
|
||||
running_tasks=running_tasks,
|
||||
latest_tasks=latest_tasks,
|
||||
record_counts=record_counts,
|
||||
endpoint_overrides=endpoint_overrides,
|
||||
config=config,
|
||||
include_endpoint=include_endpoint,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/enable")
|
||||
async def enable_datasource(
|
||||
source_id: str,
|
||||
@@ -960,6 +1070,29 @@ async def clear_datasource_data(
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{source_id}/cache")
|
||||
async def clear_datasource_cache(
|
||||
source_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
datasource = await get_datasource_record(db, source_id)
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
earth_deleted_count = invalidate_earth_layer_cache_for_source(datasource.source)
|
||||
dashboard_deleted_count = int(cache.delete("dashboard:stats")) + int(cache.delete("dashboard:summary"))
|
||||
deleted_count = earth_deleted_count + dashboard_deleted_count
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Cleared {deleted_count} cache keys for data source '{datasource.name}'",
|
||||
"deleted_count": deleted_count,
|
||||
"earth_layer_deleted_count": earth_deleted_count,
|
||||
"dashboard_deleted_count": dashboard_deleted_count,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{source_id}/task-status")
|
||||
async def get_task_status(
|
||||
source_id: str,
|
||||
|
||||
@@ -4,7 +4,8 @@ import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
import httpx
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
from dotenv import dotenv_values
|
||||
from sqlalchemy import select
|
||||
@@ -64,10 +65,12 @@ from app.services.llm_provider_catalog import (
|
||||
)
|
||||
from app.services.scheduler import sync_datasource_job
|
||||
from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings
|
||||
from app.services.persistent_logs import record_audit_log
|
||||
|
||||
router = APIRouter()
|
||||
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5
|
||||
AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test"
|
||||
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
"system": {
|
||||
@@ -130,6 +133,70 @@ DEFAULT_SETTINGS = {
|
||||
}
|
||||
|
||||
|
||||
def _user_role_value(user: User) -> str:
|
||||
role = getattr(user, "role", "")
|
||||
return role.value if hasattr(role, "value") else str(role or "")
|
||||
|
||||
|
||||
def _user_display_name(user: User) -> str | None:
|
||||
return getattr(user, "username", None) or getattr(user, "email", None)
|
||||
|
||||
|
||||
def _request_client_ip(request: Request | None) -> str | None:
|
||||
if request is None or request.client is None:
|
||||
return None
|
||||
return request.client.host
|
||||
|
||||
|
||||
def _can_reveal_integration_secrets(user: User) -> bool:
|
||||
return _user_role_value(user) in SECRET_REVEAL_ROLES
|
||||
|
||||
|
||||
async def _record_integration_secret_reveal(
|
||||
*,
|
||||
current_user: User,
|
||||
request: Request | None,
|
||||
target_id: str,
|
||||
result: str,
|
||||
details: dict,
|
||||
) -> None:
|
||||
await record_audit_log(
|
||||
action="settings.integration_secret.reveal",
|
||||
actor_id=getattr(current_user, "id", None),
|
||||
actor_name=_user_display_name(current_user),
|
||||
target_type="integration_secret",
|
||||
target_id=target_id,
|
||||
result=result,
|
||||
ip=_request_client_ip(request),
|
||||
details=details,
|
||||
)
|
||||
|
||||
|
||||
async def _ensure_secret_reveal_allowed(
|
||||
*,
|
||||
current_user: User,
|
||||
request: Request | None,
|
||||
target_id: str,
|
||||
details: dict | None = None,
|
||||
) -> None:
|
||||
if _can_reveal_integration_secrets(current_user):
|
||||
return
|
||||
await _record_integration_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=target_id,
|
||||
result="denied",
|
||||
details={
|
||||
**(details or {}),
|
||||
"role": _user_role_value(current_user),
|
||||
},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Only administrators can reveal integration secrets",
|
||||
)
|
||||
|
||||
|
||||
class SystemSettingsUpdate(BaseModel):
|
||||
system_name: str = "智能星球"
|
||||
refresh_interval: int = Field(default=60, ge=10, le=3600)
|
||||
@@ -353,9 +420,10 @@ def _get_provider_preset(provider: str) -> dict:
|
||||
"provider": provider,
|
||||
"provider_api": "openai-completions",
|
||||
"base_url": "",
|
||||
"model": "",
|
||||
"models": [],
|
||||
"api_key_env": "",
|
||||
"model": "",
|
||||
"models": [],
|
||||
"model_provider_apis": {},
|
||||
"api_key_env": "",
|
||||
}
|
||||
|
||||
|
||||
@@ -377,6 +445,9 @@ def _resolve_env_secret(*names: str) -> tuple[str, str]:
|
||||
value = env_file_values.get(name)
|
||||
if value:
|
||||
return value, "env_file"
|
||||
value = os.environ.get(name)
|
||||
if value:
|
||||
return value, "env"
|
||||
return "", ""
|
||||
|
||||
|
||||
@@ -419,9 +490,16 @@ def _provider_defaults(provider: str) -> dict:
|
||||
1200 if preset.get("provider_api") == "anthropic-messages" else 4096
|
||||
),
|
||||
"anthropic_version": "2023-06-01",
|
||||
"model_provider_apis": preset.get("model_provider_apis") or {},
|
||||
}
|
||||
|
||||
|
||||
def _selected_ai_env_provider() -> str:
|
||||
env_file_values = _read_ai_provider_env_file()
|
||||
provider = env_file_values.get("AI_PROVIDER") or os.environ.get("AI_PROVIDER") or "minimax"
|
||||
return _normalize_provider_id(provider)
|
||||
|
||||
|
||||
def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict:
|
||||
raw = dict(ai_payload or {})
|
||||
default_provider = _normalize_provider_id(raw.get("default_provider") or raw.get("provider"))
|
||||
@@ -440,6 +518,7 @@ def _normalize_ai_provider_payload(ai_payload: dict | None) -> dict:
|
||||
"api_key",
|
||||
"max_tokens",
|
||||
"anthropic_version",
|
||||
"model_provider_apis",
|
||||
)
|
||||
if raw.get(key) not in (None, "")
|
||||
}
|
||||
@@ -477,7 +556,12 @@ def _resolve_provider_api_key(provider: str, provider_config: dict) -> tuple[str
|
||||
return str(saved_key), "runtime"
|
||||
preset = _get_provider_preset(provider)
|
||||
api_key_env = preset.get("api_key_env") or ""
|
||||
return _resolve_env_secret(api_key_env, "AI_API_KEY")
|
||||
value, source = _resolve_env_secret(api_key_env)
|
||||
if value:
|
||||
return value, source
|
||||
if _normalize_provider_id(provider) == _selected_ai_env_provider():
|
||||
return _resolve_env_secret("AI_API_KEY")
|
||||
return "", ""
|
||||
|
||||
|
||||
def _resolve_service_token(ai_payload: dict) -> tuple[str, str]:
|
||||
@@ -509,7 +593,12 @@ def _is_secret_placeholder(value: Optional[str], current_preview: str = "") -> b
|
||||
|
||||
def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrationUpdate) -> dict:
|
||||
current_ai = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
|
||||
provider_id = _normalize_provider_id(update.default_provider or update.provider)
|
||||
provider_id = _normalize_provider_id(update.provider)
|
||||
default_provider = (
|
||||
_normalize_provider_id(update.default_provider)
|
||||
if update.default_provider is not None
|
||||
else current_ai["default_provider"]
|
||||
)
|
||||
current_providers = {
|
||||
provider: dict(config or {})
|
||||
for provider, config in current_ai.get("providers", {}).items()
|
||||
@@ -543,7 +632,7 @@ def _build_ai_provider_payload(current_payload: dict, update: AIProviderIntegrat
|
||||
"service_url": update.service_url.strip()
|
||||
or app_settings.AI_PROVIDER_SERVICE_URL,
|
||||
"service_token": current_ai.get("service_token") or "",
|
||||
"default_provider": provider_id,
|
||||
"default_provider": default_provider,
|
||||
"providers": current_providers,
|
||||
"timeout_seconds": update.timeout_seconds,
|
||||
"retry_attempts": update.retry_attempts,
|
||||
@@ -577,6 +666,8 @@ def _runtime_config_from_ai_payload(ai_payload: dict) -> dict:
|
||||
"api_key": api_key,
|
||||
"max_tokens": int(provider_config.get("max_tokens") or 1200),
|
||||
"anthropic_version": provider_config.get("anthropic_version") or "2023-06-01",
|
||||
"model_provider_apis": provider_config.get("model_provider_apis") or {},
|
||||
"preset_models": _get_provider_preset(default_provider).get("models") or [],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -631,6 +722,159 @@ async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _join_provider_url(base_url: str, path: str) -> str:
|
||||
return f"{base_url.rstrip('/')}/{path.lstrip('/')}"
|
||||
|
||||
|
||||
def _extract_model_ids(payload: dict) -> list[str]:
|
||||
data = payload.get("data") if isinstance(payload, dict) else None
|
||||
if isinstance(data, list):
|
||||
return [
|
||||
str(item.get("id"))
|
||||
for item in data
|
||||
if isinstance(item, dict) and item.get("id")
|
||||
]
|
||||
models = payload.get("models") if isinstance(payload, dict) else None
|
||||
if isinstance(models, list):
|
||||
return [
|
||||
str(item.get("name") or item.get("model") or item.get("id") or item)
|
||||
for item in models
|
||||
if item
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
def _contains_model(model_ids: list[str], model: str) -> bool:
|
||||
normalized_model = model.strip().lower()
|
||||
return any(str(item).strip().lower() == normalized_model for item in model_ids)
|
||||
|
||||
|
||||
async def _check_ai_provider_lightweight(llm_config: dict, timeout_seconds: int) -> dict:
|
||||
provider = _normalize_provider_id(llm_config.get("provider") or "")
|
||||
configured_api = str(llm_config.get("provider_api") or "").strip() or "openai-completions"
|
||||
model = str(llm_config.get("model") or "").strip()
|
||||
base_url = str(llm_config.get("base_url") or "").strip().rstrip("/")
|
||||
api_key = str(llm_config.get("api_key") or "").strip()
|
||||
provider_api = configured_api
|
||||
model_provider_apis = llm_config.get("model_provider_apis")
|
||||
if isinstance(model_provider_apis, dict):
|
||||
provider_api = str(model_provider_apis.get(model) or provider_api)
|
||||
preset_models = [
|
||||
str(item)
|
||||
for item in (llm_config.get("preset_models") or [])
|
||||
if str(item).strip()
|
||||
]
|
||||
|
||||
if not provider or not base_url or not model:
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": "当前 provider/base_url/model 未完整配置。",
|
||||
"mode": "lightweight_config",
|
||||
}
|
||||
if provider_api != "ollama-generate" and not api_key:
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": "当前 provider 未配置 API Key。",
|
||||
"mode": "lightweight_config",
|
||||
}
|
||||
|
||||
if provider == "opencode-go":
|
||||
url = _join_provider_url(base_url, "/models")
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
elif provider_api == "ollama-generate":
|
||||
url = _join_provider_url(base_url, "/api/tags")
|
||||
headers: dict[str, str] = {}
|
||||
elif provider_api == "openai-completions":
|
||||
url = _join_provider_url(base_url, "/models")
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
elif provider_api == "anthropic-messages":
|
||||
url = _join_provider_url(base_url, "/models")
|
||||
headers = {
|
||||
"x-api-key": api_key,
|
||||
"anthropic-version": str(llm_config.get("anthropic_version") or "2023-06-01"),
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": f"当前 provider_api 不支持轻量连通性测试: {provider_api}",
|
||||
"mode": "lightweight_unsupported",
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=min(timeout_seconds, AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS)) as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
detail = exc.response.text or exc.response.reason_phrase
|
||||
if exc.response.status_code == 404 and _contains_model(preset_models, model):
|
||||
return {
|
||||
"success": True,
|
||||
"connected": True,
|
||||
"message": "轻量连通性测试通过;当前 provider 不提供可用的模型目录,已按内置模型预设确认。",
|
||||
"mode": "lightweight_preset",
|
||||
"provider": provider,
|
||||
"provider_api": provider_api,
|
||||
"model": model,
|
||||
"url": url,
|
||||
}
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": f"轻量连通性测试失败: HTTP {exc.response.status_code} {detail}",
|
||||
"mode": "lightweight_models",
|
||||
"url": url,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": f"轻量连通性测试失败: {exc}",
|
||||
"mode": "lightweight_models",
|
||||
"url": url,
|
||||
}
|
||||
|
||||
model_ids = _extract_model_ids(payload)
|
||||
if model_ids and not _contains_model(model_ids, model):
|
||||
if _contains_model(preset_models, model):
|
||||
return {
|
||||
"success": True,
|
||||
"connected": True,
|
||||
"message": "轻量连通性测试通过;provider 模型目录未返回当前别名,已按内置模型预设确认。",
|
||||
"mode": "lightweight_models_with_preset_alias",
|
||||
"provider": provider,
|
||||
"provider_api": provider_api,
|
||||
"model": model,
|
||||
"models_count": len(model_ids),
|
||||
"url": url,
|
||||
}
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": f"连接可用,但模型目录中没有当前模型: {model}",
|
||||
"mode": "lightweight_models",
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"models_count": len(model_ids),
|
||||
"url": url,
|
||||
}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"connected": True,
|
||||
"message": "轻量连通性测试通过",
|
||||
"mode": "lightweight_models",
|
||||
"provider": provider,
|
||||
"provider_api": provider_api,
|
||||
"model": model,
|
||||
"models_count": len(model_ids),
|
||||
"url": url,
|
||||
}
|
||||
|
||||
|
||||
def _web_search_provider_defaults(provider: str) -> dict:
|
||||
return web_search_provider_defaults(provider).model_dump()
|
||||
|
||||
@@ -691,12 +935,21 @@ def _normalize_web_search_payload(web_search_payload: dict | None) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _resolve_web_search_api_key(provider: str, provider_config: dict) -> tuple[str, str]:
|
||||
def _resolve_web_search_api_key(
|
||||
provider: str,
|
||||
provider_config: dict,
|
||||
default_provider: str | None = None,
|
||||
) -> tuple[str, str]:
|
||||
saved_key = provider_config.get("api_key") or ""
|
||||
if saved_key:
|
||||
return str(saved_key), "runtime"
|
||||
preset = get_web_search_provider_preset(provider)
|
||||
return _resolve_web_search_env_secret(preset.get("api_key_env") or "", "WEB_SEARCH_API_KEY")
|
||||
value, source = _resolve_web_search_env_secret(preset.get("api_key_env") or "")
|
||||
if value:
|
||||
return value, source
|
||||
if normalize_web_search_provider(provider) == normalize_web_search_provider(default_provider or "tavily"):
|
||||
return _resolve_web_search_env_secret("WEB_SEARCH_API_KEY")
|
||||
return "", ""
|
||||
|
||||
|
||||
def _build_web_search_payload(
|
||||
@@ -706,13 +959,22 @@ def _build_web_search_payload(
|
||||
current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {})
|
||||
if update is None:
|
||||
return current_web_search
|
||||
provider_id = normalize_web_search_provider(update.default_provider or update.provider)
|
||||
provider_id = normalize_web_search_provider(update.provider)
|
||||
default_provider = (
|
||||
normalize_web_search_provider(update.default_provider)
|
||||
if update.default_provider is not None
|
||||
else current_web_search["default_provider"]
|
||||
)
|
||||
current_providers = {
|
||||
provider: dict(config or {})
|
||||
for provider, config in current_web_search.get("providers", {}).items()
|
||||
}
|
||||
current_provider = current_providers.get(provider_id) or _web_search_provider_defaults(provider_id)
|
||||
current_key, current_key_source = _resolve_web_search_api_key(provider_id, current_provider)
|
||||
current_key, current_key_source = _resolve_web_search_api_key(
|
||||
provider_id,
|
||||
current_provider,
|
||||
current_web_search["default_provider"],
|
||||
)
|
||||
current_key_preview = _mask_secret(current_key, current_key_source)["preview"]
|
||||
provider_payload = {
|
||||
**_web_search_provider_defaults(provider_id),
|
||||
@@ -745,7 +1007,7 @@ def _build_web_search_payload(
|
||||
current_providers[provider_id] = provider_payload
|
||||
return {
|
||||
"enabled": update.enabled,
|
||||
"default_provider": provider_id,
|
||||
"default_provider": default_provider,
|
||||
"providers": current_providers,
|
||||
}
|
||||
|
||||
@@ -754,12 +1016,12 @@ def _runtime_config_from_web_search_payload(web_search_payload: dict) -> WebSear
|
||||
normalized = _normalize_web_search_payload(web_search_payload)
|
||||
provider_id = normalized["default_provider"]
|
||||
provider_config = normalized["providers"].get(provider_id) or _web_search_provider_defaults(provider_id)
|
||||
api_key, _source = _resolve_web_search_api_key(provider_id, provider_config)
|
||||
api_key, _source = _resolve_web_search_api_key(provider_id, provider_config, provider_id)
|
||||
provider_models = {
|
||||
provider: WebSearchProviderConfig(**{
|
||||
**config,
|
||||
"api_key": (
|
||||
api_key if provider == provider_id else _resolve_web_search_api_key(provider, config)[0]
|
||||
api_key if provider == provider_id else _resolve_web_search_api_key(provider, config, provider_id)[0]
|
||||
),
|
||||
})
|
||||
for provider, config in normalized["providers"].items()
|
||||
@@ -893,7 +1155,11 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
|
||||
normalized_web_search["providers"].get(provider_id)
|
||||
or _web_search_provider_defaults(provider_id)
|
||||
)
|
||||
api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config)
|
||||
api_key, api_key_source = _resolve_web_search_api_key(
|
||||
provider_id,
|
||||
provider_config,
|
||||
normalized_web_search["default_provider"],
|
||||
)
|
||||
web_search_providers_payload[provider_id] = {
|
||||
**{
|
||||
key: value
|
||||
@@ -980,10 +1246,7 @@ async def save_external_integrations_payload(
|
||||
update: ExternalIntegrationsUpdate,
|
||||
) -> dict:
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
current_ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
|
||||
ai_payload = _build_ai_provider_payload(current_payload, update.ai_provider)
|
||||
if _ai_provider_runtime_fingerprint(ai_payload) != _ai_provider_runtime_fingerprint(current_ai_payload):
|
||||
await _validate_ai_provider_full_connection(ai_payload)
|
||||
web_search_payload = _build_web_search_payload(current_payload, update.web_search)
|
||||
ocr_payload = _build_ocr_payload(current_payload, update.ocr)
|
||||
|
||||
@@ -1331,6 +1594,10 @@ async def connect_ai_provider_integration(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
# Connection testing should validate the provider being edited, not the
|
||||
# currently saved default provider. This is a transient draft only and is
|
||||
# intentionally not persisted.
|
||||
payload = payload.model_copy(update={"default_provider": payload.provider})
|
||||
draft_ai_payload = _build_ai_provider_payload(current_payload, payload)
|
||||
runtime_config = _runtime_config_from_ai_payload(draft_ai_payload)
|
||||
quick_llm_config = {
|
||||
@@ -1354,24 +1621,16 @@ async def connect_ai_provider_integration(
|
||||
"message": "AI Provider 可访问,但当前 provider/model/key 未完整配置。",
|
||||
"status": status_result.model_dump(),
|
||||
}
|
||||
prompt = await get_effective_prompt(db, AI_CONNECTION_TEST_PROMPT_KEY)
|
||||
probe_result = await client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title="快速连接测试",
|
||||
objective=prompt.prompt,
|
||||
system_prompt=prompt.system_prompt or None,
|
||||
observations=[],
|
||||
constraints=["Output only OK."],
|
||||
)
|
||||
lightweight_result = await _check_ai_provider_lightweight(
|
||||
quick_llm_config,
|
||||
timeout_seconds=min(
|
||||
int(runtime_config["timeout_seconds"] or 60),
|
||||
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS,
|
||||
),
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"connected": True,
|
||||
"message": "连接测试通过",
|
||||
**lightweight_result,
|
||||
"status": status_result.model_dump(),
|
||||
"provider": probe_result.provider,
|
||||
"model": probe_result.model,
|
||||
"mode": "quick_probe",
|
||||
}
|
||||
except HTTPException as exc:
|
||||
return {
|
||||
@@ -1389,16 +1648,38 @@ async def connect_ai_provider_integration(
|
||||
|
||||
@router.get("/integrations/ai-provider/secrets")
|
||||
async def reveal_ai_provider_secrets(
|
||||
request: Request,
|
||||
provider: str = Query(default=""),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
requested_provider = _normalize_provider_id(provider) if provider else "default"
|
||||
await _ensure_secret_reveal_allowed(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=f"ai_provider:{requested_provider}",
|
||||
details={"kind": "ai_provider", "provider": requested_provider},
|
||||
)
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
|
||||
provider_id = _normalize_provider_id(provider or ai_payload["default_provider"])
|
||||
provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id)
|
||||
api_key, api_key_source = _resolve_provider_api_key(provider_id, provider_config)
|
||||
service_token, service_token_source = _resolve_service_token(ai_payload)
|
||||
await _record_integration_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=f"ai_provider:{provider_id}",
|
||||
result="success",
|
||||
details={
|
||||
"kind": "ai_provider",
|
||||
"provider": provider_id,
|
||||
"api_key_configured": bool(api_key),
|
||||
"api_key_source": api_key_source,
|
||||
"service_token_configured": bool(service_token),
|
||||
"service_token_source": service_token_source,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"provider": provider_id,
|
||||
"api_key": api_key,
|
||||
@@ -1417,10 +1698,18 @@ async def get_web_search_presets(
|
||||
|
||||
@router.get("/integrations/web-search/secrets")
|
||||
async def reveal_web_search_secrets(
|
||||
request: Request,
|
||||
provider: str = Query(default=""),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
requested_provider = normalize_web_search_provider(provider) if provider else "default"
|
||||
await _ensure_secret_reveal_allowed(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=f"web_search:{requested_provider}",
|
||||
details={"kind": "web_search", "provider": requested_provider},
|
||||
)
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
web_search_payload = _normalize_web_search_payload(current_payload.get("web_search") or {})
|
||||
provider_id = normalize_web_search_provider(provider or web_search_payload["default_provider"])
|
||||
@@ -1428,7 +1717,23 @@ async def reveal_web_search_secrets(
|
||||
web_search_payload["providers"].get(provider_id)
|
||||
or _web_search_provider_defaults(provider_id)
|
||||
)
|
||||
api_key, api_key_source = _resolve_web_search_api_key(provider_id, provider_config)
|
||||
api_key, api_key_source = _resolve_web_search_api_key(
|
||||
provider_id,
|
||||
provider_config,
|
||||
web_search_payload["default_provider"],
|
||||
)
|
||||
await _record_integration_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=f"web_search:{provider_id}",
|
||||
result="success",
|
||||
details={
|
||||
"kind": "web_search",
|
||||
"provider": provider_id,
|
||||
"api_key_configured": bool(api_key),
|
||||
"api_key_source": api_key_source,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"provider": provider_id,
|
||||
"api_key": api_key,
|
||||
@@ -1438,12 +1743,31 @@ async def reveal_web_search_secrets(
|
||||
|
||||
@router.get("/integrations/ocr/secrets")
|
||||
async def reveal_ocr_secrets(
|
||||
request: Request,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await _ensure_secret_reveal_allowed(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id="ocr:default",
|
||||
details={"kind": "ocr", "provider": "default"},
|
||||
)
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
ocr_payload = _normalize_ocr_payload(current_payload.get("ocr") or {})
|
||||
api_key, api_key_source = _resolve_ocr_api_key(ocr_payload)
|
||||
await _record_integration_secret_reveal(
|
||||
current_user=current_user,
|
||||
request=request,
|
||||
target_id=f"ocr:{ocr_payload['provider']}",
|
||||
result="success",
|
||||
details={
|
||||
"kind": "ocr",
|
||||
"provider": ocr_payload["provider"],
|
||||
"api_key_configured": bool(api_key),
|
||||
"api_key_source": api_key_source,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"provider": ocr_payload["provider"],
|
||||
"api_key": api_key,
|
||||
@@ -1547,9 +1871,17 @@ async def get_ai_provider_presets(
|
||||
async def refresh_ai_provider_preset(
|
||||
provider: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
try:
|
||||
return {"data": await refresh_llm_provider_preset(provider)}
|
||||
provider_id = _normalize_provider_id(provider)
|
||||
api_key = None
|
||||
if provider_id == "opencode-go":
|
||||
current_payload = await get_setting_payload(db, "external_integrations")
|
||||
ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
|
||||
provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id)
|
||||
api_key, _api_key_source = _resolve_provider_api_key(provider_id, provider_config)
|
||||
return {"data": await refresh_llm_provider_preset(provider_id, api_key=api_key)}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
|
||||
@@ -8,9 +8,13 @@ from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import ROOT_DIR
|
||||
from app.core.security import get_current_user
|
||||
from app.db.session import get_db
|
||||
from app.models.system_log import AuditLog, SystemLog
|
||||
from app.models.user import User
|
||||
from app.services.persistent_logs import record_audit_log, record_system_log
|
||||
from app.services.system_control import (
|
||||
@@ -310,7 +314,120 @@ async def get_system_log_sources(
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
ensure_super_admin(current_user)
|
||||
return {"items": list_log_sources()}
|
||||
return {
|
||||
"items": [
|
||||
*list_log_sources(),
|
||||
{
|
||||
"source_id": "system-db",
|
||||
"name": "系统事件",
|
||||
"kind": "database",
|
||||
"location": "table://system_logs",
|
||||
"description": "后端持久化系统事件、AI 和采集器操作日志。",
|
||||
"category": "database",
|
||||
"status": "ok",
|
||||
},
|
||||
{
|
||||
"source_id": "audit-db",
|
||||
"name": "审计事件",
|
||||
"kind": "database",
|
||||
"location": "table://audit_logs",
|
||||
"description": "管理员敏感操作和密钥 reveal 审计记录。",
|
||||
"category": "audit",
|
||||
"status": "ok",
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
async def read_database_log_snapshot(
|
||||
source_id: str,
|
||||
*,
|
||||
limit: int,
|
||||
level: str,
|
||||
levels: str | None,
|
||||
start_date: str | None,
|
||||
end_date: str | None,
|
||||
search: str | None,
|
||||
db: AsyncSession,
|
||||
) -> dict | None:
|
||||
selected_levels = set(normalize_log_level(item) for item in (levels or level).split(",") if item.strip())
|
||||
selected_levels.discard("all")
|
||||
search_query = (search or "").strip().lower()
|
||||
lines: list[str] = []
|
||||
|
||||
if source_id == "system-db":
|
||||
query = select(SystemLog).order_by(SystemLog.occurred_at.desc().nullslast(), SystemLog.id.desc()).limit(limit * 5)
|
||||
result = await db.execute(query)
|
||||
records = result.scalars().all()
|
||||
for record in records:
|
||||
record_level = normalize_log_level(record.level)
|
||||
if selected_levels and record_level not in selected_levels:
|
||||
continue
|
||||
occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else ""
|
||||
if start_date and occurred_at and occurred_at < start_date:
|
||||
continue
|
||||
if end_date and occurred_at and occurred_at > end_date:
|
||||
continue
|
||||
line = " ".join(
|
||||
part
|
||||
for part in [
|
||||
record.occurred_at.isoformat() if record.occurred_at else "",
|
||||
record_level.upper(),
|
||||
record.source,
|
||||
record.event or "",
|
||||
record.message,
|
||||
]
|
||||
if part
|
||||
)
|
||||
if search_query and search_query not in line.lower():
|
||||
continue
|
||||
lines.append(line)
|
||||
elif source_id == "audit-db":
|
||||
query = select(AuditLog).order_by(AuditLog.occurred_at.desc().nullslast(), AuditLog.id.desc()).limit(limit * 5)
|
||||
result = await db.execute(query)
|
||||
records = result.scalars().all()
|
||||
for record in records:
|
||||
occurred_at = record.occurred_at.date().isoformat() if record.occurred_at else ""
|
||||
if start_date and occurred_at and occurred_at < start_date:
|
||||
continue
|
||||
if end_date and occurred_at and occurred_at > end_date:
|
||||
continue
|
||||
line = " ".join(
|
||||
part
|
||||
for part in [
|
||||
record.occurred_at.isoformat() if record.occurred_at else "",
|
||||
"INFO",
|
||||
record.action,
|
||||
record.target_type or "",
|
||||
record.target_id or "",
|
||||
record.result or "",
|
||||
]
|
||||
if part
|
||||
)
|
||||
if search_query and search_query not in line.lower():
|
||||
continue
|
||||
lines.append(line)
|
||||
else:
|
||||
return None
|
||||
|
||||
lines = list(reversed(lines[:limit]))
|
||||
return {
|
||||
"source_id": source_id,
|
||||
"name": "系统事件" if source_id == "system-db" else "审计事件",
|
||||
"kind": "database",
|
||||
"location": "table://system_logs" if source_id == "system-db" else "table://audit_logs",
|
||||
"description": "数据库持久化日志",
|
||||
"category": "database" if source_id == "system-db" else "audit",
|
||||
"status": "ok" if lines else "empty",
|
||||
"level": level,
|
||||
"selected_levels": sorted(selected_levels),
|
||||
"search_query": search or "",
|
||||
"available_levels": ["all", "error", "warning", "info", "debug"],
|
||||
"daily_markers": [],
|
||||
"line_limit": limit,
|
||||
"line_count": len(lines),
|
||||
"lines": lines,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse)
|
||||
@@ -323,6 +440,7 @@ async def get_system_log_snapshot(
|
||||
end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"),
|
||||
search: str | None = Query(None, description="Case-insensitive substring search"),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
ensure_super_admin(current_user)
|
||||
|
||||
@@ -345,15 +463,26 @@ async def get_system_log_snapshot(
|
||||
if normalized_start_date and normalized_end_date and normalized_start_date > normalized_end_date:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="start_date must be earlier than or equal to end_date")
|
||||
|
||||
snapshot = read_log_snapshot(
|
||||
snapshot = await read_database_log_snapshot(
|
||||
source_id,
|
||||
limit,
|
||||
limit=limit,
|
||||
level=level,
|
||||
levels=levels,
|
||||
start_date=normalized_start_date,
|
||||
end_date=normalized_end_date,
|
||||
search=search,
|
||||
db=db,
|
||||
)
|
||||
if snapshot is None:
|
||||
snapshot = read_log_snapshot(
|
||||
source_id,
|
||||
limit,
|
||||
level=level,
|
||||
levels=levels,
|
||||
start_date=normalized_start_date,
|
||||
end_date=normalized_end_date,
|
||||
search=search,
|
||||
)
|
||||
if snapshot is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Log source not found")
|
||||
return snapshot
|
||||
|
||||
@@ -2159,6 +2159,24 @@ async def collect_compute_center_location(
|
||||
llm_failure_reason = llm_result.failure_reason
|
||||
|
||||
if not candidates:
|
||||
logger.warning_event(
|
||||
"Compute center location collection returned no candidates",
|
||||
event="visualization.compute_center.location_collect.completed",
|
||||
context={
|
||||
"source_id": source_id,
|
||||
"record_id": record_id,
|
||||
"name": name,
|
||||
"success": False,
|
||||
"llm_failure_reason": llm_failure_reason,
|
||||
"attempted_queries": list(attempted_queries),
|
||||
"context": {
|
||||
"operator": operator,
|
||||
"site": site,
|
||||
"city": city,
|
||||
"country": country,
|
||||
},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"source_id": source_id,
|
||||
"record_id": record_id,
|
||||
@@ -2180,13 +2198,34 @@ async def collect_compute_center_location(
|
||||
},
|
||||
}
|
||||
|
||||
best_candidate = candidates[0].to_dict()
|
||||
logger.info_event(
|
||||
"Compute center location collection returned candidates",
|
||||
event="visualization.compute_center.location_collect.completed",
|
||||
context={
|
||||
"source_id": source_id,
|
||||
"record_id": record_id,
|
||||
"name": name,
|
||||
"success": True,
|
||||
"candidate_count": len(candidates),
|
||||
"best_candidate": best_candidate,
|
||||
"llm_failure_reason": llm_failure_reason,
|
||||
"attempted_queries": list(attempted_queries),
|
||||
"context": {
|
||||
"operator": operator,
|
||||
"site": site,
|
||||
"city": city,
|
||||
"country": country,
|
||||
},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"source_id": source_id,
|
||||
"record_id": record_id,
|
||||
"name": name,
|
||||
"success": True,
|
||||
"candidates": [candidate.to_dict() for candidate in candidates],
|
||||
"best_candidate": candidates[0].to_dict(),
|
||||
"best_candidate": best_candidate,
|
||||
"attempted_queries": list(attempted_queries),
|
||||
"context": {
|
||||
"name": name,
|
||||
|
||||
@@ -14,6 +14,7 @@ from app.core.websocket.manager import manager
|
||||
|
||||
logger = get_logger(__name__, service="api")
|
||||
router = APIRouter()
|
||||
EARTH_UPDATES_CHANNEL = "earth_updates"
|
||||
|
||||
|
||||
async def authenticate_token(token: str) -> Optional[dict]:
|
||||
@@ -58,7 +59,7 @@ async def websocket_endpoint(
|
||||
|
||||
is_anonymous = payload is None
|
||||
user_id = str(payload.get("sub")) if payload else f"anonymous:{id(websocket)}"
|
||||
supported_channels = ["vessels", "earth_news"] if is_anonymous else [
|
||||
supported_channels = ["vessels", "earth_news", EARTH_UPDATES_CHANNEL] if is_anonymous else [
|
||||
"gpu_clusters",
|
||||
"submarine_cables",
|
||||
"ixp_nodes",
|
||||
@@ -67,6 +68,7 @@ async def websocket_endpoint(
|
||||
"datasource_tasks",
|
||||
"vessels",
|
||||
"earth_news",
|
||||
EARTH_UPDATES_CHANNEL,
|
||||
]
|
||||
await manager.connect(websocket, user_id)
|
||||
|
||||
|
||||
@@ -8,6 +8,8 @@ from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.manager import manager
|
||||
|
||||
|
||||
EARTH_UPDATES_CHANNEL = "earth_updates"
|
||||
|
||||
|
||||
class DataBroadcaster:
|
||||
"""Periodically broadcasts data to connected WebSocket clients"""
|
||||
@@ -83,6 +85,10 @@ class DataBroadcaster:
|
||||
channel=channel,
|
||||
)
|
||||
|
||||
async def broadcast_earth_update(self, data: Dict[str, Any]):
|
||||
"""Broadcast Earth visualization refresh hints to connected clients."""
|
||||
await self.broadcast_custom(EARTH_UPDATES_CHANNEL, data)
|
||||
|
||||
def enqueue_vessel_update(self, data: Dict[str, Any]):
|
||||
vessels = data.get("vessels") if isinstance(data, dict) else None
|
||||
if not isinstance(vessels, list):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import httpx
|
||||
from fastapi import Depends, HTTPException, status
|
||||
@@ -57,6 +58,9 @@ class AIProviderClient:
|
||||
value = self.llm_config.get(key)
|
||||
if value not in (None, ""):
|
||||
headers[header_name] = str(value)
|
||||
model_provider_apis = self.llm_config.get("model_provider_apis")
|
||||
if isinstance(model_provider_apis, dict) and model_provider_apis:
|
||||
headers["X-AI-Model-Provider-APIs"] = json.dumps(model_provider_apis)
|
||||
return headers
|
||||
|
||||
async def get_status(self, request_id: str | None = None) -> AIProviderStatusResponse:
|
||||
|
||||
@@ -9,12 +9,38 @@ from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.collected_data_fields import build_dynamic_metadata, get_record_field
|
||||
from app.core.config import settings
|
||||
from app.core.countries import normalize_country
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
|
||||
|
||||
EARTH_UPDATE_LAYER_HINTS: dict[str, list[str]] = {
|
||||
"ris_live_bgp": ["bgp"],
|
||||
"bgpstream_bgp": ["bgp"],
|
||||
"top500_supercomputers": ["computeCenters"],
|
||||
"epoch_ai_gpu": ["computeCenters"],
|
||||
"huggingface_models": ["computeCenters"],
|
||||
"huggingface_datasets": ["computeCenters"],
|
||||
"huggingface_spaces": ["computeCenters"],
|
||||
"telegeography_cables": ["cables"],
|
||||
"telegeography_landing_points": ["cables"],
|
||||
"telegeography_cable_systems": ["cables"],
|
||||
"arcgis_cables": ["cables"],
|
||||
"fao_landing_points": ["cables"],
|
||||
"arcgis_landing_points": ["cables"],
|
||||
"arcgis_cable_landing_relations": ["cables"],
|
||||
"spacetrack_tle": ["satellites"],
|
||||
"celestrak_tle": ["satellites"],
|
||||
"barentswatch_vessels": ["vessels"],
|
||||
"aisstream_vessels": ["vessels"],
|
||||
"news_live_streams": ["media"],
|
||||
"media_news_archive": ["news"],
|
||||
}
|
||||
|
||||
|
||||
def get_earth_update_layers_for_source(source: str) -> list[str]:
|
||||
return EARTH_UPDATE_LAYER_HINTS.get(source, [])
|
||||
|
||||
|
||||
class BaseCollector(ABC):
|
||||
"""Abstract base class for data collectors"""
|
||||
@@ -70,6 +96,29 @@ class BaseCollector(ABC):
|
||||
)
|
||||
self._last_broadcast_progress = rounded_progress
|
||||
|
||||
async def _publish_earth_update(
|
||||
self,
|
||||
*,
|
||||
action: str,
|
||||
records_processed: int,
|
||||
task_id: int | None = None,
|
||||
) -> None:
|
||||
layers = get_earth_update_layers_for_source(self.name)
|
||||
if not layers:
|
||||
return
|
||||
await broadcaster.broadcast_earth_update(
|
||||
{
|
||||
"action": action,
|
||||
"source": self.name,
|
||||
"data_type": self.data_type,
|
||||
"layers": layers,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"task_id": task_id,
|
||||
"records_processed": records_processed,
|
||||
"timestamp": to_iso8601_utc(datetime.now(UTC)),
|
||||
}
|
||||
)
|
||||
|
||||
async def update_progress(self, records_processed: int, *, commit: bool = False, force: bool = False):
|
||||
"""Update task progress - call this during data processing"""
|
||||
if self._current_task and self._db_session:
|
||||
@@ -187,7 +236,7 @@ class BaseCollector(ABC):
|
||||
|
||||
result = await db.execute(
|
||||
select(DataSnapshot)
|
||||
.where(DataSnapshot.source == self.name, DataSnapshot.is_current == True)
|
||||
.where(DataSnapshot.source == self.name, DataSnapshot.is_current.is_(True))
|
||||
.order_by(DataSnapshot.completed_at.desc().nullslast(), DataSnapshot.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
@@ -325,6 +374,11 @@ class BaseCollector(ABC):
|
||||
task.completed_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
await self._publish_earth_update(
|
||||
action="collector_completed",
|
||||
records_processed=records_count,
|
||||
task_id=task_id,
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
@@ -406,7 +460,7 @@ class BaseCollector(ABC):
|
||||
select(CollectedData)
|
||||
.where(
|
||||
CollectedData.source == self.name,
|
||||
CollectedData.is_current == True,
|
||||
CollectedData.is_current.is_(True),
|
||||
)
|
||||
.order_by(CollectedData.entity_key.asc(), CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
|
||||
)
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm.attributes import flag_modified
|
||||
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
from app.models.system_setting import SystemSetting
|
||||
@@ -135,33 +137,57 @@ DEFAULT_CREDENTIAL_GUIDES = {
|
||||
}
|
||||
|
||||
|
||||
def _normalize_provider(provider: str) -> str:
|
||||
return provider.strip().lower().replace(" ", "_")
|
||||
|
||||
|
||||
def _credential_guide_default(provider: str) -> CredentialGuideDefault:
|
||||
normalized = _normalize_provider(provider)
|
||||
known = DEFAULT_CREDENTIAL_GUIDES.get(normalized)
|
||||
if known is not None:
|
||||
return known
|
||||
title = f"{normalized or 'collector'} 凭证配置教程"
|
||||
return CredentialGuideDefault(
|
||||
provider=normalized,
|
||||
title=title,
|
||||
prompt=(
|
||||
f"请生成一份中文教程,指导开发者为 Planet 采集器配置 {normalized} 凭证。"
|
||||
"教程要面向已经有本地开发环境的人,包含官方入口或文档查找方式、"
|
||||
"获取 API Key / Token / Client credentials 的通用步骤、在 Planet 采集器配置中"
|
||||
"填写凭证字段、连接测试、保存、常见失败排查。不要编造具体页面按钮文案;"
|
||||
"如果公开资料不足,必须明确提醒以 provider 官方文档和当前控制台页面为准。"
|
||||
),
|
||||
markdown="",
|
||||
)
|
||||
|
||||
|
||||
async def _get_guide_store(db) -> tuple[SystemSetting | None, dict[str, Any]]:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == CREDENTIAL_GUIDES_CATEGORY)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
payload = dict(record.payload or {}) if record and isinstance(record.payload, dict) else {}
|
||||
payload = deepcopy(record.payload) if record and isinstance(record.payload, dict) else {}
|
||||
return record, payload
|
||||
|
||||
|
||||
async def get_credential_guide(db, provider: str) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
provider = _normalize_provider(provider)
|
||||
default = _credential_guide_default(provider)
|
||||
|
||||
_record, store = await _get_guide_store(db)
|
||||
custom = store.get(provider) if isinstance(store.get(provider), dict) else None
|
||||
has_default_markdown = bool(default.markdown.strip())
|
||||
return {
|
||||
"provider": provider,
|
||||
"title": custom.get("title") if custom else default.title,
|
||||
"markdown": custom.get("markdown") if custom else default.markdown,
|
||||
"prompt": default.prompt,
|
||||
"source": "ai" if custom else "default",
|
||||
"source": "ai" if custom else "default" if has_default_markdown else "missing",
|
||||
"sources": custom.get("sources", []) if custom else [],
|
||||
"verification_status": (
|
||||
custom.get("verification_status", "verified_with_search_evidence")
|
||||
if custom
|
||||
else "default_unverified"
|
||||
else "default_unverified" if has_default_markdown else "missing"
|
||||
),
|
||||
"verification_error": custom.get("verification_error") if custom else None,
|
||||
}
|
||||
@@ -177,9 +203,8 @@ async def save_credential_guide(
|
||||
verification_status: str = "verified_with_search_evidence",
|
||||
verification_error: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
provider = _normalize_provider(provider)
|
||||
default = _credential_guide_default(provider)
|
||||
|
||||
record, store = await _get_guide_store(db)
|
||||
store[provider] = {
|
||||
@@ -192,21 +217,21 @@ async def save_credential_guide(
|
||||
if record is None:
|
||||
db.add(SystemSetting(category=CREDENTIAL_GUIDES_CATEGORY, payload=store))
|
||||
else:
|
||||
record.payload = store
|
||||
record.payload = deepcopy(store)
|
||||
flag_modified(record, "payload")
|
||||
await db.commit()
|
||||
return await get_credential_guide(db, provider)
|
||||
|
||||
|
||||
async def reset_credential_guide(db, provider: str) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
provider = _normalize_provider(provider)
|
||||
|
||||
record, store = await _get_guide_store(db)
|
||||
if provider in store:
|
||||
store.pop(provider, None)
|
||||
if record is not None:
|
||||
record.payload = store
|
||||
record.payload = deepcopy(store)
|
||||
flag_modified(record, "payload")
|
||||
await db.commit()
|
||||
return await get_credential_guide(db, provider)
|
||||
|
||||
@@ -217,9 +242,8 @@ async def generate_credential_guide(
|
||||
ai_client: AIProviderClient,
|
||||
web_search_client: WebSearchClient | None = None,
|
||||
) -> dict[str, Any]:
|
||||
default = DEFAULT_CREDENTIAL_GUIDES.get(provider)
|
||||
if default is None:
|
||||
raise ValueError(f"Unsupported credential guide provider: {provider}")
|
||||
provider = _normalize_provider(provider)
|
||||
default = _credential_guide_default(provider)
|
||||
|
||||
search_evidence: list[dict[str, Any]] = []
|
||||
search_error: str | None = None
|
||||
|
||||
@@ -18,6 +18,7 @@ from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.services.barentswatch import (
|
||||
BarentsWatchConfig,
|
||||
_read_zshrc_env,
|
||||
fetch_barentswatch_access_token,
|
||||
resolve_barentswatch_config,
|
||||
@@ -44,6 +45,20 @@ def _resolve_spacetrack_credentials() -> tuple[str, str, str]:
|
||||
return username, password, source or "missing"
|
||||
|
||||
|
||||
def _resolve_spacetrack_credentials_with_override(
|
||||
credential_override: dict[str, str] | None = None,
|
||||
) -> tuple[str, str, str]:
|
||||
if credential_override and (
|
||||
credential_override.get("username") or credential_override.get("password")
|
||||
):
|
||||
return (
|
||||
str(credential_override.get("username") or ""),
|
||||
str(credential_override.get("password") or ""),
|
||||
"draft",
|
||||
)
|
||||
return _resolve_spacetrack_credentials()
|
||||
|
||||
|
||||
async def _resolve_aisstream_api_key(
|
||||
db=None,
|
||||
credential_override: dict[str, str] | None = None,
|
||||
@@ -126,7 +141,9 @@ async def build_builtin_connectivity_checksum(
|
||||
}
|
||||
)
|
||||
elif credential_provider == "spacetrack":
|
||||
username, password, credential_source = _resolve_spacetrack_credentials()
|
||||
username, password, credential_source = _resolve_spacetrack_credentials_with_override(
|
||||
credential_override
|
||||
)
|
||||
has_credentials = bool(username and password)
|
||||
credential_fingerprint = _sha256_json(
|
||||
{
|
||||
@@ -230,7 +247,16 @@ async def test_builtin_connectivity(
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
if credential_context["credential_provider"] == "barentswatch":
|
||||
barentswatch_config = await resolve_barentswatch_config(db)
|
||||
if credential_override:
|
||||
barentswatch_config = BarentsWatchConfig(
|
||||
endpoint=str(request_endpoint or ""),
|
||||
client_id=str(credential_override.get("client_id") or ""),
|
||||
client_secret=str(credential_override.get("client_secret") or ""),
|
||||
credential_source="draft",
|
||||
endpoint_source="draft",
|
||||
)
|
||||
else:
|
||||
barentswatch_config = await resolve_barentswatch_config(db)
|
||||
token = await fetch_barentswatch_access_token(client, barentswatch_config)
|
||||
if not token:
|
||||
return {
|
||||
@@ -243,7 +269,9 @@ async def test_builtin_connectivity(
|
||||
}
|
||||
request_headers["Authorization"] = f"Bearer {token}"
|
||||
elif credential_context["credential_provider"] == "spacetrack":
|
||||
username, password, _source = _resolve_spacetrack_credentials()
|
||||
username, password, _source = _resolve_spacetrack_credentials_with_override(
|
||||
credential_override
|
||||
)
|
||||
login_url = "https://www.space-track.org/ajaxauth/login"
|
||||
login_response = await client.post(
|
||||
login_url,
|
||||
|
||||
@@ -270,15 +270,31 @@ def apply_payload_budget(payload: dict[str, Any], policy: EarthLayerCachePolicy)
|
||||
def invalidate_earth_layer_cache_for_source(source: str) -> int:
|
||||
source_key = str(source or "").strip()
|
||||
patterns = {
|
||||
"barentswatch_vessels": ["vessels*", "summary*"],
|
||||
"aisstream_vessels": ["vessels*", "summary*"],
|
||||
"telegeography_cables": ["cables*", "landing-points*", "summary*"],
|
||||
"telegeography_landing": ["landing-points*", "summary*"],
|
||||
"telegeography_landing_points": ["landing-points*", "summary*"],
|
||||
"telegeography_systems": ["cables*", "summary*"],
|
||||
"telegeography_cable_systems": ["cables*", "summary*"],
|
||||
"arcgis_cables": ["cables*", "landing-points*", "summary*"],
|
||||
"arcgis_landing_points": ["landing-points*", "summary*"],
|
||||
"arcgis_cable_landing_relation": ["landing-points*", "summary*"],
|
||||
"arcgis_cable_landing_relations": ["landing-points*", "summary*"],
|
||||
"fao_landing_points": ["landing-points*", "summary*"],
|
||||
"celestrak_tle": ["satellites*", "summary*"],
|
||||
"spacetrack_tle": ["satellites*", "summary*"],
|
||||
"top500": ["compute-centers*", "summary*"],
|
||||
"top500_supercomputers": ["compute-centers*", "summary*"],
|
||||
"epoch_ai_gpu": ["compute-centers*", "summary*"],
|
||||
"huggingface_models": ["compute-centers*", "summary*"],
|
||||
"huggingface_datasets": ["compute-centers*", "summary*"],
|
||||
"huggingface_spaces": ["compute-centers*", "summary*"],
|
||||
"ris_live_bgp": ["bgp*", "summary*"],
|
||||
"bgpstream_bgp": ["bgp*", "summary*"],
|
||||
"iptoasn_prefix_geo": ["bgp*", "summary*"],
|
||||
"opengeofeed_prefix_geo": ["bgp*", "summary*"],
|
||||
"nro_delegated_prefix_geo": ["bgp*", "summary*"],
|
||||
}.get(source_key, [])
|
||||
deleted = 0
|
||||
for layer_pattern in patterns:
|
||||
|
||||
@@ -853,7 +853,7 @@ def _parse_feed_entries(xml_text: str, source: NewsFeedSource) -> list[ParsedNew
|
||||
if not clean_title or not link:
|
||||
continue
|
||||
|
||||
item_source = _normalize_source_name(clean_title, source.name)
|
||||
item_source = source.name
|
||||
display_title = clean_title
|
||||
if source.source_type == "aggregated" and " - " in clean_title:
|
||||
parts = clean_title.rsplit(" - ", 1)
|
||||
|
||||
@@ -7,6 +7,26 @@ from typing import Any
|
||||
import httpx
|
||||
|
||||
MODELS_DEV_URL = "https://models.dev/api.json"
|
||||
OPENCODE_GO_MODELS_URL = "https://opencode.ai/zen/go/v1/models"
|
||||
|
||||
OPENCODE_GO_MODEL_PROVIDER_APIS = {
|
||||
"minimax-m2.7": "anthropic-messages",
|
||||
"minimax-m2.5": "anthropic-messages",
|
||||
}
|
||||
OPENCODE_GO_FALLBACK_MODELS = [
|
||||
"minimax-m2.7",
|
||||
"minimax-m2.5",
|
||||
"kimi-k2.6",
|
||||
"kimi-k2.5",
|
||||
"glm-5.1",
|
||||
"glm-5",
|
||||
"deepseek-v4-pro",
|
||||
"deepseek-v4-flash",
|
||||
"qwen3.6-plus",
|
||||
"qwen3.5-plus",
|
||||
"mimo-v2.5-pro",
|
||||
"mimo-v2.5",
|
||||
]
|
||||
|
||||
|
||||
FALLBACK_LLM_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
|
||||
@@ -80,6 +100,17 @@ FALLBACK_LLM_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
|
||||
"api_key_env": "OPENROUTER_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"opencode-go": {
|
||||
"provider": "opencode-go",
|
||||
"label": "OpenCode Go",
|
||||
"provider_api": "openai-completions",
|
||||
"base_url": "https://opencode.ai/zen/go/v1",
|
||||
"model": "glm-5.1",
|
||||
"models": OPENCODE_GO_FALLBACK_MODELS,
|
||||
"model_provider_apis": OPENCODE_GO_MODEL_PROVIDER_APIS,
|
||||
"api_key_env": "OPENCODE_GO_API_KEY",
|
||||
"source": "fallback",
|
||||
},
|
||||
"ollama": {
|
||||
"provider": "ollama",
|
||||
"label": "Ollama Local",
|
||||
@@ -114,8 +145,43 @@ def get_fallback_llm_provider_preset(provider: str) -> dict[str, Any]:
|
||||
return dict(FALLBACK_LLM_PROVIDER_PRESETS[key])
|
||||
|
||||
|
||||
async def refresh_llm_provider_preset(provider: str) -> dict[str, Any]:
|
||||
def _opencode_go_model_provider_apis(model_ids: list[str]) -> dict[str, str]:
|
||||
return {
|
||||
model_id: OPENCODE_GO_MODEL_PROVIDER_APIS.get(model_id, "openai-completions")
|
||||
for model_id in model_ids
|
||||
}
|
||||
|
||||
|
||||
async def refresh_llm_provider_preset(provider: str, api_key: str | None = None) -> dict[str, Any]:
|
||||
fallback = get_fallback_llm_provider_preset(provider)
|
||||
if fallback["provider"] == "opencode-go":
|
||||
headers = {"User-Agent": "Planet/1.0"}
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
|
||||
response = await client.get(
|
||||
OPENCODE_GO_MODELS_URL,
|
||||
headers=headers,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
|
||||
data = payload.get("data") if isinstance(payload, dict) else []
|
||||
model_ids = [
|
||||
str(item.get("id"))
|
||||
for item in data
|
||||
if isinstance(item, dict) and item.get("id")
|
||||
][:120]
|
||||
if not model_ids:
|
||||
model_ids = fallback["models"]
|
||||
return {
|
||||
**fallback,
|
||||
"model": fallback["model"] if fallback["model"] in model_ids else model_ids[0],
|
||||
"models": model_ids,
|
||||
"model_provider_apis": _opencode_go_model_provider_apis(model_ids),
|
||||
"source": OPENCODE_GO_MODELS_URL,
|
||||
}
|
||||
|
||||
models_dev_key = MODELS_DEV_PROVIDER_KEYS.get(fallback["provider"])
|
||||
if not models_dev_key:
|
||||
return fallback
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import Any, Iterable
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.countries import COUNTRY_ENTRIES, normalize_country
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.ai_tasks.prompts import get_effective_prompt
|
||||
@@ -29,6 +30,9 @@ DEFAULT_MIN_CONFIDENCE = 0.55
|
||||
LOCATION_NORMALIZE_PROMPT_KEY = "location.factcheck.normalize"
|
||||
LOCATION_RESOLVE_PROMPT_KEY = "location.factcheck.resolve"
|
||||
MODEL_CONFIDENCE_WEIGHT = 0.25
|
||||
LOG_TEXT_LIMIT = 1200
|
||||
LOG_EVIDENCE_LIMIT = 5
|
||||
logger = get_logger(__name__, service="location")
|
||||
_geocode_llm_city = build_default_nominatim_geocoder()
|
||||
_LLM_LOCATION_NAME_KEYS = (
|
||||
"matched_location_name",
|
||||
@@ -97,6 +101,35 @@ class LocationEvidenceScore:
|
||||
summary: str
|
||||
|
||||
|
||||
def _truncate_log_text(value: Any, limit: int = LOG_TEXT_LIMIT) -> str:
|
||||
text = coerce_str(value)
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return f"{text[:limit]}…"
|
||||
|
||||
|
||||
def _summarize_search_evidence(evidence: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
for item in (evidence or [])[:LOG_EVIDENCE_LIMIT]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"title": _truncate_log_text(item.get("title"), 180),
|
||||
"source": _truncate_log_text(item.get("source") or item.get("name"), 120),
|
||||
"url": _truncate_log_text(item.get("url"), 240),
|
||||
"snippet": _truncate_log_text(
|
||||
item.get("snippet")
|
||||
or item.get("content")
|
||||
or item.get("text")
|
||||
or item.get("summary"),
|
||||
360,
|
||||
),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def _first_json_object(text: str) -> dict[str, Any] | None:
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
@@ -162,6 +195,64 @@ def _evidence_label(item: Any) -> str:
|
||||
return coerce_str(item)
|
||||
|
||||
|
||||
def _evidence_text(item: dict[str, Any]) -> str:
|
||||
return " ".join(
|
||||
coerce_str(item.get(key))
|
||||
for key in ("title", "source", "name", "url", "snippet", "content", "text", "quote", "summary")
|
||||
if coerce_str(item.get(key))
|
||||
)
|
||||
|
||||
|
||||
def _search_evidence_entity_match(item: dict[str, Any], query: LocationQuery) -> bool:
|
||||
haystack = normalize_text(_evidence_text(item))
|
||||
if not haystack:
|
||||
return False
|
||||
needles = [
|
||||
coerce_str(query.name),
|
||||
*[coerce_str(alias) for alias in query.aliases],
|
||||
]
|
||||
return any(normalize_text(needle) and normalize_text(needle) in haystack for needle in needles)
|
||||
|
||||
|
||||
def _evidence_has_location_assertion(item: dict[str, Any], city: str) -> bool:
|
||||
normalized_city = normalize_text(city)
|
||||
text = normalize_text(_evidence_text(item))
|
||||
if not normalized_city or normalized_city not in text:
|
||||
return False
|
||||
assertion_terms = (
|
||||
"located",
|
||||
"situated",
|
||||
"built",
|
||||
"hosted",
|
||||
"deployed",
|
||||
"installed",
|
||||
"facility",
|
||||
"campus",
|
||||
"site",
|
||||
"data center",
|
||||
"datacenter",
|
||||
"supercomputer center",
|
||||
"位于",
|
||||
"位於",
|
||||
"坐落",
|
||||
"建置",
|
||||
"設置",
|
||||
"设置",
|
||||
)
|
||||
return any(term in text for term in assertion_terms)
|
||||
|
||||
|
||||
def _city_is_unsupported_name_hint(payload: dict[str, Any], query: LocationQuery, evidence_items: list[dict[str, Any]]) -> bool:
|
||||
city = coerce_str(payload.get("city") or query.city)
|
||||
if not city:
|
||||
return False
|
||||
normalized_city = normalize_text(city)
|
||||
normalized_name = normalize_text(query.name)
|
||||
if not normalized_city or not normalized_name or normalized_city not in normalized_name:
|
||||
return False
|
||||
return not any(_evidence_has_location_assertion(item, city) for item in evidence_items)
|
||||
|
||||
|
||||
def _normalize_llm_precision(value: Any) -> str:
|
||||
text = coerce_str(value).lower()
|
||||
return LLM_PRECISION_ALIASES.get(text, text)
|
||||
@@ -588,6 +679,7 @@ def _weak_evidence_penalty(
|
||||
payload: dict[str, Any],
|
||||
evidence_items: list[dict[str, Any]],
|
||||
*,
|
||||
query: LocationQuery,
|
||||
entity_match: float,
|
||||
geography_match: float,
|
||||
conflict_penalty: float,
|
||||
@@ -598,6 +690,8 @@ def _weak_evidence_penalty(
|
||||
penalty += 0.20
|
||||
if any(_truthy_evidence_field(item, "ambiguous") for item in evidence_items):
|
||||
penalty += 0.15
|
||||
if _city_is_unsupported_name_hint(payload, query, evidence_items):
|
||||
penalty += 0.10
|
||||
if conflict_penalty == 0.0 and entity_match > 0 and geography_match >= 0.20:
|
||||
return min(penalty, 0.15)
|
||||
return min(penalty, 0.30)
|
||||
@@ -620,6 +714,7 @@ def _score_llm_location_payload(
|
||||
weak_evidence_penalty = _weak_evidence_penalty(
|
||||
payload,
|
||||
evidence_items,
|
||||
query=query,
|
||||
entity_match=entity_match,
|
||||
geography_match=geography_match,
|
||||
conflict_penalty=conflict_penalty,
|
||||
@@ -636,6 +731,8 @@ def _score_llm_location_payload(
|
||||
- weak_evidence_penalty
|
||||
)
|
||||
score = min(max(score, 0.0), 1.0)
|
||||
if _city_is_unsupported_name_hint(payload, query, evidence_items):
|
||||
score = min(score, 0.54)
|
||||
summary = (
|
||||
f"combined={score:.2f}; model={model_confidence:.2f}; "
|
||||
f"source={source_quality:.2f}; entity={entity_match:.2f}; "
|
||||
@@ -847,15 +944,43 @@ async def collect_location_search_evidence(
|
||||
) -> LocationSearchEvidenceResult:
|
||||
search_query = _location_search_query(query, entity_type)
|
||||
attempt = f"web_search:{entity_type}:{search_query}"
|
||||
logger.info_event(
|
||||
"Collecting location search evidence",
|
||||
event="location.factcheck.web_search.start",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"search_query": search_query,
|
||||
"location_query": _query_context(query),
|
||||
"max_results": max_results,
|
||||
},
|
||||
)
|
||||
try:
|
||||
evidence = await web_search_client.search(search_query, max_results=max_results)
|
||||
except WebSearchError as exc:
|
||||
logger.warning_event(
|
||||
"Location search evidence failed",
|
||||
event="location.factcheck.web_search.failed",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"search_query": search_query,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
return LocationSearchEvidenceResult(
|
||||
evidence=[],
|
||||
attempted_queries=[attempt],
|
||||
failure_reason=f"WebSearch location evidence failed: {exc}",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"Location search evidence unavailable",
|
||||
event="location.factcheck.web_search.unavailable",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"search_query": search_query,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
return LocationSearchEvidenceResult(
|
||||
evidence=[],
|
||||
attempted_queries=[attempt],
|
||||
@@ -863,11 +988,29 @@ async def collect_location_search_evidence(
|
||||
)
|
||||
normalized = normalize_search_evidence(evidence, limit=max_results)
|
||||
if not normalized:
|
||||
logger.warning_event(
|
||||
"Location search returned no usable evidence",
|
||||
event="location.factcheck.web_search.empty",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"search_query": search_query,
|
||||
},
|
||||
)
|
||||
return LocationSearchEvidenceResult(
|
||||
evidence=[],
|
||||
attempted_queries=[attempt],
|
||||
failure_reason="WebSearch returned no usable location evidence.",
|
||||
)
|
||||
logger.info_event(
|
||||
"Collected location search evidence",
|
||||
event="location.factcheck.web_search.result",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"search_query": search_query,
|
||||
"evidence_count": len(normalized),
|
||||
"evidence": _summarize_search_evidence(normalized),
|
||||
},
|
||||
)
|
||||
return LocationSearchEvidenceResult(
|
||||
evidence=normalized,
|
||||
attempted_queries=[attempt],
|
||||
@@ -947,6 +1090,15 @@ async def collect_llm_location_fallback_candidate(
|
||||
"""
|
||||
attempt = f"llm_factcheck:{entity_type}:{coerce_str(query.name) or 'unknown'}"
|
||||
if search_evidence is not None and not search_evidence:
|
||||
logger.warning_event(
|
||||
"Skipping LLM location factcheck because search evidence is empty",
|
||||
event="location.factcheck.llm.skipped_no_evidence",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"location_query": _query_context(query),
|
||||
},
|
||||
)
|
||||
return LocationLLMFallbackResult(
|
||||
candidates=[],
|
||||
attempted_queries=[attempt],
|
||||
@@ -986,20 +1138,66 @@ async def collect_llm_location_fallback_candidate(
|
||||
"Return evidence as objects when possible, including source, url, source_type, and entity_match.",
|
||||
"Include source names or URLs in evidence when known. The backend will recompute the final confidence from model confidence plus evidence quality.",
|
||||
"If search_evidence is provided, use only that evidence as factual support.",
|
||||
"Do not treat a website footer, office address, publisher address, or contact address as the entity's physical location.",
|
||||
"If the entity name contains a city name, do not choose that city unless evidence explicitly says the entity/facility/supercomputer is located, hosted, built, deployed, or installed there.",
|
||||
"Prefer the facility/site if known; otherwise use the best supported city.",
|
||||
],
|
||||
)
|
||||
logger.info_event(
|
||||
"Sending location factcheck request to LLM",
|
||||
event="location.factcheck.llm.request",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"title": request.title,
|
||||
"objective": request.objective,
|
||||
"location_query": request.context.get("location_query"),
|
||||
"observations": request.observations,
|
||||
"constraints": request.constraints,
|
||||
"search_evidence_count": len(search_evidence or []),
|
||||
"search_evidence": _summarize_search_evidence(search_evidence),
|
||||
},
|
||||
)
|
||||
try:
|
||||
response = await provider_client.analyze(request)
|
||||
except Exception as exc:
|
||||
logger.warning_event(
|
||||
"LLM location factcheck failed",
|
||||
event="location.factcheck.llm.failed",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
return LocationLLMFallbackResult(
|
||||
candidates=[],
|
||||
attempted_queries=[attempt],
|
||||
failure_reason=f"LLM location factcheck failed: {exc}",
|
||||
)
|
||||
|
||||
logger.info_event(
|
||||
"Received location factcheck response from LLM",
|
||||
event="location.factcheck.llm.response",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"provider": response.provider,
|
||||
"model": response.model,
|
||||
"content": _truncate_log_text(response.content, 2000),
|
||||
},
|
||||
)
|
||||
payload = _first_json_object(response.content)
|
||||
if payload is None:
|
||||
logger.warning_event(
|
||||
"LLM location factcheck response was not strict JSON; attempting repair",
|
||||
event="location.factcheck.llm.non_json",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"content": _truncate_log_text(response.content, 1200),
|
||||
},
|
||||
)
|
||||
payload = await _repair_location_payload_from_text(
|
||||
provider_client=provider_client,
|
||||
raw_text=response.content,
|
||||
@@ -1009,9 +1207,17 @@ async def collect_llm_location_fallback_candidate(
|
||||
)
|
||||
if payload is None:
|
||||
payload = _payload_from_free_text(response.content, query=query)
|
||||
if payload is None:
|
||||
if payload is None and entity_type != "compute_center":
|
||||
payload = _payload_from_query_name_geocode(query)
|
||||
if payload is None:
|
||||
logger.warning_event(
|
||||
"LLM location factcheck produced no parseable payload",
|
||||
event="location.factcheck.llm.unparseable",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
},
|
||||
)
|
||||
return LocationLLMFallbackResult(
|
||||
candidates=[],
|
||||
attempted_queries=[attempt],
|
||||
@@ -1032,7 +1238,7 @@ async def collect_llm_location_fallback_candidate(
|
||||
"url": item.get("url"),
|
||||
"text": item.get("snippet") or item.get("content"),
|
||||
"source_type": "web_search",
|
||||
"entity_match": True,
|
||||
"entity_match": _search_evidence_entity_match(item, query),
|
||||
}
|
||||
for item in search_evidence
|
||||
if isinstance(item, dict)
|
||||
@@ -1054,6 +1260,18 @@ async def collect_llm_location_fallback_candidate(
|
||||
if candidate is None:
|
||||
if city_geocode_failure and rejection_reason == "missing, invalid, or zero latitude/longitude":
|
||||
rejection_reason = f"{rejection_reason}; {city_geocode_failure}"
|
||||
logger.warning_event(
|
||||
"Rejected LLM location factcheck candidate",
|
||||
event="location.factcheck.llm.rejected",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"reason": rejection_reason,
|
||||
"payload": payload,
|
||||
"search_evidence_count": len(search_evidence or []),
|
||||
"search_evidence": _summarize_search_evidence(search_evidence),
|
||||
},
|
||||
)
|
||||
return LocationLLMFallbackResult(
|
||||
candidates=[],
|
||||
attempted_queries=[attempt],
|
||||
@@ -1062,6 +1280,18 @@ async def collect_llm_location_fallback_candidate(
|
||||
+ (f": {rejection_reason}." if rejection_reason else ".")
|
||||
),
|
||||
)
|
||||
logger.info_event(
|
||||
"Accepted LLM location factcheck candidate",
|
||||
event="location.factcheck.llm.accepted",
|
||||
context={
|
||||
"entity_type": entity_type,
|
||||
"attempt": attempt,
|
||||
"candidate": candidate.to_dict(),
|
||||
"payload": payload,
|
||||
"search_evidence_count": len(search_evidence or []),
|
||||
"search_evidence": _summarize_search_evidence(search_evidence),
|
||||
},
|
||||
)
|
||||
return LocationLLMFallbackResult(
|
||||
candidates=[candidate],
|
||||
attempted_queries=[attempt],
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -86,6 +87,7 @@ class LogSource:
|
||||
status: str = "ok"
|
||||
buffer_key: str | None = None
|
||||
container_name: str | None = None
|
||||
fallback_locations: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -104,22 +106,38 @@ class DailyLogMarker:
|
||||
dominant_level: str
|
||||
|
||||
|
||||
def _planet_state_dir() -> Path:
|
||||
configured = os.getenv("PLANET_STATE_DIR")
|
||||
if configured:
|
||||
return Path(configured).expanduser()
|
||||
xdg_state = os.getenv("XDG_STATE_HOME")
|
||||
if xdg_state:
|
||||
return Path(xdg_state).expanduser() / "planet"
|
||||
return Path.home() / ".local" / "state" / "planet"
|
||||
|
||||
|
||||
def _state_log_path(filename: str) -> str:
|
||||
return str(_planet_state_dir() / filename)
|
||||
|
||||
|
||||
LOG_SOURCES: dict[str, LogSource] = {
|
||||
"backend": LogSource(
|
||||
source_id="backend",
|
||||
name="后端服务",
|
||||
kind="file",
|
||||
location="/tmp/planet_backend.log",
|
||||
location=_state_log_path("backend.log"),
|
||||
description="FastAPI 后端、调度器和采集任务共享日志。",
|
||||
category="service",
|
||||
fallback_locations=("/tmp/planet_backend.log",),
|
||||
),
|
||||
"frontend": LogSource(
|
||||
source_id="frontend",
|
||||
name="前端开发服务",
|
||||
kind="file",
|
||||
location="/tmp/planet_frontend.log",
|
||||
location=_state_log_path("frontend.log"),
|
||||
description="控制台与 Earth 前端开发服务输出。",
|
||||
category="service",
|
||||
fallback_locations=("/tmp/planet_frontend.log",),
|
||||
),
|
||||
"ai-provider": LogSource(
|
||||
source_id="ai-provider",
|
||||
@@ -164,9 +182,18 @@ def normalize_log_levels(level: str | None = None, levels: str | None = None) ->
|
||||
return tuple(normalized_levels)
|
||||
|
||||
|
||||
def resolve_file_log_path(source: LogSource) -> Path:
|
||||
primary = Path(source.location).expanduser()
|
||||
candidates = (primary, *(Path(item).expanduser() for item in source.fallback_locations))
|
||||
for candidate in candidates:
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return primary
|
||||
|
||||
|
||||
def get_source_status(source: LogSource) -> str:
|
||||
if source.kind == "file":
|
||||
path = Path(source.location)
|
||||
path = resolve_file_log_path(source)
|
||||
if not path.exists():
|
||||
return "missing"
|
||||
return "ok" if path.stat().st_size > 0 else "empty"
|
||||
@@ -190,7 +217,7 @@ def list_log_sources() -> list[dict[str, str]]:
|
||||
"source_id": source.source_id,
|
||||
"name": source.name,
|
||||
"kind": source.kind,
|
||||
"location": source.location,
|
||||
"location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location,
|
||||
"description": source.description,
|
||||
"category": source.category,
|
||||
"status": get_source_status(source),
|
||||
@@ -339,7 +366,7 @@ def build_buffer_entry(payload: dict[str, Any]) -> StructuredLogEntry:
|
||||
|
||||
|
||||
def read_file_entries(source: LogSource, scan_limit: int) -> list[StructuredLogEntry]:
|
||||
path = Path(source.location)
|
||||
path = resolve_file_log_path(source)
|
||||
if not path.exists():
|
||||
return []
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
@@ -511,7 +538,7 @@ def read_log_snapshot(
|
||||
"source_id": source.source_id,
|
||||
"name": source.name,
|
||||
"kind": source.kind,
|
||||
"location": source.location,
|
||||
"location": str(resolve_file_log_path(source)) if source.kind == "file" else source.location,
|
||||
"description": source.description,
|
||||
"category": source.category,
|
||||
"status": get_source_status(source),
|
||||
|
||||
@@ -385,13 +385,18 @@ def build_public_tv_payload(
|
||||
settings_payload: dict[str, Any],
|
||||
collected_sources: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
configured_by_id = {
|
||||
source["id"]: source
|
||||
for source in settings_payload["sources"]
|
||||
if source.get("id")
|
||||
}
|
||||
configured_sources = [
|
||||
source for source in settings_payload["sources"] if source["is_enabled"]
|
||||
]
|
||||
|
||||
merged_by_id = {source["id"]: source for source in configured_sources}
|
||||
for source in collected_sources:
|
||||
if source["id"] in merged_by_id or not source["is_enabled"]:
|
||||
if source["id"] in configured_by_id or not source["is_enabled"]:
|
||||
continue
|
||||
merged_by_id[source["id"]] = source
|
||||
|
||||
|
||||
Reference in New Issue
Block a user