linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.48.0

Browse Source
This commit is contained in:
linkong
2026-05-07 18:06:06 +08:00
parent 421234301a
commit bb9183b8a4
51 changed files with 4609 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
backend/app/api/v1/datasource_config.py
View File

@@ -5,8 +5,8 @@ from datetime import datetime
import base64
import json
import re
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select, func
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import delete, select, func
from sqlalchemy.ext.asyncio import AsyncSession
from pydantic import BaseModel, Field
import httpx
@@ -17,6 +17,8 @@ from app.db.session import get_db
from app.models.user import User
from app.models.datasource_config import DataSourceConfig
from app.models.datasource_mapping import DataSourceMappingTemplate
from app.models.collected_data import CollectedData
from app.models.vessel import AISRawObservation, AISSourceHealth
from app.core.security import get_current_user
from app.core.cache import cache
from app.core.time import to_iso8601_utc
@@ -26,10 +28,19 @@ from app.services.datasource_mapping import (
MappingError,
build_heuristic_mapping,
execute_mapping,
persist_mapped_records,
redact_for_llm,
stable_payload_hash,
)
from app.services.custom_datasource_runtime import (
CustomDatasourceRuntimeError,
fetch_rest_payload,
get_custom_stream_status,
run_mapped_rest_config,
run_mapped_websocket_config,
start_custom_stream,
stop_custom_stream,
test_websocket_config,
)
from app.services.datasource_connectivity import (
get_builtin_connection_status,
save_connectivity_success,
@@ -43,7 +54,7 @@ router = APIRouter()
class DataSourceConfigCreate(BaseModel):
name: str = Field(..., min_length=1, max_length=100)
description: Optional[str] = None
source_type: str = Field(..., description="http, api, database")
source_type: str = Field(..., description="rest, websocket, http, api, database")
endpoint: str = Field(..., max_length=500)
auth_type: str = Field(default="none", description="none, bearer, api_key, basic")
auth_config: dict = Field(default={})
@@ -219,6 +230,8 @@ def _build_query_params(auth_type: str, auth_config: dict, config: dict) -> dict
async def fetch_custom_sample_from_config(config: DataSourceConfig, limit_bytes: int) -> Any:
if str(config.source_type or "").lower() in {"websocket", "ws"}:
raise HTTPException(status_code=400, detail="WebSocket sources must use connection test or run-mapped stream.")
request_config = config.config or {}
method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
if method not in {"GET", "POST"}:
@@ -488,6 +501,8 @@ async def update_config(
@router.delete("/configs/{config_id}")
async def delete_config(
config_id: int,
delete_mappings: bool = Query(False),
delete_source_data: bool = Query(False),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
@@ -498,12 +513,59 @@ async def delete_config(
if not config:
raise HTTPException(status_code=404, detail="Configuration not found")
deleted_mappings = 0
deleted_records = {
"collected_data": 0,
"ais_raw_observations": 0,
"ais_source_health": 0,
}
if delete_source_data:
collected_result = await db.execute(
delete(CollectedData).where(CollectedData.source == config.name)
)
raw_result = await db.execute(
delete(AISRawObservation).where(AISRawObservation.source == config.name)
)
health_result = await db.execute(
delete(AISSourceHealth).where(AISSourceHealth.source == config.name)
)
deleted_records = {
"collected_data": collected_result.rowcount or 0,
"ais_raw_observations": raw_result.rowcount or 0,
"ais_source_health": health_result.rowcount or 0,
}
if delete_mappings or delete_source_data:
mapping_result = await db.execute(
delete(DataSourceMappingTemplate).where(
DataSourceMappingTemplate.datasource_config_id == config_id
)
)
deleted_mappings = mapping_result.rowcount or 0
await db.delete(config)
await db.commit()
cache.delete_pattern("datasource_configs:*")
return {"message": "Configuration deleted successfully"}
if delete_source_data and (config.config or {}).get("target_schema") == "vessel_ais":
from app.core.websocket.broadcaster import broadcaster
await broadcaster.broadcast_custom(
"vessels",
{
"action": "reload",
"source": config.name,
"reason": "custom_source_deleted",
},
)
return {
"message": "Configuration deleted successfully",
"deleted_mappings": deleted_mappings,
"deleted_records": deleted_records,
}
@router.post("/configs/{config_id}/test")
@@ -520,6 +582,8 @@ async def test_config(
raise HTTPException(status_code=404, detail="Configuration not found")
try:
if str(config.source_type or "").lower() in {"websocket", "ws"}:
return await test_websocket_config(config)
result = await test_endpoint(
endpoint=config.endpoint,
auth_type=config.auth_type,
@@ -550,6 +614,18 @@ async def test_new_config(
):
"""Test a new data source configuration without saving"""
try:
if str(config_data.source_type or "").lower() in {"websocket", "ws"}:
config = DataSourceConfig(
name=config_data.name,
description=config_data.description,
source_type=config_data.source_type,
endpoint=config_data.endpoint,
auth_type=config_data.auth_type,
auth_config=config_data.auth_config,
headers=config_data.headers,
config=config_data.config,
)
return await test_websocket_config(config)
result = await test_endpoint(
endpoint=config_data.endpoint,
auth_type=config_data.auth_type,
@@ -875,6 +951,8 @@ async def update_datasource_mapping(
@router.post("/{config_id}/run-mapped")
async def run_mapped_datasource(
config_id: int,
background: bool = Query(False, description="For WebSocket sources, start a background stream task."),
debug_max_messages: int | None = Query(None, ge=1),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
@@ -883,20 +961,24 @@ async def run_mapped_datasource(
if not datasource:
raise HTTPException(status_code=404, detail="Configuration not found")
result = await db.execute(
select(DataSourceMappingTemplate)
.where(DataSourceMappingTemplate.datasource_config_id == config_id)
.where(DataSourceMappingTemplate.is_active.is_(True))
.order_by(DataSourceMappingTemplate.version.desc())
.limit(1)
)
mapping = result.scalar_one_or_none()
if not mapping:
raise HTTPException(status_code=404, detail="No active mapping template found")
try:
sample = await fetch_custom_sample_from_config(datasource, 5_000_000)
mapped = execute_mapping(sample, mapping.mapping_json, mapping.target_schema)
if str(datasource.source_type or "").lower() in {"websocket", "ws"}:
if background and debug_max_messages is None:
started = start_custom_stream(config_id)
if not started:
raise HTTPException(status_code=409, detail="Custom WebSocket source is already running")
return {
"status": "started",
"datasource_config_id": config_id,
"stream": get_custom_stream_status(config_id),
}
return await run_mapped_websocket_config(
db,
datasource,
debug_max_messages=debug_max_messages,
)
return await run_mapped_rest_config(db, datasource)
except httpx.HTTPStatusError as exc:
raise HTTPException(
status_code=exc.response.status_code,
@@ -904,36 +986,26 @@ async def run_mapped_datasource(
) from exc
except httpx.HTTPError as exc:
raise HTTPException(status_code=502, detail=f"Datasource request failed: {exc}") from exc
except (MappingError, ValueError) as exc:
except (CustomDatasourceRuntimeError, MappingError, ValueError) as exc:
raise HTTPException(status_code=400, detail=f"Mapping failed: {exc}") from exc
if mapped["failed_count"] > 0:
return {
"status": "failed",
"datasource_config_id": config_id,
"mapping_id": mapping.id,
"mapping_version": mapping.version,
"target_schema": mapping.target_schema,
"mapped_count": mapped["mapped_count"],
"failed_count": mapped["failed_count"],
"errors": mapped["errors"][:20],
}
written_count = await persist_mapped_records(
db,
datasource_name=datasource.name,
datasource_config_id=datasource.id,
target_schema=mapping.target_schema,
records=mapped["records"],
mapping_version=mapping.version,
)
@router.post("/{config_id}/stop-mapped")
async def stop_mapped_datasource(
config_id: int,
current_user: User = Depends(get_current_user),
):
stopped = await stop_custom_stream(config_id)
return {
"status": "success",
"status": "stopped" if stopped else "not_running",
"datasource_config_id": config_id,
"mapping_id": mapping.id,
"mapping_version": mapping.version,
"target_schema": mapping.target_schema,
"fetched_count": mapped["total_items"],
"mapped_count": mapped["mapped_count"],
"written_count": written_count,
"stream": get_custom_stream_status(config_id),
}
@router.get("/{config_id}/stream-status")
async def get_mapped_stream_status(
config_id: int,
current_user: User = Depends(get_current_user),
):
return get_custom_stream_status(config_id)