release: bump version to 0.62.0

2026-05-21 01:37:32 +08:00
parent 5c65ee24d6
commit fbca381512
138 changed files with 21303 additions and 5721 deletions
--- a/backend/app/api/v1/datasources.py
+++ b/backend/app/api/v1/datasources.py
@@ -7,6 +7,7 @@ from pydantic import BaseModel, Field
 from sqlalchemy import func, or_, select, text
 from sqlalchemy.ext.asyncio import AsyncSession

+from app.core.cache import cache
 from app.core.time import to_iso8601_utc
 from app.core.security import get_current_user
 from app.core.data_sources import get_data_sources_config
@@ -26,6 +27,7 @@ from app.services.scheduler import (
    run_collector_now,
    sync_datasource_job,
 )
+from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source

 router = APIRouter()
 STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90
@@ -235,6 +237,8 @@ async def _load_datasource_endpoint_overrides(
 async def _load_datasource_list_context(
    db: AsyncSession,
    datasources: list[DataSource],
+    *,
+    include_endpoint: bool = True,
 ) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, str]]:
    datasource_ids = [datasource.id for datasource in datasources]
    sources = [datasource.source for datasource in datasources]
@@ -260,10 +264,65 @@ async def _load_datasource_list_context(
        running_tasks = await _load_latest_running_tasks(db, datasource_ids)

    latest_tasks = await _load_latest_tasks(db, datasource_ids)
-    endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
+    endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources) if include_endpoint else {}
    return running_tasks, latest_tasks, endpoint_overrides


+def serialize_datasource_row(
+    datasource: DataSource,
+    *,
+    running_tasks: dict[int, CollectionTask],
+    latest_tasks: dict[int, CollectionTask],
+    record_counts: dict[str, int],
+    endpoint_overrides: dict[str, str],
+    config,
+    include_endpoint: bool,
+) -> dict:
+    running_task = running_tasks.get(datasource.id)
+    latest_task = latest_tasks.get(datasource.id)
+    display_task = running_task or latest_task
+    endpoint = None
+    if include_endpoint:
+        endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
+    last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None)
+    last_status = datasource.last_status or (latest_task.status if latest_task else None)
+    collected_records = record_counts.get(datasource.source, 0)
+
+    row = {
+        "id": datasource.id,
+        "source": datasource.source,
+        "name": datasource.name,
+        **datasource_metadata(datasource.source),
+        "product": datasource_product_key(datasource),
+        "module": datasource.module,
+        "priority": datasource.priority,
+        "frequency": format_frequency_label(datasource.frequency_minutes),
+        "frequency_minutes": datasource.frequency_minutes,
+        "is_active": datasource.is_active,
+        "collector_class": datasource.collector_class,
+        "last_run": to_iso8601_utc(last_run_at),
+        "last_run_at": to_iso8601_utc(last_run_at),
+        "last_status": last_status,
+        "is_running": running_task is not None,
+        "task_id": display_task.id if display_task else None,
+        "progress": display_task.progress if display_task else None,
+        "phase": display_task.phase if display_task else None,
+        "phase_progress": display_task.phase_progress if display_task else None,
+        "phase_message": display_task.phase_message if display_task else None,
+        "phase_current": display_task.phase_current if display_task else None,
+        "phase_total": display_task.phase_total if display_task else None,
+        "phase_unit": display_task.phase_unit if display_task else None,
+        "records_processed": display_task.records_processed if display_task else None,
+        "total_records": display_task.total_records if display_task else None,
+        "error_message": display_task.error_message if display_task else None,
+        "collected_records": collected_records,
+        "has_collected_data": collected_records > 0,
+    }
+    if include_endpoint:
+        row["endpoint"] = endpoint
+    return row
+
+
 def _apply_datasource_query_filters(
    query,
    *,
@@ -658,6 +717,7 @@ async def list_datasources(
    collected: Optional[bool] = None,
    credential_status: Optional[str] = None,
    q: Optional[str] = None,
+    include_endpoint: bool = True,
    current_user: User = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
 ):
@@ -676,7 +736,11 @@ async def list_datasources(

    collector_list = []
    config = get_data_sources_config()
-    running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
+    running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(
+        db,
+        datasources,
+        include_endpoint=include_endpoint,
+    )
    record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
    datasources = _filter_datasources_in_memory(
        datasources,
@@ -689,46 +753,16 @@ async def list_datasources(
        credential_status=credential_status,
    )
    for datasource in datasources:
-        running_task = running_tasks.get(datasource.id)
-        latest_task = latest_tasks.get(datasource.id)
-        display_task = running_task or latest_task
-        endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
-        last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None)
-        last_status = datasource.last_status or (latest_task.status if latest_task else None)
-        collected_records = record_counts.get(datasource.source, 0)
-
        collector_list.append(
-            {
-                "id": datasource.id,
-                "source": datasource.source,
-                "name": datasource.name,
-                **datasource_metadata(datasource.source),
-                "product": datasource_product_key(datasource),
-                "module": datasource.module,
-                "priority": datasource.priority,
-                "frequency": format_frequency_label(datasource.frequency_minutes),
-                "frequency_minutes": datasource.frequency_minutes,
-                "is_active": datasource.is_active,
-                "collector_class": datasource.collector_class,
-                "endpoint": endpoint,
-                "last_run": to_iso8601_utc(last_run_at),
-                "last_run_at": to_iso8601_utc(last_run_at),
-                "last_status": last_status,
-                "is_running": running_task is not None,
-                "task_id": display_task.id if display_task else None,
-                "progress": display_task.progress if display_task else None,
-                "phase": display_task.phase if display_task else None,
-                "phase_progress": display_task.phase_progress if display_task else None,
-                "phase_message": display_task.phase_message if display_task else None,
-                "phase_current": display_task.phase_current if display_task else None,
-                "phase_total": display_task.phase_total if display_task else None,
-                "phase_unit": display_task.phase_unit if display_task else None,
-                "records_processed": display_task.records_processed if display_task else None,
-                "total_records": display_task.total_records if display_task else None,
-                "error_message": display_task.error_message if display_task else None,
-                "collected_records": collected_records,
-                "has_collected_data": collected_records > 0,
-            }
+            serialize_datasource_row(
+                datasource,
+                running_tasks=running_tasks,
+                latest_tasks=latest_tasks,
+                record_counts=record_counts,
+                endpoint_overrides=endpoint_overrides,
+                config=config,
+                include_endpoint=include_endpoint,
+            )
        )

    return {"total": len(collector_list), "data": collector_list}
@@ -785,6 +819,51 @@ async def trigger_datasource_batch(
    return await _trigger_datasource_batch(db, datasources, force=payload.force)


+@router.get("/snapshots")
+async def list_datasource_snapshots(
+    source_id: Optional[str] = None,
+    current_only: Optional[bool] = None,
+    limit: int = Query(default=100, ge=1, le=500),
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    query = (
+        select(DataSnapshot, DataSource.name, DataSource.module)
+        .outerjoin(DataSource, DataSource.id == DataSnapshot.datasource_id)
+        .order_by(DataSnapshot.created_at.desc().nullslast(), DataSnapshot.id.desc())
+        .limit(limit)
+    )
+    if source_id:
+        query = query.where(DataSnapshot.source == source_id)
+    if current_only is not None:
+        query = query.where(DataSnapshot.is_current.is_(current_only))
+
+    result = await db.execute(query)
+    rows = []
+    for snapshot, datasource_name, datasource_module in result.all():
+        rows.append(
+            {
+                "id": snapshot.id,
+                "datasource_id": snapshot.datasource_id,
+                "datasource_name": datasource_name,
+                "module": datasource_module,
+                "task_id": snapshot.task_id,
+                "source": snapshot.source,
+                "snapshot_key": snapshot.snapshot_key,
+                "reference_date": to_iso8601_utc(snapshot.reference_date),
+                "started_at": to_iso8601_utc(snapshot.started_at),
+                "completed_at": to_iso8601_utc(snapshot.completed_at),
+                "record_count": snapshot.record_count,
+                "status": snapshot.status,
+                "is_current": snapshot.is_current,
+                "parent_snapshot_id": snapshot.parent_snapshot_id,
+                "summary": snapshot.summary or {},
+                "created_at": to_iso8601_utc(snapshot.created_at),
+            }
+        )
+    return {"total": len(rows), "data": rows}
+
+
@router.get("/{source_id}")
 async def get_datasource(
    source_id: str,
@@ -813,6 +892,37 @@ async def get_datasource(
    }


+@router.get("/{source_id}/row")
+async def get_datasource_row(
+    source_id: str,
+    include_endpoint: bool = True,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    datasource = await get_datasource_record(db, source_id)
+    if not datasource:
+        raise HTTPException(status_code=404, detail="Data source not found")
+
+    config = get_data_sources_config()
+    running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(
+        db,
+        [datasource],
+        include_endpoint=include_endpoint,
+    )
+    record_counts = await _load_collected_record_counts(db, [datasource.source])
+    return {
+        "data": serialize_datasource_row(
+            datasource,
+            running_tasks=running_tasks,
+            latest_tasks=latest_tasks,
+            record_counts=record_counts,
+            endpoint_overrides=endpoint_overrides,
+            config=config,
+            include_endpoint=include_endpoint,
+        )
+    }
+
+
@router.post("/{source_id}/enable")
 async def enable_datasource(
    source_id: str,
@@ -960,6 +1070,29 @@ async def clear_datasource_data(
    }


+@router.delete("/{source_id}/cache")
+async def clear_datasource_cache(
+    source_id: str,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    datasource = await get_datasource_record(db, source_id)
+    if not datasource:
+        raise HTTPException(status_code=404, detail="Data source not found")
+
+    earth_deleted_count = invalidate_earth_layer_cache_for_source(datasource.source)
+    dashboard_deleted_count = int(cache.delete("dashboard:stats")) + int(cache.delete("dashboard:summary"))
+    deleted_count = earth_deleted_count + dashboard_deleted_count
+
+    return {
+        "status": "success",
+        "message": f"Cleared {deleted_count} cache keys for data source '{datasource.name}'",
+        "deleted_count": deleted_count,
+        "earth_layer_deleted_count": earth_deleted_count,
+        "dashboard_deleted_count": dashboard_deleted_count,
+    }
+
+
@router.get("/{source_id}/task-status")
 async def get_task_status(
    source_id: str,