linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.43.0

Browse Source
This commit is contained in:
linkong
2026-04-28 16:10:17 +08:00
parent 1cd2dab0ee
commit ac69d5d354
69 changed files with 6954 additions and 1141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
backend/app/api/v1/datasources.py
View File

@@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.time import to_iso8601_utc
from app.core.security import get_current_user
from app.core.data_sources import get_data_sources_config
from app.core.datasource_defaults import DEFAULT_DATASOURCES
from app.db.session import get_db
from app.models.collected_data import CollectedData
from app.models.data_snapshot import DataSnapshot
@@ -35,6 +36,17 @@ def format_frequency_label(minutes: int) -> str:
return f"{minutes}m"
def datasource_metadata(source: str) -> dict:
info = DEFAULT_DATASOURCES.get(source, {})
return {
"display_name": info.get("display_name") or info.get("name") or source,
"is_free": bool(info.get("is_free", True)),
"requires_credentials": bool(info.get("requires_credentials", False)),
"credential_provider": info.get("credential_provider"),
"credential_status": info.get("credential_status", "none"),
}
def is_due_for_collection(datasource: DataSource, now: datetime) -> bool:
if datasource.last_run_at is None:
return True
@@ -72,31 +84,6 @@ async def _load_latest_running_tasks(
return {task.datasource_id: task for task in result.scalars().all()}
async def _load_latest_completed_tasks(
db: AsyncSession,
datasource_ids: list[int],
) -> dict[int, CollectionTask]:
if not datasource_ids:
return {}
ranked_tasks = (
select(
CollectionTask.id.label("task_id"),
_task_rank_column(CollectionTask.completed_at),
)
.where(CollectionTask.datasource_id.in_(datasource_ids))
.where(CollectionTask.completed_at.isnot(None))
.where(CollectionTask.status.in_(("success", "failed", "cancelled")))
.subquery()
)
result = await db.execute(
select(CollectionTask)
.join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id)
.where(ranked_tasks.c.row_num == 1)
)
return {task.datasource_id: task for task in result.scalars().all()}
async def _load_latest_task_ids(
db: AsyncSession,
datasource_ids: list[int],
@@ -123,21 +110,6 @@ async def _load_latest_task_ids(
return {datasource_id: task_id for datasource_id, task_id in result.all()}
async def _load_datasource_data_counts(
db: AsyncSession,
sources: list[str],
) -> dict[str, int]:
if not sources:
return {}
result = await db.execute(
select(CollectedData.source, func.count(CollectedData.id))
.where(CollectedData.source.in_(sources))
.group_by(CollectedData.source)
)
return {source: count for source, count in result.all()}
async def _load_datasource_endpoint_overrides(
db: AsyncSession,
sources: list[str],
@@ -161,7 +133,7 @@ async def _load_datasource_endpoint_overrides(
async def _load_datasource_list_context(
db: AsyncSession,
datasources: list[DataSource],
) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, int], dict[str, str]]:
) -> tuple[dict[int, CollectionTask], dict[str, str]]:
datasource_ids = [datasource.id for datasource in datasources]
sources = [datasource.source for datasource in datasources]
@@ -185,10 +157,8 @@ async def _load_datasource_list_context(
if stale_datasource_ids:
running_tasks = await _load_latest_running_tasks(db, datasource_ids)
completed_tasks = await _load_latest_completed_tasks(db, datasource_ids)
data_counts = await _load_datasource_data_counts(db, sources)
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
return running_tasks, completed_tasks, data_counts, endpoint_overrides
return running_tasks, endpoint_overrides
async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
@@ -401,27 +371,19 @@ async def list_datasources(
collector_list = []
config = get_data_sources_config()
running_tasks, completed_tasks, data_counts, endpoint_overrides = await _load_datasource_list_context(
db,
datasources,
)
running_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
for datasource in datasources:
running_task = running_tasks.get(datasource.id)
last_task = completed_tasks.get(datasource.id)
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(
datasource.source,
)
data_count = data_counts.get(datasource.source, 0)
last_run_at = datasource.last_run_at or (last_task.completed_at if last_task else None)
last_run = to_iso8601_utc(last_run_at)
last_status = datasource.last_status or (last_task.status if last_task else None)
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
last_run_at = datasource.last_run_at
last_status = datasource.last_status
collector_list.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
**datasource_metadata(datasource.source),
"module": datasource.module,
"priority": datasource.priority,
"frequency": format_frequency_label(datasource.frequency_minutes),
@@ -429,11 +391,9 @@ async def list_datasources(
"is_active": datasource.is_active,
"collector_class": datasource.collector_class,
"endpoint": endpoint,
"last_run": last_run,
"last_run": to_iso8601_utc(last_run_at),
"last_run_at": to_iso8601_utc(last_run_at),
"last_status": last_status,
"last_records_processed": last_task.records_processed if last_task else None,
"data_count": data_count,
"is_running": running_task is not None,
"task_id": running_task.id if running_task else None,
"progress": running_task.progress if running_task else None,
@@ -576,6 +536,7 @@ async def get_datasource(
return {
"id": datasource.id,
"name": datasource.name,
**datasource_metadata(datasource.source),
"module": datasource.module,
"priority": datasource.priority,
"frequency": format_frequency_label(datasource.frequency_minutes),