linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.45.0

Browse Source
This commit is contained in:
rayd1o
2026-04-29 23:43:54 +08:00
parent 9dafbf4f6e
commit ba54545ac7
32 changed files with 602 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
backend/app/api/v1/datasources.py
View File

@@ -398,6 +398,11 @@ async def list_datasources(
"task_id": running_task.id if running_task else None,
"progress": running_task.progress if running_task else None,
"phase": running_task.phase if running_task else None,
"phase_progress": running_task.phase_progress if running_task else None,
"phase_message": running_task.phase_message if running_task else None,
"phase_current": running_task.phase_current if running_task else None,
"phase_total": running_task.phase_total if running_task else None,
"phase_unit": running_task.phase_unit if running_task else None,
"records_processed": running_task.records_processed if running_task else None,
"total_records": running_task.total_records if running_task else None,
}
@@ -626,6 +631,11 @@ async def trigger_datasource(
"message": "当前采集任务尚未完成，重新触发会丢失本次未完成进度。是否强制重新采集？",
"task_id": running_task.id,
"phase": running_task.phase,
"phase_progress": running_task.phase_progress,
"phase_message": running_task.phase_message,
"phase_current": running_task.phase_current,
"phase_total": running_task.phase_total,
"phase_unit": running_task.phase_unit,
"progress": running_task.progress,
"records_processed": running_task.records_processed,
"total_records": running_task.total_records,
@@ -709,13 +719,29 @@ async def get_task_status(
task = await get_running_task(db, datasource.id)
if not task:
return {"is_running": False, "task_id": None, "progress": None, "phase": None, "status": "idle"}
return {
"is_running": False,
"task_id": None,
"progress": None,
"phase": None,
"phase_progress": None,
"phase_message": None,
"phase_current": None,
"phase_total": None,
"phase_unit": None,
"status": "idle",
}
return {
"is_running": task.status == "running",
"task_id": task.id,
"progress": task.progress,
"phase": task.phase,
"phase_progress": task.phase_progress,
"phase_message": task.phase_message,
"phase_current": task.phase_current,
"phase_total": task.phase_total,
"phase_unit": task.phase_unit,
"records_processed": task.records_processed,
"total_records": task.total_records,
"status": task.status,

24
backend/app/api/v1/tasks.py
View File

@@ -27,7 +27,9 @@ async def list_tasks(
offset = (page - 1) * page_size
query = """
SELECT ct.id, ct.datasource_id, ds.name as datasource_name, ct.status,
ct.started_at, ct.completed_at, ct.records_processed, ct.error_message
ct.started_at, ct.completed_at, ct.records_processed, ct.error_message,
ct.phase, ct.phase_progress, ct.phase_message, ct.phase_current,
ct.phase_total, ct.phase_unit, ct.total_records, ct.progress
FROM collection_tasks ct
JOIN data_sources ds ON ct.datasource_id = ds.id
WHERE 1=1
@@ -66,6 +68,14 @@ async def list_tasks(
"completed_at": to_iso8601_utc(t[5]),
"records_processed": t[6],
"error_message": t[7],
"phase": t[8],
"phase_progress": t[9],
"phase_message": t[10],
"phase_current": t[11],
"phase_total": t[12],
"phase_unit": t[13],
"total_records": t[14],
"progress": t[15],
}
for t in tasks
],
@@ -81,7 +91,9 @@ async def get_task(
result = await db.execute(
text("""
SELECT ct.id, ct.datasource_id, ds.name as datasource_name, ct.status,
ct.started_at, ct.completed_at, ct.records_processed, ct.error_message
ct.started_at, ct.completed_at, ct.records_processed, ct.error_message,
ct.phase, ct.phase_progress, ct.phase_message, ct.phase_current,
ct.phase_total, ct.phase_unit, ct.total_records, ct.progress
FROM collection_tasks ct
JOIN data_sources ds ON ct.datasource_id = ds.id
WHERE ct.id = :id
@@ -105,6 +117,14 @@ async def get_task(
"completed_at": to_iso8601_utc(task[5]),
"records_processed": task[6],
"error_message": task[7],
"phase": task[8],
"phase_progress": task[9],
"phase_message": task[10],
"phase_current": task[11],
"phase_total": task[12],
"phase_unit": task[13],
"total_records": task[14],
"progress": task[15],
}

7
backend/app/db/session.py
View File

@@ -146,7 +146,12 @@ async def init_db():
text(
"""
ALTER TABLE collection_tasks
ADD COLUMN IF NOT EXISTS phase VARCHAR(30) DEFAULT 'queued'
ADD COLUMN IF NOT EXISTS phase VARCHAR(30) DEFAULT 'queued',
ADD COLUMN IF NOT EXISTS phase_progress DOUBLE PRECISION,
ADD COLUMN IF NOT EXISTS phase_message VARCHAR(255),
ADD COLUMN IF NOT EXISTS phase_current BIGINT,
ADD COLUMN IF NOT EXISTS phase_total BIGINT,
ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30)
"""
)
)

7
backend/app/models/task.py
View File

@@ -1,6 +1,6 @@
"""Collection Task model"""
from sqlalchemy import Column, DateTime, Integer, String, Text, Float
from sqlalchemy import BigInteger, Column, DateTime, Integer, String, Text, Float
from sqlalchemy.sql import func
from app.db.session import Base
@@ -13,6 +13,11 @@ class CollectionTask(Base):
datasource_id = Column(Integer, nullable=False, index=True)
status = Column(String(20), nullable=False) # pending, running, success, failed, cancelled
phase = Column(String(30), default="queued")
phase_progress = Column(Float)
phase_message = Column(String(255))
phase_current = Column(BigInteger)
phase_total = Column(BigInteger)
phase_unit = Column(String(30))
started_at = Column(DateTime(timezone=True))
completed_at = Column(DateTime(timezone=True))
records_processed = Column(Integer, default=0)

60
backend/app/services/collectors/base.py
View File

@@ -54,6 +54,11 @@ class BaseCollector(ABC):
"task_id": self._current_task.id,
"status": self._current_task.status,
"phase": self._current_task.phase,
"phase_progress": self._current_task.phase_progress,
"phase_message": self._current_task.phase_message,
"phase_current": self._current_task.phase_current,
"phase_total": self._current_task.phase_total,
"phase_unit": self._current_task.phase_unit,
"progress": progress,
"records_processed": self._current_task.records_processed,
"total_records": self._current_task.total_records,
@@ -80,12 +85,52 @@ class BaseCollector(ABC):
await self._publish_task_update(force=force)
async def set_phase(self, phase: str):
async def set_phase(self, phase: str, *, message: str | None = None, reset_progress: bool = True):
if self._current_task and self._db_session:
self._current_task.phase = phase
self._current_task.phase_message = message
if reset_progress:
self._current_task.phase_progress = None
self._current_task.phase_current = None
self._current_task.phase_total = None
self._current_task.phase_unit = None
await self._db_session.commit()
await self._publish_task_update(force=True)
async def update_phase_progress(
self,
*,
current: int | None = None,
total: int | None = None,
unit: str | None = None,
message: str | None = None,
progress: float | None = None,
commit: bool = False,
force: bool = False,
):
"""Update progress for the current phase without changing task totals."""
if not self._current_task or not self._db_session:
return
if progress is None and current is not None and total and total > 0:
progress = (current / total) * 100
if progress is not None:
self._current_task.phase_progress = max(0.0, min(float(progress), 100.0))
if current is not None:
self._current_task.phase_current = max(0, int(current))
if total is not None:
self._current_task.phase_total = max(0, int(total))
if unit is not None:
self._current_task.phase_unit = unit
if message is not None:
self._current_task.phase_message = message
if commit:
await self._db_session.commit()
await self._publish_task_update(force=force)
@abstractmethod
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch raw data from source"""
@@ -251,7 +296,7 @@ class BaseCollector(ABC):
await self._publish_task_update(force=True)
try:
await self.set_phase("fetching")
await self.set_phase("fetching", message="正在拉取原始数据")
raw_data = await self.fetch()
task.total_records = len(raw_data)
await db.commit()
@@ -260,15 +305,20 @@ class BaseCollector(ABC):
if self.fail_on_empty and not raw_data:
raise RuntimeError(f"Collector {self.name} returned no data")
await self.set_phase("transforming")
await self.set_phase("transforming", message="正在转换采集数据")
data = self.transform(raw_data)
snapshot_id = await self._create_snapshot(db, task_id, data, start_time)
await self.set_phase("saving")
await self.set_phase("saving", message="正在保存采集数据")
records_count = await self._save_data(db, data, task_id=task_id, snapshot_id=snapshot_id)
task.status = "success"
task.phase = "completed"
task.phase_progress = 100.0
task.phase_message = "采集完成"
task.phase_current = records_count
task.phase_total = records_count
task.phase_unit = "records"
task.records_processed = records_count
task.progress = 100.0
task.completed_at = datetime.now(UTC)
@@ -285,6 +335,7 @@ class BaseCollector(ABC):
await db.rollback()
task.status = "cancelled"
task.phase = "cancelled"
task.phase_message = "采集已取消"
task.error_message = "Collection cancelled by operator and rolled back"
task.completed_at = datetime.now(UTC)
if snapshot_id is not None:
@@ -301,6 +352,7 @@ class BaseCollector(ABC):
await db.rollback()
task.status = "failed"
task.phase = "failed"
task.phase_message = str(e)
task.error_message = str(e)
task.completed_at = datetime.now(UTC)
if snapshot_id is not None:

20
backend/app/services/collectors/iptoasn.py
View File

@@ -108,6 +108,11 @@ class IPtoASNPrefixGeoCollector(BaseCollector):
self._current_task.total_records = total_expected
self._current_task.records_processed = 0
self._current_task.progress = 0.0
self._current_task.phase_progress = 0.0
self._current_task.phase_message = "正在下载 IPtoASN 数据"
self._current_task.phase_current = 0
self._current_task.phase_total = total_expected
self._current_task.phase_unit = "bytes"
await self._db_session.commit()
await self._publish_task_update(force=True)
@@ -135,7 +140,14 @@ class IPtoASNPrefixGeoCollector(BaseCollector):
return
last_emit["value"] = aggregated
last_emit["t"] = now
await self.update_progress(min(aggregated, total_expected), commit=True)
current = min(aggregated, total_expected)
await self.update_phase_progress(
current=current,
total=total_expected,
unit="bytes",
message="正在下载 IPtoASN 数据",
)
await self.update_progress(current, commit=True)
batches = await asyncio.gather(
*(
@@ -148,6 +160,12 @@ class IPtoASNPrefixGeoCollector(BaseCollector):
)
)
if total_expected > 0:
await self.update_phase_progress(
current=total_expected,
total=total_expected,
unit="bytes",
message="IPtoASN 数据下载完成",
)
await self.update_progress(total_expected, commit=True, force=True)
rows: list[dict[str, Any]] = []

11
backend/app/services/collectors/nro_delegated.py
View File

@@ -39,12 +39,23 @@ class NRODelegatedPrefixGeoCollector(BaseCollector):
self._current_task.total_records = total_expected
self._current_task.records_processed = 0
self._current_task.progress = 0.0
self._current_task.phase_progress = 0.0
self._current_task.phase_message = "正在下载 NRO delegated 数据"
self._current_task.phase_current = 0
self._current_task.phase_total = total_expected
self._current_task.phase_unit = "bytes"
await self._db_session.commit()
await self._publish_task_update(force=True)
async def on_progress(downloaded: int, total: int | None) -> None:
if not total or total <= 0:
return
await self.update_phase_progress(
current=min(downloaded, total),
total=total,
unit="bytes",
message="正在下载 NRO delegated 数据",
)
await self.update_progress(min(downloaded, total), commit=True)
body_path = await self._downloader.download_file(

11
backend/app/services/collectors/opengeofeed.py
View File

@@ -40,12 +40,23 @@ class OpenGeoFeedPrefixGeoCollector(BaseCollector):
self._current_task.total_records = total_expected
self._current_task.records_processed = 0
self._current_task.progress = 0.0
self._current_task.phase_progress = 0.0
self._current_task.phase_message = "正在下载 OpenGeoFeed 数据"
self._current_task.phase_current = 0
self._current_task.phase_total = total_expected
self._current_task.phase_unit = "bytes"
await self._db_session.commit()
await self._publish_task_update(force=True)
async def on_progress(downloaded: int, total: int | None) -> None:
if not total or total <= 0:
return
await self.update_phase_progress(
current=min(downloaded, total),
total=total,
unit="bytes",
message="正在下载 OpenGeoFeed 数据",
)
await self.update_progress(min(downloaded, total), commit=True)
body_path = await self._downloader.download_file(

26
backend/tests/test_collectors.py
View File

@@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
from app.services.collectors.top500 import TOP500Collector
from app.services.collectors.base import BaseCollector, HTTPCollector
from app.models.task import CollectionTask
class TestBaseCollector:
@@ -19,6 +20,31 @@ class TestBaseCollector:
assert collector.module == "L1"
assert collector.frequency_hours == 4
@pytest.mark.asyncio
async def test_update_phase_progress_tracks_phase_fields(self, mock_db_session):
"""Test phase-level progress updates independently from record totals"""
collector = TOP500Collector()
task = CollectionTask(datasource_id=1, status="running", phase="fetching")
collector._current_task = task
collector._db_session = mock_db_session
with patch.object(collector, "_publish_task_update", new=AsyncMock()) as publish:
await collector.update_phase_progress(
current=512,
total=1024,
unit="bytes",
message="Downloading dataset",
commit=True,
)
assert task.phase_progress == 50.0
assert task.phase_current == 512
assert task.phase_total == 1024
assert task.phase_unit == "bytes"
assert task.phase_message == "Downloading dataset"
mock_db_session.commit.assert_awaited_once()
publish.assert_awaited_once()
class TestTOP500Collector:
"""Tests for TOP500Collector"""

19
backend/tests/test_models.py
View File

@@ -121,6 +121,25 @@ class TestCollectionTaskModel:
)
assert task.records_processed == 100
def test_task_with_phase_progress(self):
"""Test collection task phase-level progress fields"""
task = CollectionTask(
datasource_id=1,
status="running",
phase="fetching",
phase_progress=42.5,
phase_message="Downloading dataset",
phase_current=1024,
phase_total=4096,
phase_unit="bytes",
)
assert task.phase == "fetching"
assert task.phase_progress == 42.5
assert task.phase_message == "Downloading dataset"
assert task.phase_current == 1024
assert task.phase_total == 4096
assert task.phase_unit == "bytes"
def test_task_error_message(self):
"""Test collection task with error message"""
task = CollectionTask(