feat: configure collector endpoints and health plan

This commit is contained in:
linkong
2026-04-08 10:15:33 +08:00
parent 2d43263b9e
commit 8bd9d34376
14 changed files with 673 additions and 53 deletions

View File

@@ -10,6 +10,7 @@ from typing import Dict, Any, List
from bs4 import BeautifulSoup
import httpx
from app.core.data_sources import get_data_sources_config
from app.services.collectors.base import BaseCollector
@@ -22,7 +23,7 @@ class TOP500Collector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch TOP500 list data and enrich each row with detail-page metadata."""
url = "https://top500.org/lists/top500/list/2025/11/"
url = self._resolved_url or ""
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
response = await client.get(url)
@@ -48,11 +49,13 @@ class TOP500Collector(BaseCollector):
return await asyncio.gather(*(enrich(entry) for entry in entries))
def _extract_system_fields(self, system_cell) -> Dict[str, str]:
config = get_data_sources_config()
top500_base_url = config.get_yaml_value("top500.base_url") or "https://top500.org"
link = system_cell.find("a")
system_name = link.get_text(" ", strip=True) if link else system_cell.get_text(" ", strip=True)
detail_url = ""
if link and link.get("href"):
detail_url = f"https://top500.org{link.get('href')}"
detail_url = f"{str(top500_base_url).rstrip('/')}{link.get('href')}"
manufacturer = ""
if link and link.next_sibling: