feat: configure collector endpoints and health plan
This commit is contained in:
@@ -10,6 +10,7 @@ from typing import Dict, Any, List
|
||||
from bs4 import BeautifulSoup
|
||||
import httpx
|
||||
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.services.collectors.base import BaseCollector
|
||||
|
||||
|
||||
@@ -22,7 +23,7 @@ class TOP500Collector(BaseCollector):
|
||||
|
||||
async def fetch(self) -> List[Dict[str, Any]]:
|
||||
"""Fetch TOP500 list data and enrich each row with detail-page metadata."""
|
||||
url = "https://top500.org/lists/top500/list/2025/11/"
|
||||
url = self._resolved_url or ""
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
|
||||
response = await client.get(url)
|
||||
@@ -48,11 +49,13 @@ class TOP500Collector(BaseCollector):
|
||||
return await asyncio.gather(*(enrich(entry) for entry in entries))
|
||||
|
||||
def _extract_system_fields(self, system_cell) -> Dict[str, str]:
|
||||
config = get_data_sources_config()
|
||||
top500_base_url = config.get_yaml_value("top500.base_url") or "https://top500.org"
|
||||
link = system_cell.find("a")
|
||||
system_name = link.get_text(" ", strip=True) if link else system_cell.get_text(" ", strip=True)
|
||||
detail_url = ""
|
||||
if link and link.get("href"):
|
||||
detail_url = f"https://top500.org{link.get('href')}"
|
||||
detail_url = f"{str(top500_base_url).rstrip('/')}{link.get('href')}"
|
||||
|
||||
manufacturer = ""
|
||||
if link and link.next_sibling:
|
||||
|
||||
Reference in New Issue
Block a user