Refine data management and collection workflows

This commit is contained in:
linkong
2026-03-25 17:19:10 +08:00
parent cc5f16f8a7
commit 020c1d5051
34 changed files with 3341 additions and 947 deletions

View File

@@ -10,6 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func
from typing import List, Dict, Any, Optional
from app.core.collected_data_fields import get_record_field
from app.db.session import get_db
from app.models.collected_data import CollectedData
from app.services.cable_graph import build_graph_from_data, CableGraph
@@ -83,9 +84,9 @@ def convert_cable_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
"rfs": metadata.get("rfs"),
"RFS": metadata.get("rfs"),
"status": metadata.get("status", "active"),
"length": record.value,
"length_km": record.value,
"SHAPE__Length": record.value,
"length": get_record_field(record, "value"),
"length_km": get_record_field(record, "value"),
"SHAPE__Length": get_record_field(record, "value"),
"url": metadata.get("url"),
"color": metadata.get("color"),
"year": metadata.get("year"),
@@ -101,8 +102,10 @@ def convert_landing_point_to_geojson(records: List[CollectedData], city_to_cable
for record in records:
try:
lat = float(record.latitude) if record.latitude else None
lon = float(record.longitude) if record.longitude else None
latitude = get_record_field(record, "latitude")
longitude = get_record_field(record, "longitude")
lat = float(latitude) if latitude else None
lon = float(longitude) if longitude else None
except (ValueError, TypeError):
continue
@@ -116,8 +119,8 @@ def convert_landing_point_to_geojson(records: List[CollectedData], city_to_cable
"id": record.id,
"source_id": record.source_id,
"name": record.name,
"country": record.country,
"city": record.city,
"country": get_record_field(record, "country"),
"city": get_record_field(record, "city"),
"is_tbd": metadata.get("is_tbd", False),
}
@@ -185,9 +188,11 @@ def convert_supercomputer_to_geojson(records: List[CollectedData]) -> Dict[str,
for record in records:
try:
lat = float(record.latitude) if record.latitude and record.latitude != "0.0" else None
latitude = get_record_field(record, "latitude")
longitude = get_record_field(record, "longitude")
lat = float(latitude) if latitude and latitude != "0.0" else None
lon = (
float(record.longitude) if record.longitude and record.longitude != "0.0" else None
float(longitude) if longitude and longitude != "0.0" else None
)
except (ValueError, TypeError):
lat, lon = None, None
@@ -203,12 +208,12 @@ def convert_supercomputer_to_geojson(records: List[CollectedData]) -> Dict[str,
"id": record.id,
"name": record.name,
"rank": metadata.get("rank"),
"r_max": record.value,
"r_peak": metadata.get("r_peak"),
"cores": metadata.get("cores"),
"power": metadata.get("power"),
"country": record.country,
"city": record.city,
"r_max": get_record_field(record, "rmax"),
"r_peak": get_record_field(record, "rpeak"),
"cores": get_record_field(record, "cores"),
"power": get_record_field(record, "power"),
"country": get_record_field(record, "country"),
"city": get_record_field(record, "city"),
"data_type": "supercomputer",
},
}
@@ -223,8 +228,10 @@ def convert_gpu_cluster_to_geojson(records: List[CollectedData]) -> Dict[str, An
for record in records:
try:
lat = float(record.latitude) if record.latitude else None
lon = float(record.longitude) if record.longitude else None
latitude = get_record_field(record, "latitude")
longitude = get_record_field(record, "longitude")
lat = float(latitude) if latitude else None
lon = float(longitude) if longitude else None
except (ValueError, TypeError):
lat, lon = None, None
@@ -238,8 +245,8 @@ def convert_gpu_cluster_to_geojson(records: List[CollectedData]) -> Dict[str, An
"properties": {
"id": record.id,
"name": record.name,
"country": record.country,
"city": record.city,
"country": get_record_field(record, "country"),
"city": get_record_field(record, "city"),
"metadata": metadata,
"data_type": "gpu_cluster",
},