Refine data management and collection workflows

scripts/backfill_collected_data_metadata.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Backfill legacy collected_data columns into metadata."""
+
+import asyncio
+import os
+import sys
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+BACKEND_DIR = os.path.join(ROOT_DIR, "backend")
+
+sys.path.insert(0, ROOT_DIR)
+sys.path.insert(0, BACKEND_DIR)
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
+from sqlalchemy.orm import sessionmaker
+
+from app.core.collected_data_fields import build_dynamic_metadata
+from app.models.collected_data import CollectedData
+
+
+async def main():
+    database_url = os.environ.get(
+        "DATABASE_URL", "postgresql+asyncpg://postgres:postgres@localhost:5432/planet_db"
+    )
+    engine = create_async_engine(database_url, echo=False)
+    async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+
+    updated = 0
+
+    async with async_session() as session:
+        result = await session.execute(select(CollectedData))
+        records = result.scalars().all()
+
+        for record in records:
+            merged_metadata = build_dynamic_metadata(
+                record.extra_data or {},
+                country=record.country,
+                city=record.city,
+                latitude=record.latitude,
+                longitude=record.longitude,
+                value=record.value,
+                unit=record.unit,
+            )
+
+            if merged_metadata != (record.extra_data or {}):
+                record.extra_data = merged_metadata
+                updated += 1
+
+        await session.commit()
+
+    await engine.dispose()
+    print(f"Backfill completed. Updated {updated} collected_data rows.")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())