first commit

backend/app/services/collectors/cloudflare.py

@@ -0,0 +1,163 @@
+"""Cloudflare Radar Traffic Collector
+
+Collects Internet traffic data from Cloudflare Radar API.
+https://developers.cloudflare.com/radar/
+
+Note: Radar API provides free access to global Internet traffic data.
+Some endpoints require authentication for higher rate limits.
+"""
+
+import asyncio
+import os
+from typing import Dict, Any, List
+from datetime import datetime
+
+import httpx
+from app.services.collectors.base import HTTPCollector
+
+# Cloudflare API token (optional - for higher rate limits)
+CLOUDFLARE_API_TOKEN = os.environ.get("CLOUDFLARE_API_TOKEN", "")
+
+
+class CloudflareRadarDeviceCollector(HTTPCollector):
+    """Collects device type distribution data (mobile vs desktop)"""
+
+    name = "cloudflare_radar_device"
+    priority = "P2"
+    module = "L3"
+    frequency_hours = 24
+    data_type = "device_stats"
+    base_url = "https://api.cloudflare.com/client/v4/radar/http/summary/device_type"
+
+    def __init__(self):
+        super().__init__()
+        self.headers = {
+            "User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
+            "Accept": "application/json",
+        }
+        if CLOUDFLARE_API_TOKEN:
+            self.headers["Authorization"] = f"Bearer {CLOUDFLARE_API_TOKEN}"
+
+    def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Parse Cloudflare Radar device type response"""
+        data = []
+        result = response.get("result", {})
+        summary = result.get("summary_0", {})
+
+        try:
+            entry = {
+                "source_id": "cloudflare_radar_device_global",
+                "name": "Global Device Distribution",
+                "country": "GLOBAL",
+                "city": "",
+                "latitude": 0.0,
+                "longitude": 0.0,
+                "metadata": {
+                    "desktop_percent": float(summary.get("desktop", 0)),
+                    "mobile_percent": float(summary.get("mobile", 0)),
+                    "other_percent": float(summary.get("other", 0)),
+                    "date_range": result.get("meta", {}).get("dateRange", {}),
+                },
+                "reference_date": datetime.utcnow().isoformat(),
+            }
+            data.append(entry)
+        except (ValueError, TypeError, KeyError):
+            pass
+
+        return data
+
+
+class CloudflareRadarTrafficCollector(HTTPCollector):
+    """Collects traffic volume trends"""
+
+    name = "cloudflare_radar_traffic"
+    priority = "P2"
+    module = "L3"
+    frequency_hours = 24
+    data_type = "traffic_stats"
+    base_url = "https://api.cloudflare.com/client/v4/radar/http/timeseries/requests"
+
+    def __init__(self):
+        super().__init__()
+        self.headers = {
+            "User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
+            "Accept": "application/json",
+        }
+        if CLOUDFLARE_API_TOKEN:
+            self.headers["Authorization"] = f"Bearer {CLOUDFLARE_API_TOKEN}"
+
+    def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Parse Cloudflare Radar traffic timeseries response"""
+        data = []
+        result = response.get("result", {})
+        timeseries = result.get("requests_0", {}).get("timeseries", [])
+
+        for item in timeseries:
+            try:
+                entry = {
+                    "source_id": f"cloudflare_traffic_{item.get('datetime', '')}",
+                    "name": f"Traffic {item.get('datetime', '')[:10]}",
+                    "country": "GLOBAL",
+                    "city": "",
+                    "latitude": 0.0,
+                    "longitude": 0.0,
+                    "metadata": {
+                        "datetime": item.get("datetime"),
+                        "requests": item.get("requests"),
+                        "visit_duration": item.get("visitDuration"),
+                    },
+                    "reference_date": item.get("datetime", datetime.utcnow().isoformat()),
+                }
+                data.append(entry)
+            except (ValueError, TypeError, KeyError):
+                continue
+
+        return data
+
+
+class CloudflareRadarTopASCollector(HTTPCollector):
+    """Collects top autonomous systems by traffic"""
+
+    name = "cloudflare_radar_top_as"
+    priority = "P2"
+    module = "L2"
+    frequency_hours = 24
+    data_type = "as_stats"
+    base_url = "https://api.cloudflare.com/client/v4/radar/http/top/locations"
+
+    def __init__(self):
+        super().__init__()
+        self.headers = {
+            "User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
+            "Accept": "application/json",
+        }
+        if CLOUDFLARE_API_TOKEN:
+            self.headers["Authorization"] = f"Bearer {CLOUDFLARE_API_TOKEN}"
+
+    def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Parse Cloudflare Radar top locations response"""
+        data = []
+        result = response.get("result", {})
+        top_locations = result.get("top_locations_0", [])
+
+        for idx, item in enumerate(top_locations):
+            try:
+                entry = {
+                    "source_id": f"cloudflare_as_{item.get('rank', idx)}",
+                    "name": item.get("location", {}).get("countryName", "Unknown"),
+                    "country": item.get("location", {}).get("countryCode", "XX"),
+                    "city": item.get("location", {}).get("cityName", ""),
+                    "latitude": float(item.get("location", {}).get("latitude", 0)),
+                    "longitude": float(item.get("location", {}).get("longitude", 0)),
+                    "metadata": {
+                        "rank": item.get("rank"),
+                        "traffic_share": item.get("trafficShare"),
+                        "country_code": item.get("location", {}).get("countryCode"),
+                    },
+                    "reference_date": datetime.utcnow().isoformat(),
+                }
+                data.append(entry)
+            except (ValueError, TypeError, KeyError):
+                continue
+
+        return data