release: bump version to 0.61.0

backend/app/services/earth_layer_cache.py

@@ -0,0 +1,392 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+import hashlib
+import json
+import time
+from typing import Any
+
+from fastapi import Response
+
+from app.core.cache import _RedisClient
+from app.core.config import settings
+from app.core.logging import get_logger
+
+
+logger = get_logger(__name__, service="earth_layer_cache")
+
+EARTH_LAYER_CACHE_PREFIX = "earth:layer:v1"
+EARTH_LAYER_LOCK_PREFIX = "earth:layer:lock:v1"
+DEFAULT_LOCK_TTL_SECONDS = 10
+DEFAULT_LOCK_WAIT_SECONDS = 0.2
+DEFAULT_MAX_FEATURES = 5000
+DEFAULT_MAX_BYTES = 5 * 1024 * 1024
+DEFAULT_BBOX_PRECISION_DEGREES = 0.1
+DEV_CACHE_KEY_HEADER = {"development", "dev", "test", "testing", "local"}
+
+
+@dataclass(frozen=True)
+class EarthLayerCachePolicy:
+    fresh_ttl_seconds: int
+    stale_ttl_seconds: int
+    max_features: int = DEFAULT_MAX_FEATURES
+    max_bytes: int = DEFAULT_MAX_BYTES
+    lock_ttl_seconds: int = DEFAULT_LOCK_TTL_SECONDS
+    lock_wait_seconds: float = DEFAULT_LOCK_WAIT_SECONDS
+
+
+@dataclass(frozen=True)
+class EarthLayerCacheResult:
+    payload: dict[str, Any]
+    state: str
+    key: str
+    features: int
+    bytes: int
+
+
+class EarthLayerCache:
+    def __init__(self) -> None:
+        self._client = None
+
+    @property
+    def client(self):
+        if self._client is None:
+            self._client = _RedisClient.get_client()
+        return self._client
+
+    @staticmethod
+    def key(layer: str, **params: Any) -> str:
+        parts = [EARTH_LAYER_CACHE_PREFIX, _safe_key_part(layer)]
+        for name in sorted(params):
+            value = params[name]
+            if value is None:
+                value = "none"
+            parts.append(f"{_safe_key_part(name)}:{_safe_key_part(value)}")
+        return ":".join(parts)
+
+    @staticmethod
+    def stale_key(key: str) -> str:
+        return f"{key}:stale"
+
+    @staticmethod
+    def lock_key(key: str) -> str:
+        digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:32]
+        return f"{EARTH_LAYER_LOCK_PREFIX}:{digest}"
+
+    def get_json(self, key: str) -> dict[str, Any] | None:
+        raw = self.client.get(key)
+        if not raw:
+            return None
+        value = json.loads(raw)
+        return value if isinstance(value, dict) else None
+
+    def set_json(self, key: str, payload: dict[str, Any], ttl_seconds: int) -> None:
+        self.client.setex(key, ttl_seconds, json.dumps(payload, ensure_ascii=False, default=str))
+
+    def acquire_lock(self, key: str, ttl_seconds: int) -> bool:
+        return bool(self.client.set(self.lock_key(key), "1", nx=True, ex=ttl_seconds))
+
+    def release_lock(self, key: str) -> None:
+        try:
+            self.client.delete(self.lock_key(key))
+        except Exception:
+            pass
+
+    def delete_pattern(self, pattern: str = f"{EARTH_LAYER_CACHE_PREFIX}:*") -> int:
+        keys = list(self.client.scan_iter(match=pattern))
+        if not keys:
+            return 0
+        return int(self.client.delete(*keys))
+
+    def status(self) -> dict[str, Any]:
+        keys = list(self.client.scan_iter(match=f"{EARTH_LAYER_CACHE_PREFIX}:*"))
+        by_layer: dict[str, dict[str, Any]] = {}
+        total_memory = 0
+        for key in keys:
+            key_str = key.decode("utf-8") if isinstance(key, bytes) else str(key)
+            layer = _layer_from_key(key_str)
+            entry = by_layer.setdefault(layer, {"keys": 0, "stale_keys": 0, "memory_bytes": 0})
+            entry["keys"] += 1
+            if key_str.endswith(":stale"):
+                entry["stale_keys"] += 1
+            try:
+                memory = int(self.client.memory_usage(key) or 0)
+            except Exception:
+                memory = 0
+            entry["memory_bytes"] += memory
+            total_memory += memory
+        return {
+            "prefix": EARTH_LAYER_CACHE_PREFIX,
+            "key_count": len(keys),
+            "memory_bytes": total_memory,
+            "layers": by_layer,
+        }
+
+
+earth_layer_cache = EarthLayerCache()
+
+
+def quantize_bbox(
+    bbox: tuple[float, float, float, float],
+    *,
+    precision: float = DEFAULT_BBOX_PRECISION_DEGREES,
+) -> tuple[float, float, float, float]:
+    return tuple(round(value / precision) * precision for value in bbox)  # type: ignore[return-value]
+
+
+def format_bbox_key(bbox: tuple[float, float, float, float]) -> str:
+    return ",".join(f"{value:.1f}" for value in bbox)
+
+
+def apply_cache_headers(response: Response | None, result: EarthLayerCacheResult) -> None:
+    if response is None:
+        return
+    response.headers["X-Planet-Cache"] = result.state
+    response.headers["X-Planet-Cache-Features"] = str(result.features)
+    response.headers["X-Planet-Cache-Bytes"] = str(result.bytes)
+    env_name = str(getattr(settings, "ENVIRONMENT", "") or "development").lower()
+    if env_name in DEV_CACHE_KEY_HEADER:
+        response.headers["X-Planet-Cache-Key"] = result.key
+
+
+async def get_or_build_layer_payload(
+    *,
+    key: str,
+    policy: EarthLayerCachePolicy,
+    builder: Callable[[], Awaitable[dict[str, Any]]],
+    response: Response | None = None,
+) -> dict[str, Any]:
+    result = await resolve_layer_payload(key=key, policy=policy, builder=builder)
+    apply_cache_headers(response, result)
+    return result.payload
+
+
+async def resolve_layer_payload(
+    *,
+    key: str,
+    policy: EarthLayerCachePolicy,
+    builder: Callable[[], Awaitable[dict[str, Any]]],
+) -> EarthLayerCacheResult:
+    started = time.perf_counter()
+    try:
+        cached = earth_layer_cache.get_json(key)
+        if cached is not None:
+            return _result(cached, state="hit", key=key)
+
+        lock_acquired = earth_layer_cache.acquire_lock(key, policy.lock_ttl_seconds)
+        if lock_acquired:
+            try:
+                payload = await _build_budgeted_payload(builder, policy)
+                _write_fresh_and_stale(key, payload, policy)
+                _log_cache_event("refresh", key, payload, started)
+                return _result(payload, state="refresh", key=key)
+            except Exception as exc:
+                stale = _read_stale(key)
+                if stale is not None:
+                    logger.warning_event(
+                        "Earth layer cache builder failed; returning stale payload",
+                        event="earth_layer_cache.stale_after_builder_error",
+                        context={"key": key, "error": str(exc)},
+                    )
+                    return _result(stale, state="stale", key=key)
+                raise
+            finally:
+                earth_layer_cache.release_lock(key)
+
+        stale = _read_stale(key)
+        if stale is not None:
+            return _result(stale, state="stale", key=key)
+
+        await asyncio.sleep(policy.lock_wait_seconds)
+        cached_after_wait = earth_layer_cache.get_json(key)
+        if cached_after_wait is not None:
+            return _result(cached_after_wait, state="hit", key=key)
+
+        payload = await _build_budgeted_payload(builder, policy)
+        _log_cache_event("miss", key, payload, started)
+        return _result(payload, state="miss", key=key)
+    except Exception as exc:
+        try:
+            payload = await _build_budgeted_payload(builder, policy)
+        except Exception:
+            raise exc
+        logger.warning_event(
+            "Earth layer cache bypassed",
+            event="earth_layer_cache.bypass",
+            context={"key": key, "error": str(exc)},
+        )
+        return _result(payload, state="bypass", key=key)
+
+
+def apply_payload_budget(payload: dict[str, Any], policy: EarthLayerCachePolicy) -> dict[str, Any]:
+    budgeted = _truncate_features(payload, policy.max_features, "feature_budget")
+    size = _payload_size(budgeted)
+    if size <= policy.max_bytes:
+        return budgeted
+
+    features = budgeted.get("features")
+    if not isinstance(features, list):
+        return _with_budget_diagnostics(
+            budgeted,
+            truncated=True,
+            reason="byte_budget",
+            bytes_before=size,
+            bytes_after=size,
+        )
+
+    low = 0
+    high = len(features)
+    best = []
+    best_size = _payload_size({**budgeted, "features": best})
+    while low <= high:
+        mid = (low + high) // 2
+        candidate_features = features[:mid]
+        candidate = _with_budget_diagnostics(
+            {**budgeted, "features": candidate_features},
+            truncated=mid < len(features),
+            reason="byte_budget",
+            bytes_before=size,
+            bytes_after=0,
+        )
+        candidate_size = _payload_size(candidate)
+        if candidate_size <= policy.max_bytes:
+            best = candidate_features
+            best_size = candidate_size
+            low = mid + 1
+        else:
+            high = mid - 1
+
+    return _with_budget_diagnostics(
+        {**budgeted, "features": best},
+        truncated=True,
+        reason="byte_budget",
+        bytes_before=size,
+        bytes_after=best_size,
+    )
+
+
+def invalidate_earth_layer_cache_for_source(source: str) -> int:
+    source_key = str(source or "").strip()
+    patterns = {
+        "arcgis_cables": ["cables*", "landing-points*", "summary*"],
+        "arcgis_landing_points": ["landing-points*", "summary*"],
+        "arcgis_cable_landing_relation": ["landing-points*", "summary*"],
+        "celestrak_tle": ["satellites*", "summary*"],
+        "spacetrack_tle": ["satellites*", "summary*"],
+        "top500": ["compute-centers*", "summary*"],
+        "epoch_ai_gpu": ["compute-centers*", "summary*"],
+        "ris_live_bgp": ["bgp*", "summary*"],
+        "bgpstream_bgp": ["bgp*", "summary*"],
+    }.get(source_key, [])
+    deleted = 0
+    for layer_pattern in patterns:
+        deleted += earth_layer_cache.delete_pattern(f"{EARTH_LAYER_CACHE_PREFIX}:{layer_pattern}")
+    return deleted
+
+
+async def _build_budgeted_payload(
+    builder: Callable[[], Awaitable[dict[str, Any]]],
+    policy: EarthLayerCachePolicy,
+) -> dict[str, Any]:
+    payload = await builder()
+    return apply_payload_budget(payload, policy)
+
+
+def _write_fresh_and_stale(key: str, payload: dict[str, Any], policy: EarthLayerCachePolicy) -> None:
+    earth_layer_cache.set_json(key, payload, policy.fresh_ttl_seconds)
+    earth_layer_cache.set_json(earth_layer_cache.stale_key(key), payload, policy.stale_ttl_seconds)
+
+
+def _read_stale(key: str) -> dict[str, Any] | None:
+    try:
+        return earth_layer_cache.get_json(earth_layer_cache.stale_key(key))
+    except Exception:
+        return None
+
+
+def _truncate_features(payload: dict[str, Any], max_features: int, reason: str) -> dict[str, Any]:
+    features = payload.get("features")
+    if not isinstance(features, list) or len(features) <= max_features:
+        return payload
+    return _with_budget_diagnostics(
+        {**payload, "features": features[:max_features]},
+        truncated=True,
+        reason=reason,
+        original_feature_count=len(features),
+    )
+
+
+def _with_budget_diagnostics(
+    payload: dict[str, Any],
+    *,
+    truncated: bool,
+    reason: str,
+    original_feature_count: int | None = None,
+    bytes_before: int | None = None,
+    bytes_after: int | None = None,
+) -> dict[str, Any]:
+    diagnostics = dict(payload.get("diagnostics") or {})
+    diagnostics.update(
+        {
+            "truncated": bool(truncated or diagnostics.get("truncated")),
+            "limit_reason": reason,
+        }
+    )
+    if original_feature_count is not None:
+        diagnostics["original_feature_count"] = original_feature_count
+    if bytes_before is not None:
+        diagnostics["bytes_before_budget"] = bytes_before
+    if bytes_after is not None:
+        diagnostics["bytes_after_budget"] = bytes_after
+    return {**payload, "diagnostics": diagnostics}
+
+
+def _result(payload: dict[str, Any], *, state: str, key: str) -> EarthLayerCacheResult:
+    return EarthLayerCacheResult(
+        payload=payload,
+        state=state,
+        key=key,
+        features=_feature_count(payload),
+        bytes=_payload_size(payload),
+    )
+
+
+def _feature_count(payload: dict[str, Any]) -> int:
+    features = payload.get("features")
+    if isinstance(features, list):
+        return len(features)
+    count = payload.get("count")
+    return int(count) if isinstance(count, int) else 0
+
+
+def _payload_size(payload: dict[str, Any]) -> int:
+    return len(json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8"))
+
+
+def _safe_key_part(value: Any) -> str:
+    raw = str(value).strip().lower()
+    return "".join(char if char.isalnum() or char in {"-", "_", ".", ","} else "_" for char in raw)[:160]
+
+
+def _layer_from_key(key: str) -> str:
+    prefix = f"{EARTH_LAYER_CACHE_PREFIX}:"
+    if not key.startswith(prefix):
+        return "unknown"
+    remainder = key[len(prefix):]
+    return remainder.split(":", 1)[0]
+
+
+def _log_cache_event(state: str, key: str, payload: dict[str, Any], started: float) -> None:
+    logger.info_event(
+        "Earth layer cache resolved",
+        event="earth_layer_cache.resolved",
+        context={
+            "state": state,
+            "key": key,
+            "features": _feature_count(payload),
+            "bytes": _payload_size(payload),
+            "duration_ms": round((time.perf_counter() - started) * 1000, 2),
+        },
+    )