Files
planet/backend/app/services/earth_layer_cache.py
2026-11-09 15:12:50 +08:00

499 lines
16 KiB
Python

from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from contextlib import suppress
from dataclasses import dataclass, replace
import hashlib
import json
import time
from typing import Any
from uuid import uuid4
from fastapi import HTTPException, Response
from app.core.cache import _RedisClient
from app.core.config import settings
from app.core.logging import get_logger
logger = get_logger(__name__, service="earth_layer_cache")
EARTH_LAYER_CACHE_PREFIX = "earth:layer:v1"
EARTH_LAYER_LOCK_PREFIX = "earth:layer:lock:v1"
DEFAULT_LOCK_TTL_SECONDS = 10
DEFAULT_LOCK_WAIT_SECONDS = 0.2
DEFAULT_BUILD_WAIT_SECONDS = 30
DEFAULT_MAX_FEATURES = 5000
DEFAULT_MAX_BYTES = 5 * 1024 * 1024
DEFAULT_BBOX_PRECISION_DEGREES = 0.1
DEV_CACHE_KEY_HEADER = {"development", "dev", "test", "testing", "local"}
@dataclass(frozen=True)
class EarthLayerCachePolicy:
fresh_ttl_seconds: int
stale_ttl_seconds: int
max_features: int = DEFAULT_MAX_FEATURES
max_bytes: int = DEFAULT_MAX_BYTES
lock_ttl_seconds: int = DEFAULT_LOCK_TTL_SECONDS
lock_wait_seconds: float = DEFAULT_LOCK_WAIT_SECONDS
build_wait_seconds: float = DEFAULT_BUILD_WAIT_SECONDS
@dataclass(frozen=True)
class EarthLayerCacheResult:
payload: dict[str, Any]
state: str
key: str
features: int
bytes: int
class EarthLayerCache:
def __init__(self) -> None:
self._client = None
self._inflight: dict[
tuple[asyncio.AbstractEventLoop, str], asyncio.Future[EarthLayerCacheResult | None]
] = {}
@property
def client(self):
if self._client is None:
self._client = _RedisClient.get_client()
return self._client
@staticmethod
def key(layer: str, **params: Any) -> str:
parts = [EARTH_LAYER_CACHE_PREFIX, _safe_key_part(layer)]
for name in sorted(params):
value = params[name]
if value is None:
value = "none"
parts.append(f"{_safe_key_part(name)}:{_safe_key_part(value)}")
return ":".join(parts)
@staticmethod
def stale_key(key: str) -> str:
return f"{key}:stale"
@staticmethod
def lock_key(key: str) -> str:
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:32]
return f"{EARTH_LAYER_LOCK_PREFIX}:{digest}"
def get_json(self, key: str) -> dict[str, Any] | None:
raw = self.client.get(key)
if not raw:
return None
value = json.loads(raw)
return value if isinstance(value, dict) else None
def acquire_lock(self, key: str, ttl_seconds: int) -> str | None:
token = uuid4().hex
return (
token if self.client.set(self.lock_key(key), token, nx=True, ex=ttl_seconds) else None
)
def renew_lock(self, key: str, token: str, ttl_seconds: int) -> bool:
return bool(
self.client.eval(
"""
if redis.call('get', KEYS[1]) == ARGV[1] then
return redis.call('expire', KEYS[1], ARGV[2])
end
return 0
""",
1,
self.lock_key(key),
token,
ttl_seconds,
)
)
def release_lock(self, key: str, token: str) -> None:
self.client.eval(
"""
if redis.call('get', KEYS[1]) == ARGV[1] then
return redis.call('del', KEYS[1])
end
return 0
""",
1,
self.lock_key(key),
token,
)
def publish(
self, key: str, token: str, payload: dict[str, Any], policy: EarthLayerCachePolicy
) -> bool:
return bool(
self.client.eval(
"""
if redis.call('get', KEYS[1]) ~= ARGV[1] then return 0 end
redis.call('setex', KEYS[2], ARGV[2], ARGV[4])
redis.call('setex', KEYS[3], ARGV[3], ARGV[4])
return 1
""",
3,
self.lock_key(key),
key,
self.stale_key(key),
token,
policy.fresh_ttl_seconds,
policy.stale_ttl_seconds,
json.dumps(payload, ensure_ascii=False, default=str),
)
)
def delete_pattern(self, pattern: str = f"{EARTH_LAYER_CACHE_PREFIX}:*") -> int:
keys = list(self.client.scan_iter(match=pattern))
if not keys:
return 0
return int(self.client.delete(*keys))
def status(self) -> dict[str, Any]:
keys = list(self.client.scan_iter(match=f"{EARTH_LAYER_CACHE_PREFIX}:*"))
by_layer: dict[str, dict[str, Any]] = {}
total_memory = 0
for key in keys:
key_str = key.decode("utf-8") if isinstance(key, bytes) else str(key)
layer = _layer_from_key(key_str)
entry = by_layer.setdefault(layer, {"keys": 0, "stale_keys": 0, "memory_bytes": 0})
entry["keys"] += 1
if key_str.endswith(":stale"):
entry["stale_keys"] += 1
try:
memory = int(self.client.memory_usage(key) or 0)
except Exception:
memory = 0
entry["memory_bytes"] += memory
total_memory += memory
return {
"prefix": EARTH_LAYER_CACHE_PREFIX,
"key_count": len(keys),
"memory_bytes": total_memory,
"layers": by_layer,
}
earth_layer_cache = EarthLayerCache()
def quantize_bbox(
bbox: tuple[float, float, float, float],
*,
precision: float = DEFAULT_BBOX_PRECISION_DEGREES,
) -> tuple[float, float, float, float]:
return tuple(round(value / precision) * precision for value in bbox) # type: ignore[return-value]
def format_bbox_key(bbox: tuple[float, float, float, float]) -> str:
return ",".join(f"{value:.1f}" for value in bbox)
def apply_cache_headers(response: Response | None, result: EarthLayerCacheResult) -> None:
if response is None:
return
response.headers["X-Planet-Cache"] = result.state
response.headers["X-Planet-Cache-Features"] = str(result.features)
response.headers["X-Planet-Cache-Bytes"] = str(result.bytes)
env_name = str(getattr(settings, "ENVIRONMENT", "") or "development").lower()
if env_name in DEV_CACHE_KEY_HEADER:
response.headers["X-Planet-Cache-Key"] = result.key
async def get_or_build_layer_payload(
*,
key: str,
policy: EarthLayerCachePolicy,
builder: Callable[[], Awaitable[dict[str, Any]]],
response: Response | None = None,
) -> dict[str, Any]:
result = await resolve_layer_payload(key=key, policy=policy, builder=builder)
apply_cache_headers(response, result)
return result.payload
async def resolve_layer_payload(
*,
key: str,
policy: EarthLayerCachePolicy,
builder: Callable[[], Awaitable[dict[str, Any]]],
) -> EarthLayerCacheResult:
flight_key = (asyncio.get_running_loop(), key)
while flight_key in earth_layer_cache._inflight:
stale = _read_stale(key)
if stale is not None:
return _result(stale, state="stale", key=key)
future = earth_layer_cache._inflight[flight_key]
done, _ = await asyncio.wait({future}, timeout=policy.build_wait_seconds)
if not done:
raise HTTPException(503, "Earth layer cache rebuild is still in progress")
result = future.result()
if result is not None:
state = "hit" if result.state == "refresh" else result.state
return replace(result, state=state)
future = asyncio.get_running_loop().create_future()
earth_layer_cache._inflight[flight_key] = future
try:
result = await _resolve_layer_payload(key=key, policy=policy, builder=builder)
future.set_result(result)
return result
except asyncio.CancelledError:
# A waiting request can take over using its own still-open DB session.
future.set_result(None)
raise
except Exception as exc:
future.set_exception(exc)
future.exception()
raise
finally:
earth_layer_cache._inflight.pop(flight_key)
async def _resolve_layer_payload(
*,
key: str,
policy: EarthLayerCachePolicy,
builder: Callable[[], Awaitable[dict[str, Any]]],
) -> EarthLayerCacheResult:
started = time.perf_counter()
try:
async with asyncio.timeout(policy.build_wait_seconds):
while True:
cached = earth_layer_cache.get_json(key)
if cached is not None:
return _result(cached, state="hit", key=key)
token = earth_layer_cache.acquire_lock(key, policy.lock_ttl_seconds)
if token is not None:
break
stale = _read_stale(key)
if stale is not None:
return _result(stale, state="stale", key=key)
await asyncio.sleep(policy.lock_wait_seconds)
except TimeoutError:
raise HTTPException(503, "Earth layer cache rebuild is still in progress") from None
except Exception as exc:
logger.warning_event(
"Earth layer cache bypassed",
event="earth_layer_cache.bypass",
context={"key": key, "error": str(exc)},
)
payload = await _build_budgeted_payload(builder, policy)
return _result(payload, state="bypass", key=key)
renewal = asyncio.create_task(_renew_cache_lock(key, token, policy))
try:
try:
payload = await _build_budgeted_payload(builder, policy)
except Exception as exc:
stale = _read_stale(key)
if stale is None:
raise
logger.warning_event(
"Earth layer cache builder failed; returning stale payload",
event="earth_layer_cache.stale_after_builder_error",
context={"key": key, "error": str(exc)},
)
return _result(stale, state="stale", key=key)
try:
published = earth_layer_cache.publish(key, token, payload, policy)
except Exception as exc:
logger.warning_event(
"Earth layer cache publish failed",
event="earth_layer_cache.bypass",
context={"key": key, "error": str(exc)},
)
published = False
state = "refresh" if published else "bypass"
_log_cache_event(state, key, payload, started)
return _result(payload, state=state, key=key)
finally:
renewal.cancel()
with suppress(asyncio.CancelledError):
await renewal
try:
earth_layer_cache.release_lock(key, token)
except Exception as exc:
logger.warning_event(
"Earth layer cache lock release failed",
event="earth_layer_cache.lock_release_failed",
context={"key": key, "error": str(exc)},
)
async def _renew_cache_lock(key: str, token: str, policy: EarthLayerCachePolicy) -> None:
try:
while True:
await asyncio.sleep(policy.lock_ttl_seconds / 3)
if not earth_layer_cache.renew_lock(key, token, policy.lock_ttl_seconds):
raise RuntimeError("Earth layer cache lock ownership lost")
except Exception as exc:
logger.warning_event(
"Earth layer cache lock renewal stopped",
event="earth_layer_cache.lock_renewal_failed",
context={"key": key, "error": str(exc)},
)
def apply_payload_budget(payload: dict[str, Any], policy: EarthLayerCachePolicy) -> dict[str, Any]:
budgeted = _truncate_features(payload, policy.max_features, "feature_budget")
size = _payload_size(budgeted)
if size <= policy.max_bytes:
return budgeted
features = budgeted.get("features")
if not isinstance(features, list):
return _with_budget_diagnostics(
budgeted,
truncated=True,
reason="byte_budget",
bytes_before=size,
bytes_after=size,
)
low = 0
high = len(features)
best = []
best_size = _payload_size({**budgeted, "features": best})
while low <= high:
mid = (low + high) // 2
candidate_features = features[:mid]
candidate = _with_budget_diagnostics(
{**budgeted, "features": candidate_features},
truncated=mid < len(features),
reason="byte_budget",
bytes_before=size,
bytes_after=0,
)
candidate_size = _payload_size(candidate)
if candidate_size <= policy.max_bytes:
best = candidate_features
best_size = candidate_size
low = mid + 1
else:
high = mid - 1
return _with_budget_diagnostics(
{**budgeted, "features": best},
truncated=True,
reason="byte_budget",
bytes_before=size,
bytes_after=best_size,
)
def invalidate_earth_layer_cache_for_source(source: str) -> int:
from app.services.earth_layer_adapters import get_earth_cache_patterns_for_source
source_key = str(source or "").strip()
patterns = get_earth_cache_patterns_for_source(source_key)
deleted = 0
for layer_pattern in patterns:
deleted += earth_layer_cache.delete_pattern(f"{EARTH_LAYER_CACHE_PREFIX}:{layer_pattern}")
return deleted
async def _build_budgeted_payload(
builder: Callable[[], Awaitable[dict[str, Any]]],
policy: EarthLayerCachePolicy,
) -> dict[str, Any]:
payload = await builder()
return apply_payload_budget(payload, policy)
def _read_stale(key: str) -> dict[str, Any] | None:
try:
return earth_layer_cache.get_json(earth_layer_cache.stale_key(key))
except Exception:
return None
def _truncate_features(payload: dict[str, Any], max_features: int, reason: str) -> dict[str, Any]:
features = payload.get("features")
if not isinstance(features, list) or len(features) <= max_features:
return payload
return _with_budget_diagnostics(
{**payload, "features": features[:max_features]},
truncated=True,
reason=reason,
original_feature_count=len(features),
)
def _with_budget_diagnostics(
payload: dict[str, Any],
*,
truncated: bool,
reason: str,
original_feature_count: int | None = None,
bytes_before: int | None = None,
bytes_after: int | None = None,
) -> dict[str, Any]:
diagnostics = dict(payload.get("diagnostics") or {})
diagnostics.update(
{
"truncated": bool(truncated or diagnostics.get("truncated")),
"limit_reason": reason,
}
)
if original_feature_count is not None:
diagnostics["original_feature_count"] = original_feature_count
if bytes_before is not None:
diagnostics["bytes_before_budget"] = bytes_before
if bytes_after is not None:
diagnostics["bytes_after_budget"] = bytes_after
return {**payload, "diagnostics": diagnostics}
def _result(payload: dict[str, Any], *, state: str, key: str) -> EarthLayerCacheResult:
return EarthLayerCacheResult(
payload=payload,
state=state,
key=key,
features=_feature_count(payload),
bytes=_payload_size(payload),
)
def _feature_count(payload: dict[str, Any]) -> int:
features = payload.get("features")
if isinstance(features, list):
return len(features)
count = payload.get("count")
return int(count) if isinstance(count, int) else 0
def _payload_size(payload: dict[str, Any]) -> int:
return len(json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8"))
def _safe_key_part(value: Any) -> str:
raw = str(value).strip().lower()
return "".join(char if char.isalnum() or char in {"-", "_", ".", ","} else "_" for char in raw)[
:160
]
def _layer_from_key(key: str) -> str:
prefix = f"{EARTH_LAYER_CACHE_PREFIX}:"
if not key.startswith(prefix):
return "unknown"
remainder = key[len(prefix) :]
return remainder.split(":", 1)[0]
def _log_cache_event(state: str, key: str, payload: dict[str, Any], started: float) -> None:
logger.info_event(
"Earth layer cache resolved",
event="earth_layer_cache.resolved",
context={
"state": state,
"key": key,
"features": _feature_count(payload),
"bytes": _payload_size(payload),
"duration_ms": round((time.perf_counter() - started) * 1000, 2),
},
)