forked from linkong/planet
499 lines
16 KiB
Python
499 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from collections.abc import Awaitable, Callable
|
|
from contextlib import suppress
|
|
from dataclasses import dataclass, replace
|
|
import hashlib
|
|
import json
|
|
import time
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from fastapi import HTTPException, Response
|
|
|
|
from app.core.cache import _RedisClient
|
|
from app.core.config import settings
|
|
from app.core.logging import get_logger
|
|
|
|
logger = get_logger(__name__, service="earth_layer_cache")
|
|
|
|
EARTH_LAYER_CACHE_PREFIX = "earth:layer:v1"
|
|
EARTH_LAYER_LOCK_PREFIX = "earth:layer:lock:v1"
|
|
DEFAULT_LOCK_TTL_SECONDS = 10
|
|
DEFAULT_LOCK_WAIT_SECONDS = 0.2
|
|
DEFAULT_BUILD_WAIT_SECONDS = 30
|
|
DEFAULT_MAX_FEATURES = 5000
|
|
DEFAULT_MAX_BYTES = 5 * 1024 * 1024
|
|
DEFAULT_BBOX_PRECISION_DEGREES = 0.1
|
|
DEV_CACHE_KEY_HEADER = {"development", "dev", "test", "testing", "local"}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EarthLayerCachePolicy:
|
|
fresh_ttl_seconds: int
|
|
stale_ttl_seconds: int
|
|
max_features: int = DEFAULT_MAX_FEATURES
|
|
max_bytes: int = DEFAULT_MAX_BYTES
|
|
lock_ttl_seconds: int = DEFAULT_LOCK_TTL_SECONDS
|
|
lock_wait_seconds: float = DEFAULT_LOCK_WAIT_SECONDS
|
|
build_wait_seconds: float = DEFAULT_BUILD_WAIT_SECONDS
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EarthLayerCacheResult:
|
|
payload: dict[str, Any]
|
|
state: str
|
|
key: str
|
|
features: int
|
|
bytes: int
|
|
|
|
|
|
class EarthLayerCache:
|
|
def __init__(self) -> None:
|
|
self._client = None
|
|
self._inflight: dict[
|
|
tuple[asyncio.AbstractEventLoop, str], asyncio.Future[EarthLayerCacheResult | None]
|
|
] = {}
|
|
|
|
@property
|
|
def client(self):
|
|
if self._client is None:
|
|
self._client = _RedisClient.get_client()
|
|
return self._client
|
|
|
|
@staticmethod
|
|
def key(layer: str, **params: Any) -> str:
|
|
parts = [EARTH_LAYER_CACHE_PREFIX, _safe_key_part(layer)]
|
|
for name in sorted(params):
|
|
value = params[name]
|
|
if value is None:
|
|
value = "none"
|
|
parts.append(f"{_safe_key_part(name)}:{_safe_key_part(value)}")
|
|
return ":".join(parts)
|
|
|
|
@staticmethod
|
|
def stale_key(key: str) -> str:
|
|
return f"{key}:stale"
|
|
|
|
@staticmethod
|
|
def lock_key(key: str) -> str:
|
|
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:32]
|
|
return f"{EARTH_LAYER_LOCK_PREFIX}:{digest}"
|
|
|
|
def get_json(self, key: str) -> dict[str, Any] | None:
|
|
raw = self.client.get(key)
|
|
if not raw:
|
|
return None
|
|
value = json.loads(raw)
|
|
return value if isinstance(value, dict) else None
|
|
|
|
def acquire_lock(self, key: str, ttl_seconds: int) -> str | None:
|
|
token = uuid4().hex
|
|
return (
|
|
token if self.client.set(self.lock_key(key), token, nx=True, ex=ttl_seconds) else None
|
|
)
|
|
|
|
def renew_lock(self, key: str, token: str, ttl_seconds: int) -> bool:
|
|
return bool(
|
|
self.client.eval(
|
|
"""
|
|
if redis.call('get', KEYS[1]) == ARGV[1] then
|
|
return redis.call('expire', KEYS[1], ARGV[2])
|
|
end
|
|
return 0
|
|
""",
|
|
1,
|
|
self.lock_key(key),
|
|
token,
|
|
ttl_seconds,
|
|
)
|
|
)
|
|
|
|
def release_lock(self, key: str, token: str) -> None:
|
|
self.client.eval(
|
|
"""
|
|
if redis.call('get', KEYS[1]) == ARGV[1] then
|
|
return redis.call('del', KEYS[1])
|
|
end
|
|
return 0
|
|
""",
|
|
1,
|
|
self.lock_key(key),
|
|
token,
|
|
)
|
|
|
|
def publish(
|
|
self, key: str, token: str, payload: dict[str, Any], policy: EarthLayerCachePolicy
|
|
) -> bool:
|
|
return bool(
|
|
self.client.eval(
|
|
"""
|
|
if redis.call('get', KEYS[1]) ~= ARGV[1] then return 0 end
|
|
redis.call('setex', KEYS[2], ARGV[2], ARGV[4])
|
|
redis.call('setex', KEYS[3], ARGV[3], ARGV[4])
|
|
return 1
|
|
""",
|
|
3,
|
|
self.lock_key(key),
|
|
key,
|
|
self.stale_key(key),
|
|
token,
|
|
policy.fresh_ttl_seconds,
|
|
policy.stale_ttl_seconds,
|
|
json.dumps(payload, ensure_ascii=False, default=str),
|
|
)
|
|
)
|
|
|
|
def delete_pattern(self, pattern: str = f"{EARTH_LAYER_CACHE_PREFIX}:*") -> int:
|
|
keys = list(self.client.scan_iter(match=pattern))
|
|
if not keys:
|
|
return 0
|
|
return int(self.client.delete(*keys))
|
|
|
|
def status(self) -> dict[str, Any]:
|
|
keys = list(self.client.scan_iter(match=f"{EARTH_LAYER_CACHE_PREFIX}:*"))
|
|
by_layer: dict[str, dict[str, Any]] = {}
|
|
total_memory = 0
|
|
for key in keys:
|
|
key_str = key.decode("utf-8") if isinstance(key, bytes) else str(key)
|
|
layer = _layer_from_key(key_str)
|
|
entry = by_layer.setdefault(layer, {"keys": 0, "stale_keys": 0, "memory_bytes": 0})
|
|
entry["keys"] += 1
|
|
if key_str.endswith(":stale"):
|
|
entry["stale_keys"] += 1
|
|
try:
|
|
memory = int(self.client.memory_usage(key) or 0)
|
|
except Exception:
|
|
memory = 0
|
|
entry["memory_bytes"] += memory
|
|
total_memory += memory
|
|
return {
|
|
"prefix": EARTH_LAYER_CACHE_PREFIX,
|
|
"key_count": len(keys),
|
|
"memory_bytes": total_memory,
|
|
"layers": by_layer,
|
|
}
|
|
|
|
|
|
earth_layer_cache = EarthLayerCache()
|
|
|
|
|
|
def quantize_bbox(
|
|
bbox: tuple[float, float, float, float],
|
|
*,
|
|
precision: float = DEFAULT_BBOX_PRECISION_DEGREES,
|
|
) -> tuple[float, float, float, float]:
|
|
return tuple(round(value / precision) * precision for value in bbox) # type: ignore[return-value]
|
|
|
|
|
|
def format_bbox_key(bbox: tuple[float, float, float, float]) -> str:
|
|
return ",".join(f"{value:.1f}" for value in bbox)
|
|
|
|
|
|
def apply_cache_headers(response: Response | None, result: EarthLayerCacheResult) -> None:
|
|
if response is None:
|
|
return
|
|
response.headers["X-Planet-Cache"] = result.state
|
|
response.headers["X-Planet-Cache-Features"] = str(result.features)
|
|
response.headers["X-Planet-Cache-Bytes"] = str(result.bytes)
|
|
env_name = str(getattr(settings, "ENVIRONMENT", "") or "development").lower()
|
|
if env_name in DEV_CACHE_KEY_HEADER:
|
|
response.headers["X-Planet-Cache-Key"] = result.key
|
|
|
|
|
|
async def get_or_build_layer_payload(
|
|
*,
|
|
key: str,
|
|
policy: EarthLayerCachePolicy,
|
|
builder: Callable[[], Awaitable[dict[str, Any]]],
|
|
response: Response | None = None,
|
|
) -> dict[str, Any]:
|
|
result = await resolve_layer_payload(key=key, policy=policy, builder=builder)
|
|
apply_cache_headers(response, result)
|
|
return result.payload
|
|
|
|
|
|
async def resolve_layer_payload(
|
|
*,
|
|
key: str,
|
|
policy: EarthLayerCachePolicy,
|
|
builder: Callable[[], Awaitable[dict[str, Any]]],
|
|
) -> EarthLayerCacheResult:
|
|
flight_key = (asyncio.get_running_loop(), key)
|
|
while flight_key in earth_layer_cache._inflight:
|
|
stale = _read_stale(key)
|
|
if stale is not None:
|
|
return _result(stale, state="stale", key=key)
|
|
future = earth_layer_cache._inflight[flight_key]
|
|
done, _ = await asyncio.wait({future}, timeout=policy.build_wait_seconds)
|
|
if not done:
|
|
raise HTTPException(503, "Earth layer cache rebuild is still in progress")
|
|
result = future.result()
|
|
if result is not None:
|
|
state = "hit" if result.state == "refresh" else result.state
|
|
return replace(result, state=state)
|
|
|
|
future = asyncio.get_running_loop().create_future()
|
|
earth_layer_cache._inflight[flight_key] = future
|
|
try:
|
|
result = await _resolve_layer_payload(key=key, policy=policy, builder=builder)
|
|
future.set_result(result)
|
|
return result
|
|
except asyncio.CancelledError:
|
|
# A waiting request can take over using its own still-open DB session.
|
|
future.set_result(None)
|
|
raise
|
|
except Exception as exc:
|
|
future.set_exception(exc)
|
|
future.exception()
|
|
raise
|
|
finally:
|
|
earth_layer_cache._inflight.pop(flight_key)
|
|
|
|
|
|
async def _resolve_layer_payload(
|
|
*,
|
|
key: str,
|
|
policy: EarthLayerCachePolicy,
|
|
builder: Callable[[], Awaitable[dict[str, Any]]],
|
|
) -> EarthLayerCacheResult:
|
|
started = time.perf_counter()
|
|
try:
|
|
async with asyncio.timeout(policy.build_wait_seconds):
|
|
while True:
|
|
cached = earth_layer_cache.get_json(key)
|
|
if cached is not None:
|
|
return _result(cached, state="hit", key=key)
|
|
token = earth_layer_cache.acquire_lock(key, policy.lock_ttl_seconds)
|
|
if token is not None:
|
|
break
|
|
stale = _read_stale(key)
|
|
if stale is not None:
|
|
return _result(stale, state="stale", key=key)
|
|
await asyncio.sleep(policy.lock_wait_seconds)
|
|
except TimeoutError:
|
|
raise HTTPException(503, "Earth layer cache rebuild is still in progress") from None
|
|
except Exception as exc:
|
|
logger.warning_event(
|
|
"Earth layer cache bypassed",
|
|
event="earth_layer_cache.bypass",
|
|
context={"key": key, "error": str(exc)},
|
|
)
|
|
payload = await _build_budgeted_payload(builder, policy)
|
|
return _result(payload, state="bypass", key=key)
|
|
|
|
renewal = asyncio.create_task(_renew_cache_lock(key, token, policy))
|
|
try:
|
|
try:
|
|
payload = await _build_budgeted_payload(builder, policy)
|
|
except Exception as exc:
|
|
stale = _read_stale(key)
|
|
if stale is None:
|
|
raise
|
|
logger.warning_event(
|
|
"Earth layer cache builder failed; returning stale payload",
|
|
event="earth_layer_cache.stale_after_builder_error",
|
|
context={"key": key, "error": str(exc)},
|
|
)
|
|
return _result(stale, state="stale", key=key)
|
|
try:
|
|
published = earth_layer_cache.publish(key, token, payload, policy)
|
|
except Exception as exc:
|
|
logger.warning_event(
|
|
"Earth layer cache publish failed",
|
|
event="earth_layer_cache.bypass",
|
|
context={"key": key, "error": str(exc)},
|
|
)
|
|
published = False
|
|
state = "refresh" if published else "bypass"
|
|
_log_cache_event(state, key, payload, started)
|
|
return _result(payload, state=state, key=key)
|
|
finally:
|
|
renewal.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await renewal
|
|
try:
|
|
earth_layer_cache.release_lock(key, token)
|
|
except Exception as exc:
|
|
logger.warning_event(
|
|
"Earth layer cache lock release failed",
|
|
event="earth_layer_cache.lock_release_failed",
|
|
context={"key": key, "error": str(exc)},
|
|
)
|
|
|
|
|
|
async def _renew_cache_lock(key: str, token: str, policy: EarthLayerCachePolicy) -> None:
|
|
try:
|
|
while True:
|
|
await asyncio.sleep(policy.lock_ttl_seconds / 3)
|
|
if not earth_layer_cache.renew_lock(key, token, policy.lock_ttl_seconds):
|
|
raise RuntimeError("Earth layer cache lock ownership lost")
|
|
except Exception as exc:
|
|
logger.warning_event(
|
|
"Earth layer cache lock renewal stopped",
|
|
event="earth_layer_cache.lock_renewal_failed",
|
|
context={"key": key, "error": str(exc)},
|
|
)
|
|
|
|
|
|
def apply_payload_budget(payload: dict[str, Any], policy: EarthLayerCachePolicy) -> dict[str, Any]:
|
|
budgeted = _truncate_features(payload, policy.max_features, "feature_budget")
|
|
size = _payload_size(budgeted)
|
|
if size <= policy.max_bytes:
|
|
return budgeted
|
|
|
|
features = budgeted.get("features")
|
|
if not isinstance(features, list):
|
|
return _with_budget_diagnostics(
|
|
budgeted,
|
|
truncated=True,
|
|
reason="byte_budget",
|
|
bytes_before=size,
|
|
bytes_after=size,
|
|
)
|
|
|
|
low = 0
|
|
high = len(features)
|
|
best = []
|
|
best_size = _payload_size({**budgeted, "features": best})
|
|
while low <= high:
|
|
mid = (low + high) // 2
|
|
candidate_features = features[:mid]
|
|
candidate = _with_budget_diagnostics(
|
|
{**budgeted, "features": candidate_features},
|
|
truncated=mid < len(features),
|
|
reason="byte_budget",
|
|
bytes_before=size,
|
|
bytes_after=0,
|
|
)
|
|
candidate_size = _payload_size(candidate)
|
|
if candidate_size <= policy.max_bytes:
|
|
best = candidate_features
|
|
best_size = candidate_size
|
|
low = mid + 1
|
|
else:
|
|
high = mid - 1
|
|
|
|
return _with_budget_diagnostics(
|
|
{**budgeted, "features": best},
|
|
truncated=True,
|
|
reason="byte_budget",
|
|
bytes_before=size,
|
|
bytes_after=best_size,
|
|
)
|
|
|
|
|
|
def invalidate_earth_layer_cache_for_source(source: str) -> int:
|
|
from app.services.earth_layer_adapters import get_earth_cache_patterns_for_source
|
|
|
|
source_key = str(source or "").strip()
|
|
patterns = get_earth_cache_patterns_for_source(source_key)
|
|
deleted = 0
|
|
for layer_pattern in patterns:
|
|
deleted += earth_layer_cache.delete_pattern(f"{EARTH_LAYER_CACHE_PREFIX}:{layer_pattern}")
|
|
return deleted
|
|
|
|
|
|
async def _build_budgeted_payload(
|
|
builder: Callable[[], Awaitable[dict[str, Any]]],
|
|
policy: EarthLayerCachePolicy,
|
|
) -> dict[str, Any]:
|
|
payload = await builder()
|
|
return apply_payload_budget(payload, policy)
|
|
|
|
|
|
def _read_stale(key: str) -> dict[str, Any] | None:
|
|
try:
|
|
return earth_layer_cache.get_json(earth_layer_cache.stale_key(key))
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _truncate_features(payload: dict[str, Any], max_features: int, reason: str) -> dict[str, Any]:
|
|
features = payload.get("features")
|
|
if not isinstance(features, list) or len(features) <= max_features:
|
|
return payload
|
|
return _with_budget_diagnostics(
|
|
{**payload, "features": features[:max_features]},
|
|
truncated=True,
|
|
reason=reason,
|
|
original_feature_count=len(features),
|
|
)
|
|
|
|
|
|
def _with_budget_diagnostics(
|
|
payload: dict[str, Any],
|
|
*,
|
|
truncated: bool,
|
|
reason: str,
|
|
original_feature_count: int | None = None,
|
|
bytes_before: int | None = None,
|
|
bytes_after: int | None = None,
|
|
) -> dict[str, Any]:
|
|
diagnostics = dict(payload.get("diagnostics") or {})
|
|
diagnostics.update(
|
|
{
|
|
"truncated": bool(truncated or diagnostics.get("truncated")),
|
|
"limit_reason": reason,
|
|
}
|
|
)
|
|
if original_feature_count is not None:
|
|
diagnostics["original_feature_count"] = original_feature_count
|
|
if bytes_before is not None:
|
|
diagnostics["bytes_before_budget"] = bytes_before
|
|
if bytes_after is not None:
|
|
diagnostics["bytes_after_budget"] = bytes_after
|
|
return {**payload, "diagnostics": diagnostics}
|
|
|
|
|
|
def _result(payload: dict[str, Any], *, state: str, key: str) -> EarthLayerCacheResult:
|
|
return EarthLayerCacheResult(
|
|
payload=payload,
|
|
state=state,
|
|
key=key,
|
|
features=_feature_count(payload),
|
|
bytes=_payload_size(payload),
|
|
)
|
|
|
|
|
|
def _feature_count(payload: dict[str, Any]) -> int:
|
|
features = payload.get("features")
|
|
if isinstance(features, list):
|
|
return len(features)
|
|
count = payload.get("count")
|
|
return int(count) if isinstance(count, int) else 0
|
|
|
|
|
|
def _payload_size(payload: dict[str, Any]) -> int:
|
|
return len(json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8"))
|
|
|
|
|
|
def _safe_key_part(value: Any) -> str:
|
|
raw = str(value).strip().lower()
|
|
return "".join(char if char.isalnum() or char in {"-", "_", ".", ","} else "_" for char in raw)[
|
|
:160
|
|
]
|
|
|
|
|
|
def _layer_from_key(key: str) -> str:
|
|
prefix = f"{EARTH_LAYER_CACHE_PREFIX}:"
|
|
if not key.startswith(prefix):
|
|
return "unknown"
|
|
remainder = key[len(prefix) :]
|
|
return remainder.split(":", 1)[0]
|
|
|
|
|
|
def _log_cache_event(state: str, key: str, payload: dict[str, Any], started: float) -> None:
|
|
logger.info_event(
|
|
"Earth layer cache resolved",
|
|
event="earth_layer_cache.resolved",
|
|
context={
|
|
"state": state,
|
|
"key": key,
|
|
"features": _feature_count(payload),
|
|
"bytes": _payload_size(payload),
|
|
"duration_ms": round((time.perf_counter() - started) * 1000, 2),
|
|
},
|
|
)
|