release: bump version to 0.61.0
Some checks failed
ci / backend (push) Has been cancelled
ci / frontend (push) Has been cancelled
release / images (push) Has been cancelled
ci / delivery (push) Has been cancelled

This commit is contained in:
rayd1o
2026-05-18 02:37:19 +08:00
parent 81970a1d05
commit 5c65ee24d6
16 changed files with 1097 additions and 11 deletions

View File

@@ -1,6 +1,6 @@
from typing import Any, Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import APIRouter, Depends, HTTPException, Query, Response
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.v1.visualization import (
@@ -110,6 +110,7 @@ async def get_vessel_layer_snapshot(
vessel_type: Optional[str] = Query(None, alias="type"),
since_minutes: int = Query(60, ge=1, le=1440),
db: AsyncSession = Depends(get_db),
response: Response = None,
):
parsed_bbox = _parse_layer_bbox(bbox)
return await build_vessel_snapshot_response(
@@ -119,6 +120,7 @@ async def get_vessel_layer_snapshot(
limit=limit,
type_filter=vessel_type,
since_minutes=since_minutes,
response=response,
)

View File

@@ -35,6 +35,7 @@ from app.services.system_logs import (
normalize_log_level,
read_log_snapshot,
)
from app.services.earth_layer_cache import earth_layer_cache
router = APIRouter()
@@ -112,6 +113,17 @@ class EarthClientLogEventResponse(BaseModel):
level: str
class EarthLayerCacheStatusResponse(BaseModel):
prefix: str
key_count: int
memory_bytes: int
layers: dict[str, dict[str, int]]
class EarthLayerCacheClearResponse(BaseModel):
deleted: int
def ensure_super_admin(current_user: User) -> None:
if not require_super_admin(current_user.role):
raise HTTPException(
@@ -132,6 +144,34 @@ def validate_log_date(raw_value: str | None, field_name: str) -> str | None:
) from exc
@router.get("/cache/earth-layers", response_model=EarthLayerCacheStatusResponse)
async def get_earth_layer_cache_status(
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
try:
return earth_layer_cache.status()
except Exception as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=f"Unable to read Earth layer cache status: {exc}",
) from exc
@router.delete("/cache/earth-layers", response_model=EarthLayerCacheClearResponse)
async def clear_earth_layer_cache(
current_user: User = Depends(get_current_user),
):
ensure_super_admin(current_user)
try:
return {"deleted": earth_layer_cache.delete_pattern()}
except Exception as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=f"Unable to clear Earth layer cache: {exc}",
) from exc
@router.post("/restart-tasks", response_model=RestartTaskResponse)
async def create_restart_task(
payload: RestartTaskCreate,

View File

@@ -2,7 +2,7 @@
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import APIRouter, Depends, HTTPException, Query, Response
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.v1.visualization import _parse_bbox, build_vessel_snapshot_response
@@ -23,6 +23,7 @@ async def get_vessel_snapshot(
limit: int = Query(1000, ge=1, le=MAX_SNAPSHOT_LIMIT),
since_minutes: int = Query(60, ge=1, le=1440),
db: AsyncSession = Depends(get_db),
response: Response = None,
):
if not bbox:
raise HTTPException(status_code=400, detail="bbox is required")
@@ -36,4 +37,5 @@ async def get_vessel_snapshot(
type_filter=type,
limit=limit,
since_minutes=since_minutes,
response=response,
)

View File

@@ -56,6 +56,13 @@ from app.services.vessel_ais_aggregation import (
get_vessel_raw_observations,
MAX_SNAPSHOT_LIMIT,
)
from app.services.earth_layer_cache import (
EarthLayerCachePolicy,
earth_layer_cache,
format_bbox_key,
get_or_build_layer_payload,
quantize_bbox,
)
from app.core.logging import get_logger
router = APIRouter()
@@ -69,6 +76,68 @@ TERRAIN_TILE_BATCH_CONCURRENCY = 16
_terrain_tile_cache: OrderedDict[tuple[int, int, int], tuple[bytes, str, dict[str, str]]] = OrderedDict()
VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE)
VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED = True
SECONDS_PER_MINUTE = 60
BYTES_PER_MIB = 1024 * 1024
CABLE_CACHE_FRESH_SECONDS = 6 * 60 * SECONDS_PER_MINUTE
CABLE_CACHE_STALE_SECONDS = 24 * 60 * SECONDS_PER_MINUTE
SATELLITE_CACHE_FRESH_SECONDS = 15 * SECONDS_PER_MINUTE
SATELLITE_CACHE_STALE_SECONDS = 2 * 60 * SECONDS_PER_MINUTE
COMPUTE_CENTER_CACHE_FRESH_SECONDS = 10 * SECONDS_PER_MINUTE
COMPUTE_CENTER_CACHE_STALE_SECONDS = 60 * SECONDS_PER_MINUTE
BGP_CACHE_FRESH_SECONDS = 60
BGP_EVENT_CACHE_FRESH_SECONDS = 30
BGP_CACHE_STALE_SECONDS = 10 * SECONDS_PER_MINUTE
VESSEL_SNAPSHOT_CACHE_FRESH_SECONDS = 5
VESSEL_SNAPSHOT_CACHE_STALE_SECONDS = 30
CABLE_CACHE_POLICY = EarthLayerCachePolicy(
CABLE_CACHE_FRESH_SECONDS,
CABLE_CACHE_STALE_SECONDS,
max_features=6000,
max_bytes=10 * BYTES_PER_MIB,
)
LANDING_POINT_CACHE_POLICY = EarthLayerCachePolicy(
CABLE_CACHE_FRESH_SECONDS,
CABLE_CACHE_STALE_SECONDS,
max_features=6000,
max_bytes=8 * BYTES_PER_MIB,
)
SATELLITE_CACHE_POLICY = EarthLayerCachePolicy(
SATELLITE_CACHE_FRESH_SECONDS,
SATELLITE_CACHE_STALE_SECONDS,
max_features=8000,
max_bytes=10 * BYTES_PER_MIB,
)
COMPUTE_CENTER_CACHE_POLICY = EarthLayerCachePolicy(
COMPUTE_CENTER_CACHE_FRESH_SECONDS,
COMPUTE_CENTER_CACHE_STALE_SECONDS,
max_features=1000,
max_bytes=4 * BYTES_PER_MIB,
)
BGP_CACHE_POLICY = EarthLayerCachePolicy(
BGP_CACHE_FRESH_SECONDS,
BGP_CACHE_STALE_SECONDS,
max_features=1000,
max_bytes=3 * BYTES_PER_MIB,
)
BGP_EVENT_CACHE_POLICY = EarthLayerCachePolicy(
BGP_EVENT_CACHE_FRESH_SECONDS,
BGP_CACHE_STALE_SECONDS,
max_features=1000,
max_bytes=3 * BYTES_PER_MIB,
)
SUMMARY_CACHE_POLICY = EarthLayerCachePolicy(
BGP_EVENT_CACHE_FRESH_SECONDS,
BGP_CACHE_STALE_SECONDS,
max_features=0,
max_bytes=512 * 1024,
)
VESSEL_SNAPSHOT_CACHE_POLICY = EarthLayerCachePolicy(
VESSEL_SNAPSHOT_CACHE_FRESH_SECONDS,
VESSEL_SNAPSHOT_CACHE_STALE_SECONDS,
max_features=1500,
max_bytes=3 * BYTES_PER_MIB,
)
class TerrariumTileRequest(BaseModel):
@@ -1010,7 +1079,40 @@ async def build_vessel_snapshot_response(
type_filter: str | None,
limit: int | None,
since_minutes: int = 60,
response: Response | None = None,
use_cache: bool = True,
) -> dict[str, Any]:
if use_cache and bbox is not None:
safe_limit_for_key = _safe_vessel_limit(limit)
safe_since_for_key = min(max(int(since_minutes or 60), 1), 1440)
cache_key = earth_layer_cache.key(
"vessels-snapshot",
bbox=format_bbox_key(quantize_bbox(bbox)),
zoom=zoom or "none",
type=type_filter or "all",
limit=safe_limit_for_key,
since=safe_since_for_key,
)
async def build_uncached() -> dict[str, Any]:
return await build_vessel_snapshot_response(
db,
bbox=bbox,
zoom=zoom,
type_filter=type_filter,
limit=limit,
since_minutes=since_minutes,
response=None,
use_cache=False,
)
return await get_or_build_layer_payload(
key=cache_key,
policy=VESSEL_SNAPSHOT_CACHE_POLICY,
builder=build_uncached,
response=response,
)
requested_types = _requested_vessel_types(type_filter)
safe_limit = _safe_vessel_limit(limit)
safe_since_minutes = min(max(int(since_minutes or 60), 1), 1440)
@@ -1444,8 +1546,20 @@ def convert_bgp_incidents_to_geojson(
@router.get("/geo/cables")
async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
async def get_cables_geojson(db: AsyncSession = Depends(get_db), response: Response = None):
"""获取海底电缆 GeoJSON 数据 (LineString)"""
async def build_payload() -> dict[str, Any]:
return await _build_cables_geojson(db)
return await get_or_build_layer_payload(
key=earth_layer_cache.key("cables"),
policy=CABLE_CACHE_POLICY,
builder=build_payload,
response=response,
)
async def _build_cables_geojson(db: AsyncSession) -> dict[str, Any]:
try:
records = await _load_current_collected_data(db, "arcgis_cables")
@@ -1478,7 +1592,19 @@ async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
@router.get("/geo/landing-points")
async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
async def get_landing_points_geojson(db: AsyncSession = Depends(get_db), response: Response = None):
async def build_payload() -> dict[str, Any]:
return await _build_landing_points_geojson(db)
return await get_or_build_layer_payload(
key=earth_layer_cache.key("landing-points"),
policy=LANDING_POINT_CACHE_POLICY,
builder=build_payload,
response=response,
)
async def _build_landing_points_geojson(db: AsyncSession) -> dict[str, Any]:
try:
records_by_source = await _load_current_collected_data_by_sources(
db,
@@ -1731,8 +1857,25 @@ async def get_satellites_geojson(
description="Maximum number of satellites to return. Omit for no limit.",
),
db: AsyncSession = Depends(get_db),
response: Response = None,
):
"""获取卫星 TLE GeoJSON 数据"""
async def build_payload() -> dict[str, Any]:
return await _build_satellites_geojson(limit=limit, db=db)
return await get_or_build_layer_payload(
key=earth_layer_cache.key("satellites", limit=limit or "all"),
policy=SATELLITE_CACHE_POLICY,
builder=build_payload,
response=response,
)
async def _build_satellites_geojson(
*,
limit: int | None,
db: AsyncSession,
) -> dict[str, Any]:
records = await _load_current_or_latest_task_data(
db,
"celestrak_tle",
@@ -1800,8 +1943,25 @@ async def get_gpu_clusters_geojson(
async def get_compute_centers_geojson(
limit: int = Query(200, ge=1, le=1000),
db: AsyncSession = Depends(get_db),
response: Response = None,
):
"""获取统一算力中心 GeoJSON 数据"""
async def build_payload() -> dict[str, Any]:
return await _build_compute_centers_geojson(limit=limit, db=db)
return await get_or_build_layer_payload(
key=earth_layer_cache.key("compute-centers", limit=limit),
policy=COMPUTE_CENTER_CACHE_POLICY,
builder=build_payload,
response=response,
)
async def _build_compute_centers_geojson(
*,
limit: int,
db: AsyncSession,
) -> dict[str, Any]:
records_by_source = await _load_current_collected_data_by_sources(
db,
["top500", "epoch_ai_gpu"],
@@ -2397,7 +2557,31 @@ async def get_bgp_anomalies_geojson(
status: Optional[str] = Query("active"),
limit: int = Query(200, ge=1, le=1000),
db: AsyncSession = Depends(get_db),
response: Response = None,
):
async def build_payload() -> dict[str, Any]:
return await _build_bgp_anomalies_geojson(
severity=severity,
status=status,
limit=limit,
db=db,
)
return await get_or_build_layer_payload(
key=earth_layer_cache.key("bgp-anomalies", severity=severity or "all", status=status or "all", limit=limit),
policy=BGP_EVENT_CACHE_POLICY,
builder=build_payload,
response=response,
)
async def _build_bgp_anomalies_geojson(
*,
severity: str | None,
status: str | None,
limit: int,
db: AsyncSession,
) -> dict[str, Any]:
stmt = select(BGPAnomaly).order_by(BGPAnomaly.created_at.desc()).limit(limit)
if severity:
stmt = stmt.where(BGPAnomaly.severity == severity)
@@ -2417,7 +2601,31 @@ async def get_bgp_incidents_geojson(
status: Optional[str] = Query("active"),
limit: int = Query(100, ge=1, le=500),
db: AsyncSession = Depends(get_db),
response: Response = None,
):
async def build_payload() -> dict[str, Any]:
return await _build_bgp_incidents_geojson(
severity=severity,
status=status,
limit=limit,
db=db,
)
return await get_or_build_layer_payload(
key=earth_layer_cache.key("bgp-incidents", severity=severity or "all", status=status or "all", limit=limit),
policy=BGP_EVENT_CACHE_POLICY,
builder=build_payload,
response=response,
)
async def _build_bgp_incidents_geojson(
*,
severity: str | None,
status: str | None,
limit: int,
db: AsyncSession,
) -> dict[str, Any]:
stmt = select(BGPIncident).order_by(BGPIncident.created_at.desc()).limit(limit)
if severity:
stmt = stmt.where(BGPIncident.severity == severity)
@@ -2432,7 +2640,19 @@ async def get_bgp_incidents_geojson(
@router.get("/geo/bgp-collectors")
async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db)):
async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db), response: Response = None):
async def build_payload() -> dict[str, Any]:
return await _build_bgp_collectors_geojson(db)
return await get_or_build_layer_payload(
key=earth_layer_cache.key("bgp-collectors"),
policy=BGP_CACHE_POLICY,
builder=build_payload,
response=response,
)
async def _build_bgp_collectors_geojson(db: AsyncSession) -> dict[str, Any]:
coverage = await build_bgp_collector_coverage(
db,
source_filter=("ris_live_bgp", "bgpstream_bgp"),
@@ -2447,8 +2667,20 @@ async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db)):
@router.get("/geo/summary")
async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db), response: Response = None):
"""Return lightweight Earth HUD counts without loading layer GeoJSON payloads."""
async def build_payload() -> dict[str, Any]:
return await _build_visualization_geo_summary(db)
return await get_or_build_layer_payload(
key=earth_layer_cache.key("summary"),
policy=SUMMARY_CACHE_POLICY,
builder=build_payload,
response=response,
)
async def _build_visualization_geo_summary(db: AsyncSession) -> dict[str, Any]:
cable_count = await _count_current_or_latest_task_data(db, "arcgis_cables")
landing_point_count = await _count_current_or_latest_task_data(db, "arcgis_landing_points")
satellite_count = await _count_current_or_latest_task_data(

View File

@@ -13,6 +13,7 @@ from app.core.config import settings
from app.core.countries import normalize_country
from app.core.time import to_iso8601_utc
from app.core.websocket.broadcaster import broadcaster
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
class BaseCollector(ABC):
@@ -531,6 +532,7 @@ class BaseCollector(ABC):
}
await db.commit()
invalidate_earth_layer_cache_for_source(self.name)
await self.update_progress(len(data), force=True)
return records_added

View File

@@ -19,6 +19,7 @@ from app.services.bgp_collector_locations import (
)
from app.services.bgp_event_locations import resolve_bgp_event_geo_dict
from app.services.bgp_incidents import create_bgp_incidents_for_anomalies
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
from app.services.bgp_detectors import (
detect_mass_withdrawal_anomalies,
detect_more_specific_burst_anomalies,
@@ -223,6 +224,8 @@ async def save_bgp_observations_for_batch(
if created:
await db.commit()
for source in {"ris_live_bgp", "bgpstream_bgp"}:
invalidate_earth_layer_cache_for_source(source)
return created

View File

@@ -0,0 +1,392 @@
from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
import hashlib
import json
import time
from typing import Any
from fastapi import Response
from app.core.cache import _RedisClient
from app.core.config import settings
from app.core.logging import get_logger
logger = get_logger(__name__, service="earth_layer_cache")
EARTH_LAYER_CACHE_PREFIX = "earth:layer:v1"
EARTH_LAYER_LOCK_PREFIX = "earth:layer:lock:v1"
DEFAULT_LOCK_TTL_SECONDS = 10
DEFAULT_LOCK_WAIT_SECONDS = 0.2
DEFAULT_MAX_FEATURES = 5000
DEFAULT_MAX_BYTES = 5 * 1024 * 1024
DEFAULT_BBOX_PRECISION_DEGREES = 0.1
DEV_CACHE_KEY_HEADER = {"development", "dev", "test", "testing", "local"}
@dataclass(frozen=True)
class EarthLayerCachePolicy:
fresh_ttl_seconds: int
stale_ttl_seconds: int
max_features: int = DEFAULT_MAX_FEATURES
max_bytes: int = DEFAULT_MAX_BYTES
lock_ttl_seconds: int = DEFAULT_LOCK_TTL_SECONDS
lock_wait_seconds: float = DEFAULT_LOCK_WAIT_SECONDS
@dataclass(frozen=True)
class EarthLayerCacheResult:
payload: dict[str, Any]
state: str
key: str
features: int
bytes: int
class EarthLayerCache:
def __init__(self) -> None:
self._client = None
@property
def client(self):
if self._client is None:
self._client = _RedisClient.get_client()
return self._client
@staticmethod
def key(layer: str, **params: Any) -> str:
parts = [EARTH_LAYER_CACHE_PREFIX, _safe_key_part(layer)]
for name in sorted(params):
value = params[name]
if value is None:
value = "none"
parts.append(f"{_safe_key_part(name)}:{_safe_key_part(value)}")
return ":".join(parts)
@staticmethod
def stale_key(key: str) -> str:
return f"{key}:stale"
@staticmethod
def lock_key(key: str) -> str:
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:32]
return f"{EARTH_LAYER_LOCK_PREFIX}:{digest}"
def get_json(self, key: str) -> dict[str, Any] | None:
raw = self.client.get(key)
if not raw:
return None
value = json.loads(raw)
return value if isinstance(value, dict) else None
def set_json(self, key: str, payload: dict[str, Any], ttl_seconds: int) -> None:
self.client.setex(key, ttl_seconds, json.dumps(payload, ensure_ascii=False, default=str))
def acquire_lock(self, key: str, ttl_seconds: int) -> bool:
return bool(self.client.set(self.lock_key(key), "1", nx=True, ex=ttl_seconds))
def release_lock(self, key: str) -> None:
try:
self.client.delete(self.lock_key(key))
except Exception:
pass
def delete_pattern(self, pattern: str = f"{EARTH_LAYER_CACHE_PREFIX}:*") -> int:
keys = list(self.client.scan_iter(match=pattern))
if not keys:
return 0
return int(self.client.delete(*keys))
def status(self) -> dict[str, Any]:
keys = list(self.client.scan_iter(match=f"{EARTH_LAYER_CACHE_PREFIX}:*"))
by_layer: dict[str, dict[str, Any]] = {}
total_memory = 0
for key in keys:
key_str = key.decode("utf-8") if isinstance(key, bytes) else str(key)
layer = _layer_from_key(key_str)
entry = by_layer.setdefault(layer, {"keys": 0, "stale_keys": 0, "memory_bytes": 0})
entry["keys"] += 1
if key_str.endswith(":stale"):
entry["stale_keys"] += 1
try:
memory = int(self.client.memory_usage(key) or 0)
except Exception:
memory = 0
entry["memory_bytes"] += memory
total_memory += memory
return {
"prefix": EARTH_LAYER_CACHE_PREFIX,
"key_count": len(keys),
"memory_bytes": total_memory,
"layers": by_layer,
}
earth_layer_cache = EarthLayerCache()
def quantize_bbox(
bbox: tuple[float, float, float, float],
*,
precision: float = DEFAULT_BBOX_PRECISION_DEGREES,
) -> tuple[float, float, float, float]:
return tuple(round(value / precision) * precision for value in bbox) # type: ignore[return-value]
def format_bbox_key(bbox: tuple[float, float, float, float]) -> str:
return ",".join(f"{value:.1f}" for value in bbox)
def apply_cache_headers(response: Response | None, result: EarthLayerCacheResult) -> None:
if response is None:
return
response.headers["X-Planet-Cache"] = result.state
response.headers["X-Planet-Cache-Features"] = str(result.features)
response.headers["X-Planet-Cache-Bytes"] = str(result.bytes)
env_name = str(getattr(settings, "ENVIRONMENT", "") or "development").lower()
if env_name in DEV_CACHE_KEY_HEADER:
response.headers["X-Planet-Cache-Key"] = result.key
async def get_or_build_layer_payload(
*,
key: str,
policy: EarthLayerCachePolicy,
builder: Callable[[], Awaitable[dict[str, Any]]],
response: Response | None = None,
) -> dict[str, Any]:
result = await resolve_layer_payload(key=key, policy=policy, builder=builder)
apply_cache_headers(response, result)
return result.payload
async def resolve_layer_payload(
*,
key: str,
policy: EarthLayerCachePolicy,
builder: Callable[[], Awaitable[dict[str, Any]]],
) -> EarthLayerCacheResult:
started = time.perf_counter()
try:
cached = earth_layer_cache.get_json(key)
if cached is not None:
return _result(cached, state="hit", key=key)
lock_acquired = earth_layer_cache.acquire_lock(key, policy.lock_ttl_seconds)
if lock_acquired:
try:
payload = await _build_budgeted_payload(builder, policy)
_write_fresh_and_stale(key, payload, policy)
_log_cache_event("refresh", key, payload, started)
return _result(payload, state="refresh", key=key)
except Exception as exc:
stale = _read_stale(key)
if stale is not None:
logger.warning_event(
"Earth layer cache builder failed; returning stale payload",
event="earth_layer_cache.stale_after_builder_error",
context={"key": key, "error": str(exc)},
)
return _result(stale, state="stale", key=key)
raise
finally:
earth_layer_cache.release_lock(key)
stale = _read_stale(key)
if stale is not None:
return _result(stale, state="stale", key=key)
await asyncio.sleep(policy.lock_wait_seconds)
cached_after_wait = earth_layer_cache.get_json(key)
if cached_after_wait is not None:
return _result(cached_after_wait, state="hit", key=key)
payload = await _build_budgeted_payload(builder, policy)
_log_cache_event("miss", key, payload, started)
return _result(payload, state="miss", key=key)
except Exception as exc:
try:
payload = await _build_budgeted_payload(builder, policy)
except Exception:
raise exc
logger.warning_event(
"Earth layer cache bypassed",
event="earth_layer_cache.bypass",
context={"key": key, "error": str(exc)},
)
return _result(payload, state="bypass", key=key)
def apply_payload_budget(payload: dict[str, Any], policy: EarthLayerCachePolicy) -> dict[str, Any]:
budgeted = _truncate_features(payload, policy.max_features, "feature_budget")
size = _payload_size(budgeted)
if size <= policy.max_bytes:
return budgeted
features = budgeted.get("features")
if not isinstance(features, list):
return _with_budget_diagnostics(
budgeted,
truncated=True,
reason="byte_budget",
bytes_before=size,
bytes_after=size,
)
low = 0
high = len(features)
best = []
best_size = _payload_size({**budgeted, "features": best})
while low <= high:
mid = (low + high) // 2
candidate_features = features[:mid]
candidate = _with_budget_diagnostics(
{**budgeted, "features": candidate_features},
truncated=mid < len(features),
reason="byte_budget",
bytes_before=size,
bytes_after=0,
)
candidate_size = _payload_size(candidate)
if candidate_size <= policy.max_bytes:
best = candidate_features
best_size = candidate_size
low = mid + 1
else:
high = mid - 1
return _with_budget_diagnostics(
{**budgeted, "features": best},
truncated=True,
reason="byte_budget",
bytes_before=size,
bytes_after=best_size,
)
def invalidate_earth_layer_cache_for_source(source: str) -> int:
source_key = str(source or "").strip()
patterns = {
"arcgis_cables": ["cables*", "landing-points*", "summary*"],
"arcgis_landing_points": ["landing-points*", "summary*"],
"arcgis_cable_landing_relation": ["landing-points*", "summary*"],
"celestrak_tle": ["satellites*", "summary*"],
"spacetrack_tle": ["satellites*", "summary*"],
"top500": ["compute-centers*", "summary*"],
"epoch_ai_gpu": ["compute-centers*", "summary*"],
"ris_live_bgp": ["bgp*", "summary*"],
"bgpstream_bgp": ["bgp*", "summary*"],
}.get(source_key, [])
deleted = 0
for layer_pattern in patterns:
deleted += earth_layer_cache.delete_pattern(f"{EARTH_LAYER_CACHE_PREFIX}:{layer_pattern}")
return deleted
async def _build_budgeted_payload(
builder: Callable[[], Awaitable[dict[str, Any]]],
policy: EarthLayerCachePolicy,
) -> dict[str, Any]:
payload = await builder()
return apply_payload_budget(payload, policy)
def _write_fresh_and_stale(key: str, payload: dict[str, Any], policy: EarthLayerCachePolicy) -> None:
earth_layer_cache.set_json(key, payload, policy.fresh_ttl_seconds)
earth_layer_cache.set_json(earth_layer_cache.stale_key(key), payload, policy.stale_ttl_seconds)
def _read_stale(key: str) -> dict[str, Any] | None:
try:
return earth_layer_cache.get_json(earth_layer_cache.stale_key(key))
except Exception:
return None
def _truncate_features(payload: dict[str, Any], max_features: int, reason: str) -> dict[str, Any]:
features = payload.get("features")
if not isinstance(features, list) or len(features) <= max_features:
return payload
return _with_budget_diagnostics(
{**payload, "features": features[:max_features]},
truncated=True,
reason=reason,
original_feature_count=len(features),
)
def _with_budget_diagnostics(
payload: dict[str, Any],
*,
truncated: bool,
reason: str,
original_feature_count: int | None = None,
bytes_before: int | None = None,
bytes_after: int | None = None,
) -> dict[str, Any]:
diagnostics = dict(payload.get("diagnostics") or {})
diagnostics.update(
{
"truncated": bool(truncated or diagnostics.get("truncated")),
"limit_reason": reason,
}
)
if original_feature_count is not None:
diagnostics["original_feature_count"] = original_feature_count
if bytes_before is not None:
diagnostics["bytes_before_budget"] = bytes_before
if bytes_after is not None:
diagnostics["bytes_after_budget"] = bytes_after
return {**payload, "diagnostics": diagnostics}
def _result(payload: dict[str, Any], *, state: str, key: str) -> EarthLayerCacheResult:
return EarthLayerCacheResult(
payload=payload,
state=state,
key=key,
features=_feature_count(payload),
bytes=_payload_size(payload),
)
def _feature_count(payload: dict[str, Any]) -> int:
features = payload.get("features")
if isinstance(features, list):
return len(features)
count = payload.get("count")
return int(count) if isinstance(count, int) else 0
def _payload_size(payload: dict[str, Any]) -> int:
return len(json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8"))
def _safe_key_part(value: Any) -> str:
raw = str(value).strip().lower()
return "".join(char if char.isalnum() or char in {"-", "_", ".", ","} else "_" for char in raw)[:160]
def _layer_from_key(key: str) -> str:
prefix = f"{EARTH_LAYER_CACHE_PREFIX}:"
if not key.startswith(prefix):
return "unknown"
remainder = key[len(prefix):]
return remainder.split(":", 1)[0]
def _log_cache_event(state: str, key: str, payload: dict[str, Any], started: float) -> None:
logger.info_event(
"Earth layer cache resolved",
event="earth_layer_cache.resolved",
context={
"state": state,
"key": key,
"features": _feature_count(payload),
"bytes": _payload_size(payload),
"duration_ms": round((time.perf_counter() - started) * 1000, 2),
},
)