release: bump version to 0.50.0

This commit is contained in:
rayd1o
2026-05-10 22:06:01 +08:00
parent e1984c7a35
commit 455b8360d0
80 changed files with 10936 additions and 298 deletions

11
motion_agent/__init__.py Normal file
View File

@@ -0,0 +1,11 @@
"""Local motion capture gesture agent for Planet Earth displays."""
from .config import MotionAgentConfig
from .events import GestureEvent, HeartbeatEvent, StatusEvent
__all__ = [
"GestureEvent",
"HeartbeatEvent",
"MotionAgentConfig",
"StatusEvent",
]

5
motion_agent/__main__.py Normal file
View File

@@ -0,0 +1,5 @@
from .cli import main
if __name__ == "__main__":
main()

147
motion_agent/cameras.py Normal file
View File

@@ -0,0 +1,147 @@
"""Camera input adapters for the local motion capture agent."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Protocol
class MotionAgentCameraError(RuntimeError):
"""Raised when a configured camera source cannot be opened."""
class MotionAgentDependencyError(RuntimeError):
"""Raised when an optional runtime dependency is missing."""
class CameraInput(Protocol):
camera_id: str
def open(self) -> None:
"""Open the camera source."""
def read(self) -> Any:
"""Return the next frame or raise a camera error."""
def close(self) -> None:
"""Release the camera source."""
@dataclass(frozen=True)
class UsbCameraSpec:
index: int
width: int = 1280
height: int = 720
fps: int = 30
@dataclass(frozen=True)
class UrlCameraSpec:
url: str
camera_id: str | None = None
class UsbCameraInput:
"""OpenCV-backed USB camera input.
OpenCV is intentionally imported lazily so tests and protocol-only runs do
not require native camera dependencies.
"""
def __init__(self, spec: UsbCameraSpec):
self.spec = spec
self.camera_id = f"usb:{spec.index}"
self._capture: Any = None
def open(self) -> None:
device_path = Path(f"/dev/video{self.spec.index}")
if Path("/dev").exists() and not device_path.exists():
raise MotionAgentCameraError(
f"USB camera index {self.spec.index} was not found at {device_path}. "
"Check camera passthrough or set MOTION_AGENT_CAMERA_INDEXES to the available indexes."
)
try:
import cv2 # type: ignore
except ImportError as exc:
raise MotionAgentDependencyError(
"OpenCV is required for camera capture. Add opencv-python with uv "
"or start the agent with --dry-run for protocol testing."
) from exc
capture = cv2.VideoCapture(self.spec.index)
if not capture or not capture.isOpened():
raise MotionAgentCameraError(f"Unable to open USB camera index {self.spec.index}.")
capture.set(cv2.CAP_PROP_FRAME_WIDTH, self.spec.width)
capture.set(cv2.CAP_PROP_FRAME_HEIGHT, self.spec.height)
capture.set(cv2.CAP_PROP_FPS, self.spec.fps)
self._capture = capture
def read(self) -> Any:
if self._capture is None:
raise MotionAgentCameraError(f"Camera {self.camera_id} is not open.")
ok, frame = self._capture.read()
if not ok:
raise MotionAgentCameraError(f"Camera {self.camera_id} did not return a frame.")
return frame
def close(self) -> None:
if self._capture is not None:
self._capture.release()
self._capture = None
class UrlCameraInput:
"""OpenCV-backed URL camera input for RTSP/HTTP/Web camera bridges."""
def __init__(self, spec: UrlCameraSpec):
self.spec = spec
self.camera_id = spec.camera_id or f"url:{spec.url}"
self._capture: Any = None
def open(self) -> None:
try:
import cv2 # type: ignore
except ImportError as exc:
raise MotionAgentDependencyError(
"OpenCV is required for URL camera capture. Add opencv-python with uv "
"or start the agent with --dry-run for protocol testing."
) from exc
capture = cv2.VideoCapture(self.spec.url)
if not capture or not capture.isOpened():
raise MotionAgentCameraError(
f"Unable to open camera URL {self.spec.url}. Check the stream URL, "
"firewall, and LAN reachability."
)
self._capture = capture
def read(self) -> Any:
if self._capture is None:
raise MotionAgentCameraError(f"Camera {self.camera_id} is not open.")
ok, frame = self._capture.read()
if not ok:
raise MotionAgentCameraError(f"Camera {self.camera_id} did not return a frame.")
return frame
def close(self) -> None:
if self._capture is not None:
self._capture.release()
self._capture = None
class NullCameraInput:
"""No-op camera source used by dry-run and tests."""
camera_id = "dry-run:null-camera"
def open(self) -> None:
return None
def read(self) -> None:
return None
def close(self) -> None:
return None

65
motion_agent/cli.py Normal file
View File

@@ -0,0 +1,65 @@
"""Command line entrypoint for the local motion capture agent."""
from __future__ import annotations
import argparse
import asyncio
import sys
from dataclasses import replace
from .cameras import MotionAgentCameraError, MotionAgentDependencyError
from .config import MotionAgentConfig
from .server import MotionAgentServer
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Run the Planet local motion capture agent.")
parser.add_argument("--host", default=None, help="WebSocket bind host.")
parser.add_argument("--port", type=int, default=None, help="WebSocket bind port.")
parser.add_argument("--camera-indexes", default=None, help="Comma-separated USB camera indexes.")
parser.add_argument("--camera-urls", default=None, help="Comma-separated RTSP/HTTP camera URLs.")
parser.add_argument("--mode", choices=["auto", "single", "dual"], default=None)
parser.add_argument("--dry-run", action="store_true", help="Start without camera/CV dependencies.")
return parser
def _parse_indexes(raw: str | None, fallback: tuple[int, ...]) -> tuple[int, ...]:
if raw is None:
return fallback
values = [item.strip() for item in raw.split(",") if item.strip()]
return tuple(int(item) for item in values) or fallback
def _parse_urls(raw: str | None, fallback: tuple[str, ...]) -> tuple[str, ...]:
if raw is None:
return fallback
return tuple(item.strip() for item in raw.split(",") if item.strip())
def config_from_args(args: argparse.Namespace) -> MotionAgentConfig:
config = MotionAgentConfig.from_env()
return replace(
config,
host=args.host or config.host,
port=args.port or config.port,
camera_indexes=_parse_indexes(args.camera_indexes, config.camera_indexes),
camera_urls=_parse_urls(args.camera_urls, config.camera_urls),
mode=args.mode or config.mode,
dry_run=args.dry_run or config.dry_run,
)
async def async_main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
config = config_from_args(args)
try:
server = MotionAgentServer(config)
await server.run()
except (MotionAgentCameraError, MotionAgentDependencyError, RuntimeError) as exc:
print(f"Motion agent failed: {exc}", file=sys.stderr)
return 2
return 0
def main(argv: list[str] | None = None) -> None:
raise SystemExit(asyncio.run(async_main(argv)))

65
motion_agent/config.py Normal file
View File

@@ -0,0 +1,65 @@
"""Configuration for the local motion capture agent."""
from __future__ import annotations
import os
from dataclasses import dataclass, field
def _parse_camera_indexes(raw: str | None) -> tuple[int, ...]:
if not raw:
return (0,)
indexes: list[int] = []
for item in raw.split(","):
value = item.strip()
if not value:
continue
indexes.append(int(value))
return tuple(indexes or [0])
def _parse_camera_urls(raw: str | None) -> tuple[str, ...]:
if not raw:
return ()
return tuple(item.strip() for item in raw.split(",") if item.strip())
@dataclass(frozen=True)
class MotionAgentConfig:
host: str = "127.0.0.1"
port: int = 8765
path: str = "/ws/gestures"
camera_indexes: tuple[int, ...] = field(default_factory=lambda: (0,))
camera_urls: tuple[str, ...] = field(default_factory=tuple)
camera_width: int = 1280
camera_height: int = 720
camera_fps: int = 30
mode: str = "auto"
confidence_threshold: float = 0.72
cooldown_ms: int = 450
heartbeat_interval_ms: int = 1000
max_event_hz: int = 20
dry_run: bool = False
@property
def websocket_url(self) -> str:
return f"ws://{self.host}:{self.port}{self.path}"
@classmethod
def from_env(cls) -> "MotionAgentConfig":
return cls(
host=os.getenv("MOTION_AGENT_HOST", "127.0.0.1"),
port=int(os.getenv("MOTION_AGENT_PORT", "8765")),
path=os.getenv("MOTION_AGENT_PATH", "/ws/gestures"),
camera_indexes=_parse_camera_indexes(os.getenv("MOTION_AGENT_CAMERA_INDEXES")),
camera_urls=_parse_camera_urls(os.getenv("MOTION_AGENT_CAMERA_URLS")),
camera_width=int(os.getenv("MOTION_AGENT_CAMERA_WIDTH", "1280")),
camera_height=int(os.getenv("MOTION_AGENT_CAMERA_HEIGHT", "720")),
camera_fps=int(os.getenv("MOTION_AGENT_CAMERA_FPS", "30")),
mode=os.getenv("MOTION_AGENT_MODE", "auto"),
confidence_threshold=float(os.getenv("MOTION_AGENT_CONFIDENCE_THRESHOLD", "0.72")),
cooldown_ms=int(os.getenv("MOTION_AGENT_COOLDOWN_MS", "450")),
heartbeat_interval_ms=int(os.getenv("MOTION_AGENT_HEARTBEAT_MS", "1000")),
max_event_hz=int(os.getenv("MOTION_AGENT_MAX_EVENT_HZ", "20")),
dry_run=os.getenv("MOTION_AGENT_DRY_RUN", "").lower() in {"1", "true", "yes"},
)

96
motion_agent/events.py Normal file
View File

@@ -0,0 +1,96 @@
"""Stable event models emitted by the local motion capture agent."""
from __future__ import annotations
import json
import time
from dataclasses import asdict, dataclass, field
from typing import Any, Literal
GestureName = Literal["rotate_left", "rotate_right", "zoom_in", "zoom_out", "confirm"]
GesturePhase = Literal["start", "active", "end", "discrete"]
def now_ms() -> int:
return int(time.time() * 1000)
@dataclass(frozen=True)
class GestureEvent:
gesture: GestureName
phase: GesturePhase = "discrete"
confidence: float = 1.0
intensity: float = 1.0
timestamp_ms: int = field(default_factory=now_ms)
seq: int = 0
source: str = "motion-agent"
mode: str = "single"
payload: dict[str, Any] = field(default_factory=dict)
type: Literal["gesture"] = "gesture"
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def to_json(self) -> str:
return json.dumps(self.to_dict(), ensure_ascii=False, separators=(",", ":"))
@dataclass(frozen=True)
class StatusEvent:
connected: bool
camera_count: int
active_camera_ids: tuple[str, ...] = ()
mode: str = "single"
recognizer: str = "mediapipe-opencv"
fps: float = 0.0
last_gesture: str | None = None
error: str | None = None
timestamp_ms: int = field(default_factory=now_ms)
source: str = "motion-agent"
type: Literal["status"] = "status"
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def to_json(self) -> str:
return json.dumps(self.to_dict(), ensure_ascii=False, separators=(",", ":"))
@dataclass(frozen=True)
class HeartbeatEvent:
timestamp_ms: int = field(default_factory=now_ms)
source: str = "motion-agent"
type: Literal["heartbeat"] = "heartbeat"
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def to_json(self) -> str:
return json.dumps(self.to_dict(), ensure_ascii=False, separators=(",", ":"))
@dataclass(frozen=True)
class SkeletonJoint:
id: str
x: float
y: float
confidence: float = 1.0
@dataclass(frozen=True)
class SkeletonEvent:
joints: list[SkeletonJoint]
bones: list[tuple[str, str]]
matched_gesture: GestureName | None = None
confidence: float = 0.0
camera_id: str = "unknown"
timestamp_ms: int = field(default_factory=now_ms)
source: str = "motion-agent"
mode: str = "single"
type: Literal["skeleton"] = "skeleton"
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def to_json(self) -> str:
return json.dumps(self.to_dict(), ensure_ascii=False, separators=(",", ":"))

118
motion_agent/recognizer.py Normal file
View File

@@ -0,0 +1,118 @@
"""Gesture recognizer interfaces.
The production recognizer is intentionally thin in this skeleton. It validates
that optional CV dependencies are available and keeps the protocol independent
from any specific model implementation.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Protocol
from .cameras import MotionAgentDependencyError
from .events import GestureName, SkeletonEvent, SkeletonJoint, now_ms
@dataclass(frozen=True)
class GestureObservation:
gesture: GestureName
confidence: float
intensity: float = 1.0
timestamp_ms: int | None = None
class GestureRecognizer(Protocol):
name: str
def recognize(self, frame: Any) -> GestureObservation | None:
"""Return a gesture observation for the current frame."""
def debug_skeleton(
self,
frame: Any,
*,
camera_id: str,
mode: str,
matched_gesture: GestureName | None = None,
confidence: float = 0.0,
) -> SkeletonEvent | None:
"""Return normalized skeleton debug data when available."""
class MediaPipeGestureRecognizer:
name = "mediapipe-opencv"
def __init__(self) -> None:
try:
import cv2 # noqa: F401
import mediapipe # noqa: F401
except ImportError as exc:
raise MotionAgentDependencyError(
"MediaPipe and OpenCV are required for live gesture recognition. "
"Add mediapipe and opencv-python with uv, or use --dry-run for protocol testing."
) from exc
def recognize(self, frame: Any) -> GestureObservation | None:
_ = frame
return None
def debug_skeleton(
self,
frame: Any,
*,
camera_id: str,
mode: str,
matched_gesture: GestureName | None = None,
confidence: float = 0.0,
) -> SkeletonEvent | None:
_ = frame, camera_id, mode, matched_gesture, confidence
return None
class NullGestureRecognizer:
name = "dry-run"
def recognize(self, frame: Any) -> GestureObservation | None:
_ = frame
return None
def debug_skeleton(
self,
frame: Any,
*,
camera_id: str,
mode: str,
matched_gesture: GestureName | None = None,
confidence: float = 0.0,
) -> SkeletonEvent | None:
_ = frame
now = now_ms()
sway = ((now // 250) % 6 - 2.5) * 0.015
joints = [
SkeletonJoint("head", 0.5, 0.18, 1.0),
SkeletonJoint("neck", 0.5, 0.3, 1.0),
SkeletonJoint("left_shoulder", 0.38, 0.34, 1.0),
SkeletonJoint("right_shoulder", 0.62, 0.34, 1.0),
SkeletonJoint("left_elbow", 0.31 + sway, 0.48, 0.95),
SkeletonJoint("right_elbow", 0.69 - sway, 0.48, 0.95),
SkeletonJoint("left_wrist", 0.24 + sway, 0.62, 0.9),
SkeletonJoint("right_wrist", 0.76 - sway, 0.62, 0.9),
]
bones = [
("head", "neck"),
("neck", "left_shoulder"),
("neck", "right_shoulder"),
("left_shoulder", "left_elbow"),
("left_elbow", "left_wrist"),
("right_shoulder", "right_elbow"),
("right_elbow", "right_wrist"),
]
return SkeletonEvent(
joints=joints,
bones=bones,
matched_gesture=matched_gesture,
confidence=confidence,
camera_id=camera_id,
mode=mode,
)

192
motion_agent/server.py Normal file
View File

@@ -0,0 +1,192 @@
"""Async WebSocket server for local gesture events."""
from __future__ import annotations
import asyncio
from collections.abc import Iterable
from contextlib import suppress
from typing import Any
from .cameras import (
MotionAgentCameraError,
NullCameraInput,
UrlCameraInput,
UrlCameraSpec,
UsbCameraInput,
UsbCameraSpec,
)
from .config import MotionAgentConfig
from .events import GestureEvent, HeartbeatEvent, SkeletonEvent, StatusEvent
from .recognizer import GestureRecognizer, MediaPipeGestureRecognizer, NullGestureRecognizer
from .state import GestureStateMachine
class MotionAgentServer:
def __init__(
self,
config: MotionAgentConfig,
cameras: Iterable[Any] | None = None,
recognizer: GestureRecognizer | None = None,
) -> None:
self.config = config
self.cameras = list(cameras) if cameras is not None else self._build_cameras(config)
self.recognizer = recognizer if recognizer is not None else self._build_recognizer(config)
self.state = GestureStateMachine(
confidence_threshold=config.confidence_threshold,
cooldown_ms=config.cooldown_ms,
mode=self._resolve_mode(),
)
self.clients: set[Any] = set()
self.last_gesture: str | None = None
self.last_error: str | None = None
self._stop = asyncio.Event()
def _build_cameras(self, config: MotionAgentConfig) -> list[Any]:
if config.dry_run:
return [NullCameraInput()]
if config.camera_urls:
return [
UrlCameraInput(UrlCameraSpec(url=url, camera_id=f"url:{index}"))
for index, url in enumerate(config.camera_urls)
]
specs = [
UsbCameraSpec(
index=index,
width=config.camera_width,
height=config.camera_height,
fps=config.camera_fps,
)
for index in config.camera_indexes
]
return [UsbCameraInput(spec) for spec in specs]
def _build_recognizer(self, config: MotionAgentConfig) -> GestureRecognizer:
if config.dry_run:
return NullGestureRecognizer()
return MediaPipeGestureRecognizer()
def _resolve_mode(self) -> str:
if self.config.mode in {"single", "dual", "auto"}:
if self.config.mode != "auto":
return self.config.mode
return "dual" if len(self.cameras) >= 2 else "single"
def open_cameras(self) -> None:
opened = []
try:
for camera in self.cameras:
camera.open()
opened.append(camera)
except Exception:
for camera in opened:
with suppress(Exception):
camera.close()
raise
def close_cameras(self) -> None:
for camera in self.cameras:
with suppress(Exception):
camera.close()
def status_event(self, connected: bool = True) -> StatusEvent:
return StatusEvent(
connected=connected,
camera_count=len(self.cameras),
active_camera_ids=tuple(camera.camera_id for camera in self.cameras),
mode=self.state.mode,
recognizer=self.recognizer.name,
last_gesture=self.last_gesture,
error=self.last_error,
)
async def broadcast(
self,
event: GestureEvent | HeartbeatEvent | SkeletonEvent | StatusEvent,
) -> None:
if not self.clients:
return
payload = event.to_json()
stale = []
for websocket in self.clients:
try:
await websocket.send(payload)
except Exception:
stale.append(websocket)
for websocket in stale:
self.clients.discard(websocket)
async def handler(self, websocket: Any, path: str | None = None) -> None:
if path is not None and path != self.config.path:
await websocket.close(code=1008, reason="Unsupported motion agent path")
return
self.clients.add(websocket)
await websocket.send(self.status_event().to_json())
try:
async for _message in websocket:
await websocket.send(self.status_event().to_json())
finally:
self.clients.discard(websocket)
async def heartbeat_loop(self) -> None:
interval = max(0.1, self.config.heartbeat_interval_ms / 1000)
while not self._stop.is_set():
await self.broadcast(HeartbeatEvent())
await asyncio.sleep(interval)
async def recognition_loop(self) -> None:
min_interval = 1 / max(1, self.config.max_event_hz)
while not self._stop.is_set():
try:
for camera in self.cameras:
frame = camera.read()
observation = self.recognizer.recognize(frame)
matched_gesture = None
matched_confidence = 0.0
if observation is None:
event = None
else:
event = self.state.accept(observation)
if event is not None:
self.last_gesture = event.gesture
matched_gesture = event.gesture
matched_confidence = event.confidence
await self.broadcast(event)
skeleton = self.recognizer.debug_skeleton(
frame,
camera_id=camera.camera_id,
mode=self.state.mode,
matched_gesture=matched_gesture,
confidence=matched_confidence,
)
if skeleton is not None:
await self.broadcast(skeleton)
except MotionAgentCameraError as exc:
self.last_error = str(exc)
await self.broadcast(self.status_event(connected=True))
await asyncio.sleep(1)
await asyncio.sleep(min_interval)
async def run(self) -> None:
try:
import websockets
except ImportError as exc:
raise RuntimeError("websockets is required to run the motion agent server.") from exc
self.open_cameras()
async with websockets.serve(self.handler, self.config.host, self.config.port):
print(f"Motion agent listening on {self.config.websocket_url}", flush=True)
heartbeat_task = asyncio.create_task(self.heartbeat_loop())
recognition_task = asyncio.create_task(self.recognition_loop())
try:
await self._stop.wait()
finally:
heartbeat_task.cancel()
recognition_task.cancel()
with suppress(asyncio.CancelledError):
await heartbeat_task
with suppress(asyncio.CancelledError):
await recognition_task
self.close_cameras()
def stop(self) -> None:
self._stop.set()

44
motion_agent/state.py Normal file
View File

@@ -0,0 +1,44 @@
"""Gesture event state machine."""
from __future__ import annotations
from dataclasses import dataclass
from .events import GestureEvent
from .recognizer import GestureObservation
@dataclass
class GestureStateMachine:
confidence_threshold: float = 0.72
cooldown_ms: int = 450
mode: str = "single"
def __post_init__(self) -> None:
self._last_emit_by_gesture: dict[str, int] = {}
self._seq = 0
def accept(self, observation: GestureObservation) -> GestureEvent | None:
if observation.confidence < self.confidence_threshold:
return None
timestamp_ms = observation.timestamp_ms
if timestamp_ms is None:
from .events import now_ms
timestamp_ms = now_ms()
last_emit_at = self._last_emit_by_gesture.get(observation.gesture)
if last_emit_at is not None and timestamp_ms - last_emit_at < self.cooldown_ms:
return None
self._seq += 1
self._last_emit_by_gesture[observation.gesture] = timestamp_ms
return GestureEvent(
gesture=observation.gesture,
confidence=round(max(0.0, min(1.0, observation.confidence)), 4),
intensity=round(max(0.0, min(1.0, observation.intensity)), 4),
timestamp_ms=timestamp_ms,
seq=self._seq,
mode=self.mode,
)