863 lines
30 KiB
JavaScript
863 lines
30 KiB
JavaScript
import { describe, expect, test } from "bun:test";
|
|
|
|
import {
|
|
createMotionControlAdapter,
|
|
normalizeMotionProvider,
|
|
} from "./motion-control.js";
|
|
import {
|
|
createBrowserCameraProvider,
|
|
recognizeGesture,
|
|
} from "./motion-browser-provider.js";
|
|
|
|
class TestEventTarget {
|
|
constructor() {
|
|
this.events = [];
|
|
this.listeners = new Map();
|
|
}
|
|
|
|
dispatchEvent(event) {
|
|
this.events.push(event);
|
|
(this.listeners.get(event.type) || []).forEach((listener) => listener(event));
|
|
return true;
|
|
}
|
|
|
|
addEventListener(type, listener) {
|
|
const listeners = this.listeners.get(type) || [];
|
|
listeners.push(listener);
|
|
this.listeners.set(type, listeners);
|
|
}
|
|
|
|
removeEventListener(type, listener) {
|
|
const listeners = this.listeners.get(type) || [];
|
|
this.listeners.set(type, listeners.filter((item) => item !== listener));
|
|
}
|
|
}
|
|
|
|
function installWindow(search = "") {
|
|
const target = new TestEventTarget();
|
|
globalThis.CustomEvent = class CustomEvent {
|
|
constructor(type, options = {}) {
|
|
this.type = type;
|
|
this.detail = options.detail;
|
|
}
|
|
};
|
|
globalThis.window = {
|
|
location: {
|
|
search,
|
|
hostname: "localhost",
|
|
},
|
|
isSecureContext: true,
|
|
localStorage: {
|
|
getItem() {
|
|
return null;
|
|
},
|
|
},
|
|
dispatchEvent: target.dispatchEvent.bind(target),
|
|
addEventListener: target.addEventListener.bind(target),
|
|
removeEventListener: target.removeEventListener.bind(target),
|
|
};
|
|
return target;
|
|
}
|
|
|
|
function installDocument() {
|
|
const tracks = [];
|
|
globalThis.document = {
|
|
body: {
|
|
appendChild() {},
|
|
},
|
|
createElement(tagName) {
|
|
expect(tagName).toBe("video");
|
|
return {
|
|
muted: false,
|
|
playsInline: false,
|
|
autoplay: false,
|
|
readyState: 4,
|
|
style: {},
|
|
srcObject: null,
|
|
videoWidth: 640,
|
|
videoHeight: 360,
|
|
addEventListener() {},
|
|
play: () => Promise.resolve(),
|
|
pause() {},
|
|
remove() {},
|
|
};
|
|
},
|
|
};
|
|
return tracks;
|
|
}
|
|
|
|
describe("motion-control provider manager", () => {
|
|
test("normalizes browser and agent provider aliases", () => {
|
|
expect(normalizeMotionProvider("browser")).toBe("browser_camera");
|
|
expect(normalizeMotionProvider("agent")).toBe("motion_agent");
|
|
expect(normalizeMotionProvider("motion_agent")).toBe("motion_agent");
|
|
expect(normalizeMotionProvider("unknown")).toBe("browser_camera");
|
|
});
|
|
|
|
test("mock skeleton message dispatches debug frame event", () => {
|
|
const events = installWindow();
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
providerFactories: {
|
|
browser_camera: ({ onMessage, onState }) => ({
|
|
start() {
|
|
onState({ connected: true });
|
|
onMessage({
|
|
type: "skeleton",
|
|
timestamp_ms: 1000,
|
|
source: "test",
|
|
joints: [{ id: "left_wrist", x: 0.2, y: 0.3, confidence: 1 }],
|
|
bones: [],
|
|
});
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
|
|
const debugEvent = events.events.find((event) => event.type === "earth:motion-debug-frame");
|
|
expect(debugEvent?.detail.joints[0]).toEqual({
|
|
id: "left_wrist",
|
|
x: 0.2,
|
|
y: 0.3,
|
|
confidence: 1,
|
|
});
|
|
});
|
|
|
|
test("mock gesture message maps to Earth control callback", () => {
|
|
installWindow();
|
|
const rotations = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
onRotate: (...args) => rotations.push(args),
|
|
providerFactories: {
|
|
browser_camera: ({ onMessage }) => ({
|
|
start() {
|
|
onMessage({
|
|
type: "gesture",
|
|
gesture: "rotate_left",
|
|
confidence: 0.91,
|
|
intensity: 0.5,
|
|
});
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
|
|
expect(rotations[0][0]).toBe("horizontal");
|
|
expect(rotations[0][1]).toBe("left");
|
|
expect(rotations[0][2]).toBe(0.5);
|
|
});
|
|
|
|
test("recognition pause suppresses gestures and matched skeleton state", () => {
|
|
const events = installWindow();
|
|
const rotations = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
cooldownMs: 0,
|
|
onRotate: (...args) => rotations.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
window.dispatchEvent(new CustomEvent("earth:motion-recognition-pause", {
|
|
detail: { paused: true },
|
|
}));
|
|
adapter.handleMessage({
|
|
type: "gesture",
|
|
gesture: "rotate_left",
|
|
confidence: 0.91,
|
|
intensity: 0.5,
|
|
});
|
|
adapter.handleMessage({
|
|
type: "skeleton",
|
|
matched_gesture: "rotate_left",
|
|
confidence: 0.91,
|
|
joints: [],
|
|
bones: [],
|
|
});
|
|
|
|
const debugEvent = events.events.findLast?.((event) => event.type === "earth:motion-debug-frame") ||
|
|
events.events.filter((event) => event.type === "earth:motion-debug-frame").at(-1);
|
|
expect(rotations).toHaveLength(0);
|
|
expect(debugEvent?.detail.matchedGesture).toBeNull();
|
|
expect(debugEvent?.detail.confidence).toBe(0);
|
|
});
|
|
|
|
test("disabled gestures are ignored and can be re-enabled at runtime", () => {
|
|
installWindow();
|
|
const rotations = [];
|
|
const sentCommands = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
cooldownMs: 0,
|
|
enabledGestures: ["zoom_in"],
|
|
onRotate: (...args) => rotations.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
sendCommand(command, payload) {
|
|
sentCommands.push({ command, payload });
|
|
return { ok: true };
|
|
},
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
adapter.handleMessage({ type: "gesture", gesture: "rotate_left", confidence: 0.91 });
|
|
expect(rotations).toHaveLength(0);
|
|
|
|
adapter.setEnabledGestures(["rotate_left"]);
|
|
adapter.handleMessage({ type: "gesture", gesture: "rotate_left", confidence: 0.91 });
|
|
|
|
expect(rotations).toHaveLength(1);
|
|
expect(sentCommands.at(-1)).toEqual({
|
|
command: "set_enabled_gestures",
|
|
payload: { gestures: ["rotate_left"] },
|
|
});
|
|
});
|
|
|
|
test("mock vertical gesture and focus gesture use dedicated callbacks", () => {
|
|
installWindow();
|
|
const rotations = [];
|
|
const focuses = [];
|
|
const layers = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
cooldownMs: 0,
|
|
onRotate: (...args) => rotations.push(args),
|
|
onFocus: (...args) => focuses.push(args),
|
|
onLayer: (...args) => layers.push(args),
|
|
providerFactories: {
|
|
browser_camera: ({ onMessage }) => ({
|
|
start() {
|
|
onMessage({
|
|
type: "gesture",
|
|
gesture: "rotate_up",
|
|
confidence: 0.91,
|
|
intensity: 0.6,
|
|
});
|
|
onMessage({
|
|
type: "gesture",
|
|
gesture: "focus_next",
|
|
confidence: 0.91,
|
|
intensity: 0.8,
|
|
});
|
|
onMessage({
|
|
type: "gesture",
|
|
gesture: "layer_next",
|
|
confidence: 0.91,
|
|
intensity: 0.8,
|
|
});
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
|
|
expect(rotations[0][0]).toBe("vertical");
|
|
expect(rotations[0][1]).toBe("up");
|
|
expect(rotations[0][2]).toBe(0.6);
|
|
expect(focuses[0][0]).toBe("next");
|
|
expect(layers[0][0]).toBe("next");
|
|
});
|
|
|
|
test("continuous rotate can retrigger after the short cooldown", () => {
|
|
installWindow();
|
|
let now = 1000;
|
|
const rotations = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
nowFn: () => now,
|
|
onRotate: (...args) => rotations.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
adapter.handleMessage({ type: "gesture", gesture: "rotate_right", confidence: 0.91 });
|
|
now += 60;
|
|
adapter.handleMessage({ type: "gesture", gesture: "rotate_right", confidence: 0.91 });
|
|
now += 70;
|
|
adapter.handleMessage({ type: "gesture", gesture: "rotate_right", confidence: 0.91 });
|
|
|
|
expect(rotations).toHaveLength(2);
|
|
});
|
|
|
|
test("focus gestures share one 900ms cooldown group", () => {
|
|
installWindow();
|
|
let now = 1000;
|
|
const focuses = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
nowFn: () => now,
|
|
onFocus: (...args) => focuses.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
adapter.handleMessage({ type: "gesture", gesture: "focus_next", confidence: 0.91 });
|
|
now += 100;
|
|
adapter.handleMessage({ type: "gesture", gesture: "focus_prev", confidence: 0.91 });
|
|
now += 900;
|
|
adapter.handleMessage({ type: "gesture", gesture: "focus_prev", confidence: 0.91 });
|
|
|
|
expect(focuses.map((args) => args[0])).toEqual(["next", "prev"]);
|
|
});
|
|
|
|
test("layer gestures share one 1400ms cooldown group", () => {
|
|
installWindow();
|
|
let now = 1000;
|
|
const layers = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
nowFn: () => now,
|
|
onLayer: (...args) => layers.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
adapter.handleMessage({ type: "gesture", gesture: "layer_next", confidence: 0.91 });
|
|
now += 1200;
|
|
adapter.handleMessage({ type: "gesture", gesture: "layer_next", confidence: 0.91 });
|
|
now += 200;
|
|
adapter.handleMessage({ type: "gesture", gesture: "layer_prev", confidence: 0.91 });
|
|
|
|
expect(layers.map((args) => args[0])).toEqual(["next", "prev"]);
|
|
});
|
|
|
|
test("confirm uses a 1200ms cooldown", () => {
|
|
installWindow();
|
|
let now = 1000;
|
|
const confirms = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
nowFn: () => now,
|
|
onConfirm: (...args) => confirms.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
adapter.handleMessage({ type: "gesture", gesture: "confirm", confidence: 0.91 });
|
|
now += 1000;
|
|
adapter.handleMessage({ type: "gesture", gesture: "confirm", confidence: 0.91 });
|
|
now += 200;
|
|
adapter.handleMessage({ type: "gesture", gesture: "confirm", confidence: 0.91 });
|
|
|
|
expect(confirms).toHaveLength(2);
|
|
});
|
|
});
|
|
|
|
function createPoseJoints(overrides = {}) {
|
|
const base = {
|
|
left_ear: { id: "left_ear", x: 0.45, y: 0.2, confidence: 1 },
|
|
right_ear: { id: "right_ear", x: 0.55, y: 0.2, confidence: 1 },
|
|
left_shoulder: { id: "left_shoulder", x: 0.42, y: 0.5, confidence: 1 },
|
|
right_shoulder: { id: "right_shoulder", x: 0.58, y: 0.5, confidence: 1 },
|
|
left_elbow: { id: "left_elbow", x: 0.4, y: 0.62, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.6, y: 0.62, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.42, y: 0.65, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.58, y: 0.65, confidence: 1 },
|
|
};
|
|
return Object.values({
|
|
...base,
|
|
...overrides,
|
|
});
|
|
}
|
|
|
|
describe("browser camera gesture semantics", () => {
|
|
test("right-arm left-facing pattern maps to rotate_right", () => {
|
|
const current = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.5, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.4, y: 0.53, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("rotate_right");
|
|
});
|
|
|
|
test("right-arm upward terminal pattern maps to rotate_up", () => {
|
|
const current = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.68, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.71, y: 0.39, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("rotate_up");
|
|
});
|
|
|
|
test("right-arm terminal can float about 30 degrees while matching horizontal wave", () => {
|
|
const current = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.5, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.39, y: 0.56, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("rotate_right");
|
|
});
|
|
|
|
test("left-hand vertical movement switches motion layer", () => {
|
|
const previous = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.42, y: 0.48, confidence: 1 },
|
|
});
|
|
const current = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.42, y: 0.38, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, previous)?.gesture).toBe("layer_prev");
|
|
});
|
|
|
|
test("centered close hands no longer trigger confirm", () => {
|
|
const current = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.48, y: 0.6, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.52, y: 0.6, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)).toBeNull();
|
|
});
|
|
|
|
test("head tilt emits focus navigation gesture", () => {
|
|
const current = createPoseJoints({
|
|
left_ear: { id: "left_ear", x: 0.45, y: 0.24, confidence: 1 },
|
|
right_ear: { id: "right_ear", x: 0.55, y: 0.18, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("focus_prev");
|
|
});
|
|
|
|
test("holding right hand high does not continuously rotate", () => {
|
|
const current = createPoseJoints({
|
|
right_wrist: { id: "right_wrist", x: 0.58, y: 0.34, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)).toBeNull();
|
|
});
|
|
|
|
test("holding the same pattern only emits once until neutral", () => {
|
|
const state = {};
|
|
const pattern = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.5, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.4, y: 0.5, confidence: 1 },
|
|
});
|
|
const neutral = createPoseJoints();
|
|
|
|
expect(recognizeGesture(pattern, neutral, { state })?.gesture).toBe("rotate_right");
|
|
expect(recognizeGesture(pattern, pattern, { state })).toBeNull();
|
|
expect(recognizeGesture(neutral, pattern, { state })).toBeNull();
|
|
expect(recognizeGesture(pattern, neutral, { state })?.gesture).toBe("rotate_right");
|
|
});
|
|
|
|
test("hands resting below the shoulders do not rotate down", () => {
|
|
const current = createPoseJoints({
|
|
right_wrist: { id: "right_wrist", x: 0.58, y: 0.82, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)).toBeNull();
|
|
});
|
|
|
|
test("moving left hand near the chest does not trigger zoom out", () => {
|
|
const previous = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.38, y: 0.62, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.55, y: 0.62, confidence: 1 },
|
|
});
|
|
const current = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.49, y: 0.62, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.55, y: 0.62, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, previous)).toBeNull();
|
|
});
|
|
|
|
test("two hands opening from center trigger zoom in", () => {
|
|
const current = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.36, y: 0.5, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.34, y: 0.4, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.64, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.66, y: 0.4, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("zoom_in");
|
|
});
|
|
|
|
test("two hands opening with tilted wrists still trigger zoom in", () => {
|
|
const current = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.37, y: 0.51, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.31, y: 0.46, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.63, y: 0.51, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.69, y: 0.58, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("zoom_in");
|
|
});
|
|
|
|
test("open arms within a 30 degree vertical fan trigger zoom in", () => {
|
|
const upwardFan = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.35, y: 0.48, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.29, y: 0.43, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.65, y: 0.52, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.72, y: 0.57, confidence: 1 },
|
|
});
|
|
const downwardFan = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.35, y: 0.52, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.29, y: 0.57, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.65, y: 0.48, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.72, y: 0.43, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(upwardFan, upwardFan)?.gesture).toBe("zoom_in");
|
|
expect(recognizeGesture(downwardFan, downwardFan)?.gesture).toBe("zoom_in");
|
|
});
|
|
|
|
test("near zoom-in pose suppresses right-arm rotate while the second hand catches up", () => {
|
|
const current = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.4, y: 0.5, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.35, y: 0.58, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.64, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.7, y: 0.39, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)).toBeNull();
|
|
});
|
|
|
|
test("two hands closing toward center trigger zoom out", () => {
|
|
const current = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.34, y: 0.55, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.46, y: 0.58, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.66, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.54, y: 0.58, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("zoom_out");
|
|
});
|
|
|
|
// Two-arm spread starts asymmetrically: the right wrist crosses the rotate
|
|
// trigger threshold a frame or two before the left wrist catches up. The
|
|
// mirror-safe span-based pose detector recognises this frame as a spread
|
|
// and emits zoom_in instead of letting the stale single-arm rotate fire.
|
|
test("mid-spread emits zoom_in (not a stray right-arm rotate) while the left arm is still extending", () => {
|
|
const midSpread = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.66, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.72, y: 0.5, confidence: 1 },
|
|
left_elbow: { id: "left_elbow", x: 0.36, y: 0.5, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.38, y: 0.55, confidence: 1 },
|
|
});
|
|
|
|
const result = recognizeGesture(midSpread, midSpread);
|
|
expect(result?.gesture).toBe("zoom_in");
|
|
});
|
|
|
|
// Earliest-spread case: the left wrist has just started to lift toward
|
|
// shoulder height while still sitting at the body line. The right wrist has
|
|
// already crossed the rotate threshold. The mirror-safe pose detector
|
|
// recognises the wide span and fires zoom_in; the at-rest gate ensures
|
|
// rotate cannot fire in this configuration either.
|
|
test("early-spread frame fires zoom_in instead of a stray right-arm rotate", () => {
|
|
const earlySpread = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.65, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.72, y: 0.5, confidence: 1 },
|
|
left_elbow: { id: "left_elbow", x: 0.41, y: 0.55, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.42, y: 0.58, confidence: 1 },
|
|
});
|
|
|
|
const result = recognizeGesture(earlySpread, earlySpread);
|
|
expect(result?.gesture).toBe("zoom_in");
|
|
});
|
|
|
|
// The suppressor must not over-fire: a deliberate single right-arm wave
|
|
// with the left arm at rest still needs to map to a rotate gesture.
|
|
test("right-arm wave with the left arm at rest still triggers rotate_left", () => {
|
|
const wave = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.65, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.74, y: 0.5, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(wave, wave)?.gesture).toBe("rotate_left");
|
|
});
|
|
|
|
// Trend-based zoom: anti-symmetric wrist motion is the strongest signal of
|
|
// intent. Pose matching alone is brittle because MediaPipe keypoints
|
|
// jitter; tracking direction-of-motion catches the gesture as soon as it
|
|
// starts.
|
|
test("wrists drifting apart trigger zoom_in via trend detection", () => {
|
|
const previous = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.42, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.58, y: 0.55, confidence: 1 },
|
|
});
|
|
const current = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.38, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.62, y: 0.55, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, previous)?.gesture).toBe("zoom_in");
|
|
});
|
|
|
|
test("wrists drifting together trigger zoom_out via trend detection", () => {
|
|
const previous = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.30, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.70, y: 0.55, confidence: 1 },
|
|
});
|
|
const current = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.34, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.66, y: 0.55, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, previous)?.gesture).toBe("zoom_out");
|
|
});
|
|
|
|
// Single-arm motion (right wrist moving while left wrist is stationary)
|
|
// must NOT trip the trend zoom — only anti-symmetric motion of both
|
|
// wrists qualifies.
|
|
test("single-arm motion does not trigger trend-based zoom", () => {
|
|
const previous = createPoseJoints({
|
|
right_wrist: { id: "right_wrist", x: 0.58, y: 0.55, confidence: 1 },
|
|
});
|
|
const current = createPoseJoints({
|
|
right_wrist: { id: "right_wrist", x: 0.70, y: 0.55, confidence: 1 },
|
|
});
|
|
|
|
const result = recognizeGesture(current, previous);
|
|
expect(result?.gesture).not.toBe("zoom_in");
|
|
expect(result?.gesture).not.toBe("zoom_out");
|
|
});
|
|
|
|
// If the two wrists are at very different heights (one resting, one
|
|
// raised), trend detection must NOT fire — that's a one-arm gesture.
|
|
test("trend zoom requires the wrists to be at roughly the same height", () => {
|
|
const previous = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.42, y: 0.80, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.58, y: 0.30, confidence: 1 },
|
|
});
|
|
const current = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.38, y: 0.80, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.62, y: 0.30, confidence: 1 },
|
|
});
|
|
|
|
const result = recognizeGesture(current, previous);
|
|
expect(result?.gesture).not.toBe("zoom_in");
|
|
expect(result?.gesture).not.toBe("zoom_out");
|
|
});
|
|
|
|
// Zoom is a sustained gesture: while the user holds a spread T-pose the
|
|
// recognizer must keep emitting zoom_in every frame so the globe keeps
|
|
// zooming. This is unlike rotate, which should emit once per wave.
|
|
test("holding a spread pose emits zoom_in on every frame", () => {
|
|
const state = {};
|
|
const spread = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.36, y: 0.5, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.34, y: 0.4, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.64, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.66, y: 0.4, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(spread, spread, { state })?.gesture).toBe("zoom_in");
|
|
expect(recognizeGesture(spread, spread, { state })?.gesture).toBe("zoom_in");
|
|
expect(recognizeGesture(spread, spread, { state })?.gesture).toBe("zoom_in");
|
|
});
|
|
|
|
// Mirror-safe trend: on a non-mirrored camera feed the subject's anatomical
|
|
// left arm appears on the image right (left_shoulder.x > right_shoulder.x).
|
|
// Spreading the arms must still fire zoom_in (not zoom_out) because the
|
|
// span between the wrists grows regardless of camera orientation.
|
|
test("non-mirrored camera: spreading wrists still triggers zoom_in", () => {
|
|
const previous = createPoseJoints({
|
|
// Swapped layout: anatomical left on image right
|
|
left_shoulder: { id: "left_shoulder", x: 0.58, y: 0.5, confidence: 1 },
|
|
right_shoulder: { id: "right_shoulder", x: 0.42, y: 0.5, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.58, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.42, y: 0.55, confidence: 1 },
|
|
});
|
|
const current = createPoseJoints({
|
|
left_shoulder: { id: "left_shoulder", x: 0.58, y: 0.5, confidence: 1 },
|
|
right_shoulder: { id: "right_shoulder", x: 0.42, y: 0.5, confidence: 1 },
|
|
// Subject's left arm extending right in the image; subject's right
|
|
// arm extending left in the image. Span widens either way.
|
|
left_wrist: { id: "left_wrist", x: 0.62, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.38, y: 0.55, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, previous)?.gesture).toBe("zoom_in");
|
|
});
|
|
|
|
// Mirror-safe trend: on the same non-mirrored layout, hands coming together
|
|
// must still fire zoom_out.
|
|
test("non-mirrored camera: closing wrists still triggers zoom_out", () => {
|
|
const previous = createPoseJoints({
|
|
left_shoulder: { id: "left_shoulder", x: 0.58, y: 0.5, confidence: 1 },
|
|
right_shoulder: { id: "right_shoulder", x: 0.42, y: 0.5, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.70, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.30, y: 0.55, confidence: 1 },
|
|
});
|
|
const current = createPoseJoints({
|
|
left_shoulder: { id: "left_shoulder", x: 0.58, y: 0.5, confidence: 1 },
|
|
right_shoulder: { id: "right_shoulder", x: 0.42, y: 0.5, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.66, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.34, y: 0.55, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, previous)?.gesture).toBe("zoom_out");
|
|
});
|
|
|
|
// Zoom_out must also be sustained — closing the hands and holding
|
|
// continues to zoom out.
|
|
test("holding a closed pose emits zoom_out on every frame", () => {
|
|
const state = {};
|
|
const closed = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.34, y: 0.55, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.46, y: 0.58, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.66, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.54, y: 0.58, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(closed, closed, { state })?.gesture).toBe("zoom_out");
|
|
expect(recognizeGesture(closed, closed, { state })?.gesture).toBe("zoom_out");
|
|
});
|
|
});
|
|
|
|
describe("browser camera provider", () => {
|
|
test("starts with mocked getUserMedia and emits active state", async () => {
|
|
installWindow();
|
|
installDocument();
|
|
const stopped = [];
|
|
const states = [];
|
|
const videoSources = [];
|
|
const provider = createBrowserCameraProvider({
|
|
mediaDevices: {
|
|
getUserMedia: () =>
|
|
Promise.resolve({
|
|
getTracks: () => [{ stop: () => stopped.push("camera") }],
|
|
}),
|
|
},
|
|
recognizerFactory: () =>
|
|
Promise.resolve({
|
|
recognize: () => [],
|
|
close() {},
|
|
}),
|
|
requestAnimationFrameFn: () => 0,
|
|
onState: (state) => states.push(state),
|
|
onVideoSource: (source) => videoSources.push(source),
|
|
});
|
|
|
|
await provider.start();
|
|
provider.stop();
|
|
|
|
expect(states.some((state) => state.connected === true)).toBe(true);
|
|
expect(stopped).toEqual(["camera"]);
|
|
expect(videoSources.some((source) => source.active === true)).toBe(true);
|
|
expect(videoSources.at(-1)).toEqual({
|
|
provider: "browser_camera",
|
|
source: null,
|
|
active: false,
|
|
});
|
|
});
|
|
|
|
test("reports permission errors without falling back to dry-run", async () => {
|
|
installWindow();
|
|
installDocument();
|
|
const statuses = [];
|
|
const closed = [];
|
|
const provider = createBrowserCameraProvider({
|
|
mediaDevices: {
|
|
getUserMedia: () =>
|
|
Promise.reject(Object.assign(new Error("denied"), { name: "NotAllowedError" })),
|
|
},
|
|
recognizerFactory: () =>
|
|
Promise.resolve({
|
|
recognize: () => [],
|
|
close: () => closed.push("recognizer"),
|
|
}),
|
|
onStatus: (message, type) => statuses.push({ message, type }),
|
|
});
|
|
|
|
const started = await provider.start();
|
|
|
|
expect(started).toBe(false);
|
|
expect(statuses[0]).toEqual({
|
|
message: "浏览器摄像头权限被拒绝",
|
|
type: "error",
|
|
});
|
|
expect(closed).toEqual(["recognizer"]);
|
|
});
|
|
|
|
test("reports model load errors separately from camera permission", async () => {
|
|
installWindow();
|
|
installDocument();
|
|
const statuses = [];
|
|
const requested = [];
|
|
const provider = createBrowserCameraProvider({
|
|
mediaDevices: {
|
|
getUserMedia: () => {
|
|
requested.push("camera");
|
|
return Promise.resolve({
|
|
getTracks: () => [],
|
|
});
|
|
},
|
|
},
|
|
recognizerFactory: () => Promise.reject(new Error("model unavailable")),
|
|
onStatus: (message, type) => statuses.push({ message, type }),
|
|
});
|
|
|
|
const started = await provider.start();
|
|
|
|
expect(started).toBe(false);
|
|
expect(requested).toEqual([]);
|
|
expect(statuses[0]).toEqual({
|
|
message: "浏览器动捕模型加载失败: model unavailable",
|
|
type: "error",
|
|
});
|
|
});
|
|
});
|