625 lines
19 KiB
JavaScript
625 lines
19 KiB
JavaScript
import { describe, expect, test } from "bun:test";
|
|
|
|
import {
|
|
createMotionControlAdapter,
|
|
normalizeMotionProvider,
|
|
} from "./motion-control.js";
|
|
import {
|
|
createBrowserCameraProvider,
|
|
recognizeGesture,
|
|
} from "./motion-browser-provider.js";
|
|
|
|
class TestEventTarget {
|
|
constructor() {
|
|
this.events = [];
|
|
this.listeners = new Map();
|
|
}
|
|
|
|
dispatchEvent(event) {
|
|
this.events.push(event);
|
|
(this.listeners.get(event.type) || []).forEach((listener) => listener(event));
|
|
return true;
|
|
}
|
|
|
|
addEventListener(type, listener) {
|
|
const listeners = this.listeners.get(type) || [];
|
|
listeners.push(listener);
|
|
this.listeners.set(type, listeners);
|
|
}
|
|
|
|
removeEventListener(type, listener) {
|
|
const listeners = this.listeners.get(type) || [];
|
|
this.listeners.set(type, listeners.filter((item) => item !== listener));
|
|
}
|
|
}
|
|
|
|
function installWindow(search = "") {
|
|
const target = new TestEventTarget();
|
|
globalThis.CustomEvent = class CustomEvent {
|
|
constructor(type, options = {}) {
|
|
this.type = type;
|
|
this.detail = options.detail;
|
|
}
|
|
};
|
|
globalThis.window = {
|
|
location: {
|
|
search,
|
|
hostname: "localhost",
|
|
},
|
|
isSecureContext: true,
|
|
localStorage: {
|
|
getItem() {
|
|
return null;
|
|
},
|
|
},
|
|
dispatchEvent: target.dispatchEvent.bind(target),
|
|
addEventListener: target.addEventListener.bind(target),
|
|
removeEventListener: target.removeEventListener.bind(target),
|
|
};
|
|
return target;
|
|
}
|
|
|
|
function installDocument() {
|
|
const tracks = [];
|
|
globalThis.document = {
|
|
body: {
|
|
appendChild() {},
|
|
},
|
|
createElement(tagName) {
|
|
expect(tagName).toBe("video");
|
|
return {
|
|
muted: false,
|
|
playsInline: false,
|
|
autoplay: false,
|
|
readyState: 4,
|
|
style: {},
|
|
srcObject: null,
|
|
videoWidth: 640,
|
|
videoHeight: 360,
|
|
addEventListener() {},
|
|
play: () => Promise.resolve(),
|
|
pause() {},
|
|
remove() {},
|
|
};
|
|
},
|
|
};
|
|
return tracks;
|
|
}
|
|
|
|
describe("motion-control provider manager", () => {
|
|
test("normalizes browser and agent provider aliases", () => {
|
|
expect(normalizeMotionProvider("browser")).toBe("browser_camera");
|
|
expect(normalizeMotionProvider("agent")).toBe("motion_agent");
|
|
expect(normalizeMotionProvider("motion_agent")).toBe("motion_agent");
|
|
expect(normalizeMotionProvider("unknown")).toBe("browser_camera");
|
|
});
|
|
|
|
test("mock skeleton message dispatches debug frame event", () => {
|
|
const events = installWindow();
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
providerFactories: {
|
|
browser_camera: ({ onMessage, onState }) => ({
|
|
start() {
|
|
onState({ connected: true });
|
|
onMessage({
|
|
type: "skeleton",
|
|
timestamp_ms: 1000,
|
|
source: "test",
|
|
joints: [{ id: "left_wrist", x: 0.2, y: 0.3, confidence: 1 }],
|
|
bones: [],
|
|
});
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
|
|
const debugEvent = events.events.find((event) => event.type === "earth:motion-debug-frame");
|
|
expect(debugEvent?.detail.joints[0]).toEqual({
|
|
id: "left_wrist",
|
|
x: 0.2,
|
|
y: 0.3,
|
|
confidence: 1,
|
|
});
|
|
});
|
|
|
|
test("mock gesture message maps to Earth control callback", () => {
|
|
installWindow();
|
|
const rotations = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
onRotate: (...args) => rotations.push(args),
|
|
providerFactories: {
|
|
browser_camera: ({ onMessage }) => ({
|
|
start() {
|
|
onMessage({
|
|
type: "gesture",
|
|
gesture: "rotate_left",
|
|
confidence: 0.91,
|
|
intensity: 0.5,
|
|
});
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
|
|
expect(rotations[0][0]).toBe("horizontal");
|
|
expect(rotations[0][1]).toBe("left");
|
|
expect(rotations[0][2]).toBe(0.5);
|
|
});
|
|
|
|
test("recognition pause suppresses gestures and matched skeleton state", () => {
|
|
const events = installWindow();
|
|
const rotations = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
cooldownMs: 0,
|
|
onRotate: (...args) => rotations.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
window.dispatchEvent(new CustomEvent("earth:motion-recognition-pause", {
|
|
detail: { paused: true },
|
|
}));
|
|
adapter.handleMessage({
|
|
type: "gesture",
|
|
gesture: "rotate_left",
|
|
confidence: 0.91,
|
|
intensity: 0.5,
|
|
});
|
|
adapter.handleMessage({
|
|
type: "skeleton",
|
|
matched_gesture: "rotate_left",
|
|
confidence: 0.91,
|
|
joints: [],
|
|
bones: [],
|
|
});
|
|
|
|
const debugEvent = events.events.findLast?.((event) => event.type === "earth:motion-debug-frame") ||
|
|
events.events.filter((event) => event.type === "earth:motion-debug-frame").at(-1);
|
|
expect(rotations).toHaveLength(0);
|
|
expect(debugEvent?.detail.matchedGesture).toBeNull();
|
|
expect(debugEvent?.detail.confidence).toBe(0);
|
|
});
|
|
|
|
test("mock vertical gesture and focus gesture use dedicated callbacks", () => {
|
|
installWindow();
|
|
const rotations = [];
|
|
const focuses = [];
|
|
const layers = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
cooldownMs: 0,
|
|
onRotate: (...args) => rotations.push(args),
|
|
onFocus: (...args) => focuses.push(args),
|
|
onLayer: (...args) => layers.push(args),
|
|
providerFactories: {
|
|
browser_camera: ({ onMessage }) => ({
|
|
start() {
|
|
onMessage({
|
|
type: "gesture",
|
|
gesture: "rotate_up",
|
|
confidence: 0.91,
|
|
intensity: 0.6,
|
|
});
|
|
onMessage({
|
|
type: "gesture",
|
|
gesture: "focus_next",
|
|
confidence: 0.91,
|
|
intensity: 0.8,
|
|
});
|
|
onMessage({
|
|
type: "gesture",
|
|
gesture: "layer_next",
|
|
confidence: 0.91,
|
|
intensity: 0.8,
|
|
});
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
|
|
expect(rotations[0][0]).toBe("vertical");
|
|
expect(rotations[0][1]).toBe("up");
|
|
expect(rotations[0][2]).toBe(0.6);
|
|
expect(focuses[0][0]).toBe("next");
|
|
expect(layers[0][0]).toBe("next");
|
|
});
|
|
|
|
test("continuous rotate can retrigger after the short cooldown", () => {
|
|
installWindow();
|
|
let now = 1000;
|
|
const rotations = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
nowFn: () => now,
|
|
onRotate: (...args) => rotations.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
adapter.handleMessage({ type: "gesture", gesture: "rotate_right", confidence: 0.91 });
|
|
now += 60;
|
|
adapter.handleMessage({ type: "gesture", gesture: "rotate_right", confidence: 0.91 });
|
|
now += 70;
|
|
adapter.handleMessage({ type: "gesture", gesture: "rotate_right", confidence: 0.91 });
|
|
|
|
expect(rotations).toHaveLength(2);
|
|
});
|
|
|
|
test("focus gestures share one 900ms cooldown group", () => {
|
|
installWindow();
|
|
let now = 1000;
|
|
const focuses = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
nowFn: () => now,
|
|
onFocus: (...args) => focuses.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
adapter.handleMessage({ type: "gesture", gesture: "focus_next", confidence: 0.91 });
|
|
now += 100;
|
|
adapter.handleMessage({ type: "gesture", gesture: "focus_prev", confidence: 0.91 });
|
|
now += 900;
|
|
adapter.handleMessage({ type: "gesture", gesture: "focus_prev", confidence: 0.91 });
|
|
|
|
expect(focuses.map((args) => args[0])).toEqual(["next", "prev"]);
|
|
});
|
|
|
|
test("layer gestures share one 1400ms cooldown group", () => {
|
|
installWindow();
|
|
let now = 1000;
|
|
const layers = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
nowFn: () => now,
|
|
onLayer: (...args) => layers.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
adapter.handleMessage({ type: "gesture", gesture: "layer_next", confidence: 0.91 });
|
|
now += 1200;
|
|
adapter.handleMessage({ type: "gesture", gesture: "layer_next", confidence: 0.91 });
|
|
now += 200;
|
|
adapter.handleMessage({ type: "gesture", gesture: "layer_prev", confidence: 0.91 });
|
|
|
|
expect(layers.map((args) => args[0])).toEqual(["next", "prev"]);
|
|
});
|
|
|
|
test("confirm uses a 1200ms cooldown", () => {
|
|
installWindow();
|
|
let now = 1000;
|
|
const confirms = [];
|
|
const adapter = createMotionControlAdapter({
|
|
enabled: true,
|
|
nowFn: () => now,
|
|
onConfirm: (...args) => confirms.push(args),
|
|
providerFactories: {
|
|
browser_camera: () => ({
|
|
start() {
|
|
return true;
|
|
},
|
|
stop() {},
|
|
isConnected: () => true,
|
|
}),
|
|
},
|
|
});
|
|
|
|
adapter.start();
|
|
adapter.handleMessage({ type: "gesture", gesture: "confirm", confidence: 0.91 });
|
|
now += 1000;
|
|
adapter.handleMessage({ type: "gesture", gesture: "confirm", confidence: 0.91 });
|
|
now += 200;
|
|
adapter.handleMessage({ type: "gesture", gesture: "confirm", confidence: 0.91 });
|
|
|
|
expect(confirms).toHaveLength(2);
|
|
});
|
|
});
|
|
|
|
function createPoseJoints(overrides = {}) {
|
|
const base = {
|
|
left_ear: { id: "left_ear", x: 0.45, y: 0.2, confidence: 1 },
|
|
right_ear: { id: "right_ear", x: 0.55, y: 0.2, confidence: 1 },
|
|
left_shoulder: { id: "left_shoulder", x: 0.42, y: 0.5, confidence: 1 },
|
|
right_shoulder: { id: "right_shoulder", x: 0.58, y: 0.5, confidence: 1 },
|
|
left_elbow: { id: "left_elbow", x: 0.4, y: 0.62, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.6, y: 0.62, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.42, y: 0.65, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.58, y: 0.65, confidence: 1 },
|
|
};
|
|
return Object.values({
|
|
...base,
|
|
...overrides,
|
|
});
|
|
}
|
|
|
|
describe("browser camera gesture semantics", () => {
|
|
test("right-arm left-facing pattern maps to rotate_right", () => {
|
|
const current = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.5, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.4, y: 0.53, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("rotate_right");
|
|
});
|
|
|
|
test("right-arm upward terminal pattern maps to rotate_up", () => {
|
|
const current = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.68, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.71, y: 0.39, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("rotate_up");
|
|
});
|
|
|
|
test("right-arm terminal can float about 30 degrees while matching horizontal wave", () => {
|
|
const current = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.5, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.39, y: 0.56, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("rotate_right");
|
|
});
|
|
|
|
test("left-hand vertical movement switches motion layer", () => {
|
|
const previous = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.42, y: 0.48, confidence: 1 },
|
|
});
|
|
const current = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.42, y: 0.38, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, previous)?.gesture).toBe("layer_prev");
|
|
});
|
|
|
|
test("centered close hands no longer trigger confirm", () => {
|
|
const current = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.48, y: 0.6, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.52, y: 0.6, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)).toBeNull();
|
|
});
|
|
|
|
test("head tilt emits focus navigation gesture", () => {
|
|
const current = createPoseJoints({
|
|
left_ear: { id: "left_ear", x: 0.45, y: 0.24, confidence: 1 },
|
|
right_ear: { id: "right_ear", x: 0.55, y: 0.18, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("focus_prev");
|
|
});
|
|
|
|
test("holding right hand high does not continuously rotate", () => {
|
|
const current = createPoseJoints({
|
|
right_wrist: { id: "right_wrist", x: 0.58, y: 0.34, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)).toBeNull();
|
|
});
|
|
|
|
test("holding the same pattern only emits once until neutral", () => {
|
|
const state = {};
|
|
const pattern = createPoseJoints({
|
|
right_elbow: { id: "right_elbow", x: 0.5, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.4, y: 0.5, confidence: 1 },
|
|
});
|
|
const neutral = createPoseJoints();
|
|
|
|
expect(recognizeGesture(pattern, neutral, { state })?.gesture).toBe("rotate_right");
|
|
expect(recognizeGesture(pattern, pattern, { state })).toBeNull();
|
|
expect(recognizeGesture(neutral, pattern, { state })).toBeNull();
|
|
expect(recognizeGesture(pattern, neutral, { state })?.gesture).toBe("rotate_right");
|
|
});
|
|
|
|
test("hands resting below the shoulders do not rotate down", () => {
|
|
const current = createPoseJoints({
|
|
right_wrist: { id: "right_wrist", x: 0.58, y: 0.82, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)).toBeNull();
|
|
});
|
|
|
|
test("moving left hand near the chest does not trigger zoom out", () => {
|
|
const previous = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.38, y: 0.62, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.55, y: 0.62, confidence: 1 },
|
|
});
|
|
const current = createPoseJoints({
|
|
left_wrist: { id: "left_wrist", x: 0.49, y: 0.62, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.55, y: 0.62, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, previous)).toBeNull();
|
|
});
|
|
|
|
test("two hands opening from center trigger zoom in", () => {
|
|
const current = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.36, y: 0.5, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.34, y: 0.4, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.64, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.66, y: 0.4, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("zoom_in");
|
|
});
|
|
|
|
test("two hands opening with tilted wrists still trigger zoom in", () => {
|
|
const current = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.37, y: 0.51, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.31, y: 0.46, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.63, y: 0.51, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.69, y: 0.58, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("zoom_in");
|
|
});
|
|
|
|
test("near zoom-in pose suppresses right-arm rotate while the second hand catches up", () => {
|
|
const current = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.4, y: 0.5, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.35, y: 0.58, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.64, y: 0.5, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.7, y: 0.39, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)).toBeNull();
|
|
});
|
|
|
|
test("two hands closing toward center trigger zoom out", () => {
|
|
const current = createPoseJoints({
|
|
left_elbow: { id: "left_elbow", x: 0.34, y: 0.55, confidence: 1 },
|
|
left_wrist: { id: "left_wrist", x: 0.46, y: 0.58, confidence: 1 },
|
|
right_elbow: { id: "right_elbow", x: 0.66, y: 0.55, confidence: 1 },
|
|
right_wrist: { id: "right_wrist", x: 0.54, y: 0.58, confidence: 1 },
|
|
});
|
|
|
|
expect(recognizeGesture(current, current)?.gesture).toBe("zoom_out");
|
|
});
|
|
});
|
|
|
|
describe("browser camera provider", () => {
|
|
test("starts with mocked getUserMedia and emits active state", async () => {
|
|
installWindow();
|
|
installDocument();
|
|
const stopped = [];
|
|
const states = [];
|
|
const videoSources = [];
|
|
const provider = createBrowserCameraProvider({
|
|
mediaDevices: {
|
|
getUserMedia: () =>
|
|
Promise.resolve({
|
|
getTracks: () => [{ stop: () => stopped.push("camera") }],
|
|
}),
|
|
},
|
|
recognizerFactory: () =>
|
|
Promise.resolve({
|
|
recognize: () => [],
|
|
close() {},
|
|
}),
|
|
requestAnimationFrameFn: () => 0,
|
|
onState: (state) => states.push(state),
|
|
onVideoSource: (source) => videoSources.push(source),
|
|
});
|
|
|
|
await provider.start();
|
|
provider.stop();
|
|
|
|
expect(states.some((state) => state.connected === true)).toBe(true);
|
|
expect(stopped).toEqual(["camera"]);
|
|
expect(videoSources.some((source) => source.active === true)).toBe(true);
|
|
expect(videoSources.at(-1)).toEqual({
|
|
provider: "browser_camera",
|
|
source: null,
|
|
active: false,
|
|
});
|
|
});
|
|
|
|
test("reports permission errors without falling back to dry-run", async () => {
|
|
installWindow();
|
|
installDocument();
|
|
const statuses = [];
|
|
const closed = [];
|
|
const provider = createBrowserCameraProvider({
|
|
mediaDevices: {
|
|
getUserMedia: () =>
|
|
Promise.reject(Object.assign(new Error("denied"), { name: "NotAllowedError" })),
|
|
},
|
|
recognizerFactory: () =>
|
|
Promise.resolve({
|
|
recognize: () => [],
|
|
close: () => closed.push("recognizer"),
|
|
}),
|
|
onStatus: (message, type) => statuses.push({ message, type }),
|
|
});
|
|
|
|
const started = await provider.start();
|
|
|
|
expect(started).toBe(false);
|
|
expect(statuses[0]).toEqual({
|
|
message: "浏览器摄像头权限被拒绝",
|
|
type: "error",
|
|
});
|
|
expect(closed).toEqual(["recognizer"]);
|
|
});
|
|
|
|
test("reports model load errors separately from camera permission", async () => {
|
|
installWindow();
|
|
installDocument();
|
|
const statuses = [];
|
|
const requested = [];
|
|
const provider = createBrowserCameraProvider({
|
|
mediaDevices: {
|
|
getUserMedia: () => {
|
|
requested.push("camera");
|
|
return Promise.resolve({
|
|
getTracks: () => [],
|
|
});
|
|
},
|
|
},
|
|
recognizerFactory: () => Promise.reject(new Error("model unavailable")),
|
|
onStatus: (message, type) => statuses.push({ message, type }),
|
|
});
|
|
|
|
const started = await provider.start();
|
|
|
|
expect(started).toBe(false);
|
|
expect(requested).toEqual([]);
|
|
expect(statuses[0]).toEqual({
|
|
message: "浏览器动捕模型加载失败: model unavailable",
|
|
type: "error",
|
|
});
|
|
});
|
|
});
|