678 lines
24 KiB
JavaScript
678 lines
24 KiB
JavaScript
const MEDIAPIPE_TASKS_VERSION = "0.10.35";
|
|
const MEDIAPIPE_TASKS_URLS = [
|
|
`https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MEDIAPIPE_TASKS_VERSION}/vision_bundle.mjs`,
|
|
`https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MEDIAPIPE_TASKS_VERSION}`,
|
|
`https://unpkg.com/@mediapipe/tasks-vision@${MEDIAPIPE_TASKS_VERSION}/vision_bundle.mjs`,
|
|
];
|
|
const MEDIAPIPE_WASM_URL = `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MEDIAPIPE_TASKS_VERSION}/wasm`;
|
|
const POSE_MODEL_URL =
|
|
"https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/latest/pose_landmarker_lite.task";
|
|
const FRAME_INTERVAL_MS = 66;
|
|
const GESTURE_COOLDOWN_MS = 420;
|
|
const VIDEO_METADATA_TIMEOUT_MS = 900;
|
|
const LEFT_WRIST_LAYER_DELTA_Y = 0.05;
|
|
const HEAD_TILT_DELTA_Y = 0.035;
|
|
const ARM_PATTERN_TERMINAL_TOLERANCE_DEG = 32;
|
|
const ARM_PATTERN_UPPER_TOLERANCE_DEG = 34;
|
|
const ARM_PATTERN_MIN_SEGMENT = 0.045;
|
|
const ARM_PATTERN_MIN_SIDE_REACH = 0.06;
|
|
const ARM_PATTERN_MIN_VERTICAL_REACH = 0.055;
|
|
const MIN_GESTURE_INTENSITY = 0.45;
|
|
const ARM_PATTERN_INTENSITY_SCALE = 5;
|
|
const WRIST_LAYER_INTENSITY_SCALE = 9;
|
|
const HEAD_TILT_INTENSITY_SCALE = 12;
|
|
const ZOOM_CLOSE_WRIST_SPREAD_FACTOR = 1.28;
|
|
const ZOOM_SUPPRESS_WRIST_SPREAD_FACTOR = 1.18;
|
|
// Trend-based zoom detection. Pose matching is brittle because MediaPipe
|
|
// keypoints jitter and the absolute "T-pose" pattern only matches in a
|
|
// narrow window. Track frame-to-frame motion instead: if both wrists are
|
|
// moving anti-symmetrically along the x axis (one moving outward, the other
|
|
// moving outward in the opposite direction), the user's intent is a zoom,
|
|
// regardless of where exactly the wrists end up.
|
|
const ZOOM_TREND_MIN_WRIST_DELTA = 0.010;
|
|
const ZOOM_TREND_HEIGHT_TOLERANCE = 0.18;
|
|
const ZOOM_TREND_INTENSITY_SCALE = 14;
|
|
const ZOOM_TREND_MIN_INTENSITY = 0.6;
|
|
// Left wrist must hang at least this far below the shoulder line for the
|
|
// arm to count as "at rest" -- distinguishes a deliberate single right-arm
|
|
// rotate from any two-arm or chest-height gesture in flight.
|
|
const LEFT_ARM_REST_HANGING_BELOW_SHOULDER = 0.13;
|
|
// Sustained pose-hold thresholds for continuous zoom emission while the
|
|
// user keeps their arms in a spread / closed pose. Mirror-safe (all checks
|
|
// are span-based, not direction-based) so they work regardless of whether
|
|
// the camera feed is mirrored.
|
|
const ZOOM_HOLD_HEIGHT_TOLERANCE = 0.18;
|
|
const ZOOM_HOLD_WRIST_BELOW_SHOULDER_LIMIT = 0.10;
|
|
const ZOOM_HOLD_SPREAD_FACTOR = 1.30;
|
|
const ZOOM_HOLD_CLOSE_FACTOR = 0.85;
|
|
const ZOOM_HOLD_ELBOW_OUT_FACTOR = 0.25;
|
|
const CAMERA_CONSTRAINTS = {
|
|
video: {
|
|
facingMode: "user",
|
|
width: { ideal: 1280 },
|
|
height: { ideal: 720 },
|
|
},
|
|
audio: false,
|
|
};
|
|
const POSE_JOINTS = [
|
|
[0, "nose"],
|
|
[7, "left_ear"],
|
|
[8, "right_ear"],
|
|
[11, "left_shoulder"],
|
|
[12, "right_shoulder"],
|
|
[13, "left_elbow"],
|
|
[14, "right_elbow"],
|
|
[15, "left_wrist"],
|
|
[16, "right_wrist"],
|
|
];
|
|
const POSE_BONES = [
|
|
["left_shoulder", "left_elbow"],
|
|
["left_elbow", "left_wrist"],
|
|
["right_shoulder", "right_elbow"],
|
|
["right_elbow", "right_wrist"],
|
|
["left_shoulder", "right_shoulder"],
|
|
];
|
|
|
|
function nowMs() {
|
|
return Math.round(performance?.now?.() || Date.now());
|
|
}
|
|
|
|
function wallClockMs() {
|
|
return Date.now();
|
|
}
|
|
|
|
function waitForVideoMetadata(video) {
|
|
if (video.videoWidth > 0 && video.videoHeight > 0) {
|
|
return Promise.resolve();
|
|
}
|
|
return new Promise((resolve) => {
|
|
const done = () => resolve();
|
|
video.addEventListener?.("loadedmetadata", done, { once: true });
|
|
video.addEventListener?.("canplay", done, { once: true });
|
|
setTimeout(done, VIDEO_METADATA_TIMEOUT_MS);
|
|
});
|
|
}
|
|
|
|
function getUserMediaErrorMessage(error) {
|
|
if (error?.name === "NotAllowedError" || error?.name === "PermissionDeniedError") {
|
|
return "浏览器摄像头权限被拒绝";
|
|
}
|
|
if (error?.name === "NotFoundError" || error?.name === "DevicesNotFoundError") {
|
|
return "没有找到可用摄像头";
|
|
}
|
|
if (error?.name === "NotReadableError") {
|
|
return "摄像头正被其他程序占用";
|
|
}
|
|
return `浏览器摄像头启动失败: ${error?.message || String(error)}`;
|
|
}
|
|
|
|
function canUseBrowserCamera(mediaDevices) {
|
|
return Boolean(
|
|
mediaDevices &&
|
|
typeof mediaDevices.getUserMedia === "function",
|
|
);
|
|
}
|
|
|
|
function isSecureCameraContext() {
|
|
if (typeof window === "undefined") return false;
|
|
const hostname = window.location?.hostname || "";
|
|
return Boolean(window.isSecureContext || hostname === "localhost" || hostname === "127.0.0.1");
|
|
}
|
|
|
|
function normalizePoseLandmarks(landmarks = []) {
|
|
return POSE_JOINTS.map(([index, id]) => {
|
|
const point = landmarks[index];
|
|
if (!point) return null;
|
|
const x = Number(point.x);
|
|
const y = Number(point.y);
|
|
if (!Number.isFinite(x) || !Number.isFinite(y)) return null;
|
|
return {
|
|
id,
|
|
x: Math.max(0, Math.min(1, x)),
|
|
y: Math.max(0, Math.min(1, y)),
|
|
confidence: Math.max(0, Math.min(1, Number(point.visibility ?? point.presence ?? 1))),
|
|
};
|
|
}).filter(Boolean);
|
|
}
|
|
|
|
function getJoint(joints, id) {
|
|
return joints.find((joint) => joint.id === id) || null;
|
|
}
|
|
|
|
function vectorBetween(start, end) {
|
|
if (!start || !end) return null;
|
|
const dx = end.x - start.x;
|
|
const dy = end.y - start.y;
|
|
return {
|
|
dx,
|
|
dy,
|
|
length: Math.hypot(dx, dy),
|
|
};
|
|
}
|
|
|
|
function vectorAngleDeg(vector) {
|
|
return Math.atan2(vector.dy, vector.dx) * 180 / Math.PI;
|
|
}
|
|
|
|
function normalizeAngleDelta(angle, target) {
|
|
let delta = angle - target;
|
|
while (delta > 180) delta -= 360;
|
|
while (delta < -180) delta += 360;
|
|
return Math.abs(delta);
|
|
}
|
|
|
|
function isAngleNear(angle, target, toleranceDeg) {
|
|
return normalizeAngleDelta(angle, target) <= toleranceDeg;
|
|
}
|
|
|
|
function isHorizontalArm(upperVector) {
|
|
if (!upperVector || upperVector.length < ARM_PATTERN_MIN_SEGMENT) return false;
|
|
const angle = vectorAngleDeg(upperVector);
|
|
return (
|
|
isAngleNear(angle, 0, ARM_PATTERN_UPPER_TOLERANCE_DEG) ||
|
|
isAngleNear(angle, 180, ARM_PATTERN_UPPER_TOLERANCE_DEG)
|
|
);
|
|
}
|
|
|
|
function isTerminalToward(vector, targetAngle) {
|
|
if (!vector || vector.length < ARM_PATTERN_MIN_SEGMENT) return false;
|
|
return isAngleNear(vectorAngleDeg(vector), targetAngle, ARM_PATTERN_TERMINAL_TOLERANCE_DEG);
|
|
}
|
|
|
|
function getRightArmPattern(rightShoulder, rightElbow, rightWrist) {
|
|
const upper = vectorBetween(rightShoulder, rightElbow);
|
|
const terminal = vectorBetween(rightElbow, rightWrist);
|
|
if (!upper || !terminal) return null;
|
|
const intensity = Math.min(1, Math.max(MIN_GESTURE_INTENSITY, terminal.length * ARM_PATTERN_INTENSITY_SCALE));
|
|
|
|
if (isTerminalToward(terminal, 180) && rightWrist.x < rightShoulder.x - ARM_PATTERN_MIN_SIDE_REACH) {
|
|
return { gesture: "rotate_right", confidence: 0.82, intensity };
|
|
}
|
|
if (isTerminalToward(terminal, 0) && rightWrist.x > rightShoulder.x + ARM_PATTERN_MIN_SIDE_REACH) {
|
|
return { gesture: "rotate_left", confidence: 0.82, intensity };
|
|
}
|
|
if (isHorizontalArm(upper) && isTerminalToward(terminal, -90) && rightWrist.y < rightElbow.y - ARM_PATTERN_MIN_VERTICAL_REACH) {
|
|
return { gesture: "rotate_up", confidence: 0.8, intensity };
|
|
}
|
|
if (isHorizontalArm(upper) && isTerminalToward(terminal, 90) && rightWrist.y > rightElbow.y + ARM_PATTERN_MIN_VERTICAL_REACH) {
|
|
return { gesture: "rotate_down", confidence: 0.8, intensity };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function isZoomCandidatePose(leftShoulder, leftElbow, leftWrist, rightShoulder, rightElbow, rightWrist, shoulderWidth) {
|
|
const wristsApart = Math.abs(rightWrist.x - leftWrist.x);
|
|
const bothHandsOutside =
|
|
leftWrist.x < leftElbow.x - ARM_PATTERN_MIN_SIDE_REACH * 0.25 &&
|
|
leftWrist.x < leftShoulder.x - ARM_PATTERN_MIN_SIDE_REACH * 0.55 &&
|
|
rightWrist.x > rightElbow.x + ARM_PATTERN_MIN_SIDE_REACH * 0.25 &&
|
|
rightWrist.x > rightShoulder.x + ARM_PATTERN_MIN_SIDE_REACH * 0.55;
|
|
const bothElbowsParticipating =
|
|
leftElbow.x <= leftShoulder.x + ARM_PATTERN_MIN_SIDE_REACH &&
|
|
rightElbow.x >= rightShoulder.x - ARM_PATTERN_MIN_SIDE_REACH;
|
|
const handsNearCenter =
|
|
leftElbow.x < leftShoulder.x - ARM_PATTERN_MIN_SIDE_REACH * 0.5 &&
|
|
rightElbow.x > rightShoulder.x + ARM_PATTERN_MIN_SIDE_REACH * 0.5 &&
|
|
leftWrist.x > leftElbow.x &&
|
|
rightWrist.x < rightElbow.x &&
|
|
wristsApart < shoulderWidth * ZOOM_CLOSE_WRIST_SPREAD_FACTOR;
|
|
return (
|
|
(bothHandsOutside && bothElbowsParticipating && wristsApart > shoulderWidth * ZOOM_SUPPRESS_WRIST_SPREAD_FACTOR) ||
|
|
handsNearCenter
|
|
);
|
|
}
|
|
|
|
// The single-arm rotate detector only looks at the right arm and cannot tell
|
|
// whether the user is mid-way through a two-arm gesture. Because the right
|
|
// wrist crosses its rotate trigger one or two frames before the left wrist
|
|
// catches up to the zoom threshold, rotate routinely fires as the user starts
|
|
// to spread their arms. Flip the predicate: a deliberate single right-arm
|
|
// wave keeps the left wrist clearly hanging at the side, so refuse to emit
|
|
// any right-arm rotate unless we can verify the left arm is at rest (wrist
|
|
// hanging well below the shoulder AND elbow + wrist sitting near the body).
|
|
// Any ambiguous left-arm state -- raised, extending outward, or held at
|
|
// chest level -- yields no gesture, letting getZoomHoldPose handle the next
|
|
// frame instead.
|
|
function isLeftArmAtRest(leftShoulder, leftElbow, leftWrist) {
|
|
const hangingBelowShoulder = leftWrist.y >= leftShoulder.y + LEFT_ARM_REST_HANGING_BELOW_SHOULDER;
|
|
const wristNearBody = leftWrist.x >= leftShoulder.x - ARM_PATTERN_MIN_SIDE_REACH;
|
|
const elbowNearBody = leftElbow.x >= leftShoulder.x - ARM_PATTERN_MIN_SIDE_REACH;
|
|
return hangingBelowShoulder && wristNearBody && elbowNearBody;
|
|
}
|
|
|
|
// Detect a two-arm zoom intent purely from frame-to-frame motion.
|
|
// Mirror-safe: measures the change in span between the wrists, not the
|
|
// per-wrist x direction. Spreading widens the span and triggers zoom_in
|
|
// regardless of whether the camera feed is mirrored; closing shrinks the
|
|
// span and triggers zoom_out. Both wrists must be actively moving (each
|
|
// crosses the noise floor) to rule out single-arm drift.
|
|
function getZoomTrend(leftWrist, rightWrist, previousLeftWrist, previousRightWrist) {
|
|
if (!previousLeftWrist || !previousRightWrist) return null;
|
|
const heightDelta = Math.abs(rightWrist.y - leftWrist.y);
|
|
if (heightDelta > ZOOM_TREND_HEIGHT_TOLERANCE) return null;
|
|
|
|
const leftMoved = Math.abs(leftWrist.x - previousLeftWrist.x);
|
|
const rightMoved = Math.abs(rightWrist.x - previousRightWrist.x);
|
|
if (leftMoved < ZOOM_TREND_MIN_WRIST_DELTA || rightMoved < ZOOM_TREND_MIN_WRIST_DELTA) return null;
|
|
|
|
const currentSpread = Math.abs(rightWrist.x - leftWrist.x);
|
|
const previousSpread = Math.abs(previousRightWrist.x - previousLeftWrist.x);
|
|
const spreadDelta = currentSpread - previousSpread;
|
|
const minSpreadDelta = ZOOM_TREND_MIN_WRIST_DELTA * 2;
|
|
const combinedSpeed = leftMoved + rightMoved;
|
|
const intensity = Math.min(1, Math.max(ZOOM_TREND_MIN_INTENSITY, combinedSpeed * ZOOM_TREND_INTENSITY_SCALE));
|
|
|
|
if (spreadDelta > minSpreadDelta) {
|
|
return { gesture: "zoom_in", confidence: 0.88, intensity };
|
|
}
|
|
if (spreadDelta < -minSpreadDelta) {
|
|
return { gesture: "zoom_out", confidence: 0.86, intensity };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// Mirror-safe sustained-pose detector. Decides whether the user is currently
|
|
// holding a "spread" or "closed" pose so zoom_in / zoom_out can keep firing
|
|
// while no motion is happening (trend would otherwise stop emitting).
|
|
// - heightDelta gate rules out one-arm-up-one-arm-down gestures.
|
|
// - wristsRaised gate ensures the wrists are at chest level or above
|
|
// (excludes "hands hanging at the hips" which would coincidentally have
|
|
// a small span).
|
|
// - span > shoulderWidth * 1.30 -> zoom_in (mirror-safe via Math.abs).
|
|
// - closed pose additionally requires both elbows clearly outside the
|
|
// shoulder line (forming a "hug"), distinguishing it from arms relaxed
|
|
// at the body's centerline.
|
|
function getZoomHoldPose(
|
|
leftShoulder,
|
|
leftElbow,
|
|
leftWrist,
|
|
rightShoulder,
|
|
rightElbow,
|
|
rightWrist,
|
|
shoulderWidth,
|
|
) {
|
|
const heightDelta = Math.abs(rightWrist.y - leftWrist.y);
|
|
if (heightDelta > ZOOM_HOLD_HEIGHT_TOLERANCE) return null;
|
|
|
|
const avgShoulderY = (leftShoulder.y + rightShoulder.y) / 2;
|
|
const wristsRaised =
|
|
leftWrist.y <= avgShoulderY + ZOOM_HOLD_WRIST_BELOW_SHOULDER_LIMIT &&
|
|
rightWrist.y <= avgShoulderY + ZOOM_HOLD_WRIST_BELOW_SHOULDER_LIMIT;
|
|
if (!wristsRaised) return null;
|
|
|
|
const span = Math.abs(rightWrist.x - leftWrist.x);
|
|
|
|
if (span > shoulderWidth * ZOOM_HOLD_SPREAD_FACTOR) {
|
|
return { gesture: "zoom_in", confidence: 0.82, intensity: 0.8 };
|
|
}
|
|
|
|
const leftElbowSpread = Math.abs(leftElbow.x - leftShoulder.x);
|
|
const rightElbowSpread = Math.abs(rightElbow.x - rightShoulder.x);
|
|
const elbowsOutward =
|
|
leftElbowSpread > shoulderWidth * ZOOM_HOLD_ELBOW_OUT_FACTOR &&
|
|
rightElbowSpread > shoulderWidth * ZOOM_HOLD_ELBOW_OUT_FACTOR;
|
|
if (elbowsOutward && span < shoulderWidth * ZOOM_HOLD_CLOSE_FACTOR) {
|
|
return { gesture: "zoom_out", confidence: 0.80, intensity: 0.7 };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function applyPoseLatch(observation, state) {
|
|
if (!state || !observation) return observation;
|
|
if (state.activePatternGesture === observation.gesture) return null;
|
|
state.activePatternGesture = observation.gesture;
|
|
return observation;
|
|
}
|
|
|
|
function recognizeGesture(joints, previousJoints, options = {}) {
|
|
const state = options.state || null;
|
|
const leftEar = getJoint(joints, "left_ear");
|
|
const rightEar = getJoint(joints, "right_ear");
|
|
const leftWrist = getJoint(joints, "left_wrist");
|
|
const rightWrist = getJoint(joints, "right_wrist");
|
|
const leftElbow = getJoint(joints, "left_elbow");
|
|
const rightElbow = getJoint(joints, "right_elbow");
|
|
const leftShoulder = getJoint(joints, "left_shoulder");
|
|
const rightShoulder = getJoint(joints, "right_shoulder");
|
|
const previousLeftWrist = getJoint(previousJoints, "left_wrist");
|
|
const previousRightWrist = getJoint(previousJoints, "right_wrist");
|
|
if (!leftWrist || !rightWrist || !leftElbow || !rightElbow || !leftShoulder || !rightShoulder) return null;
|
|
|
|
// Trend-based zoom runs first: anti-symmetric wrist motion expresses
|
|
// the user's intent directly and is far more reliable than waiting for
|
|
// an absolute pose to match. It also bypasses the layer / focus / rotate
|
|
// detectors, which would otherwise intercept mid-spread frames.
|
|
const trend = getZoomTrend(leftWrist, rightWrist, previousLeftWrist, previousRightWrist);
|
|
if (trend) {
|
|
if (state) state.activePatternGesture = trend.gesture;
|
|
return trend;
|
|
}
|
|
|
|
const shoulderWidth = Math.max(0.08, Math.abs(rightShoulder.x - leftShoulder.x));
|
|
const leftRaised = leftWrist.y < leftShoulder.y - 0.05;
|
|
const rightRaised = rightWrist.y < rightShoulder.y - 0.05;
|
|
const leftDeltaX = previousLeftWrist ? leftWrist.x - previousLeftWrist.x : 0;
|
|
const leftDeltaY = previousLeftWrist ? leftWrist.y - previousLeftWrist.y : 0;
|
|
const headTiltY = leftEar && rightEar ? rightEar.y - leftEar.y : 0;
|
|
|
|
if (!rightRaised && leftRaised && leftDeltaY < -LEFT_WRIST_LAYER_DELTA_Y) {
|
|
return { gesture: "layer_prev", confidence: 0.78, intensity: Math.min(1, Math.abs(leftDeltaY) * WRIST_LAYER_INTENSITY_SCALE) };
|
|
}
|
|
if (!rightRaised && leftRaised && leftDeltaY > LEFT_WRIST_LAYER_DELTA_Y) {
|
|
return { gesture: "layer_next", confidence: 0.78, intensity: Math.min(1, Math.abs(leftDeltaY) * WRIST_LAYER_INTENSITY_SCALE) };
|
|
}
|
|
|
|
if (headTiltY < -HEAD_TILT_DELTA_Y) {
|
|
return { gesture: "focus_prev", confidence: 0.78, intensity: Math.min(1, Math.abs(headTiltY) * HEAD_TILT_INTENSITY_SCALE) };
|
|
}
|
|
if (headTiltY > HEAD_TILT_DELTA_Y) {
|
|
return { gesture: "focus_next", confidence: 0.78, intensity: Math.min(1, Math.abs(headTiltY) * HEAD_TILT_INTENSITY_SCALE) };
|
|
}
|
|
|
|
// Pose-based zoom hold: while the user sustains a spread (zoom_in) or
|
|
// closed (zoom_out) pose without further motion, keep emitting the same
|
|
// zoom direction every frame. Bypasses the pattern latch so the gesture
|
|
// can fire repeatedly; downstream cooldownMs (120ms) rate-limits to
|
|
// ~8 emissions/sec, which produces smooth continuous zooming on the globe
|
|
// until the user changes their pose. Uses the mirror-safe span detector
|
|
// so it works on non-mirrored camera feeds where the absolute left/right
|
|
// pose checks would otherwise fail.
|
|
const zoomPattern = getZoomHoldPose(
|
|
leftShoulder,
|
|
leftElbow,
|
|
leftWrist,
|
|
rightShoulder,
|
|
rightElbow,
|
|
rightWrist,
|
|
shoulderWidth,
|
|
);
|
|
if (zoomPattern) {
|
|
if (state) state.activePatternGesture = zoomPattern.gesture;
|
|
return zoomPattern;
|
|
}
|
|
|
|
// Two safeguards must both hold before a single right-arm rotate fires:
|
|
// (1) zoom is not currently a likely interpretation of the pose,
|
|
// (2) the left arm is verifiably at rest. This kills the right-leads-left
|
|
// race that previously emitted a stray rotate at the start of a spread.
|
|
const rotateAllowed =
|
|
!isZoomCandidatePose(
|
|
leftShoulder,
|
|
leftElbow,
|
|
leftWrist,
|
|
rightShoulder,
|
|
rightElbow,
|
|
rightWrist,
|
|
shoulderWidth,
|
|
) && isLeftArmAtRest(leftShoulder, leftElbow, leftWrist);
|
|
const rotatePattern = rotateAllowed
|
|
? getRightArmPattern(rightShoulder, rightElbow, rightWrist)
|
|
: null;
|
|
// Rotate stays latched: one deliberate wave = one rotation step. Without
|
|
// the latch, holding the arm out would spin the globe continuously, which
|
|
// is the opposite of what the user wants for navigation.
|
|
if (rotatePattern) return applyPoseLatch(rotatePattern, state);
|
|
if (state) state.activePatternGesture = null;
|
|
return null;
|
|
}
|
|
|
|
async function createDefaultRecognizer() {
|
|
const { FilesetResolver, PoseLandmarker } = await importMediaPipeTasksVision();
|
|
const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_WASM_URL);
|
|
const pose = await PoseLandmarker.createFromOptions(vision, {
|
|
baseOptions: {
|
|
modelAssetPath: POSE_MODEL_URL,
|
|
delegate: "GPU",
|
|
},
|
|
runningMode: "VIDEO",
|
|
numPoses: 1,
|
|
});
|
|
|
|
return {
|
|
recognize(video, timestampMs) {
|
|
const result = pose.detectForVideo(video, timestampMs);
|
|
const landmarks = result?.landmarks?.[0] || [];
|
|
return normalizePoseLandmarks(landmarks);
|
|
},
|
|
close() {
|
|
pose.close?.();
|
|
},
|
|
};
|
|
}
|
|
|
|
async function importMediaPipeTasksVision() {
|
|
const failures = [];
|
|
for (const moduleUrl of MEDIAPIPE_TASKS_URLS) {
|
|
try {
|
|
const module = await import(moduleUrl);
|
|
if (module?.FilesetResolver && module?.PoseLandmarker) {
|
|
return module;
|
|
}
|
|
failures.push(`${moduleUrl}: missing MediaPipe exports`);
|
|
} catch (error) {
|
|
failures.push(`${moduleUrl}: ${error?.message || String(error)}`);
|
|
}
|
|
}
|
|
const error = new Error("无法加载 MediaPipe Tasks Vision 模块,请检查网络或切换 Motion Agent");
|
|
error.details = failures;
|
|
throw error;
|
|
}
|
|
|
|
export function createBrowserCameraProvider(options = {}) {
|
|
const {
|
|
mediaDevices = typeof navigator !== "undefined" ? navigator.mediaDevices : null,
|
|
recognizerFactory = createDefaultRecognizer,
|
|
requestAnimationFrameFn =
|
|
typeof requestAnimationFrame !== "undefined"
|
|
? requestAnimationFrame.bind(globalThis)
|
|
: (callback) => setTimeout(() => callback(nowMs()), 16),
|
|
cancelAnimationFrameFn =
|
|
typeof cancelAnimationFrame !== "undefined"
|
|
? cancelAnimationFrame.bind(globalThis)
|
|
: clearTimeout,
|
|
onMessage = () => {},
|
|
onState = () => {},
|
|
onStatus = () => {},
|
|
onVideoSource = () => {},
|
|
} = options;
|
|
|
|
let disposed = false;
|
|
let connected = false;
|
|
let stream = null;
|
|
let video = null;
|
|
let recognizer = null;
|
|
let rafId = null;
|
|
let lastFrameAt = 0;
|
|
let lastGestureAt = 0;
|
|
let seq = 0;
|
|
let previousJoints = [];
|
|
const gestureState = {};
|
|
|
|
function emitState(detail = {}) {
|
|
onState({
|
|
provider: "browser_camera",
|
|
connected,
|
|
...detail,
|
|
});
|
|
}
|
|
|
|
function emitStatus(message, type = "info", extra = {}) {
|
|
onStatus(message, type);
|
|
emitState({ message, ...extra });
|
|
}
|
|
|
|
function stopStream() {
|
|
onVideoSource({
|
|
provider: "browser_camera",
|
|
source: null,
|
|
active: false,
|
|
});
|
|
if (stream) {
|
|
stream.getTracks?.().forEach((track) => track.stop?.());
|
|
stream = null;
|
|
}
|
|
if (video) {
|
|
video.pause?.();
|
|
video.srcObject = null;
|
|
video.remove?.();
|
|
video = null;
|
|
}
|
|
}
|
|
|
|
function emitSkeleton(joints, matchedGesture = null, confidence = 0) {
|
|
onMessage({
|
|
type: "skeleton",
|
|
timestamp_ms: wallClockMs(),
|
|
source: "browser-camera",
|
|
mode: "single",
|
|
camera_id: "browser:getUserMedia",
|
|
matched_gesture: matchedGesture,
|
|
confidence,
|
|
joints,
|
|
bones: POSE_BONES,
|
|
});
|
|
}
|
|
|
|
function emitGesture(observation) {
|
|
seq += 1;
|
|
onMessage({
|
|
type: "gesture",
|
|
gesture: observation.gesture,
|
|
phase: "discrete",
|
|
confidence: observation.confidence,
|
|
intensity: observation.intensity,
|
|
timestamp_ms: wallClockMs(),
|
|
seq,
|
|
source: "browser-camera",
|
|
mode: "single",
|
|
payload: {},
|
|
});
|
|
}
|
|
|
|
function scheduleFrame() {
|
|
if (disposed) return;
|
|
rafId = requestAnimationFrameFn(processFrame);
|
|
}
|
|
|
|
function processFrame(timestamp) {
|
|
if (disposed || !video || !recognizer) return;
|
|
if (timestamp - lastFrameAt < FRAME_INTERVAL_MS) {
|
|
scheduleFrame();
|
|
return;
|
|
}
|
|
lastFrameAt = timestamp;
|
|
|
|
try {
|
|
const joints = recognizer.recognize(video, timestamp) || [];
|
|
const currentWallMs = wallClockMs();
|
|
const observation = recognizeGesture(joints, previousJoints, {
|
|
state: gestureState,
|
|
timestampMs: currentWallMs,
|
|
});
|
|
const canEmitGesture = observation && currentWallMs - lastGestureAt >= GESTURE_COOLDOWN_MS;
|
|
if (canEmitGesture) {
|
|
lastGestureAt = currentWallMs;
|
|
emitGesture(observation);
|
|
}
|
|
emitSkeleton(
|
|
joints,
|
|
observation?.gesture || null,
|
|
observation?.confidence || 0,
|
|
);
|
|
previousJoints = joints;
|
|
} catch (error) {
|
|
emitStatus(`浏览器动捕识别失败: ${error?.message || String(error)}`, "error", {
|
|
error: "recognition_failed",
|
|
});
|
|
}
|
|
scheduleFrame();
|
|
}
|
|
|
|
async function startCamera() {
|
|
if (!canUseBrowserCamera(mediaDevices)) {
|
|
emitStatus("当前浏览器不支持 getUserMedia 摄像头接口", "error", {
|
|
error: "get_user_media_unavailable",
|
|
});
|
|
return false;
|
|
}
|
|
if (!isSecureCameraContext()) {
|
|
emitStatus("浏览器摄像头需要 HTTPS 或 localhost 环境", "error", {
|
|
error: "insecure_context",
|
|
});
|
|
return false;
|
|
}
|
|
|
|
try {
|
|
recognizer = await recognizerFactory();
|
|
} catch (error) {
|
|
connected = false;
|
|
emitStatus(`浏览器动捕模型加载失败: ${error?.message || String(error)}`, "error", {
|
|
error: "model_load_failed",
|
|
});
|
|
return false;
|
|
}
|
|
|
|
try {
|
|
stream = await mediaDevices.getUserMedia(CAMERA_CONSTRAINTS);
|
|
video = document.createElement("video");
|
|
video.muted = true;
|
|
video.playsInline = true;
|
|
video.autoplay = true;
|
|
video.style.display = "none";
|
|
video.srcObject = stream;
|
|
document.body.appendChild(video);
|
|
await video.play();
|
|
await waitForVideoMetadata(video);
|
|
onVideoSource({
|
|
provider: "browser_camera",
|
|
source: video,
|
|
active: true,
|
|
});
|
|
} catch (error) {
|
|
connected = false;
|
|
recognizer?.close?.();
|
|
recognizer = null;
|
|
stopStream();
|
|
emitStatus(getUserMediaErrorMessage(error), "error", {
|
|
error: "browser_camera_failed",
|
|
});
|
|
return false;
|
|
}
|
|
|
|
connected = true;
|
|
emitStatus("浏览器摄像头动捕已连接", "info");
|
|
scheduleFrame();
|
|
return true;
|
|
}
|
|
|
|
return {
|
|
provider: "browser_camera",
|
|
async start() {
|
|
if (disposed) return false;
|
|
return startCamera();
|
|
},
|
|
stop() {
|
|
disposed = true;
|
|
if (rafId) {
|
|
cancelAnimationFrameFn(rafId);
|
|
rafId = null;
|
|
}
|
|
recognizer?.close?.();
|
|
recognizer = null;
|
|
stopStream();
|
|
connected = false;
|
|
emitState({ connected: false });
|
|
},
|
|
isConnected() {
|
|
return connected;
|
|
},
|
|
};
|
|
}
|
|
|
|
export {
|
|
CAMERA_CONSTRAINTS,
|
|
importMediaPipeTasksVision,
|
|
POSE_BONES,
|
|
recognizeGesture,
|
|
};
|