import { fetchProfile, MotionController } from "@webxr-input-profiles/motion-controllers"; import { AxesHelper, Euler, MathUtils, Matrix4, Object3D, Quaternion, Ray, Vector3 } from "three"; import { Context } from "../engine_context.js"; import { Gizmos } from "../engine_gizmos.js"; import { type InputEventNames, InputEvents, IPointerHitEventReceiver, NEPointerEvent, type NEPointerEventInit } from "../engine_input.js"; import { getTempQuaternion, getTempVector, getWorldQuaternion } from "../engine_three_utils.js"; import type { ButtonName, IGameObject, Vec3, XRControllerButtonName, XRGestureName } from "../engine_types.js"; import { getParam } from "../engine_utils.js"; import { flipForwardMatrix, flipForwardQuaternion } from "./internal.js"; import type { NeedleXRHitTestResult, NeedleXRSession } from "./NeedleXRSession.js"; const debug = getParam("debugwebxr"); /** when enabled we will not use the browser select event but instead * we will emit the input event based on our own pinch detection * this is a workaround for visionOS not emitting the select events, see https://linear.app/needle/issue/NE-4212 */ const debugCustomGesture = getParam("debugcustomgesture"); /** true when selectstart was ever received. * On VisionOS 1.1 we always have select events (as per the spec), so this is always true */ // let _didReceiveSelectStartEvent = false; // https://github.com/immersive-web/webxr-input-profiles/blob/4484a05e30bcd43fe86bb4e06b7a707861a26796/packages/registry/profiles/meta/meta-quest-touch-plus.json declare type ControllerAxes = "xr-standard-thumbstick" | "xr-standard-touchpad"; declare type StickName = "xr-standard-thumbstick" | "xr-standard-touchpad"; declare type Mapping = "xr-standard"; declare type ComponentType = "button" | "thumbstick" | "squeeze" | "touchpad"; declare type GamepadKey = "button" | "xAxis" | "yAxis"; declare type NeedleXRControllerButtonName = ButtonName | "primary-button" | "primary"; declare type ComponentMap = { type: ComponentType, rootNodeName?: string, gamepadIndices?: { [key in GamepadKey]?: number }, visualResponses?: { [key: string]: { states: Array } } } declare type InputDeviceLayout = { selectComponentId: string, components: { [key: string]: ComponentMap } mapping: Mapping; gamepad: Array, axes: Array<{ componentId: ControllerAxes, axis: "x-axis" | "y-axis", }>, } declare type InputDeviceProfile = { profileId: string, fallbackProfileIds: string[], layouts: [ left: InputDeviceLayout, right: InputDeviceLayout ] } // https://github.com/immersive-web/webxr-input-profiles/blob/4484a05e30bcd43fe86bb4e06b7a707861a26796/packages/registry/profiles/meta/meta-quest-touch-plus.json const DEFAULT_PROFILES_PATH = 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/assets@1.0/dist/profiles'; const DEFAULT_PROFILE = 'generic-trigger'; const metacarpalToGripQuaternion = new Quaternion().setFromEuler(new Euler(MathUtils.degToRad(0), MathUtils.degToRad(-90), MathUtils.degToRad(-90))) const metacarpalToGripPosition = new Vector3(0.04, -0.04, 0.0); /** * A NeedleXRController wraps a connected XRInputDevice that is either a physical controller or a hand * You can access specific buttons using `getButton` and `getStick` * To get spatial data in rig space (position, rotation) use the `gripPosition`, `gripQuaternion`, `rayPosition` and `rayQuaternion` properties * To get spatial data in world space use the `gripWorldPosition`, `gripWorldQuaternion`, `rayWorldPosition` and `rayWorldQuaternion` properties * Inputs will also be emitted as pointer events on `this.context.input` - so you can receive controller inputs on objects using the appropriate input events on your components (e.g. `onPointerDown`, `onPointerUp` etc) - use the `pointerType` property to check if the event is from a controller or not * @link https://developer.mozilla.org/en-US/docs/Web/API/XRInputSource * @category XR */ export class NeedleXRController implements IPointerHitEventReceiver { /** the Needle XR Session */ readonly xr: NeedleXRSession; get context() { return this.xr.context; } /** * https://developer.mozilla.org/en-US/docs/Web/API/XRInputSource */ readonly inputSource: XRInputSource; /** the input source index */ readonly index: number = 0; /** When enabled the controller will create input events in the Needle Engine input system (e.g. when a button is pressed or the controller is moved) * You can disable this if you don't want inputs to go through the input system but be aware that this will result in `onPointerDown` component callbacks to not be invoked anymore for this XRController */ emitEvents = true; /** Is the controller still connected? */ get connected() { return this._connected; } private _connected: boolean = true; get isTracking() { return this._isTracking; } private _isTracking: boolean = false; /** the input source gamepad giving raw access to the gamepad values * You should usually use the `getButton` and `getStick` methods instead to get access to named buttons and sticks */ get gamepad() { return this.__gamepad ??= this.inputSource.gamepad; } private __gamepad?: Gamepad; /** @returns true if this is a hand (otherwise this is a controller) */ get isHand() { return this.hand != undefined; } /** * If this is a hand then this is the hand info (XRHand) * @link https://developer.mozilla.org/en-US/docs/Web/API/XRHand */ get hand() { return this.__hand ??= this.inputSource.hand; } private __hand?: XRHand; /** threejs XRHandSpace, shorthand for `context.renderer.xr.getHand(controllerIndex)` * @link https://threejs.org/docs/#api/en/renderers/webxr/WebXRManager.getHand */ get handObject() { return this.context.renderer.xr.getHand(this.index); } /** The input source profiles */ get profiles() { return this.inputSource.profiles; } /** The device input layout */ get layout() { return this._layout; } /** shorthand for `inputSource.targetRayMode` */ get targetRayMode(): (XRTargetRayMode | "transient-pointer") { return this.inputSource.targetRayMode; } /** shorthand for `inputSource.targetRaySpace` */ get targetRaySpace() { return this.inputSource.targetRaySpace; } /** shorthand for `inputSource.gripSpace` */ get gripSpace() { return this.inputSource.gripSpace; } /** * If the controller if held in the left or right hand (or if it's a left or right hand) **/ get side() { return this.__side ??= this.inputSource.handedness; } private __side: XRHandedness | undefined = undefined; /** is right side. shorthand for `side === 'right'` */ get isRight() { return this.side === 'right'; } /** is left side. shorthand for `side === 'left'` */ get isLeft() { return this.side === 'left'; } /** is XR stylus, e.g. Logitech MX Ink */ get isStylus() { return this._isMxInk; } /** The XRTransientInputHitTestSource can be used to perform hit tests with the controller ray against the real world. * see https://developer.mozilla.org/en-US/docs/Web/API/XRSession/requestHitTestSourceForTransientInput for more information * Requires the hit-test feature to be enabled in the XRSession * * NOTE: The hit test source should be cancelled once it's not needed anymore. Call `cancelHitTestSource` to do this */ getHitTestSource() { if (!this._hitTestSource) this._requestHitTestSource(); return this._hitTestSource; } get hasHitTestSource() { return this._hitTestSource; } /** Make sure to cancel the hittest source once it's not needed anymore */ cancelHitTestSource() { if (this._hitTestSource) { this._hitTestSource.cancel(); this._hitTestSource = undefined; } } private _hitTestSource: XRTransientInputHitTestSource | undefined = undefined; private _hasSelectEvent = false; get hasSelectEvent() { return this._hasSelectEvent; } private _isMxInk = false; private _isMetaQuestTouchController = false; /** Perform a hit test against the XR planes or meshes. shorthand for `xr.getHitTest(controller)` * @returns the hit test result (with position and rotation in worldspace) or null if no hit was found */ getHitTest(): NeedleXRHitTestResult | null { return this.xr.getHitTest(this); } /** This is cleared at the beginning of each frame */ private readonly _handJointPoses: Map = new Map(); /** Get the hand joint pose from the current XRFrame. Results are cached for a frame to avoid calling getJointPose multiple times */ getHandJointPose(joint: XRJointSpace, frame?: XRFrame) { frame = frame || this.xr.frame; if (!this.hand || !frame?.getJointPose || !this.xr.referenceSpace) return null; let pose = this._handJointPoses?.get(joint); if (pose) return pose; pose = frame.getJointPose(joint, this.xr.referenceSpace); if (pose) this._handJointPoses.set(joint, pose); return pose; } /** Grip matrix in grip space */ private readonly _gripMatrix = new Matrix4(); /** Grip position in grip space */ private readonly _gripPosition = new Vector3(); /** Grip rotation in grip space */ private readonly _gripQuaternion = new Quaternion(); private readonly _linearVelocity: Vector3 = new Vector3(); private readonly _rayPositionRaw = new Vector3(); private readonly _rayRotationRaw = new Quaternion(); /** ray matrix in grip space */ private readonly _rayMatrix = new Matrix4(); /** Ray position in rig space */ private readonly _rayPosition = new Vector3(); /** Ray rotation in rig space */ private readonly _rayQuaternion = new Quaternion(); /** Grip position in rig space */ get gripPosition() { return getTempVector(this._gripPosition) } /** Grip rotation in rig space */ get gripQuaternion() { return getTempQuaternion(this._gripQuaternion) } get gripMatrix() { return this._gripMatrix; } /** Grip linear velocity in rig space * @link https://developer.mozilla.org/en-US/docs/Web/API/XRPose/linearVelocity */ get gripLinearVelocity() { return getTempVector(this._linearVelocity).applyQuaternion(flipForwardQuaternion); } /** Ray position in rig space */ get rayPosition() { return getTempVector(this._rayPosition) } /** Ray rotation in rig space */ get rayQuaternion() { return getTempQuaternion(this._rayQuaternion) } /** Controller grip position in worldspace */ get gripWorldPosition() { return getTempVector(this._gripWorldPosition); } private readonly _gripWorldPosition: Vector3 = new Vector3(); /** Controller grip rotation in wordspace */ get gripWorldQuaternion() { return getTempQuaternion(this._gripWorldQuaternion); } private readonly _gripWorldQuaternion: Quaternion = new Quaternion(); /** Controller ray position in worldspace (this value is calculated once per frame by default - call `updateRayWorldPosition` to force an update) */ get rayWorldPosition() { return getTempVector(this._rayWorldPosition); } private readonly _rayWorldPosition: Vector3 = new Vector3(); /** Recalculates the ray world position */ updateRayWorldPosition() { const parent = this.xr.context.mainCamera?.parent; this._rayWorldPosition.copy(this._rayPositionRaw); if (parent) this._rayWorldPosition.applyMatrix4(parent.matrixWorld); } /** Controller ray rotation in wordspace (this value is calculated once per frame by default - call `updateRayWorldQuaternion` to force an update) */ get rayWorldQuaternion() { return getTempQuaternion(this._rayWorldQuaternion); } private readonly _rayWorldQuaternion: Quaternion = new Quaternion(); get pinchPosition() { return getTempVector(this._pinchPosition); } private readonly _pinchPosition: Vector3 = new Vector3(); /** Recalculates the ray world quaternion */ updateRayWorldQuaternion() { const parent = this.xr.context.mainCamera?.parent; const parentWorldQuaternion = parent ? getWorldQuaternion(parent) : undefined; this._rayWorldQuaternion.copy(this._rayRotationRaw) // flip forward because we want +Z to be forward .multiply(flipForwardQuaternion); if (parentWorldQuaternion) this._rayWorldQuaternion.premultiply(parentWorldQuaternion) } /** The controller ray in worldspace */ get ray(): Ray { this._ray.origin.copy(this.rayWorldPosition); this._ray.direction.copy(getTempVector(0, 0, 1).applyQuaternion(this.rayWorldQuaternion)); return this._ray; } private readonly _ray; /** Recalculated once per update */ private _hand_wristDotUp: number | undefined = undefined; /** * The dot product of the hand palm with the up vector. * This is a number between -1 and 1, where 1 means the palm is directly up and -1 means the palm is directly down (upside down). * This value is undefined if there's no hand */ get handWristDotUp(): number | undefined { if (this._hand_wristDotUp !== undefined) return this._hand_wristDotUp; const handPalm = this.handObject?.joints["wrist"]; if (handPalm) { const up = getTempVector(0, 1, 0).applyQuaternion(handPalm.quaternion); const dot = getTempVector(0, 1, 0).dot(up); return this._hand_wristDotUp = dot; } return undefined; } /** * @returns true if the hand is upside down */ get isHandUpsideDown() { return this.handWristDotUp !== undefined ? this.handWristDotUp < -.7 : false; } /** * @returns true if the hand is upside down and we got a pinch down event this frame. */ get isTeleportGesture() { return this.isHandUpsideDown && this.getGesture("pinch")?.isDown; } /** The controller object space. * You can use it to attach objects to the controller. * Children will be automatically detached and put into the scene when the controller disconnects */ get object() { return this._object; } private readonly _object: IGameObject; private readonly _gripSpaceObject?: IGameObject; private readonly _raySpaceObject?: IGameObject; /** Assigned the model that you use for rendering. This can be used as a hint for other components */ model: Object3D | null = null; private readonly _debugAxesHelper = new AxesHelper(.15); private readonly _debugGripAxesHelper = new AxesHelper(.07); private readonly _debugRayAxesHelper = new AxesHelper(.07); /** returns the URL of the default controller model */ async getModelUrl(): Promise { return this.getMotionController?.then(res => res?.assetUrl || null); } constructor(session: NeedleXRSession, device: XRInputSource, index: number) { this.xr = session; this.inputSource = device; this.index = index; this._object = new Object3D() as unknown as IGameObject; this._object.name = `NeedleXRController_${index}`; if (debug) { this._object.add(this._debugAxesHelper); this._gripSpaceObject = new Object3D() as unknown as IGameObject; this._raySpaceObject = new Object3D() as unknown as IGameObject; this._gripSpaceObject.name = `NeedleXRController_${index}_gripSpace`; this._raySpaceObject.name = `NeedleXRController_${index}_raySpace`; this._gripSpaceObject.add(this._debugGripAxesHelper); this._raySpaceObject.add(this._debugRayAxesHelper); this.xr.context.scene.add(this._gripSpaceObject); this.xr.context.scene.add(this._raySpaceObject); } this.xr.context.scene.add(this._object); this._ray = new Ray(); this.pointerInit = { origin: this, pointerType: this.hand ? "hand" : "controller", deviceIndex: this.index, pointerId: -1, // < this will be updated in the emitPointerEvent method mode: this.inputSource.targetRayMode, ray: this._ray, device: this._object, buttonName: "none", } this.initialize(); this.subscribeEvents(); } private _hitTestSourcePromise: Promise | null = null; private _requestHitTestSource(): Promise | null { if (this._hitTestSourcePromise) return this._hitTestSourcePromise; // We only request a hit test source when we need it - meaning e.g. when we want to place the scene in AR // Make sure to cancel the hittest source when we don't need it anymore for performance reasons // // TODO: change this to check if we have hit-testing enabled instead of pass through. if (this.xr.mode === "immersive-ar" && this.inputSource.targetRayMode === "tracked-pointer" && this.xr.session.requestHitTestSourceForTransientInput) { // request hittest source return this._hitTestSourcePromise = this.xr.session.requestHitTestSourceForTransientInput({ profile: this.inputSource.profiles[0], offsetRay: new XRRay(), })?.then(hitTestSource => { this._hitTestSourcePromise = null; if (!this.connected) { hitTestSource.cancel(); return null; } return this._hitTestSource = hitTestSource; }) ?? null; } return null; } onPointerHits = _evt => { } onUpdate(frame: XRFrame) { this.onUpdateFrame(frame); this.updateInputEvents(); this.onUpdateMove(); //performance.mark('NeedleXRController onUpdate end'); //performance.measure('NeedleXRController onUpdate', 'NeedleXRController onUpdate start', 'NeedleXRController onUpdate end'); } onRenderDebug() { Gizmos.DrawSphere(this.rayWorldPosition, .003); Gizmos.DrawDirection(this.rayWorldPosition, getTempVector(0, 0, 10).applyQuaternion(this.rayWorldQuaternion)); const labelPosition = this.inputSource.gripSpace ? this.gripWorldPosition : this.object.worldPosition; const debugLabelPosition = labelPosition.sub(this.object.worldForward.multiplyScalar(.1)); const profileStr = this.inputSource.profiles.join("\n"); let debugStr = `Controller[${this.index}] (${this.inputSource.targetRayMode}, ${this.side}) C:${this.connected ? "x" : "-"} T:${this.isTracking ? "x" : "-"} Hand:${this.inputSource.hand ? "x" : "-"} Pen: ${this._isMxInk ? "x" : "-"}`; if (this.inputSource.hand) debugStr += `\nPinch: ${this.getGesture("pinch")?.value.toFixed(3)}`; debugStr += "\n" + profileStr; debugStr += "\n" + (this.inputSource.targetRaySpace ? `Ray: x` : "Ray: -") + (this.inputSource.gripSpace ? " Grip: x" : " Grip: -") + (this.inputSource.gamepad ? ` Gamepad: ${this.inputSource.gamepad.mapping}` : " Gamepad: -"); if (this.inputSource.gamepad) { const gp = this.inputSource.gamepad; let gamepadStr = "[btns " + gp.buttons.length + "]: " + gp.buttons.map(b => b.value.toPrecision(1)).join(","); gamepadStr += "\n[axes " + gp.axes.length + "]: " + gp.axes.map(a => a.toPrecision(1)).join(","); debugStr += "\n" + gamepadStr; } if (this._layout) { debugStr += "\nLayout: "; for (const component of Object.keys(this._layout.components || {})) { const val = this.getStick(component as StickName); const indices = this._layout.components[component]?.gamepadIndices; const indicesAsString = indices ? Object.entries(indices).map(e => e[0][0].toUpperCase() + e[0].slice(1) + "=" + e[1]).join(",") : ""; debugStr += `\n ${component}: ${this._layout.components[component]?.type} [${indicesAsString}] (${val.x.toPrecision(2)},${val.y.toPrecision(2)})`; } } Gizmos.DrawLabel(debugLabelPosition, debugStr, .006); } private onUpdateFrame(frame: XRFrame) { // make sure this is cleared every frame this._handJointPoses.clear(); this._hand_wristDotUp = undefined; if (!this.xr.referenceSpace || !this.inputSource.gamepad?.connected) { this._isTracking = false; return; } const rayPose = frame.getPose(this.inputSource.targetRaySpace, this.xr.referenceSpace); this._isTracking = rayPose != null; let gripPositionRaw: Vector3 | null = null; let gripQuaternionRaw: Quaternion | null = null; let rayPositionRaw: Vector3 | null = null; let rayQuaternionRaw: Quaternion | null = null; if (rayPose) { const t = rayPose.transform; this._rayMatrix .fromArray(t.matrix) .premultiply(flipForwardMatrix); this._rayMatrix.decompose(this._rayPosition, this._rayQuaternion, getTempVector(1, 1, 1)); rayPositionRaw = getTempVector(t.position); rayQuaternionRaw = getTempQuaternion(t.orientation); this._rayPositionRaw.copy(rayPositionRaw); this._rayRotationRaw.copy(rayQuaternionRaw); } if (this.inputSource.gripSpace) { const gripPose = frame.getPose(this.inputSource.gripSpace, this.xr.referenceSpace!); if (gripPose) { const t = gripPose.transform; gripPositionRaw = getTempVector(t.position); gripQuaternionRaw = getTempQuaternion(t.orientation); this._gripMatrix .fromArray(t.matrix) .premultiply(flipForwardMatrix); this._gripMatrix.decompose(this._gripPosition, this._gripQuaternion, getTempVector(1, 1, 1)); if ("linearVelocity" in gripPose && gripPose.linearVelocity) { const p = gripPose.linearVelocity as DOMPointReadOnly; this._linearVelocity.set(p.x, p.y, p.z); } } } // update controller object parent – needs to be parented to the rig, which // implicitly is the same object as the camera parent. if (this.xr.context.mainCamera?.parent) { if (this._object.parent !== this.xr.context.mainCamera?.parent) this.xr.context.mainCamera.parent.add(this._object); if (this._gripSpaceObject !== undefined && this._gripSpaceObject?.parent !== this.xr.context.mainCamera?.parent) this.xr.context.mainCamera.parent.add(this._gripSpaceObject); if (this._raySpaceObject !== undefined && this._raySpaceObject?.parent !== this.xr.context.mainCamera?.parent) this.xr.context.mainCamera.parent.add(this._raySpaceObject); } // for controllers, we set the position and rotation of the object to the ray position and rotation // for hands, we take the wrist position and rotation const hand = this.hand; if (hand) { // https://www.w3.org/TR/webxr-hand-input-1/#xrhand-interface let gotWrist = false; // TODO check why types are not correct here const wrist = hand.get("wrist"); const wristPose = wrist && this.getHandJointPose(wrist, frame); if (wristPose) { gotWrist = true; const p = wristPose.transform.position; const q = wristPose.transform.orientation; this._object.position.set(p.x, p.y, p.z); this._object.quaternion.set(q.x, q.y, q.z, q.w).multiply(flipForwardQuaternion); } if (!gotWrist) { this._object.position.copy(this._rayPosition); this._object.quaternion.copy(this._rayQuaternion).multiply(flipForwardQuaternion); } //@ts-ignore const middle = hand.get("middle-finger-metacarpal"); const middlePose = middle && this.getHandJointPose(middle, frame); if (middlePose) { // for some reason the grip rotation is different from the wrist rotation // but we want to use the wrist rotation for the grip this._gripMatrix .fromArray(middlePose.transform.matrix) .premultiply(flipForwardMatrix); this._gripMatrix.decompose(this._gripPosition, this._gripQuaternion, getTempVector(1, 1, 1)); // If we don't have a grip space, we update the data from the metacarpal bone instead. // this way, things looking for a grip pose will still find one (e.g. XRControllerFollow). // For example, hands on VisionOS do not provide a gripSpace. if (true || !this.inputSource.gripSpace) { gripPositionRaw = getTempVector().copy(middlePose.transform.position); gripQuaternionRaw = getTempQuaternion().copy(middlePose.transform.orientation); gripQuaternionRaw.multiply(metacarpalToGripQuaternion); gripPositionRaw.add(getTempVector(metacarpalToGripPosition).applyQuaternion(gripQuaternionRaw)); } } } // on VisionOS we get a gripSpace that matches where the controller is for transient input sources else if (this.inputSource.gripSpace && this.targetRayMode === "transient-pointer" && gripPositionRaw && gripQuaternionRaw) { this._object.position.copy(gripPositionRaw); this._object.quaternion.copy(gripQuaternionRaw).multiply(flipForwardQuaternion); } else if (rayPositionRaw && rayQuaternionRaw) { this._object.position.copy(rayPositionRaw); this._object.quaternion.copy(rayQuaternionRaw).multiply(flipForwardQuaternion); } if (debug) { if (rayPositionRaw && rayQuaternionRaw) { this._raySpaceObject?.position.copy(rayPositionRaw); this._raySpaceObject?.quaternion.copy(rayQuaternionRaw).multiply(flipForwardQuaternion); } if (gripPositionRaw && gripQuaternionRaw) { this._gripSpaceObject?.position.copy(gripPositionRaw); this._gripSpaceObject?.quaternion.copy(gripQuaternionRaw).multiply(flipForwardQuaternion); } } // UPDATE WORLD TRANSFORM DATA const parent = this.xr.context.mainCamera?.parent; const parentWorldQuaternion = parent ? getWorldQuaternion(parent) : undefined; // GRIP if (gripPositionRaw && gripQuaternionRaw) { this._gripWorldPosition.copy(gripPositionRaw); if (parent) this._gripWorldPosition.applyMatrix4(parent.matrixWorld); this._gripWorldQuaternion.copy(gripQuaternionRaw); // flip forward because we want +Z to be forward this._gripWorldQuaternion.multiply(flipForwardQuaternion); if (parentWorldQuaternion) this._gripWorldQuaternion.premultiply(parentWorldQuaternion) } // RAY this.updateRayWorldPosition(); this.updateRayWorldQuaternion(); } /** Called when the input source disconnects */ onDisconnected() { this._connected = false; if (debug) console.warn("Controller disconnected", this.index); // move all attached objects into the scene for (const child of this._object.children) { this.xr.context.scene.attach(child); } this._object?.removeFromParent(); this._debugAxesHelper?.removeFromParent(); this._debugGripAxesHelper?.removeFromParent(); this._debugRayAxesHelper?.removeFromParent(); this._gripSpaceObject?.removeFromParent(); this._raySpaceObject?.removeFromParent(); this.unsubscribeEvents(); if (this._hitTestSource) { this._hitTestSource.cancel(); this._hitTestSource = undefined; } } /** * Get a gamepad button * @link https://github.com/immersive-web/webxr-gamepads-module/blob/main/gamepads-module-explainer.md * @param key the controller button name e.g. x-button * @returns the gamepad button if it exists on the controller - otherwise undefined */ getButton(key: NeedleXRControllerButtonName): NeedleGamepadButton | undefined | null { if (!this._layout) return undefined; switch (key) { case "primary-button": if (this.isLeft) key = "x-button"; else if (this.isRight) key = "a-button"; else return undefined; break; case "primary": if (this.hand) { return this.getGesture("pinch"); } return this.toNeedleGamepadButton(0, key); case "xr-standard-trigger": if (this.inputSource.gamepad) { return this.toNeedleGamepadButton(0, key); } break; case "xr-standard-squeeze": if (this.inputSource.gamepad) { return this.toNeedleGamepadButton(1, key); } break; case "xr-standard-thumbstick": if (this.inputSource.gamepad) { return this.toNeedleGamepadButton(3, key); } break; } if (this._buttonMap.has(key)) { return this.toNeedleGamepadButton(this._buttonMap.get(key)!, key); } const componentModel = this._layout?.components[key]; if (componentModel?.gamepadIndices) { switch (componentModel.type) { case "button": case "squeeze": if (this.inputSource.gamepad) { const index = componentModel.gamepadIndices!.button!; this._buttonMap.set(key, index); return this.toNeedleGamepadButton(index, key); } break; default: console.warn("Unsupported component type", componentModel.type); break; } } this._buttonMap.set(key, undefined!); return undefined; } /** Get a gesture state */ getGesture(key: XRGestureName): NeedleGamepadButton | null { const state = this.states[key]; if (!state) return null; this.states[key] = state; const needleButton = this._needleGamepadButtons[key] || new NeedleGamepadButton(undefined, key); needleButton.pressed = state.pressed; needleButton.value = state.value; needleButton.isDown = state.isDown; needleButton.isUp = state.isUp; this._needleGamepadButtons[key] = needleButton; return needleButton; } /** * Get the pointer id for a specific button of this input device. * This is useful if you want to check if a button (e.g. trigger) is currently being in use which can be queried on the inputsystem. * @returns the pointer id for the button or undefined if the button is not supported * @example * ```ts * const pointerId = controller.getPointerId("primary"); * if (pointerId !== undefined) { * const isUsed = this.context.input.getPointerUsed(pointerId); * console.log(controller.side, "used?", isUsed); * } * ``` */ getPointerId(button: number): number; getPointerId(button: NeedleXRControllerButtonName | XRGestureName): number | undefined; getPointerId(button: number | NeedleXRControllerButtonName | XRGestureName): number | undefined { if (button === "primary") { button = 0; } else if (button === "pinch") { button = 0; } if (typeof button !== "number") { const needleButton = this._buttonMap.get(button); if (needleButton === undefined) { return undefined; } button = needleButton; } return this.index * 10 + button; } private readonly _needleGamepadButtons: { [key: number | string]: NeedleGamepadButton } = {}; /** combine the InputState information + the GamepadButton information (since GamepadButtons can not be extended) */ private toNeedleGamepadButton(index: number, name: string): NeedleGamepadButton | undefined { if (!this.inputSource.gamepad?.buttons) return undefined const button = this.inputSource.gamepad?.buttons[index]; const state = this.states[index]; const needleButton = this._needleGamepadButtons[index] || new NeedleGamepadButton(index, name); if (button) { needleButton.pressed = button.pressed; needleButton.value = button.value; needleButton.touched = button.touched; } if (state) { needleButton.isDown = state.isDown; needleButton.isUp = state.isUp; } this._needleGamepadButtons[index] = needleButton; return needleButton; } /** * Get the values of a controller joystick * @link https://github.com/immersive-web/webxr-gamepads-module/blob/main/gamepads-module-explainer.md * @returns the stick values where x is left/right, y is up/down and z is the button value */ getStick(key: StickName | "primary"): Vec3 { if (!this._layout) return { x: 0, y: 0, z: 0 }; // Hands dont have thumbsicks if (this.isHand) { return { x: 0, y: 0, z: 0 }; } if (key === "primary") { if (this._layout.components["xr-standard-thumbstick"]) key = "xr-standard-thumbstick"; // const x = this.inputSource.gamepad?.axes[0] || 0; // const y = this.inputSource.gamepad?.axes[1] || 0; // // the primary thumbstick is button 3 (see gamepads module explainer) // const z = this.inputSource.gamepad?.buttons[3]?.value || 0; // return { x, y, z } } const componentModel = this._layout?.components[key]; if (componentModel?.gamepadIndices) { switch (componentModel.type) { case "thumbstick": case "touchpad": if (this.inputSource.gamepad) { const xIndex = componentModel.gamepadIndices!.xAxis!; const yIndex = componentModel.gamepadIndices!.yAxis!; let x = this.inputSource.gamepad.axes[xIndex] || 0; let y = this.inputSource.gamepad.axes[yIndex] || 0; x *= -1; y *= -1; const buttonIndex = componentModel.gamepadIndices!.button!; const z = this.inputSource.gamepad?.buttons[buttonIndex]?.value || 0; return { x, y, z } } } } return { x: 0, y: 0, z: 0 } } private readonly _buttonMap = new Map(); // the motion controller contains the controller scheme, we use this to simplify button access private _motioncontroller?: MotionController; private _layout: InputDeviceLayout | undefined; private getMotionController!: Promise; private initialize() { // WORKAROUND for hand controllers that don't have a select event this._hasSelectEvent = this.profiles.includes("generic-hand-select") || this.profiles.some(p => p.startsWith("generic-trigger")); // Used to determine special layout for Quest controllers, e.g. last button is menu button this._isMetaQuestTouchController = this.profiles.includes("meta-quest-touch-plus") || this.profiles.includes("oculus-touch-v3"); // Proper profile starting with v69 and browser 35.1 this._isMxInk = this.profiles.includes("logitech-mx-ink"); // For debugging to see ALL available profiles /** @ts-ignore */ // fetchProfilesList(DEFAULT_PROFILES_PATH).then(list => console.log("Available controller profiles", list)); if (!this._layout) { // Ignore transient-pointer since we likely don't want to spawn a controller visual just for a temporary pointer. // TODO we should check how this is actually handled on Quest Browser when the transient-pointer flag is on. if (this.inputSource.targetRayMode as XRTargetRayMode | "transient-pointer" === "transient-pointer") return; // TODO: we should fetch the profiles or better yet the profile list once and cache it const fetchProfileCall = fetchProfile(this.inputSource, DEFAULT_PROFILES_PATH, DEFAULT_PROFILE); /** @ts-ignore */ this.getMotionController = fetchProfileCall.then(res => { if (!this.connected) return null; this._motioncontroller = new MotionController( this.inputSource, res.profile, res.assetPath || "" ); // const overrideProfile = await fetch(DEFAULT_PROFILES_PATH + "/htc-vive-focus-3/profile.json").then(r => r.json()); const profile = res.profile as InputDeviceProfile; const layout = profile.layouts[this.inputSource.handedness]; this._layout = layout; if (this._layout) { if (!this._layout.gamepad?.length) { this._layout.gamepad = []; for (const key in this._layout.components) { const component = this._layout.components[key]; this._layout.gamepad[component.gamepadIndices!.button!] = key as XRControllerButtonName; } } // If we have 4 axes and no thumbstick defined, we define thumbstick for axis 3+4 // This is a workaround for HTC Vive Focus 3 controllers, which have the profile for Vive Focus Plus... // This workaround fixes it for HTC Vive Focus 3 but does not change anything for Vive Focus Plus controllers if (this.profiles.length >= 1 && this.profiles[0] === "htc-vive-focus-plus") { if (this.inputSource.gamepad && this.inputSource.gamepad.axes.length === 4 && !this._layout.components["xr-standard-thumbstick"]) { this._layout.components["xr-standard-thumbstick"] = { type: "thumbstick", gamepadIndices: { xAxis: 2, yAxis: 3, } } } } } // if (debug) console.log(this._layout, this.inputSource); // debugger; // this.getButton("a-button") return this._motioncontroller; }).catch(err => { if (this.inputSource) console.warn("Couldn't initialize motion controller profile for ", this.inputSource, err); return null; }); } } /** * When enabled the controller will automatically emit pointer down events to the Needle Engine Input System. * @default true */ emitPointerDownEvent: boolean = true; /** * When enabled the controller will automatically emit pointer up events to the Needle Engine Input System. * @default true */ emitPointerUpEvent: boolean = true; /** * When enabled the controller will automatically emit pointer move events to the Needle Engine Input System. * @default true */ emitPointerMoveEvent: boolean = true; /** * The distance threshold for pointer move events. This value is in units in rig space * @default 0.03 */ pointerMoveDistanceThreshold = 0.03; /** * The angle threshold for pointer move events. This value is in radians. * @default 0.05 */ pointerMoveAngleThreshold = 0.05; private subscribeEvents() { // https://developer.mozilla.org/en-US/docs/Web/API/XRSession/selectstart_event this.xr.session.addEventListener("selectstart", this.onSelectStart); this.xr.session.addEventListener("selectend", this.onSelectEnd); // https://developer.mozilla.org/en-US/docs/Web/API/XRSession/squeeze_event this.xr.session.addEventListener("squeezestart", this.onSequeezeStart); this.xr.session.addEventListener("squeezeend", this.onSequeezeEnd); } private unsubscribeEvents() { this.xr.session.removeEventListener("selectstart", this.onSelectStart); this.xr.session.removeEventListener("selectend", this.onSelectEnd); this.xr.session.removeEventListener("squeezestart", this.onSequeezeStart); this.xr.session.removeEventListener("squeezeend", this.onSequeezeEnd); } private _selectButtonIndex: number | undefined = undefined; private _squeezeButtonIndex: number | undefined = undefined; private onSelectStart = (evt: XRInputSourceEvent) => { if (!this.emitPointerDownEvent) return; if (this.inputSource !== evt.inputSource) return; // if a selectstart event happens right after an input source is connected, we may even receive this event before // requestAnimationFrame callback with the current session. So, we need to update the frame here. this.onUpdateFrame(evt.frame); // if we receive a select event we can be true that this device supports select events this._hasSelectEvent = true; const selectComponentId = this._layout?.selectComponentId; const i = this._layout?.components[selectComponentId!]?.gamepadIndices?.button; if (i !== undefined) this._selectButtonIndex = i; if (debugCustomGesture) return; /* if (!_didReceiveSelectStartEvent) { _didReceiveSelectStartEvent = true; // safeguard first pinch event - check if the pinch gesture is already down const pinch = this.getGesture("pinch"); if (pinch?.pressed) { console.warn("Select start event was received but the pinch gesture is already down. This might happen the first time you start pinching", this.index, this.side); return; } } */ if (debug) Gizmos.DrawDirection(this.rayWorldPosition, getTempVector(0, .01, 1).applyQuaternion(this.rayWorldQuaternion), 0xff0000, 10); this.emitPointerEvent(InputEvents.PointerDown, this._selectButtonIndex || 0, "xr-standard-trigger", true, evt); } private onSelectEnd = (evt: XRInputSourceEvent) => { if (!this.emitPointerUpEvent) return; if (debugCustomGesture) return; if (this.inputSource !== evt.inputSource) return; this.emitPointerEvent(InputEvents.PointerUp, this._selectButtonIndex || 0, "xr-standard-trigger", true, evt); } private onSequeezeStart = (evt: XRInputSourceEvent) => { if (!this.emitPointerDownEvent) return; if (this.inputSource !== evt.inputSource) return; this._squeezeButtonIndex = this._layout?.components["xr-standard-squeeze"]?.gamepadIndices?.button; if (this._squeezeButtonIndex !== undefined) { if (debug) Gizmos.DrawDirection(this.rayWorldPosition, getTempVector(0, .01, 1).applyQuaternion(this.rayWorldQuaternion), 0x0000ff, 10); this.emitPointerEvent(InputEvents.PointerDown, this._squeezeButtonIndex || 0, "xr-standard-squeeze", true, evt); } }; private onSequeezeEnd = (evt: XRInputSourceEvent) => { if (!this.emitPointerUpEvent) return; if (this.inputSource !== evt.inputSource) return; if (this._squeezeButtonIndex !== undefined) this.emitPointerEvent(InputEvents.PointerUp, this._squeezeButtonIndex || 0, "xr-standard-squeeze", true, evt); }; /** Index = button index */ private readonly states: { [key: number | string]: InputState } = {}; // If we want to invoke button events for ALL buttons we need to keep track of the previous state // instead of using XR input select start events which is only raised for the primary button // we should probably do both but then we need to ignore the primary index in the following function (to not raise an event for the same button twice) // and start with index = 1 private updateInputEvents() { // https://immersive-web.github.io/webxr-gamepads-module/#xr-standard-heading if (this.gamepad?.buttons) { for (let index = 0; index < this.gamepad.buttons.length; index++) { const button = this.gamepad.buttons[index]; const state = this.states[index] || new InputState(); let eventName: InputEventNames | null = null; // Special handling for MX Ink stylus on Quest OS v69+. // We're never getting a "pressed" state here, so we determine pressed state based on the value. if (this._isMxInk && (index === 4 || index === 5)) { if (button.value > 0 && !state.pressed) { eventName = "pointerdown"; state.isDown = true; state.isUp = false; } else if (button.value === 0 && state.pressed) { eventName = "pointerup"; state.isDown = false; state.isUp = true; } else if (state.pressed) { eventName = "pointermove"; state.isDown = false; state.isUp = false; } state.pressed = button.value > 0; state.value = button.value; } // Regular controller handling. else { // is down if (button.pressed && !state.pressed) { eventName = "pointerdown"; state.isDown = true; state.isUp = false; } // is up else if (!button.pressed && state.pressed) { eventName = "pointerup" state.isDown = false; state.isUp = true; } else { state.isDown = false; state.isUp = false; } state.pressed = button.pressed; state.value = button.value; } this.states[index] = state; // the selection event is handled in the "selectstart" callback const emitEvent = index !== this._selectButtonIndex && index !== this._squeezeButtonIndex; if (eventName != null && emitEvent) { let name = this._layout?.gamepad[index]; if (this._isMxInk && index === 4) name = "stylus-touch"; if (this._isMxInk && index === 5) name = "stylus-tip"; if (debug || debugCustomGesture) console.log("Emitting pointer event", eventName, index, name, button.value, this.gamepad, this._layout); this.emitPointerEvent(eventName, index, name ?? "none", false, null, button.value); } } // For Quest controllers, the last button is the menu button if (this._isMetaQuestTouchController) { const menuButtonIndex = this.gamepad.buttons.length - 1; const menuButtonState = this.states[menuButtonIndex]; if (menuButtonState) { if (menuButtonState.isDown) { const menu = this.context.menu; if (menu.spatialMenuIsVisible) menu.setSpatialMenuVisible(false); else this.context.menu.setSpatialMenuVisible(true); } } } } // update hand gesture states if (this.hand) { const handObject = this.handObject; if (handObject) { // update pinch state const indexTip = handObject.joints["index-finger-tip"]; const thumbTip = handObject.joints["thumb-tip"]; if (indexTip && thumbTip) { const distance = indexTip.position.distanceTo(thumbTip.position); // upddate position of the pinch point this._pinchPosition.lerpVectors(indexTip.position, thumbTip.position, .5); const parent = this.xr.context.mainCamera?.parent; if (parent) this._pinchPosition.applyMatrix4(parent.matrixWorld); if (distance !== 0) { // ignore exactly 0 which happens when we switch from hands to controllers const pinchThreshold = .02; const pinchHysteresis = .01; const state = this.states["pinch"] || new InputState(); const maxDistance = (pinchThreshold + pinchHysteresis) * 1.5; state.value = 1 - ((distance - pinchThreshold) / maxDistance); const isPressed = distance < (pinchThreshold - pinchHysteresis); const isReleased = distance > (pinchThreshold + pinchHysteresis); if (isPressed && !state.pressed) { if (debugCustomGesture) console.log("pinch start", distance); state.isDown = true; state.isUp = false; state.pressed = true; } else if (isReleased && state.pressed) { state.isDown = false; state.isUp = true; state.pressed = false; } else { state.isDown = false; state.isUp = false; } this.states["pinch"] = state; } /** Workaround for visionOS not emitting selectstart. See https://linear.app/needle/issue/NE-4212 * If a selectstart event was never received we do a manual check here if the user is pinching * Update: VisionOS 1.1 now properly emits select events from transient input sources, based on gaze. * We're keeping this code commented for now since there may be future changes before VisionOS WebXR ships. */ /* if (!_didReceiveSelectStartEvent && (state.isDown || state.isUp)) { const eventName = isPressed ? "pointerdown" : "pointerup"; const pressure = distance / pinchThreshold; if (debugCustomGesture) { const p = this.rayWorldPosition.add(this.object.worldForward.multiplyScalar(.2)); p.y += .05; p.y += Math.random() * .02; Gizmos.DrawLabel(p, "pinch:" + eventName + ", " + this.index + ", " + this.side + "\n" + handObject.uuid, 0.01, 5, 0x000000, new RGBAColor(1, 1, 1, .1)); } this.emitPointerEvent(eventName, 0, "pinch", false, null, pressure); } */ } } } } private _didMoveLastFrame = false; private readonly _lastPointerMovePosition = new Vector3(); private readonly _lastPointerMoveQuaternion = new Quaternion(); private onUpdateMove() { if (!this.emitPointerMoveEvent) return; let didMove = false; const dist = this._lastPointerMovePosition.distanceTo(this.gripWorldPosition); if (dist > this.pointerMoveDistanceThreshold * this.xr.rigScale) didMove = true; if (!didMove) { const angle = this._lastPointerMoveQuaternion.angleTo(this.gripWorldQuaternion); if (angle > this.pointerMoveAngleThreshold) didMove = true; } if (didMove) { this._didMoveLastFrame = true; this._lastPointerMovePosition.copy(this.gripWorldPosition); this._lastPointerMoveQuaternion.copy(this.gripWorldQuaternion); if (debug) Gizmos.DrawLabel(this.rayWorldPosition.add(this.object.worldForward.multiplyScalar(.1)), "move", .01); let button = this.xr.context.input.getFirstPressedButtonForPointer(this.index); if (button === undefined) button = 0; const pressure = this.gamepad?.buttons[button]?.value; this.emitPointerEvent("pointermove", button, "none", false, null, pressure); } else { this._didMoveLastFrame = false; } } /** cached spatial pointer init object. We re-use it to not have */ private readonly pointerInit: NEPointerEventInit; private emitPointerEvent(type: InputEventNames, button: number, buttonName: ButtonName | "none", primary: boolean, source: Event | null = null, pressure?: number) { if (!this.emitEvents) { if (debug && type !== InputEvents.PointerMove) console.warn("Pointer events are disabled for this controller", this.index, type, button); return; } // Currently we do only want to emit pointer events for NON screen based events // that means if the input device is spatial (AR touch on a screen should be handled via touchdown etc still) // Not sure if *this* is enough to determine if the event is spatial or not if (this.xr.mode === "immersive-vr" || this.xr.isPassThrough) { this.pointerInit.origin = this; this.pointerInit.pointerId = this.getPointerId(button); this.pointerInit.pointerType = this.hand ? "hand" : "controller"; this.pointerInit.button = button; this.pointerInit.buttonName = buttonName; this.pointerInit.isPrimary = primary; this.pointerInit.mode = this.inputSource.targetRayMode; this.pointerInit.ray = this.ray; this.pointerInit.device = this.object; this.pointerInit.pressure = pressure; this.pointerInit.clientX = this._rayPosition.x / this.xr.rigScale; this.pointerInit.clientY = this._rayPosition.y / this.xr.rigScale; this.pointerInit.clientZ = this._rayPosition.z / this.xr.rigScale; const prevContext = Context.Current; Context.Current = this.xr.context; if (debug && type !== "pointermove") console.warn("Pointer event", type, button, buttonName, { ...this.pointerInit }); this.xr.context.input.createInputEvent(new NEPointerEvent(type, source, this.pointerInit)); Context.Current = prevContext; } } } class InputState { /** if the button was pressed the last update */ isDown: boolean = false; /** if the button was released the last update */ isUp: boolean = false; pressed: boolean = false; value: number = 0; }; /** Enhanced GamepadButton with `isDown` and `isUp` information */ class NeedleGamepadButton { /** The index of the button in the input gamepad */ readonly index: number | undefined; readonly name: string; touched: boolean = false; pressed: boolean = false; value: number = 0; /** was the button just pressed down the last update */ isDown: boolean = false; /** was the button just released the last update */ isUp: boolean = false; constructor(index: number | undefined, name: string) { this.index = index; this.name = name; } }