import { Object3D } from "three";
import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
import { AssetReference } from "../../engine/engine_addressables.js";
import { findObjectOfType } from "../../engine/engine_components.js";
import { serializable } from "../../engine/engine_serialization.js";
import { delayForFrames, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
import { type NeedleXREventArgs, NeedleXRSession } from "../../engine/engine_xr.js";
import { ButtonsFactory } from "../../engine/webcomponents/buttons.js";
import { WebXRButtonFactory } from "../../engine/webcomponents/WebXRButtons.js";
import { PlayerSync } from "../../engine-components-experimental/networking/PlayerSync.js";
import { Behaviour, GameObject } from "../Component.js";
import { USDZExporter } from "../export/usdz/USDZExporter.js";
import { NeedleMenu } from "../NeedleMenu.js";
import { SpatialGrabRaycaster } from "../ui/Raycaster.js";
import { Avatar } from "./Avatar.js";
import { XRControllerModel } from "./controllers/XRControllerModel.js";
import { XRControllerMovement } from "./controllers/XRControllerMovement.js";
import { WebARSessionRoot } from "./WebARSessionRoot.js";
import { XRState, XRStateFlag } from "./XRFlag.js";
const debug = getParam("debugwebxr");
const debugQuicklook = getParam("debugusdz");
/**
* Use the [WebXR](https://engine.needle.tools/docs/api/WebXR) component to enable VR and AR on **iOS and Android** in your scene. VisionOS support is also provided via QuickLook USDZ export.
*
* The WebXR component is a simple to use wrapper around the {@link NeedleXRSession} API and adds some additional features like creating buttons for AR, VR, enabling default movement behaviour ({@link XRControllerMovement}) and controller rendering ({@link XRControllerModel}), as well as handling AR placement and Quicklook USDZ export.
*
* 
*
*
* @example Enable VR and AR support using code
* ```ts
* import { onStart, WebXR } from "@needle-tools/engine";
* onStart(context => {
* const webxr = context.scene.addComponent(WebXR, { createVRButton: true, createARButton: true });
* });
* ```
*
* @example Customize VR movement
* ```ts
* import { onStart, WebXR } from "@needle-tools/engine";
* onStart(context => {
* const webxr = context.scene.addComponent(WebXR, { createVRButton: true });
* const movement = webxr.setDefaultMovementEnabled(true);
* if (movement) {
* movement.enableTeleport = false; // disable teleport, only use smooth locomotion
* movement.smoothMovementSpeed = 2; // increase speed
* // NOTE: you can also disable default movement and write your own movement component (or derive and extend the {@link XRControllerMovement} class)
* }
* });
* ```
*
*
* @example Start AR session with placement reticle and touch to place and adjust the scene
* ```ts
* import { onStart, WebXR } from "@needle-tools/engine";
* onStart(context => {
* const webxr = context.scene.addComponent(WebXR);
* webxr.autoPlace = false; // disable auto placement, we want the user to tap to place the scene
* webxr.usePlacementReticle = true; // show the placement reticle to help the user find surfaces to place the scene
* webxr.usePlacementAdjustment = true; // allow the user to adjust the position, rotation and scale of the scene with touch after placing
* webxr.arScale = 2; // set the initial scale of the scene in AR. Larger values make the scene appear smaller in AR.
* webxr.enterAR(); // start AR session
* });
*
* @summary WebXR Component for VR and AR support
* @category XR
* @group Components
* @see {@link NeedleXRSession} for low-level XR API
* @see {@link XRControllerMovement} for VR locomotion
* @see {@link WebARSessionRoot} for AR session configuration and an AR content placement event
* @see {@link Avatar} for networked user avatars
* @see {@link screenshot2} for taking screenshots in XR (including AR camera feed compositing)
* @link https://engine.needle.tools/docs/xr.html
* @link https://engine.needle.tools/samples/?overlay=samples&tag=xr
* @link https://engine.needle.tools/samples/collaborative-sandbox
*/
export class WebXR extends Behaviour {
// UI
/**
* When enabled, a button will be automatically added to {@link NeedleMenu} that allows users to enter VR mode.
*/
@serializable()
createVRButton: boolean = true;
/**
* When enabled, a button will be automatically added to {@link NeedleMenu} that allows users to enter AR mode.
*/
@serializable()
createARButton: boolean = true;
/**
* When enabled, a button to send the experience to an Oculus Quest will be shown if the current device does not support VR.
* This helps direct users to compatible devices for optimal VR experiences.
*/
@serializable()
createSendToQuestButton: boolean = true;
/**
* When enabled, a QR code will be generated and displayed on desktop devices to allow easy opening of the experience on mobile devices.
*/
@serializable()
createQRCode: boolean = true;
// VR Settings
/**
* When enabled, default movement controls will be automatically added to the scene when entering VR.
* This includes teleportation and smooth locomotion options for VR controllers.
*/
@serializable()
useDefaultControls: boolean = true;
/**
* When enabled, 3D models representing the user's VR controllers will be automatically created and rendered in the scene.
*/
@serializable()
showControllerModels: boolean = true;
/**
* When enabled, 3D models representing the user's hands will be automatically created and rendered when hand tracking is available.
*/
@serializable()
showHandModels: boolean = true;
// AR Settings
/**
* When enabled, a reticle will be displayed to help place the scene in AR. The user must tap on a detected surface to position the scene.
*/
@serializable()
usePlacementReticle: boolean = true;
/**
* Optional custom 3D object to use as the AR placement reticle instead of the default one.
*/
@serializable(AssetReference)
customARPlacementReticle?: AssetReference;
/**
* When enabled, users can adjust the position, rotation, and scale of the AR scene with one or two fingers after initial placement.
*/
@serializable()
usePlacementAdjustment: boolean = true;
/**
* Determines the scale of the user relative to the scene in AR. Larger values make the 3D content appear smaller.
* Only applies when `usePlacementReticle` is enabled.
*/
@serializable()
arScale: number = 1;
/**
* When enabled, an XRAnchor will be created for the AR scene and its position will be regularly updated to match the anchor.
* This can help with spatial persistence in AR experiences.
* @experimental
*/
@serializable()
useXRAnchor: boolean = false;
/**
* When enabled, the scene will be automatically placed as soon as a suitable surface is detected in AR,
* without requiring the user to tap to confirm placement.
*/
@serializable()
autoPlace: boolean = false;
/**
* When enabled, the AR session root center will be automatically adjusted to place the center of the scene.
* This helps ensure the scene is properly aligned with detected surfaces.
*
* **Note**: This option overrides the placement of the {@link WebARSessionRoot} component if both are used.
*/
@serializable()
autoCenter: boolean = false;
/**
* When enabled, a USDZExporter component will be automatically added to the scene if none is found.
* This allows iOS and visionOS devices to view 3D content using Apple's AR QuickLook.
*/
@serializable()
useQuicklookExport: boolean = false;
/**
* When enabled, the 'depth-sensing' WebXR feature will be requested to provide real-time depth occlusion.
* Currently only supported on Oculus Quest devices.
* @see https://developer.mozilla.org/en-US/docs/Web/API/XRDepthInformation
* @experimental
*/
@serializable()
useDepthSensing: boolean = false;
/**
* When enabled, a {@link SpatialGrabRaycaster} will be added or enabled in the scene,
* allowing users to interact with objects at a distance in VR/AR.
* @default true
*/
@serializable()
useSpatialGrab: boolean = true;
/**
* Specifies the avatar representation that will be created when entering a WebXR session.
* Can be a reference to a 3D model or a boolean to use the default avatar.
*/
@serializable(AssetReference)
defaultAvatar?: AssetReference | boolean;
private _playerSync?: PlayerSync;
/** these components were created by the WebXR component on session start and will be cleaned up again in session end */
private readonly _createdComponentsInSession: Behaviour[] = [];
private _usdzExporter?: USDZExporter;
static activeWebXRComponent: WebXR | null = null;
/**
* Initializes the WebXR component by obtaining the XR sync object for this context.
* @internal
*/
awake() {
NeedleXRSession.getXRSync(this.context);
}
/**
* Sets up the WebXR component when it's enabled. Checks for HTTPS connection,
* sets up USDZ export if enabled, creates UI buttons, and configures avatar settings.
* @internal
*/
onEnable(): void {
// check if we're on a secure connection:
if (window.location.protocol !== "https:") {
showBalloonWarning("WebXR only works on secure connections (https).");
}
// Showing the QuickLook button depends on whether we're on iOS or visionOS –
// on iOS we have AppClip support, so we don't need QuickLook button there,
// while on visionOS we use QuickLook for AR experiences for now (unless it happens to have WebXR support by then).
navigator.xr?.isSessionSupported("immersive-ar").catch(() => false).then((arSupported) => {
const isVisionOSFallback = DeviceUtilities.isVisionOS() && !arSupported;
if (this.useQuicklookExport || isVisionOSFallback) {
const existingUSDZExporter = GameObject.findObjectOfType(USDZExporter);
if (!existingUSDZExporter) {
// if no USDZ Exporter is found we add one and assign the scene to be exported
if (debug) console.log("WebXR: Adding USDZExporter");
this._usdzExporter = GameObject.addComponent(this.gameObject, USDZExporter);
this._usdzExporter.objectToExport = this.context.scene;
this._usdzExporter.autoExportAnimations = true;
this._usdzExporter.autoExportAudioSources = true;
}
}
});
this.handleCreatingHTML();
this.handleOfferSession();
// If the avatar is undefined (meaning not set and also not null) we will use the fallback default avatar
if (this.defaultAvatar === true) {
if (debug) console.warn("WebXR: No default avatar set, using static default avatar")
this.defaultAvatar = new AssetReference("https://cdn.needle.tools/static/avatars/DefaultAvatar.glb")
}
if (this.defaultAvatar) {
this._playerSync = this.gameObject.getOrAddComponent(PlayerSync);
this._playerSync.autoSync = false;
}
if (this._playerSync && typeof this.defaultAvatar != "boolean") {
this._playerSync.asset = this.defaultAvatar;
this._playerSync.onPlayerSpawned?.removeEventListener(this.onAvatarSpawned);
this._playerSync.onPlayerSpawned?.addEventListener(this.onAvatarSpawned);
}
}
/**
* Cleans up resources when the component is disabled.
* Destroys the USDZ exporter if one was created and removes UI buttons.
* @internal
*/
onDisable(): void {
this._usdzExporter?.destroy();
this.removeButtons();
}
/**
* Checks if WebXR is supported and offers an appropriate session.
* This is used to show the WebXR session joining prompt in browsers that support it.
* @returns A Promise that resolves to true if a session was offered, false otherwise
*/
private async handleOfferSession() {
if (this.createVRButton) {
const hasVRSupport = await NeedleXRSession.isVRSupported();
if (hasVRSupport && this.createVRButton) {
return NeedleXRSession.offerSession("immersive-vr", "default", this.context);
}
}
if (this.createARButton) {
const hasARSupport = await NeedleXRSession.isARSupported();
if (hasARSupport && this.createARButton) {
return NeedleXRSession.offerSession("immersive-ar", "default", this.context);
}
}
return false;
}
/** the currently active webxr input session */
get session(): NeedleXRSession | null {
return NeedleXRSession.active ?? null;
}
/** immersive-vr or immersive-ar */
get sessionMode(): XRSessionMode | null {
return NeedleXRSession.activeMode ?? null;;
}
/** While AR: this will return the currently active WebARSessionRoot component.
* You can also query this component in your scene with `findObjectOfType(WebARSessionRoot)`
*/
get arSessionRoot() {
return this._activeWebARSessionRoot;
}
/** Call to start an WebVR session.
*
* This is a shorthand for `NeedleXRSession.start("immersive-vr", init, this.context)`
*/
async enterVR(init?: XRSessionInit): Promise {
return NeedleXRSession.start("immersive-vr", init, this.context);
}
/** Call to start an WebAR session
*
* This is a shorthand for `NeedleXRSession.start("immersive-ar", init, this.context)`
*/
async enterAR(init?: XRSessionInit): Promise {
return NeedleXRSession.start("immersive-ar", init, this.context);
}
/** Call to end a WebXR (AR or VR) session.
*
* This is a shorthand for `NeedleXRSession.stop()`
*/
exitXR() {
NeedleXRSession.stop();
}
private _exitXRMenuButton?: HTMLElement;
private _previousXRState: number = 0;
private _spatialGrabRaycaster?: SpatialGrabRaycaster;
private _activeWebARSessionRoot: WebARSessionRoot | null = null;
private get isActiveWebXR() {
return !WebXR.activeWebXRComponent || WebXR.activeWebXRComponent === this;
}
/**
* Called before entering a WebXR session. Sets up optional features like depth sensing, if needed.
* @param _mode The XR session mode being requested (immersive-ar or immersive-vr)
* @param args The XRSessionInit object that will be passed to the WebXR API
* @internal
*/
onBeforeXR(_mode: XRSessionMode, args: XRSessionInit): void {
if (!this.isActiveWebXR) {
console.warn(`WebXR: another WebXR component is already active (${WebXR.activeWebXRComponent?.name}). This is ignored: ${this.name}`);
return;
}
if (this.activeAndEnabled === false || this.destroyed) {
console.debug("[WebXR] onBeforeXR called on disabled or destroyed component");
return;
}
WebXR.activeWebXRComponent = this;
if (_mode == "immersive-ar" && this.useDepthSensing) {
args.optionalFeatures = args.optionalFeatures || [];
args.optionalFeatures.push("depth-sensing");
}
}
/**
* Called when a WebXR session begins. Sets up the scene for XR by configuring controllers,
* AR placement, and other features based on component settings.
* @param args Event arguments containing information about the started XR session
* @internal
*/
async onEnterXR(args: NeedleXREventArgs) {
if (!this.isActiveWebXR) return;
if (debug) console.log("WebXR onEnterXR")
// set XR flags
this._previousXRState = XRState.Global.Mask;
const isVR = args.xr.isVR;
XRState.Global.Set(isVR ? XRStateFlag.VR : XRStateFlag.AR);
// Handle AR session root
if (args.xr.isAR) {
let sessionroot = GameObject.findObjectOfType(WebARSessionRoot, this.context, false);
// Only create a WebARSessionRoot if none is in the scene already
if (!sessionroot) {
if (this.usePlacementReticle) {
const implicitSessionRoot = new Object3D();
for (const ch of this.context.scene.children)
implicitSessionRoot.add(ch);
this.context.scene.add(implicitSessionRoot);
sessionroot = GameObject.addComponent(implicitSessionRoot, WebARSessionRoot)!;
this._createdComponentsInSession.push(sessionroot);
}
else if (debug || isDevEnvironment()) {
console.warn("WebXR: No WebARSessionRoot found in scene and usePlacementReticle is disabled in WebXR component.")
}
}
this._activeWebARSessionRoot = sessionroot;
if (sessionroot) {
// sessionroot.enabled = this.usePlacementReticle; // < not sure if we want to disable the session root when placement reticle if OFF...
sessionroot.customReticle = this.customARPlacementReticle;
sessionroot.arScale = this.arScale;
sessionroot.arTouchTransform = this.usePlacementAdjustment;
sessionroot.autoPlace = this.autoPlace;
sessionroot.autoCenter = this.autoCenter;
sessionroot.useXRAnchor = this.useXRAnchor;
}
}
// handle VR controls
if (this.useDefaultControls) {
this.setDefaultMovementEnabled(true);
}
if (this.showControllerModels || this.showHandModels) {
this.setDefaultControllerRenderingEnabled(true);
}
// ensure we have a spatial grab raycaster for close grabs
if (this.useSpatialGrab) {
this._spatialGrabRaycaster = GameObject.findObjectOfType(SpatialGrabRaycaster) ?? undefined;
if (!this._spatialGrabRaycaster) {
this._spatialGrabRaycaster = this.gameObject.addComponent(SpatialGrabRaycaster);
}
}
this.createLocalAvatar(args.xr);
// for mobile screen AR we have the "X" button to exit and don't need to add an extra menu button to close
// https://linear.app/needle/issue/NE-5716
if (args.xr.isScreenBasedAR) {
}
else {
this._exitXRMenuButton = this.context.menu.appendChild({
label: "Quit XR",
onClick: () => this.exitXR(),
icon: "exit_to_app",
priority: 20_000,
});
}
}
/**
* Called every frame during an active WebXR session.
* Updates components that depend on the current XR state.
* @param _args Event arguments containing information about the current XR session frame
* @internal
*/
onUpdateXR(_args: NeedleXREventArgs): void {
if (!this.isActiveWebXR) return;
if (this._spatialGrabRaycaster) {
this._spatialGrabRaycaster.enabled = this.useSpatialGrab;
}
}
/**
* Called when a WebXR session ends. Restores pre-session state,
* removes temporary components, and cleans up resources.
* @param _ Event arguments containing information about the ended XR session
* @internal
*/
onLeaveXR(_: NeedleXREventArgs): void {
this._exitXRMenuButton?.remove();
if (!this.isActiveWebXR) return;
// revert XR flags
XRState.Global.Set(this._previousXRState);
this._playerSync?.destroyInstance();
for (const comp of this._createdComponentsInSession) {
comp.destroy();
}
this._createdComponentsInSession.length = 0;
this._activeWebARSessionRoot = null;
this.handleOfferSession();
delayForFrames(1).then(() => WebXR.activeWebXRComponent = null);
}
/** Call to enable or disable default controller behaviour */
setDefaultMovementEnabled(enabled: boolean): XRControllerMovement | null {
let movement = this.gameObject.getComponent(XRControllerMovement)
if (!movement && enabled) {
movement = this.gameObject.addComponent(XRControllerMovement)!;
this._createdComponentsInSession.push(movement);
}
if (movement) movement.enabled = enabled;
return movement;
}
/** Call to enable or disable default controller rendering */
setDefaultControllerRenderingEnabled(enabled: boolean): XRControllerModel | null {
let models = this.gameObject.getComponent(XRControllerModel);
if (!models && enabled) {
models = this.gameObject.addComponent(XRControllerModel)!;
this._createdComponentsInSession.push(models);
models.createControllerModel = this.showControllerModels;
models.createHandModel == this.showHandModels;
}
if (models) models.enabled = enabled;
return models;
}
/**
* Creates and instantiates the user's avatar representation in the WebXR session.
* @param xr The active session
*/
protected async createLocalAvatar(xr: NeedleXRSession) {
if (this._playerSync && xr.running && typeof this.defaultAvatar != "boolean") {
this._playerSync.asset = this.defaultAvatar;
await this._playerSync.getInstance();
}
}
/**
* Event handler called when a player avatar is spawned.
* Ensures the avatar has the necessary Avatar component.
* @param instance The spawned avatar 3D object
*/
private onAvatarSpawned = (instance: Object3D) => {
// spawned webxr avatars must have a avatar component
if (debug) console.log("WebXR.onAvatarSpawned", instance);
let avatar = GameObject.getComponentInChildren(instance, Avatar);
avatar ??= GameObject.addComponent(instance, Avatar)!;
};
// HTML UI
/** @deprecated use {@link getButtonsFactory} or directly access {@link WebXRButtonFactory.getOrCreate} */
getButtonsContainer(): WebXRButtonFactory {
return this.getButtonsFactory();
}
/**
* Returns the WebXR button factory, creating one if it doesn't exist.
* Use this to access and modify WebXR UI buttons.
* @returns The WebXRButtonFactory instance
*/
getButtonsFactory(): WebXRButtonFactory {
if (!this._buttonFactory) {
this._buttonFactory = WebXRButtonFactory.getOrCreate();
}
return this._buttonFactory;
}
/**
* Reference to the WebXR button factory used by this component.
*/
private _buttonFactory?: WebXRButtonFactory;
/**
* Creates and sets up UI elements for WebXR interaction based on component settings
* and device capabilities. Handles creating AR, VR, QuickLook buttons and utility buttons like QR codes.
*/
private handleCreatingHTML() {
if (this.createARButton || this.createVRButton || this.useQuicklookExport) {
// Quicklook / iOS
if ((DeviceUtilities.isiOS() && DeviceUtilities.isSafari()) || debugQuicklook) {
if (this.useQuicklookExport) {
const usdzExporter = GameObject.findObjectOfType(USDZExporter);
if (!usdzExporter || (usdzExporter && usdzExporter.allowCreateQuicklookButton)) {
const button = this.getButtonsFactory().createQuicklookButton();
this.addButton(button);
}
}
}
// WebXR
if (this.createARButton) {
const arbutton = this.getButtonsFactory().createARButton();
this.addButton(arbutton);
}
if (this.createVRButton) {
const vrbutton = this.getButtonsFactory().createVRButton();
this.addButton(vrbutton);
}
}
if (this.createSendToQuestButton && !DeviceUtilities.isQuest()) {
NeedleXRSession.isVRSupported().then(supported => {
if (!supported) {
const button = this.getButtonsFactory().createSendToQuestButton();
this.addButton(button);
}
});
}
if (this.createQRCode) {
const menu = findObjectOfType(NeedleMenu);
if (menu && menu.createQRCodeButton === false) {
// If the menu exists and the QRCode option is disabled we dont create it (NE-4919)
if (isDevEnvironment()) console.warn("WebXR: QRCode button is disabled in the Needle Menu component")
}
else if (!DeviceUtilities.isMobileDevice()) {
const qrCode = ButtonsFactory.getOrCreate().createQRCode();
this.addButton(qrCode,);
}
}
}
/**
* Storage for UI buttons created by this component.
*/
private readonly _buttons: HTMLElement[] = [];
/**
* Adds a button to the UI with the specified priority.
* @param button The HTML element to add
* @param priority The button's priority value (lower numbers appear first)
*/
private addButton(button: HTMLElement) {
this._buttons.push(button);
this.context.menu.appendChild(button);
}
/**
* Removes all buttons created by this component from the UI.
*/
private removeButtons() {
for (const button of this._buttons) {
button.remove();
}
this._buttons.length = 0;
}
}