// %BANNER_BEGIN% // --------------------------------------------------------------------- // %COPYRIGHT_BEGIN% // Copyright (c) (2024) Magic Leap, Inc. All Rights Reserved. // Use of this file is governed by the Software License Agreement, located here: https://www.magicleap.com/software-license-agreement-ml2 // Terms and conditions applicable to third-party materials accompanying this distribution may also be found in the top-level NOTICE file appearing herein. // %COPYRIGHT_END% // --------------------------------------------------------------------- // %BANNER_END% using System; using System.Collections.Generic; using System.Linq; using MagicLeap.Android; using MagicLeap.OpenXR.Constants; using MagicLeap.OpenXR.Features.Planes; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; using Unity.Jobs; using UnityEngine; using UnityEngine.Scripting; using UnityEngine.XR.ARSubsystems; using UnityEngine.XR.MagicLeap; using UnityEngine.XR.OpenXR; using UnityEngine.XR.OpenXR.NativeTypes; namespace MagicLeap.OpenXR.Subsystems { /// /// The Magic Leap implementation of the XRPlaneSubsystem. Do not create this directly. /// Use PlanesSubsystemDescriptor.Create() instead. /// [Preserve] public sealed partial class MLXrPlaneSubsystem : XRPlaneSubsystem { private static PlanesQuery QueryInternal; public static PlanesQuery Query { get => QueryInternal; set { QuerySet = true; QueryInternal = value; } } private static bool QuerySet { get; set; } public struct PlanesQuery { /// /// The flags to apply to this query. /// public MLPlanesQueryFlags Flags; /// /// The center of the bounding box which defines where planes extraction should occur. /// public Vector3 BoundsCenter; /// /// The rotation of the bounding box where planes extraction will occur. /// public Quaternion BoundsRotation; /// /// The size of the bounding box where planes extraction will occur. /// public Vector3 BoundsExtents; /// /// The maximum number of results that should be returned. /// public uint MaxResults; /// /// The minimum area (in square meters) of planes to be returned. This value /// cannot be lower than 0.04 (lower values will be capped to this minimum). /// public float MinPlaneArea; } private const ulong PlaneTrackableIdSalt = 0xf52b75076e45ad88; private class MagicLeapProvider : Provider { private readonly HashSet currentSet = new(); private readonly HashSet invalidatedPlanes = new(); private readonly Dictionary planes = new(); private readonly Dictionary boundariesTable = new(); private MLPlanesQueryFlags currentPlaneDetectionModeInternal; private MLPlanesQueryFlags defaultQueryFlags = MLPlanesQueryFlags.AllOrientations | MLPlanesQueryFlags.SemanticAll; private uint lastNumResults; private uint maxResults = 10; private XrPlaneDetector planesTracker = Values.NullHandle; private uint previousLastNumResults; private MLPlanesQueryFlags requestedPlaneDetectionModeInternal; private ScanState currentScanState = ScanState.Stopped; private PlanesNativeFunctions nativeFunctions; private MagicLeapPlanesFeature planesFeature; private PlanesQuery DefaultPlanesQuery { get { if (QuerySet) { return Query; } return new PlanesQuery { Flags = defaultQueryFlags, BoundsCenter = Vector3.zero, BoundsRotation = Quaternion.identity, BoundsExtents = Vector3.one * 20f, MaxResults = maxResults, MinPlaneArea = 10, }; } } public override PlaneDetectionMode requestedPlaneDetectionMode { get => requestedPlaneDetectionModeInternal.ToPlaneDetectionMode(); set { requestedPlaneDetectionModeInternal = value.ToMLXrQueryFlags(); defaultQueryFlags = requestedPlaneDetectionModeInternal | MLPlanesQueryFlags.SemanticAll; } } public override PlaneDetectionMode currentPlaneDetectionMode => currentPlaneDetectionModeInternal.ToPlaneDetectionMode(); private void CreateClient() { unsafe { if (planesTracker != Values.NullHandle) { return; } if (!Permissions.CheckPermission(Permissions.SpatialMapping)) { return; } var createInfo = new XrPlaneDetectorCreateInfo { Type = XrPlaneStructTypes.PlaneDetectorCreateInfo, Flags = XrPlaneDetectorFlags.XrPlaneDetectorEnableContourBit }; var result = nativeFunctions.XrCreatePlaneDetector(planesFeature.AppSession, in createInfo, out planesTracker); if (!Utils.DidXrCallSucceed(result, nameof(nativeFunctions.XrCreatePlaneDetector))) { return; } boundariesTable.Clear(); } } public override void Start() { SubsystemFeatures.SetFeatureRequested(Feature.PlaneTracking, true); planesFeature = OpenXRSettings.Instance.GetFeature(); nativeFunctions = planesFeature.PlanesNativeFunctions; } public override void Stop() { currentScanState = ScanState.Stopped; if (planesTracker != Values.NullHandle) { unsafe { nativeFunctions.XrDestroyPlaneDetector(planesTracker); planesTracker = Values.NullHandle; boundariesTable.Clear(); } } SubsystemFeatures.SetFeatureRequested(Feature.PlaneTracking, false); } public override void Destroy() { } private PlaneBoundary GetBoundaryOfPlane(in TrackableId trackableId) { if (!planes.TryGetValue(trackableId, out _)) { return default; } return boundariesTable.GetValueOrDefault(trackableId); } public override void GetBoundary(TrackableId trackableId, Allocator allocator, ref NativeArray convexHullOut) { var boundary = GetBoundaryOfPlane(in trackableId); if (boundary?.PolygonVertexCount > 0) { var polygon = boundary.GetPolygon(Allocator.Temp); ConvexHullGenerator.GiftWrap(polygon, allocator, ref convexHullOut); return; } if (planes.TryGetValue(trackableId, out var plane)) { var halfHeight = plane.height * 0.5f; var halfWidth = plane.width * 0.5f; var calculatedBoundaries = new NativeArray(4, Allocator.Temp); calculatedBoundaries[0] = new Vector2(halfWidth, halfHeight); calculatedBoundaries[1] = new Vector2(-halfWidth, halfHeight); calculatedBoundaries[2] = new Vector2(-halfWidth, -halfHeight); calculatedBoundaries[3] = new Vector2(halfWidth, -halfHeight); ConvexHullGenerator.GiftWrap(calculatedBoundaries, allocator, ref convexHullOut); return; } CreateOrResizeNativeArrayIfNecessary(0, allocator, ref convexHullOut); } public override unsafe TrackableChanges GetChanges(BoundedPlane defaultPlane, Allocator allocator) { if(planesFeature.SessionState != XrSessionState.Focused) { return default; } if (invalidatedPlanes.Count > 0) { var invalidPlanes = new TrackableChanges(0, 0, invalidatedPlanes.Count, allocator); invalidPlanes.removed.CopyFrom(new NativeArray(invalidatedPlanes.ToArray(), Allocator.Temp)); invalidatedPlanes.Clear(); planes.Clear(); return invalidPlanes; } if (planesTracker == Values.NullHandle) { CreateClient(); return default; } if (currentScanState == ScanState.Stopped) { BeginNewQuery(); } //If query has already begun, track the changes if (currentScanState != ScanState.Scanning) { return default; } //Check the state to make sure we are done scanning var stateResult = nativeFunctions.XrGetPlaneDetectionState(planesTracker, out var scanState); if (!Utils.DidXrCallSucceed(stateResult, nameof(nativeFunctions.XrGetPlaneDetectionState))) { return default; } switch (scanState) { case XrPlaneDetectionState.Done: break; case XrPlaneDetectionState.Pending: case XrPlaneDetectionState.None: return default; case XrPlaneDetectionState.Error: case XrPlaneDetectionState.Fatal: default: currentScanState = ScanState.Stopped; var message = $"[MLXrPlaneSubsystem] Plane detection state: {scanState.ToString().ToUpper()}"; if (!Application.isEditor) { throw new ApplicationException(message); } Debug.LogWarning(message); return default; } //Successfully scanned, so get the results //Get the plane results first var getInfo = new XrPlaneDetectorGetInfo { Type = XrPlaneStructTypes.PlaneDetectorGetInfo, Space = planesFeature.AppSpace, Time = planesFeature.NextPredictedDisplayTime }; Utils.OpenXRStructHelpers.Create(XrPlaneStructTypes.PlaneDetectorLocations, out var location); var result = nativeFunctions.XrGetPlaneDetections(planesTracker, in getInfo, out location); if (!Utils.DidXrCallSucceed(result, nameof(nativeFunctions.XrGetPlaneDetections))) { currentScanState = ScanState.Stopped; return default; } var planeTrackableIds = new NativeArray((int)location.PlaneLocationCountOutput, Allocator.TempJob); if (location.PlaneLocationCountOutput > 0) { //Now we have the count and we can assign the locations var locationArray = new NativeArray((int)location.PlaneLocationCountOutput, Allocator.Temp); location.PlaneLocations = (XrPlaneDetectorLocation*)locationArray.GetUnsafePtr(); NativeCopyUtility.FillArrayWithValue(locationArray, new XrPlaneDetectorLocation { Type = XrPlaneStructTypes.PlaneDetectorLocation, }); location.PlaneLocationCapacityInput = location.PlaneLocationCountOutput; result = nativeFunctions.XrGetPlaneDetections(planesTracker, in getInfo, out location); if (!Utils.DidXrCallSucceed(result, nameof(nativeFunctions.XrGetPlaneDetections))) { currentScanState = ScanState.Stopped; return default; } //Now we have all the locations. So we can start creating the boundary boundariesTable.Clear(); var planeCountTable = new Dictionary(); for (var i = 0; i < location.PlaneLocationCountOutput; i++) { var planeLocation = location.PlaneLocations[i]; var planeBoundary = new PlaneBoundary(); Utils.OpenXRStructHelpers.Create(XrPlaneStructTypes.PlaneDetectorPolygonBuffer, out var polygonBuffer); //Get polygon result = nativeFunctions.XrGetPlanePolygonBuffer(planesTracker, planeLocation.PlaneId, 0, out polygonBuffer); if (!Utils.DidXrCallSucceed(result, nameof(nativeFunctions.XrGetPlanePolygonBuffer))) { currentScanState = ScanState.Stopped; return default; } polygonBuffer.Vertices = (Vector2*)new NativeArray((int)polygonBuffer.VertexCountOutput, Allocator.Temp).GetUnsafePtr(); polygonBuffer.VertexCapacityInput = polygonBuffer.VertexCountOutput; result = nativeFunctions.XrGetPlanePolygonBuffer(planesTracker, planeLocation.PlaneId, 0, out polygonBuffer); if (!Utils.DidXrCallSucceed(result, $"{nameof(nativeFunctions.XrGetPlanePolygonBuffer)} Querying Length")) { currentScanState = ScanState.Stopped; return default; } planeBoundary.Polygon = polygonBuffer; if (planeLocation.PolygonBufferCount > 0) { var holeBuffers = new List(); for (uint holeIndex = 1; holeIndex < planeLocation.PolygonBufferCount; holeIndex++) { Utils.OpenXRStructHelpers.Create(XrPlaneStructTypes.PlaneDetectorPolygonBuffer, out var holeBuffer); result = nativeFunctions.XrGetPlanePolygonBuffer(planesTracker, planeLocation.PlaneId, holeIndex, out holeBuffer); if (!Utils.DidXrCallSucceed(result, $"{nameof(nativeFunctions.XrGetPlanePolygonBuffer)} Querying Holes Length")) { return default; } holeBuffer.Vertices = (Vector2*)new NativeArray((int)holeBuffer.VertexCountOutput, Allocator.Temp).GetUnsafePtr(); holeBuffer.VertexCapacityInput = holeBuffer.VertexCountOutput; result = nativeFunctions.XrGetPlanePolygonBuffer(planesTracker, planeLocation.PlaneId, holeIndex, out holeBuffer); if (!Utils.DidXrCallSucceed(result, $"{nameof(nativeFunctions.XrGetPlanePolygonBuffer)} Getting Holes")) { return default; } holeBuffers.Add(holeBuffer); } planeBoundary.Holes = new NativeArray(holeBuffers.ToArray(), Allocator.Temp); } if (!planeCountTable.TryGetValue(planeLocation.PlaneId, out var count)) { count = 0; } var trackableId = new TrackableId(planeLocation.PlaneId, PlaneTrackableIdSalt + count); planeCountTable[planeLocation.PlaneId] = ++count; boundariesTable[trackableId] = planeBoundary; planeTrackableIds[i] = trackableId; } } previousLastNumResults = lastNumResults; lastNumResults = location.PlaneLocationCountOutput; currentScanState = ScanState.Stopped; using var uPlanes = new NativeArray((int)location.PlaneLocationCountOutput, Allocator.TempJob); // Perform Unity plane conversion new CopyPlaneResultsJob { PlaneTrackableIds = planeTrackableIds, PlanesIn = location.PlaneLocations, PlanesOut = uPlanes }.Schedule((int)location.PlaneLocationCountOutput, 1).Complete(); planeTrackableIds.Dispose(); // Update plane states var added = new NativeFixedList((int)location.PlaneLocationCountOutput, Allocator.Temp); var updated = new NativeFixedList((int)location.PlaneLocationCountOutput, Allocator.Temp); var removed = new NativeFixedList((int)previousLastNumResults, Allocator.Temp); currentSet.Clear(); for (var i = 0; i < location.PlaneLocationCountOutput; ++i) { var uPlane = uPlanes[i]; var trackableId = uPlane.trackableId; currentSet.Add(trackableId); if (planes.ContainsKey(trackableId)) { updated.Add(uPlane); } else { added.Add(uPlane); } planes[trackableId] = uPlane; } // Look for removed planes foreach (var kvp in planes) { var trackableId = kvp.Key; if (!currentSet.Contains(trackableId)) { removed.Add(trackableId); } } foreach (var trackableId in removed) { planes.Remove(trackableId); } using (added) using (updated) using (removed) { var changes = new TrackableChanges(added.Length, updated.Length, removed.Length, allocator); added.CopyTo(changes.added); updated.CopyTo(changes.updated); removed.CopyTo(changes.removed); return changes; } } private unsafe void BeginNewQuery() { maxResults = QuerySet switch { // We hit the max, so increase for next time false when maxResults == lastNumResults => maxResults * 3 / 2, true => DefaultPlanesQuery.MaxResults, _ => maxResults }; DefaultPlanesQuery.Flags.ToMLXrOrientationsAndSemanticTypes(out var orientationValues, out var semanticValues); var orientationsArray = new NativeArray(orientationValues.Count, Allocator.Temp); orientationsArray.CopyFrom(orientationValues.ToArray()); var semanticArray = new NativeArray(semanticValues.Count, Allocator.Temp); semanticArray.CopyFrom(semanticValues.ToArray()); var beginInfo = new XrPlaneDetectorBeginInfo { Type = XrPlaneStructTypes.PlaneDetectorBeginInfo, Space = planesFeature.AppSpace, Time = planesFeature.NextPredictedDisplayTime, OrientationCount = (uint)orientationsArray.Length, Orientations = (XrPlaneDetectorOrientation*)orientationsArray.GetUnsafePtr(), SemanticTypesCount = (uint)semanticArray.Length, SemanticTypes = (XrPlaneDetectorSemanticTypes*)semanticArray.GetUnsafePtr(), MaxPlanes = maxResults, MinArea = DefaultPlanesQuery.MinPlaneArea, BoundingBoxPose = new XrPose { Position = DefaultPlanesQuery.BoundsCenter.InvertZ(), Rotation = DefaultPlanesQuery.BoundsRotation.InvertXY() }, BoundingBoxExtents = DefaultPlanesQuery.BoundsExtents, }; var result = nativeFunctions.XrBeginPlaneDetection(planesTracker, in beginInfo); if (!Utils.DidXrCallSucceed(result, nameof(nativeFunctions.XrBeginPlaneDetection))) { return; } currentScanState = ScanState.Scanning; currentPlaneDetectionModeInternal = requestedPlaneDetectionModeInternal; } internal void InvalidateCurrentPlanes() { invalidatedPlanes.Clear(); invalidatedPlanes.UnionWith(currentSet); } private enum ScanState { Scanning, Stopped } } internal void InvalidateCurrentPlanes() { (provider as MagicLeapProvider)?.InvalidateCurrentPlanes(); } public static void RegisterDescriptor() { XRPlaneSubsystemDescriptor.Create(new XRPlaneSubsystemDescriptor.Cinfo { id = MagicLeapXrProvider.PlanesSubsystemId, providerType = typeof(MagicLeapProvider), subsystemTypeOverride = typeof(MLXrPlaneSubsystem), supportsVerticalPlaneDetection = true, supportsArbitraryPlaneDetection = true, supportsBoundaryVertices = true, supportsClassification = true }); } } public static class SubsystemExtensions { public static PlaneDetectionMode ToPlaneDetectionMode(this MLXrPlaneSubsystem.MLPlanesQueryFlags planesQueryFlags) { var outDetectionMode = PlaneDetectionMode.None; if ((planesQueryFlags & MLXrPlaneSubsystem.MLPlanesQueryFlags.Horizontal) != 0) { outDetectionMode |= PlaneDetectionMode.Horizontal; } if ((planesQueryFlags & MLXrPlaneSubsystem.MLPlanesQueryFlags.Vertical) != 0) { outDetectionMode |= PlaneDetectionMode.Vertical; } return outDetectionMode; } public static MLXrPlaneSubsystem.MLPlanesQueryFlags ToMLXrQueryFlags(this PlaneDetectionMode planeDetectionMode) { var outFlags = MLXrPlaneSubsystem.MLPlanesQueryFlags.None; if ((planeDetectionMode & PlaneDetectionMode.Horizontal) != 0) { outFlags |= MLXrPlaneSubsystem.MLPlanesQueryFlags.Horizontal; } if ((planeDetectionMode & PlaneDetectionMode.Vertical) != 0) { outFlags |= MLXrPlaneSubsystem.MLPlanesQueryFlags.Vertical; } return outFlags; } internal static void ToMLXrOrientationsAndSemanticTypes(this MLXrPlaneSubsystem.MLPlanesQueryFlags flags, out IList orientations, out IList semanticTypes) { orientations = new List(); semanticTypes = new List(); if ((flags & MLXrPlaneSubsystem.MLPlanesQueryFlags.Horizontal) != 0) { orientations.Add(XrPlaneDetectorOrientation.HorizontalDownward); orientations.Add(XrPlaneDetectorOrientation.HorizontalUpward); } if ((flags & MLXrPlaneSubsystem.MLPlanesQueryFlags.Arbitrary) != 0) { orientations.Add(XrPlaneDetectorOrientation.Arbitrary); } if ((flags & MLXrPlaneSubsystem.MLPlanesQueryFlags.Vertical) != 0) { orientations.Add(XrPlaneDetectorOrientation.Vertical); } //Semantic types if ((flags & MLXrPlaneSubsystem.MLPlanesQueryFlags.SemanticCeiling) != 0) { semanticTypes.Add(XrPlaneDetectorSemanticTypes.Ceiling); } if ((flags & MLXrPlaneSubsystem.MLPlanesQueryFlags.SemanticFloor) != 0) { semanticTypes.Add(XrPlaneDetectorSemanticTypes.Floor); } if ((flags & MLXrPlaneSubsystem.MLPlanesQueryFlags.SemanticPlatform) != 0) { semanticTypes.Add(XrPlaneDetectorSemanticTypes.Platform); } if ((flags & MLXrPlaneSubsystem.MLPlanesQueryFlags.SemanticWall) != 0) { semanticTypes.Add(XrPlaneDetectorSemanticTypes.Wall); } } } }