// %BANNER_BEGIN% // --------------------------------------------------------------------- // %COPYRIGHT_BEGIN% // Copyright (c) (2021-2022) Magic Leap, Inc. All Rights Reserved. // Use of this file is governed by the Software License Agreement, located here: https://www.magicleap.com/software-license-agreement-ml2 // Terms and conditions applicable to third-party materials accompanying this distribution may also be found in the top-level NOTICE file appearing herein. // %COPYRIGHT_END% // --------------------------------------------------------------------- // %BANNER_END% using System; using System.Collections; using System.Collections.Generic; using UnityEngine.Serialization; using UnityEngine.XR.ARSubsystems; using UnityEngine.XR.MagicLeap.Native; using UnityEngine.XR.Management; #if UNITY_EDITOR using UnityEditor; //using UnityEditor.XR.MagicLeap.Remote; #endif namespace UnityEngine.XR.MagicLeap { [DisallowMultipleComponent] public sealed class MeshingSubsystemComponent : MonoBehaviour { private const float SubsystemStartUpTime = 1f; /// /// What type of mesh to generate: a triangle mesh or a point cloud /// public enum MeshType { /// /// Generate triangle meshes /// Triangles, /// /// Generate a point cloud (a mesh with MeshTopology.Points) /// PointCloud } [SerializeField] GameObject m_MeshPrefab; /// /// Get or set the prefab which should be instantiated to create individual mesh instances. /// May have a mesh renderer and an optional mesh collider for physics. /// public GameObject meshPrefab { get { return m_MeshPrefab; } set { m_MeshPrefab = value; } } public void OnMeshingPropertyChanged() => m_SettingsDirty = true; [SerializeField] [OnChangedCall(nameof(OnMeshingPropertyChanged))] bool m_ComputeNormals = true; /// /// When enabled, the system will compute the normals for the triangle vertices. /// public bool computeNormals { get { return m_ComputeNormals; } set { if (m_ComputeNormals != value) { m_ComputeNormals = value; m_SettingsDirty = true; } } } public static MeshingSubsystem.Extensions.MLMeshing.LevelOfDetail FromDensityToLevelOfDetail(float density) { if (density < 0.33f) return MeshingSubsystem.Extensions.MLMeshing.LevelOfDetail.Minimum; else if (density < 0.66f) return MeshingSubsystem.Extensions.MLMeshing.LevelOfDetail.Medium; else return MeshingSubsystem.Extensions.MLMeshing.LevelOfDetail.Maximum; } public static float FromLevelOfDetailToDensity(MeshingSubsystem.Extensions.MLMeshing.LevelOfDetail lod) { if (lod == MeshingSubsystem.Extensions.MLMeshing.LevelOfDetail.Minimum) return 0.0f; else if (lod == MeshingSubsystem.Extensions.MLMeshing.LevelOfDetail.Medium) return 0.5f; else return 1.0f; } [SerializeField, Tooltip("Determines the level of detail that the batched mesh blocks should be. This property is not used if custom mesh block requests are created via the SetCustomMeshBlockRequests() method.")] [OnChangedCall(nameof(OnMeshingPropertyChanged))] float m_Density = 1.0f; public float density { get { return m_Density; } set { if (m_Density != value) { m_Density = value; m_SettingsDirty = true; } } } [SerializeField,Tooltip("Determines how many mesh blocks should be requested by the meshing subsystem at once. This property is not used if custom mesh block requests are created via the SetCustomMeshBlockRequests() method.")] [OnChangedCall(nameof(OnMeshingPropertyChanged))] int m_BatchSize = 16; /// /// How many meshes to update per batch. Larger values are more efficient, but have higher latency. /// public int batchSize { get { return m_BatchSize; } set { if (m_BatchSize != value) { m_BatchSize = value; m_SettingsDirty = true; } } } [SerializeField] Transform m_MeshParent; /// /// The parent transform for generated meshes. /// public Transform meshParent { get { return m_MeshParent; } set { m_MeshParent = value; } } [SerializeField] [FormerlySerializedAs("m_MeshType")] [OnChangedCall(nameof(OnMeshingPropertyChanged))] MeshType m_RequestedMeshType = MeshType.Triangles; /// /// The current mesh type being surfaced by the subsystem provider. /// public MeshType currentMeshType => SubsystemFeatures.currentFeatures.HasFlag(Feature.Meshing) ? MeshType.Triangles : MeshType.PointCloud; /// /// Request Magic Leap to generate a triangle mesh or point cloud points. /// public MeshType requestedMeshType { get { return m_RequestedMeshType; } set { if (m_RequestedMeshType != value) { m_RequestedMeshType = value; m_SettingsDirty = true; } } } [SerializeField] [OnChangedCall(nameof(OnMeshingPropertyChanged))] float m_FillHoleLength = 1.0f; /// /// Boundary distance (in meters) of holes you wish to have filled. /// public float fillHoleLength { get { return m_FillHoleLength; } set { if (m_FillHoleLength != value) { m_FillHoleLength = value; m_SettingsDirty = true; } } } [SerializeField] [OnChangedCall(nameof(OnMeshingPropertyChanged))] bool m_Planarize = false; /// /// When enabled, the system will planarize the returned mesh (planar regions will be smoothed out). /// public bool planarize { get { return m_Planarize; } set { if (m_Planarize != value) { m_Planarize = value; m_SettingsDirty = true; } } } [SerializeField] [OnChangedCall(nameof(OnMeshingPropertyChanged))] float m_DisconnectedComponentArea = 0.25f; /// /// Any component that is disconnected from the main mesh and which has an area less than this size will be removed. /// public float disconnectedComponentArea { get { return m_DisconnectedComponentArea; } set { if (m_DisconnectedComponentArea != value) { m_DisconnectedComponentArea = value; m_SettingsDirty = true; } } } [SerializeField] uint m_MeshQueueSize = 4; /// /// Controls the number of meshes to queue for generation at once. Larger numbers will lead to higher CPU usage. /// public uint meshQueueSize { get { return m_MeshQueueSize; } set { m_MeshQueueSize = value; } } [SerializeField] float m_PollingRate = 0.25f; /// /// How often to check for updates, in seconds. More frequent updates will increase CPU usage. /// public float pollingRate { get { return m_PollingRate; } set { m_PollingRate = value; } } [SerializeField] [OnChangedCall(nameof(OnMeshingPropertyChanged))] bool m_RequestVertexConfidence = false; /// /// When enabled, the system will generate confidence values for each vertex, ranging from 0-1. /// /// public bool requestVertexConfidence { get { return m_RequestVertexConfidence; } set { if (m_RequestVertexConfidence != value) { m_RequestVertexConfidence = value; m_SettingsDirty = true; } } } [SerializeField] [OnChangedCall(nameof(OnMeshingPropertyChanged))] bool m_RemoveMeshSkirt = false; /// /// When enabled, the mesh skirt (overlapping area between two mesh blocks) will be removed. This field is only valid when the Mesh Type is Blocks. /// public bool removeMeshSkirt { get { return m_RemoveMeshSkirt; } set { if (m_RemoveMeshSkirt != value) { m_RemoveMeshSkirt = value; m_SettingsDirty = true; } } } private MeshRenderer prefabRenderer; public MeshRenderer PrefabRenderer { get { if (prefabRenderer == null) { prefabRenderer = meshPrefab.GetComponent(); } return prefabRenderer; } } Vector3 boundsExtents { get { return transform.localScale; } } [SerializeField] private int objectPoolSize = 200; [SerializeField] private float objectPoolGrowthRate = 0.5f; /// /// A Dictionary which maps mesh ids to their GameObjects. /// public Dictionary meshIdToGameObjectMap { get; private set; } /// /// An event which is invoked whenever a new mesh is added /// public event Action meshAdded; /// /// An event which is invoked whenever an existing mesh is updated (regenerated). /// public event Action meshUpdated; /// /// An event which is invoked whenever an existing mesh is removed. /// public event Action meshRemoved; private InputDevice headDevice; private Coroutine startupRoutine = null; private bool shouldSubsystemBeRunning = false; /// /// Retrieve the confidence values associated with a mesh. Confidence values /// range from 0..1. must be enabled. /// /// /// The unique MeshId of the mesh. /// A List of floats, one for each vertex in the mesh. /// True if confidence values were successfully retrieved for the mesh with id . public bool TryGetConfidence(MeshId meshId, List confidenceOut) { if (confidenceOut == null) { throw new ArgumentNullException(nameof(confidenceOut)); } if (MeshingSubsystemLifecycle.MeshSubsystem == null) { return false; } int count = 0; var floatPtr = MeshingSubsystem.Extensions.MLMeshing.Config.AcquireConfidence(meshId, out count); if (floatPtr == IntPtr.Zero) { return false; } confidenceOut.Clear(); if (count > 0) { Span floatSpan; unsafe { floatSpan = new Span(floatPtr.ToPointer(), count); } for (int i = 0; i < count; ++i) { confidenceOut.Add(floatSpan[i]); } } MeshingSubsystem.Extensions.MLMeshing.Config.ReleaseConfidence(meshId); return true; } /// /// Destroy all mesh GameObjects created by this . /// The will also be cleared. /// public void DestroyAllMeshes() { foreach (var kvp in meshIdToGameObjectMap) { var go = kvp.Value; ResetGameObject(go); } meshIdToGameObjectMap.Clear(); m_MeshesBeingGenerated.Clear(); m_MeshesNeedingGeneration.Clear(); Resources.UnloadUnusedAssets(); GC.Collect(); } /// /// 'Refresh' a single mesh. This forces the mesh to be regenerated with the current settings. /// /// The MeshId of the mesh to regenerate. public void RefreshMesh(MeshId meshId) { if (m_MeshesBeingGenerated.ContainsKey(meshId)) { return; } m_MeshesNeedingGeneration[meshId] = new MeshInfo { MeshId = meshId, ChangeState = MeshChangeState.Updated, PriorityHint = Time.frameCount }; } /// /// 'Refresh' all known meshes (meshes that are in ). /// This will force all meshes to be regenerated with the current settings. /// public void RefreshAllMeshes() { foreach (var kvp in meshIdToGameObjectMap) { var meshId = kvp.Key; RefreshMesh(meshId); } } public static void SetCustomMeshBlockRequests(MeshingSubsystem.Extensions.MLMeshing.OnMeshBlockRequests onBlockRequests) => MeshingSubsystem.Extensions.MLMeshing.Config.SetCustomMeshBlockRequests(onBlockRequests); #if UNITY_EDITOR MeshingSubsystem.Extensions.MLMeshing.Config.Settings m_CachedSettings; float m_CachedDensity; bool haveSettingsChanged { get { var currentSettings = GetMeshingSettings(); return (m_CachedDensity != density) || (m_CachedSettings.fillHoleLength != currentSettings.fillHoleLength) || (m_CachedSettings.flags != currentSettings.flags) || (m_CachedSettings.disconnectedComponentArea != currentSettings.disconnectedComponentArea); } } #endif MeshingSubsystem.Extensions.MLMeshing.Config.Settings GetMeshingSettings() { var flags = MeshingSubsystem.Extensions.MLMeshing.Config.Flags.IndexOrderCW; if (computeNormals) flags |= MeshingSubsystem.Extensions.MLMeshing.Config.Flags.ComputeNormals; if (requestVertexConfidence) flags |= MeshingSubsystem.Extensions.MLMeshing.Config.Flags.ComputeConfidence; if (planarize) flags |= MeshingSubsystem.Extensions.MLMeshing.Config.Flags.Planarize; if (removeMeshSkirt) flags |= MeshingSubsystem.Extensions.MLMeshing.Config.Flags.RemoveMeshSkirt; if (requestedMeshType == MeshType.PointCloud) flags |= MeshingSubsystem.Extensions.MLMeshing.Config.Flags.PointCloud; var settings = new MeshingSubsystem.Extensions.MLMeshing.Config.Settings { flags = flags, fillHoleLength = fillHoleLength, disconnectedComponentArea = disconnectedComponentArea }; return settings; } void OnDrawGizmosSelected() { Gizmos.color = new Color(0, .5f, 0, .35f); Gizmos.matrix = transform.localToWorldMatrix; Gizmos.DrawCube(Vector3.zero, Vector3.one); } // Create new GameObject and parent to ourself GameObject CreateGameObject(MeshId meshId) { if (gameObjectPool.Count == 0) { int amountToAdd = (int)(meshIdToGameObjectMap.Count * objectPoolGrowthRate); while (gameObjectPool.Count < amountToAdd) { AddNewObjectToPool(); } objectPoolSize += (int)(objectPoolSize * objectPoolGrowthRate); Debug.Log($"added {amountToAdd} new gameObjects to pool. current pool count: {gameObjectPool.Count}"); } GameObject newGameObject = gameObjectPool.Dequeue(); newGameObject.GetComponent().sharedMaterial = PrefabRenderer.sharedMaterial; newGameObject.name = $"Mesh {meshId}"; newGameObject.SetActive(true); return newGameObject; } GameObject GetOrCreateGameObject(MeshId meshId) { if (!meshIdToGameObjectMap.TryGetValue(meshId, out var go)) { go = CreateGameObject(meshId); meshIdToGameObjectMap[meshId] = go; } return go; } void AddNewObjectToPool() { var go = Instantiate(meshPrefab, meshParent); go.SetActive(false); gameObjectPool.Enqueue(go); } void ResetGameObject(GameObject go) { if (gameObjectPool.Count < objectPoolSize) { go.SetActive(false); gameObjectPool.Enqueue(go); } else { Debug.Log($"destroying excess gameObject since gameObjectPool.Count >= size {objectPoolSize}"); DestroyImmediate(go); } } private bool gameObjectPoolInitialized = false; void Awake() { meshIdToGameObjectMap = new Dictionary(); m_MeshesNeedingGeneration = new Dictionary(); m_MeshesBeingGenerated = new Dictionary(); gameObjectPool = new Queue(); } IEnumerator Init() { yield return StartCoroutine(MeshingSubsystemLifecycle.WaitUntilInited()); while (meshPrefab == null) { yield return null; } if (!gameObjectPoolInitialized) { while (gameObjectPool.Count < objectPoolSize) { AddNewObjectToPool(); } gameObjectPoolInitialized = true; } UpdateSettings(); UpdateBounds(); UpdateBatchSize(); StartSubsystem(); } void StartSubsystem() { MeshingSubsystemLifecycle.StartSubsystem(); startupRoutine = StartCoroutine(LetSubsystemToStart()); MLSpace.OnLocalizationEvent += MLSpaceOnOnLocalizationChanged; } private IEnumerator LetSubsystemToStart() { shouldSubsystemBeRunning = false; yield return new WaitForSeconds(SubsystemStartUpTime); shouldSubsystemBeRunning = true; } private void MLSpaceOnOnLocalizationChanged(MLSpace.LocalizationResult result) { m_SettingsDirty = true; } void StopSubsystem() { MeshingSubsystemLifecycle.StopSubsystem(); SubsystemFeatures.SetCurrentFeatureEnabled(Feature.Meshing | Feature.PointCloud, false); if (startupRoutine != null) { StopCoroutine(startupRoutine); } MLSpace.OnLocalizationEvent -= MLSpaceOnOnLocalizationChanged; } void OnEnable() { StartCoroutine(Init()); } void OnDisable() { StopSubsystem(); } void AddToQueueIfNecessary(MeshInfo meshInfo) { if (m_MeshesNeedingGeneration.ContainsKey(meshInfo.MeshId)) { return; } meshInfo.PriorityHint = Time.frameCount; m_MeshesNeedingGeneration[meshInfo.MeshId] = meshInfo; } void CheckHeadTrackingMapEvents() { if (!headDevice.isValid) { headDevice = InputSubsystem.Utils.FindMagicLeapDevice(InputDeviceCharacteristics.HeadMounted | InputDeviceCharacteristics.TrackedDevice); } if (headDevice.isValid && InputSubsystem.Extensions.MLHeadTracking.TryGetMapEvents(headDevice, out var mapEvents)) { if ((uint)(mapEvents & InputSubsystem.Extensions.MLHeadTracking.MapEvents.NewSession) != 0) { // clear all the meshes if headtracking is starting a new session DestroyAllMeshes(); } } } void UpdateSettings() { DestroyAllMeshes(); UpdateBatchSize(); var settings = GetMeshingSettings(); MeshingSubsystem.Extensions.MLMeshing.Config.meshingSettings = settings; MeshingSubsystem.Extensions.MLMeshing.Config.density = density; m_SettingsDirty = false; #if UNITY_EDITOR m_CachedSettings = settings; m_CachedDensity = density; #endif } void UpdateBounds() { MeshingSubsystem.Extensions.MLMeshing.Config.SetBounds(transform, boundsExtents); transform.hasChanged = false; } void UpdateBatchSize() { MeshingSubsystem.Extensions.MLMeshing.Config.batchSize = batchSize; } // When returning from an application pause, refresh the meshes to prevent potential excess // meshing data from rendering if a head tracking pose resets within another application. void OnApplicationPause(bool pauseStatus) { if (!pauseStatus) { RefreshAllMeshes(); } } // Every frame, poll the MeshSubsystem for mesh updates (Added, Updated, Removed) // If the mesh is Added or Updated, then add it to the generation queue. // // Create generation requests for each mesh needing it until all have // been added to the asynchronous queue, or the queue is full. void Update() { if (MeshingSubsystemLifecycle.MeshSubsystem == null) return; if (!shouldSubsystemBeRunning) return; if (!MeshingSubsystemLifecycle.MeshSubsystem.running) { Debug.LogError($"MeshingSubsystemLifecycle.MeshSubsystem.running {MeshingSubsystemLifecycle.MeshSubsystem.running}"); return; } #if UNITY_EDITOR m_SettingsDirty |= haveSettingsChanged; #endif CheckHeadTrackingMapEvents(); if (m_SettingsDirty) UpdateSettings(); if (transform.hasChanged) UpdateBounds(); // Since meshing is a two pass asynchronous API, we need to poll at twice the configured rate. // Once to request the mesh info, and a second time to request the associated mesh blocks. // If a block request is pending, a new mesh info request will not be made, the inverse is also true. // Polling at twice the rate will ensure that data appears at the desired interval. float timeSinceLastUpdate = (float)(DateTime.Now - m_TimeLastUpdated).TotalSeconds; bool allowUpdate = (timeSinceLastUpdate > (m_PollingRate / 2.0f)); bool gotMeshInfos = MeshingSubsystemLifecycle.MeshSubsystem.TryGetMeshInfos(meshInfos); if (allowUpdate && gotMeshInfos) { foreach (var meshInfo in meshInfos) { switch (meshInfo.ChangeState) { case MeshChangeState.Added: case MeshChangeState.Updated: AddToQueueIfNecessary(meshInfo); break; case MeshChangeState.Removed: meshRemoved?.Invoke(meshInfo.MeshId); // Remove from processing queue if (m_MeshesNeedingGeneration.ContainsKey(meshInfo.MeshId)) { m_MeshesNeedingGeneration.Remove(meshInfo.MeshId); } // Destroy the GameObject if (meshIdToGameObjectMap.TryGetValue(meshInfo.MeshId, out var meshGameObject)) { ResetGameObject(meshGameObject); meshIdToGameObjectMap.Remove(meshInfo.MeshId); } break; case MeshChangeState.Unchanged: default: break; } } m_TimeLastUpdated = DateTime.Now; } if (meshPrefab != null) { while (m_MeshesBeingGenerated.Count < meshQueueSize && m_MeshesNeedingGeneration.Count > 0) { MeshId meshId = GetNextMeshToGenerate(); GameObject meshGameObject = GetOrCreateGameObject(meshId); var meshCollider = meshGameObject.GetComponent(); var meshFilter = meshGameObject.GetComponent(); var meshAttributes = computeNormals ? MeshVertexAttributes.Normals : MeshVertexAttributes.None; MeshingSubsystemLifecycle.MeshSubsystem.GenerateMeshAsync(meshId, meshFilter.mesh, meshCollider, meshAttributes, OnMeshGenerated); m_MeshesBeingGenerated.Add(meshId, m_MeshesNeedingGeneration[meshId]); if (m_MeshesNeedingGeneration.ContainsKey(meshId)) { m_MeshesNeedingGeneration.Remove(meshId); } } } } // Find the oldest one. Prioritize new ones. private MeshId GetNextMeshToGenerate() { KeyValuePair? highestPriorityPair = null; foreach (var pair in m_MeshesNeedingGeneration) { // Skip meshes currently being generated if (m_MeshesBeingGenerated.ContainsKey(pair.Key)) continue; if (!highestPriorityPair.HasValue) { highestPriorityPair = pair; continue; } var consideredMeshInfo = pair.Value; var selectedMeshInfo = highestPriorityPair.Value.Value; // If the selected change type is less than this one, // then ignore entirely. if (consideredMeshInfo.ChangeState > selectedMeshInfo.ChangeState) continue; // If this info has a higher priority change type // (e.g. Added rather than Updated) use it instead. if (consideredMeshInfo.ChangeState < selectedMeshInfo.ChangeState) { highestPriorityPair = pair; continue; } // If changeTypes are the same, but this one is older, // then use it. if (consideredMeshInfo.PriorityHint < selectedMeshInfo.PriorityHint) { highestPriorityPair = pair; } } if (highestPriorityPair.HasValue) { return highestPriorityPair.Value.Key; } return MeshId.InvalidId; } void OnMeshGenerated(MeshGenerationResult result) { if (result.Status == MeshGenerationStatus.Success) { // The mesh may have been removed by external code if (!m_MeshesBeingGenerated.TryGetValue(result.MeshId, out var meshInfo)) return; m_MeshesBeingGenerated.Remove(result.MeshId); switch (meshInfo.ChangeState) { case MeshChangeState.Added: meshAdded?.Invoke(result.MeshId); break; case MeshChangeState.Updated: meshUpdated?.Invoke(result.MeshId); break; // Removed/unchanged meshes don't get generated. case MeshChangeState.Removed: break; case MeshChangeState.Unchanged: break; default: break; } if (meshIdToGameObjectMap.TryGetValue(result.MeshId, out var meshGameObject)) { // Disable the collision mesh if we're in point cloud mode var meshCollider = meshGameObject.GetComponent(); if (meshCollider != null) { meshCollider.enabled = currentMeshType != MeshType.PointCloud; } } } else { m_MeshesBeingGenerated.Remove(result.MeshId); } } bool m_SettingsDirty; DateTime m_TimeLastUpdated = DateTime.MinValue; Dictionary m_MeshesNeedingGeneration; Dictionary m_MeshesBeingGenerated; List meshInfos = new List(); Queue gameObjectPool; #if UNITY_XR_MAGICLEAP_PROVIDER MagicLeapLoader m_Loader; #endif XRMeshSubsystem m_MeshSubsystem; } }