// %BANNER_BEGIN% // --------------------------------------------------------------------- // %COPYRIGHT_BEGIN% // Copyright (c) (2018-2022) Magic Leap, Inc. All Rights Reserved. // Use of this file is governed by the Software License Agreement, located here: https://www.magicleap.com/software-license-agreement-ml2 // Terms and conditions applicable to third-party materials accompanying this distribution may also be found in the top-level NOTICE file appearing herein. // %COPYRIGHT_END% // --------------------------------------------------------------------- // %BANNER_END% // Disabling deprecated warning for the internal project #pragma warning disable 618 namespace UnityEngine.XR.MagicLeap { using System; using System.Collections.Generic; using System.Runtime.InteropServices; using System.Threading.Tasks; using UnityEngine.XR.MagicLeap.Native; public sealed partial class MLMarkerTracker { /// /// Creates the MLMarkerTracker API instance on the native side. /// This must be called before any other native call. /// /// The initial settings for the marker scanner to use. /// MLResult indicating the success or failure of the operation. /// /// private static MLResult.Code MLMarkerTrackerCreate(TrackerSettings settings) { var nativeSettings = new NativeBindings.MLMarkerTrackerSettings(settings); if (!MLPermissions.CheckPermission(MLPermission.MarkerTracking).IsOk) { Debug.LogError($"Unable to create MLMarkerTracker because the permission {MLPermission.MarkerTracking} has not been granted."); return MLResult.Code.PermissionDenied; } MLResult.Code resultCode = NativeBindings.MLMarkerTrackerCreate(nativeSettings, out Instance.Handle); MLResult.DidNativeCallSucceed(resultCode, nameof(NativeBindings.MLMarkerTrackerCreate)); return resultCode; } /// /// Marshal binary data struct into string. /// /// /// A string representing the binary data. /// /// private static byte[] MarshalBinaryData(IntPtr binaryDataStructPtr) { var binaryDataStruct = Marshal.PtrToStructure(binaryDataStructPtr); // the actual binary data is at an offset of the binary data struct IntPtr offsetPtr = IntPtr.Add(binaryDataStructPtr, Marshal.SizeOf(binaryDataStruct)); byte[] bytes = new byte[binaryDataStruct.Size]; Marshal.Copy(offsetPtr, bytes, 0, bytes.Length); return bytes; } /// /// Poll the native API for marker scanner results. /// /// /// An array of MarkerData that contains the results /// that the scanner has collected since the last call to this function. This array /// may be empty if there are no new results. /// private static MarkerData[] MLMarkerTrackerGetResults() { var scannerResults = new NativeBindings.MLMarkerTrackerResultArray(1); var resultCode = NativeBindings.MLMarkerTrackerGetResult(Instance.Handle, ref scannerResults); // get results from native api if (MLResult.DidNativeCallSucceed(resultCode, nameof(NativeBindings.MLMarkerTrackerGetResult))) { var managedResults = new MarkerData[((int)scannerResults.Count)]; for (ulong i = 0; i < scannerResults.Count.ToUInt64(); i++) { // marshal native array into native structs long address = scannerResults.Detections.ToInt64() + (Marshal.SizeOf() * (int)i); NativeBindings.MLMarkerTrackerResult detectedResult = Marshal.PtrToStructure(Marshal.ReadIntPtr(new IntPtr(address))); Pose pose = Pose.identity; if (detectedResult.IsValidPose) { resultCode = MagicLeapXrProviderNativeBindings.GetUnityPose(detectedResult.CoordinateFrameUID, out pose); if (MLResult.IsOK(resultCode)) { // Update marker pose to be rotated 180 degrees on it's z axis to accommodate for how the marker face is interpreted by the capi. // Without this update the marker positions will be upside down. pose = new Pose(pose.position, pose.rotation * Quaternion.AngleAxis(180, Vector3.forward)); } else Debug.LogError($"Marker Scanner could not get pose data for coordinate frame id '{detectedResult.CoordinateFrameUID}'"); } var decodedDataType = Marshal.PtrToStructure(detectedResult.DecodedData.Data); NativeBindings.MLMarkerTrackerDecodedArucoData arucoData = default; byte[] binaryData = null; var markerType = MarkerType.None; switch (decodedDataType.Type) { case NativeBindings.DecodedDataType.Aruco: arucoData = Marshal.PtrToStructure(detectedResult.DecodedData.Data); markerType = MarkerType.Aruco_April; break; case NativeBindings.DecodedDataType.QR: binaryData = MarshalBinaryData(detectedResult.DecodedData.Data); markerType = MarkerType.QR; break; case NativeBindings.DecodedDataType.EAN_13: binaryData = MarshalBinaryData(detectedResult.DecodedData.Data); markerType = MarkerType.EAN_13; break; case NativeBindings.DecodedDataType.UPC_A: binaryData = MarshalBinaryData(detectedResult.DecodedData.Data); markerType = MarkerType.UPC_A; break; } managedResults[i] = new MarkerData ( markerType, arucoData, binaryData, pose, detectedResult.ReprojectionError ); } if (scannerResults.Count.ToUInt64() > 0) { // release native memory so results can be polled again if (MLResult.DidNativeCallSucceed(NativeBindings.MLMarkerTrackerReleaseResult(ref scannerResults), nameof(NativeBindings.MLMarkerTrackerReleaseResult))) return managedResults; else { MLPluginLog.Error($"MLMarkerTracker.NativeBindings.MLMarkerTrackerReleaseResult failed when trying to release the results' memory. Reason: {MLResult.CodeToString(resultCode)}"); return managedResults; } } else { return managedResults; } } else { MLPluginLog.Error($"MLMarkerTracker.MLMarkerTrackerGetResult failed to obtain a result. Reason: {resultCode}"); return new MarkerData[0]; } } private static Task MLMarkerTrackerSettingsUpdate(TrackerSettings settings) { var handle = Instance.Handle; var nativeSettings = new NativeBindings.MLMarkerTrackerSettings(settings); var resultCode = NativeBindings.MLMarkerTrackerUpdateSettings(handle, in nativeSettings); MLResult.DidNativeCallSucceed(resultCode, nameof(NativeBindings.MLMarkerTrackerUpdateSettings)); return MLResult.Create(resultCode); } internal class NativeBindings : MagicLeapNativeBindings { /// /// Data type of the decoded marker data. /// public enum DecodedDataType { None, /// /// This covers Aruco and AprilTag /// Aruco, QR, EAN_13, UPC_A } /// /// Create a Marker Scanner. Requires CameraCapture, LowLatencyLightwear priveledges. /// /// /// List of settings of type MLMarkerTrackerSettings that configure the scanner. /// /// /// A pointer to an MLHandle to the newly created Marker Scanner. If this /// operation fails, handle will be ML_INVALID_HANDLE . /// /// /// MLResult_InvalidParam : Failed to create Marker Scanner due to invalid /// out_handle. MLResult_Ok Successfully : created Marker Scanner. /// MLResult_PermissionDenied Failed : to create scanner due to lack of /// permission(s). MLResult_UnspecifiedFailure : Failed to create the Marker /// Scanner due to an internal error. /// [DllImport(MLPerceptionClientDll, CallingConvention = CallingConvention.Cdecl)] public static extern MLResult.Code MLMarkerTrackerCreate(in MLMarkerTrackerSettings settings, out ulong handle); /// /// Destroy a Marker Scanner. Requires CameraCapture, LowLatencyLightwear priveleges. /// /// MLHandle to the Marker Scanner created by MLMarkerTrackerCreate(). /// /// MLResult_Ok : Successfully destroyed the Marker Scanner.\n /// MLResult_UnspecifiedFailure : Failed to destroy the scanner due to an /// internal error. /// [DllImport(MLPerceptionClientDll, CallingConvention = CallingConvention.Cdecl)] public static extern MLResult.Code MLMarkerTrackerDestroy(ulong scannerHandle); /// /// brief Get the results for Marker Scanning.= This function can be used to poll /// results from the scanner. This will allocate memory for the results array that /// will have to be freed later. /// /// /// MLHandle to the Marker Scanner created by MLMarkerTrackerCreate(). /// /// /// out_data Pointer to an array of pointers to MLMarkerTrackerResult. The content /// will be freed by the MLMarkerTrackerReleaseResult. /// /// /// MLResult_InvalidParam Failed to return detection data due to invalid out_data. /// /// MLResult_Ok Successfully fetched and returned all detections. /// \retval MLResult_UnspecifiedFailure Failed to return detections due to an internal error. [DllImport(MLPerceptionClientDll, CallingConvention = CallingConvention.Cdecl)] public static extern MLResult.Code MLMarkerTrackerGetResult(ulong scanner_handle, ref MLMarkerTrackerResultArray data); /// /// Release the resources for the results array. /// /// The list of detections to be freed. /// /// MLResult_InvaldParam Failed to free structure due to invalid data. /// MLResult_Ok Successfully freed data structure. /// MLResult_UnspecifiedFailure Failed to free data due to an internal error. /// /// [DllImport(MLPerceptionClientDll, CallingConvention = CallingConvention.Cdecl)] public static extern MLResult.Code MLMarkerTrackerReleaseResult(ref MLMarkerTrackerResultArray data); /// /// Update the Marker Scanner with new settings. Requires CameraCapture, /// LowLatencyLightwear priveledges. /// /// MLHandle to the Marker Scanner created by MLArucoScannerCreate(). /// List of new Marker Scanner settings. /// /// MLResult_InvalidParam : Failed to update the settings due to invalid /// scanner_settings. MLResult_Ok Successfully : updated the Marker Scanner /// settings. MLResult_PermissionDenied : Failed to update the settings due /// to lack of permission(s). MLResult_UnspecifiedFailure : Failed to update /// the settings due to an internal error. /// [DllImport(MLPerceptionClientDll, CallingConvention = CallingConvention.Cdecl)] public static extern MLResult.Code MLMarkerTrackerUpdateSettings(ulong scanner_handle, in MLMarkerTrackerSettings scanner_settings); /// /// Different Marker Decoders will produce different data. Use this /// structure to find what the data structure is. /// [StructLayout(LayoutKind.Sequential)] public readonly struct MLMarkerTrackerDecodedTypedData { /// /// Type selector for the structure. /// public readonly DecodedDataType Type; } /// /// Aruco decoded data. /// [StructLayout(LayoutKind.Sequential)] public readonly struct MLMarkerTrackerDecodedArucoData { /// /// Type selector for the structure. /// public readonly DecodedDataType Type; /// /// Dictionary used by the Aruco Marker. /// public readonly ArucoDictionaryName Dictionary; /// /// Type selector for the structure. /// public readonly uint Id; } /// /// Aruco decoded data. /// [StructLayout(LayoutKind.Sequential)] public readonly struct MLMarkerTrackerDecodedBinaryData { /// /// Type selector for the structure. /// public readonly DecodedDataType Type; /// /// Binary data size. /// public readonly uint Size; } /// /// Represents the decoded data encoded in the marker. Markers can encode binary /// data, strings, URLs and more. This struct represents the decoded data read from /// a marker. /// [StructLayout(LayoutKind.Sequential)] public readonly struct MLMarkerTrackerDecodedData { /// /// Data field contents depends on the selected detector. /// The Data's Type field indicates which structure this actually contains. /// public readonly IntPtr Data; /// /// Length of the decoded data. /// public readonly uint Size; public override string ToString() => $"DataSize: {Size}"; // -1 is for null terminated C strings } /// /// A list of these results will be returned by the Marker Scanner, after /// processing a video frame succesfully. /// [StructLayout(LayoutKind.Sequential)] public readonly struct MLMarkerTrackerResult { /// /// The data which was encoded in the marker. /// public readonly MLMarkerTrackerDecodedData DecodedData; /// /// The type of marker that was detected. /// public readonly MarkerType Type; /// /// This indicates if coord_frame_marker holds a valid pose. /// If false do not use the CoordinateFrameUID. /// [MarshalAs(UnmanagedType.I1)] public readonly bool IsValidPose; /// /// MLCoordinateFrameUID of the QR code. This FrameUID is only useful if the /// marker is of type #MLMarkerTypeQR This should be passed to the /// MLSnapshotGetTransform() function to get the 6 DOF pose of the QR code. Any /// marker that isn't a QR code will have an invalid FrameUID here. /// public readonly MLCoordinateFrameUID CoordinateFrameUID; /// /// The reprojection error of this QR code detection in degrees. /// /// The reprojection error is only useful when tracking QR codes. A high /// reprojection error means that the estimated pose of the QR code doesn't /// match well with the 2D detection on the processed video frame and thus the /// pose might be inaccurate. The error is given in degrees, signifying by how /// much either camera or QR code would have to be moved or rotated to create a /// perfect reprojection. The further away your QR code is, the smaller this /// reprojection error will be for the same displacement error of the code. /// public readonly float ReprojectionError; public override string ToString() => $"{DecodedData}\nType: {Enum.GetName(typeof(MarkerType), Type)}\nCoordFrameID: {CoordinateFrameUID}\nReproj Error: {ReprojectionError}"; } /// /// An array of all the detection results from the marker scanning. /// [StructLayout(LayoutKind.Sequential)] public readonly struct MLMarkerTrackerResultArray { public readonly uint Version; /// /// Pointer to an array of pointers for MLMarkerResult. /// public readonly IntPtr Detections; /// /// Number of markers being tracked. /// public readonly UIntPtr Count; public MLMarkerTrackerResultArray(uint version) { Version = version; Detections = IntPtr.Zero; Count = UIntPtr.Zero; } } /// /// When creating the Marker Scanner, this list of settings needs to be passed to /// configure the scanner properly.The estimated poses will only be correct if the /// marker length has been set correctly. /// [StructLayout(LayoutKind.Sequential)] public readonly partial struct MLMarkerTrackerSettings { /// /// Version of the struct. /// public readonly uint Version; /// /// If true, Marker Tracker will detect and track the enabled marker types. /// Marker Tracker should be disabled when app is paused and enabled when app resumes. /// When enabled, Marker Tracker will gain access to the camera and start analysing camera frames. /// When disabled Marker Tracker will release the camera and stop tracking markers. /// Internal state of the tracker will be maintained. /// [MarshalAs(UnmanagedType.I1)] public readonly bool EnableMarkerScanning; /// /// The marker types that are enabled for this scanner. Enable markers by /// combining any number of MLMarkerType flags using '|' (bitwise 'or'). /// public readonly uint EnabledDetectorTypes; /// /// Aruco Dictionary or April Tag name from which markers shall be tracked. /// public readonly ArucoDictionaryName ArucoDicitonary; /// /// The physical size of the Aruco marker that shall be tracked. /// /// The physical size is important to know, because once a Aruco marker is detected /// we can only determine its 3D position when we know its correct size. The /// size of the Aruco marker is given in meters and represents the length of one side /// of the square marker(without the outer margin). /// /// Min size: As a rule of thumb the size of a Aruco marker should be at least a 10th /// of the distance you intend to scan it with a camera device. Higher version /// markers with higher information density might need to be larger than that /// to be detected reliably. /// /// Max size: Our camera needs to see the whole marker at once. If it's too /// large, we won't detect it. /// public readonly float ArucoMarkerSize; /// /// The physical size of the QR code that shall be tracked. /// /// The physical size is important to know, because once a QR code is detected /// we can only determine its 3D position when we know its correct size. The /// size of the QR code is given in meters and represents the length of one side /// of the square code(without the outer margin). /// /// Min size: As a rule of thumb the size of a QR code should be at least a 10th /// of the distance you intend to scan it with a camera device. Higher version /// markers with higher information density might need to be larger than that /// to be detected reliably. /// /// Max size: Our camera needs to see the whole marker at once. If it's too /// large, we won't detect it. /// public readonly float QRCodeSize; /// /// Tracker profile to be used. /// public readonly Profile TrackerProfile; /// /// Custom tracker profile to be used if the TrackerProfile member is the Custom value (see MLMarkerTracker.Profile enum). /// public readonly MLMarkerTrackerCustomProfile CustomTrackerProfile; /// /// Sets the native structures from the user facing properties. /// public MLMarkerTrackerSettings(TrackerSettings settings) { this.Version = 6; this.EnableMarkerScanning = settings.EnableMarkerScanning; this.EnabledDetectorTypes = (uint)settings.MarkerTypes; this.ArucoDicitonary = settings.ArucoDicitonary; this.ArucoMarkerSize = settings.ArucoMarkerSize; this.QRCodeSize = settings.QRCodeSize; this.TrackerProfile = settings.TrackerProfile; this.CustomTrackerProfile = new MLMarkerTrackerCustomProfile(settings.CustomTrackerProfile); } } /// /// Marker Tracker system provides a set of standard tracking profiles (see MLMarkerTracker.Profile enum) /// to configure the tracker settings. This is the structure that defines a custom tracker profile. /// [StructLayout(LayoutKind.Sequential)] public readonly struct MLMarkerTrackerCustomProfile { /// /// Hint used for all detectors. /// public readonly FPSHint FPSHint; /// /// The resolution hint for all detectors. /// public readonly ResolutionHint ResolutionHint; /// /// The camera hint for all detectors. /// public readonly CameraHint CameraHint; /// /// This option provides control over corner refinement methods and a way to /// balance detection rate, speed and pose accuracy. Always available and /// applicable for Aruco and April tags. /// public readonly CornerRefineMethod CornerRefineMethod; /// /// Run refinement step that uses marker edges to generate even more accurate /// corners, but slow down tracking rate overall by consuming more compute. /// Aruco/April tags only. /// [MarshalAs(UnmanagedType.I1)] public readonly bool UseEdgeRefinement; /// /// In order to improve performance, the detectors don't always run on the full /// frame.Full frame analysis is however necessary to detect new markers that /// weren't detected before. Use this option to control how often the detector may /// detect new markers and its impact on tracking performance. /// public readonly FullAnalysisIntervalHint FullAnalysisIntervalHint; /// /// Sets the native structures from the user facing properties. /// public MLMarkerTrackerCustomProfile(TrackerSettings.CustomProfile customProfile) { this.FPSHint = customProfile.FPSHint; this.ResolutionHint = customProfile.ResolutionHint; this.CameraHint = customProfile.CameraHint; this.CornerRefineMethod = customProfile.CornerRefineMethod; this.UseEdgeRefinement = customProfile.UseEdgeRefinement; this.FullAnalysisIntervalHint = customProfile.FullAnalysisIntervalHint; } } } } }