using System; using System.Collections.Concurrent; using System.Collections.Generic; using Unity.XR.CoreUtils; using UnityEngine; using UnityEngine.Assertions; using PCD = Phantom.XRMOD.QuestModule.Runtime.PassthroughCameraDebugger; namespace Phantom.XRMOD.QuestModule.Runtime { /// /// Utility class for interacting with the Meta Quest Passthrough Camera API. /// /// Provides methods to access camera intrinsics, poses, and frames via Android Java calls. /// /// public class PassthroughCameraUtils { // The Horizon OS starts supporting PCA with v74. /// /// Minimum supported Horizon OS version for Passthrough Camera Access (v74). /// public const int MINSUPPORTOSVERSION = 74; // The only pixel format supported atm private const int YUV_420_888 = 0x00000023; private static AndroidJavaObject s_currentActivity; private static AndroidJavaObject s_cameraManager; private static bool? s_isSupported; private static int? s_horizonOsVersion; static Dictionary headsetDictionary = new() { {"miramar", "Quest1"}, {"hollywood", "Quest2"}, {"eureka", "Quest3"} }; // Caches internal static readonly Dictionary CameraEyeToCameraIdMap = new(); private static readonly ConcurrentDictionary> s_cameraOutputSizes = new(); private static readonly ConcurrentDictionary s_cameraCharacteristicsMap = new(); private static readonly Pose?[] s_cachedCameraPosesRelativeToHead = new Pose?[2]; /// /// Get the Horizon OS version number on the headset /// public static int? HorizonOSVersion { get { if (!s_horizonOsVersion.HasValue) { var vrosClass = new AndroidJavaClass("vros.os.VrosBuild"); s_horizonOsVersion = vrosClass.CallStatic("getSdkVersion"); #if OVR_INTERNAL_CODE // 10000 means that the build doesn't have a proper release version, and it is still in Mainline, // not in a release branch. #endif // OVR_INTERNAL_CODE if (s_horizonOsVersion == 10000) { s_horizonOsVersion = -1; } } return s_horizonOsVersion.Value != -1 ? s_horizonOsVersion.Value : null; } } /// /// Returns true if the current headset supports Passthrough Camera API /// public static bool IsSupported { get { if (!s_isSupported.HasValue) { var tmp_Build = new AndroidJavaClass("android.os.Build"); var tmp_Device = tmp_Build.GetStatic("DEVICE"); return headsetDictionary.ContainsKey(tmp_Device) && (!HorizonOSVersion.HasValue || HorizonOSVersion >= MINSUPPORTOSVERSION); } return s_isSupported.Value; } } /// /// Provides a list of resolutions supported by the passthrough camera. Developers should use one of those /// when initializing the camera. /// /// The passthrough camera public static List GetOutputSizes(PassthroughCameraEye cameraEye) { return s_cameraOutputSizes.GetOrAdd(cameraEye, GetOutputSizesInternal(cameraEye)); } /// /// Returns the camera intrinsics for a specified passthrough camera. All the intrinsics values are provided /// in pixels. The resolution value is the maximum resolution available for the camera. /// /// The passthrough camera public static PassthroughCameraIntrinsics GetCameraIntrinsics(PassthroughCameraEye cameraEye) { var cameraCharacteristics = GetCameraCharacteristics(cameraEye); var intrinsicsArr = GetCameraValueByKey(cameraCharacteristics, "LENS_INTRINSIC_CALIBRATION"); // Querying the camera resolution for which the intrinsics are provided // https://developer.android.com/reference/android/hardware/camera2/CameraCharacteristics#SENSOR_INFO_PRE_CORRECTION_ACTIVE_ARRAY_SIZE // This is a Rect of 4 elements: [bottom, left, right, top] with (0,0) at top-left corner. var sensorSize = GetCameraValueByKey(cameraCharacteristics, "SENSOR_INFO_PRE_CORRECTION_ACTIVE_ARRAY_SIZE"); return new PassthroughCameraIntrinsics { FocalLength = new Vector2(intrinsicsArr[0], intrinsicsArr[1]), PrincipalPoint = new Vector2(intrinsicsArr[2], intrinsicsArr[3]), Resolution = new Vector2Int(sensorSize.Get("right"), sensorSize.Get("bottom")), Skew = intrinsicsArr[4] }; } /// /// Returns an Android Camera2 API's cameraId associated with the passthrough camera specified in the argument. /// /// The passthrough camera /// Throws an exception if the code was not able to find cameraId public static string GetCameraIdByEye(PassthroughCameraEye cameraEye) { _ = EnsureInitialized(); return !CameraEyeToCameraIdMap.TryGetValue(cameraEye, out var value) ? throw new ApplicationException($"Cannot find cameraId for the eye {cameraEye}") : value.id; } /// /// Returns the world pose of a passthrough camera at a given time. /// The LENS_POSE_TRANSLATION and LENS_POSE_ROTATION keys in 'android.hardware.camera2' are relative to the origin, so they can be cached to improve performance. /// /// The passthrough camera /// The passthrough camera's world pose public static Pose GetCameraPoseInWorld(PassthroughCameraEye cameraEye) { var index = cameraEye == PassthroughCameraEye.Left ? 0 : 1; if (s_cachedCameraPosesRelativeToHead[index] == null) { var cameraId = GetCameraIdByEye(cameraEye); var cameraCharacteristics = s_cameraManager.Call("getCameraCharacteristics", cameraId); var cameraTranslation = GetCameraValueByKey(cameraCharacteristics, "LENS_POSE_TRANSLATION"); var p_headFromCamera = new Vector3(cameraTranslation[0], cameraTranslation[1], -cameraTranslation[2]); var cameraRotation = GetCameraValueByKey(cameraCharacteristics, "LENS_POSE_ROTATION"); var q_cameraFromHead = new Quaternion(-cameraRotation[0], -cameraRotation[1], cameraRotation[2], cameraRotation[3]); var q_headFromCamera = Quaternion.Inverse(q_cameraFromHead); s_cachedCameraPosesRelativeToHead[index] = new Pose() { position = p_headFromCamera, rotation = q_headFromCamera }; } // var headFromCamera = s_cachedCameraPosesRelativeToHead[index].Value; // var worldFromHead = OVRPlugin.GetNodePoseStateImmediate(OVRPlugin.Node.Head).Pose.ToOVRPose(); // var worldFromCamera = worldFromHead * headFromCamera; // worldFromCamera.orientation *= Quaternion.Euler(180, 0, 0); // return new Pose(worldFromCamera.position, worldFromCamera.orientation); var headFromCamera = s_cachedCameraPosesRelativeToHead[index].Value; return headFromCamera; } /// /// Returns a 3D ray in the world space which starts from the passthrough camera origin and passes through the /// 2D camera pixel. /// /// The passthrough camera /// A 2D point on the camera texture. The point is positioned relative to the /// maximum available camera resolution. This resolution can be obtained using /// or methods. /// public static Ray ScreenPointToRayInWorld(PassthroughCameraEye cameraEye, Vector2Int screenPoint) { var rayInCamera = ScreenPointToRayInCamera(cameraEye, screenPoint); var cameraPoseInWorld = GetCameraPoseInWorld(cameraEye); var rayDirectionInWorld = cameraPoseInWorld.rotation * rayInCamera.direction; return new Ray(cameraPoseInWorld.position, rayDirectionInWorld); } /// /// Returns a 3D ray in the camera space which starts from the passthrough camera origin - which is always /// (0, 0, 0) - and passes through the 2D camera pixel. /// /// The passthrough camera /// A 2D point on the camera texture. The point is positioned relative to the /// maximum available camera resolution. This resolution can be obtained using /// or methods. /// public static Ray ScreenPointToRayInCamera(PassthroughCameraEye cameraEye, Vector2Int screenPoint) { var intrinsics = GetCameraIntrinsics(cameraEye); var directionInCamera = new Vector3 { x = (screenPoint.x - intrinsics.PrincipalPoint.x) / intrinsics.FocalLength.x, y = (screenPoint.y - intrinsics.PrincipalPoint.y) / intrinsics.FocalLength.y, z = 1 }; return new Ray(Vector3.zero, directionInCamera); } #region Private methods internal static bool EnsureInitialized() { if (CameraEyeToCameraIdMap.Count == 2) { return true; } Debug.Log($"PCA: PassthroughCamera - Initializing..."); using var activityClass = new AndroidJavaClass("com.unity3d.player.UnityPlayer"); s_currentActivity = activityClass.GetStatic("currentActivity"); s_cameraManager = s_currentActivity.Call("getSystemService", "camera"); Assert.IsNotNull(s_cameraManager, "Camera manager has not been provided by the Android system"); var cameraIds = GetCameraIdList(); Debug.Log($"PCA: PassthroughCamera - cameraId list is {string.Join(", ", cameraIds)}"); for (var idIndex = 0; idIndex < cameraIds.Length; idIndex++) { var cameraId = cameraIds[idIndex]; CameraSource? cameraSource = null; CameraPosition? cameraPosition = null; var cameraCharacteristics = GetCameraCharacteristics(cameraId); using var keysList = cameraCharacteristics.Call("getKeys"); var size = keysList.Call("size"); for (var i = 0; i < size; i++) { using var key = keysList.Call("get", i); var keyName = key.Call("getName"); if (string.Equals(keyName, "com.meta.extra_metadata.camera_source", StringComparison.OrdinalIgnoreCase)) { // Both `com.meta.extra_metadata.camera_source` and `com.meta.extra_metadata.camera_source` are // custom camera fields which are stored as arrays of size 1, instead of single values. // We have to read those values correspondingly var cameraSourceArr = GetCameraValueByKey(cameraCharacteristics, key); if (cameraSourceArr == null || cameraSourceArr.Length != 1) continue; cameraSource = (CameraSource) cameraSourceArr[0]; } else if (string.Equals(keyName, "com.meta.extra_metadata.position", StringComparison.OrdinalIgnoreCase)) { var cameraPositionArr = GetCameraValueByKey(cameraCharacteristics, key); if (cameraPositionArr == null || cameraPositionArr.Length != 1) continue; cameraPosition = (CameraPosition) cameraPositionArr[0]; } } if (!cameraSource.HasValue || !cameraPosition.HasValue || cameraSource.Value != CameraSource.Passthrough) continue; switch (cameraPosition) { case CameraPosition.Left: Debug.Log($"PCA: Found left passthrough cameraId = {cameraId}"); CameraEyeToCameraIdMap[PassthroughCameraEye.Left] = (cameraId, idIndex); break; case CameraPosition.Right: Debug.Log($"PCA: Found right passthrough cameraId = {cameraId}"); CameraEyeToCameraIdMap[PassthroughCameraEye.Right] = (cameraId, idIndex); break; default: throw new ApplicationException($"Cannot parse Camera Position value {cameraPosition}"); } } return CameraEyeToCameraIdMap.Count == 2; } private static string[] GetCameraIdList() { return s_cameraManager.Call("getCameraIdList"); } private static List GetOutputSizesInternal(PassthroughCameraEye cameraEye) { _ = EnsureInitialized(); var cameraId = GetCameraIdByEye(cameraEye); var cameraCharacteristics = GetCameraCharacteristics(cameraId); using var configurationMap = GetCameraValueByKey(cameraCharacteristics, "SCALER_STREAM_CONFIGURATION_MAP"); var outputSizes = configurationMap.Call("getOutputSizes", YUV_420_888); var result = new List(); foreach (var outputSize in outputSizes) { var width = outputSize.Call("getWidth"); var height = outputSize.Call("getHeight"); result.Add(new Vector2Int(width, height)); } foreach (var obj in outputSizes) { obj?.Dispose(); } return result; } private static AndroidJavaObject GetCameraCharacteristics(string cameraId) { return s_cameraCharacteristicsMap.GetOrAdd(cameraId, _ => s_cameraManager.Call("getCameraCharacteristics", cameraId)); } private static AndroidJavaObject GetCameraCharacteristics(PassthroughCameraEye eye) { var cameraId = GetCameraIdByEye(eye); return GetCameraCharacteristics(cameraId); } private static T GetCameraValueByKey(AndroidJavaObject cameraCharacteristics, string keyStr) { var key = cameraCharacteristics.GetStatic(keyStr); return GetCameraValueByKey(cameraCharacteristics, key); } private static T GetCameraValueByKey(AndroidJavaObject cameraCharacteristics, AndroidJavaObject key) { return cameraCharacteristics.Call("get", key); } private enum CameraSource { Passthrough = 0 } private enum CameraPosition { Left = 0, Right = 1 } #endregion Private methods } /// /// Contains camera intrinsics, which describe physical characteristics of a passthrough camera /// public struct PassthroughCameraIntrinsics { /// /// The focal length in pixels /// public Vector2 FocalLength; /// /// The principal point from the top-left corner of the image, expressed in pixels /// public Vector2 PrincipalPoint; /// /// The resolution in pixels for which the intrinsics are defined /// public Vector2Int Resolution; /// /// The skew coefficient which represents the non-perpendicularity of the image sensor's x and y axes /// public float Skew; } }