// %BANNER_BEGIN% // --------------------------------------------------------------------- // %COPYRIGHT_BEGIN% // Copyright (c) 2022-2023 Magic Leap, Inc. All Rights Reserved. // Use of this file is governed by the Magic Leap 2 Software License Agreement, located here: https://www.magicleap.com/software-license-agreement-ml2 // Terms and conditions applicable to third-party materials accompanying this distribution may also be found in the top-level NOTICE file appearing herein. // %COPYRIGHT_END% // --------------------------------------------------------------------- // %BANNER_END% using System; using System.Collections; using System.Collections.Concurrent; using System.IO; using System.Text; using UnityEngine; using UnityEngine.SceneManagement; using static MagicLeap.Spectator.SpectatorMessages; namespace MagicLeap.Spectator { [RequireComponent(typeof(Host), typeof(DeviceDiscovery), typeof(MarkerDetector))] public class MLSpectator: MonoBehaviour { #region Static private static MLSpectator _instance = null; public static MLSpectator Instance { get { if (_instance == null) _instance = FindObjectOfType(true); if (_instance == null) Debug.LogWarning("MLSpectator: Instance is missing or has already been destroyed"); return _instance; } } private static readonly byte[] encodedAudioBaseVersion = new byte[] { 1, 3, 6 }; #endregion #region Public enums public enum State { Disconnected, Connecting, Connected, LookingForMarker, FoundMarker, Running, } #endregion #region Inspector variables [SerializeField, Tooltip("Prefab for visualizing our spectator camera placement in the scene")] private GameObject spectatorPrefab = null; [SerializeField, Tooltip("Show render preview in Game View")] private bool showPreviewInEditor = true; // Disabling this in inspector until we have a default way of answering confirmation requests //[SerializeField, Tooltip("Require our connection to the headset to be accepted by the user (requires a way to call AcceptConnection)")] private bool requireConfirmation = false; [SerializeField, Tooltip("Automatically enable MLSectator on Awake")] private bool autoEnable = false; [SerializeField, Tooltip("If your only scene light is on your main camera this can be used to achieve the same lighting for spectator")] private bool controlSceneLights = false; [SerializeField, Tooltip("The audio clip that plays when a marker is detected")] private AudioClip audioClipForMarkerEvent = null; [Space] [SerializeField, Tooltip("Whether we want to use a custom layer mask instead of copying from the main camera")] private bool useCustomLayerMask = false; [SerializeField, Tooltip("The custom layer mask to use in place of the main camera's")] private LayerMask customLayerMask = new LayerMask(); #endregion #region Private variables // Our marker detector private MarkerDetector markerDetector = null; // Our spectator renderer private MLSpectatorRenderer spectatorRenderer = null; // Our audio capture component private MLSpectatorAudio spectatorAudio = null; // Our current state private State _state = State.Disconnected; private object stateLock = new object(); // Thread safe Queue of states to force update in Update private ConcurrentQueue stateQueue = new(); // Our initializiation data (thread safe) private byte[] _initData = null; private object initLock = new object(); private byte[] initData { get { lock (initLock) { return _initData; } } set { lock (initLock) { _initData = value; } } } // Whether or not we are active private SafeBool active = new(); // Our heartbeat coroutine private IEnumerator heartbeats = null; // Our user identifier private string _userId = string.Empty; private object idLock = new(); // The identifying metadata that will be transmitted via DeviceDiscovery private MLSpectatorDeviceMetaData deviceMetaData = null; // Used to follow the main camera's parent private MLSpectatorFollow follow = null; // Marker world pose private Pose markerWorldPose; // Audio source for marker detection event private AudioSource audioSource = null; // The guid of our last session (for autoreconnect) private Guid lastSessionId = new Guid(); #endregion #region Public properties public string Version { get; private set; } = "1.0.0"; public bool IsActive => active; public Camera mainCam { get; private set; } = null; public State state { get { lock (stateLock) { return _state; } } private set { lock (stateLock) { _state = value; } //onStateChanged?.Invoke(value); stateQueue.Enqueue(value); } } public string userId { get { lock (idLock) { return _userId; } } set { lock (idLock) { _userId = value; } onUserIdChanged?.Invoke(value); } } public bool ControlSceneLights => controlSceneLights; public bool UseCustomLayerMask => useCustomLayerMask; public LayerMask CustomLayerMask => customLayerMask; public MLSpectatorAudio Audio => spectatorAudio; // The version the connected phone is running public byte[] PhoneVersion { get; private set; } = null; // Whether or not our phone version expectes encoded audio public bool EncodeAudio { get => // If we haven't received a version, we can stop right here PhoneVersion != null && // If our major is greater, then we're good PhoneVersion[0] > encodedAudioBaseVersion[0] || // If our major is equal and our minor is greater, then we're good (PhoneVersion[0] == encodedAudioBaseVersion[0] && PhoneVersion [1] > encodedAudioBaseVersion[1]) || // If our major and minor are equal, and our revision is equal or greater, then we're good (PhoneVersion[0] == encodedAudioBaseVersion[0] && PhoneVersion[1] == encodedAudioBaseVersion[1] && PhoneVersion [2] >= encodedAudioBaseVersion[2]); } // Whether or not we should pack alpha into the green channel // (same as encoded audio version) public bool PackGreen => EncodeAudio; #endregion #region Public events // Callback for state change notification public event Action onStateChanged = null; // Callback for custom render event public event Action onRenderRequest = null; #endregion #region Private events // Callback for userID change notification private event Action onUserIdChanged = null; #endregion #region MonoBehaviour methods private void Awake() { // If we are a root gameobject if (transform.parent == null) { // Make sure we aren't destroyed between scenes DontDestroyOnLoad(this); } else Debug.LogWarning("MLSpectator: Awake() - Must be root object to persist between scenes"); // Get our spectator version var spectatorVersion = Resources.Load("Prefabs/MLSpectatorVersion"); if (!spectatorVersion) Debug.LogError("MLSpectator: Awake() - MLSpectator version not found!"); else Version = spectatorVersion.Version; // Subscribe to scene changes SceneManager.sceneLoaded += SceneLoaded; // Kick off Awake like a scene load SceneLoaded(SceneManager.GetActiveScene(), LoadSceneMode.Single); // Subscribe to host connection Host.Instance.OnConnected += OnConnected; #if !UNITY_EDITOR // Need this unless we decide to make marker detector a singleton markerDetector = GetComponent(); #endif if (audioClipForMarkerEvent != null) { audioSource = gameObject.AddComponent(); audioSource.clip = audioClipForMarkerEvent; audioSource.loop = false; audioSource.volume = 1.0f; audioSource.playOnAwake = false; audioSource.spatialize = true; audioSource.spatialBlend = 1.0f; } // Set script to gather device meta data to transmit via DeviceDiscovery deviceMetaData = new MLSpectatorDeviceMetaData(); onStateChanged += deviceMetaData.UpdateState; onUserIdChanged += deviceMetaData.UpdateUserId; DeviceDiscovery.Instance.SetDeviceMetaData(deviceMetaData); // Auto-enable ourself if directed to if (autoEnable) Enable(); } private void OnDestroy() { // Unsubscribe from scene changes SceneManager.sceneLoaded -= SceneLoaded; if (Host.Instance) { // Unsubscribe to host connection / disconnection Host.Instance.OnConnected -= OnConnected; Host.Instance.OnDisconnect -= OnDisconnect; } } private void OnEnable() { Host.Instance.RegisterCallback(ConnectRequest, RequestConnect); Host.Instance.RegisterCallback(SpectatorRequest, HandleConnect); } private void OnDisable() { Host.Instance.UnregisterCallback(ConnectRequest, RequestConnect); Host.Instance.UnregisterCallback(SpectatorRequest, HandleConnect); } #endregion #region SceneManager callbacks private void SceneLoaded(Scene scene, LoadSceneMode mode) { // Get a reference to the main camera in our new scene mainCam = Camera.main; // If our cam camera is parented to something if (mainCam.transform.parent != null) { // Get or create a follow object for translating into the camera's parent space follow = mainCam.transform.parent.GetComponentInChildren(); if (follow == null) { GameObject followObject = new GameObject("MLSpectatorFollow"); followObject.transform.parent = mainCam.transform.parent; follow = followObject.AddComponent(); follow.onPoseUpdated += pose => { var position = follow.transform.position; var rotation = follow.transform.rotation; if (markerDetector != null) rotation *= Quaternion.Euler(-90, 180, 0); transform.SetPositionAndRotation(position, rotation); }; } follow.transform.localPosition = markerWorldPose.position; follow.transform.localRotation = markerWorldPose.rotation; } } #endregion #region Network methods private void OnConnected() { Debug.Log("MLSpectator: OnConnected()"); // Subscribe to host disconnect Host.Instance.OnDisconnect += OnDisconnect; // Send heartbeats once a second heartbeats = SendHeartbeats(); StartCoroutine(heartbeats); } private void OnDisconnect() { Debug.Log("MLSpectator: OnDisconnect()"); // Unsubscribe from host disconnect Host.Instance.OnDisconnect -= OnDisconnect; // Distroy our spectator renderer if we have one if (spectatorRenderer) Destroy(spectatorRenderer.gameObject); // Turn off our marker detector if it is still on if (markerDetector) markerDetector.enabled = false; // Throw out any data we might be sitting on initData = null; // Update our state state = State.Disconnected; } private void RequestConnect(Message message) { Debug.Log("MLSpectator: RequestConnect() - Begin"); if (message.size > 0) { string appVersion = Encoding.ASCII.GetString(message.data, 0, message.size); string[] theirParts = appVersion.Split("."); string[] myParts = Version.Split("."); // Save our phone version PhoneVersion = new[] { byte.Parse(theirParts[0]), byte.Parse(theirParts[1]), byte.Parse(theirParts[2]) }; // Major.Minor.Revision - revision changes shouldn't be breaking so we only care about the first two if (theirParts[0] != myParts[0] || theirParts[1] != myParts[1]) { Debug.LogWarning("MLSpectator: RequestConnect() - Version mismatch, Plugin: " + $"{myParts[0]}.{myParts[1]}.{myParts[2]}, Phone: {theirParts[0]}.{theirParts[1]}.{theirParts[2]}"); using (MemoryStream stream = new()) { StreamHelper.Write(stream, (int)ConnectionResponse.RejectedFromVersionMismatch); StreamHelper.Write(stream, byte.Parse(myParts[0])); StreamHelper.Write(stream, byte.Parse(myParts[1])); StreamHelper.Write(stream, byte.Parse(myParts[2])); Host.Instance.Send(new Message(ConnectResponse, stream.GetBuffer())); } return; } } state = State.Connecting; if (!requireConfirmation) AcceptConnection(); Debug.Log("MLSpectator: RequestConnect() - End"); } private void HandleConnect(Message message) { Debug.Log("MLSpectator: HandleConnect()"); // If we don't already have an initialization message if (initData == null) { // Copy our message data initData = new byte[message.size]; Array.Copy(message.data, initData, message.size); Debug.Log("MLSpectator: HandleConnect() - Received initialization message"); } } private IEnumerator SendHeartbeats() { while (true) { yield return new WaitForSeconds(1); Host.Instance.Send(Heartbeat); //Debug.Log("Heartbeat"); } } #endregion #region Marker detector methods private void OnMarkerEvent(string id, GameObject marker) { Debug.Log("MLSpectator: OnMarkerEvent()"); // Unsubscribe from our marker events markerDetector.markerEvent.RemoveListener(OnMarkerEvent); Debug.Log("MLSpectator: OnMarkerEvent() - Removed listener"); if (!markerDetector.isActiveAndEnabled) { Debug.LogWarning("MLSpectator: OnMarkerEvent() - Marker detector is already disabled"); return; } // Update our state state = State.FoundMarker; // Hide our marker marker.SetActive(false); markerWorldPose = new Pose(marker.transform.localPosition, marker.transform.localRotation); if (follow) { follow.transform.localPosition = markerWorldPose.position; follow.transform.localRotation = markerWorldPose.rotation; } // Update our position to match our marker var position = marker.transform.position; var rotation = marker.transform.rotation; transform.SetPositionAndRotation(position, rotation * Quaternion.Euler(-90, 180, 0)); // Move our marker back to it's original position marker.transform.position = position; marker.transform.rotation = rotation; // Copy our marker to keep it around var markerCopy = Instantiate(marker, position, rotation, transform); markerCopy.name = "Marker Copy"; markerCopy.SetActive(true); // Make sure our marker fades away var originMarker = markerCopy.AddComponent(); originMarker.FadeAway(2.0f, 1.0f); // Start waiting for our marker detector to stop StartCoroutine(WaitForMarkerDetector()); // Disable our marker detector Debug.Log("MLSpectator: OnMarkerEvent() - Disabling marker detector"); markerDetector.enabled = false; // Play our audio clip for marker event if (audioSource != null) audioSource.Play(); } private IEnumerator WaitForMarkerDetector() { // Wait for the marker detector to actually stop yield return new WaitUntil(() => markerDetector.state == MarkerDetector.State.Stopped); Debug.Log("MLSpectator: WaitForMarkerDetector() - Marker detector disabled"); // Create our spectator camera CreateSpectatorCamera(); } #endregion #region Private methods private IEnumerator WaitForInitMessage() { while (active) { // Wait until we have an initialization message and we're not looking for our marker if (initData == null || state == State.LookingForMarker || state == State.FoundMarker) { yield return null; continue; } // Check if auto reconnect if (initData.Length > 48 && BitConverter.ToBoolean(initData, 48)) { Debug.Log("MLSpectator: WaitForInitMessage() - Autoreconnection"); // If our session id doesn't match if (new Guid(new ReadOnlySpan(initData, 49, 16)) != lastSessionId) { // Send our connection response and dump our init data Debug.LogWarning("MLSpectator: WaitForInitMessage() - Session mismatch"); Host.Instance.Send(new Message(SpectatorResponse, lastSessionId.ToByteArray())); initData = null; } // Otherwise, answer our initialization request here else CreateSpectatorCamera(true); } // Otherwise, if we have a marker detector else if (markerDetector != null) { // Subscribe to marker events markerDetector.markerEvent.AddListener(OnMarkerEvent); // Enable the marker detector markerDetector.enabled = true; // Update our state state = State.LookingForMarker; Debug.Log("MLSpectator: WaitForInitMessage() - Waiting for marker event"); } // Otherwise, answer our initialization request here else CreateSpectatorCamera(); } } private IEnumerator NotifyStateChanges() { while (active) { if (stateQueue.TryDequeue(out State state)) onStateChanged?.Invoke(state); yield return null; } } private void CreateSpectatorCamera(bool reconnection = false) { if (initData == null) { Debug.LogWarning("MLSpectator: CreateSpectatorCamera() - Cannot create specator camera without init data"); return; } // Set state to running state = State.Running; // If we already have a spectator if (spectatorRenderer != null) { // Send our response, clear our data, and break Host.Instance.Send(SpectatorResponse); initData = null; return; } GameObject spectator = null; // If we have a prefab to use, instantiate it if (spectatorPrefab) { spectator = Instantiate(spectatorPrefab); spectator.name = "MLSpectatorCamera"; } // Otherwise, create an empty game object else spectator = new GameObject("MLSpectatorCamera"); // Initialize our spectator spectator.transform.SetParent(transform); spectator.transform.SetLocalPositionAndRotation(Vector3.zero, Quaternion.identity); spectatorRenderer = spectator.AddComponent(); spectatorRenderer.onRenderRequest += onRenderRequest; spectatorRenderer.Initialize(initData, showPreviewInEditor); initData = null; // Create a new session id if not a reconnection if (!reconnection) lastSessionId = Guid.NewGuid(); // Send response once we're done initializing - include our session id Host.Instance.Send(new Message(SpectatorResponse, lastSessionId.ToByteArray())); // Stop sending heartbeats StopCoroutine(heartbeats); } #endregion #region Public methods public void Enable() { // Set ourselves active active.Set(true); // Get or add our audio capture component spectatorAudio = gameObject.GetComponent(); if (spectatorAudio == null) spectatorAudio = gameObject.AddComponent(); // Enable our host Host.Instance.enabled = true; // Enable device descovery DeviceDiscovery.Instance.enabled = true; // Start waiting for an init message StartCoroutine(WaitForInitMessage()); // Notify state changes on the main thread StartCoroutine(NotifyStateChanges()); } public void Disable() { // Set ourselves inactive active.Set(false); // Clear our queue of state changes stateQueue.Clear(); // If we're looking for a marker, unsubcribe if (state == State.LookingForMarker) markerDetector.markerEvent.RemoveListener(OnMarkerEvent); // Disable our marker tracker if(markerDetector) markerDetector.enabled = false; // Disable our host Host.Instance.enabled = false; // Disable device discovery DeviceDiscovery.Instance.enabled = false; } public void Toggle() { if (active) Disable(); else Enable(); } public void AcceptConnection() { if (state == State.Connecting) { using (MemoryStream stream = new()) { string[] myParts = Version.Split("."); StreamHelper.Write(stream, (int)ConnectionResponse.Accepted); StreamHelper.Write(stream, byte.Parse(myParts[0])); StreamHelper.Write(stream, byte.Parse(myParts[1])); StreamHelper.Write(stream, byte.Parse(myParts[2])); Host.Instance.Send(new Message(ConnectResponse, stream.GetBuffer())); } state = State.Connected; } else Debug.LogWarning($"MLSpectatorRenderer: AcceptConnection() - Cannot accept connection from state {state}"); } public void RejectConnection() { if (state == State.Connecting) { Host.Instance.Send(new Message(ConnectResponse, BitConverter.GetBytes((int)ConnectionResponse.RejectedByUser))); state = State.Disconnected; } else Debug.LogWarning($"MLSpectatorRenderer: AcceptConnection() - Cannot reject connection from state {state}"); } #endregion } }