// %BANNER_BEGIN%
// ---------------------------------------------------------------------
// %COPYRIGHT_BEGIN%
// Copyright (c) (2018-2022) Magic Leap, Inc. All Rights Reserved.
// Use of this file is governed by the Software License Agreement, located here: https://www.magicleap.com/software-license-agreement-ml2
// Terms and conditions applicable to third-party materials accompanying this distribution may also be found in the top-level NOTICE file appearing herein.
// %COPYRIGHT_END%
// ---------------------------------------------------------------------
// %BANNER_END%
namespace UnityEngine.XR.MagicLeap
{
using System;
using System.Collections.Generic;
using System.Threading;
using UnityEngine.XR.MagicLeap.Native;
///
/// MLWebRTC class contains the API to interface with the
/// WebRTC C API.
///
public partial class MLWebRTC
{
///
/// Class that represents a video sink used by the MLWebRTC API.
/// Video sinks are fed data by media sources and produces frames to render.
///
public partial class VideoSink : Sink
{
///
/// Buffer for the image planes array to use to hold the image plane data.
///
private CircularBuffer imagePlanesBuffer = CircularBuffer.Create(new Frame.PlaneInfo[Frame.PlaneInfo.MaxImagePlanes], new Frame.PlaneInfo[Frame.PlaneInfo.MaxImagePlanes], new Frame.PlaneInfo[Frame.PlaneInfo.MaxImagePlanes]);
///
/// The newest frame handle that the video sink knows of.
///
private ulong newFrameHandle;
private uint streamWidth = 0;
private uint streamHeight = 0;
private AutoResetEvent updateVideoEvent = new AutoResetEvent(true);
public delegate void OnFrameResolutionChangedDelegate(uint newWidth, uint newHeight);
public delegate void OnDestroySinkDelegate(VideoSink videoSink);
public delegate void OnStreamChangedDelegate(MediaStream stream);
public event OnDestroySinkDelegate OnDestroySink = delegate { };
public event OnFrameResolutionChangedDelegate OnFrameResolutionChanged = delegate { };
public event OnStreamChangedDelegate OnStreamChanged;
///
/// Initializes a new instance of the class.
///
internal VideoSink()
{
this.Type = MediaStream.Track.Type.Video;
}
///
/// Initializes a new instance of the class.
///
/// The Handle of the video sink.
internal VideoSink(ulong Handle) : base(Handle)
{
this.Type = MediaStream.Track.Type.Video;
}
///
/// Creates an initialized VideoSink object.
///
/// The MLResult object of the inner platform call(s).
/// An initialized VideoSink object.
public static VideoSink Create(out MLResult result)
{
VideoSink videoSink = null;
List sinks = MLWebRTC.Instance.sinks;
ulong Handle = MagicLeapNativeBindings.InvalidHandle;
MLResult.Code resultCode = NativeBindings.MLWebRTCVideoSinkCreate(out Handle);
if (!MLResult.DidNativeCallSucceed(resultCode, nameof(NativeBindings.MLWebRTCVideoSinkCreate)))
{
result = MLResult.Create(resultCode);
return videoSink;
}
videoSink = new VideoSink(Handle);
if (MagicLeapNativeBindings.MLHandleIsValid(videoSink.Handle))
{
sinks.Add(videoSink);
}
result = MLResult.Create(resultCode);
return videoSink;
}
public bool IsNewFrameAvailable()
{
MLResult.Code resultCode = NativeBindings.MLWebRTCVideoSinkIsNewFrameAvailable(this.Handle, out bool newFrameAvailable);
MLResult.DidNativeCallSucceed(resultCode, nameof(NativeBindings.MLWebRTCVideoSinkIsNewFrameAvailable));
return newFrameAvailable;
}
public bool AcquireNextAvailableFrame(out Frame newFrame)
{
newFrame = new Frame();
ulong frameHandle = MagicLeapNativeBindings.InvalidHandle;
MLResult.Code resultCode = NativeBindings.MLWebRTCVideoSinkAcquireNextAvailableFrame(this.Handle, out frameHandle);
if (!MLResult.DidNativeCallSucceed(resultCode, nameof(NativeBindings.MLWebRTCVideoSinkAcquireNextAvailableFrame)))
{
return false;
}
Frame.NativeBindings.MLWebRTCFrame nativeFrame = Frame.NativeBindings.MLWebRTCFrame.Create(Frame.OutputFormat.YUV_420_888);
resultCode = Frame.NativeBindings.MLWebRTCFrameGetData(frameHandle, ref nativeFrame);
if (MLResult.DidNativeCallSucceed(resultCode, nameof(Frame.NativeBindings.MLWebRTCFrameGetData)))
{
newFrameHandle = frameHandle;
newFrame = Frame.Create(frameHandle, nativeFrame, imagePlanesBuffer.Get());
if (streamWidth != newFrame.NativeFrame.Width || streamHeight != newFrame.NativeFrame.Height)
{
streamWidth = newFrame.NativeFrame.Width;
streamHeight = newFrame.NativeFrame.Height;
MLThreadDispatch.Call(newFrame.NativeFrame.Width, newFrame.NativeFrame.Height, OnFrameResolutionChanged);
Debug.LogWarning($"new frame acquired has new size: {streamWidth} x {streamHeight}");
}
return true;
}
return false;
}
public void ReleaseFrame()
{
if (MagicLeapNativeBindings.MLHandleIsValid(newFrameHandle))
{
var resultCode = NativeBindings.MLWebRTCVideoSinkReleaseFrame(Handle, newFrameHandle);
MLResult.DidNativeCallSucceed(resultCode, nameof(NativeBindings.MLWebRTCVideoSinkReleaseFrame));
newFrameHandle = MagicLeapNativeBindings.InvalidHandle;
}
}
///
/// Sets the track of the video sink.
///
/// The track to use.
///
/// MLResult.Result will be MLResult.Code.Ok if destroying all handles was successful.
/// MLResult.Result will be MLResult.Code.WebRTCResultInstanceNotCreated if MLWebRTC instance was not created.
/// MLResult.Result will be MLResult.Code.WebRTCResultMismatchingHandle if an incorrect handle was sent.
///
protected override MLResult SetTrack(MediaStream.Track track)
{
ulong sourceHandle = track != null ? track.Handle : MagicLeapNativeBindings.InvalidHandle;
MLResult.Code resultCode = NativeBindings.MLWebRTCVideoSinkSetSource(this.Handle, sourceHandle);
MLResult.DidNativeCallSucceed(resultCode, nameof(NativeBindings.MLWebRTCVideoSinkSetSource));
return MLResult.Create(resultCode);
}
///
/// Sets the stream of the video sink sink.
///
/// The stream to use.
///
/// MLResult.Result will be MLResult.Code.Ok if destroying all handles was successful.
/// MLResult.Result will be MLResult.Code.WebRTCResultInstanceNotCreated if MLWebRTC instance was not created.
/// MLResult.Result will be MLResult.Code.WebRTCResultMismatchingHandle if an incorrect handle was sent.
///
public MLResult SetStream(MediaStream stream)
{
if (stream == Stream)
{
return MLResult.Create(MLResult.Code.Ok);
}
Stream = stream;
if (OnStreamChanged != null)
{
OnStreamChanged(Stream);
}
if (Stream == null)
{
return SetTrack(null);
}
return SetTrack(Stream.ActiveVideoTrack);
}
///
/// Destroys the video sink.
///
///
/// MLResult.Result will be MLResult.Code.Ok if destroying all handles was successful.
/// MLResult.Result will be MLResult.Code.WebRTCResultInstanceNotCreated if MLWebRTC instance was not created.
/// MLResult.Result will be MLResult.Code.WebRTCResultMismatchingHandle if an incorrect handle was sent.
///
public override MLResult Destroy()
{
if (!MagicLeapNativeBindings.MLHandleIsValid(this.Handle))
{
return MLResult.Create(MLResult.Code.InvalidParam, "Handle is invalid.");
}
OnDestroySink(this);
updateVideoEvent.WaitOne(250);
this.SetStream(null);
// TODO : synchronize with renderer
MLResult.Code resultCode = NativeBindings.MLWebRTCVideoSinkDestroy(this.Handle);
MLResult.DidNativeCallSucceed(resultCode, nameof(NativeBindings.MLWebRTCVideoSinkDestroy));
this.InvalidateHandle();
MLWebRTC.Instance.sinks.Remove(this);
return MLResult.Create(resultCode);
}
}
}
}