/* * VideoKit * Copyright © 2025 Yusuf Olokoba. All Rights Reserved. */ #nullable enable namespace VideoKit { using AOT; using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Runtime.InteropServices; using System.Runtime.Serialization; using System.Text; using System.Threading.Tasks; using UnityEngine; using UnityEngine.Networking; using Function.Types; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using NJsonSchema; using NJsonSchema.Generation; using Internal; using Status = Internal.VideoKit.Status; /// /// Media asset. /// public sealed class MediaAsset { #region --Enumerations-- /// /// Media type. /// public enum MediaType : int { // CHECK // `VideoKit.h` /// /// Unknown or unsupported media type. /// [EnumMember(Value = @"unknown")] Unknown = 0, /// /// Image. /// [EnumMember(Value = @"image")] Image = 1, /// /// Audio. /// [EnumMember(Value = @"audio")] Audio = 2, /// /// Video. /// [EnumMember(Value = @"video")] Video = 3, /// /// Text. /// [EnumMember(Value = @"text")] Text = 4, /// /// Sequence. /// [EnumMember(Value = @"sequence")] Sequence = 5, } /// /// Audio narration voice. /// [JsonConverter(typeof(StringEnumConverter))] public enum NarrationVoice : int { /// /// Default narration voice. /// [EnumMember(Value = @"default")] Default = 0, /// /// Male 1 narration voice. /// [EnumMember(Value = @"kevin")] Kevin = 1, /// /// Male 2 narration voice. /// [EnumMember(Value = @"arjun")] Arjun = 2, /// /// Male 3 narration voice. /// [EnumMember(Value = @"dami")] Dami = 3, /// /// Male 4 narration voice. /// [EnumMember(Value = @"juan")] Juan = 4, /// /// Female 1 narration voice. /// [EnumMember(Value = @"rhea")] Rhea = 5, /// /// Female 2 narration voice. /// [EnumMember(Value = @"aliyah")] Aliyah = 6, /// /// Female 3 narration voice. /// [EnumMember(Value = @"kristen")] Kristen = 7, /// /// Female 4 narration voice. /// [EnumMember(Value = @"salma")] Salma = 8, } #endregion #region --Properties-- /// /// Path to media asset. /// This is `null` for sequence assets. /// public string? path { get { var sb = new StringBuilder(2048); var status = asset.GetMediaAssetPath(sb, sb.Capacity); return status == Status.Ok ? sb.ToString() : null; } } /// /// Asset media type. /// public MediaType type => asset.GetMediaAssetMediaType(out var type).Throw() == Status.Ok ? type : default; /// /// Image or video width. /// public int width => asset.GetMediaAssetWidth(out var width) == Status.Ok ? width : default; /// /// Image or video height. /// public int height => asset.GetMediaAssetHeight(out var height) == Status.Ok ? height : default; /// /// Video frame rate. /// public float frameRate => asset.GetMediaAssetFrameRate(out var frameRate) == Status.Ok ? frameRate : default; /// /// Audio sample rate. /// public int sampleRate => asset.GetMediaAssetSampleRate(out var sampleRate) == Status.Ok ? sampleRate : default; /// /// Audio channel count. /// public int channelCount => asset.GetMediaAssetChannelCount(out var channelCount) == Status.Ok ? channelCount : default; /// /// Video or audio duration in seconds. /// public float duration => asset.GetMediaAssetDuration(out var duration) == Status.Ok ? duration : default; /// /// Media assets contained within this asset. /// This is only populated for `Sequence` assets. /// public IReadOnlyList assets => new NativeMediaSequence(this); #endregion #region --Creators-- /// /// Create a media aseet from a file. /// /// Path to media file. /// Media asset. public static Task FromFile (string path) { var tcs = new TaskCompletionSource(); var handle = GCHandle.Alloc(tcs, GCHandleType.Normal); try { VideoKit.CreateMediaAsset(path, OnCreateAsset, (IntPtr)handle).Throw(); } catch (Exception ex) { handle.Free(); tcs.SetException(ex); } return tcs.Task; } /// /// Create a media aseet from a texture. /// /// Texture. /// Image asset. public static Task FromTexture (Texture2D texture) { // Check if (texture == null) return Task.FromException(new ArgumentNullException(nameof(texture))); // Check if (!texture.isReadable) return Task.FromException(new ArgumentException(@"Cannot create media asset from texture that is not readable")); // Write to file var encoded = texture.EncodeToPNG(); var name = Guid.NewGuid().ToString("N"); var path = Path.Combine(Application.temporaryCachePath, $"{name}.png"); File.WriteAllBytes(path, encoded); // Create asset return FromFile(path); } /// /// Create a media aseet from an audio clip. /// NOTE: This requires an active VideoKit plan. /// /// Audio clip. /// Media format used to encode the audio. /// Audio asset. public static async Task FromAudioClip ( AudioClip clip, MediaRecorder.Format format = MediaRecorder.Format.WAV ) { // Create audio buffer var sampleBuffer = new float[clip.samples * clip.channels]; clip.GetData(sampleBuffer, 0); using var audioBuffer = new AudioBuffer(clip.frequency, clip.channels, sampleBuffer); // Create asset var recorder = await MediaRecorder.Create( format, sampleRate: audioBuffer.sampleRate, channelCount: audioBuffer.channelCount ); recorder.Append(audioBuffer); var asset = await recorder.FinishWriting(); // Return return asset; } /// /// Create a media aseet from plain text. /// /// Text. /// Text asset. public static Task FromText (string text) { // Write to file var name = Guid.NewGuid().ToString("N"); var path = Path.Combine(Application.temporaryCachePath, $"{name}.txt"); File.WriteAllText(path, text); // Create asset return FromFile(path); } /// /// Create a media asset by prompting the user to select an image or video from the camera roll. /// NOTE: This requires iOS 14+. /// /// Desired asset type. /// Media asset. public static Task FromCameraRoll (MediaType type) { var tcs = new TaskCompletionSource(); var handle = GCHandle.Alloc(tcs, GCHandleType.Normal); try { VideoKit.CreateMediaAssetFromCameraRoll(type, OnCreateAsset, (IntPtr)handle).Throw(); } catch (Exception ex) { handle.Free(); tcs.SetException(ex); } return tcs.Task; } /// /// Create a media asset from a file in `StreamingAssets`. /// /// Relative file path in `StreamingAssets`. /// Media asset. public static async Task FromStreamingAssets (string path) { // Get absolute path var absolutePath = await StreamingAssetsToAbsolutePath(path); if (absolutePath == null) throw new InvalidOperationException($"Failed to create media asset because file '{path}' could not be found in `StreamingAssets`"); // Create asset return await FromFile(absolutePath); } /// /// Create a media asset by performing text-to-speech on the provided text prompt. /// /// Text to synthesize speech from. /// Voice to use for generation. See https://videokit.ai/reference/mediaasset for more information. /// Generated audio asset. internal static async Task FromGeneratedSpeech ( // INCOMPLETE string prompt, NarrationVoice voice = 0 ) { return default; } /// /// Create a media asset by performing text-to-image on the provided text prompt. /// /// Text prompt to use to generate the image. /// Desired image width. NOTE: The generated image is not guaranteed to have this width. /// Desired image height. NOTE: The generated image is not guaranteed to have this height. /// Generated image asset. internal static async Task FromGeneratedImage ( // INCOMPLETE string prompt, int desiredWidth = 1024, int desiredHeight = 1024 ) { return default; } #endregion #region --Converters-- /// /// Create a texture from the media asset. /// This can only be used on image and video assets. /// /// Time to extract the texture from. This is only supported for video assets. /// Texture. public async Task ToTexture (float time = 0f) { // Check video if (type == MediaType.Video) throw new NotImplementedException(@"`MediaAsset.ToTexture` is not yet supported for video assets"); // Check type if (type != MediaType.Image) throw new ArgumentException(@"`MediaAsset.ToTexture` can only be used on image assets"); // Load data var uri = path![0] == '/' ? $"file://{path}" : path; using var request = UnityWebRequestTexture.GetTexture(uri); request.SendWebRequest(); while (!request.isDone) await Task.Yield(); // Check if (request.result != UnityWebRequest.Result.Success) throw new InvalidOperationException($"Image asset could not be loaded with error: {request.error}"); // Return return DownloadHandlerTexture.GetContent(request); } /// /// Create an audio clip from the media asset. /// /// Audio clip. public async Task ToAudioClip () { // CHECK // WAV only for now // Check type if (type != MediaType.Audio) throw new ArgumentException($"Cannot create audio clip from asset because asset has invalid type: {type}"); // Load data var uri = path![0] == '/' ? $"file://{path}" : path; using var request = UnityWebRequestMultimedia.GetAudioClip(uri, AudioType.WAV); request.SendWebRequest(); while (!request.isDone) await Task.Yield(); // Check if (request.result != UnityWebRequest.Result.Success) throw new InvalidOperationException($"Audio clip could not be loaded with error: {request.error}"); // Return return DownloadHandlerAudioClip.GetContent(request); } #endregion #region --Reading-- /// /// Read sample buffers in the media asset. /// /// Sample buffers in the media asset. public IEnumerable Read () where T : struct { var type = GetMediaType(); foreach (var sampleBuffer in Read(type)) { if (type == MediaType.Video) yield return (T)(object)new PixelBuffer(sampleBuffer); // sexy code is worth boxing overhead :p else if (type == MediaType.Audio) yield return (T)(object)new AudioBuffer(sampleBuffer); else break; } } #endregion #region --Sharing-- /// /// Share the media asset using the native sharing UI. /// /// Optional message to share with the media asset. /// Receiving app bundle ID or `null` if the user did not complete the share action. public Task Share (string? message = null) { // Check if (type == MediaType.Sequence) throw new InvalidOperationException(@"Sequence assets cannot be shared"); // Share var tcs = new TaskCompletionSource(); var handle = GCHandle.Alloc(tcs, GCHandleType.Normal); try { asset.ShareMediaAsset( message, OnShare, (IntPtr)handle ).Throw(); } catch (NotImplementedException) { tcs.SetResult(null); } catch (Exception ex) { tcs.SetException(ex); } finally { handle.Free(); } // Return return tcs.Task; } /// /// Save the media asset to the camera roll. /// /// Optional album to save media asset to. /// Whether the asset was successfully saved to the camera roll. public Task SaveToCameraRoll (string? album = null) { // Check if (type == MediaType.Sequence) throw new InvalidOperationException(@"Sequence assets cannot be saved to the camera roll"); // Save var tcs = new TaskCompletionSource(); var handle = GCHandle.Alloc(tcs, GCHandleType.Normal); try { asset.SaveMediaAssetToCameraRoll( album, OnSaveToCameraRoll, (IntPtr)handle ).Throw(); } catch (NotImplementedException) { tcs.SetResult(false); } catch (Exception ex) { tcs.SetException(ex); } finally { handle.Free(); } // Return return tcs.Task; } #endregion #region --AI-- /// /// Parse the text asset into a structure. /// /// Structure to parse into. /// Parsed structure. internal async Task Parse () { // Check if (type != MediaType.Text) throw new ArgumentException($"Cannot perform structured parsing on media asset because asset is not a text asset"); // Generate schema var settings = new JsonSchemaGeneratorSettings { GenerateAbstractSchemas = false, GenerateExamples = false, UseXmlDocumentation = false, ResolveExternalXmlDocumentation = false, FlattenInheritanceHierarchy = false, }; var schema = JsonSchema.FromType(settings); // Parse return default; } /// /// Transcribe the audio asset by performing speech-to-text. /// internal async Task Transcribe () { // Check type if (type != MediaType.Audio) throw new InvalidOperationException($"Cannot caption media asset because asset is not an audio asset"); // Transcribe return default; } #endregion #region --Operations-- private readonly IntPtr asset; internal MediaAsset (IntPtr asset) => this.asset = asset; ~MediaAsset () => asset.ReleaseMediaAsset(); private IEnumerable Read (MediaType type) { asset.CreateMediaReader(type, out var reader).Throw(); try { for (;;) { var status = reader.ReadNextSampleBuffer(out var sampleBuffer); if (status == Status.InvalidOperation) break; if (sampleBuffer == default) continue; yield return sampleBuffer; sampleBuffer.ReleaseSampleBuffer(); } } finally { reader.ReleaseMediaReader(); } } public static implicit operator IntPtr (MediaAsset asset) => asset.asset; #endregion #region --Callbacks-- [MonoPInvokeCallback(typeof(VideoKit.MediaAssetHandler))] private static void OnCreateAsset (IntPtr context, IntPtr asset) { try { if (!VideoKit.IsAppDomainLoaded) return; var handle = (GCHandle)context; var tcs = handle.Target as TaskCompletionSource; handle.Free(); tcs?.SetResult(asset != IntPtr.Zero ? new MediaAsset(asset) : null); } catch (Exception ex) { Debug.LogException(ex); } } [MonoPInvokeCallback(typeof(VideoKit.MediaAssetShareHandler))] private static void OnShare (IntPtr context, IntPtr receiver) { try { // Check if (!VideoKit.IsAppDomainLoaded) return; // Get tcs var handle = (GCHandle)context; var tcs = handle.Target as TaskCompletionSource; handle.Free(); // Complete task var result = Marshal.PtrToStringUTF8(receiver); tcs?.SetResult(result); } catch (Exception ex) { Debug.LogException(ex); } } [MonoPInvokeCallback(typeof(VideoKit.MediaAssetShareHandler))] private static void OnSaveToCameraRoll (IntPtr context, IntPtr receiver) { try { // Check if (!VideoKit.IsAppDomainLoaded) return; // Get tcs var handle = (GCHandle)context; var tcs = handle.Target as TaskCompletionSource; handle.Free(); // Complete task var result = receiver != IntPtr.Zero; tcs?.SetResult(result); } catch (Exception ex) { Debug.LogException(ex); } } #endregion #region --Utilities-- private static async Task StreamingAssetsToAbsolutePath (string relativePath) { var fullPath = Path.Combine(Application.streamingAssetsPath, relativePath); if (Application.platform != RuntimePlatform.Android) return File.Exists(fullPath) ? fullPath : null; var persistentPath = Path.Combine(Application.persistentDataPath, relativePath); if (File.Exists(persistentPath)) return persistentPath; var directory = Path.GetDirectoryName(persistentPath); Directory.CreateDirectory(directory); using var request = UnityWebRequest.Get(fullPath); request.SendWebRequest(); while (!request.isDone) await Task.Yield(); if (request.result != UnityWebRequest.Result.Success) return null; File.WriteAllBytes(persistentPath, request.downloadHandler.data); return persistentPath; } private static string GetValueExtension (Dtype type) => type switch { Dtype.Image => ".png", Dtype.Audio => ".wav", Dtype.Video => ".mp4", _ => string.Empty, }; private static MediaType GetMediaType () => typeof(T) switch { var x when x == typeof(AudioBuffer) => MediaType.Audio, var x when x == typeof(PixelBuffer) => MediaType.Video, _ => MediaType.Unknown, }; private readonly struct NativeMediaSequence : IReadOnlyList { private readonly IntPtr asset; public NativeMediaSequence (IntPtr asset) => this.asset = asset; public int Count => asset.GetMediaAssetSubAssetCount(out var count) == Status.Ok ? count : default; public MediaAsset? this [int index] => asset.GetMediaAssetSubAsset(index, out var subAsset).Throw() == Status.Ok ? new MediaAsset(subAsset) : default; IEnumerator IEnumerable.GetEnumerator () { for (var idx = 0; idx < Count; ++idx) yield return this[idx]; } IEnumerator IEnumerable.GetEnumerator () => (this as IEnumerable).GetEnumerator(); } #endregion } }