/*
* VideoKit
* Copyright © 2025 Yusuf Olokoba. All Rights Reserved.
*/
#nullable enable
namespace VideoKit {
using AOT;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Text;
using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.Networking;
using Function.Types;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using NJsonSchema;
using NJsonSchema.Generation;
using Internal;
using Status = Internal.VideoKit.Status;
///
/// Media asset.
///
public sealed class MediaAsset {
#region --Enumerations--
///
/// Media type.
///
public enum MediaType : int { // CHECK // `VideoKit.h`
///
/// Unknown or unsupported media type.
///
[EnumMember(Value = @"unknown")]
Unknown = 0,
///
/// Image.
///
[EnumMember(Value = @"image")]
Image = 1,
///
/// Audio.
///
[EnumMember(Value = @"audio")]
Audio = 2,
///
/// Video.
///
[EnumMember(Value = @"video")]
Video = 3,
///
/// Text.
///
[EnumMember(Value = @"text")]
Text = 4,
///
/// Sequence.
///
[EnumMember(Value = @"sequence")]
Sequence = 5,
}
///
/// Audio narration voice.
///
[JsonConverter(typeof(StringEnumConverter))]
public enum NarrationVoice : int {
///
/// Default narration voice.
///
[EnumMember(Value = @"default")]
Default = 0,
///
/// Male 1 narration voice.
///
[EnumMember(Value = @"kevin")]
Kevin = 1,
///
/// Male 2 narration voice.
///
[EnumMember(Value = @"arjun")]
Arjun = 2,
///
/// Male 3 narration voice.
///
[EnumMember(Value = @"dami")]
Dami = 3,
///
/// Male 4 narration voice.
///
[EnumMember(Value = @"juan")]
Juan = 4,
///
/// Female 1 narration voice.
///
[EnumMember(Value = @"rhea")]
Rhea = 5,
///
/// Female 2 narration voice.
///
[EnumMember(Value = @"aliyah")]
Aliyah = 6,
///
/// Female 3 narration voice.
///
[EnumMember(Value = @"kristen")]
Kristen = 7,
///
/// Female 4 narration voice.
///
[EnumMember(Value = @"salma")]
Salma = 8,
}
#endregion
#region --Properties--
///
/// Path to media asset.
/// This is `null` for sequence assets.
///
public string? path {
get {
var sb = new StringBuilder(2048);
var status = asset.GetMediaAssetPath(sb, sb.Capacity);
return status == Status.Ok ? sb.ToString() : null;
}
}
///
/// Asset media type.
///
public MediaType type => asset.GetMediaAssetMediaType(out var type).Throw() == Status.Ok ? type : default;
///
/// Image or video width.
///
public int width => asset.GetMediaAssetWidth(out var width) == Status.Ok ? width : default;
///
/// Image or video height.
///
public int height => asset.GetMediaAssetHeight(out var height) == Status.Ok ? height : default;
///
/// Video frame rate.
///
public float frameRate => asset.GetMediaAssetFrameRate(out var frameRate) == Status.Ok ? frameRate : default;
///
/// Audio sample rate.
///
public int sampleRate => asset.GetMediaAssetSampleRate(out var sampleRate) == Status.Ok ? sampleRate : default;
///
/// Audio channel count.
///
public int channelCount => asset.GetMediaAssetChannelCount(out var channelCount) == Status.Ok ? channelCount : default;
///
/// Video or audio duration in seconds.
///
public float duration => asset.GetMediaAssetDuration(out var duration) == Status.Ok ? duration : default;
///
/// Media assets contained within this asset.
/// This is only populated for `Sequence` assets.
///
public IReadOnlyList assets => new NativeMediaSequence(this);
#endregion
#region --Creators--
///
/// Create a media aseet from a file.
///
/// Path to media file.
/// Media asset.
public static Task FromFile (string path) {
var tcs = new TaskCompletionSource();
var handle = GCHandle.Alloc(tcs, GCHandleType.Normal);
try {
VideoKit.CreateMediaAsset(path, OnCreateAsset, (IntPtr)handle).Throw();
} catch (Exception ex) {
handle.Free();
tcs.SetException(ex);
}
return tcs.Task;
}
///
/// Create a media aseet from a texture.
///
/// Texture.
/// Image asset.
public static Task FromTexture (Texture2D texture) {
// Check
if (texture == null)
return Task.FromException(new ArgumentNullException(nameof(texture)));
// Check
if (!texture.isReadable)
return Task.FromException(new ArgumentException(@"Cannot create media asset from texture that is not readable"));
// Write to file
var encoded = texture.EncodeToPNG();
var name = Guid.NewGuid().ToString("N");
var path = Path.Combine(Application.temporaryCachePath, $"{name}.png");
File.WriteAllBytes(path, encoded);
// Create asset
return FromFile(path);
}
///
/// Create a media aseet from an audio clip.
/// NOTE: This requires an active VideoKit plan.
///
/// Audio clip.
/// Media format used to encode the audio.
/// Audio asset.
public static async Task FromAudioClip (
AudioClip clip,
MediaRecorder.Format format = MediaRecorder.Format.WAV
) {
// Create audio buffer
var sampleBuffer = new float[clip.samples * clip.channels];
clip.GetData(sampleBuffer, 0);
using var audioBuffer = new AudioBuffer(clip.frequency, clip.channels, sampleBuffer);
// Create asset
var recorder = await MediaRecorder.Create(
format,
sampleRate: audioBuffer.sampleRate,
channelCount: audioBuffer.channelCount
);
recorder.Append(audioBuffer);
var asset = await recorder.FinishWriting();
// Return
return asset;
}
///
/// Create a media aseet from plain text.
///
/// Text.
/// Text asset.
public static Task FromText (string text) {
// Write to file
var name = Guid.NewGuid().ToString("N");
var path = Path.Combine(Application.temporaryCachePath, $"{name}.txt");
File.WriteAllText(path, text);
// Create asset
return FromFile(path);
}
///
/// Create a media asset by prompting the user to select an image or video from the camera roll.
/// NOTE: This requires iOS 14+.
///
/// Desired asset type.
/// Media asset.
public static Task FromCameraRoll (MediaType type) {
var tcs = new TaskCompletionSource();
var handle = GCHandle.Alloc(tcs, GCHandleType.Normal);
try {
VideoKit.CreateMediaAssetFromCameraRoll(type, OnCreateAsset, (IntPtr)handle).Throw();
} catch (Exception ex) {
handle.Free();
tcs.SetException(ex);
}
return tcs.Task;
}
///
/// Create a media asset from a file in `StreamingAssets`.
///
/// Relative file path in `StreamingAssets`.
/// Media asset.
public static async Task FromStreamingAssets (string path) {
// Get absolute path
var absolutePath = await StreamingAssetsToAbsolutePath(path);
if (absolutePath == null)
throw new InvalidOperationException($"Failed to create media asset because file '{path}' could not be found in `StreamingAssets`");
// Create asset
return await FromFile(absolutePath);
}
///
/// Create a media asset by performing text-to-speech on the provided text prompt.
///
/// Text to synthesize speech from.
/// Voice to use for generation. See https://videokit.ai/reference/mediaasset for more information.
/// Generated audio asset.
internal static async Task FromGeneratedSpeech ( // INCOMPLETE
string prompt,
NarrationVoice voice = 0
) {
return default;
}
///
/// Create a media asset by performing text-to-image on the provided text prompt.
///
/// Text prompt to use to generate the image.
/// Desired image width. NOTE: The generated image is not guaranteed to have this width.
/// Desired image height. NOTE: The generated image is not guaranteed to have this height.
/// Generated image asset.
internal static async Task FromGeneratedImage ( // INCOMPLETE
string prompt,
int desiredWidth = 1024,
int desiredHeight = 1024
) {
return default;
}
#endregion
#region --Converters--
///
/// Create a texture from the media asset.
/// This can only be used on image and video assets.
///
/// Time to extract the texture from. This is only supported for video assets.
/// Texture.
public async Task ToTexture (float time = 0f) {
// Check video
if (type == MediaType.Video)
throw new NotImplementedException(@"`MediaAsset.ToTexture` is not yet supported for video assets");
// Check type
if (type != MediaType.Image)
throw new ArgumentException(@"`MediaAsset.ToTexture` can only be used on image assets");
// Load data
var uri = path![0] == '/' ? $"file://{path}" : path;
using var request = UnityWebRequestTexture.GetTexture(uri);
request.SendWebRequest();
while (!request.isDone)
await Task.Yield();
// Check
if (request.result != UnityWebRequest.Result.Success)
throw new InvalidOperationException($"Image asset could not be loaded with error: {request.error}");
// Return
return DownloadHandlerTexture.GetContent(request);
}
///
/// Create an audio clip from the media asset.
///
/// Audio clip.
public async Task ToAudioClip () { // CHECK // WAV only for now
// Check type
if (type != MediaType.Audio)
throw new ArgumentException($"Cannot create audio clip from asset because asset has invalid type: {type}");
// Load data
var uri = path![0] == '/' ? $"file://{path}" : path;
using var request = UnityWebRequestMultimedia.GetAudioClip(uri, AudioType.WAV);
request.SendWebRequest();
while (!request.isDone)
await Task.Yield();
// Check
if (request.result != UnityWebRequest.Result.Success)
throw new InvalidOperationException($"Audio clip could not be loaded with error: {request.error}");
// Return
return DownloadHandlerAudioClip.GetContent(request);
}
#endregion
#region --Reading--
///
/// Read sample buffers in the media asset.
///
/// Sample buffers in the media asset.
public IEnumerable Read () where T : struct {
var type = GetMediaType();
foreach (var sampleBuffer in Read(type)) {
if (type == MediaType.Video)
yield return (T)(object)new PixelBuffer(sampleBuffer); // sexy code is worth boxing overhead :p
else if (type == MediaType.Audio)
yield return (T)(object)new AudioBuffer(sampleBuffer);
else
break;
}
}
#endregion
#region --Sharing--
///
/// Share the media asset using the native sharing UI.
///
/// Optional message to share with the media asset.
/// Receiving app bundle ID or `null` if the user did not complete the share action.
public Task Share (string? message = null) {
// Check
if (type == MediaType.Sequence)
throw new InvalidOperationException(@"Sequence assets cannot be shared");
// Share
var tcs = new TaskCompletionSource();
var handle = GCHandle.Alloc(tcs, GCHandleType.Normal);
try {
asset.ShareMediaAsset(
message,
OnShare,
(IntPtr)handle
).Throw();
} catch (NotImplementedException) {
tcs.SetResult(null);
} catch (Exception ex) {
tcs.SetException(ex);
} finally {
handle.Free();
}
// Return
return tcs.Task;
}
///
/// Save the media asset to the camera roll.
///
/// Optional album to save media asset to.
/// Whether the asset was successfully saved to the camera roll.
public Task SaveToCameraRoll (string? album = null) {
// Check
if (type == MediaType.Sequence)
throw new InvalidOperationException(@"Sequence assets cannot be saved to the camera roll");
// Save
var tcs = new TaskCompletionSource();
var handle = GCHandle.Alloc(tcs, GCHandleType.Normal);
try {
asset.SaveMediaAssetToCameraRoll(
album,
OnSaveToCameraRoll,
(IntPtr)handle
).Throw();
} catch (NotImplementedException) {
tcs.SetResult(false);
} catch (Exception ex) {
tcs.SetException(ex);
} finally {
handle.Free();
}
// Return
return tcs.Task;
}
#endregion
#region --AI--
///
/// Parse the text asset into a structure.
///
/// Structure to parse into.
/// Parsed structure.
internal async Task Parse () {
// Check
if (type != MediaType.Text)
throw new ArgumentException($"Cannot perform structured parsing on media asset because asset is not a text asset");
// Generate schema
var settings = new JsonSchemaGeneratorSettings {
GenerateAbstractSchemas = false,
GenerateExamples = false,
UseXmlDocumentation = false,
ResolveExternalXmlDocumentation = false,
FlattenInheritanceHierarchy = false,
};
var schema = JsonSchema.FromType(settings);
// Parse
return default;
}
///
/// Transcribe the audio asset by performing speech-to-text.
///
internal async Task Transcribe () {
// Check type
if (type != MediaType.Audio)
throw new InvalidOperationException($"Cannot caption media asset because asset is not an audio asset");
// Transcribe
return default;
}
#endregion
#region --Operations--
private readonly IntPtr asset;
internal MediaAsset (IntPtr asset) => this.asset = asset;
~MediaAsset () => asset.ReleaseMediaAsset();
private IEnumerable Read (MediaType type) {
asset.CreateMediaReader(type, out var reader).Throw();
try {
for (;;) {
var status = reader.ReadNextSampleBuffer(out var sampleBuffer);
if (status == Status.InvalidOperation)
break;
if (sampleBuffer == default)
continue;
yield return sampleBuffer;
sampleBuffer.ReleaseSampleBuffer();
}
} finally {
reader.ReleaseMediaReader();
}
}
public static implicit operator IntPtr (MediaAsset asset) => asset.asset;
#endregion
#region --Callbacks--
[MonoPInvokeCallback(typeof(VideoKit.MediaAssetHandler))]
private static void OnCreateAsset (IntPtr context, IntPtr asset) {
try {
if (!VideoKit.IsAppDomainLoaded)
return;
var handle = (GCHandle)context;
var tcs = handle.Target as TaskCompletionSource;
handle.Free();
tcs?.SetResult(asset != IntPtr.Zero ? new MediaAsset(asset) : null);
} catch (Exception ex) {
Debug.LogException(ex);
}
}
[MonoPInvokeCallback(typeof(VideoKit.MediaAssetShareHandler))]
private static void OnShare (IntPtr context, IntPtr receiver) {
try {
// Check
if (!VideoKit.IsAppDomainLoaded)
return;
// Get tcs
var handle = (GCHandle)context;
var tcs = handle.Target as TaskCompletionSource;
handle.Free();
// Complete task
var result = Marshal.PtrToStringUTF8(receiver);
tcs?.SetResult(result);
} catch (Exception ex) {
Debug.LogException(ex);
}
}
[MonoPInvokeCallback(typeof(VideoKit.MediaAssetShareHandler))]
private static void OnSaveToCameraRoll (IntPtr context, IntPtr receiver) {
try {
// Check
if (!VideoKit.IsAppDomainLoaded)
return;
// Get tcs
var handle = (GCHandle)context;
var tcs = handle.Target as TaskCompletionSource;
handle.Free();
// Complete task
var result = receiver != IntPtr.Zero;
tcs?.SetResult(result);
} catch (Exception ex) {
Debug.LogException(ex);
}
}
#endregion
#region --Utilities--
private static async Task StreamingAssetsToAbsolutePath (string relativePath) {
var fullPath = Path.Combine(Application.streamingAssetsPath, relativePath);
if (Application.platform != RuntimePlatform.Android)
return File.Exists(fullPath) ? fullPath : null;
var persistentPath = Path.Combine(Application.persistentDataPath, relativePath);
if (File.Exists(persistentPath))
return persistentPath;
var directory = Path.GetDirectoryName(persistentPath);
Directory.CreateDirectory(directory);
using var request = UnityWebRequest.Get(fullPath);
request.SendWebRequest();
while (!request.isDone)
await Task.Yield();
if (request.result != UnityWebRequest.Result.Success)
return null;
File.WriteAllBytes(persistentPath, request.downloadHandler.data);
return persistentPath;
}
private static string GetValueExtension (Dtype type) => type switch {
Dtype.Image => ".png",
Dtype.Audio => ".wav",
Dtype.Video => ".mp4",
_ => string.Empty,
};
private static MediaType GetMediaType () => typeof(T) switch {
var x when x == typeof(AudioBuffer) => MediaType.Audio,
var x when x == typeof(PixelBuffer) => MediaType.Video,
_ => MediaType.Unknown,
};
private readonly struct NativeMediaSequence : IReadOnlyList {
private readonly IntPtr asset;
public NativeMediaSequence (IntPtr asset) => this.asset = asset;
public int Count => asset.GetMediaAssetSubAssetCount(out var count) == Status.Ok ? count : default;
public MediaAsset? this [int index] => asset.GetMediaAssetSubAsset(index, out var subAsset).Throw() == Status.Ok ? new MediaAsset(subAsset) : default;
IEnumerator IEnumerable.GetEnumerator () {
for (var idx = 0; idx < Count; ++idx)
yield return this[idx];
}
IEnumerator IEnumerable.GetEnumerator () => (this as IEnumerable).GetEnumerator();
}
#endregion
}
}