/*
* VideoKit
* Copyright © 2026 Yusuf Olokoba. All Rights Reserved.
*/
#nullable enable
namespace VideoKit.UI {
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.EventSystems;
using UnityEngine.Serialization;
using UnityEngine.UI;
using Unity.Collections.LowLevel.Unsafe;
using Muna;
using Internal;
using Facing = VideoKitCameraManager.Facing;
using Image = Muna.Image;
///
/// VideoKit UI component for displaying the camera preview from a camera manager.
///
[Tooltip(@"VideoKit UI component for displaying the camera preview from a camera manager.")]
[RequireComponent(typeof(RawImage), typeof(AspectRatioFitter), typeof(EventTrigger))]
[HelpURL(@"https://videokit.ai/reference/videokitcameraview")]
[DisallowMultipleComponent]
public sealed partial class VideoKitCameraView : MonoBehaviour, IPointerUpHandler, IBeginDragHandler, IDragHandler {
#region --Enumerations--
///
/// View mode.
///
public enum ViewMode : int {
///
/// Display the camera texture.
///
CameraTexture = 0,
///
/// Display the human texture.
///
HumanTexture = 1,
}
///
/// Gesture mode.
///
public enum GestureMode : int {
///
/// Do not respond to gestures.
///
None = 0,
///
/// Detect tap gestures.
///
Tap = 1,
///
/// Detect two-finger pinch gestures.
///
Pinch = 2,
///
/// Detect single-finger drag gestures.
/// This gesture mode is recommended when the user is holding a button to record a video.
///
Drag = 3,
}
#endregion
#region --Inspector--
[Header(@"Configuration")]
///
/// VideoKit camera manager.
///
[Tooltip(@"VideoKit camera manager.")]
public VideoKitCameraManager? cameraManager;
///
/// Desired camera facing to display.
///
[Tooltip(@"Desired camera facing to display.")]
public Facing facing = Facing.User | Facing.World;
///
/// View mode of the view.
///
[Tooltip(@"View mode of the view.")]
public ViewMode viewMode = ViewMode.CameraTexture;
[Header(@"Gestures")]
///
/// Focus gesture.
///
[Tooltip(@"Focus gesture."), FormerlySerializedAs(@"focusMode")]
public GestureMode focusGesture = GestureMode.None;
///
/// Exposure gesture.
///
[Tooltip(@"Exposure gesture."), FormerlySerializedAs(@"exposureMode")]
public GestureMode exposureGesture = GestureMode.None;
///
/// Zoom gesture.
///
[Tooltip(@"Zoom gesture."), FormerlySerializedAs(@"zoomMode")]
public GestureMode zoomGesture = GestureMode.None;
[Header(@"Events")]
///
/// Event raised when a new camera frame is available in the preview texture.
///
[Tooltip(@"Event raised when a new camera frame is available.")]
public UnityEvent? OnCameraFrame;
#endregion
#region --Client API--
///
/// Get the camera device that this view displays.
///
internal CameraDevice? device => VideoKitCameraManager // CHECK // Should we make this public??
.EnumerateCameraDevices(cameraManager?.device)
.FirstOrDefault(device => facing.HasFlag(VideoKitCameraManager.GetCameraFacing(device)));
///
/// Get the camera preview texture.
///
public Texture2D? texture { get; private set; }
///
/// Get or set the camera preview rotation.
/// This is automatically reset when the component is enabled.
///
public PixelBuffer.Rotation rotation { get; set; }
///
/// Event raised when a new pixel buffer is available.
/// Unlike the `VideoKitCameraManager`, this pixel buffer has an `RGBA8888` format and is rotated upright.
/// This event is invoked on a dedicated camera thread, NOT the Unity main thread.
///
public event Action? OnPixelBuffer;
#endregion
#region --Operations--
private PixelBuffer pixelBuffer;
private RawImage rawImage;
private AspectRatioFitter aspectFitter;
private readonly object fence = new();
private static readonly List OrientationSupport = new() {
RuntimePlatform.Android,
RuntimePlatform.IPhonePlayer
};
private void Reset() {
cameraManager = FindFirstObjectByType();
}
private void Awake() {
rawImage = GetComponent();
aspectFitter = GetComponent();
}
private void OnEnable() {
rotation = GetPreviewRotation(Screen.orientation);
if (cameraManager != null)
cameraManager.OnPixelBuffer += OnCameraBuffer;
}
private unsafe void Update() {
bool upload = false;
lock (fence) {
if (pixelBuffer == IntPtr.Zero)
return;
if (
texture != null &&
(texture.width != pixelBuffer.width || texture.height != pixelBuffer.height)
) {
Texture2D.Destroy(texture);
texture = null;
}
if (texture == null)
texture = new Texture2D(
pixelBuffer.width,
pixelBuffer.height,
TextureFormat.RGBA32,
false
);
if (viewMode == ViewMode.CameraTexture) {
using var buffer = new PixelBuffer(texture);
pixelBuffer.CopyTo(buffer);
upload = true;
} else if (viewMode == ViewMode.HumanTexture) {
var muna = VideoKitClient.Instance!.muna;
var prediction = muna.Predictions.Create(
tag: VideoKitCameraManager.HumanTextureTag,
inputs: new () {
["image"] = new Image(
(byte*)pixelBuffer.data.GetUnsafePtr(),
pixelBuffer.width,
pixelBuffer.height,
4
)
}
).Throw().Result;
var image = (Image)prediction.results![0]!;
image.CopyTo(texture);
upload = true;
}
}
if (upload)
texture.Apply();
rawImage.texture = texture;
aspectFitter.aspectRatio = (float)texture.width / texture.height;
OnCameraFrame?.Invoke();
}
private unsafe void OnCameraBuffer(
CameraDevice cameraDevice,
PixelBuffer cameraBuffer
) {
if ((VideoKitCameraManager.GetCameraFacing(cameraDevice) & facing) == 0)
return;
var (width, height) = GetPreviewTextureSize(
cameraBuffer.width,
cameraBuffer.height,
rotation
);
lock (fence) {
if (
pixelBuffer != IntPtr.Zero &&
(pixelBuffer.width != width || pixelBuffer.height != height)
) {
pixelBuffer.Dispose();
pixelBuffer = default;
}
if (pixelBuffer == IntPtr.Zero)
pixelBuffer = new PixelBuffer(
width,
height,
PixelBuffer.Format.RGBA8888,
mirrored: true
);
cameraBuffer.CopyTo(pixelBuffer, rotation: rotation);
}
OnPixelBuffer?.Invoke(pixelBuffer);
}
private void OnDisable() {
if (cameraManager != null)
cameraManager.OnPixelBuffer -= OnCameraBuffer;
}
private void OnDestroy() {
pixelBuffer.Dispose();
pixelBuffer = default;
}
#endregion
#region --Handlers--
void IPointerUpHandler.OnPointerUp(PointerEventData data) {
// Check device
var device = this.device;
if (device == null)
return;
// Check focus mode
if (focusGesture != GestureMode.Tap && exposureGesture != GestureMode.Tap)
return;
// Get press position
var rectTransform = transform as RectTransform;
if (!RectTransformUtility.ScreenPointToLocalPointInRectangle(
rectTransform,
data.position,
data.pressEventCamera, // or `enterEventCamera`
out var localPoint
))
return;
// Focus
var point = Rect.PointToNormalized(rectTransform!.rect, localPoint);
if (device.focusPointSupported && focusGesture == GestureMode.Tap)
device.SetFocusPoint(point.x, point.y);
if (device.exposurePointSupported && exposureGesture == GestureMode.Tap)
device.SetExposurePoint(point.x, point.y);
}
void IBeginDragHandler.OnBeginDrag(PointerEventData data) {
}
void IDragHandler.OnDrag(PointerEventData data) {
}
#endregion
#region --Utilities--
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static PixelBuffer.Rotation GetPreviewRotation(ScreenOrientation orientation) => orientation switch {
var _ when !OrientationSupport.Contains(Application.platform) => PixelBuffer.Rotation._0,
ScreenOrientation.LandscapeLeft => PixelBuffer.Rotation._0,
ScreenOrientation.Portrait => PixelBuffer.Rotation._90,
ScreenOrientation.LandscapeRight => PixelBuffer.Rotation._180,
ScreenOrientation.PortraitUpsideDown => PixelBuffer.Rotation._270,
_ => PixelBuffer.Rotation._0
};
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static (int width, int height) GetPreviewTextureSize(
int width,
int height,
PixelBuffer.Rotation rotation
) => rotation == PixelBuffer.Rotation._90 || rotation == PixelBuffer.Rotation._270 ?
(height, width) :
(width, height);
#endregion
}
}