using System;
using System.Collections.Generic;
using System.Threading;
using Unity.Collections;
using Unity.Jobs;
using UnityEngine;
using UnityEngine.Assertions;
using UnityEngine.Rendering;
#if TFLITE_UNITASK_ENABLED
using Cysharp.Threading.Tasks;
#endif // TFLITE_UNITASK_ENABLED
using DataType = TensorFlowLite.Interpreter.DataType;
namespace TensorFlowLite
{
///
/// Converts Texture to Tensor with arbitrary matrix transformation
/// then return it as a NativeArray (NHWC layout)
///
public abstract class TextureToNativeTensor : IDisposable
{
[Serializable]
public class Options
{
public ComputeShader compute = null;
public int kernel = 0;
public int width = 0;
public int height = 0;
public int channels = 0;
public DataType inputType = DataType.Float32;
}
protected static readonly Lazy DefaultComputeShaderFloat32 = new(()
=> Resources.Load("com.github.asus4.tflite.common/TextureToNativeTensorFloat32"));
public static ComputeShader CloneDefaultComputeShaderFloat32()
{
return UnityEngine.Object.Instantiate(DefaultComputeShaderFloat32.Value);
}
private static readonly int _InputTex = Shader.PropertyToID("_InputTex");
private static readonly int _OutputTensor = Shader.PropertyToID("_OutputTensor");
private static readonly int _OutputSize = Shader.PropertyToID("_OutputSize");
private static readonly int _TransformMatrix = Shader.PropertyToID("_TransformMatrix");
private static readonly Matrix4x4 PopMatrix = Matrix4x4.Translate(new Vector3(0.5f, 0.5f, 0));
private static readonly Matrix4x4 PushMatrix = Matrix4x4.Translate(new Vector3(-0.5f, -0.5f, 0));
public readonly ComputeShader compute;
public readonly bool hasCustomCompute;
public readonly int kernel;
public readonly int width;
public readonly int height;
public readonly int channels;
private bool disposed = false;
private readonly GraphicsBuffer tensorBuffer;
protected NativeArray tensor;
protected readonly List runningJobs = new List();
protected TextureToNativeTensor(int stride, Options options)
{
bool isSupported = SystemInfo.supportsAsyncGPUReadback && SystemInfo.supportsComputeShaders;
if (!isSupported)
{
// Note: Async GPU Readback is supported on most platforms
// including OpenGL ES 3.0 since Unity 2021 LTS
throw new NotSupportedException("AsyncGPUReadback and ComputeShader are required to use TextureToNativeTensor");
}
hasCustomCompute = options.compute != null;
compute = hasCustomCompute
? options.compute
: CloneDefaultComputeShaderFloat32();
kernel = options.kernel;
width = options.width;
height = options.height;
channels = options.channels;
Assert.IsTrue(kernel >= 0, $"Kernel must be set");
Assert.IsTrue(width > 0, $"Width must be greater than 0");
Assert.IsTrue(height > 0, $"Height must be greater than 0");
Assert.IsTrue(channels > 0 && channels <= 4, $"Channels must be 1 to 4");
int length = width * height * channels;
tensorBuffer = new GraphicsBuffer(GraphicsBuffer.Target.Structured, length, stride);
tensor = new NativeArray(length * stride, Allocator.Persistent);
// Set constant values
compute.SetInts(_OutputSize, width, height);
compute.SetBuffer(kernel, _OutputTensor, tensorBuffer);
}
~TextureToNativeTensor()
{
Dispose(false);
}
public virtual void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
private void Dispose(bool disposing)
{
if (disposed)
{
return;
}
if (disposing)
{
// Complete running jobs if exists
foreach (var job in runningJobs)
{
job.Complete();
}
runningJobs.Clear();
// Dispose resources
if (!hasCustomCompute)
{
UnityEngine.Object.Destroy(compute);
}
tensor.Dispose();
tensorBuffer.Dispose();
}
disposed = true;
}
public virtual NativeArray Transform(Texture input, in Matrix4x4 t)
{
compute.SetTexture(kernel, _InputTex, input, 0);
compute.SetMatrix(_TransformMatrix, t);
compute.Dispatch(kernel, Mathf.CeilToInt(width / 8f), Mathf.CeilToInt(height / 8f), 1);
var request = AsyncGPUReadback.RequestIntoNativeArray(ref tensor, tensorBuffer);
request.WaitForCompletion();
return tensor;
}
public NativeArray Transform(Texture input, AspectMode aspectMode)
{
return Transform(input, GetAspectScaledMatrix(input, aspectMode));
}
public Matrix4x4 GetAspectScaledMatrix(Texture input, AspectMode aspectMode)
{
float srcAspect = (float)input.width / input.height;
float dstAspect = (float)width / height;
Vector2 scale = GetAspectScale(srcAspect, dstAspect, aspectMode);
return PopMatrix * Matrix4x4.Scale(new Vector3(scale.x, scale.y, 1)) * PushMatrix;
}
public static Vector2 GetAspectScale(float srcAspect, float dstAspect, AspectMode mode)
{
bool isSrcWider = srcAspect > dstAspect;
return (mode, isSrcWider) switch
{
(AspectMode.None, _) => new Vector2(1, 1),
(AspectMode.Fit, true) => new Vector2(1, srcAspect / dstAspect),
(AspectMode.Fit, false) => new Vector2(dstAspect / srcAspect, 1),
(AspectMode.Fill, true) => new Vector2(dstAspect / srcAspect, 1),
(AspectMode.Fill, false) => new Vector2(1, srcAspect / dstAspect),
_ => throw new Exception("Unknown aspect mode"),
};
}
// Available when UniTask is installed
#if TFLITE_UNITASK_ENABLED
public virtual async UniTask> TransformAsync(Texture input, Matrix4x4 t, CancellationToken cancellationToken)
{
compute.SetTexture(kernel, _InputTex, input, 0);
compute.SetMatrix(_TransformMatrix, t);
compute.Dispatch(kernel, Mathf.CeilToInt(width / 8f), Mathf.CeilToInt(height / 8f), 1);
// Didn't work due to error: AsyncGPUReadback - NativeArray does not have read/write access
// var request = AsyncGPUReadback.RequestIntoNativeArray(ref tensor, tensorBuffer);
// https://forum.unity.com/threads/asyncgpureadback-requestintonativearray-causes-invalidoperationexception-on-nativearray.1011955/
// await request.ToUniTask(cancellationToken: cancellationToken);
var request = AsyncGPUReadback.Request(tensorBuffer, (request) =>
{
if (request.hasError)
{
throw new Exception("GPU readback error detected");
}
if (cancellationToken.IsCancellationRequested)
{
Debug.LogWarning("TransformAsync task was cancelled");
return;
}
var tmpBuffer = request.GetData();
Assert.AreEqual(tmpBuffer.Length, tensor.Length);
tensor.CopyFrom(tmpBuffer);
});
await request.ToUniTask(cancellationToken: cancellationToken);
return tensor;
}
public UniTask> TransformAsync(Texture input, AspectMode aspectMode, CancellationToken cancellationToken)
{
return TransformAsync(input, GetAspectScaledMatrix(input, aspectMode), cancellationToken);
}
#endif // TFLITE_UNITASK_ENABLED
///
/// Find the appropriate TextureToNativeTensor class for the given input type
///
/// An options
/// An instance
public static TextureToNativeTensor Create(Options options)
{
return options.inputType switch
{
DataType.Float32 => new TextureToNativeTensorFloat32(options),
DataType.UInt8 => new TextureToNativeTensorUInt8(options),
DataType.Int32 => new TextureToNativeTensorInt32(options),
_ => throw new NotImplementedException(
$"input type {options.inputType} is not implemented yet. Create our own TextureToNativeTensor class and override it."),
};
}
}
}