using System; using System.Threading; using Unity.Profiling; using UnityEngine; using UnityEngine.Assertions; #if TFLITE_UNITASK_ENABLED using Cysharp.Threading.Tasks; #endif // TFLITE_UNITASK_ENABLED using TensorInfo = TensorFlowLite.Interpreter.TensorInfo; namespace TensorFlowLite { /// /// Base class for vision task that takes a Texture as an input /// public abstract class BaseVisionTask : IDisposable { protected Interpreter interpreter; protected int inputTensorIndex = 0; protected int width; protected int height; protected int channels; protected TextureToNativeTensor textureToTensor; private bool isDisposed = false; public AspectMode AspectMode { get; set; } = AspectMode.None; // Profilers protected static readonly ProfilerMarker preprocessPerfMarker = new($"{typeof(BaseVisionTask).Name}.Preprocess"); protected static readonly ProfilerMarker runPerfMarker = new($"{typeof(BaseVisionTask).Name}.Session.Run"); protected static readonly ProfilerMarker postprocessPerfMarker = new($"{typeof(BaseVisionTask).Name}.Postprocess"); /// /// Load model from byte array /// /// /// public virtual void Load(byte[] model, InterpreterOptions options) { try { interpreter = new Interpreter(model, options); } catch (Exception e) { interpreter?.Dispose(); throw e; } interpreter.LogIOInfo(); var inputTensorInfo = interpreter.GetInputTensorInfo(inputTensorIndex); InitializeInputsOutputs(inputTensorInfo); textureToTensor = CreateTextureToTensor(inputTensorInfo); } public virtual void Dispose() { if (!isDisposed) { interpreter?.Dispose(); textureToTensor?.Dispose(); isDisposed = true; } } /// /// Run the model with the input texture /// /// A texture for model input public virtual void Run(Texture texture) { if (isDisposed) { throw new ObjectDisposedException(nameof(BaseVisionTask)); } // Pre process preprocessPerfMarker.Begin(); PreProcess(texture); preprocessPerfMarker.End(); // Run inference runPerfMarker.Begin(); interpreter.Invoke(); runPerfMarker.End(); // Post process postprocessPerfMarker.Begin(); PostProcess(); postprocessPerfMarker.End(); } /// /// Pre process the input texture /// Set all input tensors for the model /// /// An input texture protected virtual void PreProcess(Texture texture) { var input = textureToTensor.Transform(texture, AspectMode); interpreter.SetInputTensorData(inputTensorIndex, input); } /// /// Get the output tensors and do post process in subclass /// protected abstract void PostProcess(); // Only available when UniTask is installed #if TFLITE_UNITASK_ENABLED public virtual async UniTask RunAsync(Texture texture, CancellationToken cancellationToken) { await RunAsync(texture, cancellationToken, true); } public virtual async UniTask RunAsync( Texture texture, CancellationToken cancellationToken, bool waitForMainThread) { if (isDisposed) { throw new ObjectDisposedException(nameof(BaseVisionTask)); } // Pre process // Note: Profiler doesn't work with async await PreProcessAsync(texture, cancellationToken); // Run inference in BG thread await UniTask.SwitchToThreadPool(); runPerfMarker.Begin(); interpreter.Invoke(); runPerfMarker.End(); // Post process await PostProcessAsync(cancellationToken); if (waitForMainThread) { // Back to main thread await UniTask.SwitchToMainThread(); } } protected virtual async UniTask PreProcessAsync(Texture texture, CancellationToken cancellationToken) { var input = await textureToTensor.TransformAsync(texture, AspectMode, cancellationToken); interpreter.SetInputTensorData(inputTensorIndex, input); } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously protected virtual async UniTask PostProcessAsync(CancellationToken cancellationToken) { PostProcess(); } #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously #endif // TFLITE_UNITASK_ENABLED /// /// Default implementation of InitializeInputsOutputs /// Override this in subclass if needed /// protected virtual void InitializeInputsOutputs(TensorInfo inputTensorInfo) { int[] inputShape = inputTensorInfo.shape; Assert.AreEqual(4, inputShape.Length); Assert.AreEqual(1, inputShape[0], $"The batch size of the model must be 1. But got {inputShape[0]}"); height = inputShape[1]; width = inputShape[2]; channels = inputShape[3]; int inputCount = interpreter.GetInputTensorCount(); for (int i = 0; i < inputCount; i++) { int[] shape = interpreter.GetInputTensorInfo(i).shape; interpreter.ResizeInputTensor(i, shape); } interpreter.AllocateTensors(); } /// /// Create TextureToTensor for this model. /// Override this in subclass if needed /// /// A TextureToNativeTensor instance protected virtual TextureToNativeTensor CreateTextureToTensor(TensorInfo inputTensorInfo) { return TextureToNativeTensor.Create(new() { compute = null, kernel = 0, width = width, height = height, channels = channels, inputType = inputTensorInfo.type, }); } } }