using System.Threading; using UnityEngine; #if TFLITE_UNITASK_ENABLED using Cysharp.Threading.Tasks; #endif // TFLITE_UNITASK_ENABLED namespace TensorFlowLite { /// /// Converts Texture to Tensor (NHWC layout) /// [System.Obsolete("TextureToTensor is obsolete, use TextureToNativeTensor instead")] public class TextureToTensor : System.IDisposable { private Texture2D fetchTexture; private ComputeBuffer tensorBuffer; private readonly ComputeShader compute; public Texture2D texture => fetchTexture; static readonly int InputTexture = Shader.PropertyToID("InputTexture"); static readonly int OutputFloatTensor = Shader.PropertyToID("OutputFloatTensor"); static readonly int TextureWidth = Shader.PropertyToID("TextureWidth"); static readonly int TextureHeight = Shader.PropertyToID("TextureHeight"); public TextureToTensor() { compute = Resources.Load("com.github.asus4.tflite.common/TextureToTensor"); } public void Dispose() { DisposeUtil.TryDispose(fetchTexture); DisposeUtil.TryDispose(tensorBuffer); } public void ToTensor(RenderTexture texture, byte[,,] inputs) { var pixels = FetchToTexture2D(texture).GetRawTextureData(); int width = texture.width; int height = texture.height - 1; for (int i = 0; i < pixels.Length; i++) { int y = height - i / width; int x = i % width; inputs[y, x, 0] = pixels[i].r; inputs[y, x, 1] = pixels[i].b; inputs[y, x, 2] = pixels[i].g; } } public void ToTensor(RenderTexture texture, int[,,] inputs) { var pixels = FetchToTexture2D(texture).GetRawTextureData(); int width = texture.width; int height = texture.height - 1; for (int i = 0; i < pixels.Length; i++) { int y = height - i / width; int x = i % width; inputs[y, x, 0] = pixels[i].r; inputs[y, x, 1] = pixels[i].g; inputs[y, x, 2] = pixels[i].b; } } public void ToTensor(RenderTexture texture, float[,,] inputs) { if (texture.width % 8 != 0 || texture.height % 8 != 0) { ToTensorCPU(texture, inputs); } else { ToTensorGPU(texture, inputs); } } public void ToTensor(RenderTexture texture, out ComputeBuffer buffer) { Debug.Assert(texture.width % 8 == 0); Debug.Assert(texture.height % 8 == 0); ToTensorGPU(texture); buffer = tensorBuffer; } public void ToTensor(RenderTexture texture, float[,,] inputs, float offset, float scale) { // TODO: optimize this var pixels = FetchToTexture2D(texture).GetRawTextureData(); int width = texture.width; int height = texture.height - 1; for (int i = 0; i < pixels.Length; i++) { int y = height - i / width; int x = i % width; inputs[y, x, 0] = (pixels[i].r - offset) * scale; inputs[y, x, 1] = (pixels[i].g - offset) * scale; inputs[y, x, 2] = (pixels[i].b - offset) * scale; } } #if TFLITE_UNITASK_ENABLED public async UniTask ToTensorAsync(RenderTexture texture, float[,,] inputs, CancellationToken cancellationToken) { await UniTask.SwitchToMainThread(PlayerLoopTiming.FixedUpdate, cancellationToken); var pixels = FetchToTexture2D(texture).GetRawTextureData(); int width = texture.width; int height = texture.height - 1; await UniTask.SwitchToThreadPool(); const float scale = 255f; for (int i = 0; i < pixels.Length; i++) { int y = height - i / width; int x = i % width; inputs[y, x, 0] = pixels[i].r / scale; inputs[y, x, 1] = pixels[i].g / scale; inputs[y, x, 2] = pixels[i].b / scale; } return true; } public async UniTask ToTensorAsync(RenderTexture texture, byte[,,] inputs, CancellationToken cancellationToken) { await UniTask.SwitchToMainThread(PlayerLoopTiming.FixedUpdate, cancellationToken); var pixels = FetchToTexture2D(texture).GetRawTextureData(); int width = texture.width; int height = texture.height - 1; await UniTask.SwitchToThreadPool(); for (int i = 0; i < pixels.Length; i++) { int y = height - i / width; int x = i % width; inputs[y, x, 0] = pixels[i].r; inputs[y, x, 1] = pixels[i].g; inputs[y, x, 2] = pixels[i].b; } return true; } public async UniTask ToTensorAsync(RenderTexture texture, int[,,] inputs, CancellationToken cancellationToken) { await UniTask.SwitchToMainThread(PlayerLoopTiming.FixedUpdate, cancellationToken); var pixels = FetchToTexture2D(texture).GetRawTextureData(); int width = texture.width; int height = texture.height - 1; await UniTask.SwitchToThreadPool(); for (int i = 0; i < pixels.Length; i++) { int y = height - i / width; int x = i % width; inputs[y, x, 0] = pixels[i].r; inputs[y, x, 1] = pixels[i].g; inputs[y, x, 2] = pixels[i].b; } return true; } #endif // TFLITE_UNITASK_ENABLED private void ToTensorCPU(RenderTexture texture, float[,,] inputs) { var pixels = FetchToTexture2D(texture).GetRawTextureData(); int width = texture.width; int height = texture.height - 1; const float scale = 255f; for (int i = 0; i < pixels.Length; i++) { int y = height - i / width; int x = i % width; inputs[y, x, 0] = (float)(pixels[i].r) / scale; inputs[y, x, 1] = (float)(pixels[i].g) / scale; inputs[y, x, 2] = (float)(pixels[i].b) / scale; } } private void ToTensorGPU(RenderTexture texture) { int width = texture.width; int height = texture.height; if (tensorBuffer == null || tensorBuffer.count != width * height) { DisposeUtil.TryDispose(tensorBuffer); tensorBuffer = new ComputeBuffer(width * height, sizeof(float) * 3); } int kernel = compute.FindKernel("TextureToFloatTensor"); compute.SetTexture(kernel, InputTexture, texture); compute.SetBuffer(kernel, OutputFloatTensor, tensorBuffer); compute.SetInt(TextureWidth, width); compute.SetInt(TextureHeight, height); compute.Dispatch(kernel, width / 8, height / 8, 1); } private void ToTensorGPU(RenderTexture texture, float[,,] inputs) { ToTensorGPU(texture); tensorBuffer.GetData(inputs); } private Texture2D FetchToTexture2D(RenderTexture texture) { if (fetchTexture == null || !IsSameSize(fetchTexture, texture)) { // Due to GetRawTextureData() returns incorrect bytes length, we cant use liner color space options // https://fogbugz.unity3d.com/default.asp?1271542_g3prspfd30mhavcb // fetchTexture = new Texture2D(texture.width, texture.height, TextureFormat.RGB24, 0, false); fetchTexture = new Texture2D(texture.width, texture.height, TextureFormat.RGBA32, false); } var prevRT = RenderTexture.active; RenderTexture.active = texture; fetchTexture.ReadPixels(new Rect(0, 0, texture.width, texture.height), 0, 0); fetchTexture.Apply(); RenderTexture.active = prevRT; return fetchTexture; } private static bool IsSameSize(Texture a, Texture b) { return a.width == b.width && a.height == b.height; } } }