Android image process ImageProxy

Question

I am working on react-native-vision-camera.

I am trying to do 2 things:

• Edit frames (image processing)
• Replace a frame with a loaded jpeg image (that way I can control what goes into my frames processor afterwards and test my image processing frame processor plugin)

I tried to replace a frame that way:

package com.mrousavy.camera.example;

import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.ImageFormat;
import android.graphics.Rect;
import android.graphics.YuvImage;
import android.util.Log;
import androidx.camera.core.ImageProxy.PlaneProxy;
import androidx.camera.core.ImageProxy;
import com.facebook.react.bridge.WritableNativeArray;
import com.facebook.react.bridge.WritableNativeMap;
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.ByteBuffer;
import java.util.Arrays;
import org.jetbrains.annotations.NotNull;

public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
    private void encodeYUV420SP(Bitmap image, byte[] Y, byte[] U, byte[] V) {
        int inputWidth = image.getWidth();
        int inputHeight = image.getHeight();
        int [] argb = new int[inputWidth * inputHeight];
        image.getPixels(argb, 0, inputWidth, 0, 0, inputWidth, inputHeight);

        byte [] yuv420sp = new byte[inputWidth*inputHeight*3/2];
        final int frameSize = inputWidth * inputHeight;

        int yIndex = 0;

        int a, R, G, B;
        int index = 0;
        int uvIndex = 0;
        for (int j = 0; j < inputHeight; j++) {
            for (int i = 0; i < inputWidth; i++) {

                a = (argb[index] & 0xff000000) >> 24; // a is not used obviously
                R = (argb[index] & 0xff0000) >> 16;
                G = (argb[index] & 0xff00) >> 8;
                B = (argb[index] & 0xff) >> 0;

                int Ypixel = ( (  66 * R + 129 * G +  25 * B + 128) >> 8) +  16;
                int Upixel = ( ( -38 * R -  74 * G + 112 * B + 128) >> 8) + 128;
                int Vpixel = ( ( 112 * R -  94 * G -  18 * B + 128) >> 8) + 128;

                Y[index] = (byte) ((Ypixel < 0) ? 0 : ((Ypixel > 255) ? 255 : Ypixel));
                if (j % 2 == 0 && index % 2 == 0) { 
                    V[uvIndex] = (byte)((Vpixel<0) ? 0 : ((Vpixel > 255) ? 255 : Vpixel));
                    U[uvIndex] = (byte)((Upixel<0) ? 0 : ((Upixel > 255) ? 255 : Upixel));
                    uvIndex++;
                }

                index ++;
            }
        }
        System.out.println("UVINDEX: " + uvIndex);
    }

    @Override
    public Object callback(@NotNull ImageProxy image, @NotNull Object[] params) {
        try {
            // Get path of image to load
            String param = params[0].toString();

            // Get image
            Bitmap bmImg = BitmapFactory.decodeFile(param);

            // Try to transform bitmap to Y U V array (representing their eponym channel)
            byte[] Y = new byte[bmImg.getWidth() * bmImg.getHeight()];
            byte[] U = new byte[bmImg.getWidth() * bmImg.getHeight()];
            byte[] V = new byte[bmImg.getWidth() * bmImg.getHeight()];
            encodeYUV420SP(bmImg, Y, U, V);
            PlaneProxy[] planes = image.getPlanes();


            ByteBuffer bufferY = ByteBuffer.wrap(Y);
            planes[0].getBuffer().put(bufferY);

            ByteBuffer bufferU = ByteBuffer.wrap(U);
            planes[1].getBuffer().put(bufferU);

            ByteBuffer bufferV = ByteBuffer.wrap(V);
            planes[2].getBuffer().put(bufferV);

            return image;
            //return test;
        } catch (Exception e) {
            System.out.println("CRASHED");
            e.printStackTrace();
        }
        return null;
    }

    ExampleFrameProcessorPlugin() {
        super("example_plugin");
    }
} 

But I m not able to do what I explained (I have an undefined frame in typescript side). Even if "encodeYUV420SP" is not the good way to Bitmap -> YUV420888, I don't even know if it s possible to create an ImageProxy object and return it or edit the existing one ...

Can someone tell me if it s possible and how to do it or give me a sample code? (I searched a lot but couldn't find any) ?