Create Tensors from Image for ONNX.js

Question

I am trying to use ONNX.js to run a ONNX object detection model in browser. I know that in Tensorflow.js you have to pass only an Image Object to model and Tensorflow automatically create Tensor required by the model, but in ONNX we have to create Tensors from image and then pass it to model.

I read the offical Repository - https://github.com/Microsoft/onnxjs but i can't find any resource on how to create Tensors from image in Javascript.

If anyone knows anything please help ?

I am using following code to run Object Detection in Browser using ONNX.js

<html>

<head> </head>

<body>
    <div onclick="runOD()">
        Run Model
    </div>
    <img src="box.jpg" id="image">

    <!-- Load ONNX.js -->
    <script src="https://cdn.jsdelivr.net/npm/onnxjs/dist/onnx.min.js"></script>
    <!-- Code that consume ONNX.js -->
    <script>
        async function runOD() {
            // Creat the session and load the pre-trained model
            const session = new onnx.InferenceSession({
                backendHint: 'webgl'
            });

            await session.loadModel("./model.onnx");

            //Function to load image to Canvas and Convert it to Tensor
            const preprocessedData = loadImage();

            const inputTensor = new onnx.Tensor(preprocessedData, 'float32', [1, 3, 244, 244]);

            // Run model with Tensor inputs and get the result.
            const outputMap = await session.run([inputTensor]);
            const outputData = outputMap.values().next().value.data;
            console.log(`model output tensor: ${outputData.data}.`);
        }
    </script>
</body>

</html>

score 3 · Accepted Answer · answered Feb 19 '20 at 13:37

Following code worked for me

<script>
const imageSize = 416;
var count = 0;

async function runOD() {
    // Creat the session and load the pre-trained model
    const session = new onnx.InferenceSession({
        backendHint: 'webgl'
    });

    await session.loadModel("./model.onnx");

    // Run model with Tensor inputs and get the result.

        // Load image.
        const imageLoader = new ImageLoader(imageSize, imageSize);
        const imageData = await imageLoader.getImageData('./box.jpg');

        // Preprocess the image data to match input dimension requirement
        const width = imageSize;
        const height = imageSize;
        const preprocessedData = preprocess(imageData.data, width, height);

        const inputTensor = new onnx.Tensor(preprocessedData, 'float32', [1, 3, width, height]);
        const outputMap = await session.run([inputTensor]);
        const outputData = outputMap.values().next().value.data;
        var x = outputData.toString();
        var blob = new Blob([x], { type: "text/plain;charset=utf-8" });
        saveAs(blob, "data.txt");

        console.log(outputData);
        count = count+1;
        document.getElementById('count').innerHTML = count;
}
/**
 * Preprocess raw image data to match Resnet50 requirement.
 */
function preprocess(data, width, height) {
    const dataFromImage = ndarray(new Float32Array(data), [width, height, 4]);
    const dataProcessed = ndarray(new Float32Array(width * height * 3), [1, 3, height, width]);

    // Normalize 0-255 to (-1)-1
    ndarray.ops.divseq(dataFromImage, 128.0);
    ndarray.ops.subseq(dataFromImage, 1.0);

    // Realign imageData from [224*224*4] to the correct dimension [1*3*224*224].
    ndarray.ops.assign(dataProcessed.pick(0, 0, null, null), dataFromImage.pick(null, null, 2));
    ndarray.ops.assign(dataProcessed.pick(0, 1, null, null), dataFromImage.pick(null, null, 1));
    ndarray.ops.assign(dataProcessed.pick(0, 2, null, null), dataFromImage.pick(null, null, 0));

    return dataProcessed.data;
}

You need to install and import it first. When you use yarn you can simply enter `yarn add ndarray` while you are in the project folder and import it using `import ndarray from "ndarray";`. — subjord, Mar 26 '21 at 19:07
Since this answer, ops live in a separate package, called `ndarray-ops`. You need to install it, and import it as `const ops = require('ndarray-ops')`. Then replace ndarray.ops.whatever to ops.whatever. — f4z3k4s, Sep 23 '21 at 13:00
@AakashBhadana I can't find the definition/import of `preprocess(...)` — Bilal, May 17 '23 at 20:16

Create Tensors from Image for ONNX.js

1 Answers1