Not able to stream object array from transform stream nodejs

Question

I am trying to read an array of objects, transform them and write to a file in Nodejs, it is giving me error "The "chunk" argument must be of type string or an instance of Buffer or Uint8Array. Received an instance of Object"

My demo Code:

const readStream = stream.Readable.from(res); // I create stream from array of objects
This is my custom transform stream:
var Transform = stream.Transform;

function Flatten(options) {
  Transform.call(this, options);
};
util.inherits(Flatten, Transform);
Flatten.prototype._transform = function (chunk, enc, cb) {
  
    let obj = chunk;
    let obj2;
    for (let j=0; j<obj.inner.length; j++) {
        let obj1 = obj.inner[j];
        obj2 = {...obj, ...obj1};
        
        delete obj2.inner;
        this.push(new Buffer(obj2));
    }
    
  cb();
};
Then I have a writable stream:
let writeStream = fs.createWriteStream('test1.txt');

Then I pipe all these await pipeline(readStream, tranf, writeStream);
that time I am getting the above error, tried using objectMode: true.

Could you please help

score 0 · Answer 1 · answered Jun 15 '22 at 06:20

first of all you need to add an option to work with object objectMode: true. then instead of passing Buffer to this.push() change to string with JSON.stringify

const stream = require('stream');
const fs = require('fs');
const res = [{ inner: [1, 2, 3] }, { inner: [4, 5, 6] }, { inner: [7, 8, 9] }];
const Transform = stream.Transform;

class Flatten extends Transform {
    constructor(options) {
        super(options)
    }

    _transform(chunk, enc, cb) {
        let obj = chunk;
        let obj2;
        for (let j = 0; j < obj.inner.length; j++) {
            let obj1 = obj.inner[j];
            obj2 = { ...obj, ...obj1 };
            delete obj2.inner;
            this.push(JSON.stringify(obj2));
        }
        cb();
    };

};


const readStream = stream.Readable.from(res);
const writeStream = fs.createWriteStream('test1.txt');
const transform = new Flatten({ objectMode: true })

readStream.pipe(transform).pipe(writeStream);

Not able to stream object array from transform stream nodejs

1 Answers1