Read multiples file line by line in parallel (Nodejs)

Question

I have 2 files (or more) 1.txt and 2.txt.
1.txt

1-a
1-b
1-c

2.txt

2-a
2-b
2-c

Now I want 1.txt and 2.txt were read in parallel line by line, and write that line into result.txt after read. result.txt.

1.a + 2.a
1.b + 2.b
1.c + 2.c

How can I do this?
edit: this is a simple example. I want to work with large files and so I want to save my memory as much as possible, I don't want to use arrays.

What have you tried? Do you want to use asynchronous file I/O or is synchronous file I/O OK for what you're doing? — jfriend00, Feb 02 '18 at 04:18
asynchronous. I tried to read each file line by line and store in an array. and then write that array into the destination file. but it's not a good solution. So now I'm looking for another way. read line 1 of each file, write line 1 and then read line 2 of each file, write line 2 and etc — Vu Thuy, Feb 02 '18 at 05:29
Did you ever get a solution to this? I'm trying to do the same thing. — Ian H, Mar 27 '19 at 00:01

score 1 · Answer 1 · answered Feb 02 '18 at 07:18

I thought this would be an interesting challenge to use async I/O and as little memory as possible (buffering only what a readStream gives you on a data event) and outputting each line as you go.

So, here's a way to do it. This is not extensively tested on very large files, but does seem to work in the test cases I tried. If the two files have a different number of lines, it will stop outputting lines when the shorter file has been read. Here's the code:

const LineReader = require('line-by-line');
const fs = require('fs');

class DualLineReader extends LineReader {
    constructor(filename, outputStream) {
        super(filename);
        this.myBuffer = [];
        this.output = outputStream;
    }
    setOtherReader(lr) {
        this.otherLineReader = lr;
    }

    resumeIfEmpty() {
        if (!this.myBuffer.length) {
            this.resume();
        }
    }

    // if data in both buffers, write the next line
    // call the callback when OK to call writeLine again
    writeLine(cb) {
        // if both buffers contain at least one line, output the first line in each
        if (this.myBuffer.length && this.otherLineReader.myBuffer.length) {
            let ready = this.output.write(this.myBuffer.shift() + " + " + this.otherLineReader.myBuffer.shift() + '\n');
            if (!ready) {
                // need to wait for drain event before writing any more
                this.output.once('drain', () => {
                    cb(true);
                });
            } else {
                process.nextTick(() => {
                    cb(true);
                });
            }
        } else {
            // nothing else to write at the moment
            // call the callback on next tick
            process.nextTick(() => {
                cb(false);
            });
        }
    }

    closeOutput(cb) {
        if (!this.output.closed) {
            this.output.end(cb);
        }
    }

    // loop asynchronously until no more data in buffer to write
    // call callback when done
    writeAllLines(cb = function() {}) {
        this.writeLine(more => {
            if (more) {
                this.writeAllLines();
            } else {
                // if either buffer is empty, start it flowing again
                this.resumeIfEmpty();
                this.otherLineReader.resumeIfEmpty();
                cb();
            }
        });
    }

    run(cb) {
        this.on('line', line => {
            this.myBuffer.push(line);
            this.pause();
            this.writeAllLines();
        });

        this.on('end', () => {
            this.writeAllLines(() => {
                this.close();
                this.otherLineReader.close();
                this.closeOutput(cb);
            });
        });

        this.on('error', (err) => {
            console.log(err);
            this.close();
            this.otherLineReader.close();
            this.closeOutput(() => {
                cb(err);
            });
        });

    }
}

let output = fs.createWriteStream("results.txt");
output.on('close', () => {
    this.closed = true;
});

let lr1 = new DualLineReader("file1.txt", output);
let lr2 = new DualLineReader("file2.txt", output);

lr1.setOtherReader(lr2);
lr2.setOtherReader(lr1);

function done() {
    console.log("all done");
}
lr1.run(done);
lr2.run(done);

@vuthuy - Did you try this? Did it work for you? I wrote a whole new piece of code to solve this somewhat unique problem and have not heard anything back from you. — jfriend00, Feb 04 '18 at 01:52

score 0 · Answer 2 · answered Feb 02 '18 at 04:28

0

i think the easy way to do like you want is read all to file and plus them. you can use ReadFileSync function to read file. i give you an example for this. Hope can help you.

var fs = require("fs");

var file_1 = fs.readFileSync("files/1.txt");
var file_2 = fs.readFileSync("files/2.txt");

array_file_1 = file_1.toString().split("\r\n");
array_file_2 = file_2.toString().split("\r\n");

string_file = '';
array_file_1.forEach((element, index) => {
    string_file += element + array_file_2[index] + '\r\n';
});

fs.writeFileSync("files/result.txt", string_file);

console.log(string_file);

answered Feb 02 '18 at 04:28

xuan hung Nguyen

390
3
12

The OP specifically doesn't want to read both files all the way into memory. They want something that works for really large files and does not use a lot of memory. – jfriend00 Feb 02 '18 at 04:31
Thanks but I want something that works for large files – Vu Thuy Feb 02 '18 at 05:30

score 0 · Answer 3 · answered Aug 26 '20 at 15:57

Spent 2 hours+ on reading 10+ articles to find a proper solution. Below works fine.

Install a node module:

npm install --save n-readlines

Code:

const lineByLine = require("n-readlines");

var filePath1 = "./file1.txt", // <------ NOTE: Make sure that this file exists
    filePath2 = "./file2.txt"; // <------ NOTE: Make sure that this file exists
var file1 = new lineByLine(filePath1),
    file2 = new lineByLine(filePath2);
var line1, line2;

while (true) {
    line1 = file1.next(); // Read next line from 1st file
    if (!line1) break; // EOF

    line2 = file2.next(); // Read next line from 2nd file
    if (!line2) break; // EOF

    line1 = line1.toString("ascii").trim(); // Buffer to text, then trim spaces
    line2 = line2.toString("ascii").trim(); // Buffer to text, then trim spaces

    console.log(line1, line2);
}

Read multiples file line by line in parallel (Nodejs)

3 Answers3