Download functionality using streams for large files in NodeJS

Question

I am trying to implement a download functionality using streams in NodeJS.

In the code I am trying to simulate an endpoint that sends data in chunks, something similar to paginated data, for example chunks in size of 5000. Or to make it further clear, we can send top and skip parameters to the endpoint to get a particular chunk of data. If no parameters are provided, it send the first 5000 entries.

There are 2 cases that I am trying to take care of:

When the user cancels the download from the browser, how do I handle the continuous fetching of data from the endpoint
When the user pauses the download from the browser, how do I pause the data fetching, and then resume once user resumes it

The first case can be taken care of using 'close' event of request. When the connection between the client and the server get cancelled, I disconnect.

If anyone has a better way of implementing this please suggest.

I am having trouble handling the second case when the user pauses.

If anyone could guide me through this, or even provide a better solution to the overall problem(incl. handling the chunks of data), it would be really helpful.

const {createServer} = require('http');
const {Transform} = require('stream');
const axios = require('axios');

var c = 0;
class ApiStream extends Transform {
    constructor(apiCallback, res, req) {
        super();
        this.apiCallback = apiCallback;
        this.isPipeSetup = false;
        this.res = res;
        this.req = req
    }
    //Will get data continuously 
    async start() {
        let response;
        try {
            response = await this.apiCallback();
        } catch (e) {
            response = null;
        }
        if (!this.isPipeSetup) {
            this.pipe(this.res);
            this.isPipeSetup = true;
        }
        if (response) {
            response = response.data
            if (Array.isArray(response)) {
                response.forEach((item) => {
                    this.push(JSON.stringify(item) + "\n");
                });
            } else if (typeof response === "object") {
                this.push(JSON.stringify(response) + "\n");
            } else if (typeof response === "string") {
                this.push(response + "\n");
            }
            this.start()
        }else{
            this.push(null);
            console.log('Stream ended')
        }
    }
}

const server = createServer(async (req, res, stream) => {
    res.setHeader("Content-disposition", "attachment; filename=download.json");
    res.setHeader("Content-type", "text/plain");
    let disconnected = false;
    const filestream = new ApiStream(async () => {
        let response;
        try {
            if(disconnected){
                console.log('Client connection closed')
                return null;
            }

            c++;
            response = await axios.get("https://jsonplaceholder.typicode.com/users");

            //Simulate delay in data fetching
            let z = 0;
            if(c>=200) response = null;
            while(z<10000){
                let b = 0;
                while(b<10000){
                    b+=0.5;
                }
                z +=0.5;
            }
        } catch (error) {
            res.status(500).send(error);
        }
        if (response) {
            return response;
        }
        return null;
    }, res, req);
    await filestream.start();

    req.on('close', (err) => {
        disconnected = true;
    })
})
server.listen(5050, () => console.log('server running on port 5050'));

Download functionality using streams for large files in NodeJS

0 Answers0