Node FTP doesn't execute `once('close')` callback when downloading file is large

Question

I have a method to download file from FTP server and it works fine on smaller files, but when I use it to download file of ~5GB size of zip type, it downloads it, but after that it doesn't do anything. When it reaches 100% of downloading, then script doesn't continue. Should I wait if it's actually doing something in the background after download is complete? Is there filesize limit?

const FTP = require('ftp')

which can be found on npm

downloadFile: params => {
    return new Promise((resolve, reject) => {
      let ftpClient = new FTP()
      let total = params.state.fileSize
      let progress = 0
      ftpClient.on('ready', _ => {
        console.log(`Downloading ${params.targetedFile} ...`);
        ftpClient.get(params.targetedFile, (err, stream) => {
          if (err) reject(err)
          stream.on('data', buffer => {
            progress += buffer.length
            process.stdout.write(`Progress: ${(progress/total*100).toFixed(2)}% (${progress}/${total})  \r`)
          })
          stream.once('close', _ => {
            ftpClient.end()
            console.log(`Saved downloaded file to ${params.localDir}`);
            resolve(params.localDir)
          })
          stream.pipe(fs.createWriteStream(params.localDir))
        })
      })
      ftpClient.connect(params.auth)
    })
  }

Basically, the callback for stream.once('close', ...) doesn't get executed when large file is downloaded. And it gets executed for smaller file of same type.

Answer 1

I recommend You to handle event of closing write stream.

Reason is simple: we read from ftp's read stream and pipe to write stream, everything is ok, when file is successfully closed.

So code:

downloadFile: params => {
    return new Promise((resolve, reject) => {
      let ftpClient = new FTP()
      let total = params.state.fileSize
      let progress = 0
      ftpClient.on('ready', _ => {
        console.log(`Downloading ${params.targetedFile} ...`);
        ftpClient.get(params.targetedFile, (err, stream) => {
          if (err) {
            ftpClient.end();
            return reject(err);
          }

          stream.on('data', buffer => {
            progress += buffer.length
            process.stdout.write(`Progress: ${(progress/total*100).toFixed(2)}% (${progress}/${total})  \r`)
          });

          // opening writeStream to file
          let finished = false;
          const writeStream = fs.createWriteStream(params.localDir);

          writeStream.on('finish', (result) => { // handling finish
            finished = true;
            ftpClient.end();
            console.log(`Finish triggered ${params.localDir}`);
            console.log(result);
            resolve(params.localDir);
          });

          writeStream.on('close', (result) => { // handling close
            ftpClient.end();
            console.log(`Close triggered ${params.localDir}`);
            console.log(result);
            resolve(params.localDir);
          })

          // piping readStream to writeStream
          stream.pipe(writeStream);
        })
      })
      ftpClient.connect(params.auth)
    })
  }

Answer 2

This code might give you idea how to handle this in a bit hacky way.

Basically this method allows you to download file from FTP server and save it to local filesystem. It outputs current progress complete_percentage% (current/total) in a single line. Upon finish, it resolves promise, returns path to local file, the same one that you passed as param.

/**
   * @name downloadFile
   * @desc downloads file from FTP server
   * @param  params, Object of params
   *   @prop auth: object, or null, authorization params
   *   @prop targetedFile: {String} filename e.g. data.txt
   *   @prop localDir: {String} filename on local disk
   *   @prop state: {Object} fileinfo object, {Int} .fileSize property is required
   * @return Promise, resolves given localDir
   */
  downloadFile: params => {
    return new Promise((resolve, reject) => {
      // validate param types
      if(typeof params.auth !== 'object'
      || typeof params.targetedFile !== 'string'
      || typeof params.localDir !== 'string'
      || typeof params.state !== 'object'
      || typeof params.state.fileSize !== 'number'
      ) throw new Error('You are either missing properties or passed wrong types')

      // initialize
      let ftpClient = new FTP()
      let total = params.state.fileSize
      let progress = 0

      //
      ftpClient.on('ready', _ => {
        console.log(`Downloading ${params.targetedFile} ...`)
        // get file
        ftpClient.get(params.targetedFile, (err, stream) => {
          if (err){
            ftpClient.end()
            return reject(err)
          }

          // upon data receive
          stream.on('data', buffer => {
            progress += buffer.length
            // if progress is complete
            if(progress === total){
              // start checking if local filesize matches server filesize
              let interval = setInterval(_ => {
                if(fs.statSync(params.localDir).size === total){
                  console.log(`Downloading file complete. Location: ${params.localDir}`);
                  clearInterval(interval)
                  ftpClient.end()
                  resolve(params.localDir)
                }
              })
            }
            // show current progress in percentages and bytes
            process.stdout.write(`Progress: ${(progress/total*100).toFixed(2)}% (${progress}/${total})  \r`)
          })
          // pipe writestream to filesystem to write these bytes
          stream.pipe(fs.createWriteStream(params.localDir))
        })
      })
      ftpClient.connect(params.auth)
    })//promise
  }