How do I recover from failure when uploading multiple files via my server to S3?

Question

New to NodeJS and S3, I wrote the following exploratory code to upload files to S3 via my NodeJS server without saving the file to disk or memory:

var express = require('express');
var Busboy = require('busboy');
var S3 = require('../utils/s3Util');
var router = express.Router(); // mounted at /uploads

router.post("/", function (req, res, next) {
    let bb = new Busboy({ headers: req.headers });
    const uploads = [];
    bb.on('file', (fieldname, stream, filename, encoding, mimeType) => {
        console.log(`Uploaded fieldname: ${fieldname}; filename: ${filename}, mimeType: ${mimeType}`);
        uploads.push(S3.svc.upload({ Bucket: 'my-test-bucket', Key: filename, Body: stream }).promise());
    });
    bb.on('finish', () => {
        console.log("# of promises:", uploads.length);
        Promise.all(uploads).then(retVals => {
            for (let i = 0; retVals && i < retVals.length; i++) {
                console.log(`File ${i + 1}::`, retVals[i]); 
            }
            res.end();
        }).catch(err => {
            console.log("Error::", err);
            res.status(500).send(`${err.name}: ${err.message}`);
        });
    });
    req.pipe(bb);
});

module.exports = router;

In the general failure case, how do I handle the scenario where the upload of 1 or more of x files being uploaded fails? Some uploads would have succeeded, some would have failed. However, in the catch clause I wouldn't know which ones have failed...

It would be good to be able to make this upload process somewhat transactional (i.e., either all uploads succeed, or none do). When errors happen, ideally I would be able to "rollback" the subset of successful uploads.

Answer 1

You could do it like this:

Push an object into uploads, with the data you need to retry, so:

uploads.push({ 
  fieldname, 
  filename, 
  mimeType,
  uploaded: S3.svc.upload({ Bucket: 'my-test-bucket', Key: filename, Body: stream })
    .promise()
    .then(() => true)
    .catch(() => false)
});

...

const failed = await 
  (Promise.all(uploads.map(async upload => ({...upload, uploaded: await upload.uploaded})))).then(u => u.filter(upload => !upload.uploaded))

const failedFiles = failed.join(', ')

console.log(`The following files failed to upload: ${failedFiles}`);

You need to make your event handlers async to use await inside them, so, for example:

bb.on('file', async (fieldname, stream, filename, encoding, mimeType) => {

Answer 2

I finally went with the following code, which is an expansion of @JoshWulf's answer:

function handleUpload(req, res, bucket, key) {
    let bb = new Busboy({ headers: req.headers });
    const uploads = [];
    bb.on('file', (fieldname, stream, filename, encoding, mimeType) => {
        console.log(`Uploaded fieldname: ${fieldname}; filename: ${filename}, mimeType: ${mimeType}`);
        const params = { Bucket: bucket, Key: key, Body: stream, ContentType: mimeType };
        uploads.push({ filename, result: S3.svc.upload(params).promise().then(data => data).catch(err => err) });
    });
    bb.on('finish', async () => {
        const results = await Promise.all(uploads.map(async (upload) => ({ ...upload, result: await upload.result })));
        // handle success/failure with their respective objects
    });
    req.pipe(bb);
}

The difference here from @Josh Wulf's answer is that in my upload promise I am returning the returned data object (if successful) and the returned error object (in case of failure) as-is. This then enables me to later use them as I need.