Pipe multipart form uploads to another server

Question

I am trying to handle a POST request on my Node Express server to deal with multipart form uploads, in my case the user is uploading images.

I want to pipe the upload to another server via my Express app which is currently setup to use body parser, which I also see does not support multipart bodes and instead recommends using some other libraries.

I have seen multiparty but I am unsure how to use this with my client side application.

In my client side code I am posting a FormData object like so:

function create(data, name) {
  var formData = new FormData();
  formData.append('file', data, name);
  return this.parentBase.one('photos').withHttpConfig({transformRequest: angular.identity}).customPOST(formData, undefined, undefined, {'Content-Type': undefined});
}

Note: I am using the Restangular library for AngularJS as documented here

So from what I understand looking at the multiparty docs, I have to handle the form upload events and act upon it further once the form has finished uploading.

The thing is, I was hoping I could just pipe the upload directly to another server. Beforehand my client side app was making direct calls to this other server, but I am now trying to get everything routed through Express, is this possible, or do I have to use something like multiparty?

The request documentation gives an example of using formData, but I am unsure how this would work with the multiparty examples I have seen. For example once the upload completes in Express using mutliparty, do I then have to construct another formData object to then make a further request with, or would I have to pipe each part to the other server?

I'm confused, please can someone help clear this up for me?

Thanks

EDIT

OK, I have taken a look at multer following @yarons comments and this seems to be the kind of thing I want to be using, I have attempted to use this with my express router setup as per below:

routes.js

var express = require('express'),
  router = express.Router(),
  customers = require('./customers.controller.js'),
  multer = require('multer'),
  upload = multer();

router.post('/customers/:customerId/photos/', upload.single('file'), customers.createPhoto);

controller.js

module.exports.createPhoto = function(req, res) {
  console.log(req.file);
  var options = prepareCustomersAPIHeaders(req);
  options.formData = req.file;
  request(options).pipe(res);
};

Logging out the req.file property in the above controller I see this:

{ fieldname: 'file',
  originalname: '4da2e703044932e33b8ceec711c35582.jpg',
  encoding: '7bit',
  mimetype: 'image/png',
  buffer: <Buffer 89 50 4e 47 0d 0a 1a 0a 00 00 00 0d 49 48 44 52 00 00 00 fa 00
 00 00 fa 08 06 00 00 00 88 ec 5a 3d 00 00 20 00 49 44 41 54 78 5e ac bd f9 8f e
6 e9 7a ... >,
  size: 105868 }

Which is what I am posting through from the client side code using:

var formData = new FormData();
      formData.append('file', data, name);
      return this.parentBase.one('photos').withHttpConfig({transformRequest: angular.identity}).customPOST(formData, undefined, undefined, {'Content-Type': undefined});

Is what I have tried sensible? Only it doesn't work, I get an error back from the server I'm trying post to. Beforehand where I was making this post request directly to the server it all worked fine, so I must have something wrong in my Express\Multer setup

EDIT 2

Ok, so after more hunting around I came across this article using multiparty which I have manager to get working in my setup like so:

var request = require('request'),
  multiparty = require('multiparty'),
  FormData = require('form-data');

module.exports.createPhoto = function(req, res) {
  //console.log(req.file);
  var options = prepareCustomersAPIHeaders(req),
    form = new multiparty.Form();
  options.headers['Transfer-Encoding'] = 'chunked';

  form.on('part', function(part){
    if(part.filename) {
      var form = new FormData(), r;
      form.append(part.name, part, {filename: part.filename, contentType: part['content-type']});


      r = request(options, function(err, response, body){
        res.status(response.statusCode).send(body);
      });
      r._form = form
    }
  });

  form.on('error', function(error){
    console.log(error);
  });

  form.parse(req);
};  

This is now uploading the files for me as expected to my other server, whilst this solution works, I dont like the line:

r._form = form

Seems to be assigning a private form variable to the request object, plus I can't see anything that is documented in this way on multiparty pages

Can anyone offer any comments on this possible solution?

Answer 1

We use something like the following:

CLIENT

//HTML
    <input type="file" ng-file-select uploader="info.uploadPath" />


//DIRECTIVES
  // It is attached to <input type="file" /> element
  .directive('ngFileSelect', function() {
    return {
      link: function($scope, $element) {
        $element.bind('change', function() {
          $scope.$emit('file:add', this.files ? this.files : this);
        });
      }
    };
  })

//OTHER
    var uploadPath = '/api/things/' + $stateParams.thingId + '/add_photo'

    var uploadInfo = {
              headers: {
                'Authorization': authToken
              },
              form: {
                title: scope.info.name
              }
            }


//SERVICE:
  $rootScope.$on('file:add', function(event, items) {
    this.addToQueue(items);
  }.bind(this));
  ...
  addToQueue: function(items) {
    var length = this.queue.length;
    angular.forEach(items.length ? items : [items], function(item) {
      var isValid = !this.filters.length ? true : !!this.filters.filter(function(filter) {
        return filter.apply(this, [item]);
      }, this).length;

      if (isValid) {
        item = new Item({
          url: this.url,
          alias: this.alias,
          removeAfterUpload: this.removeAfterUpload,
          uploader: this,
          file: item
        });

        this.queue.push(item);
      }
    }, this);

    this.uploadAll();
  },
  getNotUploadedItems: function() {
    return this.queue.filter(function(item) {
      return !item.isUploaded;
    });
  },

  /**
   * Upload a item from the queue
   * @param {Item|Number} value
   */
  uploadItem: function(value, uploadInfo) {
    if (this.isUploading) {
      return;
    }

    var index = angular.isObject(value) ? this.getIndexOfItem(value) : value;
    var item = this.queue[index];
    var transport = item.file._form ? '_iframeTransport' : '_xhrTransport';
    this.isUploading = true;
    this[transport](item, uploadInfo);
  },

  uploadAll: function(uploadInfo) {
    var item = this.getNotUploadedItems()[0];
    this._uploadNext = !!item;
    this._uploadNext && this.uploadItem(item, uploadInfo);
  },

  _xhrTransport: function(item, uploadInfo) {
    var xhr = new XMLHttpRequest();
    var form = new FormData();
    var that = this;

    form.append(item.alias, item.file);

    angular.forEach(uploadInfo.form, function(value, name) {
      form.append(name, value);
    });

    xhr.upload.addEventListener('progress', function(event) {
      var progress = event.lengthComputable ? event.loaded * 100 / event.total : 0;
      that._scope.$emit('in:progress', item, Math.round(progress));
    }, false);

    xhr.addEventListener('load', function() {
      xhr.status === 200 && that._scope.$emit('in:success', xhr, item);
      xhr.status !== 200 && that._scope.$emit('in:error', xhr, item);
      that._scope.$emit('in:complete', xhr, item);
    }, false);

    xhr.addEventListener('error', function() {
      that._scope.$emit('in:error', xhr, item);
      that._scope.$emit('in:complete', xhr, item);
    }, false);

    xhr.addEventListener('abort', function() {
      that._scope.$emit('in:complete', xhr, item);
    }, false);

    this._scope.$emit('beforeupload', item);

    xhr.open('POST', item.url, true);

    angular.forEach(uploadInfo.headers, function(value, name) {
      xhr.setRequestHeader(name, value);
    });

    xhr.send(form);
  },

SERVER

//things.router
app.route('/api/things/:thingId/add_photo')
  .post(things.uploadPhoto);

//things.controller
exports.uploadPhoto = function(req, res) {
  var formidable = require('formidable');

  var form = new formidable.IncomingForm();

  form.parse(req, function(err, fields, files) {
    var data = files.qqfile;
    //actual file is at data.path
    fs.createReadStream(data.path).pipe(request.put(uploadUrl));
  }
}