How to use Joyent Manta client.createWriteStream(path, options) correctly

Question

I'm trying to use the Manta's Node.js SDK, from Joyent, to write a file from a readable stream.

The example given on the webpage is this:

var MemoryStream = require('memorystream');

var message = 'Hello World'
var opts = {
    copies: 3,
    headers: {
        'access-control-allow-origin': '*',
        'access-control-allow-methods': 'GET'
    },
    md5: crypto.createHash('md5').update(message).digest('base64'),
    size: Buffer.byteLength(message),
    type: 'text/plain'
};
var stream = new MemoryStream();
var w = client.createWriteStream('/jill/stor/hello_world.txt', opts);

stream.pipe(w);

w.once('close', function (res) {
    console.log('all done');
});

stream.end(message);

But this is the case were we use a writeStream to put a file/object/string already into the program, what if I want to pass a readable stream to be written, like this?

 function saveCopyToManta(cb){
    var stream = fs.createReadStream('/tmp/' + filename);

    var opts = {
        copies: 3,
        headers: {
            'access-control-allow-origin': '*',
            'access-control-allow-methods': 'GET'
        },
        md5: crypto.createHash('md5').update(message).digest('base64'),
        size: filesize,
        type: 'text/plain'
    };

    client.put('/someuser/stor/logs/+filename, stream, opts, function (err) {
      if(err) cb(err);
      else cb();
    });
  };

What should I do for this to work, regarding this line:

md5: crypto.createHash('md5').update(message).digest('base64'),

Thank you

score 1 · Accepted Answer · answered Nov 15 '13 at 18:06

TL;DR don't send the md5.

If you don't mind reading the stream twice, you can calculate the md5 beforehand like this:

var fs = require('fs');
var crypto = require('crypto');

function calculateStreamHash(readableStream, algorithm, encoding, cb) {
  var hash = crypto.createHash(algorithm);

  readableStream.on('data', updateHash);
  readableStream.on('end', finish);

  function updateHash(d) {
    hash.update(d);
  }

  function finish() {
    var d = hash.digest(encoding);
    cb(d);
  }
}

calculateStreamHash(fs.createReadStream('/tmp/' + filename), 'md5', 'base64', function(md5) {
  // use md5
});

If reading the stream twice is not an option, I suggest not sending an MD5 at all. Looking at the Manta's Node.js SDK it seems to be an optional field. See https://github.com/joyent/node-manta/blob/master/lib/client.js#L169

In fact, it looks like that in the specific method you're using createWriteStream(), the md5 option field is completely ignored. See https://github.com/joyent/node-manta/blob/master/lib/client.js#L1399

So you should probably ignore that md5 field.

How to use Joyent Manta client.createWriteStream(path, options) correctly

1 Answers1