Bulk upload attachments to couchDB using node.js nano module

Question

I'm trying to bulk upload attachments to CouchDB using node.js and nano. First, the walk module is used to find all files in upload folder and create array from them. Next, each file from the array is supposed to be inserted into CouchDB via pipe and nano module. However, the final result is that only one attachment has been uploaded.

var nano = require('nano')('http://localhost:5984')
var alice = nano.use('alice');
var fs = require('fs');
var walk = require('walk');
var files = [];

// Walker options
var walker = walk.walk('./uploads', {
    followLinks: false
});

// find all files and add to array
walker.on('file', function (root, stat, next) {
    files.push(root + '/' + stat.name);
    next();
});

walker.on('end', function () {
    // files array ["./uploads/2.jpg","./uploads/3.jpg","./uploads/1.jpg"]
    files.forEach(function (file) {
        //extract file name
        fname = file.split("/")[2]

        alice.get('rabbit', {revs_info: true}, function (err, body) {

                fs.createReadStream(file).pipe(

                    alice.attachment.insert('rabbit', fname, null, 'image/jpeg', {
                        rev: body._rev
                    }, function (err, body) {
                        if (!err) console.log(body);
                    })


                )


        });



    });


});

score 1 · Accepted Answer · answered Dec 19 '12 at 18:00

1

This is because you are mixing an asynchronous api with assumptions of this being synchronous.

After the first request you will get conflicts, cause the rabbit document has changed.

Can you confirm this using NANO_ENV=testing node yourapp.js?

I recommend using async if this is the problem

answered Dec 19 '12 at 18:00

dscape

2,506
1
22
20

It was the case.Thank You for pointing the problem- i found a tutorial on flow control and modified the code according to it - below is the working example – user1276919 Dec 19 '12 at 20:57

score 0 · Answer 2 · answered Dec 19 '12 at 20:58

var nano = require('nano')('http://localhost:5984')
var alice = nano.use('alice');
var fs = require('fs');
var walk = require('walk');
var files = [];

// Walker options
var walker = walk.walk('./uploads', {
    followLinks: false
});

walker.on('file', function (root, stat, next) {
    files.push(root + '/' + stat.name);
    next();
});

walker.on('end', function () {
    series(files.shift());

});



function async(arg, callback) {
    setTimeout(function () {callback(arg); }, 100);
}


function final() {console.log('Done');}


function series(item) {
    if (item) { 
        async(item, function (result) {
            fname = item.split("/")[2]

            alice.get('rabbit', { revs_info: true }, function (err, body) {
                if (!err) {

                    fs.createReadStream(item).pipe(
                    alice.attachment.insert('rabbit', fname, null, 'image/jpeg', {
                        rev: body._rev
                    }, function (err, body) {
                        if (!err) console.log(body);
                    })


                    )

                }
            });

            return series(files.shift());
        });
    } 

    else {
        return final();
    }
}

Just curious, With your definition of "async", why the 100ms delay? why not 0ms, indicating that async should start as soon as the current execution stack completes? Just want to make sure I'm not missing a nuance to node or js. — Andrew Theken, Jan 14 '13 at 01:11

Bulk upload attachments to couchDB using node.js nano module

2 Answers2