couchdb - Import json file

Question

I have a file with data in the json format. The data is odd in that the rows have varying subsets of a set of properties that are not all known in advance (over time they will build up). For example:

[{"firstName":"Joe","education":"highschool","sex":"male"},
 {"lastName":"Edwards","address":"ohio","sex":"male"},
{"favoriteSport":"cycling","bicycle":"raleigh","ownsBoat":"yes","ownsDog":"yes","ownsHouse":"yes"}]

A large amount of data exists already in a file so I would like to import it in to couchdb rather than enter the data item by item. I followed the procedures from a post here but, while a db was created, it was empty. I used:

curl -X PUT -d @../Data/data.json http://127.0.0.1:5984/test_import

UPDATE: Since I'm working with nodejs (newbie), I thought I'd try using 'cradle'. My thought was to take the import the data as an array, and bulk load that using 'cradle's dbsave(). But using the following:

var fs = require('fs');
var cradle = require('cradle');

var data = fs.readFile( '../Data/data.json', function (err, data) {
  if (err) {
    throw err; 
  }
.
.
.
makeDB(bigdata,'test_import');  // where 'bigdata' is an array of json objects/couchdb 'documents'
});

function makeDB (p,filename) {

var db = new(cradle.Connection)().database(filename);
console.log(db);

db.save(p, function(err,res) {
if (err) {
    console.log(err);
} else {
    console.log('Success!');
}
});

};

The latter seems to work!! A database is created and filled but it does however throw the following errors:

k:\nodejs\node_modules\cradle\lib\cradle.js:198
            return callback(body);
                   ^
TypeError: undefined is not a function

OR

k:\nodejs\node_modules\cradle\lib\cradle.js:201
        callback(null, self.options.raw ? body : new cradle.Response(body, res
        ^
TypeError: undefined is not a function
    at Request.cradle.Connection.request [as _callback] (k:\nodejs\node_modules\cradle\lib\cradle.
js:201:9)
    at Request.init.self.callback (k:\nodejs\node_modules\request\main.js:120:22)
    at Request.EventEmitter.emit (events.js:91:17)
    at Request.<anonymous> (k:\nodejs\node_modules\request\main.js:633:16)
    at Request.EventEmitter.emit (events.js:88:17)
    at IncomingMessage.Request.start.self.req.self.httpModule.request.buffer (k:\nodejs\node_modul
es\request\main.js:595:14)
    at IncomingMessage.EventEmitter.emit (events.js:115:20)
    at IncomingMessage._emitEnd (http.js:366:10)
    at HTTPParser.parserOnMessageComplete [as onMessageComplete] (http.js:149:23)
    at Socket.socketOnData [as ondata] (http.js:1366:20)

Could you provide a `console.log(p)` right before you call `db.save`? — dignifiedquire, Aug 19 '12 at 18:48
I think what you pasted was `console.log(db)` and not `console.log(p)`. Because `p` should be the data you are saving. — dignifiedquire, Aug 19 '12 at 19:59
Can't post the actual data. I'll try and cobble together something sanitized. That said, the data does get imported and shows up in couchdb. I have been playing with views, so the data _is_ there. What should I look for in the data that might cause the error? — MikeB, Aug 19 '12 at 20:30
Sure. My guess would be that there is an syntactic error in the data. Because the error message points toward something that the callback that you attach to your save function as second argument is not correct. — dignifiedquire, Aug 19 '12 at 21:52

score 1 · Accepted Answer · answered Aug 21 '12 at 02:26

[SOLVED]

The answers to my questions are that yes, of course, couchdb is perfectly suited to that kind of data. The easiest way I found to do the bulk import with node.js is using cradle (whose creator provided solution to problem). The preceding code works error free with the following changes to the makeDB function:

//  Take the array of objects and create a couchdb database

function makeDB (data,filename) {

var db = new(cradle.Connection)().database(filename);
//console.log(db);
db.create(function(err){
    if (err) console.log(err); 
    });
db.save(data, function(err) {
    if (err) console.log(err);
    console.log(filename + ' is created.');
    });

};

couchdb - Import json file

1 Answers1