Nodejs stream modification for elasticsearch bulk

Question

i wanted to append below object to every object in stream

{"index":{"_index":"tvseries","_type":"internindex"}}

my stream looks like this

[
  {"showname":"The X Files","episode":"04","content":"Before what?","season":"1"},
  {"showname":"The X Files","episode":"04","content":"Before what?","season":"1"},
  {"showname":"The X Files","episode":"01","content":"What?","season":"1"}
]

what my stream should look like !

> -> POST http://localhost:9200/_bulk   {"index":{"_index":"tvseries","_type":"internindex"}}  
> {"showname":"The X Files","episode":"04","content":"Before
> what?","season":"1"}  
> {"index":{"_index":"tvseries","_type":"internindex"}}  
> {"showname":"The X
> Files","episode":"04","content":"Great.","season":"1"}  
> {"index":{"_index":"tvseries","_type":"internindex"}}  
> {"showname":"The X
> Files","episode":"01","content":"What?","season":"1"}

how can i achieve this using jsonstream in my existing below codebase

var stream = new ElasticsearchWritableStream(client, {
  highWaterMark: 256,
  flushTimeout: 500
});

pg.connect(connectionString,function(err, client, done) {
  if(err) throw err;
  var query = new QueryStream('SELECT * FROM srt limit 2')
  var streams = client.query(query)

  //release the client when the stream is finished
  streams.on('end', done)
  streams.pipe(JSONStream.stringify()).pipe(stream)
})

the npm packages currently i am using

For bulk insertion in elasticsearch !

elasticsearch-writable-stream

for getting data from postgres into streams !

pg-query-stream

missing piece is converting postgres streams into elastic writable streams ! any suggestion, pointers ,recommendations on how to achieve this !

score 0 · Answer 1 · answered Jul 13 '17 at 14:00

so basically, the only feasible options without much code change would be building format required for bulk insert into elastic search from postgres itself rather than in node.js object !

"SELECT 'tvseries' as index,'internindex' as type, json_build_object('showname', showname, 'epsiode', ep,'content',content,'season',season) AS body"
+" FROM   srt  where shownameid=4"

Nodejs stream modification for elasticsearch bulk

1 Answers1