I am pushing data to elastic via logstash , In logstash I have aggerated the data as parent child.
This pushing data but missed some data while aggerate, for example I have one card and that have 10 chapters then it never push 10 chapters to "chapters" somtimes it pushed only 3, sometimes 5 , some times 7. But in some cards it pushed all the chapters this behaves very strange and have no fixed pattern. I did not findout the reason why this happened, experts can guide me where can be issue and what point I am missing in this.
logstash config file is below
input {
jdbc {
jdbc_connection_string => "my connection string"
jdbc_user => "my user"
jdbc_password => "my_password"
jdbc_driver_library => "/usr/share/logstash/logstash-core/lib/jars/jdbc-mssql.jar"
jdbc_driver_class => "com.microsoft.sqlserver.jdbc.SQLServerDriver"
last_run_metadata_path => "/etc/logstash/.logstash_jdbc_last_run_qa_flashcard"
type => "card"
statement => "select * from carddetailview where dateupdated > :sql_last_value"
tracking_column => "dateupdated"
tracking_column_type => "timestamp"
use_column_value => "true"
schedule => "* * * * *"
}
}
filter{
aggregate {
task_id => "%{cardid}"
code => "
map['cardid'] = event.get('cardid')
map['topicname'] = event.get('topicname')
map['l1subject'] = event.get('l1subject')
map['l2subject'] = event.get('l2subject')
map['dateupdated'] = event.get('dateupdated')
map['chapters'] ||= []
map['chapters'] <<
{
'detailid' => event.get('detailid'),
'front' => event.get('front'),
'back' => event.get('back')
}
"
push_previous_map_as_event => true
timeout => 60
timeout_tags => ['aggregated']
}
mutate { remove_field => ["detailid", "front", "back"] }
}
output {
elasticsearch {
hosts => [ "http://localhost:9200" ]
user => 'elastic'
password => 'my_password'
index => "myindex"
document_type => "card"
document_id => "%{cardid}"
}
stdout { codec => "json_lines" }
}
Below is the data inserted in elastic. here only 4 chapters pushed to elastic but these should be 21 as per my data. Sometimes 5 chapters pushed , sometimes 6 or sometimes 9.
{
"userid" : 20,
"chapters" : [
{
"carddetailid" : 1246,
"backscore" : 36,
"front" : "Flamebait",
"back" : "A"
},
{
"carddetailid" : 1247,
"backscore" : 42,
"front" : "Meme",
"back" : "B"
},
{
"carddetailid" : 1248,
"backscore" : 40,
"front" : "Posts",
"back" : "C"
},
{
"carddetailid" : 1249,
"backscore" : 38,
"front" : "Chats",
"back" : "D"
}
],
"keyword" : "A program that appears desirable",
"cardid" : "1"
}