0

I am pushing data to elastic via logstash , In logstash I have aggerated the data as parent child.

This pushing data but missed some data while aggerate, for example I have one card and that have 10 chapters then it never push 10 chapters to "chapters" somtimes it pushed only 3, sometimes 5 , some times 7. But in some cards it pushed all the chapters this behaves very strange and have no fixed pattern. I did not findout the reason why this happened, experts can guide me where can be issue and what point I am missing in this.

logstash config file is below
input { 
    jdbc {
        jdbc_connection_string => "my connection string"
        jdbc_user => "my user"
        jdbc_password => "my_password"
        jdbc_driver_library => "/usr/share/logstash/logstash-core/lib/jars/jdbc-mssql.jar"
        jdbc_driver_class => "com.microsoft.sqlserver.jdbc.SQLServerDriver"
        last_run_metadata_path => "/etc/logstash/.logstash_jdbc_last_run_qa_flashcard"
        type => "card"
        statement => "select * from carddetailview where dateupdated > :sql_last_value"
        tracking_column => "dateupdated"
        tracking_column_type => "timestamp"
        use_column_value => "true"
        schedule => "* * * * *"
    }
}
filter{
    aggregate {
        task_id => "%{cardid}"
        code => "
        map['cardid']          = event.get('cardid')
        map['topicname']    = event.get('topicname')
        map['l1subject']    = event.get('l1subject')
        map['l2subject']    = event.get('l2subject')
        map['dateupdated']  = event.get('dateupdated')
        map['chapters'] ||= []
        map['chapters'] << 
                {   
                    'detailid' => event.get('detailid'),
                    'front'    => event.get('front'),
                    'back'    => event.get('back')
                }
        "
        push_previous_map_as_event => true
        timeout => 60
        timeout_tags => ['aggregated']
    }
    mutate { remove_field => ["detailid", "front", "back"] }        
}
output {
    elasticsearch {
      hosts => [ "http://localhost:9200" ]
          user => 'elastic'
          password => 'my_password'
          index => "myindex"
          document_type => "card"
          document_id => "%{cardid}" 
    }   
    stdout { codec => "json_lines" }
}

Below is the data inserted in elastic. here only 4 chapters pushed to elastic but these should be 21 as per my data. Sometimes 5 chapters pushed , sometimes 6 or sometimes 9.

{
      "userid" : 20,
      "chapters" : [
        {
          "carddetailid" : 1246,
          "backscore" : 36,
          "front" : "Flamebait",
          "back" : "A"
        },
        {
          "carddetailid" : 1247,
          "backscore" : 42,
          "front" : "Meme",
          "back" : "B"
        },
        {
          "carddetailid" : 1248,
          "backscore" : 40,
          "front" : "Posts",
          "back" : "C"
        },
        {
          "carddetailid" : 1249,
          "backscore" : 38,
          "front" : "Chats",
          "back" : "D"
        }
      ],
      "keyword" : "A program that appears desirable",
      "cardid" : "1"
    }

0 Answers0