0

I have this configuration file

input {
    file {
        path => ["/var/log/notifications/some.log"]
        type => 'some'
    }

   file {
       path => ["/var/log/somenotifications/somenotification.log"]
       type => 'notification'
   }

   file {
        path => ["/var/log/somenotifications/application.log.201607*", "/var/log/somenotifications/application.log.201608*", "/var/log/somenotifications/application.log.201609*"]
        exclude => ["/var/log/somenotifications/application.log.201607*.gz", "/var/log/somenotifications/application.log.201608*.gz", "/var/log/somenotifications/application.log.201609*.gz"]
        type => 'old'
        start_position => beginning
        sincedb_path => "/dev/null"
   }

   file {
         path => ["/var/log/somenotifications/someapplication.log"]
         type => "application"
    }
}

filter {
     if [type] == "some" {
          grok {
                match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} \| \"%{WORD:msisdn}\" \"%{WORD:operator}\" \"%{URI:page}\" \"%{DATA:affpartner}\"" }
          }
     }

     if [type] == "notification" {
          grok {
                match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} \| \"%{WORD:service}\" \"%{WORD:transactionId}\" \"%{WORD:subsId}\" \"%{WORD:status}\" \"%{WORD:errorCode}\" \"%{WORD:errorDescription}\" \"%{WORD:billingType}\" \"%{WORD:affpartner}\" \"%{WORD:operatorId}\""}
          }
     }

     if [type] == "application" or [type] == "old" or [type] == "payment" or [type] == "subscribe" {
             grok {
                 match => {"message" => "%{SYSLOG5424SD:timestamp} notifications.DEBUG: >>>>>>>> %{WORD:method} %{URIPATH}%{URIPARAM:params}"}
             }
             kv {
                 field_split => "&"
                 source => "params"
             }
             mutate {
                add_field => {
                     "payout_c" => "%{payout}"
                }
                convert => { "payout_c" => "float" }
             }
      }

    date {
       match => [ "timestamp", "[yyyy-MM-dd HH:mm:ss]" ]
       target => "@timestamp"
       locale => "en"
    }
}

output {
    if [type] == "payment" or [type] == "subscribe" or [type] == "application" or [type] == "old" {
        if "_grokparsefailure" not in [tags] {
            elasticsearch {
                hosts => ["localhost:9200", "otherhost:9200", "otherhost2:9200"]
                index => "notifications-sent"
            }
        }
    } else {
        if [type] != "old" {
            elasticsearch {
                hosts => ["localhost:9200", "otherhost:9200", "otherhost2:9200"]
            }
        }
    }
}

Can't understand why I see in my elasticsearch only notifications for first july and first august files. Does not have any sense :( The problem is in file type "old", others file-parsing works good

M_M
  • 79
  • 1
  • 2
  • 10
  • Are you sure that logs from after the first of the month are stored in /var/log/somenotifications/application.log.201607*? Can you look in the file and verify that there are indeed logs after the first of the month and they aren't gzipped as a part of log rotation? – fylie Oct 07 '16 at 15:10
  • Also, instead of `if [type] == "payment" or [type] == "subscribe" or [type] == "application" or [type] == "old"` you can do `if [type] in ["payment","subscribe","application","old"]` – fylie Oct 08 '16 at 21:05

0 Answers0