0

I am new to programming/Linux/ELK etc. My background is Windows so this project is a big leap for me.

I seem to have reached a point that I cannot overcome and would like another set of eyes to review my work.

When viewing the output in Kibana 3 all of the custom field come back empty even though in logstash rubydebug they show as populated. See rubydebug output below:

 "message" => "<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454722] ",
          "@version" => "1",
        "@timestamp" => "2015-03-23T21:46:49.000Z",
              "host" => "1.1.1.1",
    "rsyslogprepend" => "<158>Mar 23 16:46:52 servername server-log",
         "timestamp" => "Mon Mar 23 16:46:49 2015",
             "bon01" => "43227.23454683",
          "username" => "dummy.user",
         "ipaddress" => [
        [0] "2.2.2.2",
        [1] "2.2.2.2"
    ],
             "bon02" => "23454722"
}
filter received {:event=>{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1"}, :level=>:debug, :file=>"(eval)", :line=>"24"}
Running grok filter {:event=>#<LogStash::Event:0x370ea56c @accessors=#<LogStash::Util::Accessors:0x228e71b1 @store={"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1"}, @lut={"host"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1"}, "host"]}>, @data={"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1"}, @cancelled=false>, :level=>:debug, :file=>"logstash/filters/grok.rb", :line=>"280"}
Event now:  {:event=>#<LogStash::Event:0x370ea56c @accessors=#<LogStash::Util::Accessors:0x228e71b1 @store={"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, @lut={"host"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "host"], "message"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "message"], "rsyslogprepend"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "rsyslogprepend"], "timestamp"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "timestamp"], "bon01"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "bon01"], "username"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "username"], "ipaddress"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "ipaddress"], "bon02"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "bon02"]}>, @data={"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, @cancelled=false>, :level=>:debug, :file=>"logstash/filters/grok.rb", :line=>"300"}
Date filter: received event {:type=>nil, :level=>:debug, :file=>"logstash/filters/date.rb", :line=>"178"}
Date filter looking for field {:type=>nil, :field=>"timestamp", :level=>:debug, :file=>"logstash/filters/date.rb", :line=>"181"}
Date parsing done {:value=>"Mon Mar 23 16:46:49 2015", :level=>:debug, :file=>"logstash/filters/date.rb", :line=>"210"}
output received {:event=>{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T21:46:49.000Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, :level=>:debug, :file=>"(eval)", :line=>"57"}
{
           "message" => "<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ",
          "@version" => "1",
        "@timestamp" => "2015-03-23T21:46:49.000Z",
              "host" => "1.1.1.1",
    "rsyslogprepend" => "<158>Mar 23 16:46:52 servername server-log",
         "timestamp" => "Mon Mar 23 16:46:49 2015",
             "bon01" => "43227.23454683",
          "username" => "dummy.user",
         "ipaddress" => [
        [0] "1.1.1.1",
        [1] "1.1.1.1"
    ],
             "bon02" => "23454723"
}


logstash conf file below:

    # syslog input

    input {

     tcp {
        port => 514
        #type => syslog
      }
      udp {
        port => 514
       #type => syslog
      }
    }

    filter {
                    grok {
                                    patterns_dir => "opt/logstash/patterns"

                                    #       match => [ "message", "%{NESSUS_MUTATE_RSYSLOG:syslog_prepend}" ]
    #       remove_field => [ "syslog_prepend" ]

    #               }

    #                mutate {
    #        remove_field => [ "syslog_prepend" ]
    #                }

    #               grok {

    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] user %{USERNAME:username} : testing %{IPV4:ipaddress} \(%{IPV4:ipaddress}\) \[%{NUMBER:bon02}\]"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] user %{USERNAME:username} : The remote host \(%{IPV4:ipaddress}\) is dead"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] \[nessusd_www_server\] User %{USERNAME:username} \(%{IPV4:ipaddress}\) successfully logged out"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] \[nessusd_www_server\] successful login of \'%{USERNAME:username}\' from %{IPV4:ipaddress} via %{NESSUS_PROTOCOL:protocol}"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] Finished testing %{IPV4:ipaddress}. Time : %{NESSUS_DURATION:duration}"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] User \'%{USERNAME:username}\' logged in via the XMLRPC interface"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] Full audit trail enabled"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] User %{USERNAME:username} starts a new scan \(%{NESSUS_SCANID:scanid}\)"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] user %{USERNAME:username} starts a new scan. Target\(s\) : %{IPV4:ipaddress}-%{IPV4:ipaddress}, with max_hosts = %{NESSUS_MAXHOSTS:maxhosts} and max_checks = %{NESSUS_MAXCHECKS:maxchecks}"]

                    }

                    date {
            match => [ "timestamp", "EEE MMM dd HH:mm:ss yyyy" ]
            target => "@timestamp"
                    }
    }

    output {
      stdout {codec => rubydebug }
      elasticsearch {
        host => "1.1.1.1"
        port => "9200"
        protocol => "http"
        index => "nessus_scanners-%{+YYYY.MM.dd}"
      }
    #  gelf {
    #    host => "1.1.1.1"
    #  }
alecxe
  • 462,703
  • 120
  • 1,088
  • 1,195
Wanderer
  • 1

1 Answers1

0

Have a look in Elasticsearch to see if your results are actually in there,

Try:

curl -XGET 'https://localhost:9200/nessus_scanners-2015.03.23/_search?pretty=true&q=*:*'

By Default, Kibana is looking for indexes with index pattern [logstash]YYYY.MM.DD

Idgoo
  • 43
  • 5
  • There is data in that index. I can see it populating in Elasticseach head as well. I do have Kibana set to [nessus-scanners-] idices. my custom fields show in Kibana but all are empty. – Wanderer Mar 23 '15 at 17:53
  • When importing a log file via stdin from the command line Kibana populates fine. – Wanderer Mar 23 '15 at 18:02
  • Sorry for pointing out the obvious but I fell into that trap and it took me sometime to figure it out. Kibana by default is set to display only the logs of the last couple of minutes. So if your data is from longer ago you wont see any log entries unless you change the default time filter. – markus Mar 24 '15 at 19:48