1

I'm trying to create a logstash pipeline that polls a ActiveMQ jolokia endpoint. I'm wanting to collect all the metrics for the queues on the broker. I have the following pipeline.

input {
  http_poller {
    urls => {
      health_metrics => {
        method => "get"
        url => "http://localhost:8161/api/jolokia/read/org.apache.activemq:type=Broker,brokerName=localhost,destinationType=Queue,destinationName=*"
        headers => {
          "Content-Type" => "application/json"
        }
        auth => {
          user => "admin"
          password => "admin"
        }       
      }
    }
    request_timeout => 30
    keepalive => false
    interval => 5
    codec => "json"
    type => "activemq_broker_queue"
  }
}

filter {  
    json_encode {
      source => "value"
    }
    json {
      source => "value"
    }
    mutate {
      remove_field => ["request", "value", "timestamp"]
    }

}

output {
  elasticsearch {
    hosts => "localhost"
    # An index is created for each type of metrics inpout
    index => "logstash-activmq" 
    document_type => "%{type}" 
  }
   stdout {
    codec => rubydebug
  }
}

My jolokia response is in this format.

{
    request: {
        mbean: "org.apache.activemq:brokerName=localhost,destinationName=*,destinationType=Queue,type=Broker",
        type: "read"
    },
    value: {
        org.apache.activemq: brokerName=localhost,
        destinationName=SEARCH,
        destinationType=Queue,
        type=Broker: {
            ProducerFlowControl: true,
            Options: "",
            AlwaysRetroactive: false,
            MemoryUsageByteCount: 0,
            AverageBlockedTime: 0,
            MemoryPercentUsage: 0,
            CursorMemoryUsage: 0,
            InFlightCount: 0,
            Subscriptions: [],
            CacheEnabled: true,
            ForwardCount: 0,
            DLQ: false,
            StoreMessageSize: 0,
            AverageEnqueueTime: 0,
            Name: "SEARCH",
            BlockedSends: 0,
            TotalBlockedTime: 0,
            MaxAuditDepth: 2048,
            QueueSize: 0,
            MaxPageSize: 200,
            PrioritizedMessages: false,
            MemoryUsagePortion: 1,
            Paused: false,
            EnqueueCount: 0,
            MessageGroups: {

            },
            ConsumerCount: 0,
            AverageMessageSize: 0,
            CursorFull: false,
            ExpiredCount: 0,
            MaxProducersToAudit: 1024,
            CursorPercentUsage: 0,
            MinEnqueueTime: 0,
            MemoryLimit: 668309914,
            MinMessageSize: 0,
            DispatchCount: 0,
            MaxEnqueueTime: 0,
            DequeueCount: 0,
            BlockedProducerWarningInterval: 30000,
            ProducerCount: 0,
            MessageGroupType: "cached",
            MaxMessageSize: 0,
            UseCache: true,
            SlowConsumerStrategy: null
        },
        org.apache.activemq: brokerName=localhost,
        destinationName=weather,
        destinationType=Queue,
        type=Broker: {
            ProducerFlowControl: true,
            Options: "",
            AlwaysRetroactive: false,
            MemoryUsageByteCount: 0,
            AverageBlockedTime: 0,
            MemoryPercentUsage: 0,
            CursorMemoryUsage: 0,
            InFlightCount: 0,
            Subscriptions: [],
            CacheEnabled: true,
            ForwardCount: 0,
            DLQ: false,
            StoreMessageSize: 0,
            AverageEnqueueTime: 0,
            Name: "weather",
            BlockedSends: 0,
            TotalBlockedTime: 0,
            MaxAuditDepth: 2048,
            QueueSize: 0,
            MaxPageSize: 200,
            PrioritizedMessages: false,
            MemoryUsagePortion: 1,
            Paused: false,
            EnqueueCount: 0,
            MessageGroups: {

            },
            ConsumerCount: 0,
            AverageMessageSize: 0,
            CursorFull: false,
            ExpiredCount: 0,
            MaxProducersToAudit: 1024,
            CursorPercentUsage: 0,
            MinEnqueueTime: 0,
            MemoryLimit: 668309914,
            MinMessageSize: 0,
            DispatchCount: 0,
            MaxEnqueueTime: 0,
            DequeueCount: 0,
            BlockedProducerWarningInterval: 30000,
            ProducerCount: 0,
            MessageGroupType: "cached",
            MaxMessageSize: 0,
            UseCache: true,
            SlowConsumerStrategy: null
        }
    },
    timestamp: 1453588727,
    status: 200
}

I would like to be able to split the two queue destinations into two different documents and then save them to ES.

Currently I'm get an error about cannot contain '.'

zachariahyoung
  • 821
  • 4
  • 17
  • 29
  • Use the split filter to take an existing document and make two out of it. It can be a lot of work to get the remaining two documents to look the way you want. But, if you can't change the input, there's little choice. – Alain Collins Jan 24 '16 at 04:26

0 Answers0