How to access metrics of Alpakka CommittableSource with back off?

Question

Accessing the metrics of an Alpakka PlainSource seems fairly straight forward, but how can I do the same thing with a CommittableSource?

I currently have a simple consumer, something like this:

class Consumer(implicit val ma: ActorMaterializer, implicit val ec: ExecutionContext) extends Actor {

  private val settings = ConsumerSettings(
    context.system,
    new ByteArrayDeserializer,
    new StringDeserializer)
    .withProperties(...)

  override def receive: Receive = Actor.emptyBehavior

  RestartSource
    .withBackoff(minBackoff = 2.seconds, maxBackoff = 20.seconds, randomFactor = 0.2)(consumer)
    .runForeach { handleMessage }

  private def consumer() = {
    AkkaConsumer
      .committableSource(settings, Subscriptions.topics(Set(topic)))
      .log(getClass.getSimpleName)
      .withAttributes(ActorAttributes.supervisionStrategy(_ => Supervision.Resume))
  }

  private def handleMessage(message: CommittableMessage[Array[Byte], String]): Unit = {
    ...
  }
}

How can I get access to the consumer metrics in this case?

score 0 · Accepted Answer · answered Dec 09 '19 at 10:27

We are using the Java prometheus client and I solved my issue with a custom collector that fetches its metrics directly from JMX:

import java.lang.management.ManagementFactory
import java.util

import io.prometheus.client.Collector
import io.prometheus.client.Collector.MetricFamilySamples
import io.prometheus.client.CounterMetricFamily
import io.prometheus.client.GaugeMetricFamily
import javax.management.ObjectName

import scala.collection.JavaConverters._
import scala.collection.mutable

class ConsumerMetricsCollector(val labels: Map[String, String] = Map.empty) extends Collector {
  val metrics: mutable.Map[String, MetricFamilySamples] = mutable.Map.empty

  def collect: util.List[MetricFamilySamples] = {
    val server = ManagementFactory.getPlatformMBeanServer
    for {
      attrType <- List("consumer-metrics", "consumer-coordinator-metrics", "consumer-fetch-manager-metrics")
      name <- server.queryNames(new ObjectName(s"kafka.consumer:type=$attrType,client-id=*"), null).asScala
      attrInfo <- server.getMBeanInfo(name).getAttributes.filter { _.getType == "double" }
    } yield {
      val attrName = attrInfo.getName
      val metricLabels = attrName.split(",").map(_.split("=").toList).collect {
        case "client-id" :: (id: String) :: Nil => ("client-id", id)
      }.toList ++ labels

      val metricName = "kafka_consumer_" + attrName.replaceAll(raw"""[^\p{Alnum}]+""", "_")
      val labelKeys = metricLabels.map(_._1).asJava

      val metric = metrics.getOrElseUpdate(metricName,
        if(metricName.endsWith("_total") || metricName.endsWith("_sum")) {
          new CounterMetricFamily(metricName, attrInfo.getDescription, labelKeys)
        } else {
          new GaugeMetricFamily(metricName, attrInfo.getDescription, labelKeys)
        }: MetricFamilySamples
      )

      val metricValue = server.getAttribute(name, attrName).asInstanceOf[Double]
      val labelValues = metricLabels.map(_._2).asJava
      metric match {
        case f: CounterMetricFamily => f.addMetric(labelValues, metricValue)
        case f: GaugeMetricFamily => f.addMetric(labelValues, metricValue)
        case _ =>
      }
    }
    metrics.values.toList.asJava
  }
}

How to access metrics of Alpakka CommittableSource with back off?

1 Answers1