4

I am testing a cloudwatch alarm filter (to be precise checking if lambda maxed out in memory) to send to SNS which then is sent to SQS queue. However, I do not see the filter appeared in logs. The setup is cloudwatch (filtered alarm) -> SNS -> SQS ->splunk

What I have so far:

resource "aws_cloudwatch_metric_alarm" "general_lambda_error" {
    depends_on = [
      "aws_cloudwatch_log_metric_filter.max_memory_time_out",
    ]   
    alarm_name                = "general_lambda_error"
    comparison_operator       = "GreaterThanOrEqualToThreshold" 
    evaluation_periods        = "1" 
    metric_name               = "Errors" 
    namespace                 = "AWS/Lambda" 
    period                    = "60" 
    statistic                 = "SampleCount" 
    threshold                 = "2"
    alarm_description         = "This metric monitors Lambda Memory Max Usage and other Errors: threshold=2"
    alarm_actions             = [ "some-arn" ]
    dimensions {
      FunctionName = "lambda-test"
      Resource = "lambda-test"
    }
}
resource "aws_cloudwatch_log_metric_filter" "max_memory_time_out" {
    name                      = "max_memory_time_out"
    pattern                   = "[report_name=\"REPORT\", 
  request_id_name=\"RequestId:\", request_id_value, duration_name=\"Duration:\", duration_value, duration_unit=\"ms\", billed_duration_name_1=\"Billed\", bill_duration_name_2=\"Duration:\", billed_duration_value, billed_duration_unit=\"ms\", memory_size_name_1=\"Memory\", memory_size_name_2=\"Size:\", memory_size_value, memory_size_unit=\"MB\", max_memory_used_name_1=\"Max\", max_memory_used_name_2=\"Memory\", max_memory_used_name_3=\"Used:\", max_memory_used_value, max_memory_used_unit=\"MB\"]"
    log_group_name            = "/aws/lambda/lambda-test"
    metric_transformation {
      name      = "SampleCount"
      namespace = "cloudwatch_filter"
      value     = "1"
    }
}

How can I send the filtered message? I have found this link which describes similar problem but solution is to create a lambda function. Can I do it without creating lambda function?

Zafar
  • 1,111
  • 2
  • 11
  • 23
  • 1
    You are mistaking a log filter with a log subscription filter. Log filters are use to create metrics, which can then be used to create alarms. Log subscription filters, are using to send filtered content to a destination such as Kinesis. I've done this by creating a firehose to send the data to S3, and then creating an S3 notfication to send to SQS. Then splunk can read from SQS. – victor m Mar 28 '19 at 15:13
  • You also have a subscription filter to send logs to another lambda function, and have that Lambda function write the log entries to SQS – victor m Mar 28 '19 at 15:23
  • @victorm could you provide small example of using kinesis firehose? Does that mean I can use log metric filter to create alarms? – Zafar Mar 29 '19 at 14:02
  • see below for an example. – victor m Mar 30 '19 at 23:26

1 Answers1

0

Here is the code for getting cloudwatch logs to S3:

resource "aws_iam_role" "cloudwatchToFirehose" {
  name               = "${var.env}-${var.name}FirehoseCWL-Role"

  assume_role_policy = <<JSON
{
  "Version": "2012-10-17",
  "Statement": [{
      "Action": "sts:AssumeRole",
      "Principal": { "Service": "logs.${var.region}.amazonaws.com"},
      "Effect": "Allow"
  }]
}
JSON
}

resource "aws_iam_policy" "cloudwatchToFirehose" {
  name   = "${var.env}-${var.name}FirehoseCWL-Policy"

  policy = <<JSON
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect":"Allow",
      "Action": [
                "firehose:DeleteDeliveryStream",
                "firehose:PutRecord",
                "firehose:PutRecordBatch",
                "firehose:UpdateDestination"
      ],
      "Resource": ["${aws_kinesis_firehose_delivery_stream.firehoseToS3.arn}"]
    }
  ]
}
JSON
}

resource "aws_iam_role_policy_attachment" "cloudwatchToFirehose" {
  role       = "${aws_iam_role.cloudwatchToFirehose.name}"
  policy_arn = "${aws_iam_policy.cloudwatchToFirehose.arn}"
}

resource "aws_iam_role" "firehoseToS3" {
  name               = "${var.env}-${var.name}FirehoseS3-Role"
  assume_role_policy = <<JSON
{
  "Version": "2012-10-17",
  "Statement": [{
      "Action": "sts:AssumeRole",
      "Principal": { "Service": "firehose.amazonaws.com"},
      "Effect": "Allow"
  }]
}
JSON

  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_iam_policy" "firehoseToS3" {
  name   = "${var.env}-${var.name}FirehoseS3-Policy"
  policy = <<JSON
{
  "Version": "2012-10-17",
  "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:AbortMultipartUpload",
                "s3:GetBucketLocation",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:ListBucketMultipartUploads",
                "s3:PutObject",
                "s3:PutObjectAcl"
            ],
            "Resource": [
                "arn:aws:s3:::${var.logs_bucket}",
                "arn:aws:s3:::${var.logs_bucket}/*"
            ]
        },
        {
            "Effect": "Allow",
            "Action": [
                "kinesis:DescribeStream",
                "kinesis:GetShardIterator",
                "kinesis:GetRecords"
            ],
            "Resource": "${aws_kinesis_firehose_delivery_stream.firehoseToS3.arn}"
        },
        {
           "Effect": "Allow",
           "Action": [
               "kms:Decrypt",
               "kms:GenerateDataKey"
           ],
           "Resource": [
               "${var.kms_general_key}"
           ]
        },
        {
           "Effect": "Allow",
           "Action": [
               "logs:PutLogEvents"
           ],
           "Resource": [
               "arn:aws:logs:*:*:log-group:${var.org}-${var.group}-${var.environment}/Firehose:*"
           ]
        },
        {
           "Effect": "Allow",
           "Action": [
               "lambda:InvokeFunction",
               "lambda:GetFunctionConfiguration"
           ],
           "Resource": [
               "arn:aws:lambda:*:*:function:*:*"
           ]
        }
  ]
}
JSON

  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_iam_role_policy_attachment" "firehoseToS3" {
  role       = "${aws_iam_role.firehoseToS3.name}"
  policy_arn = "${aws_iam_policy.firehoseToS3.arn}"
  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_kinesis_firehose_delivery_stream" "firehoseToS3" {
  name        = "${var.env}-${var.name}Firehose-Stream"
  destination = "s3"

  s3_configuration {
    role_arn        = "${aws_iam_role.firehoseToS3.arn}"
    bucket_arn      = "arn:aws:s3:::${var.logs_bucket}"
    buffer_interval = "300"
    buffer_size     = "10"
    prefix          = "${var.name}"
  }
}

resource "aws_cloudwatch_log_subscription_filter" "cloudwatchToFirehose" {
  count           = "1"
  name            = "${var.env}-${var.name}Filter-Subscription"
  role_arn        = "${aws_iam_role.cloudwatchToFirehose.arn}"
  log_group_name  = "${element(var.log_groups, count.index)}"
  filter_pattern  = ""
  destination_arn = "${aws_kinesis_firehose_delivery_stream.firehoseToS3.arn}"
}
victor m
  • 2,012
  • 2
  • 14
  • 23