3

I'm trying to use the kinesis_firehose_delivery_stream resource to create a Kinesis Firehose with a Direct PUT source, no data transformation, and an extended_s3 destination.

I've modified the code in this example (to remove the lambda function) so now it looks like this:

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 3.0"
    }
  }
}

provider "aws" {
  region = "us-east-1"
}

resource "aws_s3_bucket" "bucket" {
  bucket = "test-kinesis-destination-bucket"
  acl    = "private"
}

resource "aws_kinesis_firehose_delivery_stream" "kinesis_event_stream" {
  name        = "kinesis-test-stream"
  destination = "extended_s3"

  extended_s3_configuration {
    role_arn   = aws_iam_role.firehose_role.arn
    bucket_arn = aws_s3_bucket.bucket.arn
    buffer_size = 1
    buffer_interval = 60
  }
}

resource "aws_iam_role" "firehose_role" {
  name = "firehose_test_role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "firehose.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

Terraform is able to successfully apply everything but Firehose doesn't seem to be able to write to S3.

Am I missing something in my IAM role? and if so how can I fix it?

Edit

I've updated my terraform file to update the IAM policy, per @Marcin 's answer, to give Firehose permission to write to S3.

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 3.0"
    }
  }
}

provider "aws" {
  region = "us-west-2"
}

resource "aws_s3_bucket" "bucket" {
  bucket = "test-kinesis-destination-bucket"
  acl    = "private"
}

resource "aws_kinesis_firehose_delivery_stream" "kinesis_event_stream" {
  name        = "kinesis-test-stream"
  destination = "extended_s3"

  extended_s3_configuration {
    role_arn   = aws_iam_role.firehose_role.arn
    bucket_arn = aws_s3_bucket.bucket.arn
    buffer_size = 1
    buffer_interval = 60
  }
}

resource "aws_iam_role" "firehose_role" {
  name = "firehose_test_role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",  
  "Statement":
  [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "firehose.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF

  inline_policy {
    name = "kinesis-s3-inline-policy"
    policy = jsonencode({
      Version = "2012-10-17"
      Statement = [
        {      
          Effect = "Allow",      
          Action = [
            "s3:AbortMultipartUpload",
            "s3:GetBucketLocation",
            "s3:GetObject",
            "s3:ListBucket",
            "s3:ListBucketMultipartUploads",
            "s3:PutObject"
          ]      
          Resource = [        
            "arn:aws:s3:::test-kinesis-destination-bucket",
            "arn:aws:s3:::test-kinesis-destination-bucket/*"            
          ]    
        },
        {
          Effect = "Allow"
          Action = [
            "kinesis:DescribeStream",
            "kinesis:GetShardIterator",
            "kinesis:GetRecords",
            "kinesis:ListShards"
          ]
          Resource = aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn
        }
      ]
    })
  }
}

But when I run terraform plan I get the following error:

$ terraform plan
╷
│ Error: Cycle: aws_kinesis_firehose_delivery_stream.kinesis_event_stream, aws_iam_role.firehose_role
│ 
│ 
|

How can I reference Firehose's ARN inside its IAM policy?

A Poor
  • 856
  • 10
  • 26

3 Answers3

5

You have created role firehose_role with only trust relatinship, but no actual S3 permissions. Your role should have the following permissions as explained in the docs (you can trim it down if you don't use lambda with kinesis, kms or other services that firehose can use):

{
    "Version": "2012-10-17",  
    "Statement":
    [    
        {      
            "Effect": "Allow",      
            "Action": [
                "s3:AbortMultipartUpload",
                "s3:GetBucketLocation",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:ListBucketMultipartUploads",
                "s3:PutObject"
            ],      
            "Resource": [        
                "arn:aws:s3:::bucket-name",
                "arn:aws:s3:::bucket-name/*"            
            ]    
        },        
        {
            "Effect": "Allow",
            "Action": [
                "kinesis:DescribeStream",
                "kinesis:GetShardIterator",
                "kinesis:GetRecords",
                "kinesis:ListShards"
            ],
            "Resource": "arn:aws:kinesis:region:account-id:stream/stream-name"
        },
        {
           "Effect": "Allow",
           "Action": [
               "kms:Decrypt",
               "kms:GenerateDataKey"
           ],
           "Resource": [
               "arn:aws:kms:region:account-id:key/key-id"           
           ],
           "Condition": {
               "StringEquals": {
                   "kms:ViaService": "s3.region.amazonaws.com"
               },
               "StringLike": {
                   "kms:EncryptionContext:aws:s3:arn": "arn:aws:s3:::bucket-name/prefix*"
               }
           }
        },
        {
           "Effect": "Allow",
           "Action": [
               "logs:PutLogEvents"
           ],
           "Resource": [
               "arn:aws:logs:region:account-id:log-group:log-group-name:log-stream:log-stream-name"
           ]
        },
        {
           "Effect": "Allow", 
           "Action": [
               "lambda:InvokeFunction", 
               "lambda:GetFunctionConfiguration" 
           ],
           "Resource": [
               "arn:aws:lambda:region:account-id:function:function-name:function-version"
           ]
        }
    ]
}
Marcin
  • 215,873
  • 14
  • 235
  • 294
  • Thanks for your answer. How can I reference Kinesis's ARN in my policy? When I try to use `Resource = aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn` I get a cycle error. – A Poor May 18 '21 at 02:06
  • 1
    @APoor Can you update the code with its new version and the new error? Or alternatively, make a new question? – Marcin May 18 '21 at 02:07
  • I've updated my question with the new code/error – A Poor May 18 '21 at 02:19
  • 1
    @APoor You need to manually construct the arn, as your kinesis resource does not exist yet. – Marcin May 18 '21 at 02:22
3

@A Poor: But when I run terraform plan I get the following error:

You are in loop:

  • aws_kinesis_firehose_delivery_stream is wating aws_iam_role to get aws_iam_role.firehose_role.arn

  • aws_iam_role is waiting: aws_kinesis_firehose_delivery_stream to get aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn

So I use: "Resource": "arn:aws:kinesis:ap-southeast-1:${var.account_id}:stream/${var.kinesis_firehose_delivery_stream_name}"

PaPu
  • 731
  • 7
  • 12
0

To avoid the cycle terraform is complaining about, you can convert your inline_policy into a aws_iam_role_policy instead.

With this approach, you can reference the aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn without the cycle error.

For example

resource "aws_iam_role" "firehose_role" {
  name = "${local.service}-${var.environment}-kinesis-firehose-role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement":
  [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "firehose.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_role_policy" "firehose_role_policy" {
  name = "${local.service}-${var.environment}-kinesis-firehose-inline-policy"
  role = aws_iam_role.firehose_role.id
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow",
        Action = [
          "s3:AbortMultipartUpload",
          "s3:GetBucketLocation",
          "s3:GetObject",
          "s3:ListBucket",
          "s3:ListBucketMultipartUploads",
          "s3:PutObject"
        ]
        Resource = [
          "arn:aws:s3:::${aws_s3_bucket.bucket.id}",
          "arn:aws:s3:::${aws_s3_bucket.bucket.id}/*"
        ]
      },
      {
        Effect = "Allow"
        Action = [
          "kinesis:DescribeStream",
          "kinesis:GetShardIterator",
          "kinesis:GetRecords",
          "kinesis:ListShards"
        ]
        Resource = aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn
      }
    ]
  })
}

ref

Ulad Kasach
  • 11,558
  • 11
  • 61
  • 87