I wrote next implementation to read messages from AWS Kinesis and deployed it on AWS ECS:
Application.java
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import com.amazonaws.event.ProgressListener;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
@SpringBootApplication
public class Application {
public static void main(final String[] args) {
SpringApplication.run(Application.class, args);
}
@Bean
public ObjectMapper objectMapper() {
ObjectMapper objectMapper = new ObjectMapper();
SimpleModule simpleModule = new SimpleModule()
.addAbstractTypeMapping(ProgressListener.class, ProgressListener.NoOpProgressListener.class);
objectMapper.registerModule(simpleModule);
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
return objectMapper;
}
}
ConsumerConfiguration.java
import java.nio.charset.StandardCharsets;
import java.util.function.Consumer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.amazonaws.services.kinesis.model.PutRecordsRequest;
import com.amazonaws.services.kinesis.model.PutRecordsRequestEntry;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import reactor.util.Logger;
import reactor.util.Loggers;
@Configuration
@RequiredArgsConstructor
public class ConsumerConfiguration {
private static final Logger LOG = Loggers.getLogger(ConsumerConfiguration.class);
private final ObjectMapper objectMapper;
private void processRecords(byte[] bytes) {
String jsonRecords = new String(bytes, StandardCharsets.UTF_8);
LOG.info("New Records consumed...");
try {
PutRecordsRequest putRecordsRequest = objectMapper.readValue(jsonRecords, PutRecordsRequest.class);
LOG.info("Records size = {}", putRecordsRequest.getRecords().size());
putRecordsRequest.getRecords().stream()
.map(PutRecordsRequestEntry::getData)
.map(data -> new String(data.array(), StandardCharsets.UTF_8))
.forEach(data -> LOG.info("Record data length received {}", data.length()));
} catch (JsonProcessingException e) {
LOG.error("Error while reading records from jsonRecords string.", e);
}
}
@Bean
public Consumer<byte[]> myConsumer() {
return this::processRecords;
}
}
application.yml
management:
metrics:
export:
cloudwatch:
namespace: kinesis-reader-app-java
batchSize: 20
spring:
application:
name: kinesis-reader-app
cloud:
stream:
bindings:
myConsumer-in-0:
group: kinesis-reader-app-group
consumer:
concurrency: 5
headerMode: none
use-native-decoding: true
destination: my-kinesis-stream
kinesis:
bindings:
my-kinesis-stream:
consumer:
checkpoint-mode: periodic
checkpoint-interval: 3000
idle-between-polls: ${KINESIS_CONSUMER_IDLE_BETWEEN_POLLS:1000}
consumer-backoff: ${KINESIS_CONSUMER_BACKOFF:1000}
records-limit: ${KINESIS_CONSUMER_RECORDS_LIMIT:5000}
shard-iterator-type: TRIM_HORIZON
worker-id: kinesis-reader-worker-id
binder:
checkpoint:
table: kinesis-reader-stream-metadata
read-capacity: 50
locks:
table: kinesis-reader-lock-registry
lease-duration: 30
refresh-period: 3000
read-capacity: 50
kpl-kcl-enabled: true
auto-create-stream: false
auto-add-shards: false
cloud:
aws:
region:
static: eu-west-1
auto: false
stack:
auto: false
My typescript AWS CDK stack to deploy on AWS:
kinesis-reader-infrastructure-stack.ts
import {RemovalPolicy, Stack, StackProps} from 'aws-cdk-lib';
import {Construct} from 'constructs';
import * as cdk from 'aws-cdk-lib';
import * as dynamodb from 'aws-cdk-lib/aws-dynamodb';
import * as ec2 from 'aws-cdk-lib/aws-ec2';
import * as ecs from 'aws-cdk-lib/aws-ecs';
import * as iam from 'aws-cdk-lib/aws-iam';
import * as logs from 'aws-cdk-lib/aws-logs';
import * as ecsPatterns from 'aws-cdk-lib/aws-ecs-patterns';
import * as ecr from 'aws-cdk-lib/aws-ecr';
// Create and prepare the infrastructure needed for KinesisReader application to work
// Like the Kinesis stream that will receive data from the application and the ECS cluster
export class KinesisReaderInfrastructureStack extends Stack {
constructor(scope: Construct, id: string, props?: StackProps) {
super(scope, id, props);
// The code that defines your stack goes here
// Create repository and docker image in it to contain the java app that will produce load to kinesis
const imageRepository = ecr.Repository.fromRepositoryName(this, 'KinesisReaderRepository', 'sandbox/kinesis-reader-app');
const appDockerImage = ecs.ContainerImage.fromEcrRepository(imageRepository);
// Create DynamoDb
new dynamodb.Table(this, 'kinesis-delivery-stream-checkpoint', {
partitionKey: {
name: "id",
type: dynamodb.AttributeType.STRING
},
billingMode: dynamodb.BillingMode.PAY_PER_REQUEST,
removalPolicy: RemovalPolicy.DESTROY
});
new dynamodb.Table(this, 'kinesis-delivery-stream-locks', {
partitionKey: {
name: "id",
type: dynamodb.AttributeType.STRING
},
billingMode: dynamodb.BillingMode.PAY_PER_REQUEST,
removalPolicy: RemovalPolicy.DESTROY,
});
// Lookup existing VPC
const vpc = ec2.Vpc.fromLookup(this, 'KinesisReaderVpc', {
// specify a 'vpcName' or 'tags'.
vpcId: 'vpc-xxxxxxxxxxxxxxxxx',
isDefault: false
});
// Create ECS Cluster
const cluster = new ecs.Cluster(this, 'KinesisReaderCluster', {
clusterName: 'kinesis-reader-cluster',
vpc: vpc
});
// Permissions added to ECS role
const ecsPermissions = new iam.PolicyStatement({
actions: [
'kinesis:Get*',
'kinesis:ListShards',
'kinesis:ListStreams',
'kinesis:DescribeStreamSummary',
'dynamodb:Scan',
'dynamodb:GetItem',
'dynamodb:PutItem',
'dynamodb:UpdateItem',
'dynamodb:CreateTable',
'dynamodb:DescribeTable',
'cloudwatch:PutMetricData'
],
resources: ['*'],
});
// Create the ECS Role and attach policy to the role
const ecsTaskRole = new iam.Role(this, 'KinesisReaderTaskRole', {
roleName: 'kinesis-reader-task-role',
assumedBy: new iam.ServicePrincipal('ecs-tasks.amazonaws.com'),
});
ecsTaskRole.addToPolicy(ecsPermissions);
// Create Cloudwatch log group
const logGroup = new logs.LogGroup(this, 'KinesisReaderLogGroup', {
logGroupName: 'kinesis-reader-log-group',
removalPolicy: cdk.RemovalPolicy.DESTROY,
retention: logs.RetentionDays.ONE_DAY,
});
const fargateKinesisReaderService = new ecsPatterns.ApplicationLoadBalancedFargateService(this, 'FargateKinesisReaderService', {
serviceName: 'kinesis-reader-service',
cluster: cluster,
cpu: 2048,
desiredCount: 30,
memoryLimitMiB: 4096,
healthCheckGracePeriod: cdk.Duration.minutes(4),
assignPublicIp: true,
publicLoadBalancer: true,
taskImageOptions: {
image: appDockerImage,
taskRole: ecsTaskRole,
containerPort: 1080,
environment: {
KINESIS_CONSUMER_IDLE_BETWEEN_POLLS: '2000',
KINESIS_CONSUMER_BACKOFF: '1000',
KINESIS_CONSUMER_RECORDS_LIMIT: '10000'
},
logDriver: ecs.LogDrivers.awsLogs({
streamPrefix: 'reader',
mode: ecs.AwsLogDriverMode.NON_BLOCKING,
logGroup: logGroup,
}),
},
});
const healthCheck = {
path: "/actuator/health",
interval: cdk.Duration.seconds(45),
timeout: cdk.Duration.seconds(44)
};
fargateKinesisReaderService.targetGroup.configureHealthCheck(healthCheck);
}
}
But after deploying, instances goes quickly shutdown and a new instances created due to heap memory exception, even when i tried to increase CPU and Memory, here is the exception i got and as you see i read a different sizes of payload:
INFO 1 --- [askExecutor-116] c.v.kinesis.ConsumerConfiguration : Record data length received 482
INFO 1 --- [askExecutor-116] c.v.kinesis.ConsumerConfiguration : Record data length received 482
INFO 1 --- [askExecutor-116] c.v.kinesis.ConsumerConfiguration : Record data length received 482
INFO 1 --- [askExecutor-137] c.v.kinesis.ConsumerConfiguration : New Records consumed...
INFO 1 --- [askExecutor-137] c.v.kinesis.ConsumerConfiguration : Records size = 6
INFO 1 --- [askExecutor-137] c.v.kinesis.ConsumerConfiguration : Record data length received 121128
INFO 1 --- [askExecutor-137] c.v.kinesis.ConsumerConfiguration : Record data length received 121128
INFO 1 --- [askExecutor-137] c.v.kinesis.ConsumerConfiguration : Record data length received 121128
INFO 1 --- [askExecutor-137] c.v.kinesis.ConsumerConfiguration : Record data length received 121128
INFO 1 --- [askExecutor-137] c.v.kinesis.ConsumerConfiguration : Record data length received 121128
Native Memory Tracking:
Total: reserved=5102531491, committed=1291543459
- Java Heap (reserved=4294967296, committed=1073741824)
(mmap: reserved=4294967296, committed=1073741824)
- Class (reserved=268005852, committed=76214748)
(classes #13566)
( instance classes #12782, array classes #784)
(malloc=2097628 #34162)
(mmap: reserved=265908224, committed=74117120)
( Metadata: )
( reserved=65011712, committed=64393216)
( used=63054472)
( free=1338744)
( waste=0 =0.00%)
( Class space:)
( reserved=200896512, committed=9723904)
( used=8903056)
( free=820848)
( waste=0 =0.00%)
- Thread (reserved=57068704, committed=5647520)
(thread #54)
(stack: reserved=56807424, committed=5386240)
(malloc=198448 #326)
(arena=62832 #106)
- Code (reserved=255618480, committed=28605872)
(malloc=1985968 #9551)
(mmap: reserved=253632512, committed=26619904)
- GC (reserved=202784183, committed=83246519)
(malloc=9285047 #18310)
(mmap: reserved=193499136, committed=73961472)
- Compiler (reserved=1365978, committed=1365978)
(malloc=471498 #941)
(arena=894480 #11)
- Internal (reserved=824358, committed=824358)
(malloc=791590 #2936)
(mmap: reserved=32768, committed=32768)
- Other (reserved=8, committed=8)
(malloc=8 #1)
- Symbol (reserved=17039968, committed=17039968)
(malloc=14609464 #168653)
(arena=2430504 #1)
- Native Memory Tracking (reserved=3852024, committed=3852024)
(malloc=15096 #188)
(tracking overhead=3836928)
- Arena Chunk (reserved=510560, committed=510560)
(malloc=510560)
- Tracing (reserved=97, committed=97)
(malloc=97 #5)
- Logging (reserved=4572, committed=4572)
(malloc=4572 #192)
- Arguments (reserved=19099, committed=19099)
(malloc=19099 #495)
- Module (reserved=251304, committed=251304)
(malloc=251304 #2088)
- Synchronizer (reserved=210816, committed=210816)
(malloc=210816 #1724)
- Safepoint (reserved=8192, committed=8192)
(mmap: reserved=8192, committed=8192)
I don't know why i got heap memory exception which leads to shutdown the instances, and there is no much details in the exception logged, i even tried this with small, medium and large (CPUs and Memory) instances, but same behaviour happen on all type of AWS instances. So what the problem is??
I am using spring-cloud-stream-binder version 2.1.0
