Ensure MSK Cluster logging is enabled
Managed streaming for Kafka can log to Cloud Watch, Kinesis Firehose and S3, at least one of these locations should be logged to
Impact
Without logging it is difficult to trace issues
Recommended Actions
Follow the appropriate remediation steps below to resolve the issue.
Enable logging
1
2
3
4
5
6
7
8
9
10
11
12
13
|
AWSTemplateFormatVersion: 2010-09-09
Description: Good example
Resources:
Cluster:
Type: AWS::MSK::Cluster
Properties:
LoggingInfo:
BrokerLogs:
S3:
Enabled: true
|
Enable logging
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
|
resource "aws_msk_cluster" "example" {
cluster_name = "example"
kafka_version = "2.4.1"
number_of_broker_nodes = 3
broker_node_group_info {
instance_type = "kafka.m5.large"
ebs_volume_size = 1000
client_subnets = [
aws_subnet.subnet_az1.id,
aws_subnet.subnet_az2.id,
aws_subnet.subnet_az3.id,
]
security_groups = [aws_security_group.sg.id]
}
logging_info {
broker_logs {
firehose {
enabled = false
delivery_stream = aws_kinesis_firehose_delivery_stream.test_stream.name
}
s3 {
enabled = true
bucket = aws_s3_bucket.bucket.id
prefix = "logs/msk-"
}
}
}
tags = {
foo = "bar"
}
}
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
|
resource "aws_msk_cluster" "example" {
cluster_name = "example"
kafka_version = "2.4.1"
number_of_broker_nodes = 3
broker_node_group_info {
instance_type = "kafka.m5.large"
ebs_volume_size = 1000
client_subnets = [
aws_subnet.subnet_az1.id,
aws_subnet.subnet_az2.id,
aws_subnet.subnet_az3.id,
]
security_groups = [aws_security_group.sg.id]
}
logging_info {
broker_logs {
cloudwatch_logs {
enabled = false
log_group = aws_cloudwatch_log_group.test.name
}
firehose {
enabled = true
delivery_stream = aws_kinesis_firehose_delivery_stream.test_stream.name
}
}
}
tags = {
foo = "bar"
}
}
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
|
resource "aws_msk_cluster" "example" {
cluster_name = "example"
kafka_version = "2.4.1"
number_of_broker_nodes = 3
broker_node_group_info {
instance_type = "kafka.m5.large"
ebs_volume_size = 1000
client_subnets = [
aws_subnet.subnet_az1.id,
aws_subnet.subnet_az2.id,
aws_subnet.subnet_az3.id,
]
security_groups = [aws_security_group.sg.id]
}
logging_info {
broker_logs {
cloudwatch_logs {
enabled = true
log_group = aws_cloudwatch_log_group.test.name
}
firehose {
enabled = false
delivery_stream = aws_kinesis_firehose_delivery_stream.test_stream.name
}
s3 {
enabled = true
bucket = aws_s3_bucket.bucket.id
prefix = "logs/msk-"
}
}
}
tags = {
foo = "bar"
}
}
|
Links