# siem-ingest / aws / t1 "Standard" # # Several log groups into Splunk through one Firehose stream, refused events # kept encrypted for resending, the stream's own errors logged, and alarms # when delivery fails or falls behind. A pipe that stops quietly leaves the # SIEM blind with nothing to show for it. data "aws_caller_identity" "current" {} data "aws_region" "current" {} locals { tags = merge({ owner = var.owner environment = var.environment data_classification = "confidential" managed_by = "terraform" }, var.extra_tags) account = data.aws_caller_identity.current.account_id region = data.aws_region.current.region } # One key for the stream and for refused events. The account statement lets # IAM decide who uses it, and the Firehose role's policy below grants it. resource "aws_kms_key" "failed" { description = "SIEM stream ${var.name} and its refused events" enable_key_rotation = true deletion_window_in_days = 30 policy = jsonencode({ Version = "2012-10-17" Statement = [{ Sid = "AccountManagesKeyThroughIAM" Effect = "Allow" Principal = { AWS = "arn:aws:iam::${local.account}:root" } Action = "kms:*" Resource = "*" }] }) } resource "aws_s3_bucket" "failed" { bucket_prefix = "${var.name}-failed-" } resource "aws_s3_bucket_public_access_block" "failed" { bucket = aws_s3_bucket.failed.id block_public_acls = true block_public_policy = true ignore_public_acls = true restrict_public_buckets = true } resource "aws_s3_bucket_server_side_encryption_configuration" "failed" { bucket = aws_s3_bucket.failed.id rule { apply_server_side_encryption_by_default { sse_algorithm = "aws:kms" kms_master_key_id = aws_kms_key.failed.arn } bucket_key_enabled = true } } resource "aws_s3_bucket_lifecycle_configuration" "failed" { bucket = aws_s3_bucket.failed.id rule { id = "expire" status = "Enabled" filter {} expiration { days = var.failed_retention_days } abort_incomplete_multipart_upload { days_after_initiation = 1 } } } resource "aws_cloudwatch_log_group" "firehose" { name = "/aws/kinesisfirehose/${var.name}" retention_in_days = 90 } resource "aws_cloudwatch_log_stream" "firehose" { name = "delivery" log_group_name = aws_cloudwatch_log_group.firehose.name } data "aws_iam_policy_document" "firehose_trust" { statement { actions = ["sts:AssumeRole"] principals { type = "Service" identifiers = ["firehose.amazonaws.com"] } condition { test = "StringEquals" variable = "sts:ExternalId" values = [local.account] } } } resource "aws_iam_role" "firehose" { name = "${var.name}-firehose" assume_role_policy = data.aws_iam_policy_document.firehose_trust.json } resource "aws_iam_role_policy" "firehose" { name = "failed-events-and-errors" role = aws_iam_role.firehose.id policy = jsonencode({ Version = "2012-10-17" Statement = [ { Effect = "Allow" Action = ["s3:PutObject", "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:ListBucket"] Resource = [aws_s3_bucket.failed.arn, "${aws_s3_bucket.failed.arn}/*"] }, { Effect = "Allow" Action = ["kms:GenerateDataKey", "kms:Decrypt"] Resource = aws_kms_key.failed.arn }, { Effect = "Allow" Action = ["logs:PutLogEvents"] Resource = "${aws_cloudwatch_log_group.firehose.arn}:*" }, ] }) } resource "aws_kinesis_firehose_delivery_stream" "splunk" { name = var.name destination = "splunk" server_side_encryption { enabled = true key_type = "CUSTOMER_MANAGED_CMK" key_arn = aws_kms_key.failed.arn } splunk_configuration { hec_endpoint = var.hec_endpoint hec_token = var.hec_token hec_endpoint_type = "Raw" hec_acknowledgment_timeout = 300 retry_duration = 3600 s3_backup_mode = "FailedEventsOnly" s3_configuration { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.failed.arn compression_format = "GZIP" kms_key_arn = aws_kms_key.failed.arn } cloudwatch_logging_options { enabled = true log_group_name = aws_cloudwatch_log_group.firehose.name log_stream_name = aws_cloudwatch_log_stream.firehose.name } processing_configuration { enabled = true processors { type = "Decompression" parameters { parameter_name = "CompressionFormat" parameter_value = "GZIP" } } processors { type = "CloudWatchLogProcessing" parameters { parameter_name = "DataMessageExtraction" parameter_value = "true" } } } } } data "aws_iam_policy_document" "logs_trust" { statement { actions = ["sts:AssumeRole"] principals { type = "Service" identifiers = ["logs.${local.region}.amazonaws.com"] } condition { test = "StringEquals" variable = "aws:SourceAccount" values = [local.account] } } } resource "aws_iam_role" "logs" { name = "${var.name}-logs" assume_role_policy = data.aws_iam_policy_document.logs_trust.json } resource "aws_iam_role_policy" "logs" { name = "put-to-firehose" role = aws_iam_role.logs.id policy = jsonencode({ Version = "2012-10-17" Statement = [{ Effect = "Allow", Action = ["firehose:PutRecord", "firehose:PutRecordBatch"], Resource = aws_kinesis_firehose_delivery_stream.splunk.arn }] }) } resource "aws_cloudwatch_log_subscription_filter" "to_splunk" { for_each = toset(var.log_groups) name = "to-${var.name}" log_group_name = each.value filter_pattern = "" destination_arn = aws_kinesis_firehose_delivery_stream.splunk.arn role_arn = aws_iam_role.logs.arn } # Some deliveries failing: the success ratio drops below 1. resource "aws_cloudwatch_metric_alarm" "delivery_failing" { alarm_name = "${var.name}-delivery-failing" alarm_description = "Splunk is refusing some events. Check the stream's error log and the failed events bucket." namespace = "AWS/Firehose" metric_name = "DeliveryToSplunk.Success" dimensions = { DeliveryStreamName = aws_kinesis_firehose_delivery_stream.splunk.name } statistic = "Average" period = 300 evaluation_periods = 3 threshold = 1 comparison_operator = "LessThanThreshold" treat_missing_data = "notBreaching" alarm_actions = [var.alert_topic_arn] } # Falling behind: the oldest record waiting is over 15 minutes old. resource "aws_cloudwatch_metric_alarm" "delivery_late" { alarm_name = "${var.name}-delivery-late" alarm_description = "Events are waiting more than 15 minutes to reach Splunk." namespace = "AWS/Firehose" metric_name = "DeliveryToSplunk.DataFreshness" dimensions = { DeliveryStreamName = aws_kinesis_firehose_delivery_stream.splunk.name } statistic = "Maximum" period = 300 evaluation_periods = 1 threshold = 900 comparison_operator = "GreaterThanThreshold" treat_missing_data = "notBreaching" alarm_actions = [var.alert_topic_arn] }