# siem-ingest / gcp / t1 "Standard" # # Audit logs from a project or a whole organisation into Pub/Sub for the SIEM, # undeliverable messages kept aside, and an alert when the collector falls an # hour behind. A pipe that stops quietly leaves the SIEM blind with nothing to # show for it. data "google_project" "this" { project_id = var.project_id } locals { org = var.organization_id != "" writer = local.org ? google_logging_organization_sink.siem[0].writer_identity : google_logging_project_sink.siem[0].writer_identity pubsub_sa = "serviceAccount:service-${data.google_project.this.number}@gcp-sa-pubsub.iam.gserviceaccount.com" collectors = length(var.collector_members) > 0 ? var.collector_members : ["serviceAccount:${google_service_account.collector[0].email}"] } resource "google_project_service" "pubsub" { project = var.project_id service = "pubsub.googleapis.com" disable_on_destroy = false } # With your own key, Pub/Sub's service agent needs to use it, or publishing # fails. resource "google_kms_crypto_key_iam_member" "pubsub_uses_key" { count = var.kms_key_name == "" ? 0 : 1 crypto_key_id = var.kms_key_name role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" member = local.pubsub_sa } resource "google_pubsub_topic" "siem" { project = var.project_id name = "siem-audit-logs" message_retention_duration = "604800s" kms_key_name = var.kms_key_name == "" ? null : var.kms_key_name depends_on = [google_project_service.pubsub, google_kms_crypto_key_iam_member.pubsub_uses_key] } # Messages the collector fails on five times move here, so one bad message # cannot block the rest. Read this subscription during a review. resource "google_pubsub_topic" "dead_letter" { project = var.project_id name = "siem-audit-logs-undeliverable" kms_key_name = var.kms_key_name == "" ? null : var.kms_key_name depends_on = [google_project_service.pubsub, google_kms_crypto_key_iam_member.pubsub_uses_key] } resource "google_pubsub_subscription" "dead_letter" { project = var.project_id name = "siem-audit-logs-undeliverable" topic = google_pubsub_topic.dead_letter.id message_retention_duration = "604800s" expiration_policy { ttl = "" } } resource "google_pubsub_subscription" "siem" { project = var.project_id name = "siem-collector" topic = google_pubsub_topic.siem.id ack_deadline_seconds = 60 message_retention_duration = "604800s" expiration_policy { ttl = "" } dead_letter_policy { dead_letter_topic = google_pubsub_topic.dead_letter.id max_delivery_attempts = 5 } retry_policy { minimum_backoff = "10s" maximum_backoff = "600s" } } # Pub/Sub moves dead letters as its own service agent, which needs rights on # both ends. resource "google_pubsub_topic_iam_member" "agent_publishes_dead_letters" { project = var.project_id topic = google_pubsub_topic.dead_letter.id role = "roles/pubsub.publisher" member = local.pubsub_sa } resource "google_pubsub_subscription_iam_member" "agent_forwards_dead_letters" { project = var.project_id subscription = google_pubsub_subscription.siem.id role = "roles/pubsub.subscriber" member = local.pubsub_sa } resource "google_logging_project_sink" "siem" { count = local.org ? 0 : 1 project = var.project_id name = "siem-audit-logs" destination = "pubsub.googleapis.com/${google_pubsub_topic.siem.id}" filter = var.filter unique_writer_identity = true dynamic "exclusions" { for_each = var.exclusions content { name = exclusions.key filter = exclusions.value } } } resource "google_logging_organization_sink" "siem" { count = local.org ? 1 : 0 org_id = var.organization_id name = "siem-audit-logs" destination = "pubsub.googleapis.com/${google_pubsub_topic.siem.id}" filter = var.filter include_children = true dynamic "exclusions" { for_each = var.exclusions content { name = exclusions.key filter = exclusions.value } } } resource "google_pubsub_topic_iam_member" "sink_publishes" { project = var.project_id topic = google_pubsub_topic.siem.id role = "roles/pubsub.publisher" member = local.writer } resource "google_service_account" "collector" { count = length(var.collector_members) > 0 ? 0 : 1 project = var.project_id account_id = "siem-collector" display_name = "Reads audit logs for the SIEM" } resource "google_pubsub_subscription_iam_member" "collector" { for_each = toset(local.collectors) project = var.project_id subscription = google_pubsub_subscription.siem.id role = "roles/pubsub.subscriber" member = each.value } resource "google_monitoring_notification_channel" "email" { for_each = toset(var.alert_emails) project = var.project_id display_name = "SIEM pipe: ${each.value}" type = "email" labels = { email_address = each.value } } resource "google_monitoring_alert_policy" "behind" { project = var.project_id display_name = "SIEM collector over an hour behind" combiner = "OR" notification_channels = [for c in google_monitoring_notification_channel.email : c.id] conditions { display_name = "oldest unread message over an hour old" condition_threshold { filter = "metric.type=\"pubsub.googleapis.com/subscription/oldest_unacked_message_age\" AND resource.type=\"pubsub_subscription\" AND resource.label.subscription_id=\"${google_pubsub_subscription.siem.name}\"" comparison = "COMPARISON_GT" threshold_value = 3600 duration = "300s" aggregations { alignment_period = "300s" per_series_aligner = "ALIGN_MAX" } } } documentation { content = "The SIEM has not read audit logs for over an hour. Check the collector. Messages wait for seven days, so nothing is lost yet." } }