# k8s-audit-logging / azure / t1 "Standard" # # Five detections on the AKS audit log, with the log kept for your own # period. The alerts read it under an identity of their own, and every event # can also go to your SIEM's event hub. Five native alerts read the # subscription's activity log without the workspace, and locks stop the # workspace, the action group, and every alert being deleted. locals { tags = merge({ owner = var.owner environment = var.environment data_classification = "confidential" managed_by = "terraform" }, var.extra_tags) category = var.full_audit ? "kube-audit" : "kube-audit-admin" table = var.full_audit ? "AKSAudit" : "AKSAuditAdmin" # Each query keeps the rows that arrived in the last five minutes of a # one-hour window, so a late row is rarely missed or alerted twice. recent = "${local.table} | where ingestion_time() > ago(5m) and Stage == \"ResponseComplete\"" detections = merge({ # Whatever the verb: newer clients open exec as a get over a WebSocket. pod-exec = "${local.recent} | where tostring(ObjectRef.resource) == \"pods\" and tostring(ObjectRef.subresource) in (\"exec\", \"attach\") | where toint(ResponseStatus.code) < 300" cluster-role-binding = "${local.recent} | where Verb == \"create\" and tostring(ObjectRef.resource) == \"clusterrolebindings\" | where toint(ResponseStatus.code) < 300" # Anonymous calls to the health and version paths are ordinary on a # public API server, so only calls that name an object count. anonymous-request = "${local.recent} | where tostring(User.username) == \"system:anonymous\" and isnotempty(tostring(ObjectRef.resource))" # Refused by authorisation. A single one is ordinary; ten from one user in # five minutes is someone probing what a stolen credential can do. forbidden-request = "${local.recent} | where toint(ResponseStatus.code) == 403 | summarize Denied = count() by User = tostring(User.username) | where Denied > 9" }, var.full_audit ? { # People reading secrets. Controllers read them constantly under system: # names, so those are left out. secret-read-by-person = "${local.recent} | where Verb in (\"get\", \"list\") and tostring(ObjectRef.resource) == \"secrets\" | where tostring(User.username) !startswith \"system:\"" } : {}) siem = var.siem_eventhub_authorization_rule_id != "" && var.siem_eventhub_name != "" # Workspace names allow letters, digits, and hyphens, up to 63 characters. cluster_slug = substr(replace(var.cluster_name, "/[^A-Za-z0-9-]/", "-"), 0, 53) # Changes the workspace might never hear of. A deleted or rewritten setting # stops the record of its own change, a rule without its read access reads # nothing, and a disabled rule does not run. Deleting a lock is the step # before deleting anything it covers. native = { diagnostic-setting-deleted = { operation = "Microsoft.Insights/diagnosticSettings/delete" doc = "A diagnostic setting was deleted in the subscription, possibly the one that sends the cluster's audit log." } # Policy remediation and other teams' monitoring write diagnostic # settings too, so expect some of these. diagnostic-setting-written = { operation = "Microsoft.Insights/diagnosticSettings/write" doc = "A diagnostic setting was created or changed in the subscription. A rewrite can drop the audit category or send the log somewhere else." } # Azure writes this when a log search rule's query has failed for a week. rule-disabled = { operation = "Microsoft.Insights/ScheduledQueryRules/disable/action" doc = "Azure disabled a log search alert rule in the subscription because its query kept failing." } lock-deleted = { operation = "Microsoft.Authorization/locks/delete" doc = "A management lock was deleted in the subscription, possibly one that keeps the audit alerts from being deleted." } # Only the rules' own assignments, named in the alert below. rule-access-removed = { operation = "Microsoft.Authorization/roleAssignments/delete" doc = "A role assignment that lets an audit alert rule read the workspace was deleted. That rule now reads nothing, and Azure disables it after a week." } } } data "azurerm_kubernetes_cluster" "this" { name = var.cluster_name resource_group_name = var.cluster_resource_group } resource "azurerm_log_analytics_workspace" "audit" { name = "law-${local.cluster_slug}-audit" location = data.azurerm_kubernetes_cluster.this.location resource_group_name = var.cluster_resource_group sku = "PerGB2018" retention_in_days = var.retention_days # No daily cap. A cap drops events once it is reached, silently, and an # attacker who can make noise can make the log stop. daily_quota_gb = -1 tags = local.tags } resource "azurerm_monitor_diagnostic_setting" "audit" { name = "k8s-audit" target_resource_id = data.azurerm_kubernetes_cluster.this.id log_analytics_workspace_id = azurerm_log_analytics_workspace.audit.id log_analytics_destination_type = "Dedicated" enabled_log { category = local.category } } # Optional: the same events, straight from the cluster to the SIEM's event # hub. A second setting on the cluster, so it depends on nothing in the # workspace. resource "azurerm_monitor_diagnostic_setting" "to_siem" { count = local.siem ? 1 : 0 name = "k8s-audit-to-siem" target_resource_id = data.azurerm_kubernetes_cluster.this.id eventhub_authorization_rule_id = var.siem_eventhub_authorization_rule_id eventhub_name = var.siem_eventhub_name enabled_log { category = local.category } } resource "azurerm_monitor_action_group" "audit" { name = "${var.cluster_name}-audit-alerts" resource_group_name = var.cluster_resource_group short_name = "k8saudit" tags = local.tags # Receiver names are numbered, because names must differ regardless of # case and two addresses can reduce to the same name. dynamic "email_receiver" { for_each = var.alert_emails content { name = "email-${email_receiver.key}" email_address = email_receiver.value } } } # Azure evaluates these alerts on the activity log itself, with no workspace # in between, and they cost nothing. They still notify through the one action # group, so someone who can edit it can disable it or change its addresses. resource "azurerm_monitor_activity_log_alert" "native" { for_each = local.native name = "${var.cluster_name}-${each.key}" resource_group_name = var.cluster_resource_group location = "global" scopes = ["/subscriptions/${var.subscription_id}"] tags = local.tags description = each.value.doc # Microsoft records a role assignment in the activity log under its full # ID, scope included, which is what resource_ids matches. The verify output # shows how to test it. criteria { category = "Administrative" operation_name = each.value.operation resource_ids = each.key == "rule-access-removed" ? [for r in azurerm_role_assignment.rule_reads_workspace : r.id] : null } action { action_group_id = azurerm_monitor_action_group.audit.id } } # Each rule reads the workspace as itself. Without an identity a rule reads # as whoever last saved it, and stops working when that person's access does. # # The table does not exist until the first event arrives, and a rule's query # is checked against the workspace when the rule is made. Skipping that check # lets the rules exist before the log does. The rules are made after the # setting, though a setting can still take up to 90 minutes to start sending. resource "azurerm_monitor_scheduled_query_rules_alert_v2" "detection" { for_each = local.detections name = "${var.cluster_name}-${each.key}" resource_group_name = var.cluster_resource_group location = data.azurerm_kubernetes_cluster.this.location scopes = [azurerm_log_analytics_workspace.audit.id] description = "Seen in the AKS audit log. In the workspace, run: ${each.value}" severity = 1 evaluation_frequency = "PT5M" window_duration = "PT1H" skip_query_validation = true tags = local.tags identity { type = "SystemAssigned" } criteria { query = each.value time_aggregation_method = "Count" operator = "GreaterThan" threshold = 0 } action { action_groups = [azurerm_monitor_action_group.audit.id] } depends_on = [azurerm_monitor_diagnostic_setting.audit] } resource "azurerm_role_assignment" "rule_reads_workspace" { for_each = azurerm_monitor_scheduled_query_rules_alert_v2.detection scope = azurerm_log_analytics_workspace.audit.id role_definition_name = "Log Analytics Reader" principal_id = each.value.identity[0].principal_id principal_type = "ServicePrincipal" } # Workloads share the cluster's resource group, so the locks go on the alert # path alone: the workspace, the action group, and every alert. A lock on the # workspace also stops the rules' role assignments there being deleted, and # any locked resource stops its resource group being deleted. Deleting a lock # first fires the lock-deleted alert. A cannot-delete lock does not stop a # workspace purge, or an edit such as disabling the action group. # # The locks are made last and removed first, so terraform destroy can run. # Any change that replaces or removes a locked resource fails while its lock # is on. resource "azurerm_management_lock" "alert_path" { for_each = merge( { workspace = azurerm_log_analytics_workspace.audit.id action-group = azurerm_monitor_action_group.audit.id }, { for k, r in azurerm_monitor_scheduled_query_rules_alert_v2.detection : "rule-${k}" => r.id }, { for k, a in azurerm_monitor_activity_log_alert.native : "native-${k}" => a.id }, ) name = "k8s-audit-logging" scope = each.value lock_level = "CanNotDelete" notes = "Keeps the AKS audit log and its alerts from being deleted. Managed by Terraform." depends_on = [azurerm_role_assignment.rule_reads_workspace] }