# ai-model-logging / azure / t1 "Standard" # # The quick start's record and eight alerts, for every account you name, with # named callers left out, each rule reading under its own identity, and the # workspace kept for your own period. A fourth alert outside the workspace # reports a rule's read access removed, and a lock stops anything in the # security group being deleted. tokens.tf alerts when one account processes # more tokens in an hour than you expect. policy.tf reports or refuses AI # accounts that accept keys. queries.tf saves the queries behind every alert, # and two for reading the calls themselves. locals { tags = merge({ owner = var.owner environment = var.environment data_classification = "confidential" managed_by = "terraform" }, var.extra_tags) # Each query keeps the rows that arrived in the last five minutes of a # one-hour window, so a late row is rarely missed or alerted twice. # Operation names are matched case-insensitively, because the activity log # mixes cases. Every detection matches its operations on any account in the # subscription, named or not. Named accounts must sit in that subscription. expected = jsonencode([for c in var.expected_callers : lower(c)]) recent = "AzureActivity | where ingestion_time() > ago(5m) | where not(tolower(Caller) in (dynamic(${local.expected}))) and not(tolower(tostring(Claims_d.appid)) in (dynamic(${local.expected})))" succeeded = "ActivityStatusValue in~ (\"Success\", \"Succeeded\")" detections = { # A key works for whoever has it, with no identity behind it. Storm-2139 # reached Azure OpenAI with customer keys scraped from public sources. A # connection on an account or its project stores credentials for other # services, and listing its secrets returns them. keys-fetched = { query = "${local.recent} | where ${local.succeeded} | where OperationNameValue in~ (\"Microsoft.CognitiveServices/accounts/listKeys/action\", \"Microsoft.CognitiveServices/accounts/regenerateKey/action\", \"Microsoft.CognitiveServices/accounts/connections/listsecrets/action\", \"Microsoft.CognitiveServices/accounts/projects/connections/listsecrets/action\")" doc = "Someone fetched or regenerated the key of an Azure OpenAI or AI Services account, or listed the secrets of a connection on one or its project." } filters-changed = { query = "${local.recent} | where ${local.succeeded} | where OperationNameValue in~ (\"Microsoft.CognitiveServices/accounts/raiPolicies/write\", \"Microsoft.CognitiveServices/accounts/raiPolicies/delete\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/write\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/delete\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/raiBlocklistItems/write\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/raiBlocklistItems/delete\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/addRaiBlocklistItems/action\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/deleteRaiBlocklistItems/action\")" doc = "A content filter policy, blocklist, or blocklist item on an AI account was created, changed, or deleted, or blocklist items were added or deleted in bulk." } deployment-changed = { query = "${local.recent} | where ${local.succeeded} | where OperationNameValue in~ (\"Microsoft.CognitiveServices/accounts/deployments/write\", \"Microsoft.CognitiveServices/accounts/deployments/delete\")" doc = "A model deployment on an AI account was created, changed, or deleted." } account-changed = { query = "${local.recent} | where ${local.succeeded} | where OperationNameValue in~ (\"Microsoft.CognitiveServices/accounts/write\", \"Microsoft.CognitiveServices/accounts/delete\", \"Microsoft.CognitiveServices/locations/resourceGroups/deletedAccounts/delete\", \"Microsoft.CognitiveServices/accounts/defenderForAISettings/write\", \"Microsoft.CognitiveServices/accounts/defenderForAISettings/delete\")" doc = "An AI account was changed, deleted, or purged, or its Defender for AI setting was changed or deleted." } # Diagnostic settings route logs, workspaces keep them, and alert rules # and action groups are what anyone hears. Metric alerts cover the token # alert, policy assignments the key policy, and an exemption takes a # scope out of the key policy's evaluation, Deny included. Deleting the # lock is the step before deleting anything in the security group. logging-weakened = { query = "${local.recent} | where ${local.succeeded} | where OperationNameValue in~ (\"Microsoft.Insights/diagnosticSettings/delete\", \"Microsoft.Insights/diagnosticSettings/write\", \"Microsoft.OperationalInsights/workspaces/delete\", \"Microsoft.OperationalInsights/workspaces/write\", \"Microsoft.OperationalInsights/workspaces/purge/action\", \"Microsoft.OperationalInsights/workspaces/tables/write\", \"Microsoft.Insights/scheduledQueryRules/delete\", \"Microsoft.Insights/scheduledQueryRules/write\", \"Microsoft.Insights/activityLogAlerts/delete\", \"Microsoft.Insights/activityLogAlerts/write\", \"Microsoft.Insights/actionGroups/delete\", \"Microsoft.Insights/actionGroups/write\", \"Microsoft.AlertsManagement/actionRules/write\", \"Microsoft.Insights/MetricAlerts/Write\", \"Microsoft.Insights/MetricAlerts/Delete\", \"Microsoft.Authorization/policyAssignments/write\", \"Microsoft.Authorization/policyAssignments/delete\", \"Microsoft.Authorization/policyExemptions/write\", \"Microsoft.Authorization/locks/delete\", \"Microsoft.Resources/subscriptions/resourceGroups/delete\")" doc = "A diagnostic setting, workspace, alert rule, action group, or policy assignment was changed or deleted. Or a workspace was purged, table settings changed, a lock or resource group deleted, or an alert processing rule or policy exemption written." } } # Changes the workspace might never hear of. A deleted or rewritten export # stops the record of its own change, a rule without its read access reads # nothing, and a disabled rule does not run. native = { diagnostic-setting-deleted = { operation = "Microsoft.Insights/diagnosticSettings/delete" doc = "A diagnostic setting was deleted in the subscription, possibly one that feeds the AI model record or its alerts." } # Policy remediation and other teams' monitoring write diagnostic # settings too, so expect some of these. diagnostic-setting-written = { operation = "Microsoft.Insights/diagnosticSettings/write" doc = "A diagnostic setting was created or changed in the subscription. A rewrite can drop a category or send the logs somewhere else." } # Azure writes this when a log search rule's query has failed for a week. rule-disabled = { operation = "Microsoft.Insights/ScheduledQueryRules/disable/action" doc = "Azure disabled a log search alert rule in the subscription because its query kept failing." } # Only the rules' own assignments, named in the alert below. rule-access-removed = { operation = "Microsoft.Authorization/roleAssignments/delete" doc = "A role assignment that lets an AI model logging rule read the workspace was deleted. That rule now reads nothing, and Azure disables it after a week." } } } data "azurerm_resource_group" "security" { name = var.resource_group_name } resource "azurerm_log_analytics_workspace" "ai" { name = "law-ai-model-logging" location = data.azurerm_resource_group.security.location resource_group_name = data.azurerm_resource_group.security.name sku = "PerGB2018" retention_in_days = var.retention_days # No daily cap. A cap drops events once it is reached, with no alert. daily_quota_gb = -1 tags = local.tags } # Each account's resource logs, into the workspace. Azure keeps none of them # until a setting like this one exists. # # RequestResponse has a row for each inference call. Each row names the # operation, deployment, status, and duration, the caller's IP with its last # octet masked, and the caller's Entra object ID for Entra sign-ins. It # carries no prompt or completion text, by named research and Elastic's # docs; Microsoft does not say either way. Recording the text is API # Management's GatewayLlmLogs, which this template leaves out. # # Audit records administrative operations on the account, such as keys # listed. The metrics go too, so they can be queried beside the calls; they # are billed per GB in the workspace like the logs. resource "azurerm_monitor_diagnostic_setting" "ai" { for_each = toset(var.ai_account_ids) name = "ai-model-logging" target_resource_id = each.value log_analytics_workspace_id = azurerm_log_analytics_workspace.ai.id enabled_log { category = "Audit" } enabled_log { category = "RequestResponse" } enabled_metric { category = "AllMetrics" } lifecycle { precondition { condition = startswith(lower(each.value), lower("/subscriptions/${var.subscription_id}/")) error_message = "Each account in ai_account_ids must sit in subscription_id. The alerts read only that subscription's activity log." } } } # The subscription's activity log, into the workspace. Ingesting it costs # nothing. resource "azurerm_monitor_diagnostic_setting" "activity" { name = "ai-model-logging" target_resource_id = "/subscriptions/${var.subscription_id}" log_analytics_workspace_id = azurerm_log_analytics_workspace.ai.id enabled_log { category = "Administrative" } } resource "azurerm_monitor_action_group" "ai" { name = "ai-model-logging" resource_group_name = data.azurerm_resource_group.security.name short_name = "ai-logging" tags = local.tags # Receiver names are numbered, because names must differ regardless of # case and two addresses can reduce to the same name. dynamic "email_receiver" { for_each = var.alert_emails content { name = "email-${email_receiver.key}" email_address = email_receiver.value } } } # Azure evaluates these alerts on the activity log itself, with no workspace # in between, and they cost nothing. They still notify through the one action # group, so someone who can edit it can disable it or change its addresses. resource "azurerm_monitor_activity_log_alert" "native" { for_each = local.native name = "ai-${each.key}" resource_group_name = data.azurerm_resource_group.security.name location = "global" scopes = ["/subscriptions/${var.subscription_id}"] tags = local.tags description = each.value.doc # Microsoft records a role assignment in the activity log under its full # ID, scope included, which is what resource_ids matches. The verify output # shows how to test it. criteria { category = "Administrative" operation_name = each.value.operation resource_ids = each.key == "rule-access-removed" ? [for r in azurerm_role_assignment.rule_reads_workspace : r.id] : null } action { action_group_id = azurerm_monitor_action_group.ai.id } } # Each rule reads the workspace as itself. Without an identity a rule reads # as whoever last saved it, and stops when that person's access does. The # rules are made after the exports, though an export can still take up to # 90 minutes to start delivering. resource "azurerm_monitor_scheduled_query_rules_alert_v2" "detection" { for_each = local.detections name = "ai-${each.key}" resource_group_name = data.azurerm_resource_group.security.name location = data.azurerm_resource_group.security.location scopes = [azurerm_log_analytics_workspace.ai.id] description = "${each.value.doc} The saved query ai-${each.key} in the workspace shows who, from where, and on what." severity = 1 evaluation_frequency = "PT5M" window_duration = "PT1H" skip_query_validation = true tags = local.tags identity { type = "SystemAssigned" } criteria { query = each.value.query time_aggregation_method = "Count" operator = "GreaterThan" threshold = 0 } action { action_groups = [azurerm_monitor_action_group.ai.id] } depends_on = [azurerm_monitor_diagnostic_setting.activity, azurerm_monitor_diagnostic_setting.ai] } resource "azurerm_role_assignment" "rule_reads_workspace" { for_each = azurerm_monitor_scheduled_query_rules_alert_v2.detection scope = azurerm_log_analytics_workspace.ai.id role_definition_name = "Log Analytics Reader" principal_id = each.value.identity[0].principal_id principal_type = "ServicePrincipal" } # Nothing in the security group can be deleted while this lock is on: the # workspace and its saved queries, the action group, the alerts, and the # rules' role assignments. Deleting the lock first is watched by # logging-weakened. A cannot-delete lock does not stop a workspace purge, # which is watched too, or an edit such as disabling the action group. # # The lock is made last and removed first, so terraform destroy can run. # Any change that replaces or removes a resource in the group fails while # it is on. resource "azurerm_management_lock" "security" { name = "ai-model-logging" scope = data.azurerm_resource_group.security.id lock_level = "CanNotDelete" notes = "Keeps the AI model logging record and its alerts from being deleted. Managed by Terraform." depends_on = [ azurerm_log_analytics_workspace.ai, azurerm_log_analytics_saved_search.ai, azurerm_monitor_action_group.ai, azurerm_monitor_activity_log_alert.native, azurerm_monitor_scheduled_query_rules_alert_v2.detection, azurerm_role_assignment.rule_reads_workspace, azurerm_monitor_metric_alert.tokens, ] }