# ai-model-logging / azure / t0 "Quick start"
#
# A record of every call to one Azure OpenAI or AI Services account, kept for
# a year in a Log Analytics workspace, and five alerts on the subscription's
# activity log. The first is an account key or a connection's secrets
# fetched, and a key works for anyone who has it. The second is a content
# filter or blocklist changed. The third is a model deployment made, changed,
# or removed. The fourth is an account changed, deleted, or purged. The fifth
# is the logging that would show any of these turned down. Three more alerts
# run on the activity log without the workspace.
#
# It needs the account's full resource ID, an existing resource group, and
# rights to add diagnostic settings and to create a workspace, alert rules,
# and an action group. The azurerm provider registers no resource providers
# from version 5.0, so register the two this file uses before applying.
#   az provider register --namespace Microsoft.Insights
#   az provider register --namespace Microsoft.OperationalInsights
#
# Change the four values below, then:
#   az login
#   terraform init && terraform apply
#
# Verify: an export can take up to 90 minutes to start, so wait that long
# after the first apply. Then run az cognitiveservices account keys list
# --name <account> --resource-group <group>. The keys-fetched email follows
# once the entry reaches the workspace; activity log entries usually take
# three to 20 minutes. A model call can take up to two hours to show:
#   AzureDiagnostics | where Category == "RequestResponse"

locals {
  subscription_id     = "00000000-0000-0000-0000-000000000000" # change me
  resource_group_name = "rg-security"                          # change me: an existing group for the workspace and alerts
  alert_email         = "security-alerts@example.com"          # change me

  # change me: the full resource ID of an existing Azure OpenAI or AI Services account
  ai_account_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-ai/providers/Microsoft.CognitiveServices/accounts/my-openai"

  # Each query keeps the rows that arrived in the last five minutes of a
  # one-hour window, so a late row is rarely missed or alerted twice.
  # Operation names are matched case-insensitively, because the activity log
  # mixes cases. Every detection matches its operations on any account in the
  # subscription, the one above included, which must sit in that subscription.
  recent    = "AzureActivity | where ingestion_time() > ago(5m)"
  succeeded = "ActivityStatusValue in~ (\"Success\", \"Succeeded\")"

  detections = {
    # A key works for whoever has it, with no identity behind it. Storm-2139
    # reached Azure OpenAI with customer keys scraped from public sources and
    # resold the access. A connection on an account or its project stores
    # credentials for other services, and listing its secrets returns them.
    keys-fetched = {
      query = "${local.recent} and ${local.succeeded} | where OperationNameValue in~ (\"Microsoft.CognitiveServices/accounts/listKeys/action\", \"Microsoft.CognitiveServices/accounts/regenerateKey/action\", \"Microsoft.CognitiveServices/accounts/connections/listsecrets/action\", \"Microsoft.CognitiveServices/accounts/projects/connections/listsecrets/action\")"
      doc   = "Someone fetched or regenerated the key of an Azure OpenAI or AI Services account, or listed the secrets of a connection on one or its project."
    }
    # Content filters and blocklists are the guardrails on what a model will
    # answer.
    filters-changed = {
      query = "${local.recent} and ${local.succeeded} | where OperationNameValue in~ (\"Microsoft.CognitiveServices/accounts/raiPolicies/write\", \"Microsoft.CognitiveServices/accounts/raiPolicies/delete\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/write\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/delete\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/raiBlocklistItems/write\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/raiBlocklistItems/delete\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/addRaiBlocklistItems/action\", \"Microsoft.CognitiveServices/accounts/raiBlocklists/deleteRaiBlocklistItems/action\")"
      doc   = "A content filter policy, blocklist, or blocklist item on an AI account was created, changed, or deleted, or blocklist items were added or deleted in bulk."
    }
    # A new deployment is another model to call, billed to the subscription.
    deployment-changed = {
      query = "${local.recent} and ${local.succeeded} | where OperationNameValue in~ (\"Microsoft.CognitiveServices/accounts/deployments/write\", \"Microsoft.CognitiveServices/accounts/deployments/delete\")"
      doc   = "A model deployment on an AI account was created, changed, or deleted."
    }
    # The account's own settings include keys on or off and network access. A
    # purge removes a deleted account for good, with no soft-delete recovery.
    account-changed = {
      query = "${local.recent} and ${local.succeeded} | where OperationNameValue in~ (\"Microsoft.CognitiveServices/accounts/write\", \"Microsoft.CognitiveServices/accounts/delete\", \"Microsoft.CognitiveServices/locations/resourceGroups/deletedAccounts/delete\", \"Microsoft.CognitiveServices/accounts/defenderForAISettings/write\", \"Microsoft.CognitiveServices/accounts/defenderForAISettings/delete\")"
      doc   = "An AI account was changed, deleted, or purged, or its Defender for AI setting was changed or deleted."
    }
    # Diagnostic settings route logs, workspaces keep them, and alert rules
    # and action groups are what anyone hears. Metric alerts, policy
    # assignments, and locks cover the token alert, key policy, and lock that
    # Standard adds. An exemption takes a scope out of a policy's evaluation.
    logging-weakened = {
      query = "${local.recent} and ${local.succeeded} | where OperationNameValue in~ (\"Microsoft.Insights/diagnosticSettings/delete\", \"Microsoft.Insights/diagnosticSettings/write\", \"Microsoft.OperationalInsights/workspaces/delete\", \"Microsoft.OperationalInsights/workspaces/write\", \"Microsoft.OperationalInsights/workspaces/purge/action\", \"Microsoft.OperationalInsights/workspaces/tables/write\", \"Microsoft.Insights/scheduledQueryRules/delete\", \"Microsoft.Insights/scheduledQueryRules/write\", \"Microsoft.Insights/activityLogAlerts/delete\", \"Microsoft.Insights/activityLogAlerts/write\", \"Microsoft.Insights/actionGroups/delete\", \"Microsoft.Insights/actionGroups/write\", \"Microsoft.AlertsManagement/actionRules/write\", \"Microsoft.Insights/MetricAlerts/Write\", \"Microsoft.Insights/MetricAlerts/Delete\", \"Microsoft.Authorization/policyAssignments/write\", \"Microsoft.Authorization/policyAssignments/delete\", \"Microsoft.Authorization/policyExemptions/write\", \"Microsoft.Authorization/locks/delete\", \"Microsoft.Resources/subscriptions/resourceGroups/delete\")"
      doc   = "A diagnostic setting, workspace, alert rule, action group, or policy assignment was changed or deleted. Or a workspace was purged, table settings changed, a lock or resource group deleted, or an alert processing rule or policy exemption written."
    }
  }

  # Changes the workspace might never hear of. A deleted or rewritten export
  # stops the record of its own change, and a disabled rule reads nothing.
  native = {
    diagnostic-setting-deleted = {
      operation = "Microsoft.Insights/diagnosticSettings/delete"
      doc       = "A diagnostic setting was deleted in the subscription, possibly one that feeds the AI model record or its alerts."
    }
    # Policy remediation and other teams' monitoring write diagnostic
    # settings too, so expect some of these.
    diagnostic-setting-written = {
      operation = "Microsoft.Insights/diagnosticSettings/write"
      doc       = "A diagnostic setting was created or changed in the subscription. A rewrite can drop a category or send the logs somewhere else."
    }
    # Azure writes this when a log search rule's query has failed for a week.
    rule-disabled = {
      operation = "Microsoft.Insights/ScheduledQueryRules/disable/action"
      doc       = "Azure disabled a log search alert rule in the subscription because its query kept failing."
    }
  }
}

terraform {
  required_version = ">= 1.6"
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = "~> 5.0"
    }
  }
}

provider "azurerm" {
  features {}
  subscription_id = local.subscription_id
}

data "azurerm_resource_group" "security" {
  name = local.resource_group_name
}

resource "azurerm_log_analytics_workspace" "ai" {
  name                = "law-ai-model-logging"
  location            = data.azurerm_resource_group.security.location
  resource_group_name = data.azurerm_resource_group.security.name
  sku                 = "PerGB2018"
  retention_in_days   = 365
}

# The account's resource logs, into the workspace. Azure keeps none of them
# until a setting like this one exists.
#
# RequestResponse has a row for each inference call. Each row names the
# operation, deployment, status, and duration, the caller's IP with its last
# octet masked, and the caller's Entra object ID for Entra sign-ins. It
# carries no prompt or completion text, by named research and Elastic's
# docs; Microsoft does not say either way. Recording the text is API
# Management's GatewayLlmLogs, which this template leaves out.
#
# Audit records administrative operations on the account, such as keys
# listed. The metrics go too, so they can be queried beside the calls; they
# are billed per GB in the workspace like the logs.
resource "azurerm_monitor_diagnostic_setting" "ai" {
  name                       = "ai-model-logging"
  target_resource_id         = local.ai_account_id
  log_analytics_workspace_id = azurerm_log_analytics_workspace.ai.id

  enabled_log {
    category = "Audit"
  }

  enabled_log {
    category = "RequestResponse"
  }

  enabled_metric {
    category = "AllMetrics"
  }

  lifecycle {
    precondition {
      condition     = can(regex("(?i)^/subscriptions/[^/]+/resourceGroups/[^/]+/providers/Microsoft.CognitiveServices/accounts/[^/]+$", local.ai_account_id)) && startswith(lower(local.ai_account_id), lower("/subscriptions/${local.subscription_id}/"))
      error_message = "Set ai_account_id to the full resource ID of an account in subscription_id, /subscriptions/<subscription_id>/.../providers/Microsoft.CognitiveServices/accounts/<name>. The alerts read only that subscription's activity log."
    }
  }
}

# The subscription's activity log, into the workspace. Ingesting it costs
# nothing.
resource "azurerm_monitor_diagnostic_setting" "activity" {
  name                       = "ai-model-logging"
  target_resource_id         = "/subscriptions/${local.subscription_id}"
  log_analytics_workspace_id = azurerm_log_analytics_workspace.ai.id

  enabled_log {
    category = "Administrative"
  }
}

resource "azurerm_monitor_action_group" "ai" {
  name                = "ai-model-logging"
  resource_group_name = data.azurerm_resource_group.security.name
  short_name          = "ai-logging"

  email_receiver {
    name          = "security"
    email_address = local.alert_email
  }
}

# Azure evaluates these alerts on the activity log itself, with no workspace
# in between, and they cost nothing. They still notify through the one action
# group, so deleting or disabling it, or deleting the resource group,
# silences every alert here. Standard locks the group against deletes.
resource "azurerm_monitor_activity_log_alert" "native" {
  for_each            = local.native
  name                = "ai-${each.key}"
  resource_group_name = data.azurerm_resource_group.security.name
  location            = "global"
  scopes              = ["/subscriptions/${local.subscription_id}"]
  description         = each.value.doc

  criteria {
    category       = "Administrative"
    operation_name = each.value.operation
  }

  action {
    action_group_id = azurerm_monitor_action_group.ai.id
  }
}

# The AzureActivity table does not exist until the first event arrives, and a
# rule's query is checked when the rule is made. Skipping that check lets the
# rules exist first. They are made after the exports, though an export can
# still take up to 90 minutes to start delivering.
resource "azurerm_monitor_scheduled_query_rules_alert_v2" "detection" {
  for_each              = local.detections
  name                  = "ai-${each.key}"
  resource_group_name   = data.azurerm_resource_group.security.name
  location              = data.azurerm_resource_group.security.location
  scopes                = [azurerm_log_analytics_workspace.ai.id]
  description           = "${each.value.doc} In the workspace, run: ${each.value.query} | project TimeGenerated, Caller, CallerIpAddress, OperationNameValue, _ResourceId"
  severity              = 1
  evaluation_frequency  = "PT5M"
  window_duration       = "PT1H"
  skip_query_validation = true

  criteria {
    query                   = each.value.query
    time_aggregation_method = "Count"
    operator                = "GreaterThan"
    threshold               = 0
  }

  action {
    action_groups = [azurerm_monitor_action_group.ai.id]
  }

  depends_on = [azurerm_monitor_diagnostic_setting.activity, azurerm_monitor_diagnostic_setting.ai]
}
